add setup metadata to help PyPI flesh out content on pypi package page

Summary:
addresses comments in #19915
Pull Request resolved: https://github.com/pytorch/pytorch/pull/19959

Differential Revision: D15149993

Pulled By: orionr

fbshipit-source-id: 0e438cfa1a311e89d4bed7ae9d7710a9f1b19a78

.circleci/.gitignore

@@ -0,0 +1,2 @@
+*.svg
+*.png

.circleci/README.md

@@ -0,0 +1,38 @@
+CircleCI configuration generator
+================================
+
+One may no longer make changes to the `.circleci/config.yml` file directly.
+Instead, one must edit these Python scripts or files in the `verbatim-sources/` directory.
+
+
+Usage
+----------
+
+1. Make changes to these scripts.
+2. Run the `regenerate.sh` script in this directory and commit the script changes and the resulting change to `config.yml`.
+
+You'll see a build failure on TravisCI if the scripts don't agree with the checked-in version.
+
+
+Motivation
+----------
+
+These scripts establish a single, authoritative source of documentation for the CircleCI configuration matrix.
+The documentation, in the form of diagrams, is automatically generated and cannot drift out of sync with the YAML content.
+
+Furthermore, consistency is enforced within the YAML config itself, by using a single source of data to generate
+multiple parts of the file.
+
+* Facilitates one-off culling/enabling of CI configs for testing PRs on special targets
+
+Also see https://github.com/pytorch/pytorch/issues/17038
+
+
+Future direction
+----------------
+
+### Declaring sparse config subsets
+See comment [here](https://github.com/pytorch/pytorch/pull/17323#pullrequestreview-206945747):
+
+In contrast with a full recursive tree traversal of configuration dimensions,
+> in the future future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.

.circleci/cimodel/data/binary_build_data.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+
+"""
+This module models the tree of configuration variants
+for "smoketest" builds.
+
+Each subclass of ConfigNode represents a layer of the configuration hierarchy.
+These tree nodes encapsulate the logic for whether a branch of the hierarchy
+should be "pruned".
+
+In addition to generating config.yml content, the tree is also traversed
+to produce a visualization of config dimensions.
+"""
+
+from collections import OrderedDict
+
+from cimodel.lib.conf_tree import ConfigNode
+import cimodel.data.dimensions as dimensions
+
+
+LINKING_DIMENSIONS = [
+    "shared",
+    "static",
+]
+
+
+DEPS_INCLUSION_DIMENSIONS = [
+    "with-deps",
+    "without-deps",
+]
+
+
+def get_processor_arch_name(cuda_version):
+    return "cpu" if not cuda_version else "cu" + cuda_version
+
+
+LINUX_PACKAGE_VARIANTS = OrderedDict(
+    manywheel=[
+        "2.7m",
+        "2.7mu",
+        "3.5m",
+        "3.6m",
+        "3.7m",
+    ],
+    conda=dimensions.STANDARD_PYTHON_VERSIONS,
+    libtorch=[
+        "2.7m",
+    ],
+)
+
+CONFIG_TREE_DATA = OrderedDict(
+    linux=(dimensions.CUDA_VERSIONS, LINUX_PACKAGE_VARIANTS),
+    macos=([None], OrderedDict(
+        wheel=dimensions.STANDARD_PYTHON_VERSIONS,
+        conda=dimensions.STANDARD_PYTHON_VERSIONS,
+        libtorch=[
+            "2.7",
+        ],
+    )),
+)
+
+
+DEVTOOLSET_VERSIONS = [3, 7]
+
+
+class TopLevelNode(ConfigNode):
+    def __init__(self, node_name, config_tree_data, smoke):
+        super(TopLevelNode, self).__init__(None, node_name)
+
+        self.config_tree_data = config_tree_data
+        self.props["smoke"] = smoke
+
+    def get_children(self):
+        return [OSConfigNode(self, x, c, p) for (x, (c, p)) in self.config_tree_data.items()]
+
+
+class OSConfigNode(ConfigNode):
+    def __init__(self, parent, os_name, cuda_versions, py_tree):
+        super(OSConfigNode, self).__init__(parent, os_name)
+
+        self.py_tree = py_tree
+        self.props["os_name"] = os_name
+        self.props["cuda_versions"] = cuda_versions
+
+    def get_children(self):
+        return [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
+
+
+class PackageFormatConfigNode(ConfigNode):
+    def __init__(self, parent, package_format, python_versions):
+        super(PackageFormatConfigNode, self).__init__(parent, package_format)
+
+        self.props["python_versions"] = python_versions
+        self.props["package_format"] = package_format
+
+    def get_children(self):
+        if self.find_prop("os_name") == "linux" and self.find_prop("package_format") != "conda":
+            return [LinuxGccConfigNode(self, v) for v in DEVTOOLSET_VERSIONS]
+        else:
+            return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
+
+
+class LinuxGccConfigNode(ConfigNode):
+    def __init__(self, parent, devtoolset_version):
+        super(LinuxGccConfigNode, self).__init__(parent, "DEVTOOLSET=" + str(devtoolset_version))
+
+        self.props["devtoolset_version"] = devtoolset_version
+
+    def get_children(self):
+        return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
+
+
+class ArchConfigNode(ConfigNode):
+    def __init__(self, parent, cu):
+        super(ArchConfigNode, self).__init__(parent, get_processor_arch_name(cu))
+
+        self.props["cu"] = cu
+
+    def get_children(self):
+        return [PyVersionConfigNode(self, v) for v in self.find_prop("python_versions")]
+
+
+class PyVersionConfigNode(ConfigNode):
+    def __init__(self, parent, pyver):
+        super(PyVersionConfigNode, self).__init__(parent, pyver)
+
+        self.props["pyver"] = pyver
+
+    def get_children(self):
+
+        smoke = self.find_prop("smoke")
+        package_format = self.find_prop("package_format")
+        os_name = self.find_prop("os_name")
+
+        has_libtorch_variants = smoke and package_format == "libtorch" and os_name == "linux"
+        linking_variants = LINKING_DIMENSIONS if has_libtorch_variants else []
+
+        return [LinkingVariantConfigNode(self, v) for v in linking_variants]
+
+
+class LinkingVariantConfigNode(ConfigNode):
+    def __init__(self, parent, linking_variant):
+        super(LinkingVariantConfigNode, self).__init__(parent, linking_variant)
+
+    def get_children(self):
+        return [DependencyInclusionConfigNode(self, v) for v in DEPS_INCLUSION_DIMENSIONS]
+
+
+class DependencyInclusionConfigNode(ConfigNode):
+    def __init__(self, parent, deps_variant):
+        super(DependencyInclusionConfigNode, self).__init__(parent, deps_variant)
+
+        self.props["libtorch_variant"] = "-".join([self.parent.get_label(), self.get_label()])

.circleci/cimodel/data/binary_build_definitions.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+
+from collections import OrderedDict
+
+import cimodel.data.binary_build_data as binary_build_data
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.visualization as visualization
+
+
+class Conf(object):
+    def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, devtoolset_version):
+
+        self.os = os
+        self.cuda_version = cuda_version
+        self.pydistro = pydistro
+        self.parms = parms
+        self.smoke = smoke
+        self.libtorch_variant = libtorch_variant
+        self.devtoolset_version = devtoolset_version
+
+    def gen_build_env_parms(self):
+        elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
+        if self.devtoolset_version is not None:
+            elems.append("devtoolset" + str(self.devtoolset_version))
+        return elems
+
+    def gen_docker_image(self):
+
+        docker_word_substitution = {
+            "manywheel": "manylinux",
+            "libtorch": "manylinux",
+        }
+
+        docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
+
+        alt_docker_suffix = self.cuda_version or "80"
+        docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
+        return miniutils.quote("soumith/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
+
+    def get_name_prefix(self):
+        return "smoke" if self.smoke else "binary"
+
+    def gen_build_name(self, build_or_test):
+
+        parts = [self.get_name_prefix(), self.os] + self.gen_build_env_parms()
+
+        if self.smoke:
+            if self.libtorch_variant:
+                parts.append(self.libtorch_variant)
+        else:
+            parts.append(build_or_test)
+
+        return "_".join(parts)
+
+    def gen_yaml_tree(self, build_or_test):
+
+        env_tuples = [("BUILD_ENVIRONMENT", miniutils.quote(" ".join(self.gen_build_env_parms())))]
+
+        if self.libtorch_variant:
+            env_tuples.append(("LIBTORCH_VARIANT", miniutils.quote(self.libtorch_variant)))
+
+        os_name = miniutils.override(self.os, {"macos": "mac"})
+        d = {"<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test])}
+
+        if build_or_test == "test":
+
+            if not (self.smoke and self.os == "macos"):
+                env_tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
+
+            if self.cuda_version:
+                env_tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
+
+        else:
+            if self.os == "linux" and build_or_test != "upload":
+                d["docker"] = [{"image": self.gen_docker_image()}]
+
+        d["environment"] = OrderedDict(env_tuples)
+
+        if build_or_test == "test":
+            if self.cuda_version:
+                d["resource_class"] = "gpu.medium"
+
+        return d
+
+
+def get_root(smoke, name):
+
+    return binary_build_data.TopLevelNode(
+        name,
+        binary_build_data.CONFIG_TREE_DATA,
+        smoke,
+    )
+
+
+def gen_build_env_list(smoke):
+
+    root = get_root(smoke, "N/A")
+    config_list = conf_tree.dfs(root)
+
+    newlist = []
+    for c in config_list:
+        conf = Conf(
+            c.find_prop("os_name"),
+            c.find_prop("cu"),
+            c.find_prop("package_format"),
+            [c.find_prop("pyver")],
+            c.find_prop("smoke"),
+            c.find_prop("libtorch_variant"),
+            c.find_prop("devtoolset_version"),
+        )
+        newlist.append(conf)
+
+    return newlist
+
+
+def predicate_exclude_nonlinux_and_libtorch(config):
+    return config.os == "linux" and (config.smoke or config.pydistro != "libtorch")
+
+
+def add_build_entries(jobs_dict, phase, smoke, filter_predicate=lambda x: True):
+
+    configs = gen_build_env_list(smoke)
+    for conf_options in filter(filter_predicate, configs):
+        jobs_dict[conf_options.gen_build_name(phase)] = conf_options.gen_yaml_tree(phase)
+
+
+def add_binary_build_specs(jobs_dict):
+    add_build_entries(jobs_dict, "build", False)
+
+
+def add_binary_build_tests(jobs_dict):
+    add_build_entries(jobs_dict, "test", False, predicate_exclude_nonlinux_and_libtorch)
+
+
+def add_binary_build_uploads(jobs_dict):
+    add_build_entries(jobs_dict, "upload", False)
+
+
+def add_smoke_test_specs(jobs_dict):
+    add_build_entries(jobs_dict, "test", True)
+
+
+def get_nightly_tests():
+
+    configs = gen_build_env_list(False)
+    filtered_configs = filter(predicate_exclude_nonlinux_and_libtorch, configs)
+
+    mylist = []
+    for conf_options in filtered_configs:
+        d = {conf_options.gen_build_name("test"): {"requires": [conf_options.gen_build_name("build")]}}
+        mylist.append(d)
+
+    return mylist
+
+
+def get_nightly_uploads():
+
+    configs = gen_build_env_list(False)
+
+    def gen_config(conf, phase_dependency):
+        return {
+            conf.gen_build_name("upload"): OrderedDict([
+                ("context", "org-member"),
+                ("requires", [conf.gen_build_name(phase_dependency)]),
+            ]),
+        }
+
+    mylist = []
+    for conf in configs:
+        phase_dependency = "test" if predicate_exclude_nonlinux_and_libtorch(conf) else "build"
+        mylist.append(gen_config(conf, phase_dependency))
+
+    return mylist
+
+
+def gen_schedule_tree(cron_timing):
+    return [{
+        "schedule": {
+            "cron": miniutils.quote(cron_timing),
+            "filters": {
+                "branches": {
+                    "only": ["master"],
+                },
+            },
+        },
+    }]
+
+
+def add_jobs_and_render(jobs_dict, toplevel_key, smoke, cron_schedule):
+
+    jobs_list = []
+
+    configs = gen_build_env_list(smoke)
+    for build_config in configs:
+        build_name = build_config.gen_build_name("build")
+        jobs_list.append(build_name)
+
+    jobs_dict[toplevel_key] = OrderedDict(
+        triggers=gen_schedule_tree(cron_schedule),
+        jobs=jobs_list,
+    )
+
+    graph = visualization.generate_graph(get_root(smoke, toplevel_key))
+    graph.draw(toplevel_key + "-config-dimensions.png", prog="twopi")
+
+
+def add_binary_build_jobs(jobs_dict):
+    add_jobs_and_render(jobs_dict, "binarybuilds", False, "5 5 * * *")
+
+
+def add_binary_smoke_test_jobs(jobs_dict):
+    add_jobs_and_render(jobs_dict, "binarysmoketests", True, "15 16 * * *")

.circleci/cimodel/data/caffe2_build_data.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+from cimodel.lib.conf_tree import ConfigNode, X
+from cimodel.lib.conf_tree import Ver
+import cimodel.data.dimensions as dimensions
+
+
+CONFIG_TREE_DATA = [
+    (Ver("ubuntu", "14.04"), [
+        (Ver("gcc", "4.8"), [X("py2")]),
+        (Ver("gcc", "4.9"), [X("py2")]),
+    ]),
+    (Ver("ubuntu", "16.04"), [
+        (Ver("cuda", "8.0"), [X("py2")]),
+        (Ver("cuda", "9.0"), [
+            # TODO make explicit that this is a "secret TensorRT build"
+            #  (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749)
+            X("py2"),
+            X("cmake"),
+        ]),
+        (Ver("cuda", "9.1"), [X("py2")]),
+        (Ver("mkl"), [X("py2")]),
+        (Ver("gcc", "5"), [X("onnx_py2")]),
+        (Ver("clang", "3.8"), [X("py2")]),
+        (Ver("clang", "3.9"), [X("py2")]),
+        (Ver("clang", "7"), [X("py2")]),
+        (Ver("android"), [X("py2")]),
+    ]),
+    (Ver("centos", "7"), [
+        (Ver("cuda", "9.0"), [X("py2")]),
+    ]),
+    (Ver("macos", "10.13"), [
+        # TODO ios and system aren't related. system qualifies where the python comes
+        #  from (use the system python instead of homebrew or anaconda)
+        (Ver("ios"), [X("py2")]),
+        (Ver("system"), [X("py2")]),
+    ]),
+]
+
+
+class TreeConfigNode(ConfigNode):
+    def __init__(self, parent, node_name, subtree):
+        super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
+        self.subtree = subtree
+        self.init2(node_name)
+
+    # noinspection PyMethodMayBeStatic
+    def modify_label(self, label):
+        return str(label)
+
+    def init2(self, node_name):
+        pass
+
+    def get_children(self):
+        return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
+
+    def is_build_only(self):
+        return str(self.find_prop("compiler_version")) in [
+            "gcc4.9",
+            "clang3.8",
+            "clang3.9",
+            "clang7",
+            "android",
+        ] or self.find_prop("distro_version").name == "macos"
+
+
+class TopLevelNode(TreeConfigNode):
+    def __init__(self, node_name, subtree):
+        super(TopLevelNode, self).__init__(None, node_name, subtree)
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return DistroConfigNode
+
+
+class DistroConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["distro_version"] = node_name
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return CompilerConfigNode
+
+
+class CompilerConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["compiler_version"] = node_name
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return LanguageConfigNode
+
+
+class LanguageConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["language_version"] = node_name
+        self.props["build_only"] = self.is_build_only()
+
+    def get_children(self):
+
+        children = []
+        for phase in dimensions.PHASES:
+            if phase == "build" or not self.props["build_only"]:
+                children.append(PhaseConfigNode(self, phase, []))
+
+        return children
+
+
+class PhaseConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["phase_name"] = node_name

.circleci/cimodel/data/caffe2_build_definitions.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+
+from collections import OrderedDict
+
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.visualization as visualization
+from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode
+
+
+DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
+
+DOCKER_IMAGE_VERSION = 266
+
+
+class Conf(object):
+    def __init__(self, language, distro, compiler, phase, build_only):
+
+        self.language = language
+        self.distro = distro
+        self.compiler = compiler
+        self.phase = phase
+        self.build_only = build_only
+
+    # TODO: Eventually we can probably just remove the cudnn7 everywhere.
+    def get_cudnn_insertion(self):
+
+        omit = self.language == "onnx_py2" \
+            or self.compiler.name in ["android", "mkl", "clang"] \
+            or str(self.distro) in ["ubuntu14.04", "macos10.13"]
+
+        return [] if omit else ["cudnn7"]
+
+    def get_build_name_root_parts(self):
+        return [
+            "caffe2",
+            self.language,
+        ] + self.get_build_name_middle_parts()
+
+    def get_build_name_middle_parts(self):
+        return [str(self.compiler)] + self.get_cudnn_insertion() + [str(self.distro)]
+
+    def construct_phase_name(self, phase):
+        root_parts = self.get_build_name_root_parts()
+        return "_".join(root_parts + [phase]).replace(".", "_")
+
+    def get_name(self):
+        return self.construct_phase_name(self.phase)
+
+    def get_platform(self):
+        platform = self.distro.name
+        if self.distro.name != "macos":
+            platform = "linux"
+        return platform
+
+    def gen_docker_image(self):
+
+        lang_substitutions = {
+            "onnx_py2": "py2",
+            "cmake": "py2",
+        }
+
+        lang = miniutils.override(self.language, lang_substitutions)
+        parts = [lang] + self.get_build_name_middle_parts()
+        return miniutils.quote(DOCKER_IMAGE_PATH_BASE + "-".join(parts) + ":" + str(DOCKER_IMAGE_VERSION))
+
+    def gen_yaml_tree(self):
+
+        tuples = []
+
+        lang_substitutions = {
+            "onnx_py2": "onnx-py2",
+        }
+
+        lang = miniutils.override(self.language, lang_substitutions)
+
+        parts = [
+            "caffe2",
+            lang,
+        ] + self.get_build_name_middle_parts() + [self.phase]
+
+        build_env = "-".join(parts)
+        if not self.distro.name == "macos":
+            build_env = miniutils.quote(build_env)
+
+        tuples.append(("BUILD_ENVIRONMENT", build_env))
+
+        if self.compiler.name == "ios":
+            tuples.append(("BUILD_IOS", miniutils.quote("1")))
+
+        if self.phase == "test":
+            # TODO cuda should not be considered a compiler
+            if self.compiler.name == "cuda":
+                tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
+
+        if self.distro.name == "macos":
+            tuples.append(("PYTHON_VERSION", miniutils.quote("2")))
+
+        else:
+            tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
+            if self.build_only:
+                tuples.append(("BUILD_ONLY", miniutils.quote("1")))
+
+        d = OrderedDict({"environment": OrderedDict(tuples)})
+
+        if self.phase == "test":
+            resource_class = "large" if self.compiler.name != "cuda" else "gpu.medium"
+            d["resource_class"] = resource_class
+
+        d["<<"] = "*" + "_".join(["caffe2", self.get_platform(), self.phase, "defaults"])
+
+        return d
+
+
+def get_root():
+    return TopLevelNode("Caffe2 Builds", CONFIG_TREE_DATA)
+
+
+def instantiate_configs():
+
+    config_list = []
+
+    root = get_root()
+    found_configs = conf_tree.dfs(root)
+    for fc in found_configs:
+
+        c = Conf(
+            fc.find_prop("language_version"),
+            fc.find_prop("distro_version"),
+            fc.find_prop("compiler_version"),
+            fc.find_prop("phase_name"),
+            fc.find_prop("build_only"),
+        )
+
+        config_list.append(c)
+
+    return config_list
+
+
+def add_caffe2_builds(jobs_dict):
+
+    configs = instantiate_configs()
+    for conf_options in configs:
+        jobs_dict[conf_options.get_name()] = conf_options.gen_yaml_tree()
+
+    graph = visualization.generate_graph(get_root())
+    graph.draw("caffe2-config-dimensions.png", prog="twopi")
+
+
+def get_caffe2_workflows():
+
+    configs = instantiate_configs()
+
+    # TODO Why don't we build this config?
+    # See https://github.com/pytorch/pytorch/pull/17323#discussion_r259450540
+    filtered_configs = filter(lambda x: not (str(x.distro) == "ubuntu14.04" and str(x.compiler) == "gcc4.9"), configs)
+
+    x = []
+    for conf_options in filtered_configs:
+        item = conf_options.get_name()
+
+        if conf_options.phase == "test":
+            item = {conf_options.get_name(): {"requires": [conf_options.construct_phase_name("build")]}}
+        x.append(item)
+
+    return x

.circleci/cimodel/data/dimensions.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+
+PHASES = ["build", "test"]
+
+CUDA_VERSIONS = [
+    None,  # cpu build
+    "80",
+    "90",
+    "100",
+]
+
+STANDARD_PYTHON_VERSIONS = [
+    "2.7",
+    "3.5",
+    "3.6",
+    "3.7",
+]

.circleci/cimodel/data/pytorch_build_data.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+from cimodel.lib.conf_tree import ConfigNode, X
+
+
+CONFIG_TREE_DATA = [
+    ("trusty", [
+        (None, [
+            X("2.7.9"),
+            X("2.7"),
+            X("3.5"),
+            X("nightly"),
+        ]),
+        ("gcc", [
+            ("4.8", [X("3.6")]),
+            ("5.4", [("3.6", [X(False), X(True)])]),
+            ("7", [X("3.6")]),
+        ]),
+    ]),
+    ("xenial", [
+        ("clang", [
+            ("5", [X("3.6")]),
+        ]),
+        ("cuda", [
+            ("8", [X("3.6")]),
+            ("9", [
+                # Note there are magic strings here
+                # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21
+                # and
+                # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L143
+                # and
+                # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153
+                # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144)
+                X("2.7"),
+                X("3.6"),
+            ]),
+            ("9.2", [X("3.6")]),
+            ("10", [X("3.6")]),
+        ]),
+        ("android", [
+            ("r19c", [X("3.6")]),
+        ]),
+    ]),
+]
+
+
+def get_major_pyver(dotted_version):
+    parts = dotted_version.split(".")
+    return "py" + parts[0]
+
+
+class TreeConfigNode(ConfigNode):
+    def __init__(self, parent, node_name, subtree):
+        super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
+        self.subtree = subtree
+        self.init2(node_name)
+
+    def modify_label(self, label):
+        return label
+
+    def init2(self, node_name):
+        pass
+
+    def get_children(self):
+        return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
+
+
+class TopLevelNode(TreeConfigNode):
+    def __init__(self, node_name, subtree):
+        super(TopLevelNode, self).__init__(None, node_name, subtree)
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return DistroConfigNode
+
+
+class DistroConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["distro_name"] = node_name
+
+    def child_constructor(self):
+        distro = self.find_prop("distro_name")
+
+        next_nodes = {
+            "trusty": TrustyCompilerConfigNode,
+            "xenial": XenialCompilerConfigNode,
+        }
+        return next_nodes[distro]
+
+
+class TrustyCompilerConfigNode(TreeConfigNode):
+
+    def modify_label(self, label):
+        return label or "<unspecified>"
+
+    def init2(self, node_name):
+        self.props["compiler_name"] = node_name
+
+    def child_constructor(self):
+        return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
+
+
+class TrustyCompilerVersionConfigNode(TreeConfigNode):
+
+    def init2(self, node_name):
+        self.props["compiler_version"] = node_name
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return PyVerConfigNode
+
+
+class PyVerConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["pyver"] = node_name
+        self.props["abbreviated_pyver"] = get_major_pyver(node_name)
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return XlaConfigNode
+
+
+class XlaConfigNode(TreeConfigNode):
+    def modify_label(self, label):
+        return "XLA=" + str(label)
+
+    def init2(self, node_name):
+        self.props["is_xla"] = node_name
+
+
+class XenialCompilerConfigNode(TreeConfigNode):
+
+    def init2(self, node_name):
+        self.props["compiler_name"] = node_name
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return XenialCompilerVersionConfigNode
+
+
+class XenialCompilerVersionConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["compiler_version"] = node_name
+
+    # noinspection PyMethodMayBeStatic
+    def child_constructor(self):
+        return PyVerConfigNode

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+
+from collections import OrderedDict
+
+from cimodel.data.pytorch_build_data import TopLevelNode, CONFIG_TREE_DATA
+import cimodel.data.dimensions as dimensions
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.visualization as visualization
+
+
+DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
+
+DOCKER_IMAGE_VERSION = 300
+
+
+class Conf(object):
+    def __init__(self,
+                 distro,
+                 parms,
+                 pyver=None,
+                 cuda_version=None,
+                 is_xla=False,
+                 restrict_phases=None,
+                 gpu_resource=None,
+                 dependent_tests=None,
+                 parent_build=None):
+
+        self.distro = distro
+        self.pyver = pyver
+        self.parms = parms
+        self.cuda_version = cuda_version
+
+        # TODO expand this to cover all the USE_* that we want to test for
+        #  tesnrorrt, leveldb, lmdb, redis, opencv, mkldnn, ideep, etc.
+        # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453608)
+        self.is_xla = is_xla
+
+        self.restrict_phases = restrict_phases
+        self.gpu_resource = gpu_resource
+        self.dependent_tests = dependent_tests or []
+        self.parent_build = parent_build
+
+    # TODO: Eliminate the special casing for docker paths
+    # In the short term, we *will* need to support special casing as docker images are merged for caffe2 and pytorch
+    def get_parms(self, for_docker):
+        leading = ["pytorch"]
+        if self.is_xla and not for_docker:
+            leading.append("xla")
+
+        cuda_parms = []
+        if self.cuda_version:
+            cuda_parms.extend(["cuda" + self.cuda_version, "cudnn7"])
+        return leading + ["linux", self.distro] + cuda_parms + self.parms
+
+    def gen_docker_image_path(self):
+
+        parms_source = self.parent_build or self
+        base_build_env_name = "-".join(parms_source.get_parms(True))
+
+        return miniutils.quote(DOCKER_IMAGE_PATH_BASE + base_build_env_name + ":" + str(DOCKER_IMAGE_VERSION))
+
+    def get_build_job_name_pieces(self, build_or_test):
+        return self.get_parms(False) + [build_or_test]
+
+    def gen_build_name(self, build_or_test):
+        return ("_".join(map(str, self.get_build_job_name_pieces(build_or_test)))).replace(".", "_").replace("-", "_")
+
+    def get_dependents(self):
+        return self.dependent_tests or []
+
+    def gen_yaml_tree(self, build_or_test):
+
+        build_job_name_pieces = self.get_build_job_name_pieces(build_or_test)
+
+        build_env_name = "-".join(map(str, build_job_name_pieces))
+
+        env_dict = OrderedDict([
+            ("BUILD_ENVIRONMENT", build_env_name),
+            ("DOCKER_IMAGE", self.gen_docker_image_path()),
+        ])
+
+        if self.pyver:
+            env_dict["PYTHON_VERSION"] = miniutils.quote(self.pyver)
+
+        if build_or_test == "test" and self.gpu_resource:
+            env_dict["USE_CUDA_DOCKER_RUNTIME"] = miniutils.quote("1")
+
+        d = {
+            "environment": env_dict,
+            "<<": "*" + "_".join(["pytorch", "linux", build_or_test, "defaults"]),
+        }
+
+        if build_or_test == "test":
+            resource_class = "large"
+            if self.gpu_resource:
+                resource_class = "gpu." + self.gpu_resource
+
+                if self.gpu_resource == "large":
+                    env_dict["MULTI_GPU"] = miniutils.quote("1")
+
+            d["resource_class"] = resource_class
+
+        return d
+
+    def gen_workflow_yaml_item(self, phase):
+
+        if phase == "test":
+            val = OrderedDict()
+
+            # TODO When merging the caffe2 and pytorch jobs, it might be convenient for a while to make a
+            #  caffe2 test job dependent on a pytorch build job. This way we could quickly dedup the repeated
+            #  build of pytorch in the caffe2 build job, and just run the caffe2 tests off of a completed
+            #  pytorch build job (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259452641)
+
+            dependency_build = self.parent_build or self
+            val["requires"] = [dependency_build.gen_build_name("build")]
+
+            return {self.gen_build_name(phase): val}
+        else:
+            return self.gen_build_name(phase)
+
+
+# TODO This is a hack to special case some configs just for the workflow list
+class HiddenConf(object):
+    def __init__(self, name, parent_build=None):
+        self.name = name
+        self.parent_build = parent_build
+
+    def gen_workflow_yaml_item(self, phase):
+
+        return {self.gen_build_name(phase): {"requires": [self.parent_build.gen_build_name("build")]}}
+
+    def gen_build_name(self, _):
+        return self.name
+
+
+# TODO Convert these to graph nodes
+def gen_dependent_configs(xenial_parent_config):
+
+    extra_parms = [
+        (["multigpu"], "large"),
+        (["NO_AVX2"], "medium"),
+        (["NO_AVX", "NO_AVX2"], "medium"),
+        (["slow"], "medium"),
+        (["nogpu"], None),
+    ]
+
+    configs = []
+    for parms, gpu in extra_parms:
+
+        c = Conf(
+            xenial_parent_config.distro,
+            ["py3"] + parms,
+            pyver="3.6",
+            cuda_version=xenial_parent_config.cuda_version,
+            restrict_phases=["test"],
+            gpu_resource=gpu,
+            parent_build=xenial_parent_config,
+        )
+
+        configs.append(c)
+
+    for x in ["pytorch_short_perf_test_gpu", "pytorch_doc_push"]:
+        configs.append(HiddenConf(x, parent_build=xenial_parent_config))
+
+    return configs
+
+
+def get_root():
+    return TopLevelNode("PyTorch Builds", CONFIG_TREE_DATA)
+
+
+def gen_tree():
+    root = get_root()
+    configs_list = conf_tree.dfs(root)
+    return configs_list
+
+
+def instantiate_configs():
+
+    config_list = []
+
+    root = get_root()
+    found_configs = conf_tree.dfs(root)
+    restrict_phases = None
+    for fc in found_configs:
+
+        distro_name = fc.find_prop("distro_name")
+
+        python_version = None
+        if distro_name == "xenial":
+            python_version = fc.find_prop("pyver")
+            parms_list = [fc.find_prop("abbreviated_pyver")]
+        else:
+            parms_list = ["py" + fc.find_prop("pyver")]
+
+        compiler_name = fc.find_prop("compiler_name")
+
+        cuda_version = None
+        if compiler_name == "cuda":
+            cuda_version = fc.find_prop("compiler_version")
+
+        elif compiler_name == "android":
+            android_ndk_version = fc.find_prop("compiler_version")
+            # TODO: do we need clang to compile host binaries like protoc?
+            parms_list.append("clang5")
+            parms_list.append("android-ndk-" + android_ndk_version)
+            restrict_phases = ["build"]
+
+        elif compiler_name:
+            gcc_version = compiler_name + (fc.find_prop("compiler_version") or "")
+            parms_list.append(gcc_version)
+
+            if compiler_name == "clang":
+                parms_list.append("asan")
+
+        if cuda_version in ["9.2", "10"]:
+            # TODO The gcc version is orthogonal to CUDA version?
+            parms_list.append("gcc7")
+
+        is_xla = fc.find_prop("is_xla") or False
+
+        gpu_resource = None
+        if cuda_version and cuda_version != "10":
+            gpu_resource = "medium"
+
+        c = Conf(
+            distro_name,
+            parms_list,
+            python_version,
+            cuda_version,
+            is_xla,
+            restrict_phases,
+            gpu_resource,
+        )
+
+        if cuda_version == "8":
+            c.dependent_tests = gen_dependent_configs(c)
+
+        config_list.append(c)
+
+    return config_list
+
+
+def add_build_env_defs(jobs_dict):
+
+    mydict = OrderedDict()
+
+    config_list = instantiate_configs()
+    for c in config_list:
+
+        phases = c.restrict_phases or dimensions.PHASES
+
+        for phase in phases:
+
+            # TODO why does this not have a test?
+            if phase == "test" and c.cuda_version == "10":
+                continue
+
+            d = c.gen_yaml_tree(phase)
+            mydict[c.gen_build_name(phase)] = d
+
+            if phase == "test":
+                for x in filter(lambda x: type(x) is not HiddenConf, c.get_dependents()):
+
+                    d = x.gen_yaml_tree(phase)
+                    mydict[x.gen_build_name(phase)] = d
+
+    # this is the circleci api version and probably never changes
+    jobs_dict["version"] = 2
+    jobs_dict["jobs"] = mydict
+
+    graph = visualization.generate_graph(get_root())
+    graph.draw("pytorch-config-dimensions.png", prog="twopi")
+
+
+def get_workflow_list():
+
+    config_list = instantiate_configs()
+
+    x = []
+    for conf_options in config_list:
+
+        phases = conf_options.restrict_phases or dimensions.PHASES
+
+        for phase in phases:
+
+            # TODO why does this not have a test?
+            if phase == "test" and conf_options.cuda_version == "10":
+                continue
+
+            x.append(conf_options.gen_workflow_yaml_item(phase))
+
+        # TODO convert to recursion
+        for conf in conf_options.get_dependents():
+            x.append(conf.gen_workflow_yaml_item("test"))
+
+    return x

.circleci/cimodel/lib/conf_tree.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+
+def X(val):
+    """
+    Compact way to write a leaf node
+    """
+    return val, []
+
+
+class Ver(object):
+    """
+    Represents a product with a version number
+    """
+    def __init__(self, name, version=""):
+        self.name = name
+        self.version = version
+
+    def __str__(self):
+        return self.name + self.version
+
+
+class ConfigNode(object):
+    def __init__(self, parent, node_name):
+        self.parent = parent
+        self.node_name = node_name
+        self.props = {}
+
+    def get_label(self):
+        return self.node_name
+
+    # noinspection PyMethodMayBeStatic
+    def get_children(self):
+        return []
+
+    def get_parents(self):
+        return (self.parent.get_parents() + [self.parent.get_label()]) if self.parent else []
+
+    def get_depth(self):
+        return len(self.get_parents())
+
+    def get_node_key(self):
+        return "%".join(self.get_parents() + [self.get_label()])
+
+    def find_prop(self, propname, searched=None):
+        """
+        Checks if its own dictionary has
+        the property, otherwise asks parent node.
+        """
+
+        if searched is None:
+            searched = []
+
+        searched.append(self.node_name)
+
+        if propname in self.props:
+            return self.props[propname]
+        elif self.parent:
+            return self.parent.find_prop(propname, searched)
+        else:
+            # raise Exception('Property "%s" does not exist anywhere in the tree! Searched: %s' % (propname, searched))
+            return None
+
+
+def dfs_recurse(
+        node,
+        leaf_callback=lambda x: None,
+        discovery_callback=lambda x, y, z: None,
+        child_callback=lambda x, y: None,
+        sibling_index=0,
+        sibling_count=1):
+
+    discovery_callback(node, sibling_index, sibling_count)
+
+    node_children = node.get_children()
+    if node_children:
+        for i, child in enumerate(node_children):
+            child_callback(node, child)
+
+            dfs_recurse(
+                child,
+                leaf_callback,
+                discovery_callback,
+                child_callback,
+                i,
+                len(node_children),
+            )
+    else:
+        leaf_callback(node)
+
+
+def dfs(toplevel_config_node):
+
+    config_list = []
+
+    def leaf_callback(node):
+        config_list.append(node)
+
+    dfs_recurse(toplevel_config_node, leaf_callback)
+
+    return config_list

.circleci/cimodel/lib/miniutils.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+
+def quote(s):
+    return sandwich('"', s)
+
+
+def sandwich(bread, jam):
+    return bread + jam + bread
+
+
+def override(word, substitutions):
+    return substitutions.get(word, word)

.circleci/cimodel/lib/miniyaml.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+
+from collections import OrderedDict
+
+
+LIST_MARKER = "- "
+INDENTATION_WIDTH = 2
+
+
+def is_dict(data):
+    return type(data) is dict or type(data) is OrderedDict
+
+
+def is_collection(data):
+    return is_dict(data) or type(data) is list
+
+
+# TODO can eventually drop this custom sorting
+def sortkey(x):
+    k = x[0]
+    return (
+        k == "<<",
+        k != "environment",
+        k,
+    )
+
+
+def render(fh, data, depth, is_list_member=False):
+    """
+    PyYaml does not allow precise control over the quoting
+    behavior, especially for merge references.
+    Therefore, we use this custom YAML renderer.
+    """
+
+    indentation = " " * INDENTATION_WIDTH * depth
+
+    if is_dict(data):
+
+        tuples = list(data.items())
+        if type(data) is not OrderedDict:
+            tuples.sort(key=sortkey)
+
+        for i, (k, v) in enumerate(tuples):
+
+            # If this dict is itself a list member, the first key gets prefixed with a list marker
+            list_marker_prefix = LIST_MARKER if is_list_member and not i else ""
+
+            trailing_whitespace = "\n" if is_collection(v) else " "
+            fh.write(indentation + list_marker_prefix + k + ":" + trailing_whitespace)
+
+            render(fh, v, depth + 1 + int(is_list_member))
+
+        # TODO Could eventually drop this cosmetic convention
+        if depth == 2:
+            fh.write("\n")
+
+    elif type(data) is list:
+        for v in data:
+            render(fh, v, depth, True)
+
+    else:
+        list_member_prefix = indentation + LIST_MARKER if is_list_member else ""
+        fh.write(list_member_prefix + str(data) + "\n")

.circleci/cimodel/lib/visualization.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+"""
+This module encapsulates dependencies on pygraphviz
+"""
+
+import colorsys
+
+import cimodel.lib.conf_tree as conf_tree
+
+
+def rgb2hex(rgb_tuple):
+    def to_hex(f):
+        return "%02x" % int(f * 255)
+
+    return "#" + "".join(map(to_hex, list(rgb_tuple)))
+
+
+def handle_missing_graphviz(f):
+    """
+    If the user has not installed pygraphviz, this causes
+    calls to the draw() method of the returned object to do nothing.
+    """
+    try:
+        import pygraphviz  # noqa: F401
+        return f
+
+    except ModuleNotFoundError:
+
+        class FakeGraph:
+            def draw(self, *args, **kwargs):
+                pass
+
+        return lambda _: FakeGraph()
+
+
+@handle_missing_graphviz
+def generate_graph(toplevel_config_node):
+    """
+    Traverses the graph once first just to find the max depth
+    """
+
+    config_list = conf_tree.dfs(toplevel_config_node)
+
+    max_depth = 0
+    for config in config_list:
+        max_depth = max(max_depth, config.get_depth())
+
+    # color the nodes using the max depth
+
+    from pygraphviz import AGraph
+    dot = AGraph()
+
+    def node_discovery_callback(node, sibling_index, sibling_count):
+        depth = node.get_depth()
+
+        sat_min, sat_max = 0.1, 0.6
+        sat_range = sat_max - sat_min
+
+        saturation_fraction = sibling_index / float(sibling_count - 1) if sibling_count > 1 else 1
+        saturation = sat_min + sat_range * saturation_fraction
+
+        # TODO Use a hash of the node label to determine the color
+        hue = depth / float(max_depth + 1)
+
+        rgb_tuple = colorsys.hsv_to_rgb(hue, saturation, 1)
+
+        this_node_key = node.get_node_key()
+
+        dot.add_node(
+            this_node_key,
+            label=node.get_label(),
+            style="filled",
+            # fillcolor=hex_color + ":orange",
+            fillcolor=rgb2hex(rgb_tuple),
+            penwidth=3,
+            color=rgb2hex(colorsys.hsv_to_rgb(hue, saturation, 0.9))
+        )
+
+    def child_callback(node, child):
+        this_node_key = node.get_node_key()
+        child_node_key = child.get_node_key()
+        dot.add_edge((this_node_key, child_node_key))
+
+    conf_tree.dfs_recurse(toplevel_config_node, lambda x: None, node_discovery_callback, child_callback)
+    return dot

.circleci/ensure-consistency.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import tempfile
+
+import generate_config_yml
+
+
+CHECKED_IN_FILE = "config.yml"
+REGENERATION_SCRIPT = "regenerate.sh"
+
+PARENT_DIR = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
+README_PATH = os.path.join(PARENT_DIR, "README.md")
+
+ERROR_MESSAGE_TEMPLATE = """
+The checked-in CircleCI "%s" file does not match what was generated by the scripts.
+Please re-run the "%s" script in the "%s" directory and commit the result. See "%s" for more information.
+"""
+
+
+def check_consistency():
+
+    _, temp_filename = tempfile.mkstemp("-generated-config.yml")
+
+    with open(temp_filename, "w") as fh:
+        generate_config_yml.stitch_sources(fh)
+
+    try:
+        subprocess.check_call(["cmp", temp_filename, CHECKED_IN_FILE])
+    except subprocess.CalledProcessError:
+        sys.exit(ERROR_MESSAGE_TEMPLATE % (CHECKED_IN_FILE, REGENERATION_SCRIPT, PARENT_DIR, README_PATH))
+    finally:
+        os.remove(temp_filename)
+
+
+if __name__ == "__main__":
+    check_consistency()

.circleci/generate_config_yml.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+"""
+This script is the source of truth for config.yml.
+Please see README.md in this directory for details.
+"""
+
+import os
+import sys
+import shutil
+from collections import namedtuple, OrderedDict
+
+import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
+import cimodel.data.binary_build_definitions as binary_build_definitions
+import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.miniyaml as miniyaml
+
+
+class File(object):
+    """
+    Verbatim copy the contents of a file into config.yml
+    """
+    def __init__(self, filename):
+        self.filename = filename
+
+    def write(self, output_filehandle):
+        with open(os.path.join("verbatim-sources", self.filename)) as fh:
+            shutil.copyfileobj(fh, output_filehandle)
+
+
+class FunctionGen(namedtuple('FunctionGen', 'function depth')):
+    __slots__ = ()
+
+
+class Treegen(FunctionGen):
+    """
+    Insert the content of a YAML tree into config.yml
+    """
+
+    def write(self, output_filehandle):
+        build_dict = OrderedDict()
+        self.function(build_dict)
+        miniyaml.render(output_filehandle, build_dict, self.depth)
+
+
+class Listgen(FunctionGen):
+    """
+    Insert the content of a YAML list into config.yml
+    """
+    def write(self, output_filehandle):
+        miniyaml.render(output_filehandle, self.function(), self.depth)
+
+
+def horizontal_rule():
+    return "".join("#" * 78)
+
+
+class Header(object):
+
+    def __init__(self, title, summary=None):
+        self.title = title
+        self.summary_lines = summary or []
+
+    def write(self, output_filehandle):
+        text_lines = [self.title] + self.summary_lines
+        comment_lines = ["# " + x for x in text_lines]
+        lines = miniutils.sandwich([horizontal_rule()], comment_lines)
+
+        for line in filter(None, lines):
+            output_filehandle.write(line + "\n")
+
+
+# Order of this list matters to the generated config.yml.
+YAML_SOURCES = [
+    File("header-section.yml"),
+    File("linux-build-defaults.yml"),
+    File("macos-build-defaults.yml"),
+    File("nightly-binary-build-defaults.yml"),
+    File("linux-binary-build-defaults.yml"),
+    File("macos-binary-build-defaults.yml"),
+    File("nightly-build-smoke-tests-defaults.yml"),
+    Header("Job specifications job specs"),
+    Treegen(pytorch_build_definitions.add_build_env_defs, 0),
+    File("job-specs-custom.yml"),
+    Treegen(caffe2_build_definitions.add_caffe2_builds, 1),
+    File("binary_update_htmls.yml"),
+    Header("Binary build specs individual job specifications"),
+    Treegen(binary_build_definitions.add_binary_build_specs, 1),
+    Header(
+        "Binary build tests", [
+            "These are the smoke tests run right after the build, before the upload.",
+            "If these fail, the upload doesn't happen."
+        ]
+    ),
+    Treegen(binary_build_definitions.add_binary_build_tests, 1),
+    File("binary-build-tests.yml"),
+    Header("Binary build uploads"),
+    Treegen(binary_build_definitions.add_binary_build_uploads, 1),
+    Header("Smoke test specs individual job specifications"),
+    Treegen(binary_build_definitions.add_smoke_test_specs, 1),
+    File("workflows.yml"),
+    Listgen(pytorch_build_definitions.get_workflow_list, 3),
+    File("workflows-pytorch-macos-builds.yml"),
+    Listgen(caffe2_build_definitions.get_caffe2_workflows, 3),
+    File("workflows-binary-builds-smoke-subset.yml"),
+    Header("Daily smoke test trigger"),
+    Treegen(binary_build_definitions.add_binary_smoke_test_jobs, 1),
+    Header("Daily binary build trigger"),
+    Treegen(binary_build_definitions.add_binary_build_jobs, 1),
+    Header("Nightly tests"),
+    Listgen(binary_build_definitions.get_nightly_tests, 3),
+    File("workflows-nightly-uploads-header.yml"),
+    Listgen(binary_build_definitions.get_nightly_uploads, 3),
+    File("workflows-s3-html.yml"),
+]
+
+
+def stitch_sources(output_filehandle):
+    for f in YAML_SOURCES:
+        f.write(output_filehandle)
+
+
+if __name__ == "__main__":
+
+    stitch_sources(sys.stdout)

.circleci/regenerate.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -xe
+
+# Allows this script to be invoked from any directory:
+cd $(dirname "$0")
+
+NEW_FILE=$(mktemp)
+./generate_config_yml.py > $NEW_FILE
+cp $NEW_FILE config.yml

.circleci/scripts/binary_checkout.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -ex
+# This step runs on multiple executors with different envfile locations
+if [[ "$(uname)" == Darwin ]]; then
+  # macos executor (builds and tests)
+  workdir="/Users/distiller/project"
+elif [[ -d "/home/circleci/project" ]]; then
+  # machine executor (binary tests)
+  workdir="/home/circleci/project"
+else
+  # docker executor (binary builds)
+  workdir="/"
+fi
+
+# It is very important that this stays in sync with binary_populate_env.sh
+export PYTORCH_ROOT="$workdir/pytorch"
+export BUILDER_ROOT="$workdir/builder"
+
+# Clone the Pytorch branch
+git clone https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT"
+pushd "$PYTORCH_ROOT"
+if [[ -n "$CIRCLE_PR_NUMBER" ]]; then
+  # "smoke" binary build on PRs
+  git fetch --force origin "pull/${CIRCLE_PR_NUMBER}/head:remotes/origin/pull/${CIRCLE_PR_NUMBER}"
+  git reset --hard "$CIRCLE_SHA1"
+  git checkout -q -B "$CIRCLE_BRANCH"
+  git reset --hard "$CIRCLE_SHA1"
+elif [[ -n "$CIRCLE_SHA1" ]]; then
+  # "smoke" binary build on master on PR merges
+  git reset --hard "$CIRCLE_SHA1"
+  git checkout -q -B master
+else
+  # nightly binary builds. These run at 05:05 UTC every day. 
+  last_commit="$(git rev-list --before "$(date -u +%Y-%m-%d) 05:00" --max-count 1 HEAD)"
+  git checkout "$last_commit"
+fi
+git submodule update --init --recursive --quiet
+echo "Using Pytorch from "
+git --no-pager log --max-count 1
+popd
+
+# Clone the Builder master repo
+git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT"
+pushd "$BUILDER_ROOT"
+git fetch origin
+git reset origin/master --hard
+echo "Using builder from "
+git --no-pager log --max-count 1
+popd

.circleci/scripts/binary_install_miniconda.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+# This step runs on multiple executors with different envfile locations
+if [[ "$(uname)" == Darwin ]]; then
+  source "/Users/distiller/project/env"
+elif [[ -d "/home/circleci/project" ]]; then
+  # machine executor (binary tests)
+  source "/home/circleci/project/env"
+else
+  # docker executor (binary builds)
+  source "/env"
+fi
+
+conda_sh="$workdir/install_miniconda.sh"
+if [[ "$(uname)" == Darwin ]]; then
+  retry curl -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+else
+  retry curl -o "$conda_sh" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+fi
+chmod +x "$conda_sh"
+"$conda_sh" -b -p "$MINICONDA_ROOT"
+rm -f "$conda_sh"
+export PATH="$MINICONDA_ROOT/bin:$PATH"
+source "$MINICONDA_ROOT/bin/activate"
+# We can't actually add miniconda to the PATH in the envfile, because that
+# breaks 'unbuffer' in Mac jobs. This is probably because conda comes with
+# a tclsh, which then gets inserted before the tclsh needed in /usr/bin

.circleci/scripts/binary_linux_build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+echo "RUNNING ON $(uname -a) WITH $(nproc) CPUS AND $(free -m)"
+set -ex
+source /env
+
+# Defaults here so they can be changed in one place
+export MAX_JOBS=12
+
+# Parse the parameters
+if [[ "$PACKAGE_TYPE" == 'conda' ]]; then
+  build_script='conda/build_pytorch.sh'
+elif [[ "$DESIRED_CUDA" == cpu ]]; then
+  build_script='manywheel/build_cpu.sh'
+else
+  build_script='manywheel/build.sh'
+fi
+
+# We want to call unbuffer, which calls tclsh which finds the expect
+# package. The expect was installed by yum into /usr/bin so we want to
+# find /usr/bin/tclsh, but this is shadowed by /opt/conda/bin/tclsh in
+# the conda docker images.
+if [[ "$PACKAGE_TYPE" == 'conda' ]]; then
+  mkdir /just_tclsh_bin
+  ln -s /usr/bin/tclsh /just_tclsh_bin/tclsh
+  export PATH=/just_tclsh_bin:$PATH
+fi
+
+# Build the package
+SKIP_ALL_TESTS=1 unbuffer "/builder/$build_script" | ts

.circleci/scripts/binary_linux_test.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+source /home/circleci/project/env
+cat >/home/circleci/project/ci_test_script.sh <<EOL
+# =================== The following code will be executed inside Docker container ===================
+set -ex
+
+# Set up Python
+if [[ "$PACKAGE_TYPE" == conda ]]; then
+  retry conda create -qyn testenv python="$DESIRED_PYTHON"
+  source activate testenv >/dev/null
+elif [[ "$DESIRED_PYTHON" == 2.7mu ]]; then
+  export PATH="/opt/python/cp27-cp27mu/bin:\$PATH"
+else
+  python_nodot="\$(echo $DESIRED_PYTHON | tr -d m.u)"
+  export PATH="/opt/python/cp\$python_nodot-cp\${python_nodot}m/bin:\$PATH"
+fi
+
+# Install the package
+# These network calls should not have 'retry's because they are installing
+# locally
+pkg="/final_pkgs/\$(ls /final_pkgs)"
+if [[ "$PACKAGE_TYPE" == conda ]]; then
+  conda install -y "\$pkg" --offline
+else
+  pip install "\$pkg"
+fi
+
+# Test the package
+pushd /pytorch
+/builder/run_tests.sh "$PACKAGE_TYPE" "$DESIRED_PYTHON" "$DESIRED_CUDA"
+# =================== The above code will be executed inside Docker container ===================
+EOL
+echo "Prepared script to run in next step"
+cat /home/circleci/project/ci_test_script.sh

.circleci/scripts/binary_linux_upload.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Do NOT set -e
+source /home/circleci/project/env
+declare -x "AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
+declare -x "AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
+cat >/home/circleci/project/login_to_anaconda.sh <<EOL
+set +x
+echo "Trying to login to Anaconda"
+yes | anaconda login \
+    --username "$PYTORCH_BINARY_SOUMITH_CONDA_USERNAME" \
+    --password "$PYTORCH_BINARY_SOUMITH_CONDA_PASSWORD"
+set -x
+EOL
+chmod +x /home/circleci/project/login_to_anaconda.sh
+
+#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
+# DO NOT TURN -e ON BEFORE THIS LINE
+#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
+set -ex
+export PATH="$MINICONDA_ROOT/bin:$PATH"
+
+# Upload the package to the final location
+pushd /home/circleci/project/final_pkgs
+if [[ "$PACKAGE_TYPE" == conda ]]; then
+  retry conda install -yq anaconda-client
+  retry timeout 30 /home/circleci/project/login_to_anaconda.sh
+  anaconda upload "$(ls)" -u pytorch-testing --label main --no-progress --force
+elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
+  retry pip install -q awscli
+  s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
+  for pkg in $(ls); do
+    retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
+  done
+else
+  retry pip install -q awscli
+  s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
+  retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
+fi

.circleci/scripts/binary_macos_build.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -ex
+
+source "/Users/distiller/project/env"
+mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR"
+
+# For some reason `unbuffer` breaks if we change the PATH here, so we
+# write a script with the PATH change in it and unbuffer the whole
+# thing
+build_script="$workdir/build_script.sh"
+touch "$build_script"
+chmod +x "$build_script"
+
+# Build
+cat >"$build_script" <<EOL
+export PATH="$workdir/miniconda/bin:$PATH"
+if [[ "$PACKAGE_TYPE" == conda ]]; then
+  "$workdir/builder/conda/build_pytorch.sh"
+else
+  export TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '-' '_')"
+  "$workdir/builder/wheel/build_wheel.sh"
+fi
+EOL
+unbuffer "$build_script" | ts

.circleci/scripts/binary_macos_test.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -ex
+
+source "/Users/distiller/project/env"
+export "PATH=$workdir/miniconda/bin:$PATH"
+pkg="$workdir/final_pkgs/$(ls $workdir/final_pkgs)"
+
+# Don't test libtorch
+# TODO we should test libtorch
+if [[ "$PACKAGE_TYPE" == libtorch ]]; then
+  exit 0
+fi
+
+# Create a new test env
+# TODO cut all this out into a separate test job and have an entirely different
+# miniconda
+source deactivate || true
+conda create -qyn test python="$DESIRED_PYTHON"
+source activate test >/dev/null
+
+# Install the package
+if [[ "$PACKAGE_TYPE" == conda ]]; then
+  conda install -y "$pkg" --offline
+else
+  pip install "$pkg" --no-index --no-dependencies -v
+fi
+
+# Test
+pushd "$workdir/pytorch"
+$workdir/builder/run_tests.sh "$PACKAGE_TYPE" "$DESIRED_PYTHON" "$DESIRED_CUDA"

.circleci/scripts/binary_macos_upload.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Do NOT set -e
+export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
+export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
+cat >/Users/distiller/project/login_to_anaconda.sh <<EOL
+set +x
+echo "Trying to login to Anaconda"
+yes | anaconda login \
+    --username "$PYTORCH_BINARY_SOUMITH_CONDA_USERNAME" \
+    --password "$PYTORCH_BINARY_SOUMITH_CONDA_PASSWORD"
+set -x
+EOL
+chmod +x /Users/distiller/project/login_to_anaconda.sh
+
+#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
+# DO NOT TURN -e ON BEFORE THIS LINE
+#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!#!
+set -ex
+
+source "/Users/distiller/project/env"
+export "PATH=$workdir/miniconda/bin:$PATH"
+
+pushd "$workdir/final_pkgs"
+if [[ "$PACKAGE_TYPE" == conda ]]; then
+  retry conda install -yq anaconda-client
+  retry /Users/distiller/project/login_to_anaconda.sh
+  retry anaconda upload "$(ls)" -u pytorch-testing --label main --no-progress --force
+elif [[ "$PACKAGE_TYPE" == libtorch ]]; then
+  retry pip install -q awscli
+  s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
+  for pkg in $(ls); do
+    retry aws s3 cp "$pkg" "$s3_dir" --acl public-read
+  done
+else
+  retry pip install -q awscli
+  s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${DESIRED_CUDA}/"
+  retry aws s3 cp "$(ls)" "$s3_dir" --acl public-read
+fi

.circleci/scripts/binary_populate_env.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+set -ex
+export TZ=UTC
+
+# We need to write an envfile to persist these variables to following
+# steps, but the location of the envfile depends on the circleci executor
+if [[ "$(uname)" == Darwin ]]; then
+  # macos executor (builds and tests)
+  workdir="/Users/distiller/project"
+elif [[ -d "/home/circleci/project" ]]; then
+  # machine executor (binary tests)
+  workdir="/home/circleci/project"
+else
+  # docker executor (binary builds)
+  workdir="/"
+fi
+envfile="$workdir/env"
+touch "$envfile"
+chmod +x "$envfile"
+
+# Parse the BUILD_ENVIRONMENT to package type, python, and cuda
+configs=($BUILD_ENVIRONMENT)
+export PACKAGE_TYPE="${configs[0]}"
+export DESIRED_PYTHON="${configs[1]}"
+export DESIRED_CUDA="${configs[2]}"
+export DESIRED_DEVTOOLSET="${configs[3]}"
+if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
+  export BUILD_PYTHONLESS=1
+fi
+
+# Pick docker image
+if [[ "$PACKAGE_TYPE" == conda ]]; then
+  export DOCKER_IMAGE="soumith/conda-cuda"
+elif [[ "$DESIRED_CUDA" == cpu ]]; then
+  export DOCKER_IMAGE="soumith/manylinux-cuda80"
+else
+  export DOCKER_IMAGE="soumith/manylinux-cuda${DESIRED_CUDA:2}"
+fi
+
+# Upload to parallel folder for gcc abis
+if [[ "$DESIRED_DEVTOOLSET" == 'devtoolset7' ]]; then
+  export PIP_UPLOAD_FOLDER='devtoolset7/'
+  if [[ "$PACKAGE_TYPE" == 'conda' ]]; then
+    echo "We don't handle conda builds with gcc ABI of 1, since we don't"
+    echo "want to add a new package name to the conda builds"
+    exit 1
+  fi
+else
+  export PIP_UPLOAD_FOLDER=''
+fi
+
+# We put this here so that OVERRIDE_PACKAGE_VERSION below can read from it
+export DATE="$(date -u +%Y%m%d)"
+export PYTORCH_BUILD_VERSION="1.1.0"
+export PYTORCH_BUILD_NUMBER=1
+
+cat >>"$envfile" <<EOL
+# =================== The following code will be executed inside Docker container ===================
+export TZ=UTC
+echo "Running on $(uname -a) at $(date)"
+
+export PACKAGE_TYPE="$PACKAGE_TYPE"
+export DESIRED_PYTHON="$DESIRED_PYTHON"
+export DESIRED_CUDA="$DESIRED_CUDA"
+export LIBTORCH_VARIANT="$LIBTORCH_VARIANT"
+export BUILD_PYTHONLESS="$BUILD_PYTHONLESS"
+export DESIRED_DEVTOOLSET="$DESIRED_DEVTOOLSET"
+
+export DATE="$DATE"
+export NIGHTLIES_DATE_PREAMBLE=1.1.0.dev
+export PYTORCH_BUILD_VERSION="$PYTORCH_BUILD_VERSION"
+export PYTORCH_BUILD_NUMBER="$PYTORCH_BUILD_NUMBER"
+export OVERRIDE_PACKAGE_VERSION="$PYTORCH_BUILD_VERSION"
+
+export TORCH_PACKAGE_NAME='torch'
+export TORCH_CONDA_BUILD_FOLDER='pytorch-1.1.0'
+
+export NO_FBGEMM=1
+export PIP_UPLOAD_FOLDER="$PIP_UPLOAD_FOLDER"
+export DOCKER_IMAGE="$DOCKER_IMAGE"
+
+export workdir="$workdir"
+export MAC_PACKAGE_WORK_DIR="$workdir"
+export PYTORCH_ROOT="$workdir/pytorch"
+export BUILDER_ROOT="$workdir/builder"
+export MINICONDA_ROOT="$workdir/miniconda"
+export PYTORCH_FINAL_PACKAGE_DIR="$workdir/final_pkgs"
+
+export CIRCLE_TAG="$CIRCLE_TAG"
+export CIRCLE_SHA1="$CIRCLE_SHA1"
+export CIRCLE_PR_NUMBER="$CIRCLE_PR_NUMBER"
+export CIRCLE_BRANCH="$CIRCLE_BRANCH"
+# =================== The above code will be executed inside Docker container ===================
+EOL
+
+echo 'retry () {' >> "$envfile"
+echo '    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)' >> "$envfile"
+echo '}' >> "$envfile"
+echo 'export -f retry' >> "$envfile"
+
+cat "$envfile"

.circleci/scripts/binary_run_in_docker.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# This section is used in the binary_test and smoke_test jobs. It expects
+# 'binary_populate_env' to have populated /home/circleci/project/env and it
+# expects another section to populate /home/circleci/project/ci_test_script.sh
+# with the code to run in the docker
+
+# Expect all needed environment variables to be written to this file
+source /home/circleci/project/env
+echo "Running the following code in Docker"
+cat /home/circleci/project/ci_test_script.sh
+set -ex
+
+# Expect actual code to be written to this file
+chmod +x /home/circleci/project/ci_test_script.sh
+
+# Run the docker
+if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
+  export id=$(docker run --runtime=nvidia -t -d "${DOCKER_IMAGE}")
+else
+  export id=$(docker run -t -d "${DOCKER_IMAGE}")
+fi
+
+# Copy the envfile and script with all the code to run into the docker.
+docker cp /home/circleci/project/. "$id:/circleci_stuff"
+
+# Copy built packages into the docker to test. This should only exist on the
+# binary test jobs. The package should've been created from a binary build job,
+# whhich persisted the package to a CircleCI workspace, which this job then
+# copies into a GPU enabled docker for testing
+if [[ -d "/home/circleci/project/final_pkgs" ]]; then
+  docker cp /home/circleci/project/final_pkgs "$id:/final_pkgs"
+fi
+
+# Copy the needed repos into the docker. These do not exist in the smoke test
+# jobs, since the smoke test jobs do not need the Pytorch source code.
+if [[ -d "$PYTORCH_ROOT" ]]; then
+  docker cp "$PYTORCH_ROOT" "$id:/pytorch"
+fi
+if [[ -d "$BUILDER_ROOT" ]]; then
+  docker cp "$BUILDER ROOT" "$id:/builder"
+fi
+
+# Execute the test script that was populated by an earlier section
+export COMMAND='((echo "source /circleci_stuff/env && /circleci_stuff/ci_test_script.sh") | docker exec -i "$id" bash) 2>&1'
+echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

.circleci/verbatim-sources/binary-build-tests.yml

@@ -0,0 +1,25 @@
+# There is currently no testing for libtorch TODO
+#  binary_linux_libtorch_2.7m_cpu_test:
+#    environment:
+#      BUILD_ENVIRONMENT: "libtorch 2.7m cpu"
+#    resource_class: gpu.medium
+#    <<: *binary_linux_test
+#
+#  binary_linux_libtorch_2.7m_cu80_test:
+#    environment:
+#      BUILD_ENVIRONMENT: "libtorch 2.7m cu80"
+#    resource_class: gpu.medium
+#    <<: *binary_linux_test
+#
+#  binary_linux_libtorch_2.7m_cu90_test:
+#    environment:
+#      BUILD_ENVIRONMENT: "libtorch 2.7m cu90"
+#    resource_class: gpu.medium
+#    <<: *binary_linux_test
+#
+#  binary_linux_libtorch_2.7m_cu100_test:
+#    environment:
+#      BUILD_ENVIRONMENT: "libtorch 2.7m cu100"
+#    resource_class: gpu.medium
+#    <<: *binary_linux_test
+

.circleci/verbatim-sources/binary_update_htmls.yml

@@ -0,0 +1,47 @@
+# update_s3_htmls job
+  update_s3_htmls: &update_s3_htmls
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+    - run:
+        <<: *setup_linux_system_environment
+    - run:
+        <<: *binary_checkout
+    # N.B. we do not run binary_populate_env. The only variable we need is
+    # PIP_UPLOAD_FOLDER (which is 'nightly/' for the nightlies and '' for
+    # releases, and sometimes other things for special cases). Instead we
+    # expect PIP_UPLOAD_FOLDER to be passed directly in the env. This is
+    # because, unlike all the other binary jobs, these jobs only get run once,
+    # in a separate workflow. They are not a step in other binary jobs like
+    # build, test, upload.
+    #
+    # You could attach this to every job, or include it in the upload step if
+    # you wanted. You would need to add binary_populate_env in this case to
+    # make sure it has the same upload folder as the job it's attached to. This
+    # function is idempotent, so it won't hurt anything; it's just a little
+    # unnescessary"
+    - run:
+        name: Update s3 htmls
+        no_output_timeout: "1h"
+        command: |
+          echo "declare -x \"AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}\"" >> /home/circleci/project/env
+          echo "declare -x \"AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}\"" >> /home/circleci/project/env
+          source /home/circleci/project/env
+          set -ex
+          retry () {
+              $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+          }
+          retry pip install awscli==1.6
+          "/home/circleci/project/builder/cron/update_s3_htmls.sh"
+
+  # Update s3 htmls for the nightlies
+  update_s3_htmls_for_nightlies:
+    environment:
+      PIP_UPLOAD_FOLDER: "nightly/"
+    <<: *update_s3_htmls
+
+  # Update s3 htmls for the nightlies for devtoolset7
+  update_s3_htmls_for_nightlies_devtoolset7:
+    environment:
+      PIP_UPLOAD_FOLDER: "nightly/devtoolset7/"
+    <<: *update_s3_htmls

.circleci/verbatim-sources/header-section.yml

@@ -0,0 +1,288 @@
+# WARNING: DO NOT EDIT THIS FILE DIRECTLY!!!
+# See the README.md in this directory.
+
+# IMPORTANT: To update Docker image version, please first update
+# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and
+# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy,
+# and then update DOCKER_IMAGE_VERSION at the top of the following files:
+# * cimodel/data/pytorch_build_definitions.py
+# * cimodel/data/caffe2_build_definitions.py
+
+docker_config_defaults: &docker_config_defaults
+  user: jenkins
+  aws_auth:
+    # This IAM user only allows read-write access to ECR
+    aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_WRITE_V3}
+    aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_WRITE_V3}
+
+# This system setup script is meant to run before the CI-related scripts, e.g.,
+# installing Git client, checking out code, setting up CI env, and
+# building/testing.
+setup_linux_system_environment: &setup_linux_system_environment
+  name: Set Up System Environment
+  no_output_timeout: "1h"
+  command: |
+    set -ex
+
+    # Set up CircleCI GPG keys for apt, if needed
+    curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
+
+    # Stop background apt updates.  Hypothetically, the kill should not
+    # be necessary, because stop is supposed to send a kill signal to
+    # the process, but we've added it for good luck.  Also
+    # hypothetically, it's supposed to be unnecessary to wait for
+    # the process to block.  We also have that line for good luck.
+    # If you like, try deleting them and seeing if it works.
+    sudo systemctl stop apt-daily.service || true
+    sudo systemctl kill --kill-who=all apt-daily.service || true
+
+    sudo systemctl stop unattended-upgrades.service || true
+    sudo systemctl kill --kill-who=all unattended-upgrades.service || true
+
+    # wait until `apt-get update` has been killed
+    while systemctl is-active --quiet apt-daily.service
+    do
+      sleep 1;
+    done
+    while systemctl is-active --quiet unattended-upgrades.service
+    do
+      sleep 1;
+    done
+
+    # See if we actually were successful
+    systemctl list-units --all | cat
+
+    sudo apt-get purge -y unattended-upgrades
+
+    cat /etc/apt/sources.list
+
+    ps auxfww | grep [a]pt
+    ps auxfww | grep dpkg
+
+install_doc_push_script: &install_doc_push_script
+  name: Install the doc push script
+  no_output_timeout: "2m"
+  command: |
+    cat >/home/circleci/project/doc_push_script.sh <<EOL
+    # =================== The following code **should** be executed inside Docker container ===================
+
+    # This is where the local pytorch install in the docker image is located
+    pt_checkout="/var/lib/jenkins/workspace"
+
+    # Since we're cat-ing this file, we need to escape all $'s
+    echo "doc_push_script.sh: Invoked with \$*"
+
+    git clone https://yf225:${GITHUB_PYTORCHBOT_TOKEN}@github.com/pytorch/pytorch.github.io -b site
+    pushd pytorch.github.io
+
+    set -ex
+
+    # Argument 1: Where to copy the built documentation to
+    # (pytorch.github.io/$install_path)
+    install_path="\$1"
+    if [ -z "\$install_path" ]; then
+    echo "error: doc_push_script.sh: install_path (arg1) not specified"
+      exit 1
+    fi
+
+    # Argument 2: What version of the docs we are building.
+    version="\$2"
+    if [ -z "\$version" ]; then
+    echo "error: doc_push_script.sh: version (arg2) not specified"
+      exit 1
+    fi
+
+    is_master_doc=false
+    if [ "\$version" == "master" ]; then
+      is_master_doc=true
+    fi
+
+    # Argument 3: (optional) If present, we will NOT do any pushing. Used for testing.
+    dry_run=false
+    if [ "\$3" != "" ]; then
+      dry_run=true
+    fi
+
+    echo "install_path: \$install_path  version: \$version  dry_run: \$dry_run"
+
+    export LC_ALL=C
+    export PATH=/opt/conda/bin:$PATH
+
+    rm -rf pytorch || true
+
+    # Install TensorBoard in python 3 so torch.utils.tensorboard classes render
+    pip install -q https://s3.amazonaws.com/ossci-linux/wheels/tensorboard-1.14.0a0-py3-none-any.whl
+
+    # Get all the documentation sources, put them in one place
+    pushd "\$pt_checkout"
+    git clone https://github.com/pytorch/vision
+    pushd vision
+    conda install -q pillow
+    time python setup.py install
+    popd
+    pushd docs
+    rm -rf source/torchvision
+    cp -a ../vision/docs/source source/torchvision
+
+    # Build the docs
+    pip -q install -r requirements.txt || true
+    if [ "\$is_master_doc" = true ]; then
+      make html
+    else
+      make html-stable
+    fi
+
+    # Move them into the docs repo
+    popd
+    popd
+    git rm -rf "\$install_path" || true
+    mv "\$pt_checkout/docs/build/html" "\$install_path"
+
+    # Add the version handler by search and replace.
+    # XXX: Consider moving this to the docs Makefile or site build
+    if [ "\$is_master_doc" = true ]; then
+      find "\$install_path" -name "*.html" -print0 | xargs -0 perl -pi -w -e "s@master\s+\((\d\.\d\.[A-Fa-f0-9]+\+[A-Fa-f0-9]+)\s+\)@<a href='http://pytorch.org/docs/versions.html'>\1 \&#x25BC</a>@g"
+    else
+      find "\$install_path" -name "*.html" -print0 | xargs -0 perl -pi -w -e "s@master\s+\((\d\.\d\.\S+)\s+\)@<a href='http://pytorch.org/docs/versions.html'>\$version \&#x25BC</a>@g"
+    fi
+
+    git add "\$install_path" || true
+    git status
+    git config user.email "soumith+bot@pytorch.org"
+    git config user.name "pytorchbot"
+    # If there aren't changes, don't make a commit; push is no-op
+    git commit -m "auto-generating sphinx docs" || true
+    git status
+
+    if [ "\$dry_run" = false ]; then
+      echo "Pushing to pytorch.github.io:site"
+      git push origin site
+    else
+      echo "Skipping push due to dry_run"
+    fi
+
+    popd
+    # =================== The above code **should** be executed inside Docker container ===================
+    EOL
+    chmod +x /home/circleci/project/doc_push_script.sh
+
+# `setup_ci_environment` has to be run **after** the ``checkout`` step because
+# it writes into the checkout directory and otherwise CircleCI will complain
+# that
+#   Directory (/home/circleci/project) you are trying to checkout to is not empty and not git repository
+setup_ci_environment: &setup_ci_environment
+  name: Set Up CI Environment After Checkout
+  no_output_timeout: "1h"
+  command: |
+    set -ex
+
+    # Check if we should actually run
+    echo "BUILD_ENVIRONMENT: ${BUILD_ENVIRONMENT}"
+    echo "CIRCLE_PULL_REQUEST: ${CIRCLE_PULL_REQUEST}"
+    if [[ "${BUILD_ENVIRONMENT}" == *-slow-* ]]; then
+      if ! [ -z "${CIRCLE_PULL_REQUEST}" ]; then
+        # It's a PR; test for [slow ci] tag on the TOPMOST commit
+        if !(git log --format='%B' -n 1 HEAD | grep -q -e '\[slow ci\]' -e '\[ci slow\]' -e '\[test slow\]' -e '\[slow test\]'); then
+          circleci step halt
+          exit
+        fi
+      fi
+    fi
+    if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
+      if ! [ -z "${CIRCLE_PULL_REQUEST}" ]; then
+        # It's a PR; test for [xla ci] tag on the TOPMOST commit
+        if !(git log --format='%B' -n 1 HEAD | grep -q -e '\[xla ci\]' -e '\[ci xla\]' -e '\[test xla\]' -e '\[xla test\]'); then
+          # NB: This doesn't halt everything, just this job.  So
+          # the rest of the workflow will keep going and you need
+          # to make sure you halt there too.  Blegh.
+          circleci step halt
+          exit
+        fi
+      fi
+    fi
+
+    # Set up NVIDIA docker repo
+    curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+    echo "deb https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+    echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+    echo "deb https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+
+    sudo apt-get -y update
+    sudo apt-get -y remove linux-image-generic linux-headers-generic linux-generic docker-ce
+    # WARNING: Docker version is hardcoded here; you must update the
+    # version number below for docker-ce and nvidia-docker2 to get newer
+    # versions of Docker.  We hardcode these numbers because we kept
+    # getting broken CI when Docker would update their docker version,
+    # and nvidia-docker2 would be out of date for a day until they
+    # released a newer version of their package.
+    #
+    # How to figure out what the correct versions of these packages are?
+    # My preferred method is to start a Docker instance of the correct
+    # Ubuntu version (e.g., docker run -it ubuntu:16.04) and then ask
+    # apt what the packages you need are.  Note that the CircleCI image
+    # comes with Docker.
+    sudo apt-get -y install \
+      linux-headers-$(uname -r) \
+      linux-image-generic \
+      moreutils \
+      docker-ce=5:18.09.4~3-0~ubuntu-xenial \
+      nvidia-container-runtime=2.0.0+docker18.09.4-1 \
+      nvidia-docker2=2.0.3+docker18.09.4-1 \
+      expect-dev
+
+    sudo pkill -SIGHUP dockerd
+
+    sudo pip -q install awscli==1.16.35
+
+    if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
+      DRIVER_FN="NVIDIA-Linux-x86_64-410.104.run"
+      wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
+      sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
+      nvidia-smi
+    fi
+
+    if [[ "${BUILD_ENVIRONMENT}" == *-build ]]; then
+      echo "declare -x IN_CIRCLECI=1" > /home/circleci/project/env
+      echo "declare -x COMMIT_SOURCE=${CIRCLE_BRANCH}" >> /home/circleci/project/env
+      echo "declare -x PYTHON_VERSION=${PYTHON_VERSION}" >> /home/circleci/project/env
+      echo "declare -x SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> /home/circleci/project/env
+      if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
+        echo "declare -x TORCH_CUDA_ARCH_LIST=5.2" >> /home/circleci/project/env
+      fi
+      export SCCACHE_MAX_JOBS=`expr $(nproc) - 1`
+      export MEMORY_LIMIT_MAX_JOBS=8  # the "large" resource class on CircleCI has 32 CPU cores, if we use all of them we'll OOM
+      export MAX_JOBS=$(( ${SCCACHE_MAX_JOBS} > ${MEMORY_LIMIT_MAX_JOBS} ? ${MEMORY_LIMIT_MAX_JOBS} : ${SCCACHE_MAX_JOBS} ))
+      echo "declare -x MAX_JOBS=${MAX_JOBS}" >> /home/circleci/project/env
+
+      if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
+        # This IAM user allows write access to S3 bucket for sccache & bazels3cache
+        echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_AND_XLA_BAZEL_S3_BUCKET_V1}" >> /home/circleci/project/env
+        echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_AND_XLA_BAZEL_S3_BUCKET_V1}" >> /home/circleci/project/env
+      else
+        # This IAM user allows write access to S3 bucket for sccache
+        echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V3}" >> /home/circleci/project/env
+        echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V3}" >> /home/circleci/project/env
+      fi
+    fi
+
+    # This IAM user only allows read-write access to ECR
+    export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_WRITE_V3}
+    export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_WRITE_V3}
+    eval $(aws ecr get-login --region us-east-1 --no-include-email)
+
+macos_brew_update: &macos_brew_update
+  name: Brew update and install moreutils, expect and libomp
+  no_output_timeout: "1h"
+  command: |
+    set -ex
+    pwd
+    ls -lah
+    # moreutils installs a `parallel` executable by default, which conflicts
+    # with the executable from the GNU `parallel`, so we must unlink GNU
+    # `parallel` first, and relink it afterwards
+    brew update
+    brew unlink parallel
+    brew install moreutils
+    brew link parallel --overwrite
+    brew install expect
+    brew install libomp

.circleci/verbatim-sources/job-specs-custom.yml

@@ -0,0 +1,193 @@
+  pytorch_short_perf_test_gpu:
+    environment:
+      BUILD_ENVIRONMENT: pytorch-short-perf-test-gpu
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda8-cudnn7-py3:300"
+      PYTHON_VERSION: "3.6"
+      USE_CUDA_DOCKER_RUNTIME: "1"
+    resource_class: gpu.medium
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+    - run:
+        <<: *setup_linux_system_environment
+    - run:
+        <<: *setup_ci_environment
+    - run:
+        name: Perf Test
+        no_output_timeout: "1h"
+        command: |
+          set -e
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
+          docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
+          export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+
+          docker cp $id:/var/lib/jenkins/workspace/env /home/circleci/project/env
+          # This IAM user allows write access to S3 bucket for perf test numbers
+          echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_PERF_TEST_S3_BUCKET_V3}" >> /home/circleci/project/env
+          echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PERF_TEST_S3_BUCKET_V3}" >> /home/circleci/project/env
+          docker cp /home/circleci/project/env $id:/var/lib/jenkins/workspace/env
+
+          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/short-perf-test-gpu.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+  pytorch_doc_push:
+    environment:
+      BUILD_ENVIRONMENT: pytorch-doc-push
+      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda8-cudnn7-py3:300"
+    resource_class: large
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+    - run:
+        <<: *setup_linux_system_environment
+    - run:
+        <<: *setup_ci_environment
+    - run:
+        <<: *install_doc_push_script
+    - run:
+        name: Doc Build and Push
+        no_output_timeout: "1h"
+        command: |
+          set -e
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
+          docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
+          export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+
+          docker cp /home/circleci/project/doc_push_script.sh $id:/var/lib/jenkins/workspace/doc_push_script.sh
+
+          # master branch docs push
+          if [[ "${CIRCLE_BRANCH}" == "master" ]]; then
+            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh docs/master master") | docker exec -u jenkins -i "$id" bash) 2>&1'
+
+          # stable release docs push. Due to some circleci limitations, we keep
+          # an eternal PR open for merging v1.1.0 -> master for this job.
+          # XXX: The following code is only run on the v1.1.0 branch, which might
+          # not be exactly the same as what you see here.
+          elif [[ "${CIRCLE_BRANCH}" == "v1.1.0" ]]; then
+            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh docs/stable 1.1.0") | docker exec -u jenkins -i "$id" bash) 2>&1'
+
+          # For open PRs: Do a dry_run of the docs build, don't push build
+          else
+            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh docs/master master dry_run") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          fi
+
+          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+          # Save the docs build so we can debug any problems
+          export DEBUG_COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}-debug
+          docker commit "$id" ${DEBUG_COMMIT_DOCKER_IMAGE}
+          docker push ${DEBUG_COMMIT_DOCKER_IMAGE}
+
+  pytorch_macos_10_13_py3_build:
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          <<: *macos_brew_update
+      - run:
+          name: Build
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
+          no_output_timeout: "1h"
+          command: |
+            set -e
+
+            export IN_CIRCLECI=1
+
+            # Install sccache
+            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo chmod +x /usr/local/bin/sccache
+
+            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+            # This IAM user allows write access to S3 bucket for sccache
+            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V3}
+            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V3}
+
+            chmod a+x .jenkins/pytorch/macos-build.sh
+            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
+
+            mkdir -p /Users/distiller/pytorch-ci-env/workspace
+
+            # copy with -a to preserve relative structure (e.g., symlinks), and be recursive
+            cp -a /Users/distiller/project/. /Users/distiller/pytorch-ci-env/workspace
+      - persist_to_workspace:
+          root: /Users/distiller/pytorch-ci-env
+          paths:
+            - "*"
+
+  pytorch_macos_10_13_py3_test:
+    macos:
+      xcode: "9.0"
+    steps:
+      - run:
+          name: Prepare workspace
+          command: |
+            sudo mkdir -p /Users/distiller/pytorch-ci-env
+            sudo chmod -R 777 /Users/distiller/pytorch-ci-env
+      - attach_workspace:
+          at: /Users/distiller/pytorch-ci-env
+      - run:
+          <<: *macos_brew_update
+      - run:
+          name: Test
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
+          no_output_timeout: "1h"
+          command: |
+            set -e
+            export IN_CIRCLECI=1
+
+            # copy with -a to preserve relative structure (e.g., symlinks), and be recursive
+            cp -a /Users/distiller/pytorch-ci-env/workspace/. /Users/distiller/project
+
+            chmod a+x .jenkins/pytorch/macos-test.sh
+            unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts
+
+  pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          <<: *macos_brew_update
+      - run:
+          name: Build
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
+          no_output_timeout: "1h"
+          command: |
+            set -e
+
+            export IN_CIRCLECI=1
+
+            # Install CUDA 9.2
+            sudo rm -rf ~/cuda_9.2.64_mac_installer.app || true
+            curl https://s3.amazonaws.com/ossci-macos/cuda_9.2.64_mac_installer.zip -o ~/cuda_9.2.64_mac_installer.zip
+            unzip ~/cuda_9.2.64_mac_installer.zip -d ~/
+            sudo ~/cuda_9.2.64_mac_installer.app/Contents/MacOS/CUDAMacOSXInstaller --accept-eula --no-window
+            sudo cp /usr/local/cuda/lib/libcuda.dylib /Developer/NVIDIA/CUDA-9.2/lib/libcuda.dylib
+            sudo rm -rf /usr/local/cuda || true
+
+            # Install cuDNN 7.1 for CUDA 9.2
+            curl https://s3.amazonaws.com/ossci-macos/cudnn-9.2-osx-x64-v7.1.tgz -o ~/cudnn-9.2-osx-x64-v7.1.tgz
+            rm -rf ~/cudnn-9.2-osx-x64-v7.1 && mkdir ~/cudnn-9.2-osx-x64-v7.1
+            tar -xzvf ~/cudnn-9.2-osx-x64-v7.1.tgz -C ~/cudnn-9.2-osx-x64-v7.1
+            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/include/
+            sudo cp ~/cudnn-9.2-osx-x64-v7.1/cuda/lib/libcudnn* /Developer/NVIDIA/CUDA-9.2/lib/
+            sudo chmod a+r /Developer/NVIDIA/CUDA-9.2/include/cudnn.h /Developer/NVIDIA/CUDA-9.2/lib/libcudnn*
+
+            # Install sccache
+            sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+            sudo chmod +x /usr/local/bin/sccache
+            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+            # This IAM user allows write access to S3 bucket for sccache
+            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V3}
+            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V3}
+
+            git submodule sync && git submodule update -q --init
+            chmod a+x .jenkins/pytorch/macos-build.sh
+            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
+

.circleci/verbatim-sources/linux-binary-build-defaults.yml

@@ -0,0 +1,94 @@
+# binary linux build defaults
+##############################################################################
+binary_linux_build: &binary_linux_build
+  resource_class: 2xlarge+
+  steps:
+  - run:
+      <<: *binary_checkout
+  - run:
+      <<: *binary_populate_env
+  - run:
+      name: Install unbuffer and ts
+      command: |
+        set -ex
+        source /env
+        retry yum -q -y install epel-release
+        retry yum -q -y install expect moreutils
+  - run:
+      name: Upgrade gcc version (based on env var)
+      command: |
+        set -ex
+        source /env
+        if [[ "$DESIRED_DEVTOOLSET" == 'devtoolset7' ]]; then
+          source "/builder/upgrade_gcc_abi.sh"
+
+          # Env variables are not persisted into the next step
+          echo "export PATH=$PATH" > /env
+          echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" > /env
+        else
+          echo "Not upgrading gcc version"
+        fi
+  - run:
+      name: Build
+      no_output_timeout: "1h"
+      command: |
+        source "/pytorch/.circleci/scripts/binary_linux_build.sh"
+  - persist_to_workspace:
+      root: /
+      paths: final_pkgs
+
+
+# This should really just be another step of the binary_linux_build job above.
+# This isn't possible right now b/c the build job uses the docker executor
+# (otherwise they'd be really really slow) but this one uses the macine
+# executor (b/c we have to run the docker with --runtime=nvidia and we can't do
+# that on the docker executor)
+binary_linux_test: &binary_linux_test
+  machine:
+    image: ubuntu-1604:201903-01
+  steps:
+  - run:
+      <<: *setup_linux_system_environment
+  - run:
+      <<: *setup_ci_environment
+  - attach_workspace:
+      at: /home/circleci/project
+  # This checkout is only needed to access
+  # .circleci/scripts/binary_linux_test.sh, which can't be inlined because it
+  # blows up the yaml size.
+  - run:
+      <<: *binary_checkout
+  - run:
+      <<: *binary_populate_env
+  - run:
+      name: Prepare test code
+      no_output_timeout: "1h"
+      command: |
+        source "/home/circleci/project/pytorch/.circleci/scripts/binary_linux_test.sh"
+  - run:
+      <<: *binary_run_in_docker
+
+binary_linux_upload: &binary_linux_upload
+  machine:
+    image: ubuntu-1604:201903-01
+  steps:
+  - run:
+      <<: *setup_linux_system_environment
+  - run:
+      <<: *setup_ci_environment
+  - attach_workspace:
+      at: /home/circleci/project
+  # This checkout is only needed to access
+  # .circleci/scripts/binary_linux_upload.sh, which can't be inlined because it
+  # blows up the yaml size.
+  - run:
+      <<: *binary_checkout
+  - run:
+      <<: *binary_populate_env
+  - run:
+      <<: *binary_install_miniconda
+  - run:
+      name: Upload
+      no_output_timeout: "1h"
+      command: |
+        source "/home/circleci/project/pytorch/.circleci/scripts/binary_linux_upload.sh"

.circleci/verbatim-sources/linux-build-defaults.yml

@@ -0,0 +1,190 @@
+
+
+##############################################################################
+# Linux build defaults
+##############################################################################
+
+pytorch_linux_build_defaults: &pytorch_linux_build_defaults
+  resource_class: large
+  machine:
+    image: ubuntu-1604:201903-01
+  steps:
+  - run:
+      <<: *setup_linux_system_environment
+  - checkout
+  - run:
+      <<: *setup_ci_environment
+  - run:
+      name: Build
+      no_output_timeout: "1h"
+      command: |
+        set -e
+        # Pull Docker image and run build
+        echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
+        docker pull ${DOCKER_IMAGE} >/dev/null
+        export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+
+        git submodule sync && git submodule update -q --init
+
+        docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
+
+        export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+        # Push intermediate Docker image for next phase to use
+        if [ -z "${BUILD_ONLY}" ]; then
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          docker commit "$id" ${COMMIT_DOCKER_IMAGE}
+          docker push ${COMMIT_DOCKER_IMAGE}
+        fi
+
+pytorch_linux_test_defaults: &pytorch_linux_test_defaults
+  machine:
+    image: ubuntu-1604:201903-01
+  steps:
+  - run:
+      <<: *setup_linux_system_environment
+  - run:
+      <<: *setup_ci_environment
+  - run:
+      name: Test
+      no_output_timeout: "90m"
+      command: |
+        set -e
+        export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+        echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
+        docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
+        if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
+          export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+        else
+          export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+        fi
+        if [ -n "${MULTI_GPU}" ]; then
+          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        else
+          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        fi
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+caffe2_linux_build_defaults: &caffe2_linux_build_defaults
+  resource_class: large
+  machine:
+    image: ubuntu-1604:201903-01
+  steps:
+  - run:
+      <<: *setup_linux_system_environment
+  - checkout
+  - run:
+      <<: *setup_ci_environment
+  - run:
+      name: Build
+      no_output_timeout: "1h"
+      command: |
+        set -e
+        cat >/home/circleci/project/ci_build_script.sh <<EOL
+        # =================== The following code will be executed inside Docker container ===================
+        set -ex
+        export BUILD_ENVIRONMENT="$BUILD_ENVIRONMENT"
+
+        # Reinitialize submodules
+        git submodule sync && git submodule update -q --init --recursive
+
+        # conda must be added to the path for Anaconda builds (this location must be
+        # the same as that in install_anaconda.sh used to build the docker image)
+        if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
+          export PATH=/opt/conda/bin:$PATH
+          sudo chown -R jenkins:jenkins '/opt/conda'
+        fi
+
+        # Build
+        ./.jenkins/caffe2/build.sh
+
+        # Show sccache stats if it is running
+        if pgrep sccache > /dev/null; then
+          sccache --show-stats
+        fi
+        # =================== The above code will be executed inside Docker container ===================
+        EOL
+        chmod +x /home/circleci/project/ci_build_script.sh
+
+        echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
+        docker pull ${DOCKER_IMAGE} >/dev/null
+        export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+        docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
+
+        export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+
+        # Push intermediate Docker image for next phase to use
+        if [ -z "${BUILD_ONLY}" ]; then
+          if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+            export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-cmake-${CIRCLE_SHA1}
+          else
+            export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+          fi
+          docker commit "$id" ${COMMIT_DOCKER_IMAGE}
+          docker push ${COMMIT_DOCKER_IMAGE}
+        fi
+
+caffe2_linux_test_defaults: &caffe2_linux_test_defaults
+  machine:
+    image: ubuntu-1604:201903-01
+  steps:
+  - run:
+      <<: *setup_linux_system_environment
+  - run:
+      <<: *setup_ci_environment
+  - run:
+      name: Test
+      no_output_timeout: "1h"
+      command: |
+        set -e
+        # TODO: merge this into Caffe2 test.sh
+        cat >/home/circleci/project/ci_test_script.sh <<EOL
+        # =================== The following code will be executed inside Docker container ===================
+        set -ex
+
+        export BUILD_ENVIRONMENT="$BUILD_ENVIRONMENT"
+
+        # libdc1394 (dependency of OpenCV) expects /dev/raw1394 to exist...
+        sudo ln /dev/null /dev/raw1394
+
+        # conda must be added to the path for Anaconda builds (this location must be
+        # the same as that in install_anaconda.sh used to build the docker image)
+        if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
+          export PATH=/opt/conda/bin:$PATH
+        fi
+
+        # Upgrade SSL module to avoid old SSL warnings
+        pip -q install --user --upgrade pyOpenSSL ndg-httpsclient pyasn1
+
+        pip -q install --user -b /tmp/pip_install_onnx "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
+
+        # Build
+        ./.jenkins/caffe2/test.sh
+
+        # Remove benign core dumps.
+        # These are tests for signal handling (including SIGABRT).
+        rm -f ./crash/core.fatal_signal_as.*
+        rm -f ./crash/core.logging_test.*
+        # =================== The above code will be executed inside Docker container ===================
+        EOL
+        chmod +x /home/circleci/project/ci_test_script.sh
+
+        if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-cmake-${CIRCLE_SHA1}
+        else
+          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+        fi
+        echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
+        docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
+        if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
+          export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+        else
+          export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+        fi
+        docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
+
+        export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+

.circleci/verbatim-sources/macos-binary-build-defaults.yml

@@ -0,0 +1,61 @@
+
+##############################################################################
+# Macos binary build defaults
+# The root of everything is /Users/distiller/pytorch-ci-env/workspace
+##############################################################################
+binary_mac_build: &binary_mac_build
+  macos:
+    xcode: "9.0"
+  steps:
+  - run:
+      <<: *binary_checkout
+  - run:
+      <<: *binary_populate_env
+  - run:
+      <<: *macos_brew_update
+  - run:
+      <<: *binary_install_miniconda
+
+  - run:
+      name: Build
+      no_output_timeout: "1h"
+      command: |
+        set -ex
+        script="/Users/distiller/project/pytorch/.circleci/scripts/binary_macos_build.sh"
+        cat "$script"
+        source "$script"
+
+  - run:
+      name: Test
+      no_output_timeout: "1h"
+      command: |
+        set -ex
+        script="/Users/distiller/project/pytorch/.circleci/scripts/binary_macos_test.sh"
+        cat "$script"
+        source "$script"
+
+  - persist_to_workspace:
+      root: /Users/distiller/project
+      paths: final_pkgs
+
+binary_mac_upload: &binary_mac_upload
+  macos:
+    xcode: "9.0"
+  steps:
+  - run:
+      <<: *binary_checkout
+  - run:
+      <<: *binary_populate_env
+  - run:
+      <<: *macos_brew_update
+  - run:
+      <<: *binary_install_miniconda
+  - attach_workspace:
+      at: /Users/distiller/project
+  - run:
+      name: Upload
+      no_output_timeout: "10m"
+      command: |
+        script="/Users/distiller/project/pytorch/.circleci/scripts/binary_macos_test.sh"
+        cat "$script"
+        source "$script"

.circleci/verbatim-sources/macos-build-defaults.yml

@@ -0,0 +1,83 @@
+
+##############################################################################
+# Macos build defaults
+##############################################################################
+
+caffe2_macos_build_defaults: &caffe2_macos_build_defaults
+  macos:
+    xcode: "9.0"
+  steps:
+    - checkout
+    - run:
+        <<: *macos_brew_update
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          set -e
+
+          export IN_CIRCLECI=1
+
+          brew install cmake
+
+          # Reinitialize submodules
+          git submodule sync && git submodule update -q --init --recursive
+
+          # Reinitialize path (see man page for path_helper(8))
+          eval `/usr/libexec/path_helper -s`
+
+          # Use Homebrew Python if configured to do so
+          if [ "${PYTHON_INSTALLATION}" == "homebrew" ]; then
+            export PATH=/usr/local/opt/python/libexec/bin:/usr/local/bin:$PATH
+          fi
+
+          pip -q install numpy
+
+          # Install Anaconda if we need to
+          if [ -n "${CAFFE2_USE_ANACONDA}" ]; then
+            rm -rf ${TMPDIR}/anaconda
+            curl -o ${TMPDIR}/conda.sh https://repo.continuum.io/miniconda/Miniconda${ANACONDA_VERSION}-latest-MacOSX-x86_64.sh
+            chmod +x ${TMPDIR}/conda.sh
+            /bin/bash ${TMPDIR}/conda.sh -b -p ${TMPDIR}/anaconda
+            rm -f ${TMPDIR}/conda.sh
+            export PATH="${TMPDIR}/anaconda/bin:${PATH}"
+            source ${TMPDIR}/anaconda/bin/activate
+          fi
+
+          # Install sccache
+          sudo curl https://s3.amazonaws.com/ossci-macos/sccache --output /usr/local/bin/sccache
+          sudo chmod +x /usr/local/bin/sccache
+          export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+
+          # This IAM user allows write access to S3 bucket for sccache
+          export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V3}
+          export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V3}
+
+          export SCCACHE_BIN=${PWD}/sccache_bin
+          mkdir -p ${SCCACHE_BIN}
+          if which sccache > /dev/null; then
+            printf "#!/bin/sh\nexec sccache $(which clang++) \$*" > "${SCCACHE_BIN}/clang++"
+            chmod a+x "${SCCACHE_BIN}/clang++"
+
+            printf "#!/bin/sh\nexec sccache $(which clang) \$*" > "${SCCACHE_BIN}/clang"
+            chmod a+x "${SCCACHE_BIN}/clang"
+
+            export PATH="${SCCACHE_BIN}:$PATH"
+          fi
+
+          # Build
+          if [ "${BUILD_IOS:-0}" -eq 1 ]; then
+            unbuffer scripts/build_ios.sh 2>&1 | ts
+          elif [ -n "${CAFFE2_USE_ANACONDA}" ]; then
+            # All conda build logic should be in scripts/build_anaconda.sh
+            unbuffer scripts/build_anaconda.sh 2>&1 | ts
+          else
+            unbuffer scripts/build_local.sh 2>&1 | ts
+          fi
+
+          # Show sccache stats if it is running
+          if which sccache > /dev/null; then
+            sccache --show-stats
+          fi
+
+

.circleci/verbatim-sources/nightly-binary-build-defaults.yml

@@ -0,0 +1,95 @@
+
+##############################################################################
+# Binary build (nightlies nightly build) defaults
+# The binary builds use the docker executor b/c at time of writing the machine
+# executor is limited to only two cores and is painfully slow (4.5+ hours per
+# GPU build). But the docker executor cannot be run with --runtime=nvidia, and
+# so the binary test/upload jobs must run on a machine executor. The package
+# built in the build job is persisted to the workspace, which the test jobs
+# expect. The test jobs just run a few quick smoke tests (very similar to the
+# second-round-user-facing smoke tests above) and then upload the binaries to
+# their final locations. The upload part requires credentials that should only
+# be available to org-members.
+#
+# binary_checkout MUST be run before other commands here. This is because the
+# other commands are written in .circleci/scripts/*.sh , so the pytorch source
+# code must be downloaded on the machine before they can be run. We cannot
+# inline all the code into this file, since that would cause the yaml size to
+# explode past 4 MB (all the code in the command section is just copy-pasted to
+# everywhere in the .circleci/config.yml file where it appears).
+##############################################################################
+
+# Checks out the Pytorch and Builder repos (always both of them), and places
+# them in the right place depending on what executor we're running on. We curl
+# our .sh file from the interweb to avoid yaml size bloat. Note that many jobs
+# do not need both the pytorch and builder repos, so this is a little wasteful
+# (smoke tests and upload jobs do not need the pytorch repo).
+binary_checkout: &binary_checkout
+  name: Checkout
+  command: |
+    if [[ -n "$CIRCLE_SHA1" ]]; then
+      # we are on a PR or on a master-merge, but not on a timed build
+      git_url="https://raw.githubusercontent.com/pytorch/pytorch/$CIRCLE_SHA1"
+    else
+      # scheduled nightly binary builds. These run at 05:05 UTC every day. 
+      last_commit="$(git rev-list --before "$(date -u +%Y-%m-%d) 05:00" --max-count 1 HEAD)"
+      git_url="https://raw.githubusercontent.com/pytorch/pytorch/$last_commit"
+    fi
+    retry () {
+        $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+    }
+    retry curl -s "$git_url/.circleci/scripts/binary_checkout.sh" -o "binary_checkout.sh"
+    cat "binary_checkout.sh"
+    source "binary_checkout.sh"
+
+# Parses circleci arguments in a consistent way, essentially routing to the
+# correct pythonXgccXcudaXos build we want
+binary_populate_env: &binary_populate_env
+  name: Set up env
+  command: |
+    # This step runs on multiple executors with different envfile locations
+    if [[ "$(uname)" == Darwin ]]; then
+      # macos executor (builds and tests)
+      workdir="/Users/distiller/project"
+    elif [[ -d "/home/circleci/project" ]]; then
+      # machine executor (binary tests)
+      workdir="/home/circleci/project"
+    else
+      # docker executor (binary builds)
+      workdir="/"
+    fi
+    script="$workdir/pytorch/.circleci/scripts/binary_populate_env.sh"
+    cat "$script"
+    source "$script"
+
+binary_install_miniconda: &binary_install_miniconda
+  name: Install miniconda
+  no_output_timeout: "1h"
+  command: |
+    # This step runs on multiple executors with different envfile locations
+    if [[ "$(uname)" == Darwin ]]; then
+      # macos executor (builds and tests)
+      workdir="/Users/distiller/project"
+    elif [[ -d "/home/circleci/project" ]]; then
+      # machine executor (binary tests)
+      workdir="/home/circleci/project"
+    else
+      # docker executor (binary builds)
+      workdir="/"
+    fi
+    script="$workdir/pytorch/.circleci/scripts/binary_install_miniconda.sh"
+    cat "$script"
+    source "$script"
+
+# This section is used in the binary_test and smoke_test jobs. It expects
+# 'binary_populate_env' to have populated /home/circleci/project/env and it
+# expects another section to populate /home/circleci/project/ci_test_script.sh
+# with the code to run in the docker
+binary_run_in_docker: &binary_run_in_docker
+  name: Run in docker
+  command: |
+    # This step only runs on circleci linux machine executors that themselves
+    # need to start docker images
+    script="/home/circleci/project/pytorch/.circleci/scripts/binary_install_miniconda.sh"
+    cat "$script"
+    source "$script"

.circleci/verbatim-sources/nightly-build-smoke-tests-defaults.yml

@@ -0,0 +1,49 @@
+# Nighlty build smoke tests defaults
+# These are the second-round smoke tests. These make sure that the binaries are
+# correct from a user perspective, testing that they exist from the cloud are
+# are runnable. Note that the pytorch repo is never cloned into these jobs
+##############################################################################
+smoke_linux_test: &smoke_linux_test
+  machine:
+    image: ubuntu-1604:201903-01
+  steps:
+  - run:
+      <<: *setup_linux_system_environment
+  - run:
+      <<: *setup_ci_environment
+  - run:
+      <<: *binary_populate_env
+  - run:
+      name: Test
+      no_output_timeout: "1h"
+      command: |
+        set -ex
+        cat >/home/circleci/project/ci_test_script.sh <<EOL
+        # The following code will be executed inside Docker container
+        set -ex
+        git clone https://github.com/pytorch/builder.git /builder
+        /builder/smoke_test.sh
+        # The above code will be executed inside Docker container
+        EOL
+  - run:
+      <<: *binary_run_in_docker
+
+smoke_mac_test: &smoke_mac_test
+  macos:
+    xcode: "9.0"
+  steps:
+    - run:
+        <<: *binary_populate_env
+    - run:
+        <<: *macos_brew_update
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          set -ex
+          source "/Users/distiller/project/env"
+          git clone https://github.com/pytorch/builder.git
+          unbuffer ./builder/smoke_test.sh | ts
+
+
+

.circleci/verbatim-sources/workflows-binary-build-header.yml

@@ -0,0 +1,4 @@
+
+##############################################################################
+# Daily binary build trigger
+##############################################################################

.circleci/verbatim-sources/workflows-binary-builds-smoke-subset.yml

@@ -0,0 +1,26 @@
+
+      # Binary builds (subset, to smoke test that they'll work)
+      - binary_linux_manywheel_2.7mu_cpu_devtoolset3_build
+      - binary_linux_manywheel_3.7m_cu100_devtoolset3_build
+      - binary_linux_conda_2.7_cpu_build
+      # This binary build is currently broken, see https://github.com/pytorch/pytorch/issues/16710
+      # - binary_linux_conda_3.6_cu90_build
+      - binary_linux_libtorch_2.7m_cu80_devtoolset3_build
+      - binary_macos_wheel_3.6_cpu_build
+      - binary_macos_conda_2.7_cpu_build
+      - binary_macos_libtorch_2.7_cpu_build
+
+      - binary_linux_manywheel_2.7mu_cpu_devtoolset3_test:
+          requires:
+            - binary_linux_manywheel_2.7mu_cpu_devtoolset3_build
+      - binary_linux_manywheel_3.7m_cu100_devtoolset3_test:
+          requires:
+            - binary_linux_manywheel_3.7m_cu100_devtoolset3_build
+      - binary_linux_conda_2.7_cpu_test:
+          requires:
+            - binary_linux_conda_2.7_cpu_build
+      # This binary build is currently broken, see https://github.com/pytorch/pytorch/issues/16710
+      # - binary_linux_conda_3.6_cu90_test:
+      #     requires:
+      #       - binary_linux_conda_3.6_cu90_build
+

.circleci/verbatim-sources/workflows-nightly-uploads-header.yml

@@ -0,0 +1,14 @@
+      #- binary_linux_libtorch_2.7m_cpu_test:
+      #    requires:
+      #      - binary_linux_libtorch_2.7m_cpu_build
+      #- binary_linux_libtorch_2.7m_cu80_test:
+      #    requires:
+      #      - binary_linux_libtorch_2.7m_cu80_build
+      #- binary_linux_libtorch_2.7m_cu90_test:
+      #    requires:
+      #      - binary_linux_libtorch_2.7m_cu90_build
+      #- binary_linux_libtorch_2.7m_cu100_test:
+      #    requires:
+      #      - binary_linux_libtorch_2.7m_cu100_build
+
+      # Nightly uploads

.circleci/verbatim-sources/workflows-pytorch-macos-builds.yml

@@ -0,0 +1,8 @@
+
+      # Pytorch MacOS builds
+      - pytorch_macos_10_13_py3_build
+      - pytorch_macos_10_13_py3_test:
+          requires:
+            - pytorch_macos_10_13_py3_build
+      - pytorch_macos_10_13_cuda9_2_cudnn7_py3_build
+

.circleci/verbatim-sources/workflows-s3-html.yml

@@ -0,0 +1,15 @@
+
+  # Scheduled to run 4 hours after the binary jobs start
+  update_s3_htmls:
+    triggers:
+      - schedule:
+          cron: "0 9 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - update_s3_htmls_for_nightlies:
+          context: org-member
+      - update_s3_htmls_for_nightlies_devtoolset7:
+          context: org-member

.circleci/verbatim-sources/workflows.yml

@@ -0,0 +1,12 @@
+
+##############################################################################
+##############################################################################
+# Workflows
+##############################################################################
+##############################################################################
+
+# PR jobs pr builds
+workflows:
+  version: 2
+  build:
+    jobs:

.clang-tidy

@@ -1,43 +1,34 @@
 ---
-# NOTE: there must be no spaces before the '-', so put the comma first.
+# NOTE there must be no spaces before the '-', so put the comma first.
 Checks: '
-  *
-  ,modernize-*
-  ,-cert-err58-cpp
-  ,-cert-err60-cpp
-  ,-clang-diagnostic-*
+  -*
+  ,bugprone-*
+  ,-bugprone-forward-declaration-namespace
+  ,-bugprone-macro-parentheses
+  ,cppcoreguidelines-*
+  ,-cppcoreguidelines-interfaces-global-init
   ,-cppcoreguidelines-owning-memory
   ,-cppcoreguidelines-pro-bounds-array-to-pointer-decay
   ,-cppcoreguidelines-pro-bounds-constant-array-index
+  ,-cppcoreguidelines-pro-bounds-pointer-arithmetic
+  ,-cppcoreguidelines-pro-type-cstyle-cast
+  ,-cppcoreguidelines-pro-type-reinterpret-cast
   ,-cppcoreguidelines-pro-type-static-cast-downcast
+  ,-cppcoreguidelines-pro-type-union-access
   ,-cppcoreguidelines-pro-type-vararg
   ,-cppcoreguidelines-special-member-functions
-  ,-fuchsia-*
-  ,-google-build-using-namespace
-  ,-google-explicit-constructor
-  ,-google-readability-braces-around-statements
-  ,-google-readability-namespace-comments
-  ,-google-readability-todo
-  ,-google-runtime-references
-  ,-google-runtime-references
-  ,-hicpp-braces-around-statements
-  ,-hicpp-explicit-conversions
-  ,-hicpp-no-array-decay
-  ,-hicpp-special-member-functions
-  ,-hicpp-vararg
-  ,-llvm-header-guard
-  ,-llvm-namespace-comment
-  ,-misc-unused-parameters
-  ,-modernize-make-unique
+  ,hicpp-exception-baseclass
+  ,hicpp-avoid-goto
+  ,modernize-*
+  ,-modernize-return-braced-init-list
+  ,-modernize-use-auto
   ,-modernize-use-default-member-init
-  ,-performance-unnecessary-value-param
-  ,-readability-braces-around-statements
-  ,-readability-else-after-return
-  ,-readability-named-parameter
-  ,clang-analyzer-*
+  ,-modernize-use-using
+  ,performance-*
+  ,-performance-noexcept-move-constructor
   '
-WarningsAsErrors: ''
-HeaderFilterRegex: 'torch/csrc/'
+WarningsAsErrors: '*'
+HeaderFilterRegex: 'torch/csrc/.*'
 AnalyzeTemporaryDtors: false
 CheckOptions:
 ...

.ctags.d/pytorch.ctags

@@ -0,0 +1,2 @@
+--exclude=build/*
+--exclude=include/*

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+select = B,C,E,F,P,T4,W,B9
+max-line-length = 120
+# C408 ignored because we like the dict keyword argument syntax
+# E501 is not flexible enough, we're using B950 instead
+ignore =
+    E203,E305,E402,E501,E721,E741,F403,F405,F821,F841,F999,W503,W504,C408,E302,
+    # these ignores are from flake8-bugbear; please fix!
+    B007,B008,
+    # these ignores are from flake8-comprehensions; please fix!
+    C400,C401,C402,C403,C404,C405,C407,C411,
+exclude = docs/src,venv,third_party,caffe2,scripts,docs/caffe2,tools/amd_build/pyHIPIFY,torch/lib/include,torch/lib/tmp_install,build,torch/include,*.pyi

.gitattributes

@@ -0,0 +1 @@
+*.bat	text eol=crlf

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,49 @@
+---
+name: "\U0001F41B Bug Report"
+about: Submit a bug report to help us improve PyTorch
+
+---
+
+## 🐛 Bug
+
+<!-- A clear and concise description of what the bug is. -->
+
+## To Reproduce
+
+Steps to reproduce the behavior:
+
+1.
+1.
+1.
+
+<!-- If you have a code sample, error messages, stack traces, please provide it here as well -->
+
+## Expected behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+## Environment
+
+Please copy and paste the output from our
+[environment collection script](https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py)
+(or fill out the checklist below manually).
+
+You can get the script and run it with:
+```
+wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
+# For security purposes, please check the contents of collect_env.py before running it.
+python collect_env.py
+```
+
+ - PyTorch Version (e.g., 1.0):
+ - OS (e.g., Linux):
+ - How you installed PyTorch (`conda`, `pip`, source):
+ - Build command you used (if compiling from source):
+ - Python version:
+ - CUDA/cuDNN version:
+ - GPU models and configuration:
+ - Any other relevant information:
+
+## Additional context
+
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4DA Documentation"
+about: Report an issue related to https://pytorch.org/docs
+
+---
+
+## 📚 Documentation
+
+<!-- A clear and concise description of what content in https://pytorch.org/docs is an issue. If this has to do with the general https://pytorch.org website, please file an issue at https://github.com/pytorch/pytorch.github.io/issues/new/choose instead. If this has to do with https://pytorch.org/tutorials, please file an issue at https://github.com/pytorch/tutorials/issues/new -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,24 @@
+---
+name: "\U0001F680Feature Request"
+about: Submit a proposal/request for a new PyTorch feature
+
+---
+
+## 🚀 Feature
+<!-- A clear and concise description of the feature proposal -->
+
+## Motivation
+
+<!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->
+
+## Pitch
+
+<!-- A clear and concise description of what you want to happen. -->
+
+## Alternatives
+
+<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->
+
+## Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/questions-help-support.md

@@ -0,0 +1,13 @@
+---
+name: "❓Questions/Help/Support"
+about: Do you need support? We have resources.
+
+---
+
+## ❓ Questions and Help
+
+### Please note that this issue tracker is not a help form and this issue will be closed.
+
+We have a set of [listed resources available on the website](https://pytorch.org/resources). Our primary means of support is our discussion forum:
+
+- [Discussion Forum](https://discuss.pytorch.org/)

.gitignore

@@ -22,24 +22,33 @@
 aten/build/
 aten/src/ATen/Config.h
 aten/src/ATen/cuda/CUDAConfig.h
-build/
+caffe2/cpp_test/
 dist/
 docs/src/**/*
+docs/cpp/build
+docs/cpp/source/api
 test/.coverage
+test/.hypothesis/
 test/cpp/api/mnist
+test/custom_operator/model.pt
 test/data/gpu_tensors.pt
 test/data/legacy_modules.t7
 test/data/legacy_serialized.pt
 test/data/linear.pt
+dropout_model.pt
+test/generated_type_hints_smoketest.py
 test/htmlcov
+test/cpp_extensions/install/
 third_party/build/
 tools/shared/_utils_internal.py
 torch.egg-info/
+torch/__init__.pyi
 torch/csrc/autograd/generated/*
 torch/csrc/cudnn/cuDNN.cpp
 torch/csrc/generated
 torch/csrc/generic/TensorMethods.cpp
 torch/csrc/jit/generated/*
+torch/csrc/jit/fuser/config.h
 torch/csrc/nn/THCUNN.cpp
 torch/csrc/nn/THCUNN.cwrap
 torch/csrc/nn/THNN_generic.cpp
@@ -47,20 +56,35 @@ torch/csrc/nn/THNN_generic.cwrap
 torch/csrc/nn/THNN_generic.h
 torch/csrc/nn/THNN.cpp
 torch/csrc/nn/THNN.cwrap
+torch/bin/
+torch/cmake/
 torch/lib/*.a*
 torch/lib/*.dll*
+torch/lib/*.exe*
 torch/lib/*.dylib*
 torch/lib/*.h
 torch/lib/*.lib
 torch/lib/*.so*
+torch/lib/protobuf*.pc
 torch/lib/build
+torch/lib/caffe2/
 torch/lib/cmake
 torch/lib/include
 torch/lib/pkgconfig
 torch/lib/protoc
+torch/lib/protobuf/
 torch/lib/tmp_install
 torch/lib/torch_shm_manager
+torch/lib/site-packages/
+torch/lib/python*
+torch/lib64
+torch/include/
+torch/share/
+torch/test/
 torch/version.py
+# Root level file used in CI to specify certain env configs.
+# E.g., see .circleci/config.yaml
+env
 
 # IPython notebook checkpoints
 .ipynb_checkpoints
@@ -140,13 +164,12 @@ docs/source/scripts/activation_images/
 # PyCharm files
 .idea
 
-# Visual Studio Code files
-.vscode
-.vs
-
 # OSX dir files
 .DS_Store
 
+# GDB history
+.gdb_history
+
 ## Caffe2
 
 # build, distribute, and bins (+ python proto bindings)
@@ -181,7 +204,6 @@ docs/dev
 *.sst
 *.ldb
 LOCK
-LOG*
 CURRENT
 MANIFEST-*
 
@@ -194,3 +216,26 @@ caffe2.egg-info
 
 # Atom/Watchman required file
 .watchmanconfig
+
+# Files generated by CLion
+cmake-build-debug
+
+# Files generated by ctags
+CTAGS
+tags
+TAGS
+
+# BEGIN NOT-CLEAN-FILES (setup.py handles this marker. Do not change.)
+#
+# Below files are not deleted by "setup.py clean".
+
+# Visual Studio Code files
+.vscode
+.vs
+
+# YouCompleteMe config file
+.ycm_extra_conf.py
+
+# Files generated when a patch is rejected
+*.orig
+*.rej

.gitmodules

@@ -1,9 +1,3 @@
-[submodule "third_party/catch"]
-	path = third_party/catch
-	url = https://github.com/catchorg/Catch2.git
-[submodule "third_party/nanopb"]
-	path = third_party/nanopb
-	url = https://github.com/nanopb/nanopb.git
 [submodule "third_party/pybind11"]
 	path = third_party/pybind11
 	url = https://github.com/pybind/pybind11.git
@@ -16,9 +10,6 @@
 [submodule "third_party/googletest"]
 	path = third_party/googletest
 	url = https://github.com/google/googletest.git
-[submodule "third_party/nervanagpu"]
-	path = third_party/nervanagpu
-	url = https://github.com/NervanaSystems/nervanagpu.git
 [submodule "third_party/benchmark"]
 	path = third_party/benchmark
 	url = https://github.com/google/benchmark.git
@@ -61,21 +52,33 @@
 [submodule "third_party/python-six"]
 	path = third_party/python-six
 	url = https://github.com/benjaminp/six.git
-[submodule "third_party/ComputeLibrary"]
-	path = third_party/ComputeLibrary
-	url = https://github.com/ARM-software/ComputeLibrary.git
 [submodule "third_party/onnx"]
 	path = third_party/onnx
 	url = https://github.com/onnx/onnx.git
-[submodule "third_party/cereal"]
-	path = third_party/cereal
-	url = https://github.com/USCiLab/cereal
 [submodule "third_party/onnx-tensorrt"]
 	path = third_party/onnx-tensorrt
 	url = https://github.com/onnx/onnx-tensorrt
 [submodule "third_party/sleef"]
 	path = third_party/sleef
-	url = https://github.com/shibatch/sleef
+	url = https://github.com/zdevito/sleef
 [submodule "third_party/ideep"]
 	path = third_party/ideep
 	url = https://github.com/intel/ideep
+[submodule "third_party/nccl/nccl"]
+	path = third_party/nccl/nccl
+	url = https://github.com/NVIDIA/nccl
+[submodule "third_party/gemmlowp/gemmlowp"]
+	path = third_party/gemmlowp/gemmlowp
+	url = https://github.com/google/gemmlowp.git
+[submodule "third_party/QNNPACK"]
+	path = third_party/QNNPACK
+	url = https://github.com/pytorch/QNNPACK
+[submodule "third_party/neon2sse"]
+	path = third_party/neon2sse
+	url = https://github.com/intel/ARM_NEON_2_x86_SSE.git
+[submodule "third_party/fbgemm"]
+	path = third_party/fbgemm
+	url = https://github.com/pytorch/fbgemm
+[submodule "third_party/foxi"]
+	path = third_party/foxi
+	url = https://github.com/houseroad/foxi.git

.jenkins/caffe2/bench.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+# Anywhere except $ROOT_DIR should work. This is so the python import doesn't
+# get confused by any 'caffe2' directory in cwd
+cd "$INSTALL_PREFIX"
+
+if [[ $BUILD_ENVIRONMENT == *-cuda* ]]; then
+    num_gpus=$(nvidia-smi -L | wc -l)
+elif [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
+    num_gpus=$(rocminfo | grep 'Device Type.*GPU' | wc -l)
+else
+    num_gpus=0
+fi
+
+caffe2_pypath="$(cd /usr && $PYTHON -c 'import os; import caffe2; print(os.path.dirname(os.path.realpath(caffe2.__file__)))')"
+# Resnet50
+if (( $num_gpus == 0 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --train_data null --batch_size 128 --epoch_size 12800 --num_epochs 2 --use_cpu
+fi
+if (( $num_gpus >= 1 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --train_data null --batch_size 128 --epoch_size 12800 --num_epochs 2 --num_gpus 1
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --train_data null --batch_size 256 --epoch_size 25600 --num_epochs 2 --num_gpus 1 --float16_compute --dtype float16
+fi
+if (( $num_gpus >= 2 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --train_data null --batch_size 256 --epoch_size 25600 --num_epochs 2 --num_gpus 2
+fi
+if (( $num_gpus >= 4 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --train_data null --batch_size 512 --epoch_size 51200 --num_epochs 2 --num_gpus 4
+fi
+
+# ResNext
+if (( $num_gpus == 0 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --resnext_num_groups 32 --resnext_width_per_group 4 --num_layers 101 --train_data null --batch_size 32 --epoch_size 3200 --num_epochs 2 --use_cpu
+fi
+if (( $num_gpus >= 1 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --resnext_num_groups 32 --resnext_width_per_group 4 --num_layers 101 --train_data null --batch_size 32 --epoch_size 3200 --num_epochs 2 --num_gpus 1
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --resnext_num_groups 32 --resnext_width_per_group 4 --num_layers 101 --train_data null --batch_size 64 --epoch_size 3200 --num_epochs 2 --num_gpus 1 --float16_compute --dtype float16
+fi
+if (( $num_gpus >= 2 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --resnext_num_groups 32 --resnext_width_per_group 4 --num_layers 101 --train_data null --batch_size 64 --epoch_size 6400 --num_epochs 2 --num_gpus 2
+fi
+if (( $num_gpus >= 4 )); then
+    "$PYTHON" "$caffe2_pypath/python/examples/resnet50_trainer.py" --resnext_num_groups 32 --resnext_width_per_group 4 --num_layers 101 --train_data null --batch_size 128 --epoch_size 12800 --num_epochs 2 --num_gpus 4
+fi

.jenkins/caffe2/build.sh

@@ -2,46 +2,72 @@
 
 set -ex
 
-# The INSTALL_PREFIX here must match up with test.sh
-INSTALL_PREFIX="/usr/local/caffe2"
-LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
-CMAKE_ARGS=()
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
+# TODO: Migrate all centos jobs to use proper devtoolset
+if [[ "$BUILD_ENVIRONMENT" == *py2-cuda9.0-cudnn7-centos7* ]]; then
+  # There is a bug in pango packge on Centos7 that causes undefined
+  # symbols, upgrading glib2 to >=2.56.1 solves the issue. See
+  # https://bugs.centos.org/view.php?id=15495
+  sudo yum install -y -q glib2-2.56.1
+fi
 
-# Setup SCCACHE
-###############################################################################
-# Setup sccache if SCCACHE_BUCKET is set
-if [ -n "${SCCACHE_BUCKET}" ]; then
-  mkdir -p ./sccache
-
-  SCCACHE="$(which sccache)"
-  if [ -z "${SCCACHE}" ]; then
-    echo "Unable to find sccache..."
-    exit 1
-  fi
-
-  # Setup wrapper scripts
-  for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do
-    (
-      echo "#!/bin/sh"
-      echo "exec $SCCACHE $(which $compiler) \"\$@\""
-    ) > "./sccache/$compiler"
-    chmod +x "./sccache/$compiler"
+# CMAKE_ARGS are only passed to 'cmake' and the -Dfoo=bar does not work with
+# setup.py, so we build a list of foo=bars and then either convert it to
+# -Dfoo=bars or export them before running setup.py
+build_args=()
+build_to_cmake () {
+  cmake_args=()
+  for build_arg in $*; do
+    cmake_args+=("-D$build_arg")
   done
+  echo ${cmake_args[@]}
+}
 
-  if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]]; then
-    (
-      echo "#!/bin/sh"
-      echo "exec $SCCACHE $(which nvcc) \"\$@\""
-    ) > "./sccache/nvcc"
-    chmod +x "./sccache/nvcc"
+
+SCCACHE="$(which sccache)"
+if [ "$(which gcc)" != "/root/sccache/gcc" ]; then
+  # Setup SCCACHE
+  ###############################################################################
+  # Setup sccache if SCCACHE_BUCKET is set
+  if [ -n "${SCCACHE_BUCKET}" ]; then
+    mkdir -p ./sccache
+
+    SCCACHE="$(which sccache)"
+    if [ -z "${SCCACHE}" ]; then
+      echo "Unable to find sccache..."
+      exit 1
+    fi
+
+    # Setup wrapper scripts
+    wrapped="cc c++ gcc g++ x86_64-linux-gnu-gcc"
+    if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]]; then
+        wrapped="$wrapped nvcc"
+    fi
+    for compiler in $wrapped; do
+      (
+        echo "#!/bin/sh"
+
+        # TODO: if/when sccache gains native support for an
+        # SCCACHE_DISABLE flag analogous to ccache's CCACHE_DISABLE,
+        # this can be removed. Alternatively, this can be removed when
+        # https://github.com/pytorch/pytorch/issues/13362 is fixed.
+        #
+        # NOTE: carefully quoted - we want `which compiler` to be
+        # resolved as we execute the script, but SCCACHE_DISABLE and
+        # $@ to be evaluated when we execute the script
+        echo 'test $SCCACHE_DISABLE && exec '"$(which $compiler)"' "$@"'
+
+        echo "exec $SCCACHE $(which $compiler) \"\$@\""
+      ) > "./sccache/$compiler"
+      chmod +x "./sccache/$compiler"
+    done
+
+    export CACHE_WRAPPER_DIR="$PWD/sccache"
+
+    # CMake must find these wrapper scripts
+    export PATH="$CACHE_WRAPPER_DIR:$PATH"
   fi
-
-  export CACHE_WRAPPER_DIR="$PWD/sccache"
-
-  # CMake must find these wrapper scripts
-  export PATH="$CACHE_WRAPPER_DIR:$PATH"
 fi
 
 # Setup ccache if configured to use it (and not sccache)
@@ -59,6 +85,15 @@ if [ -z "${SCCACHE}" ] && which ccache > /dev/null; then
   export PATH="$CACHE_WRAPPER_DIR:$PATH"
 fi
 
+# sccache will fail for CUDA builds if all cores are used for compiling
+if [ -z "$MAX_JOBS" ]; then
+  if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]] && [ -n "${SCCACHE}" ]; then
+    MAX_JOBS=`expr $(nproc) - 1`
+  else
+    MAX_JOBS=$(nproc)
+  fi
+fi
+
 report_compile_cache_stats() {
   if [[ -n "${SCCACHE}" ]]; then
     "$SCCACHE" --show-stats
@@ -67,73 +102,61 @@ report_compile_cache_stats() {
   fi
 }
 
-###############################################################################
-# Explicitly set Python executable.
-###############################################################################
-# On Ubuntu 16.04 the default Python is still 2.7.
-PYTHON="$(which python)"
-if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
-  PYTHON=$(which "python${BASH_REMATCH[1]}")
-  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=${PYTHON}")
-fi
-
 
 ###############################################################################
-# Use special scripts for Android, conda, and setup builds
+# Use special scripts for Android and setup builds
 ###############################################################################
 if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   export ANDROID_NDK=/opt/ndk
-  CMAKE_ARGS+=("-DBUILD_BINARY=ON")
-  CMAKE_ARGS+=("-DBUILD_TEST=ON")
-  CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
-  CMAKE_ARGS+=("-DUSE_ZSTD=ON")
-  "${ROOT_DIR}/scripts/build_android.sh" ${CMAKE_ARGS[*]} "$@"
-  exit 0
-elif [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
-  "${ROOT_DIR}/scripts/build_anaconda.sh" --skip-tests --install-locally "$@"
-  report_compile_cache_stats
-
-  # This build will be tested against onnx tests, which needs onnx installed.
-  # At this point the visible protbuf installation will be in conda, since one
-  # of Caffe2's dependencies uses conda, so the correct protobuf include
-  # headers are those in conda as well
-  # This path comes from install_anaconda.sh which installs Anaconda into the
-  # docker image
-  PROTOBUF_INCDIR=/opt/conda/include pip install -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
-  report_compile_cache_stats
-  exit 0
-elif [[ $BUILD_ENVIRONMENT == *setup* ]]; then
-  rm -rf $INSTALL_PREFIX && mkdir $INSTALL_PREFIX
-  PYTHONPATH=$INSTALL_PREFIX $PYTHON setup_caffe2.py develop --install-dir $INSTALL_PREFIX
+  build_args+=("BUILD_BINARY=ON")
+  build_args+=("BUILD_TEST=ON")
+  build_args+=("USE_OBSERVERS=ON")
+  build_args+=("USE_ZSTD=ON")
+  "${ROOT_DIR}/scripts/build_android.sh" $(build_to_cmake ${build_args[@]}) "$@"
   exit 0
 fi
 
-
 ###############################################################################
-# Set cmake args
+# Set parameters
 ###############################################################################
-CMAKE_ARGS+=("-DBUILD_BINARY=ON")
-CMAKE_ARGS+=("-DBUILD_TEST=ON")
-CMAKE_ARGS+=("-DINSTALL_TEST=ON")
-CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
-CMAKE_ARGS+=("-DUSE_ZSTD=ON")
-CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}")
-
-if [[ $BUILD_ENVIRONMENT == *-aten-* ]]; then
-  if [[ CMAKE_ARGS != *USE_ATEN* ]] && [[ CMAKE_ARGS != *BUILD_ATEN* ]]; then
-    CMAKE_ARGS+=("-DBUILD_ATEN=ON")
-  fi
+if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+  build_args+=("BUILD_PYTHON=OFF")
+else
+  build_args+=("BUILD_PYTHON=ON")
+  build_args+=("PYTHON_EXECUTABLE=${PYTHON}")
 fi
 if [[ $BUILD_ENVIRONMENT == *mkl* ]]; then
-  CMAKE_ARGS+=("-DBLAS=MKL")
+  build_args+=("BLAS=MKL")
+  build_args+=("USE_MKLDNN=ON")
 fi
+build_args+=("BUILD_BINARY=ON")
+build_args+=("BUILD_TEST=ON")
+build_args+=("INSTALL_TEST=ON")
+build_args+=("USE_ZSTD=ON")
+
+if [[ $BUILD_ENVIRONMENT == *py2-cuda9.0-cudnn7-ubuntu16.04* ]]; then
+  # removing http:// duplicate in favor of nvidia-ml.list
+  # which is https:// version of the same repo
+  sudo rm -f /etc/apt/sources.list.d/nvidia-machine-learning.list
+  curl -o ./nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb
+  sudo dpkg -i ./nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb
+  sudo apt-key add /var/nvinfer-runtime-trt-repo-5.0.2-ga-cuda9.0/7fa2af80.pub
+  sudo apt-get -qq update
+  sudo apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda9.0 libnvinfer-dev=5.0.2-1+cuda9.0
+  rm ./nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda9.0_1-1_amd64.deb
+
+  build_args+=("USE_TENSORRT=ON")
+fi
+
 if [[ $BUILD_ENVIRONMENT == *cuda* ]]; then
-  CMAKE_ARGS+=("-DUSE_CUDA=ON")
-  CMAKE_ARGS+=("-DCUDA_ARCH_NAME=Maxwell")
-  CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
+  build_args+=("USE_CUDA=ON")
+  build_args+=("USE_NNPACK=OFF")
+
+  # Target only our CI GPU machine's CUDA arch to speed up the build
+  build_args+=("TORCH_CUDA_ARCH_LIST=Maxwell")
 
   # Explicitly set path to NVCC such that the symlink to ccache or sccache is used
-  CMAKE_ARGS+=("-DCUDA_NVCC_EXECUTABLE=${CACHE_WRAPPER_DIR}/nvcc")
+  build_args+=("CUDA_NVCC_EXECUTABLE=${CACHE_WRAPPER_DIR}/nvcc")
 
   # Ensure FindCUDA.cmake can infer the right path to the CUDA toolkit.
   # Setting PATH to resolve to the right nvcc alone isn't enough.
@@ -144,81 +167,103 @@ if [[ $BUILD_ENVIRONMENT == *cuda* ]]; then
   export PATH="/usr/local/cuda/bin:$PATH"
 fi
 if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
-  # TODO: This is patching the official FindHip to properly handly
-  # cmake generator expression. A PR is opened in the upstream repo here:
-  # https://github.com/ROCm-Developer-Tools/HIP/pull/516
-  # remove this hack once it's merged.
-  if [[ -f /opt/rocm/hip/cmake/FindHIP.cmake ]]; then
-    sudo sed -i 's/\ -I${dir}/\ $<$<BOOL:${dir}>:-I${dir}>/' /opt/rocm/hip/cmake/FindHIP.cmake
-  fi
+  build_args+=("USE_ROCM=ON")
+  # This is needed to enable ImageInput operator in resnet50_trainer
+  build_args+=("USE_OPENCV=ON")
+  # This is needed to read datasets from https://download.caffe2.ai/databases/resnet_trainer.zip
+  build_args+=("USE_LMDB=ON")
+  # When hcc runs out of memory, it silently exits without stopping
+  # the build process, leaving undefined symbols in the shared lib
+  # which will cause undefined symbol errors when later running
+  # tests. Setting MAX_JOBS to smaller number to make CI less flaky.
+  export MAX_JOBS=4
 
-  export LANG=C.UTF-8
-  export LC_ALL=C.UTF-8
-  export HCC_AMDGPU_TARGET=gfx900
+  ########## HIPIFY Caffe2 operators
+  ${PYTHON} "${ROOT_DIR}/tools/amd_build/build_amd.py"
+fi
+
+# building bundled nccl in this config triggers a bug in nvlink. For
+# more, see https://github.com/pytorch/pytorch/issues/14486
+if [[ "${BUILD_ENVIRONMENT}" == *-cuda8*-cudnn7* ]]; then
+    build_args+=("USE_SYSTEM_NCCL=ON")
 fi
 
 # Try to include Redis support for Linux builds
 if [ "$(uname)" == "Linux" ]; then
-  CMAKE_ARGS+=("-DUSE_REDIS=ON")
-fi
-
-# Currently, on Jenkins mac os, we will use custom protobuf. Mac OS
-# contbuild at the moment is minimal dependency - it doesn't use glog
-# or gflags either.
-if [ "$(uname)" == "Darwin" ]; then
-  CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=ON")
+  build_args+=("USE_REDIS=ON")
 fi
 
 # Use a speciallized onnx namespace in CI to catch hardcoded onnx namespace
-CMAKE_ARGS+=("-DONNX_NAMESPACE=ONNX_NAMESPACE_FOR_C2_CI")
-
-if [[ -n "$INTEGRATED" ]]; then
-    # TODO: This is a temporary hack to work around the issue that both
-    # caffe2 and pytorch have libcaffe2.so and crossfire at runtime.
-    CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")
-    CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
-    CMAKE_ARGS+=("-DCAFFE2_LINK_LOCAL_PROTOBUF=OFF")
-fi
-
-# We test the presence of cmake3 (for platforms like Centos and Ubuntu 14.04)
-# and use that if so.
-if [[ -x "$(command -v cmake3)" ]]; then
-    CMAKE_BINARY=cmake3
-else
-    CMAKE_BINARY=cmake
-fi
-# sccache will fail for CUDA builds if all cores are used for compiling
-if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]] && [ -n "${SCCACHE}" ]; then
-  MAX_JOBS=`expr $(nproc) - 1`
-else
-  MAX_JOBS=$(nproc)
-fi
-
+build_args+=("ONNX_NAMESPACE=ONNX_NAMESPACE_FOR_C2_CI")
 
 ###############################################################################
 # Configure and make
 ###############################################################################
-# Run cmake from ./build_caffe2 directory so it doesn't conflict with
-# standard PyTorch build directory. Eventually these won't need to
-# be separate.
-rm -rf build_caffe2
-mkdir build_caffe2
-cd ./build_caffe2
 
-# Configure
-${CMAKE_BINARY} "${ROOT_DIR}" ${CMAKE_ARGS[*]} "$@"
+if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+  # cmake-only non-setup.py build, to test cpp only bits. This installs into
+  # /usr/local/caffe2 and installs no Python tests
+  build_args+=("CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}")
+
+  # Run cmake from ./build_caffe2 directory so it doesn't conflict with
+  # standard PyTorch build directory. Eventually these won't need to
+  # be separate.
+  rm -rf build_caffe2
+  mkdir build_caffe2
+  cd ./build_caffe2
+
+  # We test the presence of cmake3 (for platforms like Centos and Ubuntu 14.04)
+  # and use that if so.
+  if [[ -x "$(command -v cmake3)" ]]; then
+      CMAKE_BINARY=cmake3
+  else
+      CMAKE_BINARY=cmake
+  fi
+
+  # Configure
+  ${CMAKE_BINARY} "${ROOT_DIR}" $(build_to_cmake ${build_args[@]}) "$@"
+
+  # Build
+  if [ "$(uname)" == "Linux" ]; then
+    make "-j${MAX_JOBS}" install
+  else
+    echo "Don't know how to build on $(uname)"
+    exit 1
+  fi
+
+  # This is to save test binaries for testing
+  mv "$INSTALL_PREFIX/test/" "$INSTALL_PREFIX/cpp_test/"
+
+  ls -lah $INSTALL_PREFIX
 
-# Build
-if [ "$(uname)" == "Linux" ]; then
-  make "-j${MAX_JOBS}" install
 else
-  echo "Don't know how to build on $(uname)"
-  exit 1
+  # Python build. Uses setup.py to install into site-packages
+  build_args+=("USE_LEVELDB=ON")
+  build_args+=("USE_LMDB=ON")
+  build_args+=("USE_OPENCV=ON")
+  build_args+=("BUILD_TEST=ON")
+  # These flags preserve the flags that were used before this refactor (blame
+  # me)
+  build_args+=("USE_GLOG=ON")
+  build_args+=("USE_GFLAGS=ON")
+  build_args+=("USE_FBGEMM=OFF")
+  build_args+=("USE_MKLDNN=OFF")
+  build_args+=("USE_DISTRIBUTED=ON")
+  for build_arg in "${build_args[@]}"; do
+    export $build_arg
+  done
+
+  # sccache will be stuck if  all cores are used for compiling
+  # see https://github.com/pytorch/pytorch/pull/7361
+  if [[ -n "${SCCACHE}" ]]; then
+    export MAX_JOBS=`expr $(nproc) - 1`
+  fi
+
+  $PYTHON setup.py install --user
+
+  report_compile_cache_stats
 fi
 
-report_compile_cache_stats
-
-
 ###############################################################################
 # Install ONNX
 ###############################################################################
@@ -227,47 +272,3 @@ report_compile_cache_stats
 pip install --user -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
 
 report_compile_cache_stats
-
-if [[ -n "$INTEGRATED" ]]; then
-  # sccache will be stuck if  all cores are used for compiling
-  # see https://github.com/pytorch/pytorch/pull/7361
-  if [[ -n "${SCCACHE}" ]]; then
-    export MAX_JOBS=`expr $(nproc) - 1`
-  fi
-  pip install --user -v -b /tmp/pip_install_torch "file://${ROOT_DIR}#egg=torch"
-fi
-
-report_compile_cache_stats
-
-# Symlink the caffe2 base python path into the system python path,
-# so that we can import caffe2 without having to change $PYTHONPATH.
-# Run in a subshell to contain environment set by /etc/os-release.
-#
-# This is only done when running on Jenkins!  We don't want to pollute
-# the user environment with Python symlinks and ld.so.conf.d hacks.
-#
-if [ -n "${JENKINS_URL}" ]; then
-  (
-    source /etc/os-release
-
-    function python_version() {
-      "$PYTHON" -c 'import sys; print("python%d.%d" % sys.version_info[0:2])'
-    }
-
-    # Debian/Ubuntu
-    if [[ "$ID_LIKE" == *debian* ]]; then
-      python_path="/usr/local/lib/$(python_version)/dist-packages"
-      sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
-    fi
-
-    # RHEL/CentOS
-    if [[ "$ID_LIKE" == *rhel* ]]; then
-      python_path="/usr/lib64/$(python_version)/site-packages/"
-      sudo ln -sf "${INSTALL_PREFIX}/caffe2" "${python_path}"
-    fi
-
-    # /etc/ld.so.conf.d is used on both Debian and RHEL
-    echo "${INSTALL_PREFIX}/lib" | sudo tee /etc/ld.so.conf.d/caffe2.conf
-    sudo ldconfig
-  )
-fi

.jenkins/caffe2/common.sh

@@ -0,0 +1,22 @@
+set -ex
+
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
+TEST_DIR="$ROOT_DIR/caffe2_tests"
+gtest_reports_dir="${TEST_DIR}/cpp"
+pytest_reports_dir="${TEST_DIR}/python"
+
+# Figure out which Python to use
+PYTHON="$(which python)"
+if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
+  PYTHON=$(which "python${BASH_REMATCH[1]}")
+fi
+
+# /usr/local/caffe2 is where the cpp bits are installed to in in cmake-only
+# builds. In +python builds the cpp tests are copied to /usr/local/caffe2 so
+# that the test code in .jenkins/test.sh is the same
+INSTALL_PREFIX="/usr/local/caffe2"
+
+mkdir -p "$gtest_reports_dir" || true
+mkdir -p "$pytest_reports_dir" || true
+mkdir -p "$INSTALL_PREFIX" || true

.jenkins/caffe2/test.sh

@@ -1,31 +1,6 @@
 #!/bin/bash
 
-set -ex
-
-LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
-TEST_DIR=$ROOT_DIR/caffe2_tests
-
-# Figure out which Python to use
-PYTHON="python"
-if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
-  PYTHON="python${BASH_REMATCH[1]}"
-fi
-
-# The prefix must mirror the setting from build.sh
-INSTALL_PREFIX="/usr/local/caffe2"
-
-# Anaconda builds have a special install prefix and python
-if [[ "$BUILD_ENVIRONMENT" == conda* ]]; then
-  # This path comes from install_anaconda.sh which installs Anaconda into the
-  # docker image
-  PYTHON="/opt/conda/bin/python"
-  INSTALL_PREFIX="/opt/conda/"
-fi
-
-# Add the site-packages in the caffe2 install prefix to the PYTHONPATH
-SITE_DIR=$($PYTHON -c "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix=''))")
-INSTALL_SITE_DIR="${INSTALL_PREFIX}/${SITE_DIR}"
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 # Skip tests in environments where they are not built/applicable
 if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
@@ -33,83 +8,119 @@ if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   exit 0
 fi
 
-# Set PYTHONPATH and LD_LIBRARY_PATH so that python can find the installed
-# Caffe2. This shouldn't be done on Anaconda, as Anaconda should handle this.
-if [[ "$BUILD_ENVIRONMENT" != conda* ]]; then
-  export PYTHONPATH="${PYTHONPATH}:$INSTALL_SITE_DIR"
-  export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTALL_PREFIX}/lib"
+# Find where cpp tests and Caffe2 itself are installed
+if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+  # For cmake only build we install everything into /usr/local
+  cpp_test_dir="$INSTALL_PREFIX/cpp_test"
+  ld_library_path="$INSTALL_PREFIX/lib"
+else
+  # For Python builds we install into python
+  # cd to /usr first so the python import doesn't get confused by any 'caffe2'
+  # directory in cwd
+  python_installation="$(dirname $(dirname $(cd /usr && $PYTHON -c 'import os; import caffe2; print(os.path.realpath(caffe2.__file__))')))"
+  caffe2_pypath="$python_installation/caffe2"
+  cpp_test_dir="$python_installation/torch/test"
+  ld_library_path="$python_installation/torch/lib"
 fi
 
-cd "$ROOT_DIR"
-
-if [ -d $TEST_DIR ]; then
-  echo "Directory $TEST_DIR already exists; please remove it..."
-  exit 1
-fi
-
-mkdir -p $TEST_DIR/{cpp,python}
-
-cd ${INSTALL_PREFIX}
-
-# C++ tests
+################################################################################
+# C++ tests #
+################################################################################
 echo "Running C++ tests.."
-gtest_reports_dir="${TEST_DIR}/cpp"
-junit_reports_dir="${TEST_DIR}/junit_reports"
-mkdir -p "$gtest_reports_dir" "$junit_reports_dir"
-for test in $(find "${INSTALL_PREFIX}/test" -executable -type f); do
+for test in $(find "$cpp_test_dir" -executable -type f); do
   case "$test" in
     # skip tests we know are hanging or bad
     */mkl_utils_test|*/aten/integer_divider_test)
       continue
       ;;
-    */aten/*)
-      # ATen uses test framework Catch2
-      "$test" -r=xml -o "${junit_reports_dir}/$(basename $test).xml"
+    */scalar_tensor_test|*/basic|*/native_test)
+      if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+        continue
+      else
+        LD_LIBRARY_PATH="$ld_library_path" "$test"
+      fi
       ;;
     *)
-      "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
+      # Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
+      # planning to migrate to gtest as the common PyTorch c++ test suite, we
+      # currently do NOT use the xml test reporter, because Catch doesn't
+      # support multiple reporters
+      # c.f. https://github.com/catchorg/Catch2/blob/master/docs/release-notes.md#223
+      # which means that enabling XML output means you lose useful stdout
+      # output for Jenkins.  It's more important to have useful console
+      # output than it is to have XML output for Jenkins.
+      # Note: in the future, if we want to use xml test reporter once we switch
+      # to all gtest, one can simply do:
+      LD_LIBRARY_PATH="$ld_library_path" \
+          "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
       ;;
   esac
 done
 
-# Get the relative path to where the caffe2 python module was installed
-CAFFE2_PYPATH="$INSTALL_SITE_DIR/caffe2"
+################################################################################
+# Python tests #
+################################################################################
+if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+  exit 0
+fi
+
+if [[ "$BUILD_ENVIRONMENT" == *ubuntu14.04* ]]; then
+  # Hotfix, use hypothesis 3.44.6 on Ubuntu 14.04
+  # See comments on
+  # https://github.com/HypothesisWorks/hypothesis-python/commit/eadd62e467d6cee6216e71b391951ec25b4f5830
+  sudo pip -q uninstall -y hypothesis
+  # "pip install hypothesis==3.44.6" from official server is unreliable on
+  # CircleCI, so we host a copy on S3 instead
+  sudo pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
+  sudo pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
+  sudo pip -q install hypothesis==3.44.6 -f https://s3.amazonaws.com/ossci-linux/wheels/hypothesis-3.44.6-py3-none-any.whl
+else
+  pip install --user --no-cache-dir hypothesis==3.59.0
+fi
 
 # Collect additional tests to run (outside caffe2/python)
 EXTRA_TESTS=()
 
 # CUDA builds always include NCCL support
 if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]]; then
-  EXTRA_TESTS+=("$CAFFE2_PYPATH/contrib/nccl")
+  EXTRA_TESTS+=("$caffe2_pypath/contrib/nccl")
 fi
 
-conda_ignore_test=()
-if [[ $BUILD_ENVIRONMENT == conda* ]]; then
-  # These tests both assume Caffe2 was built with leveldb, which is not the case
-  conda_ignore_test+=("--ignore $CAFFE2_PYPATH/python/dataio_test.py")
-  conda_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/checkpoint_test.py")
+rocm_ignore_test=()
+if [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
+  # Currently these tests are failing on ROCM platform:
+
+  # Unknown reasons, need to debug
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/operator_test/piecewise_linear_transform_test.py")
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/operator_test/softmax_ops_test.py")
+
+  # On ROCm, RCCL (distributed) development isn't complete.
+  # https://github.com/ROCmSoftwarePlatform/rccl
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/data_parallel_model_test.py")
 fi
 
+# NB: Warnings are disabled because they make it harder to see what
+# the actual erroring test is
+echo "Running Python tests.."
+pip install --user pytest-sugar
+"$PYTHON" \
+  -m pytest \
+  -x \
+  -v \
+  --disable-warnings \
+  --junit-xml="$pytest_reports_dir/result.xml" \
+  --ignore "$caffe2_pypath/python/test/executor_test.py" \
+  --ignore "$caffe2_pypath/python/operator_test/matmul_op_test.py" \
+  --ignore "$caffe2_pypath/python/operator_test/pack_ops_test.py" \
+  --ignore "$caffe2_pypath/python/mkl/mkl_sbn_speed_test.py" \
+  ${rocm_ignore_test[@]} \
+  "$caffe2_pypath/python" \
+  "${EXTRA_TESTS[@]}"
 
-# TODO: re-enable this for rocm CI jobs once we have more rocm workers
-if [[ $BUILD_ENVIRONMENT != *rocm* ]]; then
-  # Python tests
-  echo "Running Python tests.."
-  "$PYTHON" \
-    -m pytest \
-    -x \
-    -v \
-    --junit-xml="$TEST_DIR/python/result.xml" \
-    --ignore "$CAFFE2_PYPATH/python/test/executor_test.py" \
-    --ignore "$CAFFE2_PYPATH/python/operator_test/matmul_op_test.py" \
-    --ignore "$CAFFE2_PYPATH/python/operator_test/pack_ops_test.py" \
-    --ignore "$CAFFE2_PYPATH/python/mkl/mkl_sbn_speed_test.py" \
-    ${conda_ignore_test[@]} \
-    "$CAFFE2_PYPATH/python" \
-    "${EXTRA_TESTS[@]}"
-fi
-
-if [[ -n "$INTEGRATED" ]]; then
-  pip install --user pytest-xdist torchvision
-  "$ROOT_DIR/scripts/onnx/test.sh" -p
+#####################
+# torchvision tests #
+#####################
+if [[ "$BUILD_ENVIRONMENT" == *onnx* ]]; then
+  pip install --user torchvision
+  "$ROOT_DIR/scripts/onnx/test.sh"
 fi

.jenkins/pytorch/build-asan.sh

@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
-COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
+# shellcheck disable=SC2034
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Clang version:"
@@ -14,8 +16,24 @@ clang --version
 # symbolize=1: Gives us much better errors when things go wrong
 export ASAN_OPTIONS=detect_leaks=0:symbolize=1
 
+# FIXME: Remove the hardcoded "-pthread" option.
+# With asan build, the cmake thread CMAKE_HAVE_LIBC_CREATE[1] checking will
+# succeed because "pthread_create" is in libasan.so. However, libasan doesn't
+# have the full pthread implementation. Other advanced pthread functions doesn't
+# exist in libasan.so[2]. If we need some pthread advanced functions, we still
+# need to link the pthread library.
+# This issue is already fixed in cmake 3.13[3]. If we use the newer cmake, we
+# could remove this hardcoded option.
+#
+# [1] https://github.com/Kitware/CMake/blob/8cabaaf054a16ea9c8332ce8e9291bd026b38c62/Modules/FindThreads.cmake#L135
+# [2] https://wiki.gentoo.org/wiki/AddressSanitizer/Problems
+# [3] https://github.com/Kitware/CMake/commit/e9a1ddc594de6e6251bf06d732775dae2cabe4c8
+#
 # TODO: Make the ASAN flags a more unified env var
 CC="clang" CXX="clang++" LDSHARED="clang --shared" \
-  CFLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -shared-libasan" \
-  NO_CUDA=1 DEBUG=1 \
+  CFLAGS="-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -shared-libasan -pthread" \
+  CXX_FLAGS="-pthread" \
+  NO_CUDA=1 USE_MKLDNN=0 \
   python setup.py install
+
+assert_git_not_dirty

.jenkins/pytorch/build.sh

@@ -1,22 +1,41 @@
 #!/bin/bash
 
-if [[ "$BUILD_ENVIRONMENT" == "pytorch-linux-xenial-py3-clang5-asan" ]]; then
-  exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" $*
-fi
-
-# TODO: move this to Docker
-# TODO: add both NCCL and MPI in CI test by fixing these test first
-# sudo apt-get update
-# sudo apt-get install libnccl-dev libnccl2
-# sudo apt-get install openmpi-bin libopenmpi-dev
-
 # Required environment variable: $BUILD_ENVIRONMENT
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
-COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
+# shellcheck disable=SC2034
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
+# For distributed, four environmental configs:
+# (1) build with only NCCL
+# (2) build with NCCL and MPI
+# (3) build with only MPI
+# (4) build with neither
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+  # TODO: move this to Docker
+  sudo apt-get -qq update
+  sudo apt-get -qq install --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+fi
+
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9*gcc7* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]] || [[ "$BUILD_ENVIRONMENT" == *-trusty-py2.7.9* ]]; then
+  # TODO: move this to Docker
+  sudo apt-get -qq update
+  if [[ "$BUILD_ENVIRONMENT" == *-trusty-py2.7.9* ]]; then
+    sudo apt-get -qq install openmpi-bin libopenmpi-dev
+  else
+    sudo apt-get -qq install --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
+  fi
+  sudo apt-get -qq install --no-install-recommends openssh-client openssh-server
+  sudo mkdir -p /var/run/sshd
+fi
+
+if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-py3-clang5-asan* ]]; then
+  exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@"
+fi
+
 echo "Python version:"
 python --version
 
@@ -27,67 +46,133 @@ echo "CMake version:"
 cmake --version
 
 # TODO: Don't run this...
-pip install -r requirements.txt || true
-
-if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-  export HCC_AMDGPU_TARGET=gfx900
-  export LANG=C.UTF-8
-  export LC_ALL=C.UTF-8
-
-  sudo chown -R jenkins:jenkins /usr/local
-  rm -rf "$(dirname "${BASH_SOURCE[0]}")/../../../pytorch_amd/" || true
-  python "$(dirname "${BASH_SOURCE[0]}")/../../tools/amd_build/build_pytorch_amd.py"
-  USE_ROCM=1 python setup.py install
-  exit
-fi
+pip install -q -r requirements.txt || true
 
 # TODO: Don't install this here
 if ! which conda; then
-  pip install mkl mkl-devel
+  # In ROCm CIs, we are doing cross compilation on build machines with
+  # intel cpu and later run tests on machines with amd cpu.
+  # Also leave out two builds to make sure non-mkldnn builds still work.
+  if [[ "$BUILD_ENVIRONMENT" != *rocm* && "$BUILD_ENVIRONMENT" != *-trusty-py3.5-* && "$BUILD_ENVIRONMENT" != *-xenial-cuda8-cudnn7-py3-* ]]; then
+    pip install -q mkl mkl-devel
+    export USE_MKLDNN=1
+  else
+    export USE_MKLDNN=0
+  fi
+fi
+
+# Use special scripts for Android builds
+if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
+  export ANDROID_NDK=/opt/ndk
+  build_args=()
+  build_args+=("-DBUILD_BINARY=ON")
+  build_args+=("-DBUILD_TEST=ON")
+  build_args+=("-DUSE_OBSERVERS=ON")
+  build_args+=("-DUSE_ZSTD=ON")
+  exec ./scripts/build_android.sh "${build_args[@]}" "$@"
+fi
+
+if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+  # When hcc runs out of memory, it silently exits without stopping
+  # the build process, leaving undefined symbols in the shared lib
+  # which will cause undefined symbol errors when later running
+  # tests. Setting MAX_JOBS to smaller number to make CI less flaky.
+  export MAX_JOBS=4
+
+  # ROCm CI is using Caffe2 docker images, which needs these wrapper
+  # scripts to correctly use sccache.
+  if [ -n "${SCCACHE_BUCKET}" ]; then
+    mkdir -p ./sccache
+
+    SCCACHE="$(which sccache)"
+    if [ -z "${SCCACHE}" ]; then
+      echo "Unable to find sccache..."
+      exit 1
+    fi
+
+    # Setup wrapper scripts
+    for compiler in cc c++ gcc g++; do
+      (
+        echo "#!/bin/sh"
+        echo "exec $SCCACHE $(which $compiler) \"\$@\""
+      ) > "./sccache/$compiler"
+      chmod +x "./sccache/$compiler"
+    done
+
+    export CACHE_WRAPPER_DIR="$PWD/sccache"
+
+    # CMake must find these wrapper scripts
+    export PATH="$CACHE_WRAPPER_DIR:$PATH"
+  fi
+
+  python tools/amd_build/build_amd.py
+  # OPENCV is needed to enable ImageInput operator in caffe2 resnet5_trainer
+  # LMDB is needed to read datasets from https://download.caffe2.ai/databases/resnet_trainer.zip
+  USE_ROCM=1 USE_LMDB=1 USE_OPENCV=1 python setup.py install --user
+  exit 0
 fi
 
 # sccache will fail for CUDA builds if all cores are used for compiling
 # gcc 7 with sccache seems to have intermittent OOM issue if all cores are used
-if ([[ "$BUILD_ENVIRONMENT" == *cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *gcc7* ]]) && which sccache > /dev/null; then
-  export MAX_JOBS=`expr $(nproc) - 1`
+if [ -z "$MAX_JOBS" ]; then
+  if ([[ "$BUILD_ENVIRONMENT" == *cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *gcc7* ]]) && which sccache > /dev/null; then
+    export MAX_JOBS=$(($(nproc) - 1))
+  fi
 fi
 
 # Target only our CI GPU machine's CUDA arch to speed up the build
-export TORCH_CUDA_ARCH_LIST=5.2
+export TORCH_CUDA_ARCH_LIST="5.2"
+
+if [[ "$BUILD_ENVIRONMENT" == *ppc64le* ]]; then
+  export TORCH_CUDA_ARCH_LIST="6.0"
+fi
 
 if [[ "$BUILD_ENVIRONMENT" == *trusty-py3.6-gcc5.4* ]]; then
   export DEBUG=1
 fi
 
-WERROR=1 python setup.py install
-
-# Add the test binaries so that they won't be git clean'ed away
-git add -f build/bin
-
-# Testing ATen install
-if [[ "$BUILD_ENVIRONMENT" != *cuda* ]]; then
-  echo "Testing ATen install"
-  time tools/test_aten_install.sh
+# Patch required to build xla
+if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
+  git clone --recursive https://github.com/pytorch/xla.git
+  ./xla/scripts/apply_patches.sh
 fi
 
-# Test C FFI plugins
-# cffi install doesn't work for Python 3.7
-if [[ "$BUILD_ENVIRONMENT" != *pynightly* ]]; then
-  # TODO: Don't run this here
-  pip install cffi
-  git clone https://github.com/pytorch/extension-ffi.git
-  pushd extension-ffi/script
-  python build.py
-  popd
+
+# check that setup.py would fail with bad arguments
+echo "The next three invocations are expected to fail with invalid command error messages."
+( ! get_exit_code python setup.py bad_argument )
+( ! get_exit_code python setup.py clean] )
+( ! get_exit_code python setup.py clean bad_argument )
+
+# ppc64le build fails when WERROR=1
+# set only when building other architectures
+# only use for "python setup.py install" line
+if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
+  WERROR=1 python setup.py install
+else
+  python setup.py install
 fi
 
+assert_git_not_dirty
+
 # Test documentation build
-if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn6-py3* ]]; then
+if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn7-py3* ]]; then
   pushd docs
   # TODO: Don't run this here
-  pip install -r requirements.txt || true
-  make html
+  pip install -q -r requirements.txt || true
+  LC_ALL=C make html
   popd
+  assert_git_not_dirty
+fi
+
+# Test standalone c10 build
+if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn7-py3* ]]; then
+  mkdir -p c10/build
+  pushd c10/build
+  cmake ..
+  make -j
+  popd
+  assert_git_not_dirty
 fi
 
 # Test no-Python build
@@ -95,5 +180,69 @@ if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
   echo "Building libtorch"
   # NB: Install outside of source directory (at the same level as the root
   # pytorch folder) so that it doesn't get cleaned away prior to docker push.
-  WERROR=1 VERBOSE=1 tools/cpp_build/build_all.sh "$PWD/../cpp-build"
+  BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
+  mkdir -p ../cpp-build/caffe2
+  pushd ../cpp-build/caffe2
+  WERROR=1 VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
+  popd
+
+  # Build custom operator tests.
+  CUSTOM_OP_BUILD="$PWD/../custom-op-build"
+  CUSTOM_OP_TEST="$PWD/test/custom_operator"
+  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
+  mkdir "$CUSTOM_OP_BUILD"
+  pushd "$CUSTOM_OP_BUILD"
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake "$CUSTOM_OP_TEST"
+  make VERBOSE=1
+  popd
+  assert_git_not_dirty
+fi
+
+# Test XLA build
+if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
+  # TODO: Move this to Dockerfile.
+
+  pip install -q lark-parser
+
+  # Bazel doesn't work with sccache gcc. https://github.com/bazelbuild/bazel/issues/3642
+  sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main"
+  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
+  sudo apt-get -qq update
+
+  # Install clang-7 clang++-7 for xla
+  sudo apt-get -qq install clang-7 clang++-7
+
+  # Bazel dependencies
+  sudo apt-get -qq install pkg-config zip zlib1g-dev unzip
+  # XLA build requires Bazel
+  wget https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel-0.24.1-installer-linux-x86_64.sh
+  chmod +x bazel-*.sh
+  sudo ./bazel-*.sh
+  BAZEL="$(which bazel)"
+  if [ -z "${BAZEL}" ]; then
+    echo "Unable to find bazel..."
+    exit 1
+  fi
+
+  # Install bazels3cache for cloud cache
+  sudo apt-get -qq install npm
+  npm config set strict-ssl false
+  curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
+  sudo apt-get install -qq nodejs
+  sudo npm install -g bazels3cache
+  BAZELS3CACHE="$(which bazels3cache)"
+  if [ -z "${BAZELS3CACHE}" ]; then
+    echo "Unable to find bazels3cache..."
+    exit 1
+  fi
+
+  bazels3cache --bucket=ossci-compiler-cache-circleci-xla --maxEntrySizeBytes=0
+  pushd xla
+  export CC=clang-7 CXX=clang++-7
+  # Use cloud cache to build when available.
+  sed -i '/bazel build/ a --remote_http_cache=http://localhost:7777 \\' build_torch_xla_libs.sh
+
+  python setup.py install
+  popd
+  assert_git_not_dirty
 fi

.jenkins/pytorch/common.sh

@@ -25,6 +25,8 @@ set -ex
 # system; to find out more, grep for this string in ossci-job-dsl.
 echo "ENTERED_USER_LAND"
 
+export IS_PYTORCH_CI=1
+
 # compositional trap taken from https://stackoverflow.com/a/7287873/23845
 
 # note: printf is used instead of echo to avoid backslash
@@ -61,17 +63,33 @@ declare -f -t trap_add
 
 trap_add cleanup EXIT
 
+function assert_git_not_dirty() {
+    # TODO: we should add an option to `build_amd.py` that reverts the repo to
+    #       an unmodified state.
+    if ([[ "$BUILD_ENVIRONMENT" != *rocm* ]] && [[ "$BUILD_ENVIRONMENT" != *xla* ]]) ; then
+        git_status=$(git status --porcelain)
+        if [[ $git_status ]]; then
+            echo "Build left local git repository checkout dirty"
+            echo "git status --porcelain:"
+            echo "${git_status}"
+            exit 1
+        fi
+    fi
+}
+
 if which sccache > /dev/null; then
   # Save sccache logs to file
   sccache --stop-server || true
   rm ~/sccache_error.log || true
-  SCCACHE_ERROR_LOG=~/sccache_error.log RUST_LOG=sccache::server=error sccache --start-server
+  # increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR:
+  # https://github.com/pytorch/pytorch/pull/16645
+  SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server
 
   # Report sccache stats for easier debugging
   sccache --zero-stats
   function sccache_epilogue() {
     echo '=================== sccache compilation log ==================='
-    python $(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py ~/sccache_error.log
+    python "$(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py" ~/sccache_error.log
     echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
     sccache --show-stats
     sccache --stop-server || true
@@ -113,7 +131,8 @@ else
 fi
 
 if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda9-cudnn7-py3 ]] || \
-   [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-trusty-py3.6-gcc7* ]]; then
+   [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-trusty-py3.6-gcc7* ]] || \
+   [[ "$BUILD_ENVIRONMENT" == *pytorch_macos* ]]; then
   BUILD_TEST_LIBTORCH=1
 else
   BUILD_TEST_LIBTORCH=0
@@ -122,7 +141,7 @@ fi
 # Use conda cmake in some CI build. Conda cmake will be newer than our supported
 # min version 3.5, so we only do it in two builds that we know should use conda.
 if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda* ]]; then
-  if [[ "$BUILD_ENVIRONMENT" == *cuda8-cudnn6-py2* ]] || \
+  if [[ "$BUILD_ENVIRONMENT" == *cuda8-cudnn7-py2* ]] || \
      [[ "$BUILD_ENVIRONMENT" == *cuda9-cudnn7-py3* ]]; then
     if ! which conda; then
       echo "Expected ${BUILD_ENVIRONMENT} to use conda, but 'which conda' returns empty"
@@ -138,3 +157,11 @@ if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-cuda* ]]; then
     fi
   fi
 fi
+
+function get_exit_code() {
+  set +e
+  "$@"
+  retcode=$?
+  set -e
+  return $retcode
+}

.jenkins/pytorch/docker-build-test.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="docker-build-test"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 docker build -t pytorch .

.jenkins/pytorch/enabled-configs.txt

@@ -5,17 +5,21 @@
 # in this file will report a failure (so you don't forget to
 # reenable the tests on merge ;)
 
-pytorch-linux-xenial-cuda8-cudnn6-py3-build
-pytorch-linux-xenial-cuda8-cudnn6-py3-test
-pytorch-linux-xenial-cuda8-cudnn6-py3-multigpu-test
+pytorch-linux-xenial-cuda8-cudnn7-py3-build
+pytorch-linux-xenial-cuda8-cudnn7-py3-test
+pytorch-linux-xenial-cuda8-cudnn7-py3-multigpu-test
+pytorch-linux-xenial-cuda8-cudnn7-py3-nogpu-test
 pytorch-linux-xenial-cuda9-cudnn7-py2-build
 pytorch-linux-xenial-cuda9-cudnn7-py2-test
 pytorch-linux-xenial-cuda9-cudnn7-py3-build
 pytorch-linux-xenial-cuda9-cudnn7-py3-test
 pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7-build
 pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7-test
+pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7-build
+pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7-test
 pytorch-linux-xenial-py3-clang5-asan-build
 pytorch-linux-xenial-py3-clang5-asan-test
+pytorch-linux-xenial-py3-clang5-android-ndk-r19c-build
 pytorch-linux-trusty-py2.7.9-build
 pytorch-linux-trusty-py2.7.9-test
 pytorch-linux-trusty-py2.7-build
@@ -40,4 +44,14 @@ pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
 pytorch-docker-build-test
 short-perf-test-cpu
 short-perf-test-gpu
-py2-clang3.8-rocmnightly-ubuntu16.04-build
+py2-clang7-rocmdeb-ubuntu16.04
+py2-devtoolset7-rocmrpm-centos7.5
+pytorch-ppc64le-cuda9.2-cudnn7-py3-build
+pytorch-ppc64le-cuda9.2-cudnn7-py3-test
+pytorch-ppc64le-cuda9.1-cudnn7-py3-build
+pytorch-ppc64le-cuda9.1-cudnn7-py3-test
+pytorch-linux-xenial-cuda8-cudnn7-py3-NO_AVX2-test
+pytorch-linux-xenial-cuda8-cudnn7-py3-NO_AVX-NO_AVX2-test
+pytorch-linux-xenial-cuda8-cudnn7-py3-slow-test
+pytorch-xla-linux-trusty-py3.6-gcc5.4-build
+pytorch-xla-linux-trusty-py3.6-gcc5.4-test

.jenkins/pytorch/macos-build-test.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-build* ]]; then
+if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-build* ]]; then
   source "$(dirname "${BASH_SOURCE[0]}")/macos-build.sh"
 fi
 
-if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test* ]]; then
+if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test* ]]; then
   source "$(dirname "${BASH_SOURCE[0]}")/macos-test.sh"
 fi

.jenkins/pytorch/macos-build.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
+# shellcheck disable=SC2034
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 export PATH="/usr/local/bin:$PATH"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
@@ -17,11 +19,12 @@ source ${PYTORCH_ENV_DIR}/miniconda3/bin/activate
 conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja
 rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
 
+git submodule sync --recursive
 git submodule update --init --recursive
 export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
 
 # Build PyTorch
-if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
+if [[ "${BUILD_ENVIRONMENT}" == *cuda9.2* ]]; then
   export CUDA_VERSION=9.2
   export TORCH_CUDA_ARCH_LIST=5.2
   export PATH=/Developer/NVIDIA/CUDA-${CUDA_VERSION}/bin${PATH:+:${PATH}}
@@ -29,11 +32,15 @@ if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
   export CUDA_HOME=/Developer/NVIDIA/CUDA-${CUDA_VERSION}
   export NO_CUDA=0
 
-  # Eigen gives "explicit specialization of class must precede its first use" error
-  # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
-  export DEVELOPER_DIR=/Library/Developer/CommandLineTools
+  if [ -z "${IN_CIRCLECI}" ]; then
+    # Eigen gives "explicit specialization of class must precede its first use" error
+    # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
+    export DEVELOPER_DIR=/Library/Developer/CommandLineTools
+  fi
 else
-  export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+  if [ -z "${IN_CIRCLECI}" ]; then
+    export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+  fi
 fi
 
 export MACOSX_DEPLOYMENT_TARGET=10.9
@@ -46,7 +53,7 @@ if which sccache > /dev/null; then
   printf "#!/bin/sh\nexec sccache $(which clang) \$*" > "${PYTORCH_ENV_DIR}/clang"
   chmod a+x "${PYTORCH_ENV_DIR}/clang"
 
-  if [[ "${JOB_BASE_NAME}" == *cuda* ]]; then
+  if [[ "${BUILD_ENVIRONMENT}" == *cuda* ]]; then
     printf "#!/bin/sh\nexec sccache $(which nvcc) \$*" > "${PYTORCH_ENV_DIR}/nvcc"
     chmod a+x "${PYTORCH_ENV_DIR}/nvcc"
     export CUDA_NVCC_EXECUTABLE="${PYTORCH_ENV_DIR}/nvcc"
@@ -61,6 +68,10 @@ export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
 
 python setup.py install
 
+assert_git_not_dirty
+
 # Upload torch binaries when the build job is finished
-7z a ${IMAGE_COMMIT_TAG}.7z ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
-aws s3 cp ${IMAGE_COMMIT_TAG}.7z s3://ossci-macos-build/pytorch/${IMAGE_COMMIT_TAG}.7z --acl public-read
+if [ -z "${IN_CIRCLECI}" ]; then
+  7z a ${IMAGE_COMMIT_TAG}.7z ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+  aws s3 cp ${IMAGE_COMMIT_TAG}.7z s3://ossci-macos-build/pytorch/${IMAGE_COMMIT_TAG}.7z --acl public-read
+fi

.jenkins/pytorch/macos-test.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-test"
+# shellcheck disable=SC2034
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 export PATH="/usr/local/bin:$PATH"
@@ -15,19 +17,31 @@ if [ ! -d "${PYTORCH_ENV_DIR}/miniconda3" ]; then
 fi
 export PATH="${PYTORCH_ENV_DIR}/miniconda3/bin:$PATH"
 source ${PYTORCH_ENV_DIR}/miniconda3/bin/activate
-conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja
-rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+conda install -y mkl mkl-include numpy pyyaml setuptools cmake cffi ninja six
+pip install -q hypothesis "librosa>=0.6.2" psutil
 
+# faulthandler become built-in since 3.3
+if [[ ! $(python -c "import sys; print(int(sys.version_info >= (3, 3)))") == "1" ]]; then
+  pip install -q faulthandler
+fi
+
+if [ -z "${IN_CIRCLECI}" ]; then
+  rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+fi
+
+git submodule sync --recursive
 git submodule update --init --recursive
 export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
 
 # Test PyTorch
-if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
-  # Eigen gives "explicit specialization of class must precede its first use" error
-  # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
-  export DEVELOPER_DIR=/Library/Developer/CommandLineTools
-else
-  export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+if [ -z "${IN_CIRCLECI}" ]; then
+  if [[ "${BUILD_ENVIRONMENT}" == *cuda9.2* ]]; then
+    # Eigen gives "explicit specialization of class must precede its first use" error
+    # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
+    export DEVELOPER_DIR=/Library/Developer/CommandLineTools
+  else
+    export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+  fi
 fi
 export MACOSX_DEPLOYMENT_TARGET=10.9
 export CXX=clang++
@@ -38,43 +52,87 @@ export MAX_JOBS=2
 export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
 
 # Download torch binaries in the test jobs
-rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
-aws s3 cp s3://ossci-macos-build/pytorch/${IMAGE_COMMIT_TAG}.7z ${IMAGE_COMMIT_TAG}.7z
-7z x ${IMAGE_COMMIT_TAG}.7z -o"${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages"
+if [ -z "${IN_CIRCLECI}" ]; then
+  rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
+  aws s3 cp s3://ossci-macos-build/pytorch/${IMAGE_COMMIT_TAG}.7z ${IMAGE_COMMIT_TAG}.7z
+  7z x ${IMAGE_COMMIT_TAG}.7z -o"${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages"
+fi
+
+# Test that OpenMP is enabled
+pushd test
+if [[ ! $(python -c "import torch; print(int(torch.backends.openmp.is_available()))") == "1" ]]; then
+  echo "Build should have OpenMP enabled, but torch.backends.openmp.is_available() is False"
+  exit 1
+fi
+popd
 
 test_python_all() {
   echo "Ninja version: $(ninja --version)"
   python test/run_test.py --verbose
+  assert_git_not_dirty
 }
 
-test_cpp_api() {
+test_libtorch() {
   # C++ API
 
-  # NB: Install outside of source directory (at the same level as the root
-  # pytorch folder) so that it doesn't get cleaned away prior to docker push.
-  # But still clean it before we perform our own build.
-  #
-  CPP_BUILD="$PWD/../cpp-build"
-  rm -rf $CPP_BUILD
-  mkdir -p $CPP_BUILD
-  WERROR=1 VERBOSE=1 tools/cpp_build/build_all.sh "$CPP_BUILD"
+  if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
+    # NB: Install outside of source directory (at the same level as the root
+    # pytorch folder) so that it doesn't get cleaned away prior to docker push.
+    # But still clean it before we perform our own build.
 
-  python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+    echo "Testing libtorch"
 
-  # Unfortunately it seems like the test can't load from miniconda3
-  # without these paths being set
-  export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
-  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
-  "$CPP_BUILD"/libtorch/bin/test_api
+    CPP_BUILD="$PWD/../cpp-build"
+    rm -rf $CPP_BUILD
+    mkdir -p $CPP_BUILD/caffe2
+
+    BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
+    pushd $CPP_BUILD/caffe2
+    VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
+    popd
+
+    python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+
+    # Unfortunately it seems like the test can't load from miniconda3
+    # without these paths being set
+    export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
+    export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
+    TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" "$CPP_BUILD"/caffe2/bin/test_api
+
+    assert_git_not_dirty
+  fi
 }
 
-if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
+test_custom_script_ops() {
+  echo "Testing custom script operators"
+  pushd test/custom_operator
+  # Build the custom operator library.
+  rm -rf build && mkdir build
+  pushd build
+  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake ..
+  make VERBOSE=1
+  popd
+
+  # Run tests Python-side and export a script module.
+  python test_custom_ops.py -v
+  python model.py --export-script-module=model.pt
+  # Run tests C++-side and load the exported script module.
+  build/test_custom_ops ./model.pt
+  popd
+  assert_git_not_dirty
+}
+
+
+if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test ]]; then
   test_python_all
-  test_cpp_api
+  test_libtorch
+  test_custom_script_ops
 else
-  if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
+  if [[ "${BUILD_ENVIRONMENT}" == *-test1 ]]; then
     test_python_all
-  elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
-    test_cpp_api
+  elif [[ "${BUILD_ENVIRONMENT}" == *-test2 ]]; then
+    test_libtorch
+    test_custom_script_ops
   fi
 fi

.jenkins/pytorch/multigpu-test.sh

@@ -4,8 +4,28 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
-COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-multigpu-test"
+# shellcheck disable=SC2034
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Testing pytorch (distributed only)"
+
+if [ -n "${IN_CIRCLECI}" ]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get update
+    sudo apt-get install -y --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+  fi
+
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get update
+    sudo apt-get install -y --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
+    sudo apt-get install -y --no-install-recommends openssh-client openssh-server
+    sudo mkdir -p /var/run/sshd
+  fi
+fi
+
 time python test/run_test.py --verbose -i distributed
+assert_git_not_dirty

.jenkins/pytorch/perf_test/common.sh

@@ -10,7 +10,7 @@ get_runtime_of_command () {
   TIMEFORMAT=%R
 
   # runtime=$( { time ($@ &> /dev/null); } 2>&1 1>/dev/null)
-  runtime=$( { time $@; } 2>&1 1>/dev/null)
+  runtime=$( { time "$@"; } 2>&1 1>/dev/null)
   if [[ $runtime == *"Error"* ]]; then
     exit 1
   fi

.jenkins/pytorch/perf_test/compare_with_baseline.py

@@ -1,6 +1,6 @@
 import sys
 import json
-import numpy
+import math
 import argparse
 
 parser = argparse.ArgumentParser()
@@ -35,14 +35,25 @@ else:
 print("population mean: ", mean)
 print("population sigma: ", sigma)
 
+# Let the test pass if baseline number is NaN (which happened in
+# the past when we didn't have logic for catching NaN numbers)
+if math.isnan(mean) or math.isnan(sigma):
+    mean = sys.maxsize
+    sigma = 0.001
+
 sample_stats_data = json.loads(args.sample_stats)
 
-sample_mean = sample_stats_data['mean']
-sample_sigma = sample_stats_data['sigma']
+sample_mean = float(sample_stats_data['mean'])
+sample_sigma = float(sample_stats_data['sigma'])
 
 print("sample mean: ", sample_mean)
 print("sample sigma: ", sample_sigma)
 
+if math.isnan(sample_mean):
+    raise Exception('''Error: sample mean is NaN''')
+elif math.isnan(sample_sigma):
+    raise Exception('''Error: sample sigma is NaN''')
+
 z_value = (sample_mean - mean) / sigma
 
 print("z-value: ", z_value)
@@ -50,8 +61,10 @@ print("z-value: ", z_value)
 if z_value >= 3:
     raise Exception('''\n
 z-value >= 3, there is high chance of perf regression.\n
-To reproduce this regression, run `cd .jenkins/pytorch/perf_test/ && bash ''' + test_name + '''.sh` on your local machine and compare the runtime before/after your code change.
-''')
+To reproduce this regression, run
+`cd .jenkins/pytorch/perf_test/ && bash {}.sh` on your local machine
+and compare the runtime before/after your code change.
+'''.format(test_name))
 else:
     print("z-value < 3, no perf regression detected.")
     if args.update:

.jenkins/pytorch/perf_test/test_cpu_speed_mini_sequence_labeler.sh

@@ -19,14 +19,14 @@ test_cpu_speed_mini_sequence_labeler () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py)
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh

@@ -12,7 +12,7 @@ test_cpu_speed_mnist () {
 
   cd examples/mnist
 
-  pip install -r requirements.txt
+  pip install -q -r requirements.txt
 
   # Download data
   python main.py --epochs 0
@@ -20,7 +20,7 @@ test_cpu_speed_mnist () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -28,7 +28,7 @@ test_cpu_speed_mnist () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

.jenkins/pytorch/perf_test/test_cpu_speed_torch.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 . ./common.sh
 
 test_cpu_speed_torch () {
@@ -17,7 +19,7 @@ test_cpu_speed_torch () {
   fi
 
   if ! python perf-tests/modules/test_cpu_torch.py ${ARGS}; then
-    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash "${FUNCNAME[0]}".sh\` on your local machine and compare the runtime before/after your code change."
+    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
     exit 1
   fi
 }

.jenkins/pytorch/perf_test/test_cpu_speed_torch_tensor.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 . ./common.sh
 
 test_cpu_speed_torch_tensor () {
@@ -17,7 +19,7 @@ test_cpu_speed_torch_tensor () {
   fi
 
   if ! python perf-tests/modules/test_cpu_torch_tensor.py ${ARGS}; then
-    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash "${FUNCNAME[0]}".sh\` on your local machine and compare the runtime before/after your code change."
+    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
     exit 1
   fi
 }

.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh

@@ -19,7 +19,7 @@ test_gpu_speed_cudnn_lstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python cudnn_lstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -27,7 +27,7 @@ test_gpu_speed_cudnn_lstm () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh

@@ -19,7 +19,7 @@ test_gpu_speed_lstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python lstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -27,7 +27,7 @@ test_gpu_speed_lstm () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh

@@ -19,7 +19,7 @@ test_gpu_speed_mlstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python mlstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -27,7 +27,7 @@ test_gpu_speed_mlstm () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh

@@ -12,7 +12,7 @@ test_gpu_speed_mnist () {
 
   cd examples/mnist
 
-  pip install -r requirements.txt
+  pip install -q -r requirements.txt
 
   # Download data
   python main.py --epochs 0
@@ -20,7 +20,10 @@ test_gpu_speed_mnist () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  # Needs warm up to get accurate number
+  python main.py --epochs 1 --no-log
+
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -28,7 +31,7 @@ test_gpu_speed_mnist () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

.jenkins/pytorch/perf_test/test_gpu_speed_word_language_model.sh

@@ -28,7 +28,7 @@ test_gpu_speed_word_language_model () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --cuda --epochs 1)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -36,7 +36,7 @@ test_gpu_speed_word_language_model () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

.jenkins/pytorch/short-perf-test-cpu.sh

@@ -1,13 +1,18 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="short-perf-test-cpu"
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+SCRIPT_PARENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# shellcheck source=.jenkins/pytorch/common.sh
+source "$SCRIPT_PARENT_DIR/common.sh"
 
 cd .jenkins/pytorch/perf_test
 
 echo "Running CPU perf test for PyTorch..."
 
-pip install awscli
+pip install -q awscli
 
 # Set multipart_threshold to be sufficiently high, so that `aws s3 cp` is not a multipart read
 # More info at https://github.com/aws/aws-cli/issues/2321

.jenkins/pytorch/short-perf-test-gpu.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="short-perf-test-gpu"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 pushd .jenkins/pytorch/perf_test
 
 echo "Running GPU perf test for PyTorch..."
 
-pip install awscli
+# Trying to uninstall PyYAML can cause problem. Workaround according to:
+# https://github.com/pypa/pip/issues/5247#issuecomment-415571153
+pip install -q awscli --ignore-installed PyYAML
 
 # Set multipart_threshold to be sufficiently high, so that `aws s3 cp` is not a multipart read
 # More info at https://github.com/aws/aws-cli/issues/2321

.jenkins/pytorch/test.sh

@@ -1,32 +1,82 @@
 #!/bin/bash
 
-COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-test"
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
 # Required environment variable: $BUILD_ENVIRONMENT
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
 echo "Testing pytorch"
 
-if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-  echo "Skipping ROCm tests for now"
-  exit 0
+if [ -n "${IN_CIRCLECI}" ]; then
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get -qq update
+    sudo apt-get -qq install --allow-downgrades --allow-change-held-packages libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+  fi
+
+  if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
+    # TODO: move this to Docker
+    sudo apt-get -qq update
+    sudo apt-get -qq install --allow-downgrades --allow-change-held-packages openmpi-bin libopenmpi-dev
+    sudo apt-get -qq install --no-install-recommends openssh-client openssh-server
+    sudo mkdir -p /var/run/sshd
+  fi
+
+  if [[ "$BUILD_ENVIRONMENT" == *-slow-* ]]; then
+    export PYTORCH_TEST_WITH_SLOW=1
+    export PYTORCH_TEST_SKIP_FAST=1
+  fi
 fi
 
-# JIT C++ extensions require ninja.
-git clone https://github.com/ninja-build/ninja --quiet
-pushd ninja
-python ./configure.py --bootstrap
-export PATH="$PWD:$PATH"
-popd
+# --user breaks ppc64le builds and these packages are already in ppc64le docker
+if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
+  # JIT C++ extensions require ninja.
+  pip install -q ninja --user
+  # ninja is installed in /var/lib/jenkins/.local/bin
+  export PATH="/var/lib/jenkins/.local/bin:$PATH"
+
+  # TODO: move this to Docker
+  pip install -q hypothesis --user
+
+  # TODO: move this to Docker
+  PYTHON_VERSION=$(python -c 'import platform; print(platform.python_version())'|cut -c1)
+  echo $PYTHON_VERSION
+  if [[ $PYTHON_VERSION == "2" ]]; then
+    pip install -q https://s3.amazonaws.com/ossci-linux/wheels/tensorboard-1.14.0a0-py2-none-any.whl --user
+  else
+    pip install -q https://s3.amazonaws.com/ossci-linux/wheels/tensorboard-1.14.0a0-py3-none-any.whl --user
+  fi
+
+  # mypy will fail to install on Python <3.4.  In that case,
+  # we just won't run these tests.
+  pip install mypy --user || true
+fi
+
+# faulthandler become built-in since 3.3
+if [[ ! $(python -c "import sys; print(int(sys.version_info >= (3, 3)))") == "1" ]]; then
+  pip install -q faulthandler --user
+fi
 
 # DANGER WILL ROBINSON.  The LD_PRELOAD here could cause you problems
 # if you're not careful.  Check this if you made some changes and the
 # ASAN test is not working
 if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
-    export ASAN_OPTIONS=detect_leaks=0:symbolize=1
-    export UBSAN_OPTIONS=print_stacktrace=1
+    export ASAN_OPTIONS=detect_leaks=0:symbolize=1:strict_init_order=true
+    # We suppress the vptr volation, since we have separate copies of
+    # libprotobuf in both libtorch.so and libcaffe2.so, and it causes
+    # the following problem:
+    #    test_cse (__main__.TestJit) ... torch/csrc/jit/export.cpp:622:38:
+    #        runtime error: member call on address ... which does not point
+    #        to an object of type 'google::protobuf::MessageLite'
+    #        ...: note: object is of type 'onnx_torch::ModelProto'
+    #
+    # This problem should be solved when libtorch.so and libcaffe2.so are
+    # merged.
+    export UBSAN_OPTIONS=print_stacktrace=1:suppressions=$PWD/ubsan.supp
     export PYTORCH_TEST_WITH_ASAN=1
     export PYTORCH_TEST_WITH_UBSAN=1
     # TODO: Figure out how to avoid hard-coding these paths
@@ -35,13 +85,6 @@ if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
     # Increase stack size, because ASAN red zones use more stack
     ulimit -s 81920
 
-    function get_exit_code() {
-      set +e
-      "$@"
-      retcode=$?
-      set -e
-      return $retcode
-    }
     (cd test && python -c "import torch")
     echo "The next three invocations are expected to crash; if they don't that means ASAN/UBSAN is misconfigured"
     (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_csrc_asan(3)")
@@ -49,35 +92,51 @@ if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
     (cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_aten_asan(3)")
 fi
 
-export ATEN_DISABLE_AVX=
-export ATEN_DISABLE_AVX2=
-if [[ "${JOB_BASE_NAME}" == *-NO_AVX-* ]]; then
-  export ATEN_DISABLE_AVX=1
+if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+  export PYTORCH_TEST_WITH_ROCM=1
+  # ROCm CI is using Caffe2 docker images, which doesn't have several packages
+  # needed in testing. We install them here.
+  pip install -q psutil "librosa>=0.6.2" --user
 fi
-if [[ "${JOB_BASE_NAME}" == *-NO_AVX2-* ]]; then
-  export ATEN_DISABLE_AVX2=1
+
+if [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX-* ]]; then
+  export ATEN_CPU_CAPABILITY=default
+elif [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX2-* ]]; then
+  export ATEN_CPU_CAPABILITY=avx
 fi
 
 test_python_nn() {
   time python test/run_test.py --include nn --verbose
+  assert_git_not_dirty
 }
 
 test_python_all_except_nn() {
   time python test/run_test.py --exclude nn --verbose
+  assert_git_not_dirty
 }
 
 test_aten() {
   # Test ATen
-  if [[ "$BUILD_ENVIRONMENT" != *asan* ]]; then
+  # The following test(s) of ATen have already been skipped by caffe2 in rocm environment:
+  # scalar_tensor_test, basic, native_test
+  if ([[ "$BUILD_ENVIRONMENT" != *asan* ]] && [[ "$BUILD_ENVIRONMENT" != *rocm* ]]); then
     echo "Running ATen tests with pytorch lib"
     TORCH_LIB_PATH=$(python -c "import site; print(site.getsitepackages()[0])")/torch/lib
     # NB: the ATen test binaries don't have RPATH set, so it's necessary to
     # put the dynamic libraries somewhere were the dynamic linker can find them.
     # This is a bit of a hack.
-    ln -s "$TORCH_LIB_PATH"/libcaffe2* build/bin
-    ln -s "$TORCH_LIB_PATH"/libnccl* build/bin
+    if [[ "$BUILD_ENVIRONMENT" == *ppc64le* ]]; then
+      SUDO=sudo
+    fi
+
+    ${SUDO} ln -s "$TORCH_LIB_PATH"/libc10* build/bin
+    ${SUDO} ln -s "$TORCH_LIB_PATH"/libcaffe2* build/bin
+    ${SUDO} ln -s "$TORCH_LIB_PATH"/libmkldnn* build/bin
+    ${SUDO} ln -s "$TORCH_LIB_PATH"/libnccl* build/bin
+
     ls build/bin
     aten/tools/run_tests.sh build/bin
+    assert_git_not_dirty
   fi
 }
 
@@ -95,37 +154,72 @@ test_torchvision() {
   # this should be a transient requirement...)
   # See https://github.com/pytorch/pytorch/issues/7525
   #time python setup.py install
-  pip install .
+  pip install -q --user .
   popd
+  rm -rf vision
 }
 
 test_libtorch() {
   if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
-     echo "Testing libtorch"
-     CPP_BUILD="$PWD/../cpp-build"
-     if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
-       "$CPP_BUILD"/libtorch/bin/test_jit
-     else
-       "$CPP_BUILD"/libtorch/bin/test_jit "[cpu]"
-     fi
-     python tools/download_mnist.py --quiet -d test/cpp/api/mnist
-     OMP_NUM_THREADS=2 "$CPP_BUILD"/libtorch/bin/test_api
+    echo "Testing libtorch"
+    python test/cpp/jit/tests_setup.py setup
+    CPP_BUILD="$PWD/../cpp-build"
+    if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
+      "$CPP_BUILD"/caffe2/bin/test_jit
+    else
+      "$CPP_BUILD"/caffe2/bin/test_jit "[cpu]"
+    fi
+    python test/cpp/jit/tests_setup.py shutdown
+    python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+    OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" "$CPP_BUILD"/caffe2/bin/test_api
+    assert_git_not_dirty
   fi
 }
 
-if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
+test_custom_script_ops() {
+  if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
+    echo "Testing custom script operators"
+    CUSTOM_OP_BUILD="$PWD/../custom-op-build"
+    pushd test/custom_operator
+    cp -a "$CUSTOM_OP_BUILD" build
+    # Run tests Python-side and export a script module.
+    python test_custom_ops.py -v
+    python model.py --export-script-module=model.pt
+    # Run tests C++-side and load the exported script module.
+    build/test_custom_ops ./model.pt
+    popd
+    assert_git_not_dirty
+  fi
+}
+
+test_xla() {
+  export XLA_USE_XRT=1 XRT_DEVICE_MAP="CPU:0;/job:localservice/replica:0/task:0/device:XLA_CPU:0"
+  export XRT_WORKERS="localservice:0;grpc://localhost:40934"
+  pushd xla
+  python test/test_operations.py
+  python test/test_train_mnist.py --tidy
+  popd
+  assert_git_not_dirty
+}
+
+(cd test && python -c "import torch; print(torch.__config__.show())")
+
+if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
+  test_torchvision
+  test_xla
+elif [[ "${BUILD_ENVIRONMENT}" == *-test1 ]]; then
+  test_torchvision
+  test_python_nn
+elif [[ "${BUILD_ENVIRONMENT}" == *-test2 ]]; then
+  test_python_all_except_nn
+  test_aten
+  test_libtorch
+  test_custom_script_ops
+else
+  test_torchvision
   test_python_nn
   test_python_all_except_nn
   test_aten
-  test_torchvision
   test_libtorch
-else
-  if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
-    test_python_nn
-  elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
-    test_python_all_except_nn
-    test_aten
-    test_torchvision
-    test_libtorch
-  fi
+  test_custom_script_ops
 fi

.jenkins/pytorch/win-build.sh

@@ -9,147 +9,28 @@ if [ ! -f setup.py ]; then
   exit 1
 fi
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-build
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source "$SCRIPT_PARENT_DIR/common.sh"
 
 export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
 if [[ ${JOB_NAME} == *"develop"* ]]; then
   export IMAGE_COMMIT_TAG=develop-${IMAGE_COMMIT_TAG}
 fi
 
-mkdir -p ci_scripts/
+export TMP_DIR="${PWD}/build/win_tmp"
+export TMP_DIR_WIN=$(cygpath -w "${TMP_DIR}")
 
-cat >ci_scripts/upload_image.py << EOL
+export SCRIPT_HELPERS_DIR=$SCRIPT_PARENT_DIR/win-test-helpers
 
-import os
-import sys
-import boto3
 
-IMAGE_COMMIT_TAG = os.getenv('IMAGE_COMMIT_TAG')
+$SCRIPT_HELPERS_DIR/build_pytorch.bat
 
-session = boto3.session.Session()
-s3 = session.resource('s3')
-data = open(sys.argv[1], 'rb')
-s3.Bucket('ossci-windows-build').put_object(Key='pytorch/'+IMAGE_COMMIT_TAG+'.7z', Body=data)
-object_acl = s3.ObjectAcl('ossci-windows-build','pytorch/'+IMAGE_COMMIT_TAG+'.7z')
-response = object_acl.put(ACL='public-read')
+assert_git_not_dirty
 
-EOL
-
-cat >ci_scripts/build_pytorch.bat <<EOL
-
-set PATH=C:\\Program Files\\CMake\\bin;C:\\Program Files\\7-Zip;C:\\curl-7.57.0-win64-mingw\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Amazon\\AWSCLI;%PATH%
-
-:: Install MKL
-if "%REBUILD%"=="" (
-  if "%BUILD_ENVIRONMENT%"=="" (
-    curl -k https://s3.amazonaws.com/ossci-windows/mkl_2018.2.185.7z --output mkl.7z
-  ) else (
-    aws s3 cp s3://ossci-windows/mkl_2018.2.185.7z mkl.7z --quiet
-  )
-  7z x -aoa mkl.7z -omkl
-)
-set CMAKE_INCLUDE_PATH=%cd%\\mkl\\include
-set LIB=%cd%\\mkl\\lib;%LIB
-
-:: Install MAGMA
-if "%REBUILD%"=="" (
-  if "%BUILD_ENVIRONMENT%"=="" (
-    curl -k https://s3.amazonaws.com/ossci-windows/magma_cuda90_release_mkl_2018.2.185.7z --output magma_cuda90_release_mkl_2018.2.185.7z
-  ) else (
-    aws s3 cp s3://ossci-windows/magma_cuda90_release_mkl_2018.2.185.7z magma_cuda90_release_mkl_2018.2.185.7z --quiet
-  )
-  7z x -aoa magma_cuda90_release_mkl_2018.2.185.7z -omagma
-)
-set MAGMA_HOME=%cd%\\magma
-
-:: Install sccache
-mkdir %CD%\\tmp_bin
-if "%REBUILD%"=="" (
-  :check_sccache
-  %CD%\\tmp_bin\\sccache.exe --show-stats || (
-    taskkill /im sccache.exe /f /t || ver > nul
-    del %CD%\\tmp_bin\\sccache.exe
-    if "%BUILD_ENVIRONMENT%"=="" (
-      curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\\tmp_bin\\sccache.exe
-    ) else (
-      aws s3 cp s3://ossci-windows/sccache.exe %CD%\\tmp_bin\\sccache.exe
-    )
-    goto :check_sccache
-  )
-)
-
-:: Install Miniconda3
-if "%REBUILD%"=="" (
-  IF EXIST C:\\Jenkins\\Miniconda3 ( rd /s /q C:\\Jenkins\\Miniconda3 )
-  curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
-  .\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=C:\\Jenkins\\Miniconda3
-)
-call C:\\Jenkins\\Miniconda3\\Scripts\\activate.bat C:\\Jenkins\\Miniconda3
-if "%REBUILD%"=="" ( call conda install -y -q numpy cffi pyyaml boto3 )
-
-:: Install ninja
-if "%REBUILD%"=="" ( pip install ninja )
-
-call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64
-
-git submodule update --init --recursive
-
-set PATH=%CD%\\tmp_bin;C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\bin;C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\libnvvp;%PATH%
-set CUDA_PATH=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-set CUDA_PATH_V9_0=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-set NVTOOLSEXT_PATH=C:\\Program Files\\NVIDIA Corporation\\NvToolsExt
-set CUDNN_LIB_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\lib\\x64
-set CUDA_TOOLKIT_ROOT_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-set CUDNN_ROOT_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-
-:: Target only our CI GPU machine's CUDA arch to speed up the build
-set TORCH_CUDA_ARCH_LIST=5.2
-
-sccache --stop-server
-sccache --start-server
-sccache --zero-stats
-set CC=sccache cl
-set CXX=sccache cl
-
-set DISTUTILS_USE_SDK=1
-
-set CMAKE_GENERATOR=Ninja
-
-if not "%USE_CUDA%"=="1" (
-  if "%REBUILD%"=="" (
-    set NO_CUDA=1
-    python setup.py install
-  )
-  if errorlevel 1 exit /b 1
-  if not errorlevel 0 exit /b 1
-)
-
-if not "%USE_CUDA%"=="0" (
-  if "%REBUILD%"=="" (
-    sccache --show-stats
-    sccache --zero-stats
-    rd /s /q C:\\Jenkins\\Miniconda3\\Lib\\site-packages\\torch
-    copy %CD%\\tmp_bin\\sccache.exe tmp_bin\\nvcc.exe
-  )
-
-  set CUDA_NVCC_EXECUTABLE=%CD%\\tmp_bin\\nvcc
-
-  if "%REBUILD%"=="" set NO_CUDA=0
-
-  python setup.py install && sccache --show-stats && (
-    if "%BUILD_ENVIRONMENT%"=="" (
-      echo "NOTE: To run \`import torch\`, please make sure to activate the conda environment by running \`call C:\\Jenkins\\Miniconda3\\Scripts\\activate.bat C:\\Jenkins\\Miniconda3\` in Command Prompt before running Git Bash."
-    ) else (
-      7z a %IMAGE_COMMIT_TAG%.7z C:\\Jenkins\\Miniconda3\\Lib\\site-packages\\torch && python ci_scripts\\upload_image.py %IMAGE_COMMIT_TAG%.7z
-    )
-  )
-)
-
-EOL
-
-ci_scripts/build_pytorch.bat
-if [ ! -f $IMAGE_COMMIT_TAG.7z ] && [ ! ${BUILD_ENVIRONMENT} == "" ]; then
+if [ ! -f ${TMP_DIR}/${IMAGE_COMMIT_TAG}.7z ] && [ ! ${BUILD_ENVIRONMENT} == "" ]; then
     exit 1
 fi
 echo "BUILD PASSED"

.jenkins/pytorch/win-test-helpers/build_pytorch.bat

@@ -0,0 +1,78 @@
+if "%DEBUG%" == "1" (
+  set BUILD_TYPE=debug
+) ELSE (
+  set BUILD_TYPE=release
+)
+
+set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;%PATH%
+
+
+set INSTALLER_DIR=%SCRIPT_HELPERS_DIR%\installation-helpers
+
+call %INSTALLER_DIR%\install_mkl.bat
+call %INSTALLER_DIR%\install_magma.bat
+call %INSTALLER_DIR%\install_sccache.bat
+call %INSTALLER_DIR%\install_miniconda3.bat
+
+
+:: Install ninja
+if "%REBUILD%"=="" ( pip install -q ninja )
+
+git submodule sync --recursive
+git submodule update --init --recursive
+
+set PATH=%TMP_DIR_WIN%\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\libnvvp;%PATH%
+set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+set CUDA_PATH_V9_0=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+set NVTOOLSEXT_PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt
+set CUDNN_LIB_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\lib\x64
+set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+set CUDNN_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+
+:: Target only our CI GPU machine's CUDA arch to speed up the build
+set TORCH_CUDA_ARCH_LIST=5.2
+
+sccache --stop-server
+sccache --start-server
+sccache --zero-stats
+set CC=sccache cl
+set CXX=sccache cl
+
+set CMAKE_GENERATOR=Ninja
+
+if not "%USE_CUDA%"=="1" (
+  if "%REBUILD%"=="" (
+    set NO_CUDA=1
+    python setup.py install
+  )
+  if errorlevel 1 exit /b 1
+  if not errorlevel 0 exit /b 1
+)
+
+if not "%USE_CUDA%"=="0" (
+  if "%REBUILD%"=="" (
+    sccache --show-stats
+    sccache --zero-stats
+    rd /s /q %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch
+    for /f "delims=" %%i in ('where /R caffe2\proto *.py') do (
+      IF NOT "%%i" == "%CD%\caffe2\proto\__init__.py" (
+        del /S /Q %%i
+      )
+    )
+    copy %TMP_DIR_WIN%\bin\sccache.exe %TMP_DIR_WIN%\bin\nvcc.exe
+  )
+
+  set CUDA_NVCC_EXECUTABLE=%TMP_DIR_WIN%\bin\nvcc
+
+  if "%REBUILD%"=="" set NO_CUDA=0
+
+  python setup.py install --cmake && sccache --show-stats && (
+    if "%BUILD_ENVIRONMENT%"=="" (
+      echo NOTE: To run `import torch`, please make sure to activate the conda environment by running `call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3` in Command Prompt before running Git Bash.
+    ) else (
+      mv %CD%\build\bin\test_api.exe %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch\lib
+      7z a %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\caffe2 && python %SCRIPT_HELPERS_DIR%\upload_image.py %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z
+    )
+  )
+)
+

.jenkins/pytorch/win-test-helpers/download_image.py

@@ -0,0 +1,19 @@
+import os
+import sys
+import boto3
+import botocore
+
+IMAGE_COMMIT_TAG = os.getenv('IMAGE_COMMIT_TAG')
+
+session = boto3.session.Session()
+s3 = session.resource('s3')
+BUCKET_NAME = 'ossci-windows-build'
+KEY = 'pytorch/' + IMAGE_COMMIT_TAG + '.7z'
+LOCAL_FILE_PATH = sys.argv[1]
+try:
+    s3.Bucket(BUCKET_NAME).download_file(KEY, LOCAL_FILE_PATH)
+except botocore.exceptions.ClientError as e:
+    if e.response['Error']['Code'] == "404":
+        print("The object does not exist.")
+    else:
+        raise

.jenkins/pytorch/win-test-helpers/installation-helpers/install_magma.bat

@@ -0,0 +1,9 @@
+if "%REBUILD%"=="" (
+  if "%BUILD_ENVIRONMENT%"=="" (
+    curl -k https://s3.amazonaws.com/ossci-windows/magma_2.5.0_cuda90_%BUILD_TYPE%.7z --output %TMP_DIR_WIN%\magma_2.5.0_cuda90_%BUILD_TYPE%.7z
+  ) else (
+    aws s3 cp s3://ossci-windows/magma_2.5.0_cuda90_%BUILD_TYPE%.7z %TMP_DIR_WIN%\magma_2.5.0_cuda90_%BUILD_TYPE%.7z --quiet
+  )
+  7z x -aoa %TMP_DIR_WIN%\magma_2.5.0_cuda90_%BUILD_TYPE%.7z -o%TMP_DIR_WIN%\magma
+)
+set MAGMA_HOME=%TMP_DIR_WIN%\magma

.jenkins/pytorch/win-test-helpers/installation-helpers/install_miniconda3.bat

@@ -0,0 +1,15 @@
+if "%BUILD_ENVIRONMENT%"=="" (
+  set CONDA_PARENT_DIR=%CD%
+) else (
+  set CONDA_PARENT_DIR=C:\Jenkins
+)
+if "%REBUILD%"=="" (
+  IF EXIST %CONDA_PARENT_DIR%\Miniconda3 ( rd /s /q %CONDA_PARENT_DIR%\Miniconda3 )
+  curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe --output %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe
+  %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=%CONDA_PARENT_DIR%\Miniconda3
+)
+call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3
+if "%REBUILD%"=="" (
+  :: We have to pin Python version to 3.6.7, until mkl supports Python 3.7
+  call conda install -y -q python=3.6.7 numpy cffi pyyaml boto3
+)

.jenkins/pytorch/win-test-helpers/installation-helpers/install_mkl.bat

@@ -0,0 +1,10 @@
+if "%REBUILD%"=="" (
+  if "%BUILD_ENVIRONMENT%"=="" (
+    curl -k https://s3.amazonaws.com/ossci-windows/mkl_2018.2.185.7z --output %TMP_DIR_WIN%\mkl.7z
+  ) else (
+    aws s3 cp s3://ossci-windows/mkl_2018.2.185.7z %TMP_DIR_WIN%\mkl.7z --quiet
+  )
+  7z x -aoa %TMP_DIR_WIN%\mkl.7z -o%TMP_DIR_WIN%\mkl
+)
+set CMAKE_INCLUDE_PATH=%TMP_DIR_WIN%\mkl\include
+set LIB=%TMP_DIR_WIN%\mkl\lib;%LIB

.jenkins/pytorch/win-test-helpers/installation-helpers/install_sccache.bat

@@ -0,0 +1,15 @@
+mkdir %TMP_DIR_WIN%\bin
+
+if "%REBUILD%"=="" (
+  :check_sccache
+  %TMP_DIR_WIN%\bin\sccache.exe --show-stats || (
+    taskkill /im sccache.exe /f /t || ver > nul
+    del %TMP_DIR_WIN%\bin\sccache.exe
+    if "%BUILD_ENVIRONMENT%"=="" (
+      curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %TMP_DIR_WIN%\bin\sccache.exe
+    ) else (
+      aws s3 cp s3://ossci-windows/sccache.exe %TMP_DIR_WIN%\bin\sccache.exe
+    )
+    goto :check_sccache
+  )
+)

.jenkins/pytorch/win-test-helpers/run_python_nn_smoketests.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import subprocess
+
+
+TESTS = [
+    (
+        "Checking that caffe2.python is available",
+        "from caffe2.python import core",
+    ),
+    (
+        "Checking that MKL is available",
+        "import torch; exit(0 if torch.backends.mkl.is_available() else 1)",
+    ),
+    (
+        "Checking that CUDA archs are setup correctly",
+        "import torch; torch.randn([3,5]).cuda()",
+    ),
+    (
+        "Checking that magma is available",
+        "import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)",
+    ),
+    (
+        "Checking that CuDNN is available",
+        "import torch; exit(0 if torch.backends.cudnn.is_available() else 1)",
+    ),
+]
+
+
+if __name__ == "__main__":
+
+    for description, python_commands in TESTS:
+        print(description)
+        command_args = ["python", "-c", python_commands]
+        command_string = " ".join(command_args)
+        print("Command:", command_string)
+        subprocess.check_call(command_args)

.jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat

@@ -0,0 +1,54 @@
+if exist "%TMP_DIR%/ci_scripts/pytorch_env_restore.bat" (
+    call %TMP_DIR%/ci_scripts/pytorch_env_restore.bat
+    exit /b 0
+)
+
+set PATH=C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\Program Files\Amazon\AWSCLI;%PATH%
+
+:: Install Miniconda3
+if "%BUILD_ENVIRONMENT%"=="" (
+    set CONDA_PARENT_DIR=%CD%
+) else (
+    set CONDA_PARENT_DIR=C:\Jenkins
+)
+if NOT "%BUILD_ENVIRONMENT%"=="" (
+    IF EXIST %CONDA_PARENT_DIR%\Miniconda3 ( rd /s /q %CONDA_PARENT_DIR%\Miniconda3 )
+    curl https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe --output %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe
+    %TMP_DIR_WIN%\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=%CONDA_PARENT_DIR%\Miniconda3
+)
+call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3
+if NOT "%BUILD_ENVIRONMENT%"=="" (
+    :: We have to pin Python version to 3.6.7, until mkl supports Python 3.7
+    call conda install -y -q python=3.6.7 numpy mkl cffi pyyaml boto3 protobuf numba
+)
+pip install -q ninja future hypothesis "librosa>=0.6.2" psutil
+:: No need to install faulthandler since we only test Python >= 3.6 on Windows
+:: faulthandler is builtin since Python 3.3
+
+pushd .
+call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64
+popd
+
+set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\libnvvp;%PATH%
+set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+set CUDA_PATH_V9_0=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+set NVTOOLSEXT_PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt
+set CUDNN_LIB_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\lib\x64
+set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+set CUDNN_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0
+set PYTHONPATH=%TMP_DIR_WIN%\build;%PYTHONPATH%
+set NUMBAPRO_CUDALIB=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\bin
+set NUMBAPRO_LIBDEVICE=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\nvvm\libdevice
+set NUMBAPRO_NVVM=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\nvvm\bin\nvvm64_32_0.dll
+
+if NOT "%BUILD_ENVIRONMENT%"=="" (
+    pushd %TMP_DIR_WIN%\build
+    python %SCRIPT_HELPERS_DIR%\download_image.py %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z
+    :: 7z: -aos skips if exists because this .bat can be called multiple times
+    7z x %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z -aos
+    popd
+) else (
+    xcopy /s %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %TMP_DIR_WIN%\build\torch\
+)
+
+for /f "usebackq tokens=*" %%i in (`set`) do echo set "%%i" >> %TMP_DIR%/ci_scripts/pytorch_env_restore.bat

.jenkins/pytorch/win-test-helpers/test_custom_script_ops.bat

@@ -0,0 +1,19 @@
+call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
+
+cd test\custom_operator
+
+:: Build the custom operator library.
+mkdir build
+cd build
+:: Note: Caffe2 does not support MSVC + CUDA + Debug mode (has to be Release mode)
+cmake -DCMAKE_PREFIX_PATH=%TMP_DIR_WIN%\build\torch -DCMAKE_BUILD_TYPE=Release -GNinja ..
+ninja -v
+cd ..
+
+:: Run tests Python-side and export a script module.
+python test_custom_ops.py -v
+python model.py --export-script-module="build/model.pt"
+:: Run tests C++-side and load the exported script module.
+cd build
+set PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64;%TMP_DIR_WIN%\build\torch\lib;%PATH%
+test_custom_ops.exe model.pt

.jenkins/pytorch/win-test-helpers/test_libtorch.bat

@@ -0,0 +1,9 @@
+call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
+
+dir
+dir %TMP_DIR_WIN%\build
+dir %TMP_DIR_WIN%\build\torch
+dir %TMP_DIR_WIN%\build\torch\lib
+cd %TMP_DIR_WIN%\build\torch\lib
+set PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64;%TMP_DIR_WIN%\build\torch\lib;%PATH%
+test_api.exe --gtest_filter="-IntegrationTest.MNIST*"

.jenkins/pytorch/win-test-helpers/test_python_all_except_nn.bat

@@ -0,0 +1,2 @@
+call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
+cd test && python run_test.py --exclude nn --verbose && cd ..

.jenkins/pytorch/win-test-helpers/test_python_nn.bat

@@ -0,0 +1,16 @@
+call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
+
+
+pushd test
+
+echo Some smoke tests
+python %SCRIPT_HELPERS_DIR%\run_python_nn_smoketests.py
+if ERRORLEVEL 1 exit /b 1
+
+echo Run nn tests
+python run_test.py --include nn --verbose
+if ERRORLEVEL 1 exit /b 1
+
+popd
+
+

.jenkins/pytorch/win-test-helpers/upload_image.py

@@ -0,0 +1,12 @@
+import os
+import sys
+import boto3
+
+IMAGE_COMMIT_TAG = os.getenv('IMAGE_COMMIT_TAG')
+
+session = boto3.session.Session()
+s3 = session.resource('s3')
+with open(sys.argv[1], 'rb') as data:
+    s3.Bucket('ossci-windows-build').put_object(Key='pytorch/' + IMAGE_COMMIT_TAG + '.7z', Body=data)
+object_acl = s3.ObjectAcl('ossci-windows-build', 'pytorch/' + IMAGE_COMMIT_TAG + '.7z')
+response = object_acl.put(ACL='public-read')

.jenkins/pytorch/win-test.sh

@@ -1,93 +1,50 @@
-#!/bin/bash
+#!/bin/bash -ex
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-test
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source "$SCRIPT_PARENT_DIR/common.sh"
 
 export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
 if [[ ${JOB_NAME} == *"develop"* ]]; then
   export IMAGE_COMMIT_TAG=develop-${IMAGE_COMMIT_TAG}
 fi
 
-mkdir -p ci_scripts/
+export TMP_DIR="${PWD}/build/win_tmp"
+export TMP_DIR_WIN=$(cygpath -w "${TMP_DIR}")
 
-cat >ci_scripts/download_image.py << EOL
 
-import os
-import sys
-import boto3
-import botocore
+mkdir -p $TMP_DIR/build/torch
 
-IMAGE_COMMIT_TAG = os.getenv('IMAGE_COMMIT_TAG')
 
-session = boto3.session.Session()
-s3 = session.resource('s3')
-BUCKET_NAME = 'ossci-windows-build'
-KEY = 'pytorch/'+IMAGE_COMMIT_TAG+'.7z'
-LOCAL_FILE_PATH = sys.argv[1]
-try:
-    s3.Bucket(BUCKET_NAME).download_file(KEY, LOCAL_FILE_PATH)
-except botocore.exceptions.ClientError as e:
-    if e.response['Error']['Code'] == "404":
-        print("The object does not exist.")
-    else:
-        raise
+# This directory is used only to hold "pytorch_env_restore.bat", called via "setup_pytorch_env.bat"
+CI_SCRIPTS_DIR=$TMP_DIR/ci_scripts
+mkdir -p $CI_SCRIPTS_DIR
 
-EOL
+if [ -n "$(ls $CI_SCRIPTS_DIR/*)" ]; then
+    rm $CI_SCRIPTS_DIR/*
+fi
 
-cat >ci_scripts/setup_pytorch_env.bat <<EOL
 
-set PATH=C:\\Program Files\\CMake\\bin;C:\\Program Files\\7-Zip;C:\\curl-7.57.0-win64-mingw\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Amazon\\AWSCLI;%PATH%
+export SCRIPT_HELPERS_DIR=$SCRIPT_PARENT_DIR/win-test-helpers
 
-:: Install Miniconda3
-IF EXIST C:\\Jenkins\\Miniconda3 ( rd /s /q C:\\Jenkins\\Miniconda3 )
-curl https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
-.\Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /AddToPath=0 /D=C:\\Jenkins\\Miniconda3
-call C:\\Jenkins\\Miniconda3\\Scripts\\activate.bat C:\\Jenkins\\Miniconda3
-call conda install -y -q numpy mkl cffi pyyaml boto3
-
-pip install ninja
-
-call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x86_amd64
-
-set PATH=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\bin;C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\libnvvp;%PATH%
-set CUDA_PATH=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-set CUDA_PATH_V9_0=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-set NVTOOLSEXT_PATH=C:\\Program Files\\NVIDIA Corporation\\NvToolsExt
-set CUDNN_LIB_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0\\lib\\x64
-set CUDA_TOOLKIT_ROOT_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-set CUDNN_ROOT_DIR=C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0
-set PYTHONPATH=%CD%\\test;%PYTHONPATH%
-
-cd test/
-
-python ..\\ci_scripts\\download_image.py %IMAGE_COMMIT_TAG%.7z
-
-7z x %IMAGE_COMMIT_TAG%.7z
-
-cd ..
-
-EOL
-
-cat >ci_scripts/test_python_nn.bat <<EOL
-call ci_scripts/setup_pytorch_env.bat
-cd test/ && python run_test.py --include nn --verbose && cd ..
-EOL
-
-cat >ci_scripts/test_python_all_except_nn.bat <<EOL
-call ci_scripts/setup_pytorch_env.bat
-cd test/ && python run_test.py --exclude nn --verbose && cd ..
-EOL
 
 run_tests() {
     if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then
-        ci_scripts/test_python_nn.bat && ci_scripts/test_python_all_except_nn.bat
+        $SCRIPT_HELPERS_DIR/test_python_nn.bat && \
+        $SCRIPT_HELPERS_DIR/test_python_all_except_nn.bat && \
+        $SCRIPT_HELPERS_DIR/test_custom_script_ops.bat && \
+        $SCRIPT_HELPERS_DIR/test_libtorch.bat
     else
         if [[ "${JOB_BASE_NAME}" == *-test1 ]]; then
-            ci_scripts/test_python_nn.bat
+            $SCRIPT_HELPERS_DIR/test_python_nn.bat
         elif [[ "${JOB_BASE_NAME}" == *-test2 ]]; then
-            ci_scripts/test_python_all_except_nn.bat
+            $SCRIPT_HELPERS_DIR/test_python_all_except_nn.bat && \
+            $SCRIPT_HELPERS_DIR/test_custom_script_ops.bat && \
+            $SCRIPT_HELPERS_DIR/test_libtorch.bat
         fi
     fi
 }
 
-run_tests && echo "TEST PASSED"
+run_tests && assert_git_not_dirty && echo "TEST PASSED"

.jenkins/run-shellcheck.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -xe
+
+# One may want to invoke this script locally as follows:
+#
+#   .jenkins/run-shellcheck.sh --color=always | less -R
+
+
+EXCLUSIONS=SC2086,SC1091,SC2155,SC1090,SC2164,SC1003
+
+find .jenkins/pytorch -name *.sh | xargs shellcheck --exclude=$EXCLUSIONS --external-sources "$@" || true

.travis.aten.yml

@@ -27,5 +27,5 @@ matrix:
     include:
         env: LINT_CHECK
         python: "2.7"
-        install: pip install flake8
+        install: pip install flake8-mypy
         script: flake8