frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-21 05:34:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Migrate all the Caffe2 Centos builds to explicity use devltoolset (#28465)

Browse Source 
Summary:
Continues https://github.com/pytorch/pytorch/pull/28431 with a new branch name that can trigger all the CI

https://github.com/pytorch/pytorch/issues/28059

pytorch/ossci-job-dsl@b2c823a
Pull Request resolved: https://github.com/pytorch/pytorch/pull/28465

Differential Revision: D18104647

Pulled By: bddppq

fbshipit-source-id: 24decf44bdf73bd8a9c64d5fcaf34eec7a356f6e
This commit is contained in:
Junjie Bai
2019-10-25 12:33:41 -07:00
committed by Facebook Github Bot
parent 0253e23d3f
commit b19bbde561
9 changed files with 74 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
.circleci/cimodel/data/caffe2_build_data.py
View File

@ -4,29 +4,29 @@ from cimodel.lib.conf_tree import Ver
CONFIG_TREE_DATA = [
(Ver("ubuntu", "16.04"), [
(Ver("cuda", "9.0"), [
([Ver("cuda", "9.0")], [
# TODO make explicit that this is a "secret TensorRT build"
# (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749)
# TODO Uh oh, were we supposed to make this one important?!
X("py2"),
XImportant("cmake"),
]),
(Ver("cuda", "10.1"), [XImportant("py3.5")]), # TensorRT 6 build
(Ver("mkl"), [XImportant("py2")]),
(Ver("gcc", "5"), [XImportant("onnx_py2")]),
(Ver("clang", "3.8"), [X("py2")]),
(Ver("clang", "3.9"), [X("py2")]),
(Ver("clang", "7"), [XImportant("py2"), XImportant("onnx_py3.6")]),
(Ver("android"), [XImportant("py2")]),
([Ver("cuda", "10.1")], [XImportant("py3.5")]), # TensorRT 6 build
([Ver("mkl")], [XImportant("py2")]),
([Ver("gcc", "5")], [XImportant("onnx_py2")]),
([Ver("clang", "3.8")], [X("py2")]),
([Ver("clang", "3.9")], [X("py2")]),
([Ver("clang", "7")], [XImportant("py2"), XImportant("onnx_py3.6")]),
([Ver("android")], [XImportant("py2")]),
]),
(Ver("centos", "7"), [
(Ver("cuda", "9.0"), [X("py2")]),
([Ver("devtoolset", "7"), Ver("cuda", "9.2")], [X("py3.6")]),
]),
(Ver("macos", "10.13"), [
# TODO ios and system aren't related. system qualifies where the python comes
# from (use the system python instead of homebrew or anaconda)
(Ver("ios"), [X("py2")]),
(Ver("system"), [XImportant("py2")]),
([Ver("ios")], [X("py2")]),
([Ver("system")], [XImportant("py2")]),
]),
]
@ -50,12 +50,12 @@ class TreeConfigNode(ConfigNode):
def is_build_only(self):
if str(self.find_prop("language_version")) == "onnx_py3.6":
return False
return str(self.find_prop("compiler_version")) in [
return set(str(c) for c in self.find_prop("compiler_version")).intersection({
"clang3.8",
"clang3.9",
"clang7",
"android",
] or self.find_prop("distro_version").name == "macos"
}) or self.find_prop("distro_version").name == "macos"
class TopLevelNode(TreeConfigNode):

29
.circleci/cimodel/data/caffe2_build_definitions.py
View File

@ -12,23 +12,29 @@ from dataclasses import dataclass
DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
DOCKER_IMAGE_VERSION = 325
DOCKER_IMAGE_VERSION = 336
@dataclass
class Conf:
language: str
distro: Ver
compiler: Ver
# There could be multiple compiler versions configured (e.g. nvcc
# for gpu files and host compiler (gcc/clang) for cpu files)
compilers: [Ver]
build_only: bool
is_important: bool
@property
def compiler_names(self):
return [c.name for c in self.compilers]
# TODO: Eventually we can probably just remove the cudnn7 everywhere.
def get_cudnn_insertion(self):
omit = self.language == "onnx_py2" \
or self.language == "onnx_py3.6" \
or self.compiler.name in ["android", "mkl", "clang"] \
or set(self.compiler_names).intersection({"android", "mkl", "clang"}) \
or str(self.distro) in ["ubuntu14.04", "macos10.13"]
return [] if omit else ["cudnn7"]
@ -40,7 +46,7 @@ class Conf:
] + self.get_build_name_middle_parts()
def get_build_name_middle_parts(self):
return [str(self.compiler)] + self.get_cudnn_insertion() + [str(self.distro)]
return [str(c) for c in self.compilers] + self.get_cudnn_insertion() + [str(self.distro)]
def construct_phase_name(self, phase):
root_parts = self.get_build_name_root_parts()
@ -80,11 +86,11 @@ class Conf:
build_env_name = "-".join(parts)
parameters["build_environment"] = miniutils.quote(build_env_name)
if self.compiler.name == "ios":
if "ios" in self.compiler_names:
parameters["build_ios"] = miniutils.quote("1")
if phase == "test":
# TODO cuda should not be considered a compiler
if self.compiler.name == "cuda":
if "cuda" in self.compiler_names:
parameters["use_cuda_docker_runtime"] = miniutils.quote("1")
if self.distro.name != "macos":
@ -92,7 +98,7 @@ class Conf:
if self.build_only:
parameters["build_only"] = miniutils.quote("1")
if phase == "test":
resource_class = "large" if self.compiler.name != "cuda" else "gpu.medium"
resource_class = "large" if "cuda" not in self.compiler_names else "gpu.medium"
parameters["resource_class"] = resource_class
return parameters
@ -125,11 +131,10 @@ def instantiate_configs():
root = get_root()
found_configs = conf_tree.dfs(root)
for fc in found_configs:
c = Conf(
language=fc.find_prop("language_version"),
distro=fc.find_prop("distro_version"),
compiler=fc.find_prop("compiler_version"),
compilers=fc.find_prop("compiler_version"),
build_only=fc.find_prop("build_only"),
is_important=fc.find_prop("important"),
)
@ -143,12 +148,8 @@ def get_workflow_jobs():
configs = instantiate_configs()
# TODO Why don't we build this config?
# See https://github.com/pytorch/pytorch/pull/17323#discussion_r259450540
filtered_configs = filter(lambda x: not (str(x.distro) == "ubuntu14.04" and str(x.compiler) == "gcc4.9"), configs)
x = []
for conf_options in filtered_configs:
for conf_options in configs:
phases = ["build"]
if not conf_options.build_only:

46
.circleci/config.yml
View File

@ -1930,7 +1930,7 @@ workflows:
- master
- /ci-all\/.*/
build_environment: "caffe2-py2-cuda9.0-cudnn7-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
- caffe2_linux_test:
name: caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_test
requires:
@ -1943,14 +1943,14 @@ workflows:
- /ci-all\/.*/
build_environment: "caffe2-py2-cuda9.0-cudnn7-ubuntu16.04-test"
use_cuda_docker_runtime: "1"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
resource_class: gpu.medium
- caffe2_linux_build:
name: caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_build
requires:
- setup
build_environment: "caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
- caffe2_linux_test:
name: caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_test
requires:
@ -1958,14 +1958,14 @@ workflows:
- caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_build
build_environment: "caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04-test"
use_cuda_docker_runtime: "1"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
resource_class: gpu.medium
- caffe2_linux_build:
name: caffe2_py3_5_cuda10_1_cudnn7_ubuntu16_04_build
requires:
- setup
build_environment: "caffe2-py3.5-cuda10.1-cudnn7-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:336"
- caffe2_linux_test:
name: caffe2_py3_5_cuda10_1_cudnn7_ubuntu16_04_test
requires:
@ -1973,35 +1973,35 @@ workflows:
- caffe2_py3_5_cuda10_1_cudnn7_ubuntu16_04_build
build_environment: "caffe2-py3.5-cuda10.1-cudnn7-ubuntu16.04-test"
use_cuda_docker_runtime: "1"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:336"
resource_class: gpu.medium
- caffe2_linux_build:
name: caffe2_py2_mkl_ubuntu16_04_build
requires:
- setup
build_environment: "caffe2-py2-mkl-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:336"
- caffe2_linux_test:
name: caffe2_py2_mkl_ubuntu16_04_test
requires:
- setup
- caffe2_py2_mkl_ubuntu16_04_build
build_environment: "caffe2-py2-mkl-ubuntu16.04-test"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:336"
resource_class: large
- caffe2_linux_build:
name: caffe2_onnx_py2_gcc5_ubuntu16_04_build
requires:
- setup
build_environment: "caffe2-onnx-py2-gcc5-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:336"
- caffe2_linux_test:
name: caffe2_onnx_py2_gcc5_ubuntu16_04_test
requires:
- setup
- caffe2_onnx_py2_gcc5_ubuntu16_04_build
build_environment: "caffe2-onnx-py2-gcc5-ubuntu16.04-test"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:336"
resource_class: large
- caffe2_linux_build:
name: caffe2_py2_clang3_8_ubuntu16_04_build
@ -2013,7 +2013,7 @@ workflows:
- master
- /ci-all\/.*/
build_environment: "caffe2-py2-clang3.8-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.8-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.8-ubuntu16.04:336"
build_only: "1"
- caffe2_linux_build:
name: caffe2_py2_clang3_9_ubuntu16_04_build
@ -2025,38 +2025,38 @@ workflows:
- master
- /ci-all\/.*/
build_environment: "caffe2-py2-clang3.9-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.9-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.9-ubuntu16.04:336"
build_only: "1"
- caffe2_linux_build:
name: caffe2_py2_clang7_ubuntu16_04_build
requires:
- setup
build_environment: "caffe2-py2-clang7-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang7-ubuntu16.04:336"
build_only: "1"
- caffe2_linux_build:
name: caffe2_onnx_py3_6_clang7_ubuntu16_04_build
requires:
- setup
build_environment: "caffe2-onnx-py3.6-clang7-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:336"
- caffe2_linux_test:
name: caffe2_onnx_py3_6_clang7_ubuntu16_04_test
requires:
- setup
- caffe2_onnx_py3_6_clang7_ubuntu16_04_build
build_environment: "caffe2-onnx-py3.6-clang7-ubuntu16.04-test"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:336"
resource_class: large
- caffe2_linux_build:
name: caffe2_py2_android_ubuntu16_04_build
requires:
- setup
build_environment: "caffe2-py2-android-ubuntu16.04-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-android-ubuntu16.04:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-android-ubuntu16.04:336"
build_only: "1"
- caffe2_linux_build:
name: caffe2_py2_cuda9_0_cudnn7_centos7_build
name: caffe2_py3_6_devtoolset7_cuda9_2_cudnn7_centos7_build
requires:
- setup
filters:
@ -2064,21 +2064,21 @@ workflows:
only:
- master
- /ci-all\/.*/
build_environment: "caffe2-py2-cuda9.0-cudnn7-centos7-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:325"
build_environment: "caffe2-py3.6-devtoolset7-cuda9.2-cudnn7-centos7-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-devtoolset7-cuda9.2-cudnn7-centos7:336"
- caffe2_linux_test:
name: caffe2_py2_cuda9_0_cudnn7_centos7_test
name: caffe2_py3_6_devtoolset7_cuda9_2_cudnn7_centos7_test
requires:
- setup
- caffe2_py2_cuda9_0_cudnn7_centos7_build
- caffe2_py3_6_devtoolset7_cuda9_2_cudnn7_centos7_build
filters:
branches:
only:
- master
- /ci-all\/.*/
build_environment: "caffe2-py2-cuda9.0-cudnn7-centos7-test"
build_environment: "caffe2-py3.6-devtoolset7-cuda9.2-cudnn7-centos7-test"
use_cuda_docker_runtime: "1"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:325"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-devtoolset7-cuda9.2-cudnn7-centos7:336"
resource_class: gpu.medium
- caffe2_macos_build:
name: caffe2_py2_ios_macos10_13_build

2
.circleci/scripts/should_run_job.py
View File

@ -32,6 +32,8 @@ default_set = set([
'caffe2-py2-clang7-ubuntu16.04',
# Caffe2 CMake
'caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04',
# Caffe2 CentOS
'caffe2-py3.6-devtoolset7-cuda9.0-cudnn7-centos7',
# Binaries
'manywheel 2.7mu cpu devtoolset7',

10
.jenkins/caffe2/build.sh
View File

@ -4,16 +4,6 @@ set -ex
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
if [[ "$BUILD_ENVIRONMENT" == *centos7* ]]; then
# CentOS has gcc 4 but we need a newer compiler. Upgrade it.
sudo yum install -y centos-release-scl
sudo yum install -y devtoolset-4\*
sudo yum install -y scl-utils
source scl_source enable devtoolset-4 || yes
gcc --version
fi
# CMAKE_ARGS are only passed to 'cmake' and the -Dfoo=bar does not work with
# setup.py, so we build a list of foo=bars and then either convert it to
# -Dfoo=bars or export them before running setup.py

9
.jenkins/caffe2/test.sh
View File

@ -104,8 +104,13 @@ fi
echo "Running Python tests.."
if [[ "$BUILD_ENVIRONMENT" == *py3* ]]; then
# locale setting is required by click package with py3
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
for loc in "en_US.utf8" "C.UTF-8"; do
if locale -a | grep "$loc" >/dev/null 2>&1; then
export LC_ALL="$loc"
export LANG="$loc"
break;
fi
done
fi
pip install --user pytest-sugar

7
docker/caffe2/jenkins/centos-cuda/Dockerfile
View File

@ -8,6 +8,13 @@ ARG EC2
ADD ./install_base.sh install_base.sh
RUN bash ./install_base.sh && rm install_base.sh
# Install devtoolset
ARG DEVTOOLSET_VERSION
ADD ./install_devtoolset.sh install_devtoolset.sh
RUN bash ./install_devtoolset.sh
RUN rm install_devtoolset.sh
ENV BASH_ENV "/etc/profile"
# Compile/install ccache for faster builds
ADD ./install_ccache.sh install_ccache.sh
RUN bash ./install_ccache.sh && rm install_ccache.sh

7
docker/caffe2/jenkins/centos/Dockerfile
View File

@ -6,6 +6,13 @@ ARG EC2
ADD ./install_base.sh install_base.sh
RUN bash ./install_base.sh && rm install_base.sh
# Install devtoolset
ARG DEVTOOLSET_VERSION
ADD ./install_devtoolset.sh install_devtoolset.sh
RUN bash ./install_devtoolset.sh
RUN rm install_devtoolset.sh
ENV BASH_ENV "/etc/profile"
# Compile/install ccache for faster builds
ADD ./install_ccache.sh install_ccache.sh
RUN bash ./install_ccache.sh && rm install_ccache.sh

13
docker/caffe2/jenkins/common/install_ccache.sh
View File

@ -2,19 +2,6 @@
set -ex
# Install ccache from source.
# Needs specific branch to work with nvcc (ccache/ccache#145)
# Also pulls in a commit that disables documentation generation,
# as this requires asciidoc to be installed (which pulls in a LOT of deps).
pushd /tmp
git clone https://github.com/pietern/ccache -b ccbin
pushd ccache
./autogen.sh
./configure --prefix=/usr/local
make "-j$(nproc)" install
popd
popd
# Install sccache from pre-compiled binary.
curl https://s3.amazonaws.com/ossci-linux/sccache -o /usr/local/bin/sccache
chmod a+x /usr/local/bin/sccache