From 8c3a7bb57f0a69c45208c46b02a9aa7d108c8053 Mon Sep 17 00:00:00 2001 From: Junjie Bai Date: Thu, 9 May 2019 13:42:44 -0700 Subject: [PATCH] Move librosa and psutil installation from CI script to docker images build script (#20299) Summary: pip install librosa randomly coredump, causes CI flakiness Pull Request resolved: https://github.com/pytorch/pytorch/pull/20299 Differential Revision: D15276270 Pulled By: bddppq fbshipit-source-id: 9105106f41aaacf620751290b016359ef7d665b3 --- .../cimodel/data/caffe2_build_definitions.py | 2 +- .circleci/config.yml | 42 +++++++++---------- .jenkins/pytorch/test.sh | 3 -- .../caffe2/jenkins/common/install_python.sh | 4 +- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.circleci/cimodel/data/caffe2_build_definitions.py b/.circleci/cimodel/data/caffe2_build_definitions.py index aa67a49894a9..44ef7f4ac3d3 100644 --- a/.circleci/cimodel/data/caffe2_build_definitions.py +++ b/.circleci/cimodel/data/caffe2_build_definitions.py @@ -10,7 +10,7 @@ from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/" -DOCKER_IMAGE_VERSION = 273 +DOCKER_IMAGE_VERSION = 276 class Conf(object): diff --git a/.circleci/config.yml b/.circleci/config.yml index 69fe24f832fa..582067cbbde6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1198,144 +1198,144 @@ jobs: caffe2_py2_gcc4_8_ubuntu14_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-gcc4.8-ubuntu14.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.8-ubuntu14.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.8-ubuntu14.04:276" <<: *caffe2_linux_build_defaults caffe2_py2_gcc4_8_ubuntu14_04_test: environment: BUILD_ENVIRONMENT: "caffe2-py2-gcc4.8-ubuntu14.04-test" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.8-ubuntu14.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.8-ubuntu14.04:276" resource_class: large <<: *caffe2_linux_test_defaults caffe2_py2_gcc4_9_ubuntu14_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-gcc4.9-ubuntu14.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.9-ubuntu14.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc4.9-ubuntu14.04:276" BUILD_ONLY: "1" <<: *caffe2_linux_build_defaults caffe2_py2_cuda8_0_cudnn7_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda8.0-cudnn7-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn7-ubuntu16.04:276" <<: *caffe2_linux_build_defaults caffe2_py2_cuda8_0_cudnn7_ubuntu16_04_test: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda8.0-cudnn7-ubuntu16.04-test" USE_CUDA_DOCKER_RUNTIME: "1" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda8.0-cudnn7-ubuntu16.04:276" resource_class: gpu.medium <<: *caffe2_linux_test_defaults caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda9.0-cudnn7-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:276" <<: *caffe2_linux_build_defaults caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_test: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda9.0-cudnn7-ubuntu16.04-test" USE_CUDA_DOCKER_RUNTIME: "1" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:276" resource_class: gpu.medium <<: *caffe2_linux_test_defaults caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:276" <<: *caffe2_linux_build_defaults caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_test: environment: BUILD_ENVIRONMENT: "caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04-test" USE_CUDA_DOCKER_RUNTIME: "1" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:276" resource_class: gpu.medium <<: *caffe2_linux_test_defaults caffe2_py2_cuda9_1_cudnn7_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda9.1-cudnn7-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.1-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.1-cudnn7-ubuntu16.04:276" <<: *caffe2_linux_build_defaults caffe2_py2_cuda9_1_cudnn7_ubuntu16_04_test: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda9.1-cudnn7-ubuntu16.04-test" USE_CUDA_DOCKER_RUNTIME: "1" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.1-cudnn7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.1-cudnn7-ubuntu16.04:276" resource_class: gpu.medium <<: *caffe2_linux_test_defaults caffe2_py2_mkl_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-mkl-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:276" <<: *caffe2_linux_build_defaults caffe2_py2_mkl_ubuntu16_04_test: environment: BUILD_ENVIRONMENT: "caffe2-py2-mkl-ubuntu16.04-test" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:276" resource_class: large <<: *caffe2_linux_test_defaults caffe2_onnx_py2_gcc5_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-onnx-py2-gcc5-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:276" <<: *caffe2_linux_build_defaults caffe2_onnx_py2_gcc5_ubuntu16_04_test: environment: BUILD_ENVIRONMENT: "caffe2-onnx-py2-gcc5-ubuntu16.04-test" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:276" resource_class: large <<: *caffe2_linux_test_defaults caffe2_py2_clang3_8_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-clang3.8-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.8-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.8-ubuntu16.04:276" BUILD_ONLY: "1" <<: *caffe2_linux_build_defaults caffe2_py2_clang3_9_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-clang3.9-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.9-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.9-ubuntu16.04:276" BUILD_ONLY: "1" <<: *caffe2_linux_build_defaults caffe2_py2_clang7_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-clang7-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang7-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang7-ubuntu16.04:276" BUILD_ONLY: "1" <<: *caffe2_linux_build_defaults caffe2_py2_android_ubuntu16_04_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-android-ubuntu16.04-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-android-ubuntu16.04:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-android-ubuntu16.04:276" BUILD_ONLY: "1" <<: *caffe2_linux_build_defaults caffe2_py2_cuda9_0_cudnn7_centos7_build: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda9.0-cudnn7-centos7-build" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:276" <<: *caffe2_linux_build_defaults caffe2_py2_cuda9_0_cudnn7_centos7_test: environment: BUILD_ENVIRONMENT: "caffe2-py2-cuda9.0-cudnn7-centos7-test" USE_CUDA_DOCKER_RUNTIME: "1" - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:273" + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:276" resource_class: gpu.medium <<: *caffe2_linux_test_defaults diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh index 03c96cb16e54..431b805aef0d 100755 --- a/.jenkins/pytorch/test.sh +++ b/.jenkins/pytorch/test.sh @@ -94,9 +94,6 @@ fi if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then export PYTORCH_TEST_WITH_ROCM=1 - # ROCm CI is using Caffe2 docker images, which doesn't have several packages - # needed in testing. We install them here. - pip install -q psutil "librosa>=0.6.2" --user fi if [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX-* ]]; then diff --git a/docker/caffe2/jenkins/common/install_python.sh b/docker/caffe2/jenkins/common/install_python.sh index da79ab80bcf4..b22702a3a278 100755 --- a/docker/caffe2/jenkins/common/install_python.sh +++ b/docker/caffe2/jenkins/common/install_python.sh @@ -165,5 +165,7 @@ pip install --no-cache-dir \ mock \ typing \ typing-extensions \ - pyyaml + pyyaml \ + librosa>=0.6.2 \ + psutil