mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
CircleCI: disable output buffering to better locate test timeout (#13516)
Summary: ASAN test timeout such as https://circleci.com/gh/pytorch/pytorch/165649?utm_campaign=vcs-integration-link&utm_medium=referral&utm_source=github-build-link doesn't actually show where the timeout happened because of the bash output buffering. This PR turns off the buffering to better surface the error. Pull Request resolved: https://github.com/pytorch/pytorch/pull/13516 Differential Revision: D12952513 Pulled By: yf225 fbshipit-source-id: 48058c021470e5aa7a2246e1fcd974cfabf5df54
This commit is contained in:
Will Feng
committed by
Facebook Github Bot
Facebook Github Bot
parent
619c2f8b44
commit
bc1de6ae7d
@ -37,7 +37,8 @@ setup_ci_environment: &setup_ci_environment
|
||||
linux-headers-$(uname -r) \
|
||||
linux-image-generic \
|
||||
moreutils \
|
||||
nvidia-docker2
|
||||
nvidia-docker2 \
|
||||
expect-dev
|
||||
|
||||
sudo pkill -SIGHUP dockerd
|
||||
|
||||
@ -98,7 +99,8 @@ pytorch_linux_build_defaults: &pytorch_linux_build_defaults
|
||||
|
||||
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
|
||||
|
||||
((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
|
||||
export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
||||
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
||||
|
||||
# Push intermediate Docker image for next phase to use
|
||||
if [ -z "${BUILD_ONLY}" ]; then
|
||||
@ -122,15 +124,16 @@ pytorch_linux_test_defaults: &pytorch_linux_test_defaults
|
||||
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
|
||||
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
|
||||
if [ -n "${CUDA_VERSION}" ]; then
|
||||
id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
else
|
||||
id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
fi
|
||||
if [ -n "${MULTI_GPU}" ]; then
|
||||
((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
|
||||
export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
||||
else
|
||||
((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
|
||||
export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
||||
fi
|
||||
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
||||
|
||||
caffe2_linux_build_defaults: &caffe2_linux_build_defaults
|
||||
resource_class: large
|
||||
@ -197,7 +200,8 @@ caffe2_linux_build_defaults: &caffe2_linux_build_defaults
|
||||
export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
|
||||
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
|
||||
|
||||
((echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
|
||||
export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
||||
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
||||
|
||||
# Push intermediate Docker image for next phase to use
|
||||
if [ -z "${BUILD_ONLY}" ]; then
|
||||
@ -271,13 +275,14 @@ caffe2_linux_test_defaults: &caffe2_linux_test_defaults
|
||||
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
|
||||
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
|
||||
if [ -n "${CUDA_VERSION}" ]; then
|
||||
id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
else
|
||||
id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
fi
|
||||
docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
|
||||
|
||||
((echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
|
||||
export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
||||
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
||||
|
||||
caffe2_macos_build_defaults: &caffe2_macos_build_defaults
|
||||
macos:
|
||||
@ -296,6 +301,8 @@ caffe2_macos_build_defaults: &caffe2_macos_build_defaults
|
||||
brew install moreutils --without-parallel
|
||||
brew install cmake
|
||||
|
||||
brew install expect
|
||||
|
||||
# Reinitialize submodules
|
||||
git submodule sync && git submodule update -q --init --recursive
|
||||
|
||||
@ -342,12 +349,12 @@ caffe2_macos_build_defaults: &caffe2_macos_build_defaults
|
||||
|
||||
# Build
|
||||
if [ "${BUILD_IOS:-0}" -eq 1 ]; then
|
||||
scripts/build_ios.sh 2>&1 | ts
|
||||
unbuffer scripts/build_ios.sh 2>&1 | ts
|
||||
elif [ -n "${CAFFE2_USE_ANACONDA}" ]; then
|
||||
# All conda build logic should be in scripts/build_anaconda.sh
|
||||
scripts/build_anaconda.sh 2>&1 | ts
|
||||
unbuffer scripts/build_anaconda.sh 2>&1 | ts
|
||||
else
|
||||
scripts/build_local.sh 2>&1 | ts
|
||||
unbuffer scripts/build_local.sh 2>&1 | ts
|
||||
fi
|
||||
|
||||
# Show sccache stats if it is running
|
||||
@ -579,7 +586,7 @@ jobs:
|
||||
export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
|
||||
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
|
||||
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
|
||||
id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
|
||||
docker cp $id:/var/lib/jenkins/workspace/env /home/circleci/project/env
|
||||
# This IAM user allows write access to S3 bucket for perf test numbers
|
||||
@ -587,7 +594,8 @@ jobs:
|
||||
echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PERF_TEST_S3_BUCKET_V2}" >> /home/circleci/project/env
|
||||
docker cp /home/circleci/project/env $id:/var/lib/jenkins/workspace/env
|
||||
|
||||
((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/short-perf-test-gpu.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
|
||||
export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/short-perf-test-gpu.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
||||
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
||||
|
||||
pytorch_doc_push:
|
||||
environment:
|
||||
@ -611,7 +619,7 @@ jobs:
|
||||
export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
|
||||
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
|
||||
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
|
||||
id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
||||
|
||||
cat >/home/circleci/project/doc_push_script.sh <<EOL
|
||||
# =================== The following code will be executed inside Docker container ===================
|
||||
@ -663,7 +671,8 @@ jobs:
|
||||
chmod +x /home/circleci/project/doc_push_script.sh
|
||||
docker cp /home/circleci/project/doc_push_script.sh $id:/var/lib/jenkins/workspace/doc_push_script.sh
|
||||
|
||||
((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
|
||||
export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
||||
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
||||
|
||||
pytorch_macos_10_13_py3_build:
|
||||
macos:
|
||||
@ -694,7 +703,7 @@ jobs:
|
||||
|
||||
git submodule sync && git submodule update -q --init
|
||||
chmod a+x .jenkins/pytorch/macos-build.sh
|
||||
.jenkins/pytorch/macos-build.sh 2>&1 | ts
|
||||
unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
|
||||
|
||||
mkdir -p /Users/distiller/pytorch-ci-env/workspace
|
||||
cp -r /Users/distiller/project/. /Users/distiller/pytorch-ci-env/workspace
|
||||
@ -729,7 +738,7 @@ jobs:
|
||||
cp -r /Users/distiller/pytorch-ci-env/workspace/. /Users/distiller/project
|
||||
|
||||
chmod a+x .jenkins/pytorch/macos-test.sh
|
||||
.jenkins/pytorch/macos-test.sh 2>&1 | ts
|
||||
unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts
|
||||
|
||||
pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
|
||||
macos:
|
||||
@ -776,7 +785,7 @@ jobs:
|
||||
|
||||
git submodule sync && git submodule update -q --init
|
||||
chmod a+x .jenkins/pytorch/macos-build.sh
|
||||
.jenkins/pytorch/macos-build.sh 2>&1 | ts
|
||||
unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
|
||||
|
||||
caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_build:
|
||||
environment:
|
||||
|
||||
Reference in New Issue
Block a user