mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
Move all ONNX test dependencies to Docker (#96590)
Per title. This is the first one of a two-part process: [x] Move all ONNX test dependencies to Docker https://github.com/pytorch/pytorch/pull/96590 [ ] Move the test model used by [TestFxToOnnxWithOnnxRuntime.test_gpt2_tiny](https://hud.pytorch.org/failure/FAILED%20test%2Fonnx%2Ftest_fx_to_onnx_with_onnxruntime.py%3A%3ATestFxToOnnxWithOnnxRuntime%3A%3Atest_large_scale_exporter_with_tiny_gpt2%20-%20requests.exceptions.ReadTimeout%3A%20HTTPSConnectionPool(host%3D'huggingface.co'%2C%20port%3D443)%3A%20Read%20timed%20out.%20(read%20timeout%3D10.0)) Pull Request resolved: https://github.com/pytorch/pytorch/pull/96590 Approved by: https://github.com/ZainRizvi
This commit is contained in:
Huy Do
PyTorch MergeBot
@ -146,6 +146,7 @@ case "$image" in
|
||||
DB=yes
|
||||
VISION=yes
|
||||
CONDA_CMAKE=yes
|
||||
ONNX=yes
|
||||
;;
|
||||
pytorch-linux-focal-py3-clang7-android-ndk-r19c)
|
||||
ANACONDA_PYTHON_VERSION=3.7
|
||||
@ -340,6 +341,7 @@ docker build \
|
||||
--build-arg "UCC_COMMIT=${UCC_COMMIT}" \
|
||||
--build-arg "CONDA_CMAKE=${CONDA_CMAKE}" \
|
||||
--build-arg "TRITON=${TRITON}" \
|
||||
--build-arg "ONNX=${ONNX}" \
|
||||
-f $(dirname ${DOCKERFILE})/Dockerfile \
|
||||
-t "$tmp_tag" \
|
||||
"$@" \
|
||||
|
||||
25
.ci/docker/common/install_onnx.sh
Normal file
25
.ci/docker/common/install_onnx.sh
Normal file
@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
|
||||
|
||||
# A bunch of custom pip dependencies for ONNX
|
||||
pip_install \
|
||||
beartype==0.10.4 \
|
||||
filelock==3.9.0 \
|
||||
flatbuffers==2.0 \
|
||||
mock==5.0.1 \
|
||||
ninja==1.10.2 \
|
||||
networkx==2.0 \
|
||||
numpy==1.22.4 \
|
||||
onnx==1.13.1 \
|
||||
onnxruntime==1.14.0 \
|
||||
parameterized==0.8.1 \
|
||||
pytest-cov==4.0.0 \
|
||||
pytest-subtests==0.10.0 \
|
||||
tabulate==0.9.0 \
|
||||
transformers==4.25.1
|
||||
|
||||
# TODO: change this when onnx-script is on testPypi
|
||||
pip_install "onnx-script@git+https://github.com/microsoft/onnx-script@29241e15f5182be1384f1cf6ba203d7e2e125196"
|
||||
@ -62,7 +62,7 @@ librosa>=0.6.2 ; python_version < "3.11"
|
||||
#mkl-devel
|
||||
# see mkl
|
||||
|
||||
#mock # breaks ci/circleci: docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
|
||||
#mock
|
||||
#Description: A testing library that allows you to replace parts of your
|
||||
#system under test with mock objects
|
||||
#Pinned versions:
|
||||
|
||||
@ -143,6 +143,12 @@ COPY ci_commit_pins/triton.txt triton.txt
|
||||
RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi
|
||||
RUN rm install_triton.sh common_utils.sh triton.txt
|
||||
|
||||
ARG ONNX
|
||||
# Install ONNX dependencies
|
||||
COPY ./common/install_onnx.sh ./common/common_utils.sh ./
|
||||
RUN if [ -n "${ONNX}" ]; then bash ./install_onnx.sh; fi
|
||||
RUN rm install_onnx.sh common_utils.sh
|
||||
|
||||
# Install ccache/sccache (do this last, so we get priority in PATH)
|
||||
COPY ./common/install_cache.sh install_cache.sh
|
||||
ENV PATH /opt/cache/bin:$PATH
|
||||
|
||||
@ -3,70 +3,11 @@
|
||||
# shellcheck source=./common.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
|
||||
|
||||
if [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then
|
||||
pip install click mock tabulate networkx==2.0
|
||||
pip -q install --user "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
|
||||
fi
|
||||
|
||||
# Skip tests in environments where they are not built/applicable
|
||||
if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
|
||||
echo 'Skipping tests'
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${BUILD_ENVIRONMENT}" == *-rocm* ]]; then
|
||||
# temporary to locate some kernel issues on the CI nodes
|
||||
export HSAKMT_DEBUG_LEVEL=4
|
||||
fi
|
||||
# These additional packages are needed for circleci ROCm builds.
|
||||
if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
|
||||
# Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
|
||||
# defaults installs the most recent networkx version, so we install this lower
|
||||
# version explicitly before scikit-image pulls it in as a dependency
|
||||
pip install networkx==2.0
|
||||
# click - onnx
|
||||
pip install --progress-bar off click protobuf tabulate virtualenv mock typing-extensions
|
||||
fi
|
||||
|
||||
################################################################################
|
||||
# Python tests #
|
||||
################################################################################
|
||||
if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# If pip is installed as root, we must use sudo.
|
||||
# CircleCI docker images could install conda as jenkins user, or use the OS's python package.
|
||||
PIP=$(which pip)
|
||||
PIP_USER=$(stat --format '%U' $PIP)
|
||||
CURRENT_USER=$(id -u -n)
|
||||
if [[ "$PIP_USER" = root && "$CURRENT_USER" != root ]]; then
|
||||
MAYBE_SUDO=sudo
|
||||
fi
|
||||
|
||||
# Uninstall pre-installed hypothesis and coverage to use an older version as newer
|
||||
# versions remove the timeout parameter from settings which ideep/conv_transpose_test.py uses
|
||||
$MAYBE_SUDO pip -q uninstall -y hypothesis
|
||||
$MAYBE_SUDO pip -q uninstall -y coverage
|
||||
|
||||
# "pip install hypothesis==3.44.6" from official server is unreliable on
|
||||
# CircleCI, so we host a copy on S3 instead
|
||||
$MAYBE_SUDO pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
|
||||
$MAYBE_SUDO pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
|
||||
$MAYBE_SUDO pip -q install hypothesis==4.57.1
|
||||
|
||||
##############
|
||||
# ONNX tests #
|
||||
##############
|
||||
if [[ "$BUILD_ENVIRONMENT" == *onnx* ]]; then
|
||||
pip -q install --user "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
|
||||
# TODO: This can be removed later once vision is also part of the Docker image
|
||||
pip install -q --user --no-use-pep517 "git+https://github.com/pytorch/vision.git@$(cat .github/ci_commit_pins/vision.txt)"
|
||||
pip install -q --user transformers==4.25.1
|
||||
pip install -q --user ninja flatbuffers==2.0 numpy==1.22.4 onnxruntime==1.14.0 beartype==0.10.4
|
||||
pip install -q --user onnx==1.13.1
|
||||
# TODO: change this when onnx-script is on testPypi
|
||||
pip install 'onnx-script @ git+https://github.com/microsoft/onnx-script@29241e15f5182be1384f1cf6ba203d7e2e125196'
|
||||
# numba requires numpy <= 1.20, onnxruntime requires numpy >= 1.21.
|
||||
# We don't actually need it for our tests, but it's imported if it's present, so uninstall.
|
||||
pip uninstall -q --yes numba
|
||||
|
||||
# JIT C++ extensions require ninja, so put it into PATH.
|
||||
export PATH="/var/lib/jenkins/.local/bin:$PATH"
|
||||
"$ROOT_DIR/scripts/onnx/test.sh"
|
||||
|
||||
@ -24,15 +24,6 @@ do
|
||||
done
|
||||
set -- "${UNKNOWN[@]}" # leave UNKNOWN
|
||||
|
||||
if [[ $PARALLEL == 1 ]]; then
|
||||
pip install pytest-xdist
|
||||
fi
|
||||
|
||||
# pytest, scipy, hypothesis: these may not be necessary
|
||||
# pytest-cov: installing since `coverage run -m pytest ..` doesn't work
|
||||
# pytest-subtests: unittest subtests support for pytest
|
||||
# parameterized: parameterizing test class
|
||||
pip install pytest scipy hypothesis pytest-cov pytest-subtests parameterized
|
||||
# allows coverage to run w/o failing due to a missing plug-in
|
||||
pip install -e tools/coverage_plugins_package
|
||||
|
||||
|
||||
Reference in New Issue
Block a user