mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 12:54:11 +08:00
[CD] Build aarch64 wheels without conda (#140093)
As manylinuxaarch64-builder already comes pre-built with all versions of python runtime Refactor logic for setting path to DESIRED_PYTHON from `manywheel/build_common` into `set_desired_python.sh` and call it from aarch64_ci_setup.sh In followup PRs move scons and ninja installation into base docker image Pull Request resolved: https://github.com/pytorch/pytorch/pull/140093 Approved by: https://github.com/atalman
This commit is contained in:
Nikita Shulga
committed by
PyTorch MergeBot
PyTorch MergeBot
parent
95198f8299
commit
f3cbf67686
@ -2,36 +2,22 @@
|
||||
set -eux -o pipefail
|
||||
|
||||
# This script is used to prepare the Docker container for aarch64_ci_wheel_build.py python script
|
||||
# as we need to install conda and setup the python version for the build.
|
||||
|
||||
CONDA_PYTHON_EXE=/opt/conda/bin/python
|
||||
CONDA_EXE=/opt/conda/bin/conda
|
||||
CONDA_ENV_NAME=aarch64_env
|
||||
PATH=/opt/conda/bin:$PATH
|
||||
LD_LIBRARY_PATH=/opt/conda/envs/${CONDA_ENV_NAME}/lib/:/opt/conda/lib:$LD_LIBRARY_PATH
|
||||
|
||||
###############################################################################
|
||||
# Install conda
|
||||
# disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path"
|
||||
# when using Python version, less than the conda latest
|
||||
###############################################################################
|
||||
echo 'Installing conda-forge'
|
||||
curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh
|
||||
chmod +x /mambaforge.sh
|
||||
/mambaforge.sh -b -p /opt/conda
|
||||
rm /mambaforge.sh
|
||||
source /opt/conda/etc/profile.d/conda.sh
|
||||
conda config --set ssl_verify False
|
||||
conda create -y -c conda-forge -n "${CONDA_ENV_NAME}" python=${DESIRED_PYTHON}
|
||||
conda activate "${CONDA_ENV_NAME}"
|
||||
# By creating symlinks from desired /opt/python to /usr/local/bin/
|
||||
|
||||
NUMPY_VERSION=2.0.2
|
||||
PYGIT2_VERSION=1.15.1
|
||||
if [[ "$DESIRED_PYTHON" == "3.13" ]]; then
|
||||
pip install -q --pre numpy==2.1.2
|
||||
conda install -y -c conda-forge pyyaml==6.0.2 patchelf==0.17.2 pygit2==1.15.1 ninja==1.11.1 scons==4.7.0
|
||||
else
|
||||
pip install -q --pre numpy==2.0.2
|
||||
conda install -y -c conda-forge pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 ninja==1.11.1 scons==4.5.2
|
||||
NUMPY_VERSION=2.1.2
|
||||
PYGIT2_VERSION=1.16.0
|
||||
fi
|
||||
|
||||
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
|
||||
source $SCRIPTPATH/../manywheel/set_desired_python.sh
|
||||
|
||||
pip install -q numpy==${NUMPY_VERSION} pyyaml==6.0.2 scons==4.7.0 ninja==1.11.1 patchelf==0.17.2 pygit2==${PYGIT2_VERSION}
|
||||
|
||||
for tool in python python3 pip pip3 ninja scons patchelf; do
|
||||
ln -sf ${DESIRED_PYTHON_BIN_DIR}/${tool} /usr/local/bin;
|
||||
done
|
||||
|
||||
python --version
|
||||
conda --version
|
||||
|
||||
@ -88,7 +88,7 @@ def update_wheel(wheel_path) -> None:
|
||||
"/usr/local/cuda/lib64/libcudnn_engines_runtime_compiled.so.9",
|
||||
"/usr/local/cuda/lib64/libcudnn_engines_precompiled.so.9",
|
||||
"/usr/local/cuda/lib64/libcudnn_heuristic.so.9",
|
||||
"/opt/conda/envs/aarch64_env/lib/libgomp.so.1",
|
||||
"/lib64/libgomp.so.1",
|
||||
"/usr/lib64/libgfortran.so.5",
|
||||
"/acl/build/libarm_compute.so",
|
||||
"/acl/build/libarm_compute_graph.so",
|
||||
|
||||
@ -4,12 +4,9 @@
|
||||
set -ex
|
||||
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
||||
|
||||
source ${SOURCE_DIR}/set_desired_python.sh
|
||||
|
||||
|
||||
# Require only one python installation
|
||||
if [[ -z "$DESIRED_PYTHON" ]]; then
|
||||
echo "Need to set DESIRED_PYTHON env variable"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$BUILD_PYTHONLESS" && -z "$LIBTORCH_VARIANT" ]]; then
|
||||
echo "BUILD_PYTHONLESS is set, so need LIBTORCH_VARIANT to also be set"
|
||||
echo "LIBTORCH_VARIANT should be one of shared-with-deps shared-without-deps static-with-deps static-without-deps"
|
||||
@ -80,27 +77,7 @@ if [[ -e /opt/openssl ]]; then
|
||||
export CMAKE_INCLUDE_PATH="/opt/openssl/include":$CMAKE_INCLUDE_PATH
|
||||
fi
|
||||
|
||||
# If given a python version like 3.6m or 2.7mu, convert this to the format we
|
||||
# expect. The binary CI jobs pass in python versions like this; they also only
|
||||
# ever pass one python version, so we assume that DESIRED_PYTHON is not a list
|
||||
# in this case
|
||||
if [[ -n "$DESIRED_PYTHON" && $DESIRED_PYTHON =~ ([0-9].[0-9]+)t ]]; then
|
||||
python_digits="$(echo $DESIRED_PYTHON | tr -cd [:digit:])"
|
||||
py_majmin="${DESIRED_PYTHON}"
|
||||
DESIRED_PYTHON="cp${python_digits}-cp${python_digits}t"
|
||||
elif [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then
|
||||
python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)"
|
||||
DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}"
|
||||
if [[ ${python_nodot} -ge 310 ]]; then
|
||||
py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:2}"
|
||||
else
|
||||
py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:1}"
|
||||
fi
|
||||
fi
|
||||
|
||||
pydir="/opt/python/$DESIRED_PYTHON"
|
||||
export PATH="$pydir/bin:$PATH"
|
||||
echo "Will build for Python version: ${DESIRED_PYTHON} with ${python_installation}"
|
||||
|
||||
mkdir -p /tmp/$WHEELHOUSE_DIR
|
||||
|
||||
|
||||
30
.ci/manywheel/set_desired_python.sh
Normal file
30
.ci/manywheel/set_desired_python.sh
Normal file
@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Require only one python installation
|
||||
if [[ -z "$DESIRED_PYTHON" ]]; then
|
||||
echo "Need to set DESIRED_PYTHON env variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If given a python version like 3.6m or 2.7mu, convert this to the format we
|
||||
# expect. The binary CI jobs pass in python versions like this; they also only
|
||||
# ever pass one python version, so we assume that DESIRED_PYTHON is not a list
|
||||
# in this case
|
||||
if [[ -n "$DESIRED_PYTHON" && $DESIRED_PYTHON =~ ([0-9].[0-9]+)t ]]; then
|
||||
python_digits="$(echo $DESIRED_PYTHON | tr -cd [:digit:])"
|
||||
py_majmin="${DESIRED_PYTHON}"
|
||||
DESIRED_PYTHON="cp${python_digits}-cp${python_digits}t"
|
||||
elif [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then
|
||||
python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)"
|
||||
DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}"
|
||||
if [[ ${python_nodot} -ge 310 ]]; then
|
||||
py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:2}"
|
||||
else
|
||||
py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:1}"
|
||||
fi
|
||||
fi
|
||||
|
||||
pydir="/opt/python/$DESIRED_PYTHON"
|
||||
export DESIRED_PYTHON_BIN_DIR="${pydir}/bin"
|
||||
export PATH="$DESIRED_PYTHON_BIN_DIR:$PATH"
|
||||
echo "Will build for Python version: ${DESIRED_PYTHON}"
|
||||
Reference in New Issue
Block a user