frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:07:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Do not use builder repo for MacOS builds (#142277)

Browse Source 
Added c7564f31f7/wheel/build_wheel.sh to `.ci/wheel/` folder

Commented out call to 39532891a0/run_tests.sh, because since 2018 this script just checked that tests folder is there and exited, as there are no way to run all pytorch tests in single shard, see this logic:
```bash
#!/bin/bash
set -eux -o pipefail

# Essentially runs pytorch/test/run_test.py, but keeps track of which tests to
# skip in a centralized place.
#
# TODO Except for a few tests, this entire file is a giant TODO. Why are these
# tests # failing?
# TODO deal with Windows

# This script expects to be in the pytorch root folder
if [[ ! -d 'test' || ! -f 'test/run_test.py' ]]; then
    echo "builder/test.sh expects to be run from the Pytorch root directory " \
         "but I'm actually in $(pwd)"
    exit 2
fi

# Allow master skip of all tests
if [[ -n "${SKIP_ALL_TESTS:-}" ]]; then
    exit 0
fi
```

https://github.com/pytorch/pytorch/pull/123390 is a misread attempt to interpret above-mentioned logic, as run_tests will be skipped if `${SKIP_ALL_TESTS}` is a non-empty string
Pull Request resolved: https://github.com/pytorch/pytorch/pull/142277
Approved by: https://github.com/huydhn, https://github.com/atalman
ghstack dependencies: #142276
This commit is contained in:
Nikita Shulga
2024-12-09 09:15:17 -08:00
committed by PyTorch MergeBot
parent 5e8e1d725a
commit 0610b9730e
5 changed files with 280 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
.ci/wheel/build_wheel.sh Executable file
View File

@ -0,0 +1,279 @@
#!/usr/bin/env bash
set -ex
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
# Env variables that should be set:
# DESIRED_PYTHON
# Which Python version to build for in format 'Maj.min' e.g. '2.7' or '3.6'
#
# PYTORCH_FINAL_PACKAGE_DIR
# **absolute** path to folder where final whl packages will be stored. The
# default should not be used when calling this from a script. The default
# is 'whl', and corresponds to the default in the wheel/upload.sh script.
#
# MAC_PACKAGE_WORK_DIR
# absolute path to a workdir in which to clone an isolated conda
# installation and pytorch checkout. If the pytorch checkout already exists
# then it will not be overwritten.
# Function to retry functions that sometimes timeout or have flaky failures
retry () {
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
}
# Parameters
if [[ -n "$DESIRED_PYTHON" && -n "$PYTORCH_BUILD_VERSION" && -n "$PYTORCH_BUILD_NUMBER" ]]; then
desired_python="$DESIRED_PYTHON"
build_version="$PYTORCH_BUILD_VERSION"
build_number="$PYTORCH_BUILD_NUMBER"
else
if [ "$#" -ne 3 ]; then
echo "illegal number of parameters. Need PY_VERSION BUILD_VERSION BUILD_NUMBER"
echo "for example: build_wheel.sh 2.7 0.1.6 20"
echo "Python version should be in format 'M.m'"
exit 1
fi
desired_python=$1
build_version=$2
build_number=$3
fi
echo "Building for Python: $desired_python Version: $build_version Build: $build_number"
python_nodot="$(echo $desired_python | tr -d m.u)"
# Version: setup.py uses $PYTORCH_BUILD_VERSION.post$PYTORCH_BUILD_NUMBER if
# PYTORCH_BUILD_NUMBER > 1
if [[ -n "$OVERRIDE_PACKAGE_VERSION" ]]; then
# This will be the *exact* version, since build_number<1
build_version="$OVERRIDE_PACKAGE_VERSION"
build_number=0
build_number_prefix=''
else
if [[ $build_number -eq 1 ]]; then
build_number_prefix=""
else
build_number_prefix=".post$build_number"
fi
fi
export PYTORCH_BUILD_VERSION=$build_version
export PYTORCH_BUILD_NUMBER=$build_number
package_type="${PACKAGE_TYPE:-wheel}"
# Fill in empty parameters with defaults
if [[ -z "$TORCH_PACKAGE_NAME" ]]; then
TORCH_PACKAGE_NAME='torch'
fi
TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '-' '_')"
if [[ -z "$PYTORCH_REPO" ]]; then
PYTORCH_REPO='pytorch'
fi
if [[ -z "$PYTORCH_BRANCH" ]]; then
PYTORCH_BRANCH="v${build_version}"
fi
if [[ -z "$RUN_TEST_PARAMS" ]]; then
RUN_TEST_PARAMS=()
fi
if [[ -z "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then
if [[ -n "$BUILD_PYTHONLESS" ]]; then
PYTORCH_FINAL_PACKAGE_DIR='libtorch'
else
PYTORCH_FINAL_PACKAGE_DIR='whl'
fi
fi
mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true
# Create an isolated directory to store this builds pytorch checkout and conda
# installation
if [[ -z "$MAC_PACKAGE_WORK_DIR" ]]; then
MAC_PACKAGE_WORK_DIR="$(pwd)/tmp_wheel_conda_${DESIRED_PYTHON}_$(date +%H%M%S)"
fi
mkdir -p "$MAC_PACKAGE_WORK_DIR" || true
if [[ -n ${GITHUB_ACTIONS} ]]; then
pytorch_rootdir="${PYTORCH_ROOT:-${MAC_PACKAGE_WORK_DIR}/pytorch}"
else
pytorch_rootdir="${MAC_PACKAGE_WORK_DIR}/pytorch"
fi
whl_tmp_dir="${MAC_PACKAGE_WORK_DIR}/dist"
mkdir -p "$whl_tmp_dir"
mac_version='macosx_11_0_arm64'
libtorch_arch='arm64'
# Create a consistent wheel package name to rename the wheel to
wheel_filename_new="${TORCH_PACKAGE_NAME}-${build_version}${build_number_prefix}-cp${python_nodot}-none-${mac_version}.whl"
###########################################################
# Have a separate Pytorch repo clone
if [[ ! -d "$pytorch_rootdir" ]]; then
git clone "https://github.com/${PYTORCH_REPO}/pytorch" "$pytorch_rootdir"
pushd "$pytorch_rootdir"
if ! git checkout "$PYTORCH_BRANCH" ; then
echo "Could not checkout $PYTORCH_BRANCH, so trying tags/v${build_version}"
git checkout tags/v${build_version}
fi
popd
fi
pushd "$pytorch_rootdir"
git submodule update --init --recursive
popd
##########################
# now build the binary
export TH_BINARY_BUILD=1
export INSTALL_TEST=0 # dont install test binaries into site-packages
export MACOSX_DEPLOYMENT_TARGET=10.15
export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
SETUPTOOLS_PINNED_VERSION="=46.0.0"
PYYAML_PINNED_VERSION="=5.3"
EXTRA_CONDA_INSTALL_FLAGS=""
case $desired_python in
3.13)
echo "Using 3.13 deps"
SETUPTOOLS_PINNED_VERSION=">=68.0.0"
PYYAML_PINNED_VERSION=">=6.0.1"
NUMPY_PINNED_VERSION="=2.1.0"
;;
3.12)
echo "Using 3.12 deps"
SETUPTOOLS_PINNED_VERSION=">=68.0.0"
PYYAML_PINNED_VERSION=">=6.0.1"
NUMPY_PINNED_VERSION="=2.0.2"
;;
3.11)
echo "Using 3.11 deps"
SETUPTOOLS_PINNED_VERSION=">=46.0.0"
PYYAML_PINNED_VERSION=">=5.3"
NUMPY_PINNED_VERSION="=2.0.2"
;;
3.10)
echo "Using 3.10 deps"
SETUPTOOLS_PINNED_VERSION=">=46.0.0"
PYYAML_PINNED_VERSION=">=5.3"
NUMPY_PINNED_VERSION="=2.0.2"
;;
3.9)
echo "Using 3.9 deps"
SETUPTOOLS_PINNED_VERSION=">=46.0.0"
PYYAML_PINNED_VERSION=">=5.3"
NUMPY_PINNED_VERSION="=2.0.2"
;;
*)
echo "Using default deps"
NUMPY_PINNED_VERSION="=1.11.3"
;;
esac
# Install into a fresh env
tmp_env_name="wheel_py$python_nodot"
conda create ${EXTRA_CONDA_INSTALL_FLAGS} -yn "$tmp_env_name" python="$desired_python"
source activate "$tmp_env_name"
pip install -q "numpy=${NUMPY_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" requests
retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq llvm-openmp=14.0.6 cmake ninja "setuptools${SETUPTOOLS_PINNED_VERSION}" typing_extensions
retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true
# For USE_DISTRIBUTED=1 on macOS, need libuv and pkg-config to find libuv.
export USE_DISTRIBUTED=1
retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq libuv pkg-config
if [[ -n "$CROSS_COMPILE_ARM64" ]]; then
export CMAKE_OSX_ARCHITECTURES=arm64
fi
export USE_MKLDNN=OFF
export USE_QNNPACK=OFF
export BUILD_TEST=OFF
pushd "$pytorch_rootdir"
echo "Calling setup.py bdist_wheel at $(date)"
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
echo "Calling setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)"
BUILD_LIBTORCH_WHL=1 BUILD_PYTHON_ONLY=0 python setup.py bdist_wheel -d "$whl_tmp_dir"
echo "Finished setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)"
echo "Calling setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)"
BUILD_PYTHON_ONLY=1 BUILD_LIBTORCH_WHL=0 python setup.py bdist_wheel -d "$whl_tmp_dir" --cmake
echo "Finished setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)"
else
python setup.py bdist_wheel -d "$whl_tmp_dir"
fi
echo "Finished setup.py bdist_wheel at $(date)"
if [[ $package_type != 'libtorch' ]]; then
echo "delocating wheel dependencies"
retry pip install https://github.com/matthew-brett/delocate/archive/refs/tags/0.10.4.zip
echo "found the following wheels:"
find $whl_tmp_dir -name "*.whl"
echo "running delocate"
find $whl_tmp_dir -name "*.whl" | xargs -I {} delocate-wheel -v {}
find $whl_tmp_dir -name "*.whl"
find $whl_tmp_dir -name "*.whl" | xargs -I {} delocate-listdeps {}
echo "Finished delocating wheels at $(date)"
fi
echo "The wheel is in $(find $whl_tmp_dir -name '*.whl')"
wheel_filename_gen=$(find $whl_tmp_dir -name '*.whl' | head -n1 | xargs -I {} basename {})
popd
if [[ -z "$BUILD_PYTHONLESS" ]]; then
# Copy the whl to a final destination before tests are run
echo "Renaming Wheel file: $wheel_filename_gen to $wheel_filename_new"
cp "$whl_tmp_dir/$wheel_filename_gen" "$PYTORCH_FINAL_PACKAGE_DIR/$wheel_filename_new"
##########################
# now test the binary, unless it's cross compiled arm64
if [[ -z "$CROSS_COMPILE_ARM64" ]]; then
pip uninstall -y "$TORCH_PACKAGE_NAME" || true
pip uninstall -y "$TORCH_PACKAGE_NAME" || true
# Create new "clean" conda environment for testing
conda create ${EXTRA_CONDA_INSTALL_FLAGS} -yn "test_conda_env" python="$desired_python"
conda activate test_conda_env
pip install "$PYTORCH_FINAL_PACKAGE_DIR/$wheel_filename_new" -v
echo "$(date) :: Running tests"
# TODO: Add real tests, as run_test.sh from builder is a glorified no-op
# pushd "$pytorch_rootdir"
# "${SOURCE_DIR}/../run_tests.sh" 'wheel' "$desired_python" 'cpu'
# popd
echo "$(date) :: Finished tests"
fi
else
pushd "$pytorch_rootdir"
mkdir -p libtorch/{lib,bin,include,share}
cp -r "$(pwd)/build/lib" "$(pwd)/libtorch/"
# for now, the headers for the libtorch package will just be
# copied in from the wheel
unzip -d any_wheel "$whl_tmp_dir/$wheel_filename_gen"
if [[ -d $(pwd)/any_wheel/torch/include ]]; then
cp -r "$(pwd)/any_wheel/torch/include" "$(pwd)/libtorch/"
else
cp -r "$(pwd)/any_wheel/torch/lib/include" "$(pwd)/libtorch/"
fi
cp -r "$(pwd)/any_wheel/torch/share/cmake" "$(pwd)/libtorch/share/"
if [[ "${libtorch_arch}" == "x86_64" ]]; then
if [[ -x "$(pwd)/any_wheel/torch/.dylibs/libiomp5.dylib" ]]; then
cp -r "$(pwd)/any_wheel/torch/.dylibs/libiomp5.dylib" "$(pwd)/libtorch/lib/"
else
cp -r "$(pwd)/any_wheel/torch/lib/libiomp5.dylib" "$(pwd)/libtorch/lib/"
fi
else
cp -r "$(pwd)/any_wheel/torch/lib/libomp.dylib" "$(pwd)/libtorch/lib/"
fi
rm -rf "$(pwd)/any_wheel"
echo $PYTORCH_BUILD_VERSION > libtorch/build-version
echo "$(pushd $pytorch_rootdir && git rev-parse HEAD)" > libtorch/build-hash
zip -rq "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-$PYTORCH_BUILD_VERSION.zip" libtorch
cp "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-$PYTORCH_BUILD_VERSION.zip" \
"$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-latest.zip"
fi