frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[pytorch][mobile] change mobile build scripts to build PyTorch by default (#34203)

Browse Source 
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/34203

Currently cmake and mobile build scripts still build libcaffe2 by
default. To build pytorch mobile users have to set environment variable
BUILD_PYTORCH_MOBILE=1 or set cmake option BUILD_CAFFE2_MOBILE=OFF.

PyTorch mobile has been released for a while. It's about time to change
CMake and build scripts to build libtorch by default.

Changed caffe2 CI job to build libcaffe2 by setting BUILD_CAFFE2_MOBILE=1
environment variable. Only found android CI for libcaffe2 - do we ever
have iOS CI for libcaffe2?

Test Plan: Imported from OSS

Differential Revision: D20267274

Pulled By: ljk53

fbshipit-source-id: 9d997032a599c874d62fbcfc4f5d4fbf8323a12e
This commit is contained in:
Jiakai Liu
2020-03-05 23:38:10 -08:00
committed by Facebook Github Bot
parent b50825e011
commit 9a5e9d8cec
13 changed files with 26 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.circleci/config.yml
View File

@ -1522,7 +1522,6 @@ jobs:
if [ -n "${SELECTED_OP_LIST}" ]; then if [ -n "${SELECTED_OP_LIST}" ]; then
export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}" export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}"
fi fi
export BUILD_PYTORCH_MOBILE=1
export IOS_ARCH=${IOS_ARCH} export IOS_ARCH=${IOS_ARCH}
export IOS_PLATFORM=${IOS_PLATFORM} export IOS_PLATFORM=${IOS_PLATFORM}
unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts

3
.circleci/scripts/binary_ios_build.sh
View File

@ -5,7 +5,7 @@ echo ""
echo "DIR: $(pwd)" echo "DIR: $(pwd)"
WORKSPACE=/Users/distiller/workspace WORKSPACE=/Users/distiller/workspace
PROJ_ROOT=/Users/distiller/project PROJ_ROOT=/Users/distiller/project
export TCLLIBPATH="/usr/local/lib" export TCLLIBPATH="/usr/local/lib"
# Install conda # Install conda
curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@ -30,7 +30,6 @@ cat ${PROJ_ROOT}/scripts/build_ios.sh
echo "########################################################" echo "########################################################"
echo "IOS_ARCH: ${IOS_ARCH}" echo "IOS_ARCH: ${IOS_ARCH}"
echo "IOS_PLATFORM: ${IOS_PLATFORM}" echo "IOS_PLATFORM: ${IOS_PLATFORM}"
export BUILD_PYTORCH_MOBILE=1
export IOS_ARCH=${IOS_ARCH} export IOS_ARCH=${IOS_ARCH}
export IOS_PLATFORM=${IOS_PLATFORM} export IOS_PLATFORM=${IOS_PLATFORM}
unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts

1
.circleci/verbatim-sources/job-specs-custom.yml
View File

@ -443,7 +443,6 @@
if [ -n "${SELECTED_OP_LIST}" ]; then if [ -n "${SELECTED_OP_LIST}" ]; then
export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}" export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}"
fi fi
export BUILD_PYTORCH_MOBILE=1
export IOS_ARCH=${IOS_ARCH} export IOS_ARCH=${IOS_ARCH}
export IOS_PLATFORM=${IOS_PLATFORM} export IOS_PLATFORM=${IOS_PLATFORM}
unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts

2
.jenkins/caffe2/build.sh
View File

@ -104,7 +104,7 @@ if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
build_args+=("BUILD_TEST=ON") build_args+=("BUILD_TEST=ON")
build_args+=("USE_OBSERVERS=ON") build_args+=("USE_OBSERVERS=ON")
build_args+=("USE_ZSTD=ON") build_args+=("USE_ZSTD=ON")
"${ROOT_DIR}/scripts/build_android.sh" $(build_to_cmake ${build_args[@]}) "$@" BUILD_CAFFE2_MOBILE=1 "${ROOT_DIR}/scripts/build_android.sh" $(build_to_cmake ${build_args[@]}) "$@"
exit 0 exit 0
fi fi

1
.jenkins/pytorch/build.sh
View File

@ -105,7 +105,6 @@ if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
elif [[ "${BUILD_ENVIRONMENT}" == *-x86_64* ]]; then elif [[ "${BUILD_ENVIRONMENT}" == *-x86_64* ]]; then
build_args+=("-DANDROID_ABI=x86_64") build_args+=("-DANDROID_ABI=x86_64")
fi fi
export BUILD_PYTORCH_MOBILE=1
exec ./scripts/build_android.sh "${build_args[@]}" "$@" exec ./scripts/build_android.sh "${build_args[@]}" "$@"
fi fi

2
CMakeLists.txt
View File

@ -121,7 +121,7 @@ option(BUILD_CUSTOM_PROTOBUF "Build and use Caffe2's own protobuf under third_pa
option(BUILD_PYTHON "Build Python binaries" ON) option(BUILD_PYTHON "Build Python binaries" ON)
option(BUILD_CAFFE2_OPS "Build Caffe2 operators" ON) option(BUILD_CAFFE2_OPS "Build Caffe2 operators" ON)
option(BUILD_SHARED_LIBS "Build libcaffe2.so" ON) option(BUILD_SHARED_LIBS "Build libcaffe2.so" ON)
option(BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" ON) option(BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" OFF)
option(USE_STATIC_DISPATCH "Use static dispatch for ATen operators" OFF) option(USE_STATIC_DISPATCH "Use static dispatch for ATen operators" OFF)
cmake_dependent_option( cmake_dependent_option(
CAFFE2_LINK_LOCAL_PROTOBUF "If set, build protobuf inside libcaffe2.so." ON CAFFE2_LINK_LOCAL_PROTOBUF "If set, build protobuf inside libcaffe2.so." ON

3
android/build_test_app.sh
View File

@ -62,7 +62,7 @@ mkdir -p $OUT_DIR
pushd $PYTORCH_DIR pushd $PYTORCH_DIR
python $PYTORCH_DIR/setup.py clean python $PYTORCH_DIR/setup.py clean
ANDROID_ABI=$abi BUILD_PYTORCH_MOBILE=1 VERBOSE=1 ANDROID_DEBUG_SYMBOLS=1 $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache) ANDROID_ABI=$abi VERBOSE=1 ANDROID_DEBUG_SYMBOLS=1 $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache)
cp -R $PYTORCH_DIR/build_android/install/lib $OUT_DIR/ cp -R $PYTORCH_DIR/build_android/install/lib $OUT_DIR/
cp -R $PYTORCH_DIR/build_android/install/include $OUT_DIR/ cp -R $PYTORCH_DIR/build_android/install/include $OUT_DIR/
@ -97,4 +97,3 @@ find $PYTORCH_ANDROID_DIR -type f -name *apk
find $PYTORCH_ANDROID_DIR -type f -name *apk | xargs echo "To install apk run: $ANDROID_HOME/platform-tools/adb install -r " find $PYTORCH_ANDROID_DIR -type f -name *apk | xargs echo "To install apk run: $ANDROID_HOME/platform-tools/adb install -r "
popd popd

4
caffe2/CMakeLists.txt
View File

@ -61,7 +61,7 @@ if (NOT MSVC)
ENDIF() ENDIF()
if (NOT (INTERN_BUILD_MOBILE AND NOT BUILD_CAFFE2_MOBILE)) if (NOT (INTERN_BUILD_MOBILE AND NOT BUILD_CAFFE2_MOBILE))
set_source_files_properties( set_source_files_properties(
utils/threadpool/pthreadpool_new_if_impl.c PROPERTIES COMPILE_FLAGS -fno-openmp) utils/threadpool/pthreadpool_new_if_impl.c PROPERTIES COMPILE_FLAGS -fno-openmp)
endif() endif()
endif() endif()
@ -485,7 +485,7 @@ if (NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE)
${TORCH_SRC_DIR}/csrc/jit/mobile/type_parser.cpp ${TORCH_SRC_DIR}/csrc/jit/mobile/type_parser.cpp
) )
if (NOT INTERN_DISABLE_MOBILE_INTERP AND BUILD_CAFFE2_MOBILE) if (NOT INTERN_DISABLE_MOBILE_INTERP)
set (MOBILE_SRCS set (MOBILE_SRCS
${TORCH_SRC_DIR}/csrc/jit/mobile/function.cpp ${TORCH_SRC_DIR}/csrc/jit/mobile/function.cpp
${TORCH_SRC_DIR}/csrc/jit/mobile/import.cpp ${TORCH_SRC_DIR}/csrc/jit/mobile/import.cpp

17
ios/TestApp/README.md
View File

@ -4,7 +4,7 @@ The TestApp is being used for different purposes as described below
### Cocoapods ### Cocoapods
To quickly test our framework in Cocoapods, simply run To quickly test our framework in Cocoapods, simply run
```ruby ```ruby
pod install pod install
@ -16,29 +16,29 @@ This will pull the latest version of `LibTorch` from Cocoapods. To run the app,
The TestApp is currently being used as a dummy app by Circle CI for nightly jobs. The challenge comes when testing the arm64 build as we don't have a way to code-sign our TestApp. This is where Fastlane came to rescue. [Fastlane](https://fastlane.tools/) is a trendy automation tool for building and managing iOS applications. It also works seamlessly with Circle CI. We are going to leverage the `import_certificate` action, which can install developer certificates on CI machines. See `Fastfile` for more details. The TestApp is currently being used as a dummy app by Circle CI for nightly jobs. The challenge comes when testing the arm64 build as we don't have a way to code-sign our TestApp. This is where Fastlane came to rescue. [Fastlane](https://fastlane.tools/) is a trendy automation tool for building and managing iOS applications. It also works seamlessly with Circle CI. We are going to leverage the `import_certificate` action, which can install developer certificates on CI machines. See `Fastfile` for more details.
For simulator build, we run unit tests as the last step of our CI workflow. Those unit tests can also be run manually via the `fastlane scan` command. For simulator build, we run unit tests as the last step of our CI workflow. Those unit tests can also be run manually via the `fastlane scan` command.
### Benchmark ### Benchmark
The benchmark folder contains two scripts that help you setup the benchmark project. The `setup.rb` does the heavy-lifting jobs of setting up the XCode project, whereas the `trace_model.py` is a Python script that you can tweak to generate your model for benchmarking. Simply follow the steps below to setup the project The benchmark folder contains two scripts that help you setup the benchmark project. The `setup.rb` does the heavy-lifting jobs of setting up the XCode project, whereas the `trace_model.py` is a Python script that you can tweak to generate your model for benchmarking. Simply follow the steps below to setup the project
1. In the PyTorch root directory, run `BUILD_PYTORCH_MOBILE=1 IOS_ARCH=arm64 ./scripts/build_ios.sh` to generate the custom build from **Master** branch 1. In the PyTorch root directory, run `IOS_ARCH=arm64 ./scripts/build_ios.sh` to generate the custom build from **Master** branch
2. Navigate to the `benchmark` folder, run `python trace_model.py` to generate your model. 2. Navigate to the `benchmark` folder, run `python trace_model.py` to generate your model.
3. In the same directory, open `config.json`. Those are the input parameters you can tweak. 3. In the same directory, open `config.json`. Those are the input parameters you can tweak.
4. Again, in the same directory, run `ruby setup.rb` to setup the XCode project. 4. Again, in the same directory, run `ruby setup.rb` to setup the XCode project.
5. Open the `TestApp.xcodeproj`, you're ready to go. 5. Open the `TestApp.xcodeproj`, you're ready to go.
The benchmark code is written in C++, you can use `UI_LOG` to visualize the log. See `benchmark.mm` for more details. The benchmark code is written in C++, you can use `UI_LOG` to visualize the log. See `benchmark.mm` for more details.
### `bootstrap.sh` ### `bootstrap.sh`
For those who want to do perf testing but don't want to touch XCode, `bootstrap.sh` is the right tool for you. It'll automatically build and install the app on your device. That being said, it does require you to have For those who want to do perf testing but don't want to touch XCode, `bootstrap.sh` is the right tool for you. It'll automatically build and install the app on your device. That being said, it does require you to have
1. A valid iOS dev certificate installed on your local machine. 1. A valid iOS dev certificate installed on your local machine.
2. A valid provisioning profile for code signing 2. A valid provisioning profile for code signing
3. A valid team identifier 3. A valid team identifier
To run the script, simply type the command below and make sure your phone is connected via USB. To run the script, simply type the command below and make sure your phone is connected via USB.
```shell ```shell
./bootstrap ./bootstrap
@ -47,6 +47,3 @@ To run the script, simply type the command below and make sure your phone is con
Open the app on your device, the benchmark result will be displayed on the screen. Open the app on your device, the benchmark result will be displayed on the screen.
> Note This requires ios-deploy to be installed. Please have a look at [ios-deploy](https://github.com/ios-control/ios-deploy). To quickly install it, use `npm -g i ios-deploy` > Note This requires ios-deploy to be installed. Please have a look at [ios-deploy](https://github.com/ios-control/ios-deploy). To quickly install it, use `npm -g i ios-deploy`

6
scripts/build_android.sh
View File

@ -55,8 +55,8 @@ cd $BUILD_ROOT
CMAKE_ARGS=() CMAKE_ARGS=()
if [ -n "${BUILD_PYTORCH_MOBILE:-}" ]; then if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
CMAKE_ARGS+=("-DBUILD_CAFFE2_MOBILE=OFF") # Build PyTorch mobile
CMAKE_ARGS+=("-DUSE_STATIC_DISPATCH=ON") CMAKE_ARGS+=("-DUSE_STATIC_DISPATCH=ON")
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')") CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')") CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
@ -66,6 +66,8 @@ if [ -n "${BUILD_PYTORCH_MOBILE:-}" ]; then
CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}") CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}")
fi fi
else else
# Build Caffe2 mobile
CMAKE_ARGS+=("-DBUILD_CAFFE2_MOBILE=ON")
# Build protobuf from third_party so we have a host protoc binary. # Build protobuf from third_party so we have a host protoc binary.
echo "Building protoc" echo "Building protoc"
$CAFFE2_ROOT/scripts/build_host_protoc.sh $CAFFE2_ROOT/scripts/build_host_protoc.sh

6
scripts/build_ios.sh
View File

@ -17,8 +17,8 @@ cd $BUILD_ROOT
CMAKE_ARGS=() CMAKE_ARGS=()
if [ -n "${BUILD_PYTORCH_MOBILE:-}" ]; then if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
CMAKE_ARGS+=("-DBUILD_CAFFE2_MOBILE=OFF") # Build PyTorch mobile
CMAKE_ARGS+=("-DUSE_STATIC_DISPATCH=ON") CMAKE_ARGS+=("-DUSE_STATIC_DISPATCH=ON")
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')") CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')") CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
@ -33,6 +33,8 @@ if [ -n "${BUILD_PYTORCH_MOBILE:-}" ]; then
CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode") CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode")
fi fi
else else
# Build Caffe2 mobile
CMAKE_ARGS+=("-DBUILD_CAFFE2_MOBILE=ON")
# Build protobuf from third_party so we have a host protoc binary. # Build protobuf from third_party so we have a host protoc binary.
echo "Building protoc" echo "Building protoc"
BITCODE_FLAGS="-DCMAKE_C_FLAGS=-fembed-bitcode -DCMAKE_CXX_FLAGS=-fembed-bitcode " BITCODE_FLAGS="-DCMAKE_C_FLAGS=-fembed-bitcode -DCMAKE_CXX_FLAGS=-fembed-bitcode "

1
scripts/build_mobile.sh
View File

@ -21,7 +21,6 @@ mkdir -p $BUILD_ROOT
cd $BUILD_ROOT cd $BUILD_ROOT
CMAKE_ARGS=() CMAKE_ARGS=()
CMAKE_ARGS+=("-DBUILD_CAFFE2_MOBILE=OFF")
CMAKE_ARGS+=("-DUSE_STATIC_DISPATCH=ON") CMAKE_ARGS+=("-DUSE_STATIC_DISPATCH=ON")
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')") CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')") CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")

11
scripts/build_pytorch_android.sh
View File

@ -34,12 +34,12 @@ GRADLE_NOT_FOUND_MSG="Unable to find gradle, please add it to PATH or set GRADLE
if [ ! -x "$(command -v gradle)" ]; then if [ ! -x "$(command -v gradle)" ]; then
if [ -z "$GRADLE_HOME" ]; then if [ -z "$GRADLE_HOME" ]; then
echo GRADLE_NOT_FOUND_MSG echo GRADLE_NOT_FOUND_MSG
exit 1 exit 1
fi fi
GRADLE_PATH=$GRADLE_HOME/bin/gradle GRADLE_PATH=$GRADLE_HOME/bin/gradle
if [ ! -f "$GRADLE_PATH" ]; then if [ ! -f "$GRADLE_PATH" ]; then
echo GRADLE_NOT_FOUND_MSG echo GRADLE_NOT_FOUND_MSG
exit 1 exit 1
fi fi
fi fi
@ -51,7 +51,7 @@ if [ $# -gt 0 ]; then
ABIS_LIST=$1 ABIS_LIST=$1
CUSTOM_ABIS_LIST=true CUSTOM_ABIS_LIST=true
fi fi
echo "ABIS_LIST:$ABIS_LIST" echo "ABIS_LIST:$ABIS_LIST"
LIB_DIR=$PYTORCH_ANDROID_DIR/pytorch_android/src/main/jniLibs LIB_DIR=$PYTORCH_ANDROID_DIR/pytorch_android/src/main/jniLibs
@ -71,12 +71,12 @@ mkdir -p $OUT_DIR
pushd $PYTORCH_DIR pushd $PYTORCH_DIR
python $PYTORCH_DIR/setup.py clean python $PYTORCH_DIR/setup.py clean
ANDROID_ABI=$abi BUILD_PYTORCH_MOBILE=1 $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache) ANDROID_ABI=$abi $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache)
cp -R $PYTORCH_DIR/build_android/install/lib $OUT_DIR/ cp -R $PYTORCH_DIR/build_android/install/lib $OUT_DIR/
cp -R $PYTORCH_DIR/build_android/install/include $OUT_DIR/ cp -R $PYTORCH_DIR/build_android/install/include $OUT_DIR/
echo "$abi build output lib,include copied to $OUT_DIR" echo "$abi build output lib,include copied to $OUT_DIR"
LIB_LINK_PATH=$LIB_DIR/$abi LIB_LINK_PATH=$LIB_DIR/$abi
INCLUDE_LINK_PATH=$INCLUDE_DIR/$abi INCLUDE_LINK_PATH=$INCLUDE_DIR/$abi
@ -103,4 +103,3 @@ fi
find $PYTORCH_ANDROID_DIR -type f -name *aar | xargs ls -lah find $PYTORCH_ANDROID_DIR -type f -name *aar | xargs ls -lah
popd popd