mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
Revert "Cleanup old caffe2 scripts (#158475)"
This reverts commit 94d7f0c1ef9a4cb4db0eb5d6b1ffc55941cbeab1. Reverted https://github.com/pytorch/pytorch/pull/158475 on behalf of https://github.com/facebook-github-bot due to Diff reverted internally ([comment](https://github.com/pytorch/pytorch/pull/158475#issuecomment-3085447409))
This commit is contained in:
PyTorch MergeBot
15
.github/workflows/pull.yml
vendored
15
.github/workflows/pull.yml
vendored
@ -315,6 +315,21 @@ jobs:
|
|||||||
test-matrix: ${{ needs.linux-jammy-cuda12_8-py3_10-gcc11-build.outputs.test-matrix }}
|
test-matrix: ${{ needs.linux-jammy-cuda12_8-py3_10-gcc11-build.outputs.test-matrix }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
|
linux-jammy-py3-clang18-mobile-build:
|
||||||
|
name: linux-jammy-py3-clang18-mobile-build
|
||||||
|
uses: ./.github/workflows/_linux-build.yml
|
||||||
|
needs: get-label-type
|
||||||
|
with:
|
||||||
|
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
|
||||||
|
build-environment: linux-jammy-py3-clang12-mobile-build
|
||||||
|
docker-image-name: ci-image:pytorch-linux-jammy-py3-clang18-asan
|
||||||
|
build-generates-artifacts: false
|
||||||
|
test-matrix: |
|
||||||
|
{ include: [
|
||||||
|
{ config: "default", shard: 1, num_shards: 1 },
|
||||||
|
]}
|
||||||
|
secrets: inherit
|
||||||
|
|
||||||
linux-jammy-cuda12_8-cudnn9-py3_9-clang12-build:
|
linux-jammy-cuda12_8-cudnn9-py3_9-clang12-build:
|
||||||
name: linux-jammy-cuda12.8-cudnn9-py3.9-clang12
|
name: linux-jammy-cuda12.8-cudnn9-py3.9-clang12
|
||||||
uses: ./.github/workflows/_linux-build.yml
|
uses: ./.github/workflows/_linux-build.yml
|
||||||
|
|||||||
@ -1 +1,40 @@
|
|||||||
This directory contains the useful tools.
|
This directory contains the useful tools.
|
||||||
|
|
||||||
|
|
||||||
|
## build_android.sh
|
||||||
|
This script is to build PyTorch/Caffe2 library for Android. Take the following steps to start the build:
|
||||||
|
|
||||||
|
- set ANDROID_NDK to the location of ndk
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export ANDROID_NDK=YOUR_NDK_PATH
|
||||||
|
```
|
||||||
|
|
||||||
|
- run build_android.sh
|
||||||
|
```bash
|
||||||
|
#in your PyTorch root directory
|
||||||
|
bash scripts/build_android.sh
|
||||||
|
```
|
||||||
|
If succeeded, the libraries and headers would be generated to build_android/install directory. You can then copy these files from build_android/install to your Android project for further usage.
|
||||||
|
|
||||||
|
You can also override the cmake flags via command line, e.g., following command will also compile the executable binary files:
|
||||||
|
```bash
|
||||||
|
bash scripts/build_android.sh -DBUILD_BINARY=ON
|
||||||
|
```
|
||||||
|
|
||||||
|
## build_ios.sh
|
||||||
|
This script is to build PyTorch/Caffe2 library for iOS, and can only be performed on macOS. Take the following steps to start the build:
|
||||||
|
|
||||||
|
- Install Xcode from App Store, and configure "Command Line Tools" properly on Xcode.
|
||||||
|
- Install the dependencies:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
brew install cmake automake libtool
|
||||||
|
```
|
||||||
|
|
||||||
|
- run build_ios.sh
|
||||||
|
```bash
|
||||||
|
#in your PyTorch root directory
|
||||||
|
bash scripts/build_ios.sh
|
||||||
|
```
|
||||||
|
If succeeded, the libraries and headers would be generated to build_ios/install directory. You can then copy these files to your Xcode project for further usage.
|
||||||
|
|||||||
1
scripts/add_apache_header.sh
Executable file
1
scripts/add_apache_header.sh
Executable file
@ -0,0 +1 @@
|
|||||||
|
cat apache_header.txt $1 > _add_apache_header.txt && mv _add_apache_header.txt $1
|
||||||
15
scripts/apache_header.txt
Normal file
15
scripts/apache_header.txt
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
/**
|
||||||
|
* Copyright (c) 2016-present, Facebook, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
14
scripts/apache_python.txt
Normal file
14
scripts/apache_python.txt
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
# Copyright (c) 2016-present, Facebook, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
##############################################################################
|
||||||
189
scripts/build_android.sh
Executable file
189
scripts/build_android.sh
Executable file
@ -0,0 +1,189 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
##############################################################################
|
||||||
|
# Example command to build the android target.
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# This script shows how one can build a Caffe2 binary for the Android platform
|
||||||
|
# using android-cmake. A few notes:
|
||||||
|
#
|
||||||
|
# (1) This build also does a host build for protobuf. You will need autoconf
|
||||||
|
# to carry out this. If autoconf is not possible, you will need to provide
|
||||||
|
# a pre-built protoc binary that is the same version as the protobuf
|
||||||
|
# version under third_party.
|
||||||
|
# If you are building on Mac, you might need to install autotool and
|
||||||
|
# libtool. The easiest way is via homebrew:
|
||||||
|
# brew install automake
|
||||||
|
# brew install libtool
|
||||||
|
# (2) You will need to have android ndk installed. The current script assumes
|
||||||
|
# that you set ANDROID_NDK to the location of ndk.
|
||||||
|
# (3) The toolchain and the build target platform can be specified with the
|
||||||
|
# cmake arguments below. For more details, check out android-cmake's doc.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Android specific flags
|
||||||
|
if [ -z "$ANDROID_ABI" ]; then
|
||||||
|
ANDROID_ABI="armeabi-v7a with NEON"
|
||||||
|
fi
|
||||||
|
ANDROID_NATIVE_API_LEVEL="21"
|
||||||
|
echo "Build with ANDROID_ABI[$ANDROID_ABI], ANDROID_NATIVE_API_LEVEL[$ANDROID_NATIVE_API_LEVEL]"
|
||||||
|
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
|
||||||
|
if [ -z "$ANDROID_NDK" ]; then
|
||||||
|
echo "ANDROID_NDK not set; please set it to the Android NDK directory"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -d "$ANDROID_NDK" ]; then
|
||||||
|
echo "ANDROID_NDK not a directory; did you install it under $ANDROID_NDK?"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$PYTHON" ]; then
|
||||||
|
PYTHON=python
|
||||||
|
PYTHON_VERSION_MAJOR=$($PYTHON -c 'import sys; print(sys.version_info[0])')
|
||||||
|
if [ "${PYTHON_VERSION_MAJOR}" -le 2 ]; then
|
||||||
|
echo "Default python executable is Python-2, trying to use python3 alias"
|
||||||
|
PYTHON=python3
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
ANDROID_NDK_PROPERTIES="$ANDROID_NDK/source.properties"
|
||||||
|
[ -f "$ANDROID_NDK_PROPERTIES" ] && ANDROID_NDK_VERSION=$(sed -n 's/^Pkg.Revision[^=]*= *\([0-9]*\)\..*$/\1/p' "$ANDROID_NDK_PROPERTIES")
|
||||||
|
|
||||||
|
echo "Bash: $(/bin/bash --version | head -1)"
|
||||||
|
echo "Python: $($PYTHON -c 'import sys; print(sys.version)')"
|
||||||
|
echo "Caffe2 path: $CAFFE2_ROOT"
|
||||||
|
echo "Using Android NDK at $ANDROID_NDK"
|
||||||
|
echo "Android NDK version: $ANDROID_NDK_VERSION"
|
||||||
|
|
||||||
|
CMAKE_ARGS=()
|
||||||
|
|
||||||
|
# Build PyTorch mobile
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
|
||||||
|
CMAKE_ARGS+=("-DPython_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
|
||||||
|
|
||||||
|
# custom build with selected ops
|
||||||
|
if [ -n "${SELECTED_OP_LIST}" ]; then
|
||||||
|
SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)"
|
||||||
|
echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST"
|
||||||
|
if [ ! -r ${SELECTED_OP_LIST} ]; then
|
||||||
|
echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# If Ninja is installed, prefer it to Make
|
||||||
|
if [ -x "$(command -v ninja)" ]; then
|
||||||
|
CMAKE_ARGS+=("-GNinja")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use android-cmake to build Android project from CMake.
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake")
|
||||||
|
|
||||||
|
if [ -z "$BUILD_MOBILE_BENCHMARK" ]; then
|
||||||
|
BUILD_MOBILE_BENCHMARK=0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$BUILD_MOBILE_TEST" ]; then
|
||||||
|
BUILD_MOBILE_TEST=0
|
||||||
|
fi
|
||||||
|
# Don't build artifacts we don't need
|
||||||
|
CMAKE_ARGS+=("-DBUILD_TEST=OFF")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_BINARY=OFF")
|
||||||
|
|
||||||
|
# If there exists env variable and it equals to 0, build full jit interpreter.
|
||||||
|
# Default behavior is to build lite interpreter
|
||||||
|
# cmd: BUILD_LITE_INTERPRETER=0 ./scripts/build_android.sh
|
||||||
|
if [ "${BUILD_LITE_INTERPRETER}" == 0 ]; then
|
||||||
|
CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON")
|
||||||
|
fi
|
||||||
|
if [ "${TRACING_BASED}" == 1 ]; then
|
||||||
|
CMAKE_ARGS+=("-DTRACING_BASED=ON")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DTRACING_BASED=OFF")
|
||||||
|
fi
|
||||||
|
if [ "${USE_LIGHTWEIGHT_DISPATCH}" == 1 ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=ON")
|
||||||
|
CMAKE_ARGS+=("-DSTATIC_DISPATCH_BACKEND=CPU")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=OFF")
|
||||||
|
fi
|
||||||
|
|
||||||
|
CMAKE_ARGS+=("-DBUILD_MOBILE_BENCHMARK=$BUILD_MOBILE_BENCHMARK")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_MOBILE_TEST=$BUILD_MOBILE_TEST")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_PYTHON=OFF")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")
|
||||||
|
if (( "${ANDROID_NDK_VERSION:-0}" < 18 )); then
|
||||||
|
CMAKE_ARGS+=("-DANDROID_TOOLCHAIN=gcc")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DANDROID_TOOLCHAIN=clang")
|
||||||
|
fi
|
||||||
|
# Disable unused dependencies
|
||||||
|
CMAKE_ARGS+=("-DUSE_CUDA=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_ITT=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_MPI=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")
|
||||||
|
# Only toggle if VERBOSE=1
|
||||||
|
if [ "${VERBOSE:-}" == '1' ]; then
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Android specific flags
|
||||||
|
CMAKE_ARGS+=("-DANDROID_NDK=$ANDROID_NDK")
|
||||||
|
CMAKE_ARGS+=("-DANDROID_ABI=$ANDROID_ABI")
|
||||||
|
CMAKE_ARGS+=("-DANDROID_NATIVE_API_LEVEL=$ANDROID_NATIVE_API_LEVEL")
|
||||||
|
CMAKE_ARGS+=("-DANDROID_CPP_FEATURES=rtti exceptions")
|
||||||
|
if [ "${ANDROID_STL_SHARED:-}" == '1' ]; then
|
||||||
|
CMAKE_ARGS+=("-DANDROID_STL=c++_shared")
|
||||||
|
fi
|
||||||
|
if [ "${ANDROID_DEBUG_SYMBOLS:-}" == '1' ]; then
|
||||||
|
CMAKE_ARGS+=("-DANDROID_DEBUG_SYMBOLS=1")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${USE_VULKAN}" ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_VULKAN=ON")
|
||||||
|
if [ -n "${USE_VULKAN_FP16_INFERENCE}" ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_VULKAN_FP16_INFERENCE=ON")
|
||||||
|
fi
|
||||||
|
if [ -n "${USE_VULKAN_RELAXED_PRECISION}" ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_VULKAN_RELAXED_PRECISION=ON")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use-specified CMake arguments go last to allow overriding defaults
|
||||||
|
CMAKE_ARGS+=($@)
|
||||||
|
|
||||||
|
# Patch pocketfft (as Android does not have aligned_alloc even if compiled with c++17
|
||||||
|
if [ -f third_party/pocketfft/pocketfft_hdronly.h ]; then
|
||||||
|
sed -i -e "s/__cplusplus >= 201703L/0/" third_party/pocketfft/pocketfft_hdronly.h
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Now, actually build the Android target.
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_android"}
|
||||||
|
INSTALL_PREFIX=${BUILD_ROOT}/install
|
||||||
|
mkdir -p $BUILD_ROOT
|
||||||
|
cd $BUILD_ROOT
|
||||||
|
cmake "$CAFFE2_ROOT" \
|
||||||
|
-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release \
|
||||||
|
"${CMAKE_ARGS[@]}"
|
||||||
|
|
||||||
|
# Cross-platform parallel build
|
||||||
|
if [ -z "$MAX_JOBS" ]; then
|
||||||
|
if [ "$(uname)" == 'Darwin' ]; then
|
||||||
|
MAX_JOBS=$(sysctl -n hw.ncpu)
|
||||||
|
else
|
||||||
|
MAX_JOBS=$(nproc)
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Will install headers and libs to $INSTALL_PREFIX for further Android project usage."
|
||||||
|
cmake --build . --target install -- "-j${MAX_JOBS}"
|
||||||
|
echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your Android project directory."
|
||||||
102
scripts/build_android_gradle.sh
Executable file
102
scripts/build_android_gradle.sh
Executable file
@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eux -o pipefail
|
||||||
|
|
||||||
|
env
|
||||||
|
echo "BUILD_ENVIRONMENT:$BUILD_ENVIRONMENT"
|
||||||
|
|
||||||
|
export ANDROID_NDK_HOME=/opt/ndk
|
||||||
|
export ANDROID_NDK=/opt/ndk
|
||||||
|
export ANDROID_HOME=/opt/android/sdk
|
||||||
|
|
||||||
|
# Must be in sync with GRADLE_VERSION in docker image for android
|
||||||
|
# https://github.com/pietern/pytorch-dockerfiles/blob/master/build.sh#L155
|
||||||
|
export GRADLE_VERSION=6.8.3
|
||||||
|
export GRADLE_HOME=/opt/gradle/gradle-$GRADLE_VERSION
|
||||||
|
export GRADLE_PATH=$GRADLE_HOME/bin/gradle
|
||||||
|
|
||||||
|
# touch gradle cache files to prevent expiration
|
||||||
|
while IFS= read -r -d '' file
|
||||||
|
do
|
||||||
|
touch "$file" || true
|
||||||
|
done < <(find /var/lib/jenkins/.gradle -type f -print0)
|
||||||
|
|
||||||
|
# Patch pocketfft (as Android does not have aligned_alloc even if compiled with c++17
|
||||||
|
if [ -f ~/workspace/third_party/pocketfft/pocketfft_hdronly.h ]; then
|
||||||
|
sed -i -e "s/__cplusplus >= 201703L/0/" ~/workspace/third_party/pocketfft/pocketfft_hdronly.h
|
||||||
|
fi
|
||||||
|
|
||||||
|
export GRADLE_LOCAL_PROPERTIES=~/workspace/android/local.properties
|
||||||
|
rm -f $GRADLE_LOCAL_PROPERTIES
|
||||||
|
echo "sdk.dir=/opt/android/sdk" >> $GRADLE_LOCAL_PROPERTIES
|
||||||
|
echo "ndk.dir=/opt/ndk" >> $GRADLE_LOCAL_PROPERTIES
|
||||||
|
echo "cmake.dir=/usr/local" >> $GRADLE_LOCAL_PROPERTIES
|
||||||
|
|
||||||
|
retry () {
|
||||||
|
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run custom build script
|
||||||
|
if [[ "${BUILD_ENVIRONMENT}" == *-gradle-custom-build* ]]; then
|
||||||
|
# Install torch & torchvision - used to download & dump used ops from test model.
|
||||||
|
retry pip install torch torchvision --progress-bar off
|
||||||
|
|
||||||
|
exec "$(dirname "${BASH_SOURCE[0]}")/../android/build_test_app_custom.sh" armeabi-v7a
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run default build
|
||||||
|
BUILD_ANDROID_INCLUDE_DIR_x86=~/workspace/build_android/install/include
|
||||||
|
BUILD_ANDROID_LIB_DIR_x86=~/workspace/build_android/install/lib
|
||||||
|
|
||||||
|
BUILD_ANDROID_INCLUDE_DIR_x86_64=~/workspace/build_android_install_x86_64/install/include
|
||||||
|
BUILD_ANDROID_LIB_DIR_x86_64=~/workspace/build_android_install_x86_64/install/lib
|
||||||
|
|
||||||
|
BUILD_ANDROID_INCLUDE_DIR_arm_v7a=~/workspace/build_android_install_arm_v7a/install/include
|
||||||
|
BUILD_ANDROID_LIB_DIR_arm_v7a=~/workspace/build_android_install_arm_v7a/install/lib
|
||||||
|
|
||||||
|
BUILD_ANDROID_INCLUDE_DIR_arm_v8a=~/workspace/build_android_install_arm_v8a/install/include
|
||||||
|
BUILD_ANDROID_LIB_DIR_arm_v8a=~/workspace/build_android_install_arm_v8a/install/lib
|
||||||
|
|
||||||
|
PYTORCH_ANDROID_SRC_MAIN_DIR=~/workspace/android/pytorch_android/src/main
|
||||||
|
|
||||||
|
JNI_INCLUDE_DIR=${PYTORCH_ANDROID_SRC_MAIN_DIR}/cpp/libtorch_include
|
||||||
|
mkdir -p $JNI_INCLUDE_DIR
|
||||||
|
|
||||||
|
JNI_LIBS_DIR=${PYTORCH_ANDROID_SRC_MAIN_DIR}/jniLibs
|
||||||
|
mkdir -p $JNI_LIBS_DIR
|
||||||
|
|
||||||
|
ln -s ${BUILD_ANDROID_INCLUDE_DIR_x86} ${JNI_INCLUDE_DIR}/x86
|
||||||
|
ln -s ${BUILD_ANDROID_LIB_DIR_x86} ${JNI_LIBS_DIR}/x86
|
||||||
|
|
||||||
|
if [[ "${BUILD_ENVIRONMENT}" != *-gradle-build-only-x86_32* ]]; then
|
||||||
|
ln -s ${BUILD_ANDROID_INCLUDE_DIR_x86_64} ${JNI_INCLUDE_DIR}/x86_64
|
||||||
|
ln -s ${BUILD_ANDROID_LIB_DIR_x86_64} ${JNI_LIBS_DIR}/x86_64
|
||||||
|
|
||||||
|
ln -s ${BUILD_ANDROID_INCLUDE_DIR_arm_v7a} ${JNI_INCLUDE_DIR}/armeabi-v7a
|
||||||
|
ln -s ${BUILD_ANDROID_LIB_DIR_arm_v7a} ${JNI_LIBS_DIR}/armeabi-v7a
|
||||||
|
|
||||||
|
ln -s ${BUILD_ANDROID_INCLUDE_DIR_arm_v8a} ${JNI_INCLUDE_DIR}/arm64-v8a
|
||||||
|
ln -s ${BUILD_ANDROID_LIB_DIR_arm_v8a} ${JNI_LIBS_DIR}/arm64-v8a
|
||||||
|
fi
|
||||||
|
|
||||||
|
GRADLE_PARAMS="-p android assembleRelease --debug --stacktrace"
|
||||||
|
if [[ "${BUILD_ENVIRONMENT}" == *-gradle-build-only-x86_32* ]]; then
|
||||||
|
GRADLE_PARAMS+=" -PABI_FILTERS=x86"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${GRADLE_OFFLINE:-}" ]; then
|
||||||
|
GRADLE_PARAMS+=" --offline"
|
||||||
|
fi
|
||||||
|
|
||||||
|
$GRADLE_PATH $GRADLE_PARAMS
|
||||||
|
|
||||||
|
find . -type f -name "*.a" -exec ls -lh {} \;
|
||||||
|
|
||||||
|
while IFS= read -r -d '' file
|
||||||
|
do
|
||||||
|
echo
|
||||||
|
echo "$file"
|
||||||
|
ls -lah "$file"
|
||||||
|
zipinfo -l "$file"
|
||||||
|
done < <(find . -type f -name '*.aar' -print0)
|
||||||
|
|
||||||
|
find . -type f -name *aar -print | xargs tar cfvz ~/workspace/android/artifacts.tgz
|
||||||
59
scripts/build_host_protoc.sh
Executable file
59
scripts/build_host_protoc.sh
Executable file
@ -0,0 +1,59 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
##############################################################################
|
||||||
|
# Build script to build the protoc compiler for the host platform.
|
||||||
|
##############################################################################
|
||||||
|
# This script builds the protoc compiler for the host platform, which is needed
|
||||||
|
# for any cross-compilation as we will need to convert the protobuf source
|
||||||
|
# files to cc files.
|
||||||
|
#
|
||||||
|
# --other-flags accepts flags that should be passed to cmake. Optional.
|
||||||
|
#
|
||||||
|
# After the execution of the file, one should be able to find the host protoc
|
||||||
|
# binary at build_host_protoc/bin/protoc.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_host_protoc"}
|
||||||
|
mkdir -p $BUILD_ROOT/build
|
||||||
|
cd $BUILD_ROOT/build
|
||||||
|
|
||||||
|
CMAKE_ARGS=()
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$BUILD_ROOT")
|
||||||
|
CMAKE_ARGS+=("-Dprotobuf_BUILD_TESTS=OFF")
|
||||||
|
|
||||||
|
# If Ninja is installed, prefer it to Make
|
||||||
|
if [ -x "$(command -v ninja)" ]; then
|
||||||
|
CMAKE_ARGS+=("-GNinja")
|
||||||
|
fi
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
case "$1" in
|
||||||
|
--other-flags)
|
||||||
|
shift;
|
||||||
|
CMAKE_ARGS+=("$@")
|
||||||
|
break ;;
|
||||||
|
"")
|
||||||
|
break ;;
|
||||||
|
*)
|
||||||
|
echo "Unknown option passed as argument: $1"
|
||||||
|
break ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Use ccache if available (this path is where Homebrew installs ccache symlinks)
|
||||||
|
if [ "$(uname)" == 'Darwin' ] && [ -d /usr/local/opt/ccache/libexec ]; then
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/local/opt/ccache/libexec/gcc")
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=/usr/local/opt/ccache/libexec/g++")
|
||||||
|
fi
|
||||||
|
|
||||||
|
cmake "$CAFFE2_ROOT/third_party/protobuf/cmake" ${CMAKE_ARGS[@]}
|
||||||
|
|
||||||
|
if [ -z "$MAX_JOBS" ]; then
|
||||||
|
if [ "$(uname)" == 'Darwin' ]; then
|
||||||
|
MAX_JOBS=$(sysctl -n hw.ncpu)
|
||||||
|
else
|
||||||
|
MAX_JOBS=$(nproc)
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
cmake --build . -- "-j${MAX_JOBS}" install
|
||||||
155
scripts/build_ios.sh
Executable file
155
scripts/build_ios.sh
Executable file
@ -0,0 +1,155 @@
|
|||||||
|
#!/bin/bash -xe
|
||||||
|
##############################################################################
|
||||||
|
# Example command to build the iOS target.
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# This script shows how one can build a Caffe2 binary for the iOS platform
|
||||||
|
# using ios-cmake. This is very similar to the android-cmake - see
|
||||||
|
# build_android.sh for more details.
|
||||||
|
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
|
||||||
|
|
||||||
|
if [ -z "$PYTHON" ]; then
|
||||||
|
PYTHON=python
|
||||||
|
PYTHON_VERSION_MAJOR=$($PYTHON -c 'import sys; print(sys.version_info[0])')
|
||||||
|
if [ "${PYTHON_VERSION_MAJOR}" -le 2 ]; then
|
||||||
|
echo "Default python executable is Python-2, trying to use python3 alias"
|
||||||
|
PYTHON=python3
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Bash: $(/bin/bash --version | head -1)"
|
||||||
|
echo "Python: $($PYTHON -c 'import sys; print(sys.version)')"
|
||||||
|
echo "Caffe2 path: $CAFFE2_ROOT"
|
||||||
|
|
||||||
|
CMAKE_ARGS=()
|
||||||
|
|
||||||
|
# Build PyTorch mobile
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
|
||||||
|
CMAKE_ARGS+=("-DPython_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
|
||||||
|
|
||||||
|
# custom build with selected ops
|
||||||
|
if [ -n "${SELECTED_OP_LIST}" ]; then
|
||||||
|
SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)"
|
||||||
|
echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST"
|
||||||
|
if [ ! -r ${SELECTED_OP_LIST} ]; then
|
||||||
|
echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# bitcode
|
||||||
|
if [ "${ENABLE_BITCODE:-}" == '1' ]; then
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_C_FLAGS=-fembed-bitcode")
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use ios-cmake to build iOS project from CMake.
|
||||||
|
# This projects sets CMAKE_C_COMPILER to /usr/bin/gcc and
|
||||||
|
# CMAKE_CXX_COMPILER to /usr/bin/g++. In order to use ccache (if it is available) we
|
||||||
|
# must override these variables via CMake arguments.
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=$CAFFE2_ROOT/cmake/iOS.cmake")
|
||||||
|
if [ -n "${CCACHE_WRAPPER_PATH:-}"]; then
|
||||||
|
CCACHE_WRAPPER_PATH=/usr/local/opt/ccache/libexec
|
||||||
|
fi
|
||||||
|
if [ -d "$CCACHE_WRAPPER_PATH" ]; then
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=$CCACHE_WRAPPER_PATH/gcc")
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=$CCACHE_WRAPPER_PATH/g++")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# IOS_PLATFORM controls type of iOS platform (see ios-cmake)
|
||||||
|
if [ -n "${IOS_PLATFORM:-}" ]; then
|
||||||
|
CMAKE_ARGS+=("-DIOS_PLATFORM=${IOS_PLATFORM}")
|
||||||
|
if [ "${IOS_PLATFORM}" == "WATCHOS" ]; then
|
||||||
|
# enable bitcode by default for watchos
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_C_FLAGS=-fembed-bitcode")
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode")
|
||||||
|
# disable the QNNPACK
|
||||||
|
CMAKE_ARGS+=("-DUSE_PYTORCH_QNNPACK=OFF")
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# IOS_PLATFORM is not set, default to OS, which builds iOS.
|
||||||
|
CMAKE_ARGS+=("-DIOS_PLATFORM=OS")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${IOS_ARCH:-}" ]; then
|
||||||
|
CMAKE_ARGS+=("-DIOS_ARCH=${IOS_ARCH}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${BUILD_LITE_INTERPRETER}" == 0 ]; then
|
||||||
|
CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON")
|
||||||
|
fi
|
||||||
|
if [ "${TRACING_BASED}" == 1 ]; then
|
||||||
|
CMAKE_ARGS+=("-DTRACING_BASED=ON")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DTRACING_BASED=OFF")
|
||||||
|
fi
|
||||||
|
if [ "${USE_LIGHTWEIGHT_DISPATCH}" == 1 ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=ON")
|
||||||
|
CMAKE_ARGS+=("-DSTATIC_DISPATCH_BACKEND=CPU")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=OFF")
|
||||||
|
fi
|
||||||
|
|
||||||
|
CMAKE_ARGS+=("-DUSE_LITE_INTERPRETER_PROFILER=OFF")
|
||||||
|
|
||||||
|
# Don't build binaries or tests (only the library)
|
||||||
|
CMAKE_ARGS+=("-DBUILD_TEST=OFF")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_BINARY=OFF")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_PYTHON=OFF")
|
||||||
|
|
||||||
|
# Disable unused dependencies
|
||||||
|
CMAKE_ARGS+=("-DUSE_CUDA=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_ITT=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_MPI=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_NUMPY=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_MKLDNN=OFF")
|
||||||
|
|
||||||
|
# Metal
|
||||||
|
if [ "${USE_PYTORCH_METAL:-}" == "1" ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_PYTORCH_METAL=ON")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Core ML
|
||||||
|
if [ "${USE_COREML_DELEGATE}" == "1" ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_COREML_DELEGATE=ON")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# pthreads
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_THREAD_LIBS_INIT=-lpthread")
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_HAVE_THREADS_LIBRARY=1")
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_USE_PTHREADS_INIT=1")
|
||||||
|
|
||||||
|
# Only toggle if VERBOSE=1
|
||||||
|
if [ "${VERBOSE:-}" == '1' ]; then
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# enable ARC
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fobjc-arc")
|
||||||
|
|
||||||
|
# Now, actually build the iOS target.
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_ios"}
|
||||||
|
INSTALL_PREFIX=${BUILD_ROOT}/install
|
||||||
|
mkdir -p $BUILD_ROOT
|
||||||
|
cd $BUILD_ROOT
|
||||||
|
cmake "$CAFFE2_ROOT" \
|
||||||
|
-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
|
||||||
|
-DCMAKE_BUILD_TYPE=MinSizeRel \
|
||||||
|
-DBUILD_SHARED_LIBS=OFF \
|
||||||
|
${CMAKE_ARGS[@]} \
|
||||||
|
$@
|
||||||
|
|
||||||
|
cmake --build . -- "-j$(sysctl -n hw.ncpu)"
|
||||||
|
|
||||||
|
# copy headers and libs to install directory
|
||||||
|
echo "Will install headers and libs to $INSTALL_PREFIX for further Xcode project usage."
|
||||||
|
make install
|
||||||
|
echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your Xcode project directory."
|
||||||
82
scripts/build_local.sh
Executable file
82
scripts/build_local.sh
Executable file
@ -0,0 +1,82 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
##############################################################################
|
||||||
|
# Example command to build Caffe2
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
|
||||||
|
|
||||||
|
CMAKE_ARGS=()
|
||||||
|
|
||||||
|
# If Ninja is installed, prefer it to Make
|
||||||
|
if [ -x "$(command -v ninja)" ]; then
|
||||||
|
CMAKE_ARGS+=("-GNinja")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use ccache if available (this path is where Homebrew installs ccache symlinks)
|
||||||
|
if [ "$(uname)" == 'Darwin' ]; then
|
||||||
|
if [ -n "${CCACHE_WRAPPER_PATH:-}"]; then
|
||||||
|
CCACHE_WRAPPER_PATH=/usr/local/opt/ccache/libexec
|
||||||
|
fi
|
||||||
|
if [ -d "$CCACHE_WRAPPER_PATH" ]; then
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_C_COMPILER=$CCACHE_WRAPPER_PATH/gcc")
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=$CCACHE_WRAPPER_PATH/g++")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use special install script with Anaconda
|
||||||
|
if [ -n "${USE_ANACONDA}" ]; then
|
||||||
|
export SKIP_CONDA_TESTS=1
|
||||||
|
export CONDA_INSTALL_LOCALLY=1
|
||||||
|
"${ROOT_DIR}/scripts/build_anaconda.sh" "$@"
|
||||||
|
else
|
||||||
|
# Make sure that pyyaml is installed for the codegen of building Aten to work
|
||||||
|
if [[ -n "$(python -c 'import yaml' 2>&1)" ]]; then
|
||||||
|
echo "Installing pyyaml with pip at $(which pip)"
|
||||||
|
pip install --user pyyaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Make sure that typing is installed for the codegen of building Aten to work
|
||||||
|
if [[ -n "$(python -c 'import typing' 2>&1)" ]]; then
|
||||||
|
echo "Installing typing with pip at $(which pip)"
|
||||||
|
pip install --user typing
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Build protobuf compiler from third_party if configured to do so
|
||||||
|
if [ -n "${USE_HOST_PROTOC:-}" ]; then
|
||||||
|
echo "USE_HOST_PROTOC is set; building protoc before building Caffe2..."
|
||||||
|
"$CAFFE2_ROOT/scripts/build_host_protoc.sh"
|
||||||
|
CUSTOM_PROTOC_EXECUTABLE="$CAFFE2_ROOT/build_host_protoc/bin/protoc"
|
||||||
|
echo "Built protoc $("$CUSTOM_PROTOC_EXECUTABLE" --version)"
|
||||||
|
CMAKE_ARGS+=("-DCAFFE2_CUSTOM_PROTOC_EXECUTABLE=$CUSTOM_PROTOC_EXECUTABLE")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# We are going to build the target into build.
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
|
||||||
|
mkdir -p "$BUILD_ROOT"
|
||||||
|
cd "$BUILD_ROOT"
|
||||||
|
echo "Building Caffe2 in: $BUILD_ROOT"
|
||||||
|
|
||||||
|
cmake "$CAFFE2_ROOT" \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release \
|
||||||
|
"${CMAKE_ARGS[@]}" \
|
||||||
|
"$@"
|
||||||
|
|
||||||
|
# Determine the number of CPUs to build with.
|
||||||
|
# If the `CAFFE_MAKE_NCPUS` variable is not specified, use them all.
|
||||||
|
if [ -n "${MAX_JOBS}" ]; then
|
||||||
|
CAFFE_MAKE_NCPUS="$MAX_JOBS"
|
||||||
|
elif [ -n "${CAFFE_MAKE_NCPUS}" ]; then
|
||||||
|
CAFFE_MAKE_NCPUS="$CAFFE_MAKE_NCPUS"
|
||||||
|
elif [ "$(uname)" == 'Darwin' ]; then
|
||||||
|
CAFFE_MAKE_NCPUS="$(sysctl -n hw.ncpu)"
|
||||||
|
else
|
||||||
|
CAFFE_MAKE_NCPUS="$(nproc)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Now, actually build the target.
|
||||||
|
cmake --build . -- "-j$CAFFE_MAKE_NCPUS"
|
||||||
|
fi
|
||||||
107
scripts/build_mobile.sh
Executable file
107
scripts/build_mobile.sh
Executable file
@ -0,0 +1,107 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
##############################################################################
|
||||||
|
# Example command to build the mobile target.
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# This script shows how one can build a libtorch library optimized for mobile
|
||||||
|
# devices using host toolchain.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
export BUILD_PYTORCH_MOBILE_WITH_HOST_TOOLCHAIN=1
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
|
||||||
|
|
||||||
|
echo "Bash: $(/bin/bash --version | head -1)"
|
||||||
|
echo "Caffe2 path: $CAFFE2_ROOT"
|
||||||
|
|
||||||
|
CMAKE_ARGS=()
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
|
||||||
|
CMAKE_ARGS+=("-DPython_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")
|
||||||
|
|
||||||
|
# custom build with selected ops
|
||||||
|
if [ -n "${SELECTED_OP_LIST}" ]; then
|
||||||
|
SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)"
|
||||||
|
echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST"
|
||||||
|
if [ ! -r ${SELECTED_OP_LIST} ]; then
|
||||||
|
echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# If Ninja is installed, prefer it to Make
|
||||||
|
if [ -x "$(command -v ninja)" ]; then
|
||||||
|
CMAKE_ARGS+=("-GNinja")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Don't build artifacts we don't need
|
||||||
|
CMAKE_ARGS+=("-DBUILD_TEST=OFF")
|
||||||
|
CMAKE_ARGS+=("-DBUILD_BINARY=OFF")
|
||||||
|
|
||||||
|
# If there exists env variable and it equals to 1, build lite interpreter.
|
||||||
|
# Default behavior is to build full jit interpreter.
|
||||||
|
# cmd: BUILD_LITE_INTERPRETER=1 ./scripts/build_mobile.sh
|
||||||
|
if [ "x${BUILD_LITE_INTERPRETER}" == "x1" ]; then
|
||||||
|
CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF")
|
||||||
|
fi
|
||||||
|
if [ "x${TRACING_BASED}" == "x1" ]; then
|
||||||
|
CMAKE_ARGS+=("-DTRACING_BASED=ON")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DTRACING_BASED=OFF")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Lightweight dispatch bypasses the PyTorch Dispatcher.
|
||||||
|
if [ "${USE_LIGHTWEIGHT_DISPATCH}" == 1 ]; then
|
||||||
|
CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=ON")
|
||||||
|
CMAKE_ARGS+=("-DSTATIC_DISPATCH_BACKEND=CPU")
|
||||||
|
else
|
||||||
|
CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=OFF")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Disable unused dependencies
|
||||||
|
CMAKE_ARGS+=("-DUSE_ROCM=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_CUDA=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_ITT=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_MPI=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_MKLDNN=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_NUMPY=OFF")
|
||||||
|
CMAKE_ARGS+=("-DUSE_BLAS=OFF")
|
||||||
|
|
||||||
|
# Only toggle if VERBOSE=1
|
||||||
|
if [ "${VERBOSE:-}" == '1' ]; then
|
||||||
|
CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use-specified CMake arguments go last to allow overriding defaults
|
||||||
|
CMAKE_ARGS+=("$@")
|
||||||
|
|
||||||
|
# Now, actually build the Android target.
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_mobile"}
|
||||||
|
INSTALL_PREFIX=${BUILD_ROOT}/install
|
||||||
|
mkdir -p $BUILD_ROOT
|
||||||
|
cd $BUILD_ROOT
|
||||||
|
cmake "$CAFFE2_ROOT" \
|
||||||
|
-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release \
|
||||||
|
"${CMAKE_ARGS[@]}"
|
||||||
|
|
||||||
|
# Cross-platform parallel build
|
||||||
|
if [ -z "$MAX_JOBS" ]; then
|
||||||
|
if [ "$(uname)" == 'Darwin' ]; then
|
||||||
|
MAX_JOBS=$(sysctl -n hw.ncpu)
|
||||||
|
else
|
||||||
|
MAX_JOBS=$(nproc)
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Will install headers and libs to $INSTALL_PREFIX for further project usage."
|
||||||
|
cmake --build . --target install -- "-j${MAX_JOBS}"
|
||||||
|
echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your project directory."
|
||||||
51
scripts/build_pytorch_android.sh
Executable file
51
scripts/build_pytorch_android.sh
Executable file
@ -0,0 +1,51 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
# Master script to build PyTorch Android library with Java bindings.
|
||||||
|
##############################################################################
|
||||||
|
# Example usage:
|
||||||
|
# - Build default AARs:
|
||||||
|
# scripts/build_pytorch_android.sh
|
||||||
|
#
|
||||||
|
# - Build for specific ABI(s):
|
||||||
|
# scripts/build_pytorch_android.sh armeabi-v7a
|
||||||
|
# scripts/build_pytorch_android.sh arm64-v8a,x86,x86_64
|
||||||
|
#
|
||||||
|
# Script's workflow:
|
||||||
|
# 1. Builds libtorch for android for specified android abisi (by default for all 4).
|
||||||
|
# Custom list of android abis can be specified as a bash argument as comma separated list.
|
||||||
|
# For example just for testing on android x86 emulator we need only x86 build.
|
||||||
|
# ./scripts/build_pytorch_android.sh x86
|
||||||
|
# 2. Creates symbolic links to android/pytorch_android/src/main/jniLibs/${abi} for libtorch build output,
|
||||||
|
# android/pytorch_android/src/main/cpp/libtorch_include/${abi} for headers.
|
||||||
|
# 3. Runs pyotrch_android gradle build:
|
||||||
|
# gradle assembleRelease
|
||||||
|
|
||||||
|
PYTORCH_DIR="$(cd $(dirname $0)/..; pwd -P)"
|
||||||
|
PYTORCH_ANDROID_DIR=$PYTORCH_DIR/android
|
||||||
|
|
||||||
|
echo "PYTORCH_DIR:$PYTORCH_DIR"
|
||||||
|
|
||||||
|
source "$PYTORCH_ANDROID_DIR/common.sh"
|
||||||
|
|
||||||
|
check_android_sdk
|
||||||
|
check_gradle
|
||||||
|
parse_abis_list "$@"
|
||||||
|
build_android
|
||||||
|
|
||||||
|
# To set proxy for gradle add following lines to ./gradle/gradle.properties:
|
||||||
|
# systemProp.http.proxyHost=...
|
||||||
|
# systemProp.http.proxyPort=8080
|
||||||
|
# systemProp.https.proxyHost=...
|
||||||
|
# systemProp.https.proxyPort=8080
|
||||||
|
|
||||||
|
if [ "$CUSTOM_ABIS_LIST" = true ]; then
|
||||||
|
# Skipping clean task here as android gradle plugin 3.3.2 exteralNativeBuild has problems
|
||||||
|
# with it when abiFilters are specified.
|
||||||
|
$GRADLE_PATH -PABI_FILTERS=$ABIS_LIST -p $PYTORCH_ANDROID_DIR assembleRelease
|
||||||
|
else
|
||||||
|
$GRADLE_PATH -p $PYTORCH_ANDROID_DIR clean assembleRelease
|
||||||
|
fi
|
||||||
|
|
||||||
|
find $PYTORCH_ANDROID_DIR -type f -name *aar | xargs ls -lah
|
||||||
44
scripts/build_raspbian.sh
Executable file
44
scripts/build_raspbian.sh
Executable file
@ -0,0 +1,44 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
##############################################################################
|
||||||
|
# Example command to build the Raspbian target.
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# This script shows how one can build a Caffe2 binary for raspbian. The build
|
||||||
|
# is essentially much similar to a host build, with one additional change
|
||||||
|
# which is to specify -mfpu=neon for optimized speed.
|
||||||
|
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
|
||||||
|
echo "Caffe2 codebase root is: $CAFFE2_ROOT"
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
|
||||||
|
mkdir -p $BUILD_ROOT
|
||||||
|
echo "Build Caffe2 raspbian into: $BUILD_ROOT"
|
||||||
|
|
||||||
|
# obtain dependencies.
|
||||||
|
echo "Installing dependencies."
|
||||||
|
sudo apt-get install \
|
||||||
|
cmake \
|
||||||
|
libgflags-dev \
|
||||||
|
libgoogle-glog-dev \
|
||||||
|
libprotobuf-dev \
|
||||||
|
libpython-dev \
|
||||||
|
python-pip \
|
||||||
|
python-numpy \
|
||||||
|
protobuf-compiler \
|
||||||
|
python-protobuf
|
||||||
|
# python dependencies
|
||||||
|
sudo pip install hypothesis
|
||||||
|
|
||||||
|
# Now, actually build the raspbian target.
|
||||||
|
echo "Building caffe2"
|
||||||
|
cd $BUILD_ROOT
|
||||||
|
|
||||||
|
# Note: you can add more dependencies above if you need libraries such as
|
||||||
|
# leveldb, lmdb, etc.
|
||||||
|
cmake "$CAFFE2_ROOT" \
|
||||||
|
-DCMAKE_VERBOSE_MAKEFILE=1 \
|
||||||
|
-DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=hard" \
|
||||||
|
|| exit 1
|
||||||
|
|
||||||
|
# Note: while Raspberry pi has 4 cores, running too many builds in parallel may
|
||||||
|
# cause out of memory errors so we will simply run -j 2 only.
|
||||||
|
make -j 2 || exit 1
|
||||||
51
scripts/build_tegra_x1.sh
Executable file
51
scripts/build_tegra_x1.sh
Executable file
@ -0,0 +1,51 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
##############################################################################
|
||||||
|
# Example command to build Caffe2 on Tegra X1.
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# This script shows how one can build a Caffe2 binary for NVidia's TX1.
|
||||||
|
# The build script assumes that you have the most recent libraries installed
|
||||||
|
# via the JetPack toolkit available at
|
||||||
|
# https://developer.nvidia.com/embedded/jetpack
|
||||||
|
# and it assumes that we are starting from a fresh system after the jetpack
|
||||||
|
# installation. If you have already installed some of the dependencies, you
|
||||||
|
# may be able to skip quite a few of the apt-get installs.
|
||||||
|
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
|
||||||
|
echo "Caffe2 codebase root is: $CAFFE2_ROOT"
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
|
||||||
|
mkdir -p $BUILD_ROOT
|
||||||
|
echo "Build Caffe2 raspbian into: $BUILD_ROOT"
|
||||||
|
|
||||||
|
# obtain necessary dependencies
|
||||||
|
echo "Installing dependencies."
|
||||||
|
sudo apt-get install \
|
||||||
|
cmake \
|
||||||
|
libgflags-dev \
|
||||||
|
libgoogle-glog-dev \
|
||||||
|
libprotobuf-dev \
|
||||||
|
protobuf-compiler
|
||||||
|
|
||||||
|
# obtain optional dependencies that are usually useful to have.
|
||||||
|
echo "Installing optional dependencies."
|
||||||
|
sudo apt-get install \
|
||||||
|
libpython-dev \
|
||||||
|
python-numpy \
|
||||||
|
python-pip \
|
||||||
|
python-protobuf
|
||||||
|
|
||||||
|
# Obtain python hypothesis, which Caffe2 uses for unit testing. Note that
|
||||||
|
# the one provided by apt-get is quite old so we install it via pip
|
||||||
|
sudo pip install hypothesis
|
||||||
|
|
||||||
|
# Now, actually build the android target.
|
||||||
|
echo "Building caffe2"
|
||||||
|
cd $BUILD_ROOT
|
||||||
|
|
||||||
|
# CUDA_USE_STATIC_CUDA_RUNTIME needs to be set to off so that opencv can be
|
||||||
|
# properly used. Otherwise, opencv will complain that opencv_dep_cudart cannot
|
||||||
|
# be found.
|
||||||
|
cmake "$CAFFE2_ROOT" -DCUDA_USE_STATIC_CUDA_RUNTIME=OFF \
|
||||||
|
|| exit 1
|
||||||
|
|
||||||
|
make -j 4 || exit 1
|
||||||
118
scripts/build_tizen.sh
Executable file
118
scripts/build_tizen.sh
Executable file
@ -0,0 +1,118 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
##############################################################################
|
||||||
|
# Example command to build the Tizen target (RPi3).
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# This script shows how one can build a Caffe2 binary for a Tizen device (RPi3).
|
||||||
|
# The build is essentially much similar to a host build, with one additional change
|
||||||
|
# which is to specify -mfpu=neon for optimized speed.
|
||||||
|
|
||||||
|
setup_environment(){
|
||||||
|
# The rootfs image for a Tizen target (RPi3)is located at the below webpage:
|
||||||
|
# https://cdn.download.tizen.org/archive/releases/milestone/tizen/4.0.m1/tizen-unified_20170529.1/images/
|
||||||
|
# If you do not have a Tizen device, Please, run qemu-arm-static and chroot command.
|
||||||
|
# $ sudo chroot ~/tizen-rootfs qemu-arm-static /usr/bin/bash
|
||||||
|
|
||||||
|
CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
|
||||||
|
echo "Caffe2 codebase root is: $CAFFE2_ROOT"
|
||||||
|
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
|
||||||
|
mkdir -p $BUILD_ROOT
|
||||||
|
echo "Build Caffe2 Tizen into: $BUILD_ROOT"
|
||||||
|
}
|
||||||
|
|
||||||
|
caffe2_lite_dep_packages(){
|
||||||
|
# Obtain necessary dependencies
|
||||||
|
# You can set-up a rpm repository with zypper, yum, and dnf because Tizen
|
||||||
|
# software platform officially support rpm format such as Fedora, OpenSUSE.
|
||||||
|
# The official Tizen repository is as following:
|
||||||
|
# https://cdn.download.tizen.org/archive/releases/milestone/tizen/4.0.m1/
|
||||||
|
echo "Installing dependencies."
|
||||||
|
sudo zypper install \
|
||||||
|
make \
|
||||||
|
strace \
|
||||||
|
cmake \
|
||||||
|
gcc* \
|
||||||
|
binutils \
|
||||||
|
glibc* \
|
||||||
|
cpp \
|
||||||
|
protobuf-devel \
|
||||||
|
libstdc++*
|
||||||
|
}
|
||||||
|
|
||||||
|
caffe2_lite_build(){
|
||||||
|
# Now, actually build the android target.
|
||||||
|
echo "Building caffe2"
|
||||||
|
cd $BUILD_ROOT
|
||||||
|
|
||||||
|
# Note: add more dependencies above if you need libraries such as leveldb, lmdb, etc.
|
||||||
|
# If you have to disable a specific package due to a package absence
|
||||||
|
# from https://git.tizen.org/cgit/, append -Dxxx_xxx=OFF option before executing cmake.
|
||||||
|
cmake .. \
|
||||||
|
-DCMAKE_VERBOSE_MAKEFILE=1 \
|
||||||
|
-DUSE_GFLAGS=OFF \
|
||||||
|
-DUSE_GLOG=OFF -DUSE_NNPACK=OFF \
|
||||||
|
-DRUN_HAVE_STD_REGEX=0 \
|
||||||
|
-DRUN_HAVE_POSIX_REGEX=0 \
|
||||||
|
-DHAVE_GNU_POSIX_REGEX=0 \
|
||||||
|
-DUSE_MPI=OFF -DUSE_OPENMP=OFF \
|
||||||
|
-DBUILD_PYTHON=OFF \
|
||||||
|
-DUSE_GLOO=OFF \
|
||||||
|
-DUSE_OPENCV=OFF \
|
||||||
|
-DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \
|
||||||
|
|| exit 1
|
||||||
|
|
||||||
|
make -j`nproc` || exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
caffe2_full_dep_packages(){
|
||||||
|
# Obtain necessary dependencies
|
||||||
|
# You can set-up a rpm repository with zypper, yum, and dnf because Tizen
|
||||||
|
# software platform officially support rpm format such as Fedora, OpenSUSE.
|
||||||
|
# The official Tizen repository is as following:
|
||||||
|
# https://cdn.download.tizen.org/archive/releases/milestone/tizen/4.0.m1/
|
||||||
|
echo "Installing dependencies."
|
||||||
|
sudo zypper install \
|
||||||
|
cmake \
|
||||||
|
libgflags-dev \
|
||||||
|
libgoogle-glog-dev \
|
||||||
|
libprotobuf-dev \
|
||||||
|
protobuf-compiler
|
||||||
|
|
||||||
|
# Obtain optional dependencies that are usually useful to have.
|
||||||
|
echo "Installing optional dependencies."
|
||||||
|
sudo zypper install \
|
||||||
|
libpython-dev \
|
||||||
|
python-numpy \
|
||||||
|
python-pip \
|
||||||
|
python-protobuf
|
||||||
|
|
||||||
|
# Obtain python hypothesis, which Caffe2 uses for unit testing. Note that
|
||||||
|
# the one provided by zypper is quite old so we install it via pip
|
||||||
|
sudo pip install hypothesis
|
||||||
|
}
|
||||||
|
|
||||||
|
caffe2_full_build(){
|
||||||
|
# Now, actually build the android target.
|
||||||
|
echo "Building caffe2"
|
||||||
|
cd $BUILD_ROOT
|
||||||
|
|
||||||
|
# Note: add more dependencies above if you need libraries such as leveldb, lmdb, etc.
|
||||||
|
# If you have to disable a specific package due to a package absence
|
||||||
|
# from https://git.tizen.org/cgit/, append -Dxxx_xxx=OFF option before executing cmake.
|
||||||
|
cmake "$CAFFE2_ROOT" \
|
||||||
|
-DCMAKE_VERBOSE_MAKEFILE=1 \
|
||||||
|
-DUSE_CUDA=OFF \
|
||||||
|
-DUSE_ITT=OFF \
|
||||||
|
-DUSE_OPENCV=OFF \
|
||||||
|
-DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \
|
||||||
|
|| exit 1
|
||||||
|
|
||||||
|
make -j`nproc` || exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
#### Main
|
||||||
|
# Setup a build environment to compile Caffe2 deeplearning framework in Tizen platform.
|
||||||
|
setup_environment
|
||||||
|
# There are two build options to support 'full' version and 'lite' version (by default).
|
||||||
|
caffe2_lite_dep_packages
|
||||||
|
caffe2_lite_build
|
||||||
80
scripts/build_windows.bat
Normal file
80
scripts/build_windows.bat
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
:: #############################################################################
|
||||||
|
:: Example command to build on Windows.
|
||||||
|
:: #############################################################################
|
||||||
|
|
||||||
|
:: This script shows how one can build a Caffe2 binary for windows.
|
||||||
|
|
||||||
|
@echo off
|
||||||
|
setlocal
|
||||||
|
|
||||||
|
SET ORIGINAL_DIR=%cd%
|
||||||
|
SET CAFFE2_ROOT=%~dp0%..
|
||||||
|
|
||||||
|
if NOT DEFINED BUILD_BINARY (
|
||||||
|
set BUILD_BINARY=OFF
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED BUILD_SHARED_LIBS (
|
||||||
|
:: On CI, we test with BUILD_SHARED_LIBS=OFF.
|
||||||
|
:: By default, it will be BUILD_SHARED_LIBS=ON.
|
||||||
|
if NOT DEFINED BUILD_ENVIRONMENT (
|
||||||
|
set BUILD_SHARED_LIBS=OFF
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED CAFFE2_STATIC_LINK_CUDA (
|
||||||
|
set CAFFE2_STATIC_LINK_CUDA=OFF
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED CMAKE_BUILD_TYPE (
|
||||||
|
set CMAKE_BUILD_TYPE=Release
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED ONNX_NAMESPACE (
|
||||||
|
set ONNX_NAMESPACE=onnx_c2
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED TORCH_CUDA_ARCH_LIST (
|
||||||
|
set TORCH_CUDA_ARCH_LIST=5.0
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED USE_CUDA (
|
||||||
|
set USE_CUDA=OFF
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED USE_OBSERVERS (
|
||||||
|
set USE_OBSERVERS=OFF
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED MSVC_Z7_OVERRIDE (
|
||||||
|
set MSVC_Z7_OVERRIDE=OFF
|
||||||
|
)
|
||||||
|
|
||||||
|
if NOT DEFINED CMAKE_GENERATOR (
|
||||||
|
set CMAKE_GENERATOR=Ninja
|
||||||
|
)
|
||||||
|
|
||||||
|
set CMAKE_VERBOSE_MAKEFILE=1
|
||||||
|
|
||||||
|
:: Install pyyaml for Aten codegen
|
||||||
|
pip install pyyaml ninja
|
||||||
|
|
||||||
|
echo CAFFE2_ROOT=%CAFFE2_ROOT%
|
||||||
|
echo CMAKE_GENERATOR=%CMAKE_GENERATOR%
|
||||||
|
echo CMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%
|
||||||
|
|
||||||
|
:: Set up cmake. We will skip building the test files right now.
|
||||||
|
pushd %CAFFE2_ROOT%
|
||||||
|
python tools\build_libtorch.py || goto :label_error
|
||||||
|
popd
|
||||||
|
|
||||||
|
echo "Caffe2 built successfully"
|
||||||
|
cd %ORIGINAL_DIR%
|
||||||
|
endlocal
|
||||||
|
exit /b 0
|
||||||
|
|
||||||
|
:label_error
|
||||||
|
echo "Caffe2 building failed"
|
||||||
|
cd %ORIGINAL_DIR%
|
||||||
|
endlocal
|
||||||
|
exit /b 1
|
||||||
92
scripts/diagnose_protobuf.py
Normal file
92
scripts/diagnose_protobuf.py
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
## @package diagnose_protobuf
|
||||||
|
# Module scripts.diagnose_protobuf
|
||||||
|
"""Diagnoses the current protobuf situation.
|
||||||
|
|
||||||
|
Protocol buffer needs to be properly installed for Caffe2 to work, and
|
||||||
|
sometimes it is rather tricky. Specifically, we will need to have a
|
||||||
|
consistent version between C++ and python simultaneously. This is a
|
||||||
|
convenience script for one to quickly check if this is so on one's local
|
||||||
|
machine.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
[set your environmental variables like PATH and PYTHONPATH]
|
||||||
|
python scripts/diagnose_protobuf.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from subprocess import PIPE, Popen
|
||||||
|
|
||||||
|
|
||||||
|
# Get python protobuf version.
|
||||||
|
try:
|
||||||
|
import google.protobuf
|
||||||
|
|
||||||
|
python_version = google.protobuf.__version__
|
||||||
|
python_protobuf_installed = True
|
||||||
|
except ImportError:
|
||||||
|
print("DEBUG: cannot find python protobuf install.")
|
||||||
|
python_protobuf_installed = False
|
||||||
|
|
||||||
|
if os.name == "nt":
|
||||||
|
protoc_name = "protoc.exe"
|
||||||
|
else:
|
||||||
|
protoc_name = "protoc"
|
||||||
|
|
||||||
|
try:
|
||||||
|
p = Popen([protoc_name, "--version"], stdout=PIPE, stderr=PIPE)
|
||||||
|
out, err = p.communicate()
|
||||||
|
except:
|
||||||
|
print("DEBUG: did not find protoc binary.")
|
||||||
|
print("DEBUG: out: " + out)
|
||||||
|
print("DEBUG: err: " + err)
|
||||||
|
native_protobuf_installed = False
|
||||||
|
else:
|
||||||
|
if p.returncode:
|
||||||
|
print("DEBUG: protoc returned a non-zero return code.")
|
||||||
|
print("DEBUG: out: " + out)
|
||||||
|
print("DEBUG: err: " + err)
|
||||||
|
native_protobuf_installed = False
|
||||||
|
else:
|
||||||
|
tmp = re.search(r"\d\.\d\.\d", out)
|
||||||
|
if tmp:
|
||||||
|
native_version = tmp.group(0)
|
||||||
|
native_protobuf_installed = True
|
||||||
|
else:
|
||||||
|
print("DEBUG: cannot parse protoc version string.")
|
||||||
|
print("DEBUG: out: " + out)
|
||||||
|
native_protobuf_installed = False
|
||||||
|
|
||||||
|
PYTHON_PROTOBUF_NOT_INSTALLED = """
|
||||||
|
You have not installed python protobuf. Protobuf is needed to run caffe2. You
|
||||||
|
can install protobuf via pip or conda (if you are using anaconda python).
|
||||||
|
"""
|
||||||
|
|
||||||
|
NATIVE_PROTOBUF_NOT_INSTALLED = """
|
||||||
|
You have not installed the protoc binary. Protoc is needed to compile Caffe2
|
||||||
|
protobuf source files. Depending on the platform you are on, you can install
|
||||||
|
protobuf via:
|
||||||
|
(1) Mac: using homebrew and do brew install protobuf.
|
||||||
|
(2) Linux: use apt and do apt-get install libprotobuf-dev
|
||||||
|
(3) Windows: install from source, or from the releases here:
|
||||||
|
https://github.com/google/protobuf/releases/
|
||||||
|
"""
|
||||||
|
|
||||||
|
VERSION_MISMATCH = f"""
|
||||||
|
Your python protobuf is of version {python_version} but your native protoc version is of
|
||||||
|
version {native_version}. This will cause the installation to produce incompatible
|
||||||
|
protobuf files. This is bad in general - consider installing the same version.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Now, give actual recommendations
|
||||||
|
if not python_protobuf_installed:
|
||||||
|
print(PYTHON_PROTOBUF_NOT_INSTALLED)
|
||||||
|
|
||||||
|
if not native_protobuf_installed:
|
||||||
|
print(NATIVE_PROTOBUF_NOT_INSTALLED)
|
||||||
|
|
||||||
|
if python_protobuf_installed and native_protobuf_installed:
|
||||||
|
if python_version != native_version:
|
||||||
|
print(VERSION_MISMATCH)
|
||||||
|
else:
|
||||||
|
print("All looks good.")
|
||||||
92
scripts/fbcode-dev-setup/ccache_setup.sh
Executable file
92
scripts/fbcode-dev-setup/ccache_setup.sh
Executable file
@ -0,0 +1,92 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# This script installs CCache with CUDA support.
|
||||||
|
# Example usage:
|
||||||
|
# ./ccache_setup.sh --path /installed/folder
|
||||||
|
|
||||||
|
set -e
|
||||||
|
shopt -s expand_aliases
|
||||||
|
|
||||||
|
# Setup the proxy
|
||||||
|
alias with_proxy="HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 FTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 ftp_proxy=http://fwdproxy:8080 http_no_proxy='*.facebook.com|*.tfbnw.net|*.fb.com'"
|
||||||
|
|
||||||
|
# Parse options
|
||||||
|
path="$HOME/ccache"
|
||||||
|
force=false
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--path)
|
||||||
|
shift
|
||||||
|
path="$1"
|
||||||
|
path=$(realpath "$path")
|
||||||
|
;;
|
||||||
|
--force) # Force install
|
||||||
|
force=true
|
||||||
|
;;
|
||||||
|
--help)
|
||||||
|
echo 'usage: ./ccache_setup.py --path /installed/folder [--force]'
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Invalid option: $1"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
# Check whether you put nvcc in PATH
|
||||||
|
set +e
|
||||||
|
nvcc_path=$(which nvcc)
|
||||||
|
if [[ -z "$nvcc_path" ]]; then
|
||||||
|
nvcc_path="/usr/local/cuda/bin/nvcc"
|
||||||
|
export PATH="/usr/local/cuda/bin:$PATH"
|
||||||
|
fi
|
||||||
|
set -e
|
||||||
|
if [ ! -f "$nvcc_path" ] && ! $force; then
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
echo 'nvcc is not detected in $PATH'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "nvcc is detected at $nvcc_path"
|
||||||
|
|
||||||
|
if [ -f "$CUDA_NVCC_EXECUTABLE" ] && [[ "$CUDA_NVCC_EXECUTABLE" == *"ccache"* ]]; then # Heuristic rule
|
||||||
|
if $CUDA_NVCC_EXECUTABLE --version; then
|
||||||
|
if ! $force; then
|
||||||
|
echo "CCache with nvcc support is already installed at $CUDA_NVCC_EXECUTABLE, please add --force"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Installing CCache
|
||||||
|
echo "CCache will be installed at $path"
|
||||||
|
if [ -e "$path" ]; then
|
||||||
|
mv --backup=t -T "$path" "${path}.old"
|
||||||
|
fi
|
||||||
|
|
||||||
|
with_proxy git clone https://github.com/colesbury/ccache.git "$path" -b ccbin
|
||||||
|
cd "$path"
|
||||||
|
./autogen.sh
|
||||||
|
./configure
|
||||||
|
make install prefix="$path"
|
||||||
|
|
||||||
|
mkdir -p "$path/lib"
|
||||||
|
mkdir -p "$path/cuda"
|
||||||
|
ln -sf "$path/bin/ccache" "$path/lib/cc"
|
||||||
|
ln -sf "$path/bin/ccache" "$path/lib/c++"
|
||||||
|
ln -sf "$path/bin/ccache" "$path/lib/gcc"
|
||||||
|
ln -sf "$path/bin/ccache" "$path/lib/g++"
|
||||||
|
ln -sf "$path/bin/ccache" "$path/cuda/nvcc"
|
||||||
|
"$path/bin/ccache" -M 25Gi
|
||||||
|
|
||||||
|
# Make sure the nvcc wrapped in CCache is runnable
|
||||||
|
"$path/cuda/nvcc" --version
|
||||||
|
echo 'Congrats! The CCache with nvcc support is installed!'
|
||||||
|
echo -e "Please add the following lines to your bash init script:\\n"
|
||||||
|
echo "################ Env Var for CCache with CUDA support ################"
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
echo 'export PATH="'"$path"'/lib:$PATH"'
|
||||||
|
echo 'export CUDA_NVCC_EXECUTABLE="'"$path"'/cuda/nvcc"'
|
||||||
|
echo '######################################################################'
|
||||||
24
scripts/get_python_cmake_flags.py
Normal file
24
scripts/get_python_cmake_flags.py
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
## @package get_python_cmake_flags
|
||||||
|
# Module scripts.get_python_cmake_flags
|
||||||
|
##############################################################################
|
||||||
|
# Use this script to find your preferred python installation.
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# You can use the following to build with your preferred version of python
|
||||||
|
# if your installation is not being properly detected by CMake.
|
||||||
|
#
|
||||||
|
# mkdir -p build && cd build
|
||||||
|
# cmake $(python ../scripts/get_python_cmake_flags.py) ..
|
||||||
|
# make
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import sysconfig
|
||||||
|
|
||||||
|
|
||||||
|
flags = [
|
||||||
|
f"-DPython_EXECUTABLE:FILEPATH={sys.executable}",
|
||||||
|
]
|
||||||
|
|
||||||
|
print(" ".join(flags), end="")
|
||||||
18
scripts/proto.ps1
Normal file
18
scripts/proto.ps1
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
param(
|
||||||
|
[string]$protoc,
|
||||||
|
[string]$srcdir,
|
||||||
|
[string]$unprocessed,
|
||||||
|
[string]$processed,
|
||||||
|
[string]$out
|
||||||
|
)
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
Get-Content $unprocessed | % {$_ -Replace "caffe2/proto/caffe2.proto", "caffe2.proto"} | Set-Content $processed
|
||||||
|
Add-Content -Path $processed -Value "option optimize_for = LITE_RUNTIME;`n" -NoNewline
|
||||||
|
$dir = (Get-Item $processed).DirectoryName
|
||||||
|
|
||||||
|
copy $srcdir/caffe2/proto/caffe2.proto $srcdir/caffe2.proto
|
||||||
|
Add-Content -Path $srcdir/caffe2.proto -Value "option optimize_for = LITE_RUNTIME;`n" -NoNewline
|
||||||
|
|
||||||
|
$processed = (Get-Item $processed).Name
|
||||||
|
$cmd = "$protoc -I${dir} --cpp_out=$out $processed"
|
||||||
|
Invoke-Expression $cmd
|
||||||
13
scripts/remove_apache_header.sh
Executable file
13
scripts/remove_apache_header.sh
Executable file
@ -0,0 +1,13 @@
|
|||||||
|
if [[ "$1" == *.py ]]; then
|
||||||
|
apache_header="apache_python.txt"
|
||||||
|
else
|
||||||
|
apache_header="apache_header.txt"
|
||||||
|
fi
|
||||||
|
apache_lines=$(wc -l < "${apache_header}")
|
||||||
|
apache_md5=$(cat "${apache_header}" | md5)
|
||||||
|
header_md5=$(head -n ${apache_lines} $1 | md5)
|
||||||
|
if [ "${header_md5}" == "${apache_md5}" ]; then
|
||||||
|
keep_lines=$(($(wc -l < $1) - ${apache_lines}))
|
||||||
|
tail -n ${keep_lines} $1 > _remove_apache_header.txt
|
||||||
|
mv _remove_apache_header.txt $1
|
||||||
|
fi
|
||||||
7
scripts/temp.sh
Executable file
7
scripts/temp.sh
Executable file
@ -0,0 +1,7 @@
|
|||||||
|
find ../caffe2 -name "*.py" -exec ./remove_apache_header.sh {} \;
|
||||||
|
find ../caffe2 -name "*.h" -exec ./remove_apache_header.sh {} \;
|
||||||
|
find ../caffe2 -name "*.cc" -exec ./remove_apache_header.sh {} \;
|
||||||
|
find ../caffe2 -name "*.cpp" -exec ./remove_apache_header.sh {} \;
|
||||||
|
find ../caffe2 -name "*.cu" -exec ./remove_apache_header.sh {} \;
|
||||||
|
find ../caffe2 -name "*.mm" -exec ./remove_apache_header.sh {} \;
|
||||||
|
find ../caffe2 -name "*.m" -exec ./remove_apache_header.sh {} \;
|
||||||
76
scripts/xcode_build.rb
Normal file
76
scripts/xcode_build.rb
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
require 'optparse'
|
||||||
|
require 'xcodeproj'
|
||||||
|
|
||||||
|
options = {}
|
||||||
|
option_parser = OptionParser.new do |opts|
|
||||||
|
opts.banner = 'Tools for building PyTorch iOS framework on MacOS'
|
||||||
|
opts.on('-i', '--install_path ', 'path to the cmake install folder') { |value|
|
||||||
|
options[:install] = value
|
||||||
|
}
|
||||||
|
opts.on('-x', '--xcodeproj_path ', 'path to the XCode project file') { |value|
|
||||||
|
options[:xcodeproj] = value
|
||||||
|
}
|
||||||
|
opts.on('-p', '--platform ', 'platform for the current build, OS or SIMULATOR') { |value|
|
||||||
|
options[:platform] = value
|
||||||
|
}
|
||||||
|
end.parse!
|
||||||
|
puts options.inspect
|
||||||
|
|
||||||
|
install_path = File.expand_path(options[:install])
|
||||||
|
if not Dir.exist? (install_path)
|
||||||
|
raise "path don't exist:#{install_path}!"
|
||||||
|
end
|
||||||
|
xcodeproj_path = File.expand_path(options[:xcodeproj])
|
||||||
|
if not File.exist? (xcodeproj_path)
|
||||||
|
raise "path don't exist:#{xcodeproj_path}!"
|
||||||
|
end
|
||||||
|
|
||||||
|
project = Xcodeproj::Project.open(xcodeproj_path)
|
||||||
|
target = project.targets.first #TestApp
|
||||||
|
header_search_path = ['$(inherited)', "#{install_path}/include"]
|
||||||
|
libraries_search_path = ['$(inherited)', "#{install_path}/lib"]
|
||||||
|
other_linker_flags = ['$(inherited)', "-all_load"]
|
||||||
|
|
||||||
|
target.build_configurations.each do |config|
|
||||||
|
config.build_settings['HEADER_SEARCH_PATHS'] = header_search_path
|
||||||
|
config.build_settings['LIBRARY_SEARCH_PATHS'] = libraries_search_path
|
||||||
|
config.build_settings['OTHER_LDFLAGS'] = other_linker_flags
|
||||||
|
config.build_settings['ENABLE_BITCODE'] = 'No'
|
||||||
|
end
|
||||||
|
|
||||||
|
# link static libraries
|
||||||
|
target.frameworks_build_phases.clear
|
||||||
|
libs = ['libc10.a', 'libclog.a', 'libpthreadpool.a', 'libXNNPACK.a', 'libmicrokernels-prod.a', 'libeigen_blas.a', 'libcpuinfo.a', 'libpytorch_qnnpack.a', 'libtorch_cpu.a', 'libtorch.a', 'libkineto.a']
|
||||||
|
for lib in libs do
|
||||||
|
path = "#{install_path}/lib/#{lib}"
|
||||||
|
if File.exist?(path)
|
||||||
|
libref = project.frameworks_group.new_file(path)
|
||||||
|
target.frameworks_build_phases.add_file_reference(libref)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
# link system frameworks
|
||||||
|
frameworks = ['CoreML', 'Metal', 'MetalPerformanceShaders', 'Accelerate', 'UIKit']
|
||||||
|
if frameworks
|
||||||
|
frameworks.each do |framework|
|
||||||
|
path = "System/Library/Frameworks/#{framework}.framework"
|
||||||
|
framework_ref = project.frameworks_group.new_reference(path)
|
||||||
|
framework_ref.name = "#{framework}.framework"
|
||||||
|
framework_ref.source_tree = 'SDKROOT'
|
||||||
|
target.frameworks_build_phases.add_file_reference(framework_ref)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
project.save
|
||||||
|
|
||||||
|
sdk = nil
|
||||||
|
arch = nil
|
||||||
|
if options[:platform] == 'SIMULATOR'
|
||||||
|
sdk = 'iphonesimulator'
|
||||||
|
arch = 'arm64'
|
||||||
|
elsif options[:platform] == 'OS'
|
||||||
|
sdk = 'iphoneos'
|
||||||
|
arch = 'arm64'
|
||||||
|
else
|
||||||
|
raise "unsupported platform #{options[:platform]}"
|
||||||
|
end
|
||||||
|
|
||||||
|
exec "xcodebuild clean build -project #{xcodeproj_path} -alltargets -sdk #{sdk} -configuration Release -arch #{arch}"
|
||||||
Reference in New Issue
Block a user