From ced5cf042de1d4b573f258c9f770581d9574b990 Mon Sep 17 00:00:00 2001 From: PyTorch MergeBot Date: Thu, 17 Jul 2025 20:58:34 +0000 Subject: [PATCH] Revert "Cleanup old caffe2 scripts (#158475)" This reverts commit 94d7f0c1ef9a4cb4db0eb5d6b1ffc55941cbeab1. Reverted https://github.com/pytorch/pytorch/pull/158475 on behalf of https://github.com/facebook-github-bot due to Diff reverted internally ([comment](https://github.com/pytorch/pytorch/pull/158475#issuecomment-3085447409)) --- .github/workflows/pull.yml | 15 ++ scripts/README.md | 39 +++++ scripts/add_apache_header.sh | 1 + scripts/apache_header.txt | 15 ++ scripts/apache_python.txt | 14 ++ scripts/build_android.sh | 189 +++++++++++++++++++++++ scripts/build_android_gradle.sh | 102 ++++++++++++ scripts/build_host_protoc.sh | 59 +++++++ scripts/build_ios.sh | 155 +++++++++++++++++++ scripts/build_local.sh | 82 ++++++++++ scripts/build_mobile.sh | 107 +++++++++++++ scripts/build_pytorch_android.sh | 51 ++++++ scripts/build_raspbian.sh | 44 ++++++ scripts/build_tegra_x1.sh | 51 ++++++ scripts/build_tizen.sh | 118 ++++++++++++++ scripts/build_windows.bat | 80 ++++++++++ scripts/diagnose_protobuf.py | 92 +++++++++++ scripts/fbcode-dev-setup/ccache_setup.sh | 92 +++++++++++ scripts/get_python_cmake_flags.py | 24 +++ scripts/proto.ps1 | 18 +++ scripts/remove_apache_header.sh | 13 ++ scripts/temp.sh | 7 + scripts/xcode_build.rb | 76 +++++++++ 23 files changed, 1444 insertions(+) create mode 100755 scripts/add_apache_header.sh create mode 100644 scripts/apache_header.txt create mode 100644 scripts/apache_python.txt create mode 100755 scripts/build_android.sh create mode 100755 scripts/build_android_gradle.sh create mode 100755 scripts/build_host_protoc.sh create mode 100755 scripts/build_ios.sh create mode 100755 scripts/build_local.sh create mode 100755 scripts/build_mobile.sh create mode 100755 scripts/build_pytorch_android.sh create mode 100755 scripts/build_raspbian.sh create mode 100755 scripts/build_tegra_x1.sh create mode 100755 scripts/build_tizen.sh create mode 100644 scripts/build_windows.bat create mode 100644 scripts/diagnose_protobuf.py create mode 100755 scripts/fbcode-dev-setup/ccache_setup.sh create mode 100644 scripts/get_python_cmake_flags.py create mode 100644 scripts/proto.ps1 create mode 100755 scripts/remove_apache_header.sh create mode 100755 scripts/temp.sh create mode 100644 scripts/xcode_build.rb diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index be0bdc527cc1..59a726517380 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -315,6 +315,21 @@ jobs: test-matrix: ${{ needs.linux-jammy-cuda12_8-py3_10-gcc11-build.outputs.test-matrix }} secrets: inherit + linux-jammy-py3-clang18-mobile-build: + name: linux-jammy-py3-clang18-mobile-build + uses: ./.github/workflows/_linux-build.yml + needs: get-label-type + with: + runner_prefix: "${{ needs.get-label-type.outputs.label-type }}" + build-environment: linux-jammy-py3-clang12-mobile-build + docker-image-name: ci-image:pytorch-linux-jammy-py3-clang18-asan + build-generates-artifacts: false + test-matrix: | + { include: [ + { config: "default", shard: 1, num_shards: 1 }, + ]} + secrets: inherit + linux-jammy-cuda12_8-cudnn9-py3_9-clang12-build: name: linux-jammy-cuda12.8-cudnn9-py3.9-clang12 uses: ./.github/workflows/_linux-build.yml diff --git a/scripts/README.md b/scripts/README.md index 367e7261f6a6..a1c5ae5f93e6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1 +1,40 @@ This directory contains the useful tools. + + +## build_android.sh +This script is to build PyTorch/Caffe2 library for Android. Take the following steps to start the build: + +- set ANDROID_NDK to the location of ndk + +```bash +export ANDROID_NDK=YOUR_NDK_PATH +``` + +- run build_android.sh +```bash +#in your PyTorch root directory +bash scripts/build_android.sh +``` +If succeeded, the libraries and headers would be generated to build_android/install directory. You can then copy these files from build_android/install to your Android project for further usage. + +You can also override the cmake flags via command line, e.g., following command will also compile the executable binary files: +```bash +bash scripts/build_android.sh -DBUILD_BINARY=ON +``` + +## build_ios.sh +This script is to build PyTorch/Caffe2 library for iOS, and can only be performed on macOS. Take the following steps to start the build: + +- Install Xcode from App Store, and configure "Command Line Tools" properly on Xcode. +- Install the dependencies: + +```bash +brew install cmake automake libtool +``` + +- run build_ios.sh +```bash +#in your PyTorch root directory +bash scripts/build_ios.sh +``` +If succeeded, the libraries and headers would be generated to build_ios/install directory. You can then copy these files to your Xcode project for further usage. diff --git a/scripts/add_apache_header.sh b/scripts/add_apache_header.sh new file mode 100755 index 000000000000..a29a059d2d03 --- /dev/null +++ b/scripts/add_apache_header.sh @@ -0,0 +1 @@ +cat apache_header.txt $1 > _add_apache_header.txt && mv _add_apache_header.txt $1 diff --git a/scripts/apache_header.txt b/scripts/apache_header.txt new file mode 100644 index 000000000000..b4eff258eb04 --- /dev/null +++ b/scripts/apache_header.txt @@ -0,0 +1,15 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/scripts/apache_python.txt b/scripts/apache_python.txt new file mode 100644 index 000000000000..bc104d884515 --- /dev/null +++ b/scripts/apache_python.txt @@ -0,0 +1,14 @@ +# Copyright (c) 2016-present, Facebook, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################## diff --git a/scripts/build_android.sh b/scripts/build_android.sh new file mode 100755 index 000000000000..43f11b86828d --- /dev/null +++ b/scripts/build_android.sh @@ -0,0 +1,189 @@ +#!/bin/bash +############################################################################## +# Example command to build the android target. +############################################################################## +# +# This script shows how one can build a Caffe2 binary for the Android platform +# using android-cmake. A few notes: +# +# (1) This build also does a host build for protobuf. You will need autoconf +# to carry out this. If autoconf is not possible, you will need to provide +# a pre-built protoc binary that is the same version as the protobuf +# version under third_party. +# If you are building on Mac, you might need to install autotool and +# libtool. The easiest way is via homebrew: +# brew install automake +# brew install libtool +# (2) You will need to have android ndk installed. The current script assumes +# that you set ANDROID_NDK to the location of ndk. +# (3) The toolchain and the build target platform can be specified with the +# cmake arguments below. For more details, check out android-cmake's doc. + +set -e + +# Android specific flags +if [ -z "$ANDROID_ABI" ]; then + ANDROID_ABI="armeabi-v7a with NEON" +fi +ANDROID_NATIVE_API_LEVEL="21" +echo "Build with ANDROID_ABI[$ANDROID_ABI], ANDROID_NATIVE_API_LEVEL[$ANDROID_NATIVE_API_LEVEL]" + +CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)" +if [ -z "$ANDROID_NDK" ]; then + echo "ANDROID_NDK not set; please set it to the Android NDK directory" + exit 1 +fi + +if [ ! -d "$ANDROID_NDK" ]; then + echo "ANDROID_NDK not a directory; did you install it under $ANDROID_NDK?" + exit 1 +fi + +if [ -z "$PYTHON" ]; then + PYTHON=python + PYTHON_VERSION_MAJOR=$($PYTHON -c 'import sys; print(sys.version_info[0])') + if [ "${PYTHON_VERSION_MAJOR}" -le 2 ]; then + echo "Default python executable is Python-2, trying to use python3 alias" + PYTHON=python3 + fi +fi + +ANDROID_NDK_PROPERTIES="$ANDROID_NDK/source.properties" +[ -f "$ANDROID_NDK_PROPERTIES" ] && ANDROID_NDK_VERSION=$(sed -n 's/^Pkg.Revision[^=]*= *\([0-9]*\)\..*$/\1/p' "$ANDROID_NDK_PROPERTIES") + +echo "Bash: $(/bin/bash --version | head -1)" +echo "Python: $($PYTHON -c 'import sys; print(sys.version)')" +echo "Caffe2 path: $CAFFE2_ROOT" +echo "Using Android NDK at $ANDROID_NDK" +echo "Android NDK version: $ANDROID_NDK_VERSION" + +CMAKE_ARGS=() + +# Build PyTorch mobile +CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')") +CMAKE_ARGS+=("-DPython_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')") +CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF") + +# custom build with selected ops +if [ -n "${SELECTED_OP_LIST}" ]; then + SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)" + echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST" + if [ ! -r ${SELECTED_OP_LIST} ]; then + echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found." + exit 1 + fi + CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}") +fi + +# If Ninja is installed, prefer it to Make +if [ -x "$(command -v ninja)" ]; then + CMAKE_ARGS+=("-GNinja") +fi + +# Use android-cmake to build Android project from CMake. +CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake") + +if [ -z "$BUILD_MOBILE_BENCHMARK" ]; then + BUILD_MOBILE_BENCHMARK=0 +fi + +if [ -z "$BUILD_MOBILE_TEST" ]; then + BUILD_MOBILE_TEST=0 +fi +# Don't build artifacts we don't need +CMAKE_ARGS+=("-DBUILD_TEST=OFF") +CMAKE_ARGS+=("-DBUILD_BINARY=OFF") + +# If there exists env variable and it equals to 0, build full jit interpreter. +# Default behavior is to build lite interpreter +# cmd: BUILD_LITE_INTERPRETER=0 ./scripts/build_android.sh +if [ "${BUILD_LITE_INTERPRETER}" == 0 ]; then + CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF") +else + CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON") +fi +if [ "${TRACING_BASED}" == 1 ]; then + CMAKE_ARGS+=("-DTRACING_BASED=ON") +else + CMAKE_ARGS+=("-DTRACING_BASED=OFF") +fi +if [ "${USE_LIGHTWEIGHT_DISPATCH}" == 1 ]; then + CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=ON") + CMAKE_ARGS+=("-DSTATIC_DISPATCH_BACKEND=CPU") +else + CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=OFF") +fi + +CMAKE_ARGS+=("-DBUILD_MOBILE_BENCHMARK=$BUILD_MOBILE_BENCHMARK") +CMAKE_ARGS+=("-DBUILD_MOBILE_TEST=$BUILD_MOBILE_TEST") +CMAKE_ARGS+=("-DBUILD_PYTHON=OFF") +CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF") +if (( "${ANDROID_NDK_VERSION:-0}" < 18 )); then + CMAKE_ARGS+=("-DANDROID_TOOLCHAIN=gcc") +else + CMAKE_ARGS+=("-DANDROID_TOOLCHAIN=clang") +fi +# Disable unused dependencies +CMAKE_ARGS+=("-DUSE_CUDA=OFF") +CMAKE_ARGS+=("-DUSE_ITT=OFF") +CMAKE_ARGS+=("-DUSE_GFLAGS=OFF") +CMAKE_ARGS+=("-DUSE_OPENCV=OFF") +CMAKE_ARGS+=("-DUSE_MPI=OFF") +CMAKE_ARGS+=("-DUSE_OPENMP=OFF") +# Only toggle if VERBOSE=1 +if [ "${VERBOSE:-}" == '1' ]; then + CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1") +fi + +# Android specific flags +CMAKE_ARGS+=("-DANDROID_NDK=$ANDROID_NDK") +CMAKE_ARGS+=("-DANDROID_ABI=$ANDROID_ABI") +CMAKE_ARGS+=("-DANDROID_NATIVE_API_LEVEL=$ANDROID_NATIVE_API_LEVEL") +CMAKE_ARGS+=("-DANDROID_CPP_FEATURES=rtti exceptions") +if [ "${ANDROID_STL_SHARED:-}" == '1' ]; then + CMAKE_ARGS+=("-DANDROID_STL=c++_shared") +fi +if [ "${ANDROID_DEBUG_SYMBOLS:-}" == '1' ]; then + CMAKE_ARGS+=("-DANDROID_DEBUG_SYMBOLS=1") +fi + +if [ -n "${USE_VULKAN}" ]; then + CMAKE_ARGS+=("-DUSE_VULKAN=ON") + if [ -n "${USE_VULKAN_FP16_INFERENCE}" ]; then + CMAKE_ARGS+=("-DUSE_VULKAN_FP16_INFERENCE=ON") + fi + if [ -n "${USE_VULKAN_RELAXED_PRECISION}" ]; then + CMAKE_ARGS+=("-DUSE_VULKAN_RELAXED_PRECISION=ON") + fi +fi + +# Use-specified CMake arguments go last to allow overriding defaults +CMAKE_ARGS+=($@) + +# Patch pocketfft (as Android does not have aligned_alloc even if compiled with c++17 +if [ -f third_party/pocketfft/pocketfft_hdronly.h ]; then + sed -i -e "s/__cplusplus >= 201703L/0/" third_party/pocketfft/pocketfft_hdronly.h +fi + +# Now, actually build the Android target. +BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_android"} +INSTALL_PREFIX=${BUILD_ROOT}/install +mkdir -p $BUILD_ROOT +cd $BUILD_ROOT +cmake "$CAFFE2_ROOT" \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \ + -DCMAKE_BUILD_TYPE=Release \ + "${CMAKE_ARGS[@]}" + +# Cross-platform parallel build +if [ -z "$MAX_JOBS" ]; then + if [ "$(uname)" == 'Darwin' ]; then + MAX_JOBS=$(sysctl -n hw.ncpu) + else + MAX_JOBS=$(nproc) + fi +fi + +echo "Will install headers and libs to $INSTALL_PREFIX for further Android project usage." +cmake --build . --target install -- "-j${MAX_JOBS}" +echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your Android project directory." diff --git a/scripts/build_android_gradle.sh b/scripts/build_android_gradle.sh new file mode 100755 index 000000000000..fc27c5dd2516 --- /dev/null +++ b/scripts/build_android_gradle.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -eux -o pipefail + +env +echo "BUILD_ENVIRONMENT:$BUILD_ENVIRONMENT" + +export ANDROID_NDK_HOME=/opt/ndk +export ANDROID_NDK=/opt/ndk +export ANDROID_HOME=/opt/android/sdk + +# Must be in sync with GRADLE_VERSION in docker image for android +# https://github.com/pietern/pytorch-dockerfiles/blob/master/build.sh#L155 +export GRADLE_VERSION=6.8.3 +export GRADLE_HOME=/opt/gradle/gradle-$GRADLE_VERSION +export GRADLE_PATH=$GRADLE_HOME/bin/gradle + +# touch gradle cache files to prevent expiration +while IFS= read -r -d '' file +do + touch "$file" || true +done < <(find /var/lib/jenkins/.gradle -type f -print0) + +# Patch pocketfft (as Android does not have aligned_alloc even if compiled with c++17 +if [ -f ~/workspace/third_party/pocketfft/pocketfft_hdronly.h ]; then + sed -i -e "s/__cplusplus >= 201703L/0/" ~/workspace/third_party/pocketfft/pocketfft_hdronly.h +fi + +export GRADLE_LOCAL_PROPERTIES=~/workspace/android/local.properties +rm -f $GRADLE_LOCAL_PROPERTIES +echo "sdk.dir=/opt/android/sdk" >> $GRADLE_LOCAL_PROPERTIES +echo "ndk.dir=/opt/ndk" >> $GRADLE_LOCAL_PROPERTIES +echo "cmake.dir=/usr/local" >> $GRADLE_LOCAL_PROPERTIES + +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +# Run custom build script +if [[ "${BUILD_ENVIRONMENT}" == *-gradle-custom-build* ]]; then + # Install torch & torchvision - used to download & dump used ops from test model. + retry pip install torch torchvision --progress-bar off + + exec "$(dirname "${BASH_SOURCE[0]}")/../android/build_test_app_custom.sh" armeabi-v7a +fi + +# Run default build +BUILD_ANDROID_INCLUDE_DIR_x86=~/workspace/build_android/install/include +BUILD_ANDROID_LIB_DIR_x86=~/workspace/build_android/install/lib + +BUILD_ANDROID_INCLUDE_DIR_x86_64=~/workspace/build_android_install_x86_64/install/include +BUILD_ANDROID_LIB_DIR_x86_64=~/workspace/build_android_install_x86_64/install/lib + +BUILD_ANDROID_INCLUDE_DIR_arm_v7a=~/workspace/build_android_install_arm_v7a/install/include +BUILD_ANDROID_LIB_DIR_arm_v7a=~/workspace/build_android_install_arm_v7a/install/lib + +BUILD_ANDROID_INCLUDE_DIR_arm_v8a=~/workspace/build_android_install_arm_v8a/install/include +BUILD_ANDROID_LIB_DIR_arm_v8a=~/workspace/build_android_install_arm_v8a/install/lib + +PYTORCH_ANDROID_SRC_MAIN_DIR=~/workspace/android/pytorch_android/src/main + +JNI_INCLUDE_DIR=${PYTORCH_ANDROID_SRC_MAIN_DIR}/cpp/libtorch_include +mkdir -p $JNI_INCLUDE_DIR + +JNI_LIBS_DIR=${PYTORCH_ANDROID_SRC_MAIN_DIR}/jniLibs +mkdir -p $JNI_LIBS_DIR + +ln -s ${BUILD_ANDROID_INCLUDE_DIR_x86} ${JNI_INCLUDE_DIR}/x86 +ln -s ${BUILD_ANDROID_LIB_DIR_x86} ${JNI_LIBS_DIR}/x86 + +if [[ "${BUILD_ENVIRONMENT}" != *-gradle-build-only-x86_32* ]]; then +ln -s ${BUILD_ANDROID_INCLUDE_DIR_x86_64} ${JNI_INCLUDE_DIR}/x86_64 +ln -s ${BUILD_ANDROID_LIB_DIR_x86_64} ${JNI_LIBS_DIR}/x86_64 + +ln -s ${BUILD_ANDROID_INCLUDE_DIR_arm_v7a} ${JNI_INCLUDE_DIR}/armeabi-v7a +ln -s ${BUILD_ANDROID_LIB_DIR_arm_v7a} ${JNI_LIBS_DIR}/armeabi-v7a + +ln -s ${BUILD_ANDROID_INCLUDE_DIR_arm_v8a} ${JNI_INCLUDE_DIR}/arm64-v8a +ln -s ${BUILD_ANDROID_LIB_DIR_arm_v8a} ${JNI_LIBS_DIR}/arm64-v8a +fi + +GRADLE_PARAMS="-p android assembleRelease --debug --stacktrace" +if [[ "${BUILD_ENVIRONMENT}" == *-gradle-build-only-x86_32* ]]; then + GRADLE_PARAMS+=" -PABI_FILTERS=x86" +fi + +if [ -n "${GRADLE_OFFLINE:-}" ]; then + GRADLE_PARAMS+=" --offline" +fi + +$GRADLE_PATH $GRADLE_PARAMS + +find . -type f -name "*.a" -exec ls -lh {} \; + +while IFS= read -r -d '' file +do + echo + echo "$file" + ls -lah "$file" + zipinfo -l "$file" +done < <(find . -type f -name '*.aar' -print0) + +find . -type f -name *aar -print | xargs tar cfvz ~/workspace/android/artifacts.tgz diff --git a/scripts/build_host_protoc.sh b/scripts/build_host_protoc.sh new file mode 100755 index 000000000000..cd37db3b3171 --- /dev/null +++ b/scripts/build_host_protoc.sh @@ -0,0 +1,59 @@ +#!/bin/bash +############################################################################## +# Build script to build the protoc compiler for the host platform. +############################################################################## +# This script builds the protoc compiler for the host platform, which is needed +# for any cross-compilation as we will need to convert the protobuf source +# files to cc files. +# +# --other-flags accepts flags that should be passed to cmake. Optional. +# +# After the execution of the file, one should be able to find the host protoc +# binary at build_host_protoc/bin/protoc. + +set -e + +CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)" +BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_host_protoc"} +mkdir -p $BUILD_ROOT/build +cd $BUILD_ROOT/build + +CMAKE_ARGS=() +CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$BUILD_ROOT") +CMAKE_ARGS+=("-Dprotobuf_BUILD_TESTS=OFF") + +# If Ninja is installed, prefer it to Make +if [ -x "$(command -v ninja)" ]; then + CMAKE_ARGS+=("-GNinja") +fi + +while true; do + case "$1" in + --other-flags) + shift; + CMAKE_ARGS+=("$@") + break ;; + "") + break ;; + *) + echo "Unknown option passed as argument: $1" + break ;; + esac +done + +# Use ccache if available (this path is where Homebrew installs ccache symlinks) +if [ "$(uname)" == 'Darwin' ] && [ -d /usr/local/opt/ccache/libexec ]; then + CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/local/opt/ccache/libexec/gcc") + CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=/usr/local/opt/ccache/libexec/g++") +fi + +cmake "$CAFFE2_ROOT/third_party/protobuf/cmake" ${CMAKE_ARGS[@]} + +if [ -z "$MAX_JOBS" ]; then + if [ "$(uname)" == 'Darwin' ]; then + MAX_JOBS=$(sysctl -n hw.ncpu) + else + MAX_JOBS=$(nproc) + fi +fi +cmake --build . -- "-j${MAX_JOBS}" install diff --git a/scripts/build_ios.sh b/scripts/build_ios.sh new file mode 100755 index 000000000000..ad16cb940dcb --- /dev/null +++ b/scripts/build_ios.sh @@ -0,0 +1,155 @@ +#!/bin/bash -xe +############################################################################## +# Example command to build the iOS target. +############################################################################## +# +# This script shows how one can build a Caffe2 binary for the iOS platform +# using ios-cmake. This is very similar to the android-cmake - see +# build_android.sh for more details. + +CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)" + +if [ -z "$PYTHON" ]; then + PYTHON=python + PYTHON_VERSION_MAJOR=$($PYTHON -c 'import sys; print(sys.version_info[0])') + if [ "${PYTHON_VERSION_MAJOR}" -le 2 ]; then + echo "Default python executable is Python-2, trying to use python3 alias" + PYTHON=python3 + fi +fi + +echo "Bash: $(/bin/bash --version | head -1)" +echo "Python: $($PYTHON -c 'import sys; print(sys.version)')" +echo "Caffe2 path: $CAFFE2_ROOT" + +CMAKE_ARGS=() + +# Build PyTorch mobile +CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')") +CMAKE_ARGS+=("-DPython_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')") +CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF") + +# custom build with selected ops +if [ -n "${SELECTED_OP_LIST}" ]; then + SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)" + echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST" + if [ ! -r ${SELECTED_OP_LIST} ]; then + echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found." + exit 1 + fi + CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}") +fi + +# bitcode +if [ "${ENABLE_BITCODE:-}" == '1' ]; then + CMAKE_ARGS+=("-DCMAKE_C_FLAGS=-fembed-bitcode") + CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode") +fi + +# Use ios-cmake to build iOS project from CMake. +# This projects sets CMAKE_C_COMPILER to /usr/bin/gcc and +# CMAKE_CXX_COMPILER to /usr/bin/g++. In order to use ccache (if it is available) we +# must override these variables via CMake arguments. +CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=$CAFFE2_ROOT/cmake/iOS.cmake") +if [ -n "${CCACHE_WRAPPER_PATH:-}"]; then + CCACHE_WRAPPER_PATH=/usr/local/opt/ccache/libexec +fi +if [ -d "$CCACHE_WRAPPER_PATH" ]; then + CMAKE_ARGS+=("-DCMAKE_C_COMPILER=$CCACHE_WRAPPER_PATH/gcc") + CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=$CCACHE_WRAPPER_PATH/g++") +fi + +# IOS_PLATFORM controls type of iOS platform (see ios-cmake) +if [ -n "${IOS_PLATFORM:-}" ]; then + CMAKE_ARGS+=("-DIOS_PLATFORM=${IOS_PLATFORM}") + if [ "${IOS_PLATFORM}" == "WATCHOS" ]; then + # enable bitcode by default for watchos + CMAKE_ARGS+=("-DCMAKE_C_FLAGS=-fembed-bitcode") + CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode") + # disable the QNNPACK + CMAKE_ARGS+=("-DUSE_PYTORCH_QNNPACK=OFF") + fi +else + # IOS_PLATFORM is not set, default to OS, which builds iOS. + CMAKE_ARGS+=("-DIOS_PLATFORM=OS") +fi + +if [ -n "${IOS_ARCH:-}" ]; then + CMAKE_ARGS+=("-DIOS_ARCH=${IOS_ARCH}") +fi + +if [ "${BUILD_LITE_INTERPRETER}" == 0 ]; then + CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF") +else + CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON") +fi +if [ "${TRACING_BASED}" == 1 ]; then + CMAKE_ARGS+=("-DTRACING_BASED=ON") +else + CMAKE_ARGS+=("-DTRACING_BASED=OFF") +fi +if [ "${USE_LIGHTWEIGHT_DISPATCH}" == 1 ]; then + CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=ON") + CMAKE_ARGS+=("-DSTATIC_DISPATCH_BACKEND=CPU") +else + CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=OFF") +fi + +CMAKE_ARGS+=("-DUSE_LITE_INTERPRETER_PROFILER=OFF") + +# Don't build binaries or tests (only the library) +CMAKE_ARGS+=("-DBUILD_TEST=OFF") +CMAKE_ARGS+=("-DBUILD_BINARY=OFF") +CMAKE_ARGS+=("-DBUILD_PYTHON=OFF") + +# Disable unused dependencies +CMAKE_ARGS+=("-DUSE_CUDA=OFF") +CMAKE_ARGS+=("-DUSE_ITT=OFF") +CMAKE_ARGS+=("-DUSE_GFLAGS=OFF") +CMAKE_ARGS+=("-DUSE_OPENCV=OFF") +CMAKE_ARGS+=("-DUSE_MPI=OFF") +CMAKE_ARGS+=("-DUSE_NUMPY=OFF") +CMAKE_ARGS+=("-DUSE_NNPACK=OFF") +CMAKE_ARGS+=("-DUSE_MKLDNN=OFF") + +# Metal +if [ "${USE_PYTORCH_METAL:-}" == "1" ]; then + CMAKE_ARGS+=("-DUSE_PYTORCH_METAL=ON") +fi + +# Core ML +if [ "${USE_COREML_DELEGATE}" == "1" ]; then + CMAKE_ARGS+=("-DUSE_COREML_DELEGATE=ON") +fi + +# pthreads +CMAKE_ARGS+=("-DCMAKE_THREAD_LIBS_INIT=-lpthread") +CMAKE_ARGS+=("-DCMAKE_HAVE_THREADS_LIBRARY=1") +CMAKE_ARGS+=("-DCMAKE_USE_PTHREADS_INIT=1") + +# Only toggle if VERBOSE=1 +if [ "${VERBOSE:-}" == '1' ]; then + CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1") +fi + +# enable ARC +CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fobjc-arc") + +# Now, actually build the iOS target. +BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_ios"} +INSTALL_PREFIX=${BUILD_ROOT}/install +mkdir -p $BUILD_ROOT +cd $BUILD_ROOT +cmake "$CAFFE2_ROOT" \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DBUILD_SHARED_LIBS=OFF \ + ${CMAKE_ARGS[@]} \ + $@ + +cmake --build . -- "-j$(sysctl -n hw.ncpu)" + +# copy headers and libs to install directory +echo "Will install headers and libs to $INSTALL_PREFIX for further Xcode project usage." +make install +echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your Xcode project directory." diff --git a/scripts/build_local.sh b/scripts/build_local.sh new file mode 100755 index 000000000000..b84367150125 --- /dev/null +++ b/scripts/build_local.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# +############################################################################## +# Example command to build Caffe2 +############################################################################## +# + +set -ex + +CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)" + +CMAKE_ARGS=() + +# If Ninja is installed, prefer it to Make +if [ -x "$(command -v ninja)" ]; then + CMAKE_ARGS+=("-GNinja") +fi + +# Use ccache if available (this path is where Homebrew installs ccache symlinks) +if [ "$(uname)" == 'Darwin' ]; then + if [ -n "${CCACHE_WRAPPER_PATH:-}"]; then + CCACHE_WRAPPER_PATH=/usr/local/opt/ccache/libexec + fi + if [ -d "$CCACHE_WRAPPER_PATH" ]; then + CMAKE_ARGS+=("-DCMAKE_C_COMPILER=$CCACHE_WRAPPER_PATH/gcc") + CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=$CCACHE_WRAPPER_PATH/g++") + fi +fi + +# Use special install script with Anaconda +if [ -n "${USE_ANACONDA}" ]; then + export SKIP_CONDA_TESTS=1 + export CONDA_INSTALL_LOCALLY=1 + "${ROOT_DIR}/scripts/build_anaconda.sh" "$@" +else + # Make sure that pyyaml is installed for the codegen of building Aten to work + if [[ -n "$(python -c 'import yaml' 2>&1)" ]]; then + echo "Installing pyyaml with pip at $(which pip)" + pip install --user pyyaml + fi + + # Make sure that typing is installed for the codegen of building Aten to work + if [[ -n "$(python -c 'import typing' 2>&1)" ]]; then + echo "Installing typing with pip at $(which pip)" + pip install --user typing + fi + + # Build protobuf compiler from third_party if configured to do so + if [ -n "${USE_HOST_PROTOC:-}" ]; then + echo "USE_HOST_PROTOC is set; building protoc before building Caffe2..." + "$CAFFE2_ROOT/scripts/build_host_protoc.sh" + CUSTOM_PROTOC_EXECUTABLE="$CAFFE2_ROOT/build_host_protoc/bin/protoc" + echo "Built protoc $("$CUSTOM_PROTOC_EXECUTABLE" --version)" + CMAKE_ARGS+=("-DCAFFE2_CUSTOM_PROTOC_EXECUTABLE=$CUSTOM_PROTOC_EXECUTABLE") + fi + + # We are going to build the target into build. + BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"} + mkdir -p "$BUILD_ROOT" + cd "$BUILD_ROOT" + echo "Building Caffe2 in: $BUILD_ROOT" + + cmake "$CAFFE2_ROOT" \ + -DCMAKE_BUILD_TYPE=Release \ + "${CMAKE_ARGS[@]}" \ + "$@" + + # Determine the number of CPUs to build with. + # If the `CAFFE_MAKE_NCPUS` variable is not specified, use them all. + if [ -n "${MAX_JOBS}" ]; then + CAFFE_MAKE_NCPUS="$MAX_JOBS" + elif [ -n "${CAFFE_MAKE_NCPUS}" ]; then + CAFFE_MAKE_NCPUS="$CAFFE_MAKE_NCPUS" + elif [ "$(uname)" == 'Darwin' ]; then + CAFFE_MAKE_NCPUS="$(sysctl -n hw.ncpu)" + else + CAFFE_MAKE_NCPUS="$(nproc)" + fi + + # Now, actually build the target. + cmake --build . -- "-j$CAFFE_MAKE_NCPUS" +fi diff --git a/scripts/build_mobile.sh b/scripts/build_mobile.sh new file mode 100755 index 000000000000..7b1995a61ebc --- /dev/null +++ b/scripts/build_mobile.sh @@ -0,0 +1,107 @@ +#!/bin/bash +############################################################################## +# Example command to build the mobile target. +############################################################################## +# +# This script shows how one can build a libtorch library optimized for mobile +# devices using host toolchain. + +set -e + +export BUILD_PYTORCH_MOBILE_WITH_HOST_TOOLCHAIN=1 +CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)" + +echo "Bash: $(/bin/bash --version | head -1)" +echo "Caffe2 path: $CAFFE2_ROOT" + +CMAKE_ARGS=() +CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')") +CMAKE_ARGS+=("-DPython_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')") +CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF") +CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF") + +# custom build with selected ops +if [ -n "${SELECTED_OP_LIST}" ]; then + SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)" + echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST" + if [ ! -r ${SELECTED_OP_LIST} ]; then + echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found." + exit 1 + fi + CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}") +fi + +# If Ninja is installed, prefer it to Make +if [ -x "$(command -v ninja)" ]; then + CMAKE_ARGS+=("-GNinja") +fi + +# Don't build artifacts we don't need +CMAKE_ARGS+=("-DBUILD_TEST=OFF") +CMAKE_ARGS+=("-DBUILD_BINARY=OFF") + +# If there exists env variable and it equals to 1, build lite interpreter. +# Default behavior is to build full jit interpreter. +# cmd: BUILD_LITE_INTERPRETER=1 ./scripts/build_mobile.sh +if [ "x${BUILD_LITE_INTERPRETER}" == "x1" ]; then + CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON") +else + CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF") +fi +if [ "x${TRACING_BASED}" == "x1" ]; then + CMAKE_ARGS+=("-DTRACING_BASED=ON") +else + CMAKE_ARGS+=("-DTRACING_BASED=OFF") +fi + +# Lightweight dispatch bypasses the PyTorch Dispatcher. +if [ "${USE_LIGHTWEIGHT_DISPATCH}" == 1 ]; then + CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=ON") + CMAKE_ARGS+=("-DSTATIC_DISPATCH_BACKEND=CPU") +else + CMAKE_ARGS+=("-DUSE_LIGHTWEIGHT_DISPATCH=OFF") +fi + +# Disable unused dependencies +CMAKE_ARGS+=("-DUSE_ROCM=OFF") +CMAKE_ARGS+=("-DUSE_CUDA=OFF") +CMAKE_ARGS+=("-DUSE_ITT=OFF") +CMAKE_ARGS+=("-DUSE_GFLAGS=OFF") +CMAKE_ARGS+=("-DUSE_OPENCV=OFF") +CMAKE_ARGS+=("-DUSE_MPI=OFF") +CMAKE_ARGS+=("-DUSE_OPENMP=OFF") +CMAKE_ARGS+=("-DUSE_MKLDNN=OFF") +CMAKE_ARGS+=("-DUSE_NNPACK=OFF") +CMAKE_ARGS+=("-DUSE_NUMPY=OFF") +CMAKE_ARGS+=("-DUSE_BLAS=OFF") + +# Only toggle if VERBOSE=1 +if [ "${VERBOSE:-}" == '1' ]; then + CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1") +fi + +# Use-specified CMake arguments go last to allow overriding defaults +CMAKE_ARGS+=("$@") + +# Now, actually build the Android target. +BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_mobile"} +INSTALL_PREFIX=${BUILD_ROOT}/install +mkdir -p $BUILD_ROOT +cd $BUILD_ROOT +cmake "$CAFFE2_ROOT" \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \ + -DCMAKE_BUILD_TYPE=Release \ + "${CMAKE_ARGS[@]}" + +# Cross-platform parallel build +if [ -z "$MAX_JOBS" ]; then + if [ "$(uname)" == 'Darwin' ]; then + MAX_JOBS=$(sysctl -n hw.ncpu) + else + MAX_JOBS=$(nproc) + fi +fi + +echo "Will install headers and libs to $INSTALL_PREFIX for further project usage." +cmake --build . --target install -- "-j${MAX_JOBS}" +echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your project directory." diff --git a/scripts/build_pytorch_android.sh b/scripts/build_pytorch_android.sh new file mode 100755 index 000000000000..7b80965e34b5 --- /dev/null +++ b/scripts/build_pytorch_android.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -eux + +############################################################################## +# Master script to build PyTorch Android library with Java bindings. +############################################################################## +# Example usage: +# - Build default AARs: +# scripts/build_pytorch_android.sh +# +# - Build for specific ABI(s): +# scripts/build_pytorch_android.sh armeabi-v7a +# scripts/build_pytorch_android.sh arm64-v8a,x86,x86_64 +# +# Script's workflow: +# 1. Builds libtorch for android for specified android abisi (by default for all 4). +# Custom list of android abis can be specified as a bash argument as comma separated list. +# For example just for testing on android x86 emulator we need only x86 build. +# ./scripts/build_pytorch_android.sh x86 +# 2. Creates symbolic links to android/pytorch_android/src/main/jniLibs/${abi} for libtorch build output, +# android/pytorch_android/src/main/cpp/libtorch_include/${abi} for headers. +# 3. Runs pyotrch_android gradle build: +# gradle assembleRelease + +PYTORCH_DIR="$(cd $(dirname $0)/..; pwd -P)" +PYTORCH_ANDROID_DIR=$PYTORCH_DIR/android + +echo "PYTORCH_DIR:$PYTORCH_DIR" + +source "$PYTORCH_ANDROID_DIR/common.sh" + +check_android_sdk +check_gradle +parse_abis_list "$@" +build_android + +# To set proxy for gradle add following lines to ./gradle/gradle.properties: +# systemProp.http.proxyHost=... +# systemProp.http.proxyPort=8080 +# systemProp.https.proxyHost=... +# systemProp.https.proxyPort=8080 + +if [ "$CUSTOM_ABIS_LIST" = true ]; then + # Skipping clean task here as android gradle plugin 3.3.2 exteralNativeBuild has problems + # with it when abiFilters are specified. + $GRADLE_PATH -PABI_FILTERS=$ABIS_LIST -p $PYTORCH_ANDROID_DIR assembleRelease +else + $GRADLE_PATH -p $PYTORCH_ANDROID_DIR clean assembleRelease +fi + +find $PYTORCH_ANDROID_DIR -type f -name *aar | xargs ls -lah diff --git a/scripts/build_raspbian.sh b/scripts/build_raspbian.sh new file mode 100755 index 000000000000..b1fe85926219 --- /dev/null +++ b/scripts/build_raspbian.sh @@ -0,0 +1,44 @@ +#!/bin/bash +############################################################################## +# Example command to build the Raspbian target. +############################################################################## +# +# This script shows how one can build a Caffe2 binary for raspbian. The build +# is essentially much similar to a host build, with one additional change +# which is to specify -mfpu=neon for optimized speed. + +CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)" +echo "Caffe2 codebase root is: $CAFFE2_ROOT" +BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"} +mkdir -p $BUILD_ROOT +echo "Build Caffe2 raspbian into: $BUILD_ROOT" + +# obtain dependencies. +echo "Installing dependencies." +sudo apt-get install \ + cmake \ + libgflags-dev \ + libgoogle-glog-dev \ + libprotobuf-dev \ + libpython-dev \ + python-pip \ + python-numpy \ + protobuf-compiler \ + python-protobuf +# python dependencies +sudo pip install hypothesis + +# Now, actually build the raspbian target. +echo "Building caffe2" +cd $BUILD_ROOT + +# Note: you can add more dependencies above if you need libraries such as +# leveldb, lmdb, etc. +cmake "$CAFFE2_ROOT" \ + -DCMAKE_VERBOSE_MAKEFILE=1 \ + -DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=hard" \ + || exit 1 + +# Note: while Raspberry pi has 4 cores, running too many builds in parallel may +# cause out of memory errors so we will simply run -j 2 only. +make -j 2 || exit 1 diff --git a/scripts/build_tegra_x1.sh b/scripts/build_tegra_x1.sh new file mode 100755 index 000000000000..063e17dfe351 --- /dev/null +++ b/scripts/build_tegra_x1.sh @@ -0,0 +1,51 @@ +#!/bin/bash +############################################################################## +# Example command to build Caffe2 on Tegra X1. +############################################################################## +# +# This script shows how one can build a Caffe2 binary for NVidia's TX1. +# The build script assumes that you have the most recent libraries installed +# via the JetPack toolkit available at +# https://developer.nvidia.com/embedded/jetpack +# and it assumes that we are starting from a fresh system after the jetpack +# installation. If you have already installed some of the dependencies, you +# may be able to skip quite a few of the apt-get installs. + +CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)" +echo "Caffe2 codebase root is: $CAFFE2_ROOT" +BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"} +mkdir -p $BUILD_ROOT +echo "Build Caffe2 raspbian into: $BUILD_ROOT" + +# obtain necessary dependencies +echo "Installing dependencies." +sudo apt-get install \ + cmake \ + libgflags-dev \ + libgoogle-glog-dev \ + libprotobuf-dev \ + protobuf-compiler + +# obtain optional dependencies that are usually useful to have. +echo "Installing optional dependencies." +sudo apt-get install \ + libpython-dev \ + python-numpy \ + python-pip \ + python-protobuf + +# Obtain python hypothesis, which Caffe2 uses for unit testing. Note that +# the one provided by apt-get is quite old so we install it via pip +sudo pip install hypothesis + +# Now, actually build the android target. +echo "Building caffe2" +cd $BUILD_ROOT + +# CUDA_USE_STATIC_CUDA_RUNTIME needs to be set to off so that opencv can be +# properly used. Otherwise, opencv will complain that opencv_dep_cudart cannot +# be found. +cmake "$CAFFE2_ROOT" -DCUDA_USE_STATIC_CUDA_RUNTIME=OFF \ + || exit 1 + +make -j 4 || exit 1 diff --git a/scripts/build_tizen.sh b/scripts/build_tizen.sh new file mode 100755 index 000000000000..2262a2503c1d --- /dev/null +++ b/scripts/build_tizen.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +############################################################################## +# Example command to build the Tizen target (RPi3). +############################################################################## +# +# This script shows how one can build a Caffe2 binary for a Tizen device (RPi3). +# The build is essentially much similar to a host build, with one additional change +# which is to specify -mfpu=neon for optimized speed. + +setup_environment(){ +# The rootfs image for a Tizen target (RPi3)is located at the below webpage: +# https://cdn.download.tizen.org/archive/releases/milestone/tizen/4.0.m1/tizen-unified_20170529.1/images/ +# If you do not have a Tizen device, Please, run qemu-arm-static and chroot command. +# $ sudo chroot ~/tizen-rootfs qemu-arm-static /usr/bin/bash + +CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)" +echo "Caffe2 codebase root is: $CAFFE2_ROOT" +BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"} +mkdir -p $BUILD_ROOT +echo "Build Caffe2 Tizen into: $BUILD_ROOT" +} + +caffe2_lite_dep_packages(){ +# Obtain necessary dependencies +# You can set-up a rpm repository with zypper, yum, and dnf because Tizen +# software platform officially support rpm format such as Fedora, OpenSUSE. +# The official Tizen repository is as following: +# https://cdn.download.tizen.org/archive/releases/milestone/tizen/4.0.m1/ +echo "Installing dependencies." +sudo zypper install \ + make \ + strace \ + cmake \ + gcc* \ + binutils \ + glibc* \ + cpp \ + protobuf-devel \ + libstdc++* +} + +caffe2_lite_build(){ +# Now, actually build the android target. +echo "Building caffe2" +cd $BUILD_ROOT + +# Note: add more dependencies above if you need libraries such as leveldb, lmdb, etc. +# If you have to disable a specific package due to a package absence +# from https://git.tizen.org/cgit/, append -Dxxx_xxx=OFF option before executing cmake. +cmake .. \ + -DCMAKE_VERBOSE_MAKEFILE=1 \ + -DUSE_GFLAGS=OFF \ + -DUSE_GLOG=OFF -DUSE_NNPACK=OFF \ + -DRUN_HAVE_STD_REGEX=0 \ + -DRUN_HAVE_POSIX_REGEX=0 \ + -DHAVE_GNU_POSIX_REGEX=0 \ + -DUSE_MPI=OFF -DUSE_OPENMP=OFF \ + -DBUILD_PYTHON=OFF \ + -DUSE_GLOO=OFF \ + -DUSE_OPENCV=OFF \ + -DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \ + || exit 1 + +make -j`nproc` || exit 1 +} + +caffe2_full_dep_packages(){ +# Obtain necessary dependencies +# You can set-up a rpm repository with zypper, yum, and dnf because Tizen +# software platform officially support rpm format such as Fedora, OpenSUSE. +# The official Tizen repository is as following: +# https://cdn.download.tizen.org/archive/releases/milestone/tizen/4.0.m1/ +echo "Installing dependencies." +sudo zypper install \ + cmake \ + libgflags-dev \ + libgoogle-glog-dev \ + libprotobuf-dev \ + protobuf-compiler + +# Obtain optional dependencies that are usually useful to have. +echo "Installing optional dependencies." +sudo zypper install \ + libpython-dev \ + python-numpy \ + python-pip \ + python-protobuf + +# Obtain python hypothesis, which Caffe2 uses for unit testing. Note that +# the one provided by zypper is quite old so we install it via pip +sudo pip install hypothesis +} + +caffe2_full_build(){ +# Now, actually build the android target. +echo "Building caffe2" +cd $BUILD_ROOT + +# Note: add more dependencies above if you need libraries such as leveldb, lmdb, etc. +# If you have to disable a specific package due to a package absence +# from https://git.tizen.org/cgit/, append -Dxxx_xxx=OFF option before executing cmake. +cmake "$CAFFE2_ROOT" \ + -DCMAKE_VERBOSE_MAKEFILE=1 \ + -DUSE_CUDA=OFF \ + -DUSE_ITT=OFF \ + -DUSE_OPENCV=OFF \ + -DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \ + || exit 1 + +make -j`nproc` || exit 1 +} + +#### Main +# Setup a build environment to compile Caffe2 deeplearning framework in Tizen platform. +setup_environment +# There are two build options to support 'full' version and 'lite' version (by default). +caffe2_lite_dep_packages +caffe2_lite_build diff --git a/scripts/build_windows.bat b/scripts/build_windows.bat new file mode 100644 index 000000000000..60bfebad08c0 --- /dev/null +++ b/scripts/build_windows.bat @@ -0,0 +1,80 @@ +:: ############################################################################# +:: Example command to build on Windows. +:: ############################################################################# + +:: This script shows how one can build a Caffe2 binary for windows. + +@echo off +setlocal + +SET ORIGINAL_DIR=%cd% +SET CAFFE2_ROOT=%~dp0%.. + +if NOT DEFINED BUILD_BINARY ( + set BUILD_BINARY=OFF +) + +if NOT DEFINED BUILD_SHARED_LIBS ( + :: On CI, we test with BUILD_SHARED_LIBS=OFF. + :: By default, it will be BUILD_SHARED_LIBS=ON. + if NOT DEFINED BUILD_ENVIRONMENT ( + set BUILD_SHARED_LIBS=OFF + ) +) + +if NOT DEFINED CAFFE2_STATIC_LINK_CUDA ( + set CAFFE2_STATIC_LINK_CUDA=OFF +) + +if NOT DEFINED CMAKE_BUILD_TYPE ( + set CMAKE_BUILD_TYPE=Release +) + +if NOT DEFINED ONNX_NAMESPACE ( + set ONNX_NAMESPACE=onnx_c2 +) + +if NOT DEFINED TORCH_CUDA_ARCH_LIST ( + set TORCH_CUDA_ARCH_LIST=5.0 +) + +if NOT DEFINED USE_CUDA ( + set USE_CUDA=OFF +) + +if NOT DEFINED USE_OBSERVERS ( + set USE_OBSERVERS=OFF +) + +if NOT DEFINED MSVC_Z7_OVERRIDE ( + set MSVC_Z7_OVERRIDE=OFF +) + +if NOT DEFINED CMAKE_GENERATOR ( + set CMAKE_GENERATOR=Ninja +) + +set CMAKE_VERBOSE_MAKEFILE=1 + +:: Install pyyaml for Aten codegen +pip install pyyaml ninja + +echo CAFFE2_ROOT=%CAFFE2_ROOT% +echo CMAKE_GENERATOR=%CMAKE_GENERATOR% +echo CMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% + +:: Set up cmake. We will skip building the test files right now. +pushd %CAFFE2_ROOT% +python tools\build_libtorch.py || goto :label_error +popd + +echo "Caffe2 built successfully" +cd %ORIGINAL_DIR% +endlocal +exit /b 0 + +:label_error +echo "Caffe2 building failed" +cd %ORIGINAL_DIR% +endlocal +exit /b 1 diff --git a/scripts/diagnose_protobuf.py b/scripts/diagnose_protobuf.py new file mode 100644 index 000000000000..65af4618228d --- /dev/null +++ b/scripts/diagnose_protobuf.py @@ -0,0 +1,92 @@ +## @package diagnose_protobuf +# Module scripts.diagnose_protobuf +"""Diagnoses the current protobuf situation. + +Protocol buffer needs to be properly installed for Caffe2 to work, and +sometimes it is rather tricky. Specifically, we will need to have a +consistent version between C++ and python simultaneously. This is a +convenience script for one to quickly check if this is so on one's local +machine. + +Usage: + [set your environmental variables like PATH and PYTHONPATH] + python scripts/diagnose_protobuf.py +""" + +import os +import re +from subprocess import PIPE, Popen + + +# Get python protobuf version. +try: + import google.protobuf + + python_version = google.protobuf.__version__ + python_protobuf_installed = True +except ImportError: + print("DEBUG: cannot find python protobuf install.") + python_protobuf_installed = False + +if os.name == "nt": + protoc_name = "protoc.exe" +else: + protoc_name = "protoc" + +try: + p = Popen([protoc_name, "--version"], stdout=PIPE, stderr=PIPE) + out, err = p.communicate() +except: + print("DEBUG: did not find protoc binary.") + print("DEBUG: out: " + out) + print("DEBUG: err: " + err) + native_protobuf_installed = False +else: + if p.returncode: + print("DEBUG: protoc returned a non-zero return code.") + print("DEBUG: out: " + out) + print("DEBUG: err: " + err) + native_protobuf_installed = False + else: + tmp = re.search(r"\d\.\d\.\d", out) + if tmp: + native_version = tmp.group(0) + native_protobuf_installed = True + else: + print("DEBUG: cannot parse protoc version string.") + print("DEBUG: out: " + out) + native_protobuf_installed = False + +PYTHON_PROTOBUF_NOT_INSTALLED = """ +You have not installed python protobuf. Protobuf is needed to run caffe2. You +can install protobuf via pip or conda (if you are using anaconda python). +""" + +NATIVE_PROTOBUF_NOT_INSTALLED = """ +You have not installed the protoc binary. Protoc is needed to compile Caffe2 +protobuf source files. Depending on the platform you are on, you can install +protobuf via: + (1) Mac: using homebrew and do brew install protobuf. + (2) Linux: use apt and do apt-get install libprotobuf-dev + (3) Windows: install from source, or from the releases here: + https://github.com/google/protobuf/releases/ +""" + +VERSION_MISMATCH = f""" +Your python protobuf is of version {python_version} but your native protoc version is of +version {native_version}. This will cause the installation to produce incompatible +protobuf files. This is bad in general - consider installing the same version. +""" + +# Now, give actual recommendations +if not python_protobuf_installed: + print(PYTHON_PROTOBUF_NOT_INSTALLED) + +if not native_protobuf_installed: + print(NATIVE_PROTOBUF_NOT_INSTALLED) + +if python_protobuf_installed and native_protobuf_installed: + if python_version != native_version: + print(VERSION_MISMATCH) + else: + print("All looks good.") diff --git a/scripts/fbcode-dev-setup/ccache_setup.sh b/scripts/fbcode-dev-setup/ccache_setup.sh new file mode 100755 index 000000000000..cb461bee2dd2 --- /dev/null +++ b/scripts/fbcode-dev-setup/ccache_setup.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# This script installs CCache with CUDA support. +# Example usage: +# ./ccache_setup.sh --path /installed/folder + +set -e +shopt -s expand_aliases + +# Setup the proxy +alias with_proxy="HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 FTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 ftp_proxy=http://fwdproxy:8080 http_no_proxy='*.facebook.com|*.tfbnw.net|*.fb.com'" + +# Parse options +path="$HOME/ccache" +force=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --path) + shift + path="$1" + path=$(realpath "$path") + ;; + --force) # Force install + force=true + ;; + --help) + echo 'usage: ./ccache_setup.py --path /installed/folder [--force]' + exit 0 + ;; + *) + echo "Invalid option: $1" + exit 1 + ;; + esac + shift +done + +# Check whether you put nvcc in PATH +set +e +nvcc_path=$(which nvcc) +if [[ -z "$nvcc_path" ]]; then + nvcc_path="/usr/local/cuda/bin/nvcc" + export PATH="/usr/local/cuda/bin:$PATH" +fi +set -e +if [ ! -f "$nvcc_path" ] && ! $force; then + # shellcheck disable=SC2016 + echo 'nvcc is not detected in $PATH' + exit 1 +fi +echo "nvcc is detected at $nvcc_path" + +if [ -f "$CUDA_NVCC_EXECUTABLE" ] && [[ "$CUDA_NVCC_EXECUTABLE" == *"ccache"* ]]; then # Heuristic rule + if $CUDA_NVCC_EXECUTABLE --version; then + if ! $force; then + echo "CCache with nvcc support is already installed at $CUDA_NVCC_EXECUTABLE, please add --force" + exit 0 + fi + fi +fi + +# Installing CCache +echo "CCache will be installed at $path" +if [ -e "$path" ]; then + mv --backup=t -T "$path" "${path}.old" +fi + +with_proxy git clone https://github.com/colesbury/ccache.git "$path" -b ccbin +cd "$path" +./autogen.sh +./configure +make install prefix="$path" + +mkdir -p "$path/lib" +mkdir -p "$path/cuda" +ln -sf "$path/bin/ccache" "$path/lib/cc" +ln -sf "$path/bin/ccache" "$path/lib/c++" +ln -sf "$path/bin/ccache" "$path/lib/gcc" +ln -sf "$path/bin/ccache" "$path/lib/g++" +ln -sf "$path/bin/ccache" "$path/cuda/nvcc" +"$path/bin/ccache" -M 25Gi + +# Make sure the nvcc wrapped in CCache is runnable +"$path/cuda/nvcc" --version +echo 'Congrats! The CCache with nvcc support is installed!' +echo -e "Please add the following lines to your bash init script:\\n" +echo "################ Env Var for CCache with CUDA support ################" +# shellcheck disable=SC2016 +echo 'export PATH="'"$path"'/lib:$PATH"' +echo 'export CUDA_NVCC_EXECUTABLE="'"$path"'/cuda/nvcc"' +echo '######################################################################' diff --git a/scripts/get_python_cmake_flags.py b/scripts/get_python_cmake_flags.py new file mode 100644 index 000000000000..a49debcc884a --- /dev/null +++ b/scripts/get_python_cmake_flags.py @@ -0,0 +1,24 @@ +## @package get_python_cmake_flags +# Module scripts.get_python_cmake_flags +############################################################################## +# Use this script to find your preferred python installation. +############################################################################## +# +# You can use the following to build with your preferred version of python +# if your installation is not being properly detected by CMake. +# +# mkdir -p build && cd build +# cmake $(python ../scripts/get_python_cmake_flags.py) .. +# make +# + + +import sys +import sysconfig + + +flags = [ + f"-DPython_EXECUTABLE:FILEPATH={sys.executable}", +] + +print(" ".join(flags), end="") diff --git a/scripts/proto.ps1 b/scripts/proto.ps1 new file mode 100644 index 000000000000..a6bce82ff682 --- /dev/null +++ b/scripts/proto.ps1 @@ -0,0 +1,18 @@ +param( + [string]$protoc, + [string]$srcdir, + [string]$unprocessed, + [string]$processed, + [string]$out +) +$ErrorActionPreference = "Stop" +Get-Content $unprocessed | % {$_ -Replace "caffe2/proto/caffe2.proto", "caffe2.proto"} | Set-Content $processed +Add-Content -Path $processed -Value "option optimize_for = LITE_RUNTIME;`n" -NoNewline +$dir = (Get-Item $processed).DirectoryName + +copy $srcdir/caffe2/proto/caffe2.proto $srcdir/caffe2.proto +Add-Content -Path $srcdir/caffe2.proto -Value "option optimize_for = LITE_RUNTIME;`n" -NoNewline + +$processed = (Get-Item $processed).Name +$cmd = "$protoc -I${dir} --cpp_out=$out $processed" +Invoke-Expression $cmd diff --git a/scripts/remove_apache_header.sh b/scripts/remove_apache_header.sh new file mode 100755 index 000000000000..97980bfbb0ef --- /dev/null +++ b/scripts/remove_apache_header.sh @@ -0,0 +1,13 @@ +if [[ "$1" == *.py ]]; then + apache_header="apache_python.txt" +else + apache_header="apache_header.txt" +fi +apache_lines=$(wc -l < "${apache_header}") +apache_md5=$(cat "${apache_header}" | md5) +header_md5=$(head -n ${apache_lines} $1 | md5) +if [ "${header_md5}" == "${apache_md5}" ]; then + keep_lines=$(($(wc -l < $1) - ${apache_lines})) + tail -n ${keep_lines} $1 > _remove_apache_header.txt + mv _remove_apache_header.txt $1 +fi diff --git a/scripts/temp.sh b/scripts/temp.sh new file mode 100755 index 000000000000..18eb2b473381 --- /dev/null +++ b/scripts/temp.sh @@ -0,0 +1,7 @@ +find ../caffe2 -name "*.py" -exec ./remove_apache_header.sh {} \; +find ../caffe2 -name "*.h" -exec ./remove_apache_header.sh {} \; +find ../caffe2 -name "*.cc" -exec ./remove_apache_header.sh {} \; +find ../caffe2 -name "*.cpp" -exec ./remove_apache_header.sh {} \; +find ../caffe2 -name "*.cu" -exec ./remove_apache_header.sh {} \; +find ../caffe2 -name "*.mm" -exec ./remove_apache_header.sh {} \; +find ../caffe2 -name "*.m" -exec ./remove_apache_header.sh {} \; diff --git a/scripts/xcode_build.rb b/scripts/xcode_build.rb new file mode 100644 index 000000000000..0734167bdda1 --- /dev/null +++ b/scripts/xcode_build.rb @@ -0,0 +1,76 @@ +require 'optparse' +require 'xcodeproj' + +options = {} +option_parser = OptionParser.new do |opts| + opts.banner = 'Tools for building PyTorch iOS framework on MacOS' + opts.on('-i', '--install_path ', 'path to the cmake install folder') { |value| + options[:install] = value + } + opts.on('-x', '--xcodeproj_path ', 'path to the XCode project file') { |value| + options[:xcodeproj] = value + } + opts.on('-p', '--platform ', 'platform for the current build, OS or SIMULATOR') { |value| + options[:platform] = value + } +end.parse! +puts options.inspect + +install_path = File.expand_path(options[:install]) +if not Dir.exist? (install_path) + raise "path don't exist:#{install_path}!" +end +xcodeproj_path = File.expand_path(options[:xcodeproj]) +if not File.exist? (xcodeproj_path) + raise "path don't exist:#{xcodeproj_path}!" +end + +project = Xcodeproj::Project.open(xcodeproj_path) +target = project.targets.first #TestApp +header_search_path = ['$(inherited)', "#{install_path}/include"] +libraries_search_path = ['$(inherited)', "#{install_path}/lib"] +other_linker_flags = ['$(inherited)', "-all_load"] + +target.build_configurations.each do |config| + config.build_settings['HEADER_SEARCH_PATHS'] = header_search_path + config.build_settings['LIBRARY_SEARCH_PATHS'] = libraries_search_path + config.build_settings['OTHER_LDFLAGS'] = other_linker_flags + config.build_settings['ENABLE_BITCODE'] = 'No' +end + +# link static libraries +target.frameworks_build_phases.clear +libs = ['libc10.a', 'libclog.a', 'libpthreadpool.a', 'libXNNPACK.a', 'libmicrokernels-prod.a', 'libeigen_blas.a', 'libcpuinfo.a', 'libpytorch_qnnpack.a', 'libtorch_cpu.a', 'libtorch.a', 'libkineto.a'] +for lib in libs do + path = "#{install_path}/lib/#{lib}" + if File.exist?(path) + libref = project.frameworks_group.new_file(path) + target.frameworks_build_phases.add_file_reference(libref) + end +end +# link system frameworks +frameworks = ['CoreML', 'Metal', 'MetalPerformanceShaders', 'Accelerate', 'UIKit'] +if frameworks + frameworks.each do |framework| + path = "System/Library/Frameworks/#{framework}.framework" + framework_ref = project.frameworks_group.new_reference(path) + framework_ref.name = "#{framework}.framework" + framework_ref.source_tree = 'SDKROOT' + target.frameworks_build_phases.add_file_reference(framework_ref) + end +end +project.save + +sdk = nil +arch = nil +if options[:platform] == 'SIMULATOR' + sdk = 'iphonesimulator' + arch = 'arm64' +elsif options[:platform] == 'OS' + sdk = 'iphoneos' + arch = 'arm64' +else + raise "unsupported platform #{options[:platform]}" +end + +exec "xcodebuild clean build -project #{xcodeproj_path} -alltargets -sdk #{sdk} -configuration Release -arch #{arch}"