mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
get rid of protobuf dependencies (#25650)
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/25650 This PR removes protobuf dependencies from mobile build altogether: - caffe2/proto: protobuf files, including caffe2.proto and torch.proto; - caffe2 components that depend on caffe2.proto, including most part of caffe2/core, caffe2/utils; - libprotobuf / libprotobuf-lite dependencies; - protobuf compiler; - some utils class, e.g.: netdef_converter.cpp; - introduce a macro to disable third_party/onnx which depends on protobuf; Test Plan: - builds; - link with demo app to make sure it can load and run a model in pickle format; Differential Revision: D17183548 Pulled By: ljk53 fbshipit-source-id: fe60b48674f29c4a9b58fd1cf8ece44191491531
This commit is contained in:
Jiakai Liu
committed by
Facebook Github Bot
Facebook Github Bot
parent
9d2d31e626
commit
67c530851c
@ -300,6 +300,7 @@ if (INTERN_BUILD_MOBILE AND NOT BUILD_CAFFE2_MOBILE)
|
||||
set(NO_API ON)
|
||||
set(USE_FBGEMM OFF)
|
||||
set(USE_STATIC_DISPATCH ON)
|
||||
set(INTERN_DISABLE_ONNX ON)
|
||||
endif()
|
||||
|
||||
# ---[ Utils
|
||||
|
||||
@ -59,7 +59,6 @@ endif()
|
||||
# addressed yet.
|
||||
|
||||
add_subdirectory(core)
|
||||
add_subdirectory(proto)
|
||||
add_subdirectory(serialize)
|
||||
add_subdirectory(utils)
|
||||
add_subdirectory(perfkernels)
|
||||
@ -95,6 +94,7 @@ if (NOT INTERN_BUILD_MOBILE OR BUILD_CAFFE2_MOBILE)
|
||||
endif()
|
||||
endif()
|
||||
add_subdirectory(opt)
|
||||
add_subdirectory(proto)
|
||||
add_subdirectory(python)
|
||||
add_subdirectory(queue)
|
||||
add_subdirectory(sgd)
|
||||
@ -170,43 +170,44 @@ if (FALSE)
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
# ---[ List of libraries to link with
|
||||
add_library(caffe2_protos STATIC $<TARGET_OBJECTS:Caffe2_PROTO>)
|
||||
add_dependencies(caffe2_protos Caffe2_PROTO)
|
||||
# If we are going to link protobuf locally inside caffe2 libraries, what we will do is
|
||||
# to create a helper static library that always contains libprotobuf source files, and
|
||||
# link the caffe2 related dependent libraries to it.
|
||||
target_include_directories(caffe2_protos INTERFACE $<INSTALL_INTERFACE:include>)
|
||||
# Reason for this public dependency is as follows:
|
||||
# (1) Strictly speaking, we should not expose any Protobuf related functions. We should
|
||||
# only use function interfaces wrapped with our own public API, and link protobuf
|
||||
# locally.
|
||||
# (2) However, currently across the Caffe2 codebase, we have extensive use of protobuf
|
||||
# functionalities. For example, not only libcaffe2.so uses it, but also other
|
||||
# binaries such as python extensions etc. As a result, we will have to have a
|
||||
# transitive dependency to libprotobuf.
|
||||
#
|
||||
# Good thing is that, if we specify CAFFE2_LINK_LOCAL_PROTOBUF, then we do not need to
|
||||
# separately deploy protobuf binaries - libcaffe2.so will contain all functionalities
|
||||
# one needs. One can verify this via ldd.
|
||||
#
|
||||
# TODO item in the future includes:
|
||||
# (1) Enable using lite protobuf
|
||||
# (2) Properly define public API that do not directly depend on protobuf itself.
|
||||
# (3) Expose the libprotobuf.a file for dependent libraries to link to.
|
||||
#
|
||||
# What it means for users/developers?
|
||||
# (1) Users: nothing affecting the users, other than the fact that CAFFE2_LINK_LOCAL_PROTOBUF
|
||||
# avoids the need to deploy protobuf.
|
||||
# (2) Developers: if one simply uses core caffe2 functionality without using protobuf,
|
||||
# nothing changes. If one has a dependent library that uses protobuf, then one needs to
|
||||
# have the right protobuf version as well as linking to libprotobuf.a.
|
||||
target_link_libraries(caffe2_protos PUBLIC protobuf::libprotobuf)
|
||||
if (NOT BUILD_SHARED_LIBS)
|
||||
INSTALL(TARGETS caffe2_protos ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
|
||||
if (NOT INTERN_BUILD_MOBILE OR BUILD_CAFFE2_MOBILE)
|
||||
# ---[ List of libraries to link with
|
||||
add_library(caffe2_protos STATIC $<TARGET_OBJECTS:Caffe2_PROTO>)
|
||||
add_dependencies(caffe2_protos Caffe2_PROTO)
|
||||
# If we are going to link protobuf locally inside caffe2 libraries, what we will do is
|
||||
# to create a helper static library that always contains libprotobuf source files, and
|
||||
# link the caffe2 related dependent libraries to it.
|
||||
target_include_directories(caffe2_protos INTERFACE $<INSTALL_INTERFACE:include>)
|
||||
# Reason for this public dependency is as follows:
|
||||
# (1) Strictly speaking, we should not expose any Protobuf related functions. We should
|
||||
# only use function interfaces wrapped with our own public API, and link protobuf
|
||||
# locally.
|
||||
# (2) However, currently across the Caffe2 codebase, we have extensive use of protobuf
|
||||
# functionalities. For example, not only libcaffe2.so uses it, but also other
|
||||
# binaries such as python extensions etc. As a result, we will have to have a
|
||||
# transitive dependency to libprotobuf.
|
||||
#
|
||||
# Good thing is that, if we specify CAFFE2_LINK_LOCAL_PROTOBUF, then we do not need to
|
||||
# separately deploy protobuf binaries - libcaffe2.so will contain all functionalities
|
||||
# one needs. One can verify this via ldd.
|
||||
#
|
||||
# TODO item in the future includes:
|
||||
# (1) Enable using lite protobuf
|
||||
# (2) Properly define public API that do not directly depend on protobuf itself.
|
||||
# (3) Expose the libprotobuf.a file for dependent libraries to link to.
|
||||
#
|
||||
# What it means for users/developers?
|
||||
# (1) Users: nothing affecting the users, other than the fact that CAFFE2_LINK_LOCAL_PROTOBUF
|
||||
# avoids the need to deploy protobuf.
|
||||
# (2) Developers: if one simply uses core caffe2 functionality without using protobuf,
|
||||
# nothing changes. If one has a dependent library that uses protobuf, then one needs to
|
||||
# have the right protobuf version as well as linking to libprotobuf.a.
|
||||
target_link_libraries(caffe2_protos PUBLIC protobuf::libprotobuf)
|
||||
if (NOT BUILD_SHARED_LIBS)
|
||||
INSTALL(TARGETS caffe2_protos ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
# ==========================================================
|
||||
# formerly-libtorch
|
||||
# ==========================================================
|
||||
@ -369,7 +370,6 @@ if (NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE)
|
||||
${TORCH_SRC_DIR}/csrc/jit/ir.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/irparser.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/jit_log.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/netdef_converter.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/operator.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/register_c10_ops.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/subgraph_matcher.cpp
|
||||
@ -461,6 +461,7 @@ if (NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE)
|
||||
${TORCH_SRC_DIR}/csrc/distributed/rpc/script_ret.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/export.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/import_legacy.cpp
|
||||
${TORCH_SRC_DIR}/csrc/jit/netdef_converter.cpp
|
||||
)
|
||||
if (NOT WIN32)
|
||||
list(APPEND TORCH_SRCS
|
||||
@ -572,14 +573,15 @@ ENDIF()
|
||||
# formerly-libtorch flags
|
||||
# ==========================================================
|
||||
|
||||
if (NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE)
|
||||
|
||||
if (NOT INTERN_BUILD_MOBILE)
|
||||
# Forces caffe2.pb.h to be generated before its dependents are compiled.
|
||||
# Adding the generated header file to the ${TORCH_SRCS} list is not sufficient
|
||||
# to establish the dependency, since the generation procedure is declared in a different CMake file.
|
||||
# See https://samthursfield.wordpress.com/2015/11/21/cmake-dependencies-between-targets-and-files-and-custom-commands/#custom-commands-in-different-directories
|
||||
add_dependencies(torch Caffe2_PROTO)
|
||||
endif()
|
||||
|
||||
if (NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE)
|
||||
target_compile_definitions(torch PUBLIC _THP_CORE)
|
||||
|
||||
|
||||
@ -830,13 +832,14 @@ if (NOT WIN32 AND NOT USE_ASAN)
|
||||
target_compile_options(torch PRIVATE "-fvisibility=hidden")
|
||||
endif()
|
||||
|
||||
|
||||
caffe2_interface_library(caffe2_protos caffe2_protos_whole)
|
||||
target_link_libraries(torch PRIVATE caffe2_protos_whole)
|
||||
if (${CAFFE2_LINK_LOCAL_PROTOBUF})
|
||||
target_link_libraries(torch INTERFACE protobuf::libprotobuf)
|
||||
else()
|
||||
target_link_libraries(torch PUBLIC protobuf::libprotobuf)
|
||||
if (NOT INTERN_BUILD_MOBILE OR BUILD_CAFFE2_MOBILE)
|
||||
caffe2_interface_library(caffe2_protos caffe2_protos_whole)
|
||||
target_link_libraries(torch PRIVATE caffe2_protos_whole)
|
||||
if (${CAFFE2_LINK_LOCAL_PROTOBUF})
|
||||
target_link_libraries(torch INTERFACE protobuf::libprotobuf)
|
||||
else()
|
||||
target_link_libraries(torch PUBLIC protobuf::libprotobuf)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (USE_OPENMP AND OPENMP_FOUND)
|
||||
|
||||
@ -1,3 +1,11 @@
|
||||
if (INTERN_BUILD_MOBILE AND NOT BUILD_CAFFE2_MOBILE)
|
||||
list(APPEND Caffe2_CPU_SRCS
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/common.cc"
|
||||
)
|
||||
set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
|
||||
# ---[ GPU files
|
||||
# ------[ cuDNN
|
||||
if (USE_CUDNN)
|
||||
|
||||
@ -1,3 +1,11 @@
|
||||
if (INTERN_BUILD_MOBILE AND NOT BUILD_CAFFE2_MOBILE)
|
||||
list(APPEND Caffe2_CPU_SRCS
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/embedding_lookup_idx.cc"
|
||||
)
|
||||
set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
|
||||
# ---[ CPU files.
|
||||
file(GLOB common_srcs *.cc)
|
||||
file(GLOB avx_srcs *_avx.cc)
|
||||
|
||||
@ -1,3 +1,15 @@
|
||||
if (INTERN_BUILD_MOBILE AND NOT BUILD_CAFFE2_MOBILE)
|
||||
list(APPEND Caffe2_CPU_SRCS
|
||||
utils/string_utils.cc
|
||||
utils/threadpool/pthreadpool.cc
|
||||
utils/threadpool/pthreadpool_impl.cc
|
||||
utils/threadpool/ThreadPool.cc
|
||||
utils/threadpool/ThreadPoolMobile.cc
|
||||
)
|
||||
set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
|
||||
list(APPEND Caffe2_CPU_SRCS
|
||||
utils/bench_utils.cc
|
||||
utils/cpuid.cc
|
||||
|
||||
@ -1035,7 +1035,7 @@ if (USE_ZSTD)
|
||||
endif()
|
||||
|
||||
# ---[ Onnx
|
||||
if (CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO)
|
||||
if (CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX)
|
||||
if(EXISTS "${CAFFE2_CUSTOM_PROTOC_EXECUTABLE}")
|
||||
set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${CAFFE2_CUSTOM_PROTOC_EXECUTABLE})
|
||||
endif()
|
||||
|
||||
Reference in New Issue
Block a user