From 30587195d314eb5eb02ce63f39a9be4c943629ef Mon Sep 17 00:00:00 2001 From: Jane Xu Date: Tue, 15 Jul 2025 19:52:53 +0000 Subject: [PATCH] Migrate c10/macros/cmake_macros.h.in to torch/headeronly (#158035) Summary: As above, also changes a bunch of the build files to be better Test Plan: internal and external CI did run buck2 build fbcode//caffe2:torch and it succeeded Rollback Plan: Reviewed By: swolchok Differential Revision: D78016591 Pull Request resolved: https://github.com/pytorch/pytorch/pull/158035 Approved by: https://github.com/swolchok --- .bazelrc | 2 +- CMakeLists.txt | 1 + c10/BUCK.oss | 2 - c10/CMakeLists.txt | 20 +++--- c10/macros/Export.h | 2 +- c10/macros/Macros.h | 2 +- c10/macros/build.bzl | 18 +---- c10/macros/cmake_macros.h | 5 ++ c10/ovrsource_defs.bzl | 52 +-------------- tools/bazel.bzl | 2 +- torch/CMakeLists.txt | 1 - torch/headeronly/BUCK.oss | 26 ++++++++ torch/headeronly/BUILD.bazel | 10 +-- torch/headeronly/CMakeLists.txt | 34 ++++++++++ torch/headeronly/build.bzl | 11 ++++ torch/headeronly/macros/BUILD.bazel | 4 ++ torch/headeronly/macros/build.bzl | 28 ++++++++ .../macros/cmake_configure_file.bzl | 0 .../headeronly}/macros/cmake_macros.h.in | 2 +- torch/headeronly/ovrsource_defs.bzl | 65 ++++++++++++++++--- 20 files changed, 184 insertions(+), 103 deletions(-) create mode 100644 c10/macros/cmake_macros.h create mode 100644 torch/headeronly/BUCK.oss create mode 100644 torch/headeronly/CMakeLists.txt create mode 100644 torch/headeronly/build.bzl create mode 100644 torch/headeronly/macros/BUILD.bazel create mode 100644 torch/headeronly/macros/build.bzl rename {c10 => torch/headeronly}/macros/cmake_configure_file.bzl (100%) rename {c10 => torch/headeronly}/macros/cmake_macros.h.in (80%) diff --git a/.bazelrc b/.bazelrc index 7581b5243021..fc2995dc838c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,7 +2,7 @@ build --cxxopt=--std=c++17 build --copt=-I. # Bazel does not support including its cc_library targets as system # headers. We work around this for generated code -# (e.g. c10/macros/cmake_macros.h) by making the generated directory a +# (e.g. torch/headeronly/macros/cmake_macros.h) by making the generated directory a # system include path. build --copt=-isystem --copt bazel-out/k8-fastbuild/bin build --copt=-isystem --copt bazel-out/darwin-fastbuild/bin diff --git a/CMakeLists.txt b/CMakeLists.txt index 99c0b9e0ea0c..d1f8a13fb9fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1245,6 +1245,7 @@ if(USE_MIMALLOC AND USE_MIMALLOC_ON_MKL) endif() # ---[ Main build +add_subdirectory(torch/headeronly) # headeronly headers add_subdirectory(c10) add_subdirectory(caffe2) diff --git a/c10/BUCK.oss b/c10/BUCK.oss index 4b2cbd049b85..4ec4ab5beabb 100644 --- a/c10/BUCK.oss +++ b/c10/BUCK.oss @@ -37,8 +37,6 @@ cxx_library( ), exported_linker_flags = [], exported_preprocessor_flags = [ - '-DC10_USING_CUSTOM_GENERATED_MACROS', - '-DC10_USE_GLOG', '-DC10_USE_MINIMAL_GLOG', '-DC10_MOBILE', '-fexceptions', diff --git a/c10/CMakeLists.txt b/c10/CMakeLists.txt index 8e9d267352dd..f82e460cafc3 100644 --- a/c10/CMakeLists.txt +++ b/c10/CMakeLists.txt @@ -18,16 +18,12 @@ else() set(C10_LIB c10) endif() - # ---[ Configure macro file. - set(C10_USE_GFLAGS ${USE_GFLAGS}) # used in cmake_macros.h.in - set(C10_USE_GLOG ${USE_GLOG}) # used in cmake_macros.h.in - set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) # used in cmake_macros.h.in - set(C10_USE_NUMA ${USE_NUMA}) - set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME}) - set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT}) - configure_file( - ${CMAKE_CURRENT_LIST_DIR}/macros/cmake_macros.h.in - ${CMAKE_BINARY_DIR}/c10/macros/cmake_macros.h) +set(C10_USE_GFLAGS ${USE_GFLAGS}) # also used in torch/headeronly +set(C10_USE_GLOG ${USE_GLOG}) # also used in torch/headeronly +set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) # also used in torch/headeronly +set(C10_USE_NUMA ${USE_NUMA}) # also used in torch/headeronly +set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME}) # also used in torch/headeronly +set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT}) # also used in torch/headeronly # Note: if you want to add ANY dependency to the c10 library, make sure you # check with the core PyTorch developers as the dependency will be @@ -94,6 +90,8 @@ if(NOT BUILD_LIBTORCHLESS) if(C10_USE_GLOG) target_link_libraries(c10 PUBLIC glog::glog) endif() + + target_link_libraries(c10 PUBLIC headeronly) target_link_libraries(c10 PRIVATE fmt::fmt-header-only) target_link_libraries(c10 PRIVATE nlohmann) target_link_libraries(c10 PRIVATE moodycamel) @@ -170,8 +168,6 @@ endif() install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR} DESTINATION include FILES_MATCHING PATTERN "*.h") -install(FILES ${CMAKE_BINARY_DIR}/c10/macros/cmake_macros.h - DESTINATION include/c10/macros) if(MSVC AND C10_BUILD_SHARED_LIBS) install(FILES $ DESTINATION lib OPTIONAL) diff --git a/c10/macros/Export.h b/c10/macros/Export.h index b013910902b2..3d9126610261 100644 --- a/c10/macros/Export.h +++ b/c10/macros/Export.h @@ -2,7 +2,7 @@ #define C10_MACROS_EXPORT_H_ #ifndef C10_USING_CUSTOM_GENERATED_MACROS -#include +#include #endif // C10_USING_CUSTOM_GENERATED_MACROS #include diff --git a/c10/macros/Macros.h b/c10/macros/Macros.h index 6b51a39f2a94..55a79ee67430 100644 --- a/c10/macros/Macros.h +++ b/c10/macros/Macros.h @@ -19,7 +19,7 @@ // file. #ifndef C10_USING_CUSTOM_GENERATED_MACROS -#include +#include #endif // C10_USING_CUSTOM_GENERATED_MACROS #include diff --git a/c10/macros/build.bzl b/c10/macros/build.bzl index 129b2b1e0570..d5809d36687d 100644 --- a/c10/macros/build.bzl +++ b/c10/macros/build.bzl @@ -1,13 +1,13 @@ def define_targets(rules): rules.cc_library( name = "macros", - srcs = [":cmake_macros_h"], hdrs = [ "Macros.h", # Despite the documentation in Macros.h, Export.h is included # directly by many downstream files. Thus, we declare it as a # public header in this file. "Export.h", + "cmake_macros.h", ], linkstatic = True, local_defines = ["C10_BUILD_MAIN_LIB"], @@ -17,22 +17,6 @@ def define_targets(rules): ], ) - rules.cmake_configure_file( - name = "cmake_macros_h", - src = "cmake_macros.h.in", - out = "cmake_macros.h", - definitions = [ - "C10_BUILD_SHARED_LIBS", - "C10_USE_MSVC_STATIC_RUNTIME", - ] + rules.select({ - "//c10:using_gflags": ["C10_USE_GFLAGS"], - "//conditions:default": [], - }) + rules.select({ - "//c10:using_glog": ["C10_USE_GLOG"], - "//conditions:default": [], - }), - ) - rules.filegroup( name = "headers", srcs = rules.glob( diff --git a/c10/macros/cmake_macros.h b/c10/macros/cmake_macros.h new file mode 100644 index 000000000000..4358f6906c97 --- /dev/null +++ b/c10/macros/cmake_macros.h @@ -0,0 +1,5 @@ +// This file exists for backwards compatibility and has been moved to +// torch/headeronly/macros/cmake_macros.h.in. No end user library should be +// including this file directly anyway (cuz they should be including +// Macros.h instead). +#include diff --git a/c10/ovrsource_defs.bzl b/c10/ovrsource_defs.bzl index 4abf8b0014de..aafe5a4de8c4 100644 --- a/c10/ovrsource_defs.bzl +++ b/c10/ovrsource_defs.bzl @@ -73,8 +73,7 @@ def define_c10_ovrsource(name, is_mobile): ], }), exported_deps = [ - "//xplat/caffe2/torch/headeronly:torch_headeronly", - ":ovrsource_c10_cmake_macros.h", + "//xplat/caffe2/torch/headeronly:torch_headeronly_ovrsource", "//arvr/third-party/gflags:gflags", "//third-party/cpuinfo:cpuinfo", "//third-party/fmt:fmt", @@ -83,55 +82,6 @@ def define_c10_ovrsource(name, is_mobile): ) def define_ovrsource_targets(): - common_c10_cmake_defines = [ - ("#cmakedefine C10_BUILD_SHARED_LIBS", ""), - ("#cmakedefine C10_USE_NUMA", ""), - ("#cmakedefine C10_USE_MSVC_STATIC_RUNTIME", ""), - ("#cmakedefine C10_USE_ROCM_KERNEL_ASSERT", ""), - ] - - mobile_c10_cmake_defines = [ - ("#cmakedefine C10_USE_GLOG", ""), - ("#cmakedefine C10_USE_GFLAGS", ""), - ] - - non_mobile_c10_cmake_defines = [ - ("#cmakedefine C10_USE_GLOG", "#define C10_USE_GLOG 1"), - ("#cmakedefine C10_USE_GFLAGS", "#define C10_USE_GFLAGS 1"), - ] - - gen_cmake_header( - src = "macros/cmake_macros.h.in", - defines = common_c10_cmake_defines + mobile_c10_cmake_defines, - header = "c10/macros/cmake_macros.h", - prefix = "ovrsource_c10_mobile_", - ) - - gen_cmake_header( - src = "macros/cmake_macros.h.in", - defines = common_c10_cmake_defines + non_mobile_c10_cmake_defines, - header = "c10/macros/cmake_macros.h", - prefix = "ovrsource_c10_non_mobile_", - ) - - oxx_static_library( - name = "ovrsource_c10_cmake_macros.h", - compatible_with = [ - "ovr_config//os:android", - "ovr_config//os:iphoneos", - "ovr_config//os:linux", - "ovr_config//os:macos", - "ovr_config//os:windows", - ], - deps = select({ - "ovr_config//os:android": [":ovrsource_c10_mobile_cmake_macros.h"], - "ovr_config//os:iphoneos": [":ovrsource_c10_mobile_cmake_macros.h"], - "ovr_config//os:linux": [":ovrsource_c10_non_mobile_cmake_macros.h"], - "ovr_config//os:macos": [":ovrsource_c10_non_mobile_cmake_macros.h"], - "ovr_config//os:windows": [":ovrsource_c10_non_mobile_cmake_macros.h"], - }), - ) - c10_cuda_macros = gen_cmake_header( src = "cuda/impl/cuda_cmake_macros.h.in", defines = [ diff --git a/tools/bazel.bzl b/tools/bazel.bzl index cd263ba4d324..9b662859adb4 100644 --- a/tools/bazel.bzl +++ b/tools/bazel.bzl @@ -2,7 +2,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load("@rules_cuda//cuda:defs.bzl", "cuda_library", "requires_cuda_enabled") load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("@pip_deps//:requirements.bzl", "requirement") -load("@pytorch//c10/macros:cmake_configure_file.bzl", "cmake_configure_file") +load("@pytorch//torch/headeronly/macros:cmake_configure_file.bzl", "cmake_configure_file") load("@pytorch//tools/config:defs.bzl", "if_cuda") def _genrule(**kwds): diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt index bc92f97b3956..8d761068d1e6 100644 --- a/torch/CMakeLists.txt +++ b/torch/CMakeLists.txt @@ -29,7 +29,6 @@ endif() set(LIBSHM_SRCDIR ${TORCH_SRC_DIR}/lib/${LIBSHM_SUBDIR}) add_subdirectory(${LIBSHM_SRCDIR}) - # Generate files set(TOOLS_PATH "${TORCH_ROOT}/tools") diff --git a/torch/headeronly/BUCK.oss b/torch/headeronly/BUCK.oss new file mode 100644 index 000000000000..2b8d77e597a6 --- /dev/null +++ b/torch/headeronly/BUCK.oss @@ -0,0 +1,26 @@ +load("//tools/build_defs:glob_defs.bzl", "subdir_glob") + +cxx_library( + name = "torch_headeronly", + header_namespace = "torch/headeronly", + exported_deps = [], + compiler_flags = [ + "-Werror", + "-Wno-global-constructors", + ], + exported_headers = subdir_glob( + [ + ("", "**/*.h"), + ], + ), + exported_linker_flags = [], + exported_preprocessor_flags = [ + '-DC10_USING_CUSTOM_GENERATED_MACROS', + '-DC10_USE_GLOG', + ], + link_whole = True, + platform_preprocessor_flags = [['windows', ['-D_WINDOWS', '-D_WIN32', '-DWIN32', '-DNOMINMAX', '-D_CRT_SECURE_NO_WARNINGS', '-D_USE_MATH_DEFINES']], ['windows.*64$', ['-D_WIN64']]], + preprocessor_flags = ['-DC10_BUILD_MAIN_LIB'], + reexport_all_header_dependencies = True, + visibility = ['PUBLIC'], +) diff --git a/torch/headeronly/BUILD.bazel b/torch/headeronly/BUILD.bazel index f4a27fac1f7f..030651b12043 100644 --- a/torch/headeronly/BUILD.bazel +++ b/torch/headeronly/BUILD.bazel @@ -1,9 +1,5 @@ load("@rules_cc//cc:defs.bzl", "cc_library") +load("//:tools/bazel.bzl", "rules") +load(":build.bzl", "define_targets") -cc_library( - name = "torch_headeronly", - hdrs = glob([ - "**/*.h" - ]), - visibility = ["//visibility:public"], -) +define_targets(rules = rules) diff --git a/torch/headeronly/CMakeLists.txt b/torch/headeronly/CMakeLists.txt new file mode 100644 index 000000000000..08ad713ca845 --- /dev/null +++ b/torch/headeronly/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.27 FATAL_ERROR) + +project(headeronly CXX) + +set(CMAKE_CXX_STANDARD 17 CACHE STRING "The C++ standard whose features are requested to build this target.") +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Main build file for torch/headeronly, except there's no build cuz this lib is header-only! + +# ---[ Configure macro file. +set(C10_USE_GFLAGS ${USE_GFLAGS}) # used in cmake_macros.h.in +set(C10_USE_GLOG ${USE_GLOG}) # used in cmake_macros.h.in +set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) # used in cmake_macros.h.in +set(C10_USE_NUMA ${USE_NUMA}) # used in cmake_macros.h.in +set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME}) # used in cmake_macros.h.in +set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT}) # used in cmake_macros.h.in +configure_file( + ${CMAKE_CURRENT_LIST_DIR}/macros/cmake_macros.h.in + ${CMAKE_BINARY_DIR}/torch/headeronly/macros/cmake_macros.h) + +file(GLOB HEADERONLY_HEADERS + *.h + macros/*.h +) + +add_library(headeronly INTERFACE ${HEADERONLY_HEADERS}) + +install(FILES ${CMAKE_BINARY_DIR}/torch/headeronly/macros/cmake_macros.h + DESTINATION include/torch/headeronly/macros) + +if(NOT BUILD_LIBTORCHLESS) + # ---[ Installation copied from c10/CMakeLists.txt + install(TARGETS headeronly EXPORT Caffe2Targets DESTINATION lib) +endif() diff --git a/torch/headeronly/build.bzl b/torch/headeronly/build.bzl new file mode 100644 index 000000000000..6ec9a843e884 --- /dev/null +++ b/torch/headeronly/build.bzl @@ -0,0 +1,11 @@ +def define_targets(rules): + rules.cc_library( + name = "torch_headeronly", + hdrs = rules.glob([ + "**/*.h" + ]), + visibility = ["//visibility:public"], + deps = [ + "//torch/headeronly/macros", + ], + ) diff --git a/torch/headeronly/macros/BUILD.bazel b/torch/headeronly/macros/BUILD.bazel new file mode 100644 index 000000000000..d1a0db360d23 --- /dev/null +++ b/torch/headeronly/macros/BUILD.bazel @@ -0,0 +1,4 @@ +load("//:tools/bazel.bzl", "rules") +load(":build.bzl", "define_targets") + +define_targets(rules = rules) diff --git a/torch/headeronly/macros/build.bzl b/torch/headeronly/macros/build.bzl new file mode 100644 index 000000000000..5217c2f7d37d --- /dev/null +++ b/torch/headeronly/macros/build.bzl @@ -0,0 +1,28 @@ +def define_targets(rules): + rules.cc_library( + name = "macros", + srcs = [":cmake_macros_h"], + hdrs = [ + # Following the example from c10 + "Export.h", + ], + linkstatic = True, + local_defines = ["C10_BUILD_MAIN_LIB"], + visibility = ["//visibility:public"], + ) + + rules.cmake_configure_file( + name = "cmake_macros_h", + src = "cmake_macros.h.in", + out = "cmake_macros.h", + definitions = [ + "C10_BUILD_SHARED_LIBS", + "C10_USE_MSVC_STATIC_RUNTIME", + ] + rules.select({ + "//c10:using_gflags": ["C10_USE_GFLAGS"], + "//conditions:default": [], + }) + rules.select({ + "//c10:using_glog": ["C10_USE_GLOG"], + "//conditions:default": [], + }), + ) diff --git a/c10/macros/cmake_configure_file.bzl b/torch/headeronly/macros/cmake_configure_file.bzl similarity index 100% rename from c10/macros/cmake_configure_file.bzl rename to torch/headeronly/macros/cmake_configure_file.bzl diff --git a/c10/macros/cmake_macros.h.in b/torch/headeronly/macros/cmake_macros.h.in similarity index 80% rename from c10/macros/cmake_macros.h.in rename to torch/headeronly/macros/cmake_macros.h.in index 76c185b55236..e624221202df 100644 --- a/c10/macros/cmake_macros.h.in +++ b/torch/headeronly/macros/cmake_macros.h.in @@ -2,7 +2,7 @@ #define C10_MACROS_CMAKE_MACROS_H_ // Automatically generated header file for the C10 library. -// Do not include this file directly. Instead, include c10/macros/Macros.h. +// Do not include this file directly. Instead, include torch/headeronly/macros/Macros.h. #cmakedefine C10_BUILD_SHARED_LIBS #cmakedefine C10_USE_GLOG diff --git a/torch/headeronly/ovrsource_defs.bzl b/torch/headeronly/ovrsource_defs.bzl index 55e1947b5e76..5ba9b593c297 100644 --- a/torch/headeronly/ovrsource_defs.bzl +++ b/torch/headeronly/ovrsource_defs.bzl @@ -1,3 +1,4 @@ +load("//arvr/tools/build_defs:genrule_utils.bzl", "gen_cmake_header") load("//arvr/tools/build_defs:oxx.bzl", "oxx_static_library") cpu_supported_platforms = [ @@ -18,29 +19,77 @@ def define_torch_headeronly_ovrsource(name, is_mobile): oxx_static_library( name = name, - srcs = [] + srcs = [], compatible_with = cpu_supported_platforms, compiler_flags = select({ "DEFAULT": [], }), - include_directories = [".."], - preprocessor_flags = [], + preprocessor_flags = ["-DC10_BUILD_MAIN_LIB=1",], fbobjc_compiler_flags = [], - public_include_directories = [".."], + public_include_directories = ["../.."], public_preprocessor_flags = pp_flags, public_raw_headers = native.glob([ "macros/*.h", ]), reexport_all_header_dependencies = False, visibility = [ - "//xplat/caffe2/torch/headeronly:torch_headeronly", + "//xplat/caffe2/torch/headeronly:torch_headeronly_ovrsource", + ], + exported_deps = [ + ":ovrsource_torch_headeronly_cmake_macros.h", ], - deps = select({ - "DEFAULT": [], - }), ) def define_ovrsource_targets(): + common_c10_cmake_defines = [ + ("#cmakedefine C10_BUILD_SHARED_LIBS", ""), + ("#cmakedefine C10_USE_NUMA", ""), + ("#cmakedefine C10_USE_MSVC_STATIC_RUNTIME", ""), + ("#cmakedefine C10_USE_ROCM_KERNEL_ASSERT", ""), + ] + + mobile_c10_cmake_defines = [ + ("#cmakedefine C10_USE_GLOG", ""), + ("#cmakedefine C10_USE_GFLAGS", ""), + ] + + non_mobile_c10_cmake_defines = [ + ("#cmakedefine C10_USE_GLOG", "#define C10_USE_GLOG 1"), + ("#cmakedefine C10_USE_GFLAGS", "#define C10_USE_GFLAGS 1"), + ] + + gen_cmake_header( + src = "macros/cmake_macros.h.in", + defines = common_c10_cmake_defines + mobile_c10_cmake_defines, + header = "torch/headeronly/macros/cmake_macros.h", + prefix = "ovrsource_torch_headeronly_mobile_", + ) + + gen_cmake_header( + src = "macros/cmake_macros.h.in", + defines = common_c10_cmake_defines + non_mobile_c10_cmake_defines, + header = "torch/headeronly/macros/cmake_macros.h", + prefix = "ovrsource_torch_headeronly_non_mobile_", + ) + + oxx_static_library( + name = "ovrsource_torch_headeronly_cmake_macros.h", + compatible_with = [ + "ovr_config//os:android", + "ovr_config//os:iphoneos", + "ovr_config//os:linux", + "ovr_config//os:macos", + "ovr_config//os:windows", + ], + deps = select({ + "ovr_config//os:android": [":ovrsource_torch_headeronly_mobile_cmake_macros.h"], + "ovr_config//os:iphoneos": [":ovrsource_torch_headeronly_mobile_cmake_macros.h"], + "ovr_config//os:linux": [":ovrsource_torch_headeronly_non_mobile_cmake_macros.h"], + "ovr_config//os:macos": [":ovrsource_torch_headeronly_non_mobile_cmake_macros.h"], + "ovr_config//os:windows": [":ovrsource_torch_headeronly_non_mobile_cmake_macros.h"], + }), + ) + oxx_static_library( name = "torch_headeronly_ovrsource", compatible_with = cpu_supported_platforms,