Migrate c10/macros/cmake_macros.h.in to torch/headeronly (#158035)

Summary: As above, also changes a bunch of the build files to be better

Test Plan:
internal and external CI

did run buck2 build fbcode//caffe2:torch and it succeeded

Rollback Plan:

Reviewed By: swolchok

Differential Revision: D78016591

Pull Request resolved: https://github.com/pytorch/pytorch/pull/158035
Approved by: https://github.com/swolchok

.bazelrc

@@ -2,7 +2,7 @@ build --cxxopt=--std=c++17
 build --copt=-I.
 # Bazel does not support including its cc_library targets as system
 # headers. We work around this for generated code
-# (e.g. c10/macros/cmake_macros.h) by making the generated directory a
+# (e.g. torch/headeronly/macros/cmake_macros.h) by making the generated directory a
 # system include path.
 build --copt=-isystem --copt bazel-out/k8-fastbuild/bin
 build --copt=-isystem --copt bazel-out/darwin-fastbuild/bin

CMakeLists.txt

@@ -1245,6 +1245,7 @@ if(USE_MIMALLOC AND USE_MIMALLOC_ON_MKL)
 endif()
 
 # ---[ Main build
+add_subdirectory(torch/headeronly)  # headeronly headers
 add_subdirectory(c10)
 add_subdirectory(caffe2)
 

c10/BUCK.oss

@@ -37,8 +37,6 @@ cxx_library(
     ),
     exported_linker_flags = [],
     exported_preprocessor_flags = [
-        '-DC10_USING_CUSTOM_GENERATED_MACROS',
-        '-DC10_USE_GLOG',
         '-DC10_USE_MINIMAL_GLOG',
         '-DC10_MOBILE',
         '-fexceptions',

c10/CMakeLists.txt

@@ -18,16 +18,12 @@ else()
   set(C10_LIB c10)
 endif()
 
-  # ---[ Configure macro file.
+set(C10_USE_GFLAGS ${USE_GFLAGS})  # also used in torch/headeronly
-  set(C10_USE_GFLAGS ${USE_GFLAGS}) # used in cmake_macros.h.in
+set(C10_USE_GLOG ${USE_GLOG})  # also used in torch/headeronly
-  set(C10_USE_GLOG ${USE_GLOG}) # used in cmake_macros.h.in
+set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})  # also used in torch/headeronly
-  set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) # used in cmake_macros.h.in
+set(C10_USE_NUMA ${USE_NUMA})  # also used in torch/headeronly
-  set(C10_USE_NUMA ${USE_NUMA})
+set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME})  # also used in torch/headeronly
-  set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME})
+set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT})  # also used in torch/headeronly
-  set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT})
-  configure_file(
-      ${CMAKE_CURRENT_LIST_DIR}/macros/cmake_macros.h.in
-      ${CMAKE_BINARY_DIR}/c10/macros/cmake_macros.h)
 
   # Note: if you want to add ANY dependency to the c10 library, make sure you
   # check with the core PyTorch developers as the dependency will be
@@ -94,6 +90,8 @@ if(NOT BUILD_LIBTORCHLESS)
   if(C10_USE_GLOG)
     target_link_libraries(c10 PUBLIC glog::glog)
   endif()
+
+  target_link_libraries(c10 PUBLIC headeronly)
   target_link_libraries(c10 PRIVATE fmt::fmt-header-only)
   target_link_libraries(c10 PRIVATE nlohmann)
   target_link_libraries(c10 PRIVATE moodycamel)
@@ -170,8 +168,6 @@ endif()
 install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
         DESTINATION include
         FILES_MATCHING PATTERN "*.h")
-install(FILES ${CMAKE_BINARY_DIR}/c10/macros/cmake_macros.h
-        DESTINATION include/c10/macros)
 
 if(MSVC AND C10_BUILD_SHARED_LIBS)
   install(FILES $<TARGET_PDB_FILE:c10> DESTINATION lib OPTIONAL)

c10/macros/Export.h

@@ -2,7 +2,7 @@
 #define C10_MACROS_EXPORT_H_
 
 #ifndef C10_USING_CUSTOM_GENERATED_MACROS
-#include <c10/macros/cmake_macros.h>
+#include <torch/headeronly/macros/cmake_macros.h>
 #endif // C10_USING_CUSTOM_GENERATED_MACROS
 
 #include <torch/headeronly/macros/Export.h>

c10/macros/Macros.h

@@ -19,7 +19,7 @@
 // file.
 
 #ifndef C10_USING_CUSTOM_GENERATED_MACROS
-#include <c10/macros/cmake_macros.h>
+#include <torch/headeronly/macros/cmake_macros.h>
 #endif // C10_USING_CUSTOM_GENERATED_MACROS
 
 #include <c10/macros/Export.h>

c10/macros/build.bzl

@@ -1,13 +1,13 @@
 def define_targets(rules):
     rules.cc_library(
         name = "macros",
-        srcs = [":cmake_macros_h"],
         hdrs = [
             "Macros.h",
             # Despite the documentation in Macros.h, Export.h is included
             # directly by many downstream files. Thus, we declare it as a
             # public header in this file.
             "Export.h",
+            "cmake_macros.h",
         ],
         linkstatic = True,
         local_defines = ["C10_BUILD_MAIN_LIB"],
@@ -17,22 +17,6 @@ def define_targets(rules):
         ],
     )
 
-    rules.cmake_configure_file(
-        name = "cmake_macros_h",
-        src = "cmake_macros.h.in",
-        out = "cmake_macros.h",
-        definitions = [
-            "C10_BUILD_SHARED_LIBS",
-            "C10_USE_MSVC_STATIC_RUNTIME",
-        ] + rules.select({
-            "//c10:using_gflags": ["C10_USE_GFLAGS"],
-            "//conditions:default": [],
-        }) + rules.select({
-            "//c10:using_glog": ["C10_USE_GLOG"],
-            "//conditions:default": [],
-        }),
-    )
-
     rules.filegroup(
         name = "headers",
         srcs = rules.glob(

c10/macros/cmake_macros.h

@@ -0,0 +1,5 @@
+// This file exists for backwards compatibility and has been moved to
+// torch/headeronly/macros/cmake_macros.h.in. No end user library should be
+// including this file directly anyway (cuz they should be including
+// Macros.h instead).
+#include <torch/headeronly/macros/cmake_macros.h>

c10/ovrsource_defs.bzl

@@ -73,8 +73,7 @@ def define_c10_ovrsource(name, is_mobile):
             ],
         }),
         exported_deps = [
-            "//xplat/caffe2/torch/headeronly:torch_headeronly",
+            "//xplat/caffe2/torch/headeronly:torch_headeronly_ovrsource",
-            ":ovrsource_c10_cmake_macros.h",
             "//arvr/third-party/gflags:gflags",
             "//third-party/cpuinfo:cpuinfo",
             "//third-party/fmt:fmt",
@@ -83,55 +82,6 @@ def define_c10_ovrsource(name, is_mobile):
     )
 
 def define_ovrsource_targets():
-    common_c10_cmake_defines = [
-        ("#cmakedefine C10_BUILD_SHARED_LIBS", ""),
-        ("#cmakedefine C10_USE_NUMA", ""),
-        ("#cmakedefine C10_USE_MSVC_STATIC_RUNTIME", ""),
-        ("#cmakedefine C10_USE_ROCM_KERNEL_ASSERT", ""),
-    ]
-
-    mobile_c10_cmake_defines = [
-        ("#cmakedefine C10_USE_GLOG", ""),
-        ("#cmakedefine C10_USE_GFLAGS", ""),
-    ]
-
-    non_mobile_c10_cmake_defines = [
-        ("#cmakedefine C10_USE_GLOG", "#define C10_USE_GLOG 1"),
-        ("#cmakedefine C10_USE_GFLAGS", "#define C10_USE_GFLAGS 1"),
-    ]
-
-    gen_cmake_header(
-        src = "macros/cmake_macros.h.in",
-        defines = common_c10_cmake_defines + mobile_c10_cmake_defines,
-        header = "c10/macros/cmake_macros.h",
-        prefix = "ovrsource_c10_mobile_",
-    )
-
-    gen_cmake_header(
-        src = "macros/cmake_macros.h.in",
-        defines = common_c10_cmake_defines + non_mobile_c10_cmake_defines,
-        header = "c10/macros/cmake_macros.h",
-        prefix = "ovrsource_c10_non_mobile_",
-    )
-
-    oxx_static_library(
-        name = "ovrsource_c10_cmake_macros.h",
-        compatible_with = [
-            "ovr_config//os:android",
-            "ovr_config//os:iphoneos",
-            "ovr_config//os:linux",
-            "ovr_config//os:macos",
-            "ovr_config//os:windows",
-        ],
-        deps = select({
-            "ovr_config//os:android": [":ovrsource_c10_mobile_cmake_macros.h"],
-            "ovr_config//os:iphoneos": [":ovrsource_c10_mobile_cmake_macros.h"],
-            "ovr_config//os:linux": [":ovrsource_c10_non_mobile_cmake_macros.h"],
-            "ovr_config//os:macos": [":ovrsource_c10_non_mobile_cmake_macros.h"],
-            "ovr_config//os:windows": [":ovrsource_c10_non_mobile_cmake_macros.h"],
-        }),
-    )
-
     c10_cuda_macros = gen_cmake_header(
         src = "cuda/impl/cuda_cmake_macros.h.in",
         defines = [

tools/bazel.bzl

@@ -2,7 +2,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 load("@rules_cuda//cuda:defs.bzl", "cuda_library", "requires_cuda_enabled")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 load("@pip_deps//:requirements.bzl", "requirement")
-load("@pytorch//c10/macros:cmake_configure_file.bzl", "cmake_configure_file")
+load("@pytorch//torch/headeronly/macros:cmake_configure_file.bzl", "cmake_configure_file")
 load("@pytorch//tools/config:defs.bzl", "if_cuda")
 
 def _genrule(**kwds):

torch/CMakeLists.txt

@@ -29,7 +29,6 @@ endif()
 set(LIBSHM_SRCDIR ${TORCH_SRC_DIR}/lib/${LIBSHM_SUBDIR})
 add_subdirectory(${LIBSHM_SRCDIR})
 
-
 # Generate files
 set(TOOLS_PATH "${TORCH_ROOT}/tools")
 

torch/headeronly/BUCK.oss

@@ -0,0 +1,26 @@
+load("//tools/build_defs:glob_defs.bzl", "subdir_glob")
+
+cxx_library(
+    name = "torch_headeronly",
+    header_namespace = "torch/headeronly",
+    exported_deps = [],
+    compiler_flags = [
+        "-Werror",
+        "-Wno-global-constructors",
+    ],
+    exported_headers = subdir_glob(
+        [
+            ("", "**/*.h"),
+        ],
+    ),
+    exported_linker_flags = [],
+    exported_preprocessor_flags = [
+        '-DC10_USING_CUSTOM_GENERATED_MACROS',
+        '-DC10_USE_GLOG',
+    ],
+    link_whole = True,
+    platform_preprocessor_flags = [['windows', ['-D_WINDOWS', '-D_WIN32', '-DWIN32', '-DNOMINMAX', '-D_CRT_SECURE_NO_WARNINGS', '-D_USE_MATH_DEFINES']], ['windows.*64$', ['-D_WIN64']]],
+    preprocessor_flags = ['-DC10_BUILD_MAIN_LIB'],
+    reexport_all_header_dependencies = True,
+    visibility = ['PUBLIC'],
+)

torch/headeronly/BUILD.bazel

@@ -1,9 +1,5 @@
 load("@rules_cc//cc:defs.bzl", "cc_library")
+load("//:tools/bazel.bzl", "rules")
+load(":build.bzl", "define_targets")
 
-cc_library(
+define_targets(rules = rules)
-    name = "torch_headeronly",
-    hdrs = glob([
-        "**/*.h"
-    ]),
-    visibility = ["//visibility:public"],
-)

torch/headeronly/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.27 FATAL_ERROR)
+
+project(headeronly CXX)
+
+set(CMAKE_CXX_STANDARD 17 CACHE STRING "The C++ standard whose features are requested to build this target.")
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+# Main build file for torch/headeronly, except there's no build cuz this lib is header-only!
+
+# ---[ Configure macro file.
+set(C10_USE_GFLAGS ${USE_GFLAGS}) # used in cmake_macros.h.in
+set(C10_USE_GLOG ${USE_GLOG}) # used in cmake_macros.h.in
+set(C10_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) # used in cmake_macros.h.in
+set(C10_USE_NUMA ${USE_NUMA}) # used in cmake_macros.h.in
+set(C10_USE_MSVC_STATIC_RUNTIME ${CAFFE2_USE_MSVC_STATIC_RUNTIME}) # used in cmake_macros.h.in
+set(C10_USE_ROCM_KERNEL_ASSERT ${USE_ROCM_KERNEL_ASSERT}) # used in cmake_macros.h.in
+configure_file(
+    ${CMAKE_CURRENT_LIST_DIR}/macros/cmake_macros.h.in
+    ${CMAKE_BINARY_DIR}/torch/headeronly/macros/cmake_macros.h)
+
+file(GLOB HEADERONLY_HEADERS
+    *.h
+    macros/*.h
+)
+
+add_library(headeronly INTERFACE ${HEADERONLY_HEADERS})
+
+install(FILES ${CMAKE_BINARY_DIR}/torch/headeronly/macros/cmake_macros.h
+        DESTINATION include/torch/headeronly/macros)
+
+if(NOT BUILD_LIBTORCHLESS)
+  # ---[ Installation copied from c10/CMakeLists.txt
+  install(TARGETS headeronly EXPORT Caffe2Targets DESTINATION lib)
+endif()

torch/headeronly/build.bzl

@@ -0,0 +1,11 @@
+def define_targets(rules):
+    rules.cc_library(
+        name = "torch_headeronly",
+        hdrs = rules.glob([
+            "**/*.h"
+        ]),
+        visibility = ["//visibility:public"],
+        deps = [
+            "//torch/headeronly/macros",
+        ],
+    )

torch/headeronly/macros/BUILD.bazel

@@ -0,0 +1,4 @@
+load("//:tools/bazel.bzl", "rules")
+load(":build.bzl", "define_targets")
+
+define_targets(rules = rules)

torch/headeronly/macros/build.bzl

@@ -0,0 +1,28 @@
+def define_targets(rules):
+    rules.cc_library(
+        name = "macros",
+        srcs = [":cmake_macros_h"],
+        hdrs = [
+            # Following the example from c10
+            "Export.h",
+        ],
+        linkstatic = True,
+        local_defines = ["C10_BUILD_MAIN_LIB"],
+        visibility = ["//visibility:public"],
+    )
+
+    rules.cmake_configure_file(
+        name = "cmake_macros_h",
+        src = "cmake_macros.h.in",
+        out = "cmake_macros.h",
+        definitions = [
+            "C10_BUILD_SHARED_LIBS",
+            "C10_USE_MSVC_STATIC_RUNTIME",
+        ] + rules.select({
+            "//c10:using_gflags": ["C10_USE_GFLAGS"],
+            "//conditions:default": [],
+        }) + rules.select({
+            "//c10:using_glog": ["C10_USE_GLOG"],
+            "//conditions:default": [],
+        }),
+    )

torch/headeronly/macros/cmake_macros.h.in

@@ -2,7 +2,7 @@
 #define C10_MACROS_CMAKE_MACROS_H_
 
 // Automatically generated header file for the C10 library.
-// Do not include this file directly. Instead, include c10/macros/Macros.h.
+// Do not include this file directly. Instead, include torch/headeronly/macros/Macros.h.
 
 #cmakedefine C10_BUILD_SHARED_LIBS
 #cmakedefine C10_USE_GLOG

torch/headeronly/ovrsource_defs.bzl

@@ -1,3 +1,4 @@
+load("//arvr/tools/build_defs:genrule_utils.bzl", "gen_cmake_header")
 load("//arvr/tools/build_defs:oxx.bzl", "oxx_static_library")
 
 cpu_supported_platforms = [
@@ -18,29 +19,77 @@ def define_torch_headeronly_ovrsource(name, is_mobile):
 
     oxx_static_library(
         name = name,
-        srcs = []
+        srcs = [],
         compatible_with = cpu_supported_platforms,
         compiler_flags = select({
             "DEFAULT": [],
         }),
-        include_directories = [".."],
+        preprocessor_flags = ["-DC10_BUILD_MAIN_LIB=1",],
-        preprocessor_flags = [],
         fbobjc_compiler_flags = [],
-        public_include_directories = [".."],
+        public_include_directories = ["../.."],
         public_preprocessor_flags = pp_flags,
         public_raw_headers = native.glob([
             "macros/*.h",
         ]),
         reexport_all_header_dependencies = False,
         visibility = [
-            "//xplat/caffe2/torch/headeronly:torch_headeronly",
+            "//xplat/caffe2/torch/headeronly:torch_headeronly_ovrsource",
+        ],
+        exported_deps = [
+            ":ovrsource_torch_headeronly_cmake_macros.h",
         ],
-        deps = select({
-            "DEFAULT": [],
-        }),
     )
 
 def define_ovrsource_targets():
+    common_c10_cmake_defines = [
+        ("#cmakedefine C10_BUILD_SHARED_LIBS", ""),
+        ("#cmakedefine C10_USE_NUMA", ""),
+        ("#cmakedefine C10_USE_MSVC_STATIC_RUNTIME", ""),
+        ("#cmakedefine C10_USE_ROCM_KERNEL_ASSERT", ""),
+    ]
+
+    mobile_c10_cmake_defines = [
+        ("#cmakedefine C10_USE_GLOG", ""),
+        ("#cmakedefine C10_USE_GFLAGS", ""),
+    ]
+
+    non_mobile_c10_cmake_defines = [
+        ("#cmakedefine C10_USE_GLOG", "#define C10_USE_GLOG 1"),
+        ("#cmakedefine C10_USE_GFLAGS", "#define C10_USE_GFLAGS 1"),
+    ]
+
+    gen_cmake_header(
+        src = "macros/cmake_macros.h.in",
+        defines = common_c10_cmake_defines + mobile_c10_cmake_defines,
+        header = "torch/headeronly/macros/cmake_macros.h",
+        prefix = "ovrsource_torch_headeronly_mobile_",
+    )
+
+    gen_cmake_header(
+        src = "macros/cmake_macros.h.in",
+        defines = common_c10_cmake_defines + non_mobile_c10_cmake_defines,
+        header = "torch/headeronly/macros/cmake_macros.h",
+        prefix = "ovrsource_torch_headeronly_non_mobile_",
+    )
+
+    oxx_static_library(
+        name = "ovrsource_torch_headeronly_cmake_macros.h",
+        compatible_with = [
+            "ovr_config//os:android",
+            "ovr_config//os:iphoneos",
+            "ovr_config//os:linux",
+            "ovr_config//os:macos",
+            "ovr_config//os:windows",
+        ],
+        deps = select({
+            "ovr_config//os:android": [":ovrsource_torch_headeronly_mobile_cmake_macros.h"],
+            "ovr_config//os:iphoneos": [":ovrsource_torch_headeronly_mobile_cmake_macros.h"],
+            "ovr_config//os:linux": [":ovrsource_torch_headeronly_non_mobile_cmake_macros.h"],
+            "ovr_config//os:macos": [":ovrsource_torch_headeronly_non_mobile_cmake_macros.h"],
+            "ovr_config//os:windows": [":ovrsource_torch_headeronly_non_mobile_cmake_macros.h"],
+        }),
+    )
+
     oxx_static_library(
         name = "torch_headeronly_ovrsource",
         compatible_with = cpu_supported_platforms,