From c9cbdb411d32cb3a4572269fe766435e1aa566eb Mon Sep 17 00:00:00 2001 From: Linbin Yu Date: Wed, 22 Jun 2022 03:14:23 +0000 Subject: [PATCH] [2] move more pytorch buck targets to shared build (#79330) Summary: as title Test Plan: sandcastle and oss CI Differential Revision: D37087229 Pull Request resolved: https://github.com/pytorch/pytorch/pull/79330 Approved by: https://github.com/dhruvbird --- .buckconfig.oss | 2 + BUCK.oss | 268 +---- buckbuild.bzl | 1034 ++++++++++++++++- pt_defs.oss.bzl | 15 + tools/build_defs/buck_helpers.bzl | 46 - tools/build_defs/fb_python_binary.bzl | 3 +- tools/build_defs/fb_python_library.bzl | 9 + tools/build_defs/fb_xplat_cxx_library.bzl | 9 +- tools/build_defs/select.bzl | 8 + tools/build_defs/windows/windows_flag_map.bzl | 9 + 10 files changed, 1062 insertions(+), 341 deletions(-) create mode 100644 tools/build_defs/fb_python_library.bzl create mode 100644 tools/build_defs/select.bzl create mode 100644 tools/build_defs/windows/windows_flag_map.bzl diff --git a/.buckconfig.oss b/.buckconfig.oss index a289ddd82a14..3d1ba5109396 100644 --- a/.buckconfig.oss +++ b/.buckconfig.oss @@ -3,9 +3,11 @@ [buildfile] name = BUCK.oss + includes = //tools/build_defs/select.bzl [repositories] bazel_skylib = third_party/bazel-skylib/ + ovr_config = . [download] in_build = true diff --git a/BUCK.oss b/BUCK.oss index 125868dc9ec5..8ba89590c458 100644 --- a/BUCK.oss +++ b/BUCK.oss @@ -1,5 +1,4 @@ load("//tools/build_defs:glob_defs.bzl", "subdir_glob") -load("//tools/build_defs:fb_xplat_genrule.bzl", "fb_xplat_genrule") load( ":build_variables.bzl", "aten_cpu_source_list", @@ -21,13 +20,16 @@ load( "pt_operator_library", "get_pt_ops_deps", "aten_ufunc_generated_all_cpu_sources", + "pt_xplat_cxx_library", "TEMPLATE_SOURCE_LIST", ) load(":buckbuild.bzl", "define_buck_targets", ) -define_buck_targets() +define_buck_targets( + pt_xplat_cxx_library = pt_xplat_cxx_library, +) cxx_library( name = "pthreadpool", @@ -76,20 +78,6 @@ cxx_library( visibility = ['PUBLIC'], ) -cxx_library( - name = "common_core", - srcs = ['caffe2/core/common.cc'], - deps = [':caffe2_headers', '//c10:c10'], - exported_deps = [], - compiler_flags = ['-frtti', '-Os', '-Wno-unknown-pragmas', '-Wno-write-strings', '-Wno-unused-variable', '-Wno-unused-function', '-Wno-deprecated-declarations', '-Wno-shadow', '-Wno-global-constructors', '-Wno-missing-prototypes', '-std=gnu++17'], - preferred_linkage = "static", - header_namespace = "caffe2", - headers = [], - link_whole = True, - platform_preprocessor_flags = [['windows', ['-D_WINDOWS', '-D_WIN32', '-DWIN32', '-DNOMINMAX', '-D_CRT_SECURE_NO_WARNINGS', '-D_USE_MATH_DEFINES']], ['windows.*64$', ['-D_WIN64']]], - visibility = ['PUBLIC'], -) - build_aten_cpu( name = "aten_cpu", srcs = jit_core_sources + @@ -109,35 +97,6 @@ build_aten_cpu( ], ) -fb_xplat_genrule( - name = "generate_aten_config", - srcs = [ - "aten/src/ATen/Config.h.in", - ], - cmd = " ".join([ - "sed", - "-e 's/@AT_MKLDNN_ENABLED@/ATEN_MKLDNN_ENABLED_FBXPLAT/g'", - "-e 's/@AT_MKL_ENABLED@/ATEN_MKL_ENABLED_FBXPLAT/g'", - "-e 's/@AT_MKL_SEQUENTIAL@/ATEN_MKL_SEQUENTIAL_FBXPLAT/g'", - "-e 's/@AT_FFTW_ENABLED@/0/g'", - "-e 's/@AT_POCKETFFT_ENABLED@/0/g'", - "-e 's/@AT_NNPACK_ENABLED@/ATEN_NNPACK_ENABLED_FBXPLAT/g'", - "-e 's/@CAFFE2_STATIC_LINK_CUDA_INT@/CAFFE2_STATIC_LINK_CUDA_FBXPLAT/g'", - "-e 's/@AT_BUILD_WITH_BLAS@/USE_BLAS_FBXPLAT/g'", - "-e 's/@AT_PARALLEL_OPENMP@/AT_PARALLEL_OPENMP_FBXPLAT/g'", - "-e 's/@AT_PARALLEL_NATIVE@/AT_PARALLEL_NATIVE_FBXPLAT/g'", - "-e 's/@AT_PARALLEL_NATIVE_TBB@/AT_PARALLEL_NATIVE_TBB_FBXPLAT/g'", - "-e 's/@AT_BUILD_WITH_LAPACK@/USE_LAPACK_FBXPLAT/g'", - "-e 's/@AT_BLAS_F2C@/AT_BLAS_F2C_FBXPLAT/g'", - "-e 's/@AT_BLAS_USE_CBLAS_DOT@/AT_BLAS_USE_CBLAS_DOT_FBXPLAT/g'", - "aten/src/ATen/Config.h.in > $OUT/Config.h", - ]), - outs = { - "Config.h": ["Config.h"], - }, - default_outs = ["."], -) - gen_aten_files( name = "gen_aten", extra_flags = get_aten_codegen_extra_params(USED_PT_BACKENDS), @@ -172,87 +131,6 @@ cxx_library( ) gen_aten_libtorch_files(name = "gen_aten_libtorch") -cxx_library( - name = "torch_mobile_observer", - srcs = [ - "torch/csrc/jit/mobile/observer.cpp", - #"torch/fb/observers/MobileObserverUtil.cpp", - ], - header_namespace = "", - exported_headers = subdir_glob( - [ - ("", "torch/csrc/jit/mobile/observer.h"), - #("", "torch/fb/observers/ObserverUtil.h"), - #("", "torch/fb/observers/MobileObserverUtil.h"), - ], - ), - visibility = ["PUBLIC"], - deps = [ - "//c10:c10", - ], -) - -python_library( - name = "aten_code_template", - srcs = subdir_glob([ - ("aten", "src/ATen/code_template.py"), - ]), - base_module = "", - visibility = ["PUBLIC"], -) - - -cxx_library( - name = "torch_common", - srcs = core_sources_common, - compiler_flags = get_pt_compiler_flags(), - exported_preprocessor_flags = get_pt_preprocessor_flags(), - link_whole = True, - visibility = ["PUBLIC"], - deps = [ - ":aten_cpu", - ":generated-autograd-headers", - ":torch_headers", - "//third_party:glog", - "//c10:c10", - ], -) - - -cxx_library( - name = "torch_mobile_deserialize_common", - srcs = [ - "torch/csrc/jit/mobile/parse_bytecode.cpp", - "torch/csrc/jit/mobile/parse_operators.cpp", - "torch/csrc/jit/mobile/upgrader_mobile.cpp", - "torch/csrc/jit/serialization/import_read.cpp", - "torch/csrc/jit/serialization/unpickler.cpp", - ], - header_namespace = "", - exported_headers = [ - "torch/csrc/jit/serialization/import_read.h", - "torch/csrc/jit/serialization/unpickler.h", - ], - compiler_flags = get_pt_compiler_flags(), - link_whole = True, - linker_flags = [ - "-Wl,--no-as-needed", - ], - visibility = ["PUBLIC"], - exported_deps = [ - ":aten_cpu", - ":caffe2_headers", - ":caffe2_serialize", - ":torch_common", - ":torch_headers", - ":torch_mobile_headers", - ":torch_mobile_module", - ":torch_mobile_observer", - "//third_party:glog", - "//c10:c10", - ], -) - cxx_library( name = "caffe2_serialize", srcs = [ @@ -270,88 +148,6 @@ cxx_library( ], ) -cxx_library( - name = "torch_mobile_deserialize", - srcs = [ - "torch/csrc/jit/mobile/import.cpp", - ], - header_namespace = "", - exported_headers = [ - "torch/csrc/jit/mobile/import.h", - ], - compiler_flags = get_pt_compiler_flags(), - link_whole = True, - linker_flags = [ - "-Wl,--no-as-needed", - ], - visibility = ["PUBLIC"], - exported_deps = [ - ":aten_cpu", - ":caffe2_headers", - ":caffe2_serialize", - ":torch_common", - ":torch_headers", - ":torch_mobile_headers", - ":torch_mobile_module", - ":torch_mobile_observer", - "//third_party:glog", - "//c10:c10", - ":torch_mobile_deserialize_common", - ], -) - -cxx_library( - name = "torch_mobile_module", - srcs = [ - "torch/csrc/jit/mobile/function.cpp", - "torch/csrc/jit/mobile/interpreter.cpp", - "torch/csrc/jit/mobile/module.cpp", - ], - header_namespace = "", - exported_headers = [], - compiler_flags = get_pt_compiler_flags(), - link_whole = True, - linker_flags = [ - "-Wl,--no-as-needed", - ], - visibility = ["PUBLIC"], - exported_deps = [ - ":aten_cpu", - ":caffe2_headers", - ":torch_common", - ":torch_headers", - ":torch_mobile_headers", - ":torch_mobile_observer", - "//third_party:glog", - "//c10:c10", - ], -) - -cxx_library( - name = "torch_mobile_core", - srcs = [], - header_namespace = "", - exported_headers = [], - compiler_flags = get_pt_compiler_flags(), - exported_preprocessor_flags = get_pt_preprocessor_flags(), - link_whole = True, - linker_flags = [ - "-Wl,--no-as-needed", - # "-ldl", - ], - visibility = ["PUBLIC"], - deps = [ - ":generated-autograd-headers", - ":torch_mobile_observer", - ":torch_mobile_headers", - ], - exported_deps = [ - ":aten_cpu", - ":torch_common", - ":torch_mobile_deserialize", - ], -) - pt_operator_library( name = "torch_mobile_ops_full_dev", check_decl = False, @@ -371,62 +167,6 @@ cxx_library( ), ) -python_library( - name = "gen_oplist_lib", - srcs = subdir_glob([ - ("tools/code_analyzer", "gen_oplist.py"), - ("tools/code_analyzer", "gen_op_registration_allowlist.py"), - ]), - base_module = "", - deps = [ - "//third_party:pyyaml", - "//tools/lite_interpreter:gen_selected_mobile_ops_header", - "//torchgen:torchgen", - ], -) - -python_binary( - name = "gen_oplist", - main_module = "gen_oplist", - visibility = ["PUBLIC"], - deps = [ - ":gen_oplist_lib", - ], -) - -python_library( - name = "gen_operators_yaml_lib", - srcs = subdir_glob([ - ("tools/code_analyzer", "gen_operators_yaml.py"), - ("tools/code_analyzer", "gen_op_registration_allowlist.py"), - ]), - base_module = "", - deps = [ - "//third_party:pyyaml", - "//torchgen:torchgen", - ], -) - -python_binary( - name = "gen_aten_bin", - main_module = "torchgen.gen", - visibility = [ - "PUBLIC", - ], - deps = [ - "//torchgen:torchgen", - ], -) - -python_binary( - name = "gen_operators_yaml", - main_module = "gen_operators_yaml", - visibility = ["PUBLIC"], - deps = [ - ":gen_operators_yaml_lib", - ], -) - cxx_binary( name = 'ptmobile_benchmark', srcs = [ diff --git a/buckbuild.bzl b/buckbuild.bzl index 9150061912c8..4e4d28dfd3c8 100644 --- a/buckbuild.bzl +++ b/buckbuild.bzl @@ -1,38 +1,109 @@ -load("//tools/build_defs:expect.bzl", "expect") - # NOTE: This file is shared by internal and OSS BUCK build. # These load paths point to different files in internal and OSS environment + load("//tools/build_defs:fb_native_wrapper.bzl", "fb_native") +load("//tools/build_defs:fb_python_binary.bzl", "fb_python_binary") +load("//tools/build_defs:fb_python_library.bzl", "fb_python_library") load("//tools/build_defs:fb_xplat_cxx_library.bzl", "fb_xplat_cxx_library") +load("//tools/build_defs:fb_xplat_genrule.bzl", "fb_xplat_genrule") +load("//tools/build_defs:fbsource_utils.bzl", "is_arvr_mode") load("//tools/build_defs:glob_defs.bzl", "subdir_glob") +load("//tools/build_defs:platform_defs.bzl", "APPLETVOS", "IOS", "MACOSX") +load("//tools/build_defs/windows:windows_flag_map.bzl", "windows_convert_gcc_clang_flags") load( ":build_variables.bzl", + "core_sources_common", + "core_sources_full_mobile_no_backend_interface", + "core_trainer_sources", "jit_core_headers", + "libtorch_profiler_sources", ) -def read_bool(section, field, default): +def read_bool(section, field, default, required = True): # @lint-ignore BUCKRESTRICTEDSYNTAX - value = read_config(section, field) - if value == None: + val = read_config(section, field) + if val != None: + if val in ["true", "True", "1"]: + return True + elif val in ["false", "False", "0"]: + return False + else: + fail( + "`{}:{}`: must be one of (0, 1, true, false, True, False), but was {}".format(section, field, val), + ) + elif default != None: return default - expect( - value == "0" or value == "1", - "{}.{} == \"{}\", wanted \"0\" or \"1\".".format(section, field, value), - ) - return bool(int(value)) + elif not required: + return None + else: + fail("`{}:{}`: no value set".format(section, field)) -def is_oss_build(): - return read_bool("pt", "is_oss", False) +def get_enable_eager_symbolication(): + return read_bool("pt", "enable_eager_symbolication", default = False, required = False) + +# @lint-ignore BUCKRESTRICTEDSYNTAX +IS_OSS = read_config("pt", "is_oss", "0") == "1" # True for OSS BUCK build, and False for internal BUCK build + +NOT_OSS = not IS_OSS # for targets in caffe2 root path -ROOT_NAME = "//" if is_oss_build() else "//xplat/caffe2" +ROOT = "//" if IS_OSS else "//xplat/caffe2" # for targets in subfolders -ROOT_PATH = "//" if is_oss_build() else "//xplat/caffe2/" +ROOT_PATH = "//" if IS_OSS else "//xplat/caffe2/" + +C10 = "//c10:c10" if IS_OSS else "//xplat/caffe2/c10:c10" + +# a dictionary maps third party library name to fbsource and oss target +THIRD_PARTY_LIBS = { + "FP16": ["//third-party/FP16:FP16", "//third_party:FP16"], + "FXdiv": ["//third-party/FXdiv:FXdiv", "//third_party:FXdiv"], + "XNNPACK": ["//xplat/third-party/XNNPACK:XNNPACK", "//third_party:XNNPACK"], + "clog": ["//third-party/clog:clog", "//third_party:clog"], + "cpuinfo": ["//third-party/cpuinfo:cpuinfo", "//third_party:cpuinfo"], + "flatbuffers-api": ["//third-party/flatbuffers:flatbuffers-api", "//third_party:flatbuffers-api"], + "flatc": ["//third-party/flatbuffers:flatc", "//third_party:flatc"], + "fmt": ["//third-party/fmt:fmt", "//third_party:fmt"], + "glog": ["//third-party/glog:glog", "//third_party:glog"], + "kineto": ["//xplat/kineto/libkineto:libkineto", "//third_party:libkineto"], + "psimd": ["//third-party/psimd:psimd", "//third_party:psimd"], + "pthreadpool": ["//third-party/pthreadpool:pthreadpool", "//third_party:pthreadpool"], + "pthreadpool_header": ["//third-party/pthreadpool:pthreadpool_header", "//third_party:pthreadpool_header"], + "pyyaml": ["//third-party/pyyaml:pyyaml", "//third_party:pyyaml"], + "ruy": ["//third-party/ruy:ruy_xplat_lib", "//third_party:ruy_lib"], + "typing-extensions": ["//third-party/typing-extensions:typing-extensions", "//third_party:typing-extensions"], +} + +def third_party(name): + if name not in THIRD_PARTY_LIBS: + fail("Cannot find thrid party library " + name + ", please register it in THIRD_PARTY_LIBS first!") + return THIRD_PARTY_LIBS[name][1] if IS_OSS else THIRD_PARTY_LIBS[name][0] + +def get_pt_compiler_flags(): + return select({ + "DEFAULT": _PT_COMPILER_FLAGS + [ + "-std=gnu++17", #to accomodate for eigen + ], + "ovr_config//compiler:cl": windows_convert_gcc_clang_flags(_PT_COMPILER_FLAGS), + }) + +_PT_COMPILER_FLAGS = [ + "-frtti", + "-Os", + "-Wno-unknown-pragmas", + "-Wno-write-strings", + "-Wno-unused-variable", + "-Wno-unused-function", + "-Wno-deprecated-declarations", + "-Wno-shadow", + "-Wno-global-constructors", + "-Wno-missing-prototypes", +] # these targets are shared by internal and OSS BUCK def define_buck_targets( - feature = None, + pt_xplat_cxx_library, + c2_fbandroid_xplat_compiler_flags = [], labels = []): fb_xplat_cxx_library( name = "th_header", @@ -53,7 +124,6 @@ def define_buck_targets( ("aten/src", "THNN/generic/*.h"), ("aten/src", "THNN/generic/*.c"), ]), - feature = feature, labels = labels, ) @@ -93,7 +163,6 @@ def define_buck_targets( ("aten/src", "ATen/native/mkldnn/*.h"), ]), visibility = ["PUBLIC"], - feature = feature, labels = labels, ) @@ -106,7 +175,6 @@ def define_buck_targets( ("aten/src", "ATen/native/vulkan/ops/*.h"), ("aten/src", "ATen/vulkan/*.h"), ]), - feature = feature, labels = labels, visibility = ["PUBLIC"], ) @@ -115,7 +183,6 @@ def define_buck_targets( name = "jit_core_headers", header_namespace = "", exported_headers = subdir_glob([("", x) for x in jit_core_headers]), - feature = feature, labels = labels, ) @@ -143,7 +210,6 @@ def define_buck_targets( "torch/csrc/jit/serialization/mobile_bytecode_generated.h", ], ), - feature = feature, labels = labels, visibility = ["PUBLIC"], deps = [ @@ -167,7 +233,6 @@ def define_buck_targets( ("", "torch/csrc/jit/mobile/*.h"), ], ), - feature = feature, labels = labels, visibility = ["PUBLIC"], ) @@ -178,7 +243,6 @@ def define_buck_targets( exported_headers = { "Config.h": ":generate_aten_config[Config.h]", }, - feature = feature, labels = labels, ) @@ -192,7 +256,6 @@ def define_buck_targets( # Don't build python bindings on mobile. #"python_functions.h", }, - feature = feature, labels = labels, visibility = ["PUBLIC"], ) @@ -203,7 +266,6 @@ def define_buck_targets( exported_headers = { "version.h": ":generate-version-header[version.h]", }, - feature = feature, labels = labels, ) @@ -240,3 +302,927 @@ def define_buck_targets( "PUBLIC", ], ) + + fb_xplat_cxx_library( + name = "common_core", + srcs = [ + "caffe2/core/common.cc", + ], + apple_sdks = (IOS, MACOSX, APPLETVOS), + compiler_flags = get_pt_compiler_flags(), + labels = labels, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + visibility = ["PUBLIC"], + windows_preferred_linkage = "static" if is_arvr_mode() else None, + deps = [ + ":caffe2_headers", + C10, + ], + ) + + fb_python_library( + name = "substitutelib", + srcs = ["tools/substitute.py"], + base_module = "", + ) + + fb_python_binary( + name = "substitute", + main_module = "tools.substitute", + visibility = ["PUBLIC"], + deps = [ + ":substitutelib", + ], + ) + + # @lint-ignore BUCKLINT + fb_native.genrule( + name = "generate_aten_config", + srcs = [ + "aten/src/ATen/Config.h.in", + ], + cmd = "$(exe :substitute) " + " ".join([ + "--install_dir", + "$OUT", + "--input-file", + "aten/src/ATen/Config.h.in", + "--output-file", + "Config.h", + "--replace", + "@AT_MKLDNN_ENABLED@", + "ATEN_MKLDNN_ENABLED_FBXPLAT", + "--replace", + "@AT_MKL_ENABLED@", + "ATEN_MKL_ENABLED_FBXPLAT", + "--replace", + "@AT_MKL_SEQUENTIAL@", + "ATEN_MKL_SEQUENTIAL_FBXPLAT", + "--replace", + "@AT_FFTW_ENABLED@", + "0", + "--replace", + "@AT_POCKETFFT_ENABLED@", + "0", + "--replace", + "@AT_NNPACK_ENABLED@", + "ATEN_NNPACK_ENABLED_FBXPLAT", + "--replace", + "@CAFFE2_STATIC_LINK_CUDA_INT@", + "CAFFE2_STATIC_LINK_CUDA_FBXPLAT", + "--replace", + "@AT_BUILD_WITH_BLAS@", + "USE_BLAS_FBXPLAT", + "--replace", + "@AT_PARALLEL_OPENMP@", + "AT_PARALLEL_OPENMP_FBXPLAT", + "--replace", + "@AT_PARALLEL_NATIVE@", + "AT_PARALLEL_NATIVE_FBXPLAT", + "--replace", + "@AT_PARALLEL_NATIVE_TBB@", + "AT_PARALLEL_NATIVE_TBB_FBXPLAT", + "--replace", + "@AT_BUILD_WITH_LAPACK@", + "USE_LAPACK_FBXPLAT", + "--replace", + "@AT_BLAS_F2C@", + "AT_BLAS_F2C_FBXPLAT", + "--replace", + "@AT_BLAS_USE_CBLAS_DOT@", + "AT_BLAS_USE_CBLAS_DOT_FBXPLAT", + ]), + outs = { + "Config.h": ["Config.h"], + }, + default_outs = ["."], + ) + + fb_python_binary( + name = "gen_aten_bin", + main_module = "torchgen.gen", + visibility = [ + "PUBLIC", + ], + deps = [ + ROOT_PATH + "torchgen:torchgen", + ], + ) + + fb_python_binary( + name = "gen_unboxing_bin", + main_module = "tools.jit.gen_unboxing", + visibility = [ + "PUBLIC", + ], + deps = [ + ROOT_PATH + "tools/jit:jit", + ], + ) + + fb_python_library( + name = "gen_oplist_lib", + srcs = subdir_glob([ + ("tools/code_analyzer", "gen_oplist.py"), + ("tools/code_analyzer", "gen_op_registration_allowlist.py"), + ]), + base_module = "", + tests = [ + ":gen_oplist_test", + ], + deps = [ + third_party("pyyaml"), + ROOT_PATH + "tools/lite_interpreter:gen_selected_mobile_ops_header", + ROOT_PATH + "torchgen:torchgen", + ], + ) + + fb_python_library( + name = "gen_operators_yaml_lib", + srcs = subdir_glob([ + ("tools/code_analyzer", "gen_operators_yaml.py"), + ("tools/code_analyzer", "gen_op_registration_allowlist.py"), + ]), + base_module = "", + tests = [ + ":gen_operators_yaml_test", + ], + deps = [ + third_party("pyyaml"), + ROOT_PATH + "torchgen:torchgen", + ], + ) + + fb_python_binary( + name = "gen_aten_vulkan_spv_bin", + main_module = "aten.src.ATen.gen_vulkan_spv", + visibility = [ + "PUBLIC", + ], + deps = [ + ":gen_aten_vulkan_spv_lib", + ], + ) + + fb_python_library( + name = "gen_aten_vulkan_spv_lib", + srcs = [ + "aten/src/ATen/gen_vulkan_spv.py", + ], + base_module = "", + deps = [ + ROOT_PATH + "torchgen:torchgen", + ], + ) + + fb_python_binary( + name = "gen_oplist", + main_module = "gen_oplist", + visibility = ["PUBLIC"], + deps = [ + ":gen_oplist_lib", + ], + ) + + fb_python_binary( + name = "gen_operators_yaml", + main_module = "gen_operators_yaml", + visibility = ["PUBLIC"], + deps = [ + ":gen_operators_yaml_lib", + ], + ) + + fb_xplat_cxx_library( + name = "torch_mobile_observer", + srcs = [ + "torch/csrc/jit/mobile/observer.cpp", + ] + ([] if IS_OSS else ["torch/fb/observers/MobileObserverUtil.cpp"]), + header_namespace = "", + exported_headers = subdir_glob( + [ + ("", "torch/csrc/jit/mobile/observer.h"), + ] + ([] if IS_OSS else [ + ("", "torch/fb/observers/ObserverUtil.h"), + ("", "torch/fb/observers/MobileObserverUtil.h"), + ]), + ), + fbobjc_compiler_flags = [ + "-Wno-missing-prototypes", + ], + labels = labels, + visibility = ["PUBLIC"], + deps = [ + C10, + ], + ) + + # Base library shared by lite-interpreter and full-jit. + pt_xplat_cxx_library( + name = "torch_common", + srcs = core_sources_common, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + visibility = ["PUBLIC"], + deps = [ + ":aten_cpu", + ":generated-autograd-headers", + ":torch_headers", + C10, + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_deserialize_common", + srcs = [ + "torch/csrc/jit/mobile/parse_bytecode.cpp", + "torch/csrc/jit/mobile/parse_operators.cpp", + "torch/csrc/jit/mobile/upgrader_mobile.cpp", + "torch/csrc/jit/serialization/import_read.cpp", + "torch/csrc/jit/serialization/unpickler.cpp", + ], + header_namespace = "", + exported_headers = [ + "torch/csrc/jit/serialization/import_read.h", + "torch/csrc/jit/serialization/unpickler.h", + ], + extra_flags = { + "fbandroid_compiler_flags": ["-frtti"], + }, + # torch_mobile_deserialize brings in sources neccessary to read a module + # which depends on mobile module definition + # link_whole is enable so that all symbols neccessary for mobile module are compiled + # instead of only symbols used while loading; this prevents symbol + # found definied in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = ["-Wl,--no-as-needed"], + visibility = ["PUBLIC"], + exported_deps = [ + ":aten_cpu", + ":caffe2_headers", + ":caffe2_serialize", + ":torch_common", + ":torch_headers", + ":torch_mobile_headers", + ":torch_mobile_module", + ":torch_mobile_observer", + C10, + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_module", + srcs = [ + "torch/csrc/jit/mobile/function.cpp", + "torch/csrc/jit/mobile/interpreter.cpp", + "torch/csrc/jit/mobile/module.cpp", + ], + header_namespace = "", + exported_headers = [ + ], + extra_flags = { + "fbandroid_compiler_flags": ["-frtti"], + }, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + exported_deps = [ + ":aten_cpu", + ":caffe2_headers", + ":torch_common", + ":torch_headers", + ":torch_mobile_headers", + ":torch_mobile_observer", + C10, + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_debug_symbolication", + srcs = [ + # included in aten_cpu "torch/csrc/jit/frontend/source_range.cpp", + "torch/csrc/jit/ir/scope.cpp", + "torch/csrc/jit/mobile/debug_info.cpp", + "torch/csrc/jit/serialization/callstack_debug_info_serialization.cpp", + "torch/csrc/jit/serialization/source_range_serialization.cpp", + "torch/csrc/jit/serialization/pickle.cpp", + # pickler.cpp doesn't seem to be needed. + # "torch/csrc/jit/serialization/pickler.cpp", + # included in core_sources_common "torch/csrc/jit/serialization/unpickler.cpp", + ], + header_namespace = "", + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [ + ":torch_mobile_deserialize", + ], + exported_deps = [ + ":torch_common", + ], + ) + + pt_xplat_cxx_library( + name = "torch_model_tracer", + srcs = [ + "torch/csrc/jit/mobile/model_tracer/BuildFeatureTracer.cpp", + "torch/csrc/jit/mobile/model_tracer/CustomClassTracer.cpp", + "torch/csrc/jit/mobile/model_tracer/KernelDTypeTracer.cpp", + "torch/csrc/jit/mobile/model_tracer/OperatorCallTracer.cpp", + "torch/csrc/jit/mobile/model_tracer/TracerRunner.cpp", + ], + header_namespace = "", + extra_flags = { + "exported_preprocessor_flags": ["-DSYMBOLICATE_MOBILE_DEBUG_HANDLE"] if get_enable_eager_symbolication() else [], + }, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [ + ":generated-autograd-headers", + ":torch_mobile_deserialize", + ":torch_mobile_headers", + ":torch_mobile_observer", + ] + ([] if IS_OSS else ["//xplat/folly:molly"]), + exported_deps = [ + ":aten_cpu", + ":torch_common", + ] + ([] if IS_OSS else [ + "//xplat/caffe2/fb/custom_ops/batch_box_cox:batch_box_cox", + "//xplat/caffe2/fb/custom_ops/maskrcnn:maskrcnn", + ]), + ) + + pt_xplat_cxx_library( + name = "torch_mobile_deserialize", + srcs = [ + "torch/csrc/jit/mobile/import.cpp", + ], + header_namespace = "", + exported_headers = [ + "torch/csrc/jit/mobile/import.h", + ], + # torch_mobile_deserialize brings in sources neccessary to read a module + # which depends on mobile module definition + # link_whole is enable so that all symbols neccessary for mobile module are compiled + # instead of only symbols used while loading; this prevents symbol + # found definied in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + exported_deps = [ + ":aten_cpu", + ":caffe2_headers", + ":caffe2_serialize", + ":torch_common", + ":torch_headers", + ":torch_mobile_headers", + ":torch_mobile_module", + ":torch_mobile_observer", + ":torch_mobile_deserialize_common", + C10, + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_core", + srcs = [], + header_namespace = "", + exported_headers = [], + extra_flags = { + "exported_preprocessor_flags": (["-DSYMBOLICATE_MOBILE_DEBUG_HANDLE"] if get_enable_eager_symbolication() else []), + }, + # torch_mobile_core brings in sources neccessary to read and run a module + # link_whole is enabled so that all symbols linked + # operators, registerations and other few symbols are need in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [ + ":generated-autograd-headers", + ":torch_mobile_headers", + ":torch_mobile_observer", + ], + exported_deps = [ + ":aten_cpu", + ":torch_common", + ":torch_mobile_deserialize", + ":torch_supported_mobile_models", + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_core_pickle_and_flatbuffer", + visibility = ["PUBLIC"], + exported_deps = [ + ":torch_flatbuffer_all", + ":torch_mobile_core", + ], + ) + + pt_xplat_cxx_library( + name = "torch_core", + srcs = core_sources_full_mobile_no_backend_interface + [ + "torch/csrc/api/src/jit.cpp", + "torch/csrc/jit/serialization/export_bytecode.cpp", + "torch/csrc/jit/serialization/export_module.cpp", + ], + visibility = [ + "//xplat/caffe2/android/...", + "//xplat/caffe2/fb/...", + "//xplat/caffe2/fb/model_tracer/...", + ], + deps = [ + ":aten_cpu", + ":backend_interface_lib", + ":generated-autograd-headers", + ":torch_headers", + ":torch_mobile_deserialize", + third_party("glog"), + C10, + ] + ([] if IS_OSS else [ + "//xplat/caffe2/fb/custom_ops/batch_box_cox:batch_box_cox", + "//xplat/caffe2/fb/custom_ops/maskrcnn:maskrcnn", + "//xplat/third-party/linker_lib:rt", + ]), + exported_deps = [ + ":torch_common", + ":torch_mobile_train", + ], + ) + + pt_xplat_cxx_library( + name = "torch_train", + srcs = [ + "torch/csrc/api/src/data/samplers/random.cpp", + "torch/csrc/api/src/data/samplers/sequential.cpp", + "torch/csrc/api/src/optim/optimizer.cpp", + "torch/csrc/api/src/optim/serialize.cpp", + "torch/csrc/api/src/optim/sgd.cpp", + "torch/csrc/api/src/serialize/input-archive.cpp", + "torch/csrc/api/src/serialize/output-archive.cpp", + "torch/csrc/jit/api/module_save.cpp", + ], + visibility = ["PUBLIC"], + deps = [ + ":aten_cpu", + ":torch_headers", + ":torch", + ":torch_core", + ":torch_mobile_deserialize", + ":torch_mobile_train", + C10, + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_train", + srcs = core_trainer_sources + [ + "torch/csrc/autograd/VariableTypeManual.cpp", + "torch/csrc/autograd/FunctionsManual.cpp", + "torch/csrc/api/src/data/datasets/mnist.cpp", + "torch/csrc/jit/mobile/train/export_data.cpp", + "torch/csrc/jit/mobile/train/optim/sgd.cpp", + "torch/csrc/jit/mobile/train/random.cpp", + "torch/csrc/jit/mobile/train/sequential.cpp", + ":gen_aten_libtorch[autograd/generated/Functions.cpp]", + ], + extra_flags = { + "exported_preprocessor_flags": ["-DUSE_MOBILE_CLASSTYPE"], + }, + # torch_mobile_train brings in sources neccessary to read and run a mobile + # and save and load mobile params along with autograd + # link_whole is enabled so that all symbols linked + # operators, registerations and autograd related symbols are need in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + visibility = ["PUBLIC"], + deps = [ + ":aten_cpu", + ":generated-autograd-headers", + ":torch_headers", + ":torch_mobile_deserialize", + C10, + ], + ) + + pt_xplat_cxx_library( + name = "torch", + srcs = [ + "torch/csrc/jit/runtime/register_c10_ops.cpp", + "torch/csrc/jit/runtime/register_prim_ops_fulljit.cpp", + ], + # torch brings in all sources neccessary to read and run a mobile module/jit module + # link_whole is enabled so that all symbols linked + # operators, registerations and other few symbols are need in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + visibility = ["PUBLIC"], + deps = [ + # This is to have autograd profiler available + # in xplat/caffe2:torch which some builds are using + # notable xplate/facegen:testsAndroid + ":torch_headers", + ":torch_kineto_profiling", + ], + exported_deps = [ + ":aten_cpu", + ":torch_core", + C10, + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_train_import_data", + srcs = [ + "torch/csrc/jit/mobile/import_data.cpp", + ], + extra_flags = { + "exported_preprocessor_flags": ["-DUSE_MOBILE_CLASSTYPE"], + }, + # torch_mobile_train_import_data brings in sources neccessary to read a mobile module + # link_whole is enabled so that all symbols linked + # operators other few symbols are need in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + visibility = ["PUBLIC"], + deps = [ + ":torch_headers", + ":torch_mobile_observer", + ":torch_mobile_core", + ":torch_mobile_train", + ], + ) + + fb_xplat_cxx_library( + name = "torch_mobile_compatibility", + srcs = [ + # These .cpp brought in through core_sources_common + # "torch/csrc/jit/mobile/compatibility/runtime_compatibility.cpp", + # "torch/csrc/jit/serialization/unpickler.cpp", + "torch/csrc/jit/mobile/compatibility/model_compatibility.cpp", + "torch/csrc/jit/serialization/pickle.cpp", + "torch/csrc/jit/serialization/pickler.cpp", + ], + header_namespace = "", + exported_headers = [ + "torch/csrc/jit/mobile/compatibility/backport.h", + "torch/csrc/jit/mobile/compatibility/backport_manager.h", + "torch/csrc/jit/mobile/compatibility/model_compatibility.h", + "torch/csrc/jit/mobile/compatibility/runtime_compatibility.h", + ], + compiler_flags = [ + "-fexceptions", + "-frtti", + "-Wno-deprecated-declarations", + "-Wno-global-constructors", + ], + labels = labels, + visibility = ["PUBLIC"], + deps = [ + ":torch_mobile_deserialize", + ], + ) + + pt_xplat_cxx_library( + name = "jit_module_saving", + srcs = [ + "torch/csrc/jit/api/module_save.cpp", + "torch/csrc/jit/serialization/export_bytecode.cpp", + "torch/csrc/jit/serialization/export_module.cpp", + ], + exported_headers = [ + "torch/csrc/jit/serialization/export.h", + "torch/csrc/jit/serialization/flatbuffer_serializer_jit.h", + ], + visibility = ["PUBLIC"], + deps = [ + ":torch", + ":torch_mobile_core", + ], + ) + + pt_xplat_cxx_library( + name = "torch_mobile_model_tracer", + srcs = [ + "torch/csrc/jit/mobile/model_tracer/MobileModelRunner.cpp", + "torch/csrc/jit/mobile/model_tracer/TensorUtils.cpp", + ], + headers = [ + "torch/csrc/jit/mobile/model_tracer/MobileModelRunner.h", + "torch/csrc/jit/mobile/model_tracer/TensorUtils.h", + ], + header_namespace = "", + exported_headers = [ + "torch/csrc/jit/mobile/model_tracer/MobileModelRunner.h", + ], + extra_flags = { + "exported_preprocessor_flags": (["-DSYMBOLICATE_MOBILE_DEBUG_HANDLE"] if get_enable_eager_symbolication() else []), + }, + # torch_mobile_model_tracer brings in sources neccessary to read and run a jit module + # and trace the ops + # link_whole is enabled so that all symbols linked + # operators, registerations and other few symbols are need in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [ + ":caffe2_serialize", + ":generated-autograd-headers", + ":torch_mobile_headers", + ":torch_mobile_observer", + ":torch_mobile_core", + ] + ([] if IS_OSS else ["//xplat/folly:molly"]), + exported_deps = [ + ":aten_cpu", + ":torch_common", + ] + ([] if IS_OSS else [ + "//xplat/caffe2/fb/custom_ops/batch_box_cox:batch_box_cox", + "//xplat/caffe2/fb/custom_ops/maskrcnn:maskrcnn", + "//xplat/caffe2/fb/custom_ops/sparsenn:sparsenn-all", + ]), + ) + + pt_xplat_cxx_library( + name = "torch_mobile_core_flatbuffer", + srcs = [], + header_namespace = "", + exported_headers = [], + extra_flags = { + "exported_preprocessor_flags": ["-DSYMBOLICATE_MOBILE_DEBUG_HANDLE"] if get_enable_eager_symbolication() else [], + }, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [ + ":generated-autograd-headers", + ":torch_mobile_headers", + ":torch_mobile_observer", + ], + exported_deps = [ + ":aten_cpu", + ":torch_common", + ] + ([] if IS_OSS else [ + "//xplat/caffe2/fb/runtime:torch_mobile_deserialize_flatbuffer", + ]), + ) + + fb_xplat_cxx_library( + name = "backend_interface_lib", + srcs = [ + "torch/csrc/jit/backends/backend_debug_info.cpp", + "torch/csrc/jit/backends/backend_interface.cpp", + ], + compiler_flags = get_pt_compiler_flags(), + fbandroid_compiler_flags = c2_fbandroid_xplat_compiler_flags, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + exported_deps = [ + ":aten_cpu", + ":torch_common", + ], + ) + + pt_xplat_cxx_library( + name = "torch_kineto_profiling", + srcs = libtorch_profiler_sources, + extra_flags = { + "compiler_flags": ["-Wno-error"], + "exported_preprocessor_flags": [ + "-DUSE_KINETO", + "-DUSE_KINETO_UPDATED", + # Need this otherwise USE_KINETO is undefed + # for mobile + "-DEDGE_PROFILER_USE_KINETO", + ], + }, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [ + third_party("glog"), + third_party("kineto"), + ], + exported_deps = [ + ":aten_cpu", + ":torch_common", + ], + ) + + pt_xplat_cxx_library( + name = "torch_edge_profiling", + srcs = ["torch/csrc/jit/mobile/profiler_edge.cpp"], + extra_flags = { + "exported_preprocessor_flags": [ + "-DUSE_KINETO", + "-DUSE_KINETO_UPDATED", + "-DEDGE_PROFILER_USE_KINETO", + ], + }, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + exported_deps = [ + ":torch_common", + ":torch_kineto_profiling", + ":torch_mobile_core", + ], + ) + + fb_xplat_genrule( + name = "mobile_bytecode_header", + srcs = [ + "torch/csrc/jit/serialization/mobile_bytecode.fbs", + ], + outs = { + "mobile_bytecode_generated.h": ["mobile_bytecode_generated.h"], + }, + cmd = "$(exe {})".format(third_party("flatc")) + + " --cpp --gen-mutable --scoped-enums -o ${OUT} ${SRCS}", + default_outs = ["."], + ) + + fb_xplat_cxx_library( + name = "mobile_bytecode", + header_namespace = "", + exported_headers = { + "torch/csrc/jit/serialization/mobile_bytecode_generated.h": ":mobile_bytecode_header[mobile_bytecode_generated.h]", + }, + exported_deps = [ + third_party("flatbuffers-api"), + ], + ) + + fb_xplat_cxx_library( + name = "flatbuffer_serializer", + srcs = ["torch/csrc/jit/serialization/flatbuffer_serializer.cpp"], + exported_headers = [ + "torch/csrc/jit/serialization/flatbuffer_serializer.h", + ], + compiler_flags = [ + "-g0", + "-O3", + "-fexceptions", + "-frtti", + "-Wno-deprecated-declarations", + ], + visibility = ["PUBLIC"], + deps = [ + ":torch_mobile_module", + C10, + ], + exported_deps = [ + ":flatbuffer_loader", + ":mobile_bytecode", + ":torch_mobile_train", + third_party("flatbuffers-api"), + ], + ) + + pt_xplat_cxx_library( + name = "flatbuffer_loader", + srcs = [ + "torch/csrc/jit/mobile/flatbuffer_loader.cpp", + ], + exported_headers = [ + "torch/csrc/jit/mobile/flatbuffer_loader.h", + ], + extra_flags = { + "fbandroid_compiler_flags": ["-frtti"], + }, + # torch_mobile_deserialize brings in sources neccessary to read a module + # which depends on mobile module definition + # link_whole is enable so that all symbols neccessary for mobile module are compiled + # instead of only symbols used while loading; this prevents symbol + # found definied in runtime + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + exported_deps = [ + ":mobile_bytecode", + ":torch_mobile_deserialize", + third_party("flatbuffers-api"), + C10, + ], + ) + + fb_xplat_cxx_library( + name = "flatbuffer_serializer_jit", + srcs = ["torch/csrc/jit/serialization/flatbuffer_serializer_jit.cpp"], + exported_headers = [ + "torch/csrc/jit/serialization/flatbuffer_serializer_jit.h", + ], + compiler_flags = [ + "-g0", + "-O3", + "-fexceptions", + "-frtti", + "-Wno-deprecated-declarations", + ], + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [ + ":flatbuffer_loader", + ":flatbuffer_serializer", + ":mobile_bytecode", + ":torch_core", + ":torch_mobile_module", + third_party("flatbuffers-api"), + C10, + ], + ) + + fb_xplat_cxx_library( + name = "torch_flatbuffer_all", + visibility = ["PUBLIC"], + exported_deps = [ + ":flatbuffer_loader", + ":flatbuffer_serializer", + ":flatbuffer_serializer_jit", + ], + ) + + pt_xplat_cxx_library( + name = "torch_supported_mobile_models", + srcs = [ + "fb/supported_mobile_models/SupportedMobileModels.cpp", + ] if NOT_OSS else [], + header_namespace = "", + exported_headers = ["fb/supported_mobile_models/SupportedMobileModels.h"] if NOT_OSS else [], + extra_flags = { + "exported_preprocessor_flags": ["-DSYMBOLICATE_MOBILE_DEBUG_HANDLE"] if get_enable_eager_symbolication() else [], + }, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + linker_flags = [ + "-Wl,--no-as-needed", + ], + visibility = ["PUBLIC"], + deps = [], + exported_deps = [ + "//xplat/caffe2/fb/custom_ops/batch_box_cox:batch_box_cox", + "//xplat/caffe2/fb/custom_ops/maskrcnn:maskrcnn", + ] if NOT_OSS else [], + ) + + fb_xplat_cxx_library( + name = "static_runtime", + srcs = [ + "torch/csrc/jit/runtime/static/fusion.cpp", + "torch/csrc/jit/runtime/static/generated_ops.cpp", + "torch/csrc/jit/runtime/static/impl.cpp", + "torch/csrc/jit/runtime/static/memory_planner.cpp", + "torch/csrc/jit/runtime/static/native_ops.cpp", + "torch/csrc/jit/runtime/static/ops.cpp", + "torch/csrc/jit/runtime/static/passes.cpp", + "torch/csrc/jit/runtime/static/te_wrapper.cpp", + ], + labels = labels, + # @lint-ignore BUCKLINT link_whole + link_whole = True, + visibility = ["PUBLIC"], + windows_preferred_linkage = "static" if is_arvr_mode() else None, + deps = [ + ":aten_cpu", + ":caffe2_headers", + ":torch_core", + C10, + ], + ) diff --git a/pt_defs.oss.bzl b/pt_defs.oss.bzl index 879acb31f8b8..e1cedf0035bb 100644 --- a/pt_defs.oss.bzl +++ b/pt_defs.oss.bzl @@ -1,4 +1,5 @@ load("@bazel_skylib//lib:paths.bzl", "paths") +load("//tools/build_defs:fb_xplat_cxx_library.bzl", "fb_xplat_cxx_library") load("//tools/build_defs:fb_xplat_genrule.bzl", "fb_xplat_genrule") load("//tools/build_defs:type_defs.bzl", "is_list", "is_string") load(":build_variables.bzl", "aten_native_source_list") @@ -220,6 +221,20 @@ TEMPLATE_SOURCE_LIST = [ "torch/csrc/jit/runtime/register_special_ops.cpp", ] + aten_native_source_list +# the basic xplat library for OSS build +def pt_xplat_cxx_library(extra_flags = {}, **kwargs): + fb_xplat_cxx_library( + compiler_flags = get_pt_compiler_flags() + extra_flags.get("compiler_flags", []), + exported_preprocessor_flags = get_pt_preprocessor_flags() + extra_flags.get("exported_preprocessor_flags", []), + **kwargs + ) + +def get_aten_default_args(): + return dict( + compiler_flags = get_aten_compiler_flags(), + exported_preprocessor_flags = get_aten_preprocessor_flags(), + ) + # For selective build, we can lump the CPU and CPU kernel sources altogether # because there is only ever one vectorization variant that is compiled def aten_ufunc_generated_all_cpu_sources(gencode_pattern = "{}"): diff --git a/tools/build_defs/buck_helpers.bzl b/tools/build_defs/buck_helpers.bzl index c946f87eba06..521661239802 100644 --- a/tools/build_defs/buck_helpers.bzl +++ b/tools/build_defs/buck_helpers.bzl @@ -25,49 +25,3 @@ def filter_attributes(kwgs): if key.startswith(invalid_prefix): kwgs.pop(key) return kwgs - -# maps known fbsource deps to OSS deps -DEPS_MAP = { - "//third-party/FP16:FP16": "//third_party:FP16", - "//third-party/FXdiv:FXdiv": "//third_party:FXdiv", - "//third-party/XNNPACK:XNNPACK": "//third_party:XNNPACK", - "//third-party/clog:clog": "//third_party:clog", - "//third-party/cpuinfo:cpuinfo": "//third_party:cpuinfo", - "//third-party/fmt:fmt": "//third_party:fmt", - "//third-party/glog:glog": "//third_party:glog", - "//third-party/psimd:psimd": "//third_party:psimd", - "//third-party/pthreadpool:pthreadpool": "//third_party:pthreadpool", - "//third-party/pthreadpool:pthreadpool_header": "//third_party:pthreadpool_header", - "//third-party/ruy:ruy_xplat_lib": "//third_party:ruy_lib", -} - -# map fbsource deps to OSS deps -def to_oss_deps(deps = []): - new_deps = [] - for dep in deps: - new_deps += map_deps(dep) - return new_deps - -def map_deps(dep): - # remove @fbsource prefix - if dep.startswith("@fbsource"): - dep = dep[len("@fbsource"):] - - # ignore all fbsource linker_lib targets - if dep.startswith("//xplat/third-party/linker_lib"): - return [] - - # map targets in caffe2 root folder. Just use relative path - if dep.startswith("//xplat/caffe2:"): - return [dep[len("//xplat/caffe2"):]] - - # map targets in caffe2 subfolders - if dep.startswith("//xplat/caffe2/"): - return ["//" + dep[len("//xplat/caffe2/"):]] - - # map other known targets - if dep in DEPS_MAP: - return DEPS_MAP[dep] - - # drop other unknown deps - return [] diff --git a/tools/build_defs/fb_python_binary.bzl b/tools/build_defs/fb_python_binary.bzl index f403b4bbbda6..5e69f32881b0 100644 --- a/tools/build_defs/fb_python_binary.bzl +++ b/tools/build_defs/fb_python_binary.bzl @@ -1,7 +1,6 @@ # Only used for PyTorch open source BUCK build # @lint-ignore-every BUCKRESTRICTEDSYNTAX - -load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") +# @lint-ignore-every FBCODEBZLADDLOADS def fb_python_binary(**kwgs): if read_config("pt", "is_oss", "0") == "0": diff --git a/tools/build_defs/fb_python_library.bzl b/tools/build_defs/fb_python_library.bzl new file mode 100644 index 000000000000..e0ab86f77b7f --- /dev/null +++ b/tools/build_defs/fb_python_library.bzl @@ -0,0 +1,9 @@ +# Only used for PyTorch open source BUCK build +# @lint-ignore-every BUCKRESTRICTEDSYNTAX +# @lint-ignore-every FBCODEBZLADDLOADS + +def fb_python_library(**kwgs): + if read_config("pt", "is_oss", "0") == "0": + fail("This file is for open source pytorch build. Do not use it in fbsource!") + + python_library(**kwgs) diff --git a/tools/build_defs/fb_xplat_cxx_library.bzl b/tools/build_defs/fb_xplat_cxx_library.bzl index 785983606213..c73fa500a984 100644 --- a/tools/build_defs/fb_xplat_cxx_library.bzl +++ b/tools/build_defs/fb_xplat_cxx_library.bzl @@ -1,19 +1,18 @@ # Only used for PyTorch open source BUCK build # @lint-ignore-every BUCKRESTRICTEDSYNTAX -load(":buck_helpers.bzl", "filter_attributes", "to_oss_deps") +load( + ":buck_helpers.bzl", + "filter_attributes", +) def fb_xplat_cxx_library( name, - deps = [], - exported_deps = [], **kwgs): if read_config("pt", "is_oss", "0") == "0": fail("This file is for open source pytorch build. Do not use it in fbsource!") cxx_library( name = name, - deps = to_oss_deps(deps), - exported_deps = to_oss_deps(exported_deps), **filter_attributes(kwgs) ) diff --git a/tools/build_defs/select.bzl b/tools/build_defs/select.bzl new file mode 100644 index 000000000000..36c86e93cb8d --- /dev/null +++ b/tools/build_defs/select.bzl @@ -0,0 +1,8 @@ +# Only used for PyTorch open source BUCK build +# @lint-ignore-every BUCKRESTRICTEDSYNTAX + +def select(conditions): + if read_config("pt", "is_oss", "0") == "0": + fail("This file is for open source pytorch build. Do not use it in fbsource!") + + return conditions["DEFAULT"] diff --git a/tools/build_defs/windows/windows_flag_map.bzl b/tools/build_defs/windows/windows_flag_map.bzl new file mode 100644 index 000000000000..e25d9897e076 --- /dev/null +++ b/tools/build_defs/windows/windows_flag_map.bzl @@ -0,0 +1,9 @@ +# Only used for PyTorch open source BUCK build +# @lint-ignore-every BUCKRESTRICTEDSYNTAX + +def windows_convert_gcc_clang_flags(flags = []): + if read_config("pt", "is_oss", "0") == "0": + fail("This file is for open source pytorch build. Do not use it in fbsource!") + + # not implemented + return []