frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 12:54:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

[2] move more pytorch buck targets to shared build (#79330)

Browse Source 
Summary: as title

Test Plan: sandcastle and oss CI

Differential Revision: D37087229

Pull Request resolved: https://github.com/pytorch/pytorch/pull/79330
Approved by: https://github.com/dhruvbird
This commit is contained in:
Linbin Yu
2022-06-22 03:14:23 +00:00
committed by PyTorch MergeBot
parent 0f95e1846c
commit c9cbdb411d
10 changed files with 1062 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.buckconfig.oss
View File

@ -3,9 +3,11 @@
[buildfile]
name = BUCK.oss
includes = //tools/build_defs/select.bzl
[repositories]
bazel_skylib = third_party/bazel-skylib/
ovr_config = .
[download]
in_build = true

268
BUCK.oss
View File

@ -1,5 +1,4 @@
load("//tools/build_defs:glob_defs.bzl", "subdir_glob")
load("//tools/build_defs:fb_xplat_genrule.bzl", "fb_xplat_genrule")
load(
":build_variables.bzl",
"aten_cpu_source_list",
@ -21,13 +20,16 @@ load(
"pt_operator_library",
"get_pt_ops_deps",
"aten_ufunc_generated_all_cpu_sources",
"pt_xplat_cxx_library",
"TEMPLATE_SOURCE_LIST",
)
load(":buckbuild.bzl",
"define_buck_targets",
)
define_buck_targets()
define_buck_targets(
pt_xplat_cxx_library = pt_xplat_cxx_library,
)
cxx_library(
name = "pthreadpool",
@ -76,20 +78,6 @@ cxx_library(
visibility = ['PUBLIC'],
)
cxx_library(
name = "common_core",
srcs = ['caffe2/core/common.cc'],
deps = [':caffe2_headers', '//c10:c10'],
exported_deps = [],
compiler_flags = ['-frtti', '-Os', '-Wno-unknown-pragmas', '-Wno-write-strings', '-Wno-unused-variable', '-Wno-unused-function', '-Wno-deprecated-declarations', '-Wno-shadow', '-Wno-global-constructors', '-Wno-missing-prototypes', '-std=gnu++17'],
preferred_linkage = "static",
header_namespace = "caffe2",
headers = [],
link_whole = True,
platform_preprocessor_flags = [['windows', ['-D_WINDOWS', '-D_WIN32', '-DWIN32', '-DNOMINMAX', '-D_CRT_SECURE_NO_WARNINGS', '-D_USE_MATH_DEFINES']], ['windows.*64$', ['-D_WIN64']]],
visibility = ['PUBLIC'],
)
build_aten_cpu(
name = "aten_cpu",
srcs = jit_core_sources +
@ -109,35 +97,6 @@ build_aten_cpu(
],
)
fb_xplat_genrule(
name = "generate_aten_config",
srcs = [
"aten/src/ATen/Config.h.in",
],
cmd = " ".join([
"sed",
"-e 's/@AT_MKLDNN_ENABLED@/ATEN_MKLDNN_ENABLED_FBXPLAT/g'",
"-e 's/@AT_MKL_ENABLED@/ATEN_MKL_ENABLED_FBXPLAT/g'",
"-e 's/@AT_MKL_SEQUENTIAL@/ATEN_MKL_SEQUENTIAL_FBXPLAT/g'",
"-e 's/@AT_FFTW_ENABLED@/0/g'",
"-e 's/@AT_POCKETFFT_ENABLED@/0/g'",
"-e 's/@AT_NNPACK_ENABLED@/ATEN_NNPACK_ENABLED_FBXPLAT/g'",
"-e 's/@CAFFE2_STATIC_LINK_CUDA_INT@/CAFFE2_STATIC_LINK_CUDA_FBXPLAT/g'",
"-e 's/@AT_BUILD_WITH_BLAS@/USE_BLAS_FBXPLAT/g'",
"-e 's/@AT_PARALLEL_OPENMP@/AT_PARALLEL_OPENMP_FBXPLAT/g'",
"-e 's/@AT_PARALLEL_NATIVE@/AT_PARALLEL_NATIVE_FBXPLAT/g'",
"-e 's/@AT_PARALLEL_NATIVE_TBB@/AT_PARALLEL_NATIVE_TBB_FBXPLAT/g'",
"-e 's/@AT_BUILD_WITH_LAPACK@/USE_LAPACK_FBXPLAT/g'",
"-e 's/@AT_BLAS_F2C@/AT_BLAS_F2C_FBXPLAT/g'",
"-e 's/@AT_BLAS_USE_CBLAS_DOT@/AT_BLAS_USE_CBLAS_DOT_FBXPLAT/g'",
"aten/src/ATen/Config.h.in > $OUT/Config.h",
]),
outs = {
"Config.h": ["Config.h"],
},
default_outs = ["."],
)
gen_aten_files(
name = "gen_aten",
extra_flags = get_aten_codegen_extra_params(USED_PT_BACKENDS),
@ -172,87 +131,6 @@ cxx_library(
)
gen_aten_libtorch_files(name = "gen_aten_libtorch")
cxx_library(
name = "torch_mobile_observer",
srcs = [
"torch/csrc/jit/mobile/observer.cpp",
#"torch/fb/observers/MobileObserverUtil.cpp",
],
header_namespace = "",
exported_headers = subdir_glob(
[
("", "torch/csrc/jit/mobile/observer.h"),
#("", "torch/fb/observers/ObserverUtil.h"),
#("", "torch/fb/observers/MobileObserverUtil.h"),
],
),
visibility = ["PUBLIC"],
deps = [
"//c10:c10",
],
)
python_library(
name = "aten_code_template",
srcs = subdir_glob([
("aten", "src/ATen/code_template.py"),
]),
base_module = "",
visibility = ["PUBLIC"],
)
cxx_library(
name = "torch_common",
srcs = core_sources_common,
compiler_flags = get_pt_compiler_flags(),
exported_preprocessor_flags = get_pt_preprocessor_flags(),
link_whole = True,
visibility = ["PUBLIC"],
deps = [
":aten_cpu",
":generated-autograd-headers",
":torch_headers",
"//third_party:glog",
"//c10:c10",
],
)
cxx_library(
name = "torch_mobile_deserialize_common",
srcs = [
"torch/csrc/jit/mobile/parse_bytecode.cpp",
"torch/csrc/jit/mobile/parse_operators.cpp",
"torch/csrc/jit/mobile/upgrader_mobile.cpp",
"torch/csrc/jit/serialization/import_read.cpp",
"torch/csrc/jit/serialization/unpickler.cpp",
],
header_namespace = "",
exported_headers = [
"torch/csrc/jit/serialization/import_read.h",
"torch/csrc/jit/serialization/unpickler.h",
],
compiler_flags = get_pt_compiler_flags(),
link_whole = True,
linker_flags = [
"-Wl,--no-as-needed",
],
visibility = ["PUBLIC"],
exported_deps = [
":aten_cpu",
":caffe2_headers",
":caffe2_serialize",
":torch_common",
":torch_headers",
":torch_mobile_headers",
":torch_mobile_module",
":torch_mobile_observer",
"//third_party:glog",
"//c10:c10",
],
)
cxx_library(
name = "caffe2_serialize",
srcs = [
@ -270,88 +148,6 @@ cxx_library(
],
)
cxx_library(
name = "torch_mobile_deserialize",
srcs = [
"torch/csrc/jit/mobile/import.cpp",
],
header_namespace = "",
exported_headers = [
"torch/csrc/jit/mobile/import.h",
],
compiler_flags = get_pt_compiler_flags(),
link_whole = True,
linker_flags = [
"-Wl,--no-as-needed",
],
visibility = ["PUBLIC"],
exported_deps = [
":aten_cpu",
":caffe2_headers",
":caffe2_serialize",
":torch_common",
":torch_headers",
":torch_mobile_headers",
":torch_mobile_module",
":torch_mobile_observer",
"//third_party:glog",
"//c10:c10",
":torch_mobile_deserialize_common",
],
)
cxx_library(
name = "torch_mobile_module",
srcs = [
"torch/csrc/jit/mobile/function.cpp",
"torch/csrc/jit/mobile/interpreter.cpp",
"torch/csrc/jit/mobile/module.cpp",
],
header_namespace = "",
exported_headers = [],
compiler_flags = get_pt_compiler_flags(),
link_whole = True,
linker_flags = [
"-Wl,--no-as-needed",
],
visibility = ["PUBLIC"],
exported_deps = [
":aten_cpu",
":caffe2_headers",
":torch_common",
":torch_headers",
":torch_mobile_headers",
":torch_mobile_observer",
"//third_party:glog",
"//c10:c10",
],
)
cxx_library(
name = "torch_mobile_core",
srcs = [],
header_namespace = "",
exported_headers = [],
compiler_flags = get_pt_compiler_flags(),
exported_preprocessor_flags = get_pt_preprocessor_flags(),
link_whole = True,
linker_flags = [
"-Wl,--no-as-needed",
# "-ldl",
],
visibility = ["PUBLIC"],
deps = [
":generated-autograd-headers",
":torch_mobile_observer",
":torch_mobile_headers",
],
exported_deps = [
":aten_cpu",
":torch_common",
":torch_mobile_deserialize",
],
)
pt_operator_library(
name = "torch_mobile_ops_full_dev",
check_decl = False,
@ -371,62 +167,6 @@ cxx_library(
),
)
python_library(
name = "gen_oplist_lib",
srcs = subdir_glob([
("tools/code_analyzer", "gen_oplist.py"),
("tools/code_analyzer", "gen_op_registration_allowlist.py"),
]),
base_module = "",
deps = [
"//third_party:pyyaml",
"//tools/lite_interpreter:gen_selected_mobile_ops_header",
"//torchgen:torchgen",
],
)
python_binary(
name = "gen_oplist",
main_module = "gen_oplist",
visibility = ["PUBLIC"],
deps = [
":gen_oplist_lib",
],
)
python_library(
name = "gen_operators_yaml_lib",
srcs = subdir_glob([
("tools/code_analyzer", "gen_operators_yaml.py"),
("tools/code_analyzer", "gen_op_registration_allowlist.py"),
]),
base_module = "",
deps = [
"//third_party:pyyaml",
"//torchgen:torchgen",
],
)
python_binary(
name = "gen_aten_bin",
main_module = "torchgen.gen",
visibility = [
"PUBLIC",
],
deps = [
"//torchgen:torchgen",
],
)
python_binary(
name = "gen_operators_yaml",
main_module = "gen_operators_yaml",
visibility = ["PUBLIC"],
deps = [
":gen_operators_yaml_lib",
],
)
cxx_binary(
name = 'ptmobile_benchmark',
srcs = [

1034
buckbuild.bzl
View File

File diff suppressed because it is too large Load Diff

15
pt_defs.oss.bzl
View File

@ -1,4 +1,5 @@
load("@bazel_skylib//lib:paths.bzl", "paths")
load("//tools/build_defs:fb_xplat_cxx_library.bzl", "fb_xplat_cxx_library")
load("//tools/build_defs:fb_xplat_genrule.bzl", "fb_xplat_genrule")
load("//tools/build_defs:type_defs.bzl", "is_list", "is_string")
load(":build_variables.bzl", "aten_native_source_list")
@ -220,6 +221,20 @@ TEMPLATE_SOURCE_LIST = [
"torch/csrc/jit/runtime/register_special_ops.cpp",
] + aten_native_source_list
# the basic xplat library for OSS build
def pt_xplat_cxx_library(extra_flags = {}, **kwargs):
fb_xplat_cxx_library(
compiler_flags = get_pt_compiler_flags() + extra_flags.get("compiler_flags", []),
exported_preprocessor_flags = get_pt_preprocessor_flags() + extra_flags.get("exported_preprocessor_flags", []),
**kwargs
)
def get_aten_default_args():
return dict(
compiler_flags = get_aten_compiler_flags(),
exported_preprocessor_flags = get_aten_preprocessor_flags(),
)
# For selective build, we can lump the CPU and CPU kernel sources altogether
# because there is only ever one vectorization variant that is compiled
def aten_ufunc_generated_all_cpu_sources(gencode_pattern = "{}"):

46
tools/build_defs/buck_helpers.bzl
View File

@ -25,49 +25,3 @@ def filter_attributes(kwgs):
if key.startswith(invalid_prefix):
kwgs.pop(key)
return kwgs
# maps known fbsource deps to OSS deps
DEPS_MAP = {
"//third-party/FP16:FP16": "//third_party:FP16",
"//third-party/FXdiv:FXdiv": "//third_party:FXdiv",
"//third-party/XNNPACK:XNNPACK": "//third_party:XNNPACK",
"//third-party/clog:clog": "//third_party:clog",
"//third-party/cpuinfo:cpuinfo": "//third_party:cpuinfo",
"//third-party/fmt:fmt": "//third_party:fmt",
"//third-party/glog:glog": "//third_party:glog",
"//third-party/psimd:psimd": "//third_party:psimd",
"//third-party/pthreadpool:pthreadpool": "//third_party:pthreadpool",
"//third-party/pthreadpool:pthreadpool_header": "//third_party:pthreadpool_header",
"//third-party/ruy:ruy_xplat_lib": "//third_party:ruy_lib",
}
# map fbsource deps to OSS deps
def to_oss_deps(deps = []):
new_deps = []
for dep in deps:
new_deps += map_deps(dep)
return new_deps
def map_deps(dep):
# remove @fbsource prefix
if dep.startswith("@fbsource"):
dep = dep[len("@fbsource"):]
# ignore all fbsource linker_lib targets
if dep.startswith("//xplat/third-party/linker_lib"):
return []
# map targets in caffe2 root folder. Just use relative path
if dep.startswith("//xplat/caffe2:"):
return [dep[len("//xplat/caffe2"):]]
# map targets in caffe2 subfolders
if dep.startswith("//xplat/caffe2/"):
return ["//" + dep[len("//xplat/caffe2/"):]]
# map other known targets
if dep in DEPS_MAP:
return DEPS_MAP[dep]
# drop other unknown deps
return []

3
tools/build_defs/fb_python_binary.bzl
View File

@ -1,7 +1,6 @@
# Only used for PyTorch open source BUCK build
# @lint-ignore-every BUCKRESTRICTEDSYNTAX
load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary")
# @lint-ignore-every FBCODEBZLADDLOADS
def fb_python_binary(**kwgs):
if read_config("pt", "is_oss", "0") == "0":

9
tools/build_defs/fb_python_library.bzl Normal file
View File

@ -0,0 +1,9 @@
# Only used for PyTorch open source BUCK build
# @lint-ignore-every BUCKRESTRICTEDSYNTAX
# @lint-ignore-every FBCODEBZLADDLOADS
def fb_python_library(**kwgs):
if read_config("pt", "is_oss", "0") == "0":
fail("This file is for open source pytorch build. Do not use it in fbsource!")
python_library(**kwgs)

9
tools/build_defs/fb_xplat_cxx_library.bzl
View File

@ -1,19 +1,18 @@
# Only used for PyTorch open source BUCK build
# @lint-ignore-every BUCKRESTRICTEDSYNTAX
load(":buck_helpers.bzl", "filter_attributes", "to_oss_deps")
load(
":buck_helpers.bzl",
"filter_attributes",
)
def fb_xplat_cxx_library(
name,
deps = [],
exported_deps = [],
**kwgs):
if read_config("pt", "is_oss", "0") == "0":
fail("This file is for open source pytorch build. Do not use it in fbsource!")
cxx_library(
name = name,
deps = to_oss_deps(deps),
exported_deps = to_oss_deps(exported_deps),
**filter_attributes(kwgs)
)

8
tools/build_defs/select.bzl Normal file
View File

@ -0,0 +1,8 @@
# Only used for PyTorch open source BUCK build
# @lint-ignore-every BUCKRESTRICTEDSYNTAX
def select(conditions):
if read_config("pt", "is_oss", "0") == "0":
fail("This file is for open source pytorch build. Do not use it in fbsource!")
return conditions["DEFAULT"]

9
tools/build_defs/windows/windows_flag_map.bzl Normal file
View File

@ -0,0 +1,9 @@
# Only used for PyTorch open source BUCK build
# @lint-ignore-every BUCKRESTRICTEDSYNTAX
def windows_convert_gcc_clang_flags(flags = []):
if read_config("pt", "is_oss", "0") == "0":
fail("This file is for open source pytorch build. Do not use it in fbsource!")
# not implemented
return []