mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
Revert "[AOTI][refactor] Consolidate CppBuilder.build and CppBuilder.build_fbcode_cpu_re (#147803)"
This reverts commit 0b9da1ae0ad30ef228f132354b875bcaec214ace. Reverted https://github.com/pytorch/pytorch/pull/147803 on behalf of https://github.com/wdvr due to breaking internal tests, discussed with author ([comment](https://github.com/pytorch/pytorch/pull/147803#issuecomment-2683938121))
This commit is contained in:
PyTorch MergeBot
@ -1521,7 +1521,13 @@ class AotCodeCompiler:
|
||||
BuildOption=object_build_options,
|
||||
)
|
||||
consts_o = object_builder.get_target_file_path()
|
||||
object_builder.build()
|
||||
if fbcode_aot_cpu_re:
|
||||
# TODO: refactor fbcode_aot_cpu_re logic into CppBuilder
|
||||
consts_o = str(consts_s.with_suffix(".o"))
|
||||
object_builder.build_fbcode(str(consts_s), consts_o)
|
||||
os.chmod(consts_o, 0o644)
|
||||
else:
|
||||
object_builder.build()
|
||||
|
||||
if is_large_consts:
|
||||
with open(consts_o, "r+b") as f:
|
||||
@ -1660,7 +1666,12 @@ class AotCodeCompiler:
|
||||
object_builder.save_src_to_cmake(cmake_path, cpp_path)
|
||||
generated_files.append(cmake_path)
|
||||
else:
|
||||
object_builder.build()
|
||||
if fbcode_aot_cpu_re:
|
||||
output_o = str(cpp_path_operator.with_suffix(".o"))
|
||||
object_builder.build_fbcode(cpp_path, output_o)
|
||||
os.chmod(output_o, 0o644)
|
||||
else:
|
||||
object_builder.build()
|
||||
|
||||
if not use_mmap_weights:
|
||||
aot_constants = serialized_weights
|
||||
@ -1692,9 +1703,7 @@ class AotCodeCompiler:
|
||||
|
||||
so_builder = CppBuilder(
|
||||
name=output_name,
|
||||
sources=[output_o, consts_o, kernels_o]
|
||||
if kernels_o
|
||||
else [output_o, consts_o],
|
||||
sources=[output_o, consts_o, kernels_o],
|
||||
output_dir=output_dir,
|
||||
BuildOption=so_build_options,
|
||||
)
|
||||
@ -1741,7 +1750,16 @@ class AotCodeCompiler:
|
||||
so_builder.save_src_to_cmake(cmake_path, kernel_o)
|
||||
so_builder.save_link_cmd_to_cmake(cmake_path)
|
||||
else:
|
||||
so_builder.build()
|
||||
if fbcode_aot_cpu_re:
|
||||
output_so = (
|
||||
config.aot_inductor.output_path
|
||||
if specified_artifact_name
|
||||
else str(cpp_path_operator.with_suffix(".so"))
|
||||
)
|
||||
so_builder.build_fbcode([output_o, consts_o], output_so)
|
||||
os.chmod(output_so, 0o755)
|
||||
else:
|
||||
so_builder.build()
|
||||
|
||||
for o_file in [
|
||||
output_o,
|
||||
@ -1988,7 +2006,13 @@ def _worker_compile_cpp(
|
||||
fb_output_path if config.is_fbcode() else cpp_builder.get_target_file_path()
|
||||
)
|
||||
if not os.path.exists(binary_path):
|
||||
cpp_builder.build()
|
||||
if config.is_fbcode():
|
||||
cpp_builder.build_fbcode(
|
||||
fb_input_path,
|
||||
fb_output_path,
|
||||
)
|
||||
else:
|
||||
cpp_builder.build()
|
||||
|
||||
|
||||
# Customized Python binding for cpp kernels
|
||||
|
||||
@ -1394,8 +1394,6 @@ class CppBuilder:
|
||||
self._libraries_args = ""
|
||||
self._passthrough_parameters_args = ""
|
||||
|
||||
# When relative path is used, we need to maintain the source dir list.
|
||||
self._orig_source_paths = []
|
||||
self._output_dir = ""
|
||||
self._target_file = ""
|
||||
|
||||
@ -1430,7 +1428,8 @@ class CppBuilder:
|
||||
# Will create another temp director for building, so do NOT use
|
||||
# use the absolute path.
|
||||
inp_name = [os.path.basename(i) for i in sources]
|
||||
self._orig_source_paths = sources
|
||||
self._target_file = os.path.basename(self._target_file)
|
||||
|
||||
self._sources_args = " ".join(inp_name)
|
||||
else:
|
||||
self._sources_args = " ".join(sources)
|
||||
@ -1517,9 +1516,7 @@ class CppBuilder:
|
||||
libraries_args=self._libraries_args,
|
||||
libraries_dirs_args=self._libraries_dirs_args,
|
||||
passthrough_args=self._passthrough_parameters_args,
|
||||
target_file=os.path.basename(self._target_file)
|
||||
if self._use_relative_path
|
||||
else self._target_file,
|
||||
target_file=self._target_file,
|
||||
)
|
||||
return command_line
|
||||
|
||||
@ -1528,19 +1525,25 @@ class CppBuilder:
|
||||
|
||||
# Given a path to an input cpp file and an output path,
|
||||
# Attempts to compile the file, storing the output in "output_path"
|
||||
def build_fbcode_re(
|
||||
def build_fbcode(
|
||||
self,
|
||||
input_path: Union[str, list[str]],
|
||||
output_path: str,
|
||||
) -> None:
|
||||
from torch._inductor.codecache import cpp_prefix_path
|
||||
|
||||
with dynamo_timed("compile_file"):
|
||||
input_paths = [input_path] if isinstance(input_path, str) else input_path
|
||||
input_files = [
|
||||
os.path.basename(ip) if config.is_fbcode() else ip for ip in input_paths
|
||||
]
|
||||
command = self.get_command_line().split()
|
||||
try:
|
||||
assert config.is_fbcode(), "compile_file() is only used in fbcode"
|
||||
# Need to copy our header into the same folder as the sourcecode.
|
||||
header_path = cpp_prefix_path()
|
||||
header_name = os.path.basename(header_path)
|
||||
output_path = self._target_file
|
||||
output_name = os.path.basename(output_path)
|
||||
# When we build remotely, we need to make sure to carefully copy any files
|
||||
# that are required during the compilation process into our build directly.
|
||||
# This is where all of the ATen/c10/Torch includes come from.
|
||||
@ -1549,22 +1552,16 @@ class CppBuilder:
|
||||
# Copy everything to tmp compilation folder
|
||||
shutil.copy(header_path, os.path.join(tmp_dir, header_name))
|
||||
shutil.copy(_LINKER_SCRIPT, os.path.join(tmp_dir, "script.ld"))
|
||||
for src in self._orig_source_paths:
|
||||
shutil.copy(src, os.path.join(tmp_dir, os.path.basename(src)))
|
||||
for p, f in zip(input_paths, input_files):
|
||||
shutil.copy(p, os.path.join(tmp_dir, f))
|
||||
dest_include_path = os.path.join(tmp_dir, "include")
|
||||
shutil.copytree(torch_includes_path, dest_include_path)
|
||||
# Run the build
|
||||
tmp_output_path = _run_build_command(
|
||||
command, tmp_dir, os.path.basename(output_path)
|
||||
)
|
||||
output_file_path = _run_build_command(command, tmp_dir, output_name)
|
||||
# Copy output from the build
|
||||
if os.path.exists(output_path):
|
||||
os.remove(output_path)
|
||||
shutil.copy(tmp_output_path, output_path)
|
||||
if output_path.endswith(".o"):
|
||||
os.chmod(output_path, 0o644)
|
||||
elif output_path.endswith(".so"):
|
||||
os.chmod(output_path, 0o755)
|
||||
shutil.copy(output_file_path, output_path)
|
||||
except subprocess.CalledProcessError as e:
|
||||
output = e.output.decode("utf-8")
|
||||
raise exc.CppCompileError(command, output) from e
|
||||
@ -1574,9 +1571,6 @@ class CppBuilder:
|
||||
It is must need a temperary directory to store object files in Windows.
|
||||
After build completed, delete the temperary directory to save disk space.
|
||||
"""
|
||||
if self._use_relative_path:
|
||||
# _use_relative_path indicates this is a remote build
|
||||
return self.build_fbcode_re()
|
||||
_create_if_dir_not_exist(self._output_dir)
|
||||
_build_tmp_dir = os.path.join(
|
||||
self._output_dir, f"{self._name}_{_BUILD_TEMP_DIR}"
|
||||
|
||||
Reference in New Issue
Block a user