mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
[Split Build] Add option to create libtorch wheel and use it to build pytorch as a separate wheel (#126328)
Creates an option to just build the libtorch portion of pytorch such that we have the necessary .so files. Then it builds a torch package using the libtorch wheel. These options are enabled using ` BUILD_LIBTORCH_WHL` and `BUILD_PYTHON_ONLY`.
We run
```
BUILD_LIBTORCH_WHL=1 python setup.py install
python setup.py clean
BUILD_PYTHON_ONLY=1 python setup.py install
```
to produce
```
sahanp@devgpu086 ~/pytorch (detached HEAD|REBASE-i 3/5)> ls /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/torch/lib/ (pytorch-3.10)
libshm.so* libtorch_global_deps.so* libtorch_python.so*
sahanp@devgpu086 ~/pytorch (detached HEAD|REBASE-i 3/5)> ldd build/lib/libtorch_python.so (pytorch-3.10)
linux-vdso.so.1 (0x00007ffdc2d37000)
libtorch.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libtorch.so (0x00007f539fe99000)
libshm.so => /home/sahanp/pytorch/build/lib/libshm.so (0x00007f539fe90000)
libcudnn.so.8 => /usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudnn.so.8 (0x00007f539e800000)
libnvToolsExt.so.1 => /usr/local/cuda/lib64/libnvToolsExt.so.1 (0x00007f539e400000)
libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007f539e000000)
libm.so.6 => /lib64/libm.so.6 (0x00007f539fda5000)
libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007f539ebe5000)
libc.so.6 => /lib64/libc.so.6 (0x00007f539dc00000)
/lib64/ld-linux-x86-64.so.2 (0x00007f539fea0000)
libtorch_cpu.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libtorch_cpu.so (0x00007f5392400000)
libtorch_cuda.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libtorch_cuda.so (0x00007f5380000000)
librt.so.1 => /lib64/librt.so.1 (0x00007f539fd9e000)
libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f539fd99000)
libdl.so.2 => /lib64/libdl.so.2 (0x00007f539fd94000)
libc10.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libc10.so (0x00007f539eb07000)
libmkl_intel_lp64.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_intel_lp64.so.2 (0x00007f537ec00000)
libmkl_gnu_thread.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_gnu_thread.so.2 (0x00007f537ce00000)
libmkl_core.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_core.so.2 (0x00007f5378800000)
libomp.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/libomp.so (0x00007f539e707000)
libcupti.so.12 => /usr/local/cuda/lib64/libcupti.so.12 (0x00007f5377e00000)
libcudart.so.12 => /usr/local/cuda/lib64/libcudart.so.12 (0x00007f5377a00000)
libc10_cuda.so => /home/sahanp/.conda/envs/pytorch-3.10/lib/python3.10/site-packages/libtorch/lib/libc10_cuda.so (0x00007f539ea6a000)
libcusparse.so.12 => /usr/local/cuda/lib64/libcusparse.so.12 (0x00007f5368400000)
libcufft.so.11 => /usr/local/cuda/lib64/libcufft.so.11 (0x00007f535ee00000)
libcusolver.so.11 => /usr/local/cuda/lib64/libcusolver.so.11 (0x00007f534c800000)
libcurand.so.10 => /usr/local/cuda/lib64/libcurand.so.10 (0x00007f5346200000)
libcublas.so.12 => /usr/local/cuda/lib64/libcublas.so.12 (0x00007f533f800000)
libcublasLt.so.12 => /usr/local/cuda/lib64/libcublasLt.so.12 (0x00007f531e800000)
libutil.so.1 => /lib64/libutil.so.1 (0x00007f539ea63000)
libnvJitLink.so.12 => /usr/local/cuda/lib64/libnvJitLink.so.12 (0x00007f531b800000)
sahanp@devgpu086 ~/pytorch (detached HEAD|REBASE-i 3/5)> ldd build/lib/libtorch_global_deps.so (pytorch-3.10)
linux-vdso.so.1 (0x00007ffc265df000)
libmkl_intel_lp64.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_intel_lp64.so.2 (0x00007fa93fc00000)
libmkl_gnu_thread.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_gnu_thread.so.2 (0x00007fa93de00000)
libmkl_core.so.2 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libmkl_core.so.2 (0x00007fa939800000)
libm.so.6 => /lib64/libm.so.6 (0x00007fa940f05000)
libcudart.so.12 => /usr/local/cuda/lib64/libcudart.so.12 (0x00007fa939400000)
libnvToolsExt.so.1 => /usr/local/cuda/lib64/libnvToolsExt.so.1 (0x00007fa939000000)
libgomp.so.1 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libgomp.so.1 (0x00007fa93fb07000)
libc.so.6 => /lib64/libc.so.6 (0x00007fa938c00000)
libdl.so.2 => /lib64/libdl.so.2 (0x00007fa940efe000)
libpthread.so.0 => /lib64/libpthread.so.0 (0x00007fa940ef9000)
/lib64/ld-linux-x86-64.so.2 (0x00007fa940ff5000)
librt.so.1 => /lib64/librt.so.1 (0x00007fa940ef2000)
libstdc++.so.6 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libstdc++.so.6 (0x00007fa93921d000)
libgcc_s.so.1 => /home/sahanp/.conda/envs/pytorch-3.10/lib/libgcc_s.so.1 (0x00007fa93faec000)
```
Pull Request resolved: https://github.com/pytorch/pytorch/pull/126328
Approved by: https://github.com/atalman
This commit is contained in:
PaliC
PyTorch MergeBot
@ -22,7 +22,7 @@ set_target_properties(shm PROPERTIES
|
||||
PREFIX "lib"
|
||||
IMPORT_PREFIX "lib"
|
||||
CXX_STANDARD 17)
|
||||
target_link_libraries(shm PRIVATE torch_cpu)
|
||||
target_link_libraries(shm PRIVATE ${TORCH_CPU_LIB})
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
include(CheckLibraryExists)
|
||||
@ -60,7 +60,12 @@ if(UNIX AND NOT APPLE)
|
||||
endif()
|
||||
|
||||
add_executable(torch_shm_manager manager.cpp)
|
||||
target_link_libraries(torch_shm_manager PRIVATE shm c10)
|
||||
if(BUILD_LIBTORCHLESS)
|
||||
target_link_libraries(torch_shm_manager PRIVATE shm ${C10_LIB})
|
||||
else()
|
||||
# we need to link directly to c10 here otherwise we miss symbols
|
||||
target_link_libraries(torch_shm_manager PRIVATE shm c10)
|
||||
endif()
|
||||
set_target_properties(torch_shm_manager PROPERTIES
|
||||
INSTALL_RPATH "${_rpath_portable_origin}/../lib")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user