mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
ddff4efa26
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/31162 This should help us resolve a multitude of weird segfaults and crashes when PyTorch is imported along with other packages. Those would often happen because libtorch symbols were exposed globally and could be used as a source of relocations in shared libraries loaded after libtorch. Fixes #3059. Some of the subtleties in preparing this patch: * Getting ASAN to play ball was a pain in the ass. The basic problem is that when we load with `RTLD_LOCAL`, we now may load a library multiple times into the address space; this happens when we have custom C++ extensions. Since the libraries are usually identical, this is usually benign, but it is technically undefined behavior and UBSAN hates it. I sprayed a few ways of getting things to "work" correctly: I preload libstdc++ (so that it is seen consistently over all library loads) and added turned off vptr checks entirely. Another possibility is we should have a mode where we use RTLD_GLOBAL to load _C, which would be acceptable in environments where you're sure C++ lines up correctly. There's a long comment in the test script going into more detail about this. * Making some of our shared library dependencies load with `RTLD_LOCAL` breaks them. OpenMPI and MKL don't work; they play linker shenanigans to look up their symbols which doesn't work when loaded locally, and if we load a library with `RLTD_LOCAL` we aren't able to subsequently see it with `ctypes`. To solve this problem, we employ a clever device invented by apaszke: we create a dummy library `torch_global_deps` with dependencies on all of the libraries which need to be loaded globally, and then load that with `RTLD_GLOBAL`. As long as none of these libraries have C++ symbols, we can avoid confusion about C++ standard library. Signed-off-by: Edward Z. Yang <ezyang@fb.com> Differential Revision: D19262579 Test Plan: Imported from OSS Pulled By: ezyang fbshipit-source-id: 06a48a5d2c9036aacd535f7e8a4de0e8fe1639f2
58 lines
1.6 KiB
Python
58 lines
1.6 KiB
Python
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
import os
|
|
import inspect
|
|
|
|
# this arbitrary-looking assortment of functionality is provided here
|
|
# to have a central place for overrideable behavior. The motivating
|
|
# use is the FB build environment, where this source file is replaced
|
|
# by an equivalent.
|
|
|
|
if os.path.basename(os.path.dirname(__file__)) == 'shared':
|
|
torch_parent = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
|
else:
|
|
torch_parent = os.path.dirname(os.path.dirname(__file__))
|
|
|
|
|
|
def get_file_path(*path_components):
|
|
return os.path.join(torch_parent, *path_components)
|
|
|
|
|
|
def get_file_path_2(*path_components):
|
|
return os.path.join(*path_components)
|
|
|
|
|
|
def get_writable_path(path):
|
|
return path
|
|
|
|
|
|
def prepare_multiprocessing_environment(path):
|
|
pass
|
|
|
|
|
|
def resolve_library_path(path):
|
|
return os.path.realpath(path)
|
|
|
|
|
|
def get_source_lines_and_file(obj):
|
|
"""
|
|
Wrapper around inspect.getsourcelines and inspect.getsourcefile.
|
|
|
|
Returns: (sourcelines, file_lino, filename)
|
|
"""
|
|
filename = None # in case getsourcefile throws
|
|
try:
|
|
filename = inspect.getsourcefile(obj)
|
|
sourcelines, file_lineno = inspect.getsourcelines(obj)
|
|
except OSError as e:
|
|
raise OSError((
|
|
"Can't get source for {}. TorchScript requires source access in order to carry out compilation. " +
|
|
"Make sure original .py files are available. Original error: {}").format(filename, e))
|
|
|
|
return sourcelines, file_lineno, filename
|
|
|
|
|
|
TEST_MASTER_ADDR = '127.0.0.1'
|
|
TEST_MASTER_PORT = 29500
|
|
USE_RTLD_GLOBAL_WITH_LIBTORCH = False
|