frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Respect -q of setup.py (#14972)

Browse Source 
Summary:
1. Changes the prints along the 'rebuild' pathway to respect the '-q' flag of setup.py
A clean rebuild now only prints:

    [zdevito@devgpu172.prn2 /data/users/zdevito/pytorch] python setup.py -q rebuild develop
    [0/1] Install the project...
    -- Install configuration: "RelWithDebInfo"
    ninja: no work to do.
    ninja: no work to do.
    ninja: no work to do.
    ninja: no work to do.
    ninja: no work to do.
    ninja: no work to do.

2. Deletes apparently dead calls to `generate_code`. Now that CMake builds these files,
it appears that it is getting called twice and the second version is never used.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/14972

Reviewed By: soumith

Differential Revision: D13396330

Pulled By: zdevito

fbshipit-source-id: 83c45143bbc6a6d2c1cfee929291ec059f2b5dc3
This commit is contained in:
Zachary DeVito
2018-12-09 22:45:18 -08:00
committed by Facebook Github Bot
parent fab8085111
commit e747acbebb
2 changed files with 84 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
setup.py
View File

@ -124,6 +124,7 @@
# LD_LIBRARY_PATH # LD_LIBRARY_PATH
# we will search for libraries in these paths # we will search for libraries in these paths
from __future__ import print_function
from setuptools import setup, Extension, distutils, Command, find_packages from setuptools import setup, Extension, distutils, Command, find_packages
import setuptools.command.build_ext import setuptools.command.build_ext
import setuptools.command.install import setuptools.command.install
@ -194,6 +195,23 @@ from tools.setup_helpers.dist_check import USE_DISTRIBUTED, \
DEBUG = check_env_flag('DEBUG') DEBUG = check_env_flag('DEBUG')
REL_WITH_DEB_INFO = check_env_flag('REL_WITH_DEB_INFO') REL_WITH_DEB_INFO = check_env_flag('REL_WITH_DEB_INFO')
VERBOSE_SCRIPT = True
# see if the user passed a quiet flag to setup.py arguments and respect
# that in our parts of the build
for arg in sys.argv:
if arg == "--":
break
if arg == '-q' or arg == '--quiet':
VERBOSE_SCRIPT = False
if VERBOSE_SCRIPT:
def report(*args):
print(*args)
else:
def report(*args):
pass
IS_WINDOWS = (platform.system() == 'Windows') IS_WINDOWS = (platform.system() == 'Windows')
IS_DARWIN = (platform.system() == 'Darwin') IS_DARWIN = (platform.system() == 'Darwin')
IS_LINUX = (platform.system() == 'Linux') IS_LINUX = (platform.system() == 'Linux')
@ -279,13 +297,13 @@ else:
version += '+' + sha[:7] version += '+' + sha[:7]
except Exception: except Exception:
pass pass
print("Building wheel {}-{}".format(package_name, version)) report("Building wheel {}-{}".format(package_name, version))
class create_version_file(PytorchCommand): class create_version_file(PytorchCommand):
def run(self): def run(self):
global version, cwd global version, cwd
print('-- Building version ' + version) report('-- Building version ' + version)
version_path = os.path.join(cwd, 'torch', 'version.py') version_path = os.path.join(cwd, 'torch', 'version.py')
with open(version_path, 'w') as f: with open(version_path, 'w') as f:
f.write("__version__ = '{}'\n".format(version)) f.write("__version__ = '{}'\n".format(version))
@ -393,7 +411,8 @@ def build_libs(libs):
my_env["USE_FFMPEG"] = "ON" if USE_FFMPEG else "OFF" my_env["USE_FFMPEG"] = "ON" if USE_FFMPEG else "OFF"
my_env["USE_DISTRIBUTED"] = "ON" if USE_DISTRIBUTED else "OFF" my_env["USE_DISTRIBUTED"] = "ON" if USE_DISTRIBUTED else "OFF"
my_env["USE_SYSTEM_NCCL"] = "ON" if USE_SYSTEM_NCCL else "OFF" my_env["USE_SYSTEM_NCCL"] = "ON" if USE_SYSTEM_NCCL else "OFF"
if VERBOSE_SCRIPT:
my_env['VERBOSE_SCRIPT'] = '1'
try: try:
os.mkdir('build') os.mkdir('build')
except OSError: except OSError:
@ -402,7 +421,7 @@ def build_libs(libs):
kwargs = {'cwd': 'build'} if not IS_WINDOWS else {} kwargs = {'cwd': 'build'} if not IS_WINDOWS else {}
if subprocess.call(build_libs_cmd + libs, env=my_env, **kwargs) != 0: if subprocess.call(build_libs_cmd + libs, env=my_env, **kwargs) != 0:
print("Failed to run '{}'".format(' '.join(build_libs_cmd + libs))) report("Failed to run '{}'".format(' '.join(build_libs_cmd + libs)))
sys.exit(1) sys.exit(1)
@ -410,7 +429,7 @@ def build_libs(libs):
# protobuf python compiler) from the build folder to the root folder # protobuf python compiler) from the build folder to the root folder
# cp root/build/caffe2/proto/proto.py root/caffe2/proto/proto.py # cp root/build/caffe2/proto/proto.py root/caffe2/proto/proto.py
def copy_protos(): def copy_protos():
print('setup.py::copy_protos()') report('setup.py::copy_protos()')
for src in glob.glob( for src in glob.glob(
os.path.join(caffe2_build_dir, 'caffe2', 'proto', '*.py')): os.path.join(caffe2_build_dir, 'caffe2', 'proto', '*.py')):
dst = os.path.join( dst = os.path.join(
@ -421,13 +440,13 @@ def copy_protos():
# Build all dependent libraries # Build all dependent libraries
class build_deps(PytorchCommand): class build_deps(PytorchCommand):
def run(self): def run(self):
print('setup.py::build_deps::run()') report('setup.py::build_deps::run()')
# Check if you remembered to check out submodules # Check if you remembered to check out submodules
def check_file(f): def check_file(f):
if not os.path.exists(f): if not os.path.exists(f):
print("Could not find {}".format(f)) report("Could not find {}".format(f))
print("Did you run 'git submodule update --init --recursive'?") report("Did you run 'git submodule update --init --recursive'?")
sys.exit(1) sys.exit(1)
check_file(os.path.join(third_party_path, "gloo", "CMakeLists.txt")) check_file(os.path.join(third_party_path, "gloo", "CMakeLists.txt"))
@ -489,7 +508,7 @@ for lib in dep_libs:
class build_module(PytorchCommand): class build_module(PytorchCommand):
def run(self): def run(self):
print('setup.py::build_module::run()') report('setup.py::build_module::run()')
self.run_command('build_py') self.run_command('build_py')
self.run_command('build_ext') self.run_command('build_ext')
@ -497,7 +516,7 @@ class build_module(PytorchCommand):
class build_py(setuptools.command.build_py.build_py): class build_py(setuptools.command.build_py.build_py):
def run(self): def run(self):
print('setup.py::build_py::run()') report('setup.py::build_py::run()')
self.run_command('create_version_file') self.run_command('create_version_file')
setuptools.command.build_py.build_py.run(self) setuptools.command.build_py.build_py.run(self)
@ -505,7 +524,7 @@ class build_py(setuptools.command.build_py.build_py):
class develop(setuptools.command.develop.develop): class develop(setuptools.command.develop.develop):
def run(self): def run(self):
print('setup.py::develop::run()') report('setup.py::develop::run()')
self.run_command('create_version_file') self.run_command('create_version_file')
setuptools.command.develop.develop.run(self) setuptools.command.develop.develop.run(self)
self.create_compile_commands() self.create_compile_commands()
@ -539,9 +558,9 @@ class develop(setuptools.command.develop.develop):
f.write(new_contents) f.write(new_contents)
if not USE_NINJA: if not USE_NINJA:
print("WARNING: 'develop' is not building C++ code incrementally") report("WARNING: 'develop' is not building C++ code incrementally")
print("because ninja is not installed. Run this to enable it:") report("because ninja is not installed. Run this to enable it:")
print(" > pip install ninja") report(" > pip install ninja")
build_ext_parent = ninja_build_ext if USE_NINJA \ build_ext_parent = ninja_build_ext if USE_NINJA \
@ -551,52 +570,41 @@ build_ext_parent = ninja_build_ext if USE_NINJA \
class build_ext(build_ext_parent): class build_ext(build_ext_parent):
def run(self): def run(self):
# Print build options # report build options
if USE_NUMPY: if USE_NUMPY:
print('-- Building with NumPy bindings') report('-- Building with NumPy bindings')
else: else:
print('-- NumPy not found') report('-- NumPy not found')
if USE_CUDNN: if USE_CUDNN:
print('-- Detected cuDNN at ' + CUDNN_LIBRARY + ', ' + CUDNN_INCLUDE_DIR) report('-- Detected cuDNN at ' + CUDNN_LIBRARY + ', ' + CUDNN_INCLUDE_DIR)
else: else:
print('-- Not using cuDNN') report('-- Not using cuDNN')
if USE_MIOPEN: if USE_MIOPEN:
print('-- Detected MIOpen at ' + MIOPEN_LIBRARY + ', ' + MIOPEN_INCLUDE_DIR) report('-- Detected MIOpen at ' + MIOPEN_LIBRARY + ', ' + MIOPEN_INCLUDE_DIR)
else: else:
print('-- Not using MIOpen') report('-- Not using MIOpen')
if USE_CUDA: if USE_CUDA:
print('-- Detected CUDA at ' + CUDA_HOME) report('-- Detected CUDA at ' + CUDA_HOME)
else: else:
print('-- Not using CUDA') report('-- Not using CUDA')
if USE_MKLDNN: if USE_MKLDNN:
print('-- Using MKLDNN') report('-- Using MKLDNN')
else: else:
print('-- Not using MKLDNN') report('-- Not using MKLDNN')
if USE_NCCL and USE_SYSTEM_NCCL: if USE_NCCL and USE_SYSTEM_NCCL:
print('-- Using system provided NCCL library at ' + NCCL_SYSTEM_LIB + ', ' + NCCL_INCLUDE_DIR) report('-- Using system provided NCCL library at ' + NCCL_SYSTEM_LIB + ', ' + NCCL_INCLUDE_DIR)
elif USE_NCCL: elif USE_NCCL:
print('-- Building NCCL library') report('-- Building NCCL library')
else: else:
print('-- Not using NCCL') report('-- Not using NCCL')
if USE_DISTRIBUTED: if USE_DISTRIBUTED:
print('-- Building with THD distributed package ') report('-- Building with THD distributed package ')
if IS_LINUX: if IS_LINUX:
print('-- Building with c10d distributed package ') report('-- Building with c10d distributed package ')
else: else:
print('-- Building without c10d distributed package') report('-- Building without c10d distributed package')
else: else:
print('-- Building without distributed package') report('-- Building without distributed package')
if USE_NINJA:
ninja_builder = NinjaBuilder('global')
generate_code(ninja_builder)
# before we start the normal build make sure all generated code
# gets built
ninja_builder.run()
else:
generate_code(None)
# It's an old-style class in Python 2.7... # It's an old-style class in Python 2.7...
setuptools.command.build_ext.build_ext.run(self) setuptools.command.build_ext.build_ext.run(self)
@ -639,15 +647,15 @@ class build_ext(build_ext_parent):
continue continue
fullname = self.get_ext_fullname(ext.name) fullname = self.get_ext_fullname(ext.name)
filename = self.get_ext_filename(fullname) filename = self.get_ext_filename(fullname)
print("\nCopying extension {}".format(ext.name)) report("\nCopying extension {}".format(ext.name))
src = os.path.join(tmp_install_path, rel_site_packages, filename) src = os.path.join(tmp_install_path, rel_site_packages, filename)
if not os.path.exists(src): if not os.path.exists(src):
print("{} does not exist".format(src)) report("{} does not exist".format(src))
del self.extensions[i] del self.extensions[i]
else: else:
dst = os.path.join(os.path.realpath(self.build_lib), filename) dst = os.path.join(os.path.realpath(self.build_lib), filename)
print("Copying {} from {} to {}".format(ext.name, src, dst)) report("Copying {} from {} to {}".format(ext.name, src, dst))
dst_dir = os.path.dirname(dst) dst_dir = os.path.dirname(dst)
if not os.path.exists(dst_dir): if not os.path.exists(dst_dir):
os.makedirs(dst_dir) os.makedirs(dst_dir)
@ -658,7 +666,7 @@ class build_ext(build_ext_parent):
def get_outputs(self): def get_outputs(self):
outputs = distutils.command.build_ext.build_ext.get_outputs(self) outputs = distutils.command.build_ext.build_ext.get_outputs(self)
outputs.append(os.path.join(self.build_lib, "caffe2")) outputs.append(os.path.join(self.build_lib, "caffe2"))
print("setup.py::get_outputs returning {}".format(outputs)) report("setup.py::get_outputs returning {}".format(outputs))
return outputs return outputs
@ -682,7 +690,7 @@ class rebuild(distutils.command.build.build):
class install(setuptools.command.install.install): class install(setuptools.command.install.install):
def run(self): def run(self):
print('setup.py::run()') report('setup.py::run()')
if not self.skip_build: if not self.skip_build:
self.run_command('build_deps') self.run_command('build_deps')
@ -738,8 +746,8 @@ if IS_WINDOWS:
'/wd4275'] '/wd4275']
if sys.version_info[0] == 2: if sys.version_info[0] == 2:
if not check_env_flag('FORCE_PY27_BUILD'): if not check_env_flag('FORCE_PY27_BUILD'):
print('The support for PyTorch with Python 2.7 on Windows is very experimental.') report('The support for PyTorch with Python 2.7 on Windows is very experimental.')
print('Please set the flag `FORCE_PY27_BUILD` to 1 to continue build.') report('Please set the flag `FORCE_PY27_BUILD` to 1 to continue build.')
sys.exit(1) sys.exit(1)
# /bigobj increases number of sections in .obj file, which is needed to link # /bigobj increases number of sections in .obj file, which is needed to link
# against libaries in Python 2.7 under Windows # against libaries in Python 2.7 under Windows

40
tools/build_pytorch_libs.sh
View File

@ -8,7 +8,16 @@
# #
# TODO: Replace this with the root-level CMakeLists.txt # TODO: Replace this with the root-level CMakeLists.txt
set -ex if [[ $VERBOSE_SCRIPT == '1' ]]; then
set -ex
report() {
echo "$@"
}
else
report() {
:
}
fi
SYNC_COMMAND="cp" SYNC_COMMAND="cp"
if [ -x "$(command -v rsync)" ]; then if [ -x "$(command -v rsync)" ]; then
@ -163,7 +172,7 @@ elif [[ -n "$REL_WITH_DEB_INFO" && $REL_WITH_DEB_INFO -ne 0 ]]; then
BUILD_TYPE="RelWithDebInfo" BUILD_TYPE="RelWithDebInfo"
fi fi
echo "Building in $BUILD_TYPE mode" report "Building in $BUILD_TYPE mode"
function path_remove { function path_remove {
# Delete path by parts so we can never accidentally remove sub paths # Delete path by parts so we can never accidentally remove sub paths
@ -271,8 +280,12 @@ function build_caffe2() {
# Install Python proto files # Install Python proto files
if [[ "$BUILD_PYTHON" == 'ON' ]]; then if [[ "$BUILD_PYTHON" == 'ON' ]]; then
echo "Copying Caffe2 proto files from $(pwd)/caffe2/proto to $(cd .. && pwd)/caffe2/proto"
echo "All the files in caffe2/proto are $(find caffe2/proto)" if [[ $VERBOSE_SCRIPT == '1' ]]; then
report "Copying Caffe2 proto files from $(pwd)/caffe2/proto to $(cd .. && pwd)/caffe2/proto"
report "All the files in caffe2/proto are $(find caffe2/proto)"
fi
for proto_file in $(pwd)/caffe2/proto/*.py; do for proto_file in $(pwd)/caffe2/proto/*.py; do
cp $proto_file "$(pwd)/../caffe2/proto/" cp $proto_file "$(pwd)/../caffe2/proto/"
done done
@ -282,10 +295,10 @@ function build_caffe2() {
# Fix rpaths of shared libraries # Fix rpaths of shared libraries
if [[ $(uname) == 'Darwin' ]]; then if [[ $(uname) == 'Darwin' ]]; then
# root/torch/lib/tmp_install/lib # root/torch/lib/tmp_install/lib
echo "Updating all install_names in $INSTALL_DIR/lib" report "Updating all install_names in $INSTALL_DIR/lib"
pushd "$INSTALL_DIR/lib" pushd "$INSTALL_DIR/lib"
for lib in *.dylib; do for lib in *.dylib; do
echo "Updating install_name for $(pwd)/$lib" report "Updating install_name for $(pwd)/$lib"
install_name_tool -id @rpath/$lib $lib install_name_tool -id @rpath/$lib $lib
done done
popd popd
@ -306,28 +319,27 @@ for arg in "$@"; do
fi fi
done done
pushd $TORCH_LIB_DIR pushd $TORCH_LIB_DIR > /dev/null
# If all the builds succeed we copy the libraries, headers, # If all the builds succeed we copy the libraries, headers,
# binaries to torch/lib # binaries to torch/lib
echo "tools/build_pytorch_libs.sh succeeded at $(date)" report "tools/build_pytorch_libs.sh succeeded at $(date)"
echo "removing $INSTALL_DIR/lib/cmake and $INSTALL_DIR/lib/python" report "removing $INSTALL_DIR/lib/cmake and $INSTALL_DIR/lib/python"
rm -rf "$INSTALL_DIR/lib/cmake" rm -rf "$INSTALL_DIR/lib/cmake"
rm -rf "$INSTALL_DIR/lib/python" rm -rf "$INSTALL_DIR/lib/python"
echo "Copying $INSTALL_DIR/lib to $(pwd)" report "Copying $INSTALL_DIR/lib to $(pwd)"
$SYNC_COMMAND -r "$INSTALL_DIR/lib"/* . $SYNC_COMMAND -r "$INSTALL_DIR/lib"/* .
if [ -d "$INSTALL_DIR/lib64/" ]; then if [ -d "$INSTALL_DIR/lib64/" ]; then
$SYNC_COMMAND -r "$INSTALL_DIR/lib64"/* . $SYNC_COMMAND -r "$INSTALL_DIR/lib64"/* .
fi fi
echo "Copying $(cd ../.. && pwd)/aten/src/generic/THNN.h to $(pwd)" report "Copying $(cd ../.. && pwd)/aten/src/generic/THNN.h to $(pwd)"
$SYNC_COMMAND ../../aten/src/THNN/generic/THNN.h . $SYNC_COMMAND ../../aten/src/THNN/generic/THNN.h .
$SYNC_COMMAND ../../aten/src/THCUNN/generic/THCUNN.h . $SYNC_COMMAND ../../aten/src/THCUNN/generic/THCUNN.h .
echo "Copying $INSTALL_DIR/include to $(pwd)" report "Copying $INSTALL_DIR/include to $(pwd)"
$SYNC_COMMAND -r "$INSTALL_DIR/include" . $SYNC_COMMAND -r "$INSTALL_DIR/include" .
if [ -d "$INSTALL_DIR/bin/" ]; then if [ -d "$INSTALL_DIR/bin/" ]; then
$SYNC_COMMAND -r "$INSTALL_DIR/bin/"/* . $SYNC_COMMAND -r "$INSTALL_DIR/bin/"/* .
fi fi
popd > /dev/null
popd