Add MANIFEST.in (#52908)

Summary:
Do not build PyTorch if `setup.py` is called with  'sdist' option
Regenerate bundled license while sdist package is being built
Refactor `check_submodules` out of `build_deps` and check that submodules project are present during source package build stage.

Test that sdist package is configurable during `asan-build` step

Fixes https://github.com/pytorch/pytorch/issues/52843

Pull Request resolved: https://github.com/pytorch/pytorch/pull/52908

Reviewed By: walterddr

Differential Revision: D26685176

Pulled By: malfet

fbshipit-source-id: 972a40ae36e194c0b4e0fc31c5e1af1e7a815185

.gitignore

@@ -237,6 +237,7 @@ caffe2/version.py
 # setup.py intermediates
 .eggs
 caffe2.egg-info
+MANIFEST
 
 # Atom/Watchman required file
 .watchmanconfig

.jenkins/pytorch/build-asan.sh

@@ -36,4 +36,14 @@ CC="clang" CXX="clang++" LDSHARED="clang --shared" \
   USE_ASAN=1 USE_CUDA=0 USE_MKLDNN=0 \
   python setup.py install
 
+
+# Test building via the sdist source tarball
+python setup.py sdist
+mkdir -p /tmp/tmp
+pushd /tmp/tmp
+tar zxf "$(dirname "${BASH_SOURCE[0]}")/../../dist/"*.tar.gz
+cd torch-*
+python setup.py build --cmake-only
+popd
+
 assert_git_not_dirty

MANIFEST.in

@@ -0,0 +1,28 @@
+include MANIFEST.in
+include CMakeLists.txt
+include CITATION
+include LICENSE
+include NOTICE
+include .gitmodules
+include mypy.ini
+include requirements.txt
+include version.txt
+recursive-include android *.*
+recursive-include aten *.*
+recursive-include binaries *.*
+recursive-include c10 *.*
+recursive-include caffe2 *.*
+recursive-include cmake *.*
+recursive-include torch *.*
+recursive-include tools *.*
+recursive-include test *.*
+recursive-include docs *.*
+recursive-include ios *.*
+recursive-include third_party *
+recursive-include test *.*
+recursive-include benchmarks *.*
+recursive-include scripts *.*
+recursive-include mypy_plugins *.*
+recursive-include modules *.*
+prune */__pycache__
+global-exclude *.o *.so *.dylib *.a .git *.pyc *.swp

setup.py

@@ -194,6 +194,7 @@ from distutils.errors import DistutilsArgError
 import setuptools.command.build_ext
 import setuptools.command.install
 import distutils.command.clean
+import distutils.command.sdist
 import distutils.sysconfig
 import filecmp
 import shutil
@@ -238,7 +239,7 @@ for i, arg in enumerate(sys.argv):
         break
     if arg == '-q' or arg == '--quiet':
         VERBOSE_SCRIPT = False
-    if arg == 'clean' or arg == 'egg_info':
+    if arg in ['clean', 'egg_info', 'sdist']:
         RUN_BUILD_DEPS = False
     filtered_args.append(arg)
 sys.argv = filtered_args
@@ -286,10 +287,8 @@ report("Building wheel {}-{}".format(package_name, version))
 
 cmake = CMake()
 
-# all the work we need to do _before_ setup runs
-def build_deps():
-    report('-- Building version ' + version)
 
+def check_submodules():
     def check_file(f):
         if bool(os.getenv("USE_SYSTEM_LIBS", False)):
             return
@@ -310,6 +309,12 @@ def build_deps():
     check_file(os.path.join(third_party_path, 'onnx', 'third_party',
                             'benchmark', 'CMakeLists.txt'))
 
+
+# all the work we need to do _before_ setup runs
+def build_deps():
+    report('-- Building version ' + version)
+
+    check_submodules()
     check_pydep('yaml', 'pyyaml')
 
     build_caffe2(version=version,
@@ -599,7 +604,7 @@ else:
 
 class install(setuptools.command.install.install):
     def run(self):
-        setuptools.command.install.install.run(self)
+        super().run()
 
 
 class clean(distutils.command.clean.clean):
@@ -623,8 +628,14 @@ class clean(distutils.command.clean.clean):
                         except OSError:
                             shutil.rmtree(filename, ignore_errors=True)
 
-        # It's an old-style class in Python 2.7...
-        distutils.command.clean.clean.run(self)
+        super().run()
+
+
+class sdist(distutils.command.sdist.sdist):
+    def run(self):
+        with concat_license_files():
+            super().run()
+
 
 def configure_extension_build():
     r"""Configures extension build options according to system environment and user's choice.
@@ -762,10 +773,11 @@ def configure_extension_build():
         )
 
     cmdclass = {
+        'bdist_wheel': wheel_concatenate,
         'build_ext': build_ext,
         'clean': clean,
         'install': install,
-        'bdist_wheel': wheel_concatenate,
+        'sdist': sdist,
     }
 
     entry_points = {