Improve MANIFEST.in for source distribution (#157814)

Pull Request resolved: https://github.com/pytorch/pytorch/pull/157814
Approved by: https://github.com/XuehaiPan, https://github.com/atalman

MANIFEST.in

@@ -1,20 +1,61 @@
 # Reference: https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html
 
-# Include source files in SDist
-include CMakeLists.txt
-include *.bzl *.bazel .bazel* BUILD *.BUILD BUILD.* WORKSPACE
-include BUCK BUCK.*
-include requirements*.txt
-include version.txt
-include [Mm]akefile *.[Mm]akefile [Mm]akefile.*
-include [Dd]ockerfile *.[Dd]ockerfile [Dd]ockerfile.* .dockerignore
+# Include individual top-level files
+include CITATION.cff
+include CODEOWNERS
+include Dockerfile
+include LICENSE
+include MANIFEST.in
+include Makefile
+include NOTICE
+include .bc-linter.yml
+include .clang-format .clang-tidy
+include .cmakelintrc
+include .coveragerc
+include .dockerignore
+include .editorconfig
+include .flake8
+include .gdbinit
+include .lintrunner.toml
+include .lldbinit
+include codex_setup.sh
+include docker.Makefile
+include pyrefly.toml
+include ubsan.supp
+
+# Include bazel and BUCK related files
+include BUILD.bazel BUCK.oss
+include WORKSPACE
+include *.bzl
+include .bazelignore .bazelrc .bazelversion
+
+# Include general configuration files
+include *.ini
+# Include important top-level information
+include *.md
+# Include technical text files at the moment, comprises
+# version.txt, CMakeLists.txt, requirements.txt
+include *.txt
+
+# Include ctags configuration
+include .ctags.d/*.ctags
+
+# Include subfolders completely
+graft .devcontainer
+graft .vscode
 graft android
 graft aten
+graft benchmarks
 graft binaries
 graft c10
 graft caffe2
 graft cmake
+graft docs
 graft functorch
+graft ios
+graft mypy_plugins
+graft scripts
+graft test
 graft third_party
 graft tools
 graft torch
@@ -22,29 +63,37 @@ graft torchgen
 # FIXME: torch-xla build during codegen will fail if include this file in wheel
 exclude torchgen/BUILD.bazel
 
-# Misc files and directories in SDist
-include *.md
-include CITATION.cff
-include LICENSE NOTICE
-include mypy*.ini
-graft benchmarks
-graft docs
-graft mypy_plugins
-graft scripts
+# The following exclusions omit parts from third-party dependencies that
+# contain invalid symlinks[1] and that are not needed for pytorch, such as
+# bindings for unused languages
+prune third_party/flatbuffers/java
+prune third_party/flatbuffers/kotlin
+prune third_party/ittapi/rust
+prune third_party/nccl/pkg/debian
+prune third_party/opentelemetry-cpp/third_party/prometheus-cpp/cmake/project-import-*
+
+# The following document is also an invalid symlink[1] and superfluous
+exclude third_party/flatbuffers/docs/source/CONTRIBUTING.md
+
+# Omit autogenerated code
+prune torchgen/packaged
+
+# Omit caches, compiled, and scm related content
+prune */__pycache__
+prune **/.github
+prune **/.gitlab
+global-exclude *.o *.obj *.so *.dylib *.a *.pxd *.dll *.lib
+global-exclude *.py[cod] *.swp *~
+global-exclude .git .git-blame-ignore-revs .gitattributes .gitignore .gitmodules
+global-exclude .gitlab-ci.yml
 
 # Misc files needed for custom setuptools command
 include .gitignore
 include .gitmodules
 
-# Include test suites in SDist
-graft test
-include pytest.ini
-include .coveragerc
+# [1] Invalid symlinks for the purposes of Python source distributions are,
+# according to the source distribution format[2] links pointing outside the
+# destination directory or links with a `..` component, which is those of
+# concern here.
 
-# Prune generated/compiled files
-prune torchgen/packaged
-prune */__pycache__
-global-exclude *.o *.obj *.so *.a *.dylib *.pxd *.dll *.lib *.py[cod]
-
-prune */.git
-global-exclude .git *~ *.swp
+# [2] https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-archive-features