Revert "Switch to standard pep517 sdist generation (#152098)"

This reverts commit f16053f0c9a09fa337fbf85aaf64f88712b8dcdb.

Reverted https://github.com/pytorch/pytorch/pull/152098 on behalf of https://github.com/malfet due to IMO this PR needs to be split into few helper ones, with better test plan ([comment](https://github.com/pytorch/pytorch/pull/152098#issuecomment-3024223880))

.ci/docker/build.sh

@@ -383,7 +383,7 @@ if [[ -n "${CI:-}" ]]; then
 fi
 
 # Build image
-tar ch . | docker build \
+docker build \
        ${no_cache_flag} \
        ${progress_flag} \
        --build-arg "BUILD_ENVIRONMENT=${image}" \
@@ -422,7 +422,7 @@ tar ch . | docker build \
        -f $(dirname ${DOCKERFILE})/Dockerfile \
        -t "$tmp_tag" \
        "$@" \
-       -
+       .
 
 # NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
 # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could

.ci/docker/requirements-docs.txt

@@ -1 +0,0 @@
-../../docs/requirements.txt

.ci/docker/requirements-docs.txt

@@ -0,0 +1,61 @@
+sphinx==5.3.0
+#Description: This is used to generate PyTorch docs
+#Pinned versions: 5.3.0
+-e git+https://github.com/pytorch/pytorch_sphinx_theme.git@pytorch_sphinx_theme2#egg=pytorch_sphinx_theme2
+
+# TODO: sphinxcontrib.katex 0.9.0 adds a local KaTeX server to speed up pre-rendering
+# but it doesn't seem to work and hangs around idly. The initial thought is probably
+# something related to Docker setup. We can investigate this later
+
+sphinxcontrib.katex==0.8.6
+#Description: This is used to generate PyTorch docs
+#Pinned versions: 0.8.6
+
+sphinxext-opengraph==0.9.1
+#Description: This is used to generate PyTorch docs
+#Pinned versions: 0.9.1
+
+sphinx_sitemap==2.6.0
+#Description: This is used to generate sitemap for PyTorch docs
+#Pinned versions: 2.6.0
+
+matplotlib==3.5.3 ; python_version < "3.13"
+matplotlib==3.6.3 ; python_version >= "3.13"
+#Description: This is used to generate PyTorch docs
+#Pinned versions: 3.6.3 if python > 3.12. Otherwise 3.5.3.
+
+tensorboard==2.13.0 ; python_version < "3.13"
+tensorboard==2.18.0 ; python_version >= "3.13"
+#Description: This is used to generate PyTorch docs
+#Pinned versions: 2.13.0
+
+breathe==4.34.0
+#Description: This is used to generate PyTorch C++ docs
+#Pinned versions: 4.34.0
+
+exhale==0.2.3
+#Description: This is used to generate PyTorch C++ docs
+#Pinned versions: 0.2.3
+
+docutils==0.16
+#Description: This is used to generate PyTorch C++ docs
+#Pinned versions: 0.16
+
+bs4==0.0.1
+#Description: This is used to generate PyTorch C++ docs
+#Pinned versions: 0.0.1
+
+IPython==8.12.0
+#Description: This is used to generate PyTorch functorch docs
+#Pinned versions: 8.12.0
+
+myst-nb==0.17.2
+#Description: This is used to generate PyTorch functorch docs
+#Pinned versions: 0.13.2
+
+# The following are required to build torch.distributed.elastic.rendezvous.etcd* docs
+python-etcd==0.4.5
+sphinx-copybutton==0.5.0
+sphinx-design==0.4.0
+sphinxcontrib-mermaid==1.0.0
+myst-parser==0.18.1

.github/workflows/create_release.yml

@@ -35,7 +35,6 @@ jobs:
       contents: write
     outputs:
       pt_release_name: ${{ steps.release_name.outputs.pt_release_name }}
-      pt_pep517_release_name: ${{ steps.release_name.outputs.pt_pep517_release_name }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -54,57 +53,36 @@ jobs:
           tag_or_branch="${tag_or_branch#refs/heads/}"
           # replace directory separators with _ in branch name
           tag_or_branch="${tag_or_branch//\//_}"
-          torch_version="$(python -c 'from tools.generate_torch_version import get_torch_version; print(get_torch_version())')"
+          echo "PT_RELEASE_NAME=pytorch-$tag_or_branch" >> "$GITHUB_ENV"
-          {
+          echo "PT_RELEASE_FILE=pytorch-$tag_or_branch.tar.gz" >> "$GITHUB_ENV"
-            echo "PT_RELEASE_NAME=pytorch-$tag_or_branch";
-            echo "PT_RELEASE_FILE=pytorch-$tag_or_branch.tar.gz";
-            echo "PT_PEP517_RELEASE_FILE=torch-${torch_version}.tar.gz";
-          } >> "$GITHUB_ENV"
       - name: Checkout optional submodules
         run: python3 tools/optional_submodules.py
       - name: Create source distribution
         run: |
-          # Create new folder with specified name so extracting the archive yields that
+            # Create new folder with specified name so extracting the archive yields that
-          rm -rf "/tmp/$PT_RELEASE_NAME"
+            rm -rf "/tmp/$PT_RELEASE_NAME"
-          cp -r "$PWD" "/tmp/$PT_RELEASE_NAME"
+            cp -r "$PWD" "/tmp/$PT_RELEASE_NAME"
-          mv "/tmp/$PT_RELEASE_NAME" .
+            mv "/tmp/$PT_RELEASE_NAME" .
-          # Cleanup
+            # Cleanup
-          rm -rf "$PT_RELEASE_NAME"/{.circleci,.ci}
+            rm -rf "$PT_RELEASE_NAME"/{.circleci,.ci}
-          find "$PT_RELEASE_NAME" -name '.git*' -exec rm -rv {} \; || true
+            find "$PT_RELEASE_NAME" -name '.git*' -exec rm -rv {} \; || true
-          # Create archive
+            # Create archive
-          tar -czf "$PT_RELEASE_FILE" "$PT_RELEASE_NAME"
+            tar -czf "$PT_RELEASE_FILE" "$PT_RELEASE_NAME"
-          echo "Created source archive $PT_RELEASE_FILE with content: $(ls -a "$PT_RELEASE_NAME")"
+            echo "Created source archive $PT_RELEASE_FILE with content: $(ls -a "$PT_RELEASE_NAME")"
-      - name: Create PEP 517 compatible source distribution
-        run: |
-          pip install build==1.2.2.post1 || exit 1
-          python -m build --sdist || exit 1
-          cd dist || exit 1
       - name: Upload source distribution for release
         if: ${{ github.event_name == 'release' }}
         uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2
         with:
-          files: |
+          files: ${{env.PT_RELEASE_FILE}}
-            ${{ env.PT_RELEASE_FILE }}
-            ${{ env.PT_PEP517_RELEASE_FILE }}
       - name: Upload source distribution to GHA artifacts for release tags
         if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }}
         uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
           name: ${{ env.PT_RELEASE_FILE }}
           path: ${{ env.PT_RELEASE_FILE }}
-      - name: Upload PEP 517 source distribution to GHA artifacts for release tags
-        if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }}
-        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
-        with:
-          name: ${{ env.PT_PEP517_RELEASE_FILE }}
-          path: dist/${{ env.PT_PEP517_RELEASE_FILE }}
       - name: Set output
         id: release_name
-        run: |
+        run: echo "name=pt_release_name::${{ env.PT_RELEASE_NAME }}.tar.gz" >> "${GITHUB_OUTPUT}"
-          {
-            echo "name=pt_release_name::${{ env.PT_RELEASE_FILE }}";
-            echo "name=pt_pep517_release_name::${{ env.PT_PEP517_RELEASE_FILE }}";
-          } >> "${GITHUB_OUTPUT}"
 
   upload_source_code_to_s3:
     if: ${{ github.repository == 'pytorch/pytorch' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }}
@@ -120,9 +98,6 @@ jobs:
       - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ needs.release.outputs.pt_release_name }}
-      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
-        with:
-          name: ${{ needs.release.outputs.pt_pep517_release_name }}
       - name: Configure AWS credentials(PyTorch account)
         uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
         with:
@@ -133,9 +108,7 @@ jobs:
           s3-bucket: pytorch
           s3-prefix: source_code/test
           if-no-files-found: warn
-          path: |
+          path: ${{ needs.release.outputs.pt_release_name }}
-            ${{ needs.release.outputs.pt_release_name }}
-            ${{ needs.release.outputs.pt_pep517_release_name }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name }}

MANIFEST.in

@@ -1,91 +1,31 @@
-# Include individual top-level files
 include MANIFEST.in
-include BUCK.oss
+include CMakeLists.txt
-include BUILD.bazel
 include CITATION.cff
-include CODEOWNERS
-include Dockerfile
 include LICENSE
-include Makefile
 include NOTICE
-include WORKSPACE
+include .gitmodules
-include .bazelignore .bazelrc .bazelversion
+include build_variables.bzl
-include .clang-format .clang-tidy
+include mypy.ini
-include .cmakelintrc
+include requirements.txt
-include .coveragerc
+include ufunc_defs.bzl
-include .dockerignore
+include version.txt
-include .flake8
+recursive-include android *.*
-include .gdbinit
+recursive-include aten *.*
-include .lintrunner.toml
+recursive-include binaries *.*
-include .lldbinit
+recursive-include c10 *.*
-include docker.Makefile
+recursive-include caffe2 *.*
-include ubsan.supp
+recursive-include cmake *.*
-
+recursive-include torch *.*
-# Include bazel related files
+recursive-include tools *.*
-include *.bzl
+recursive-include test *.*
-# Include general configuration files
+recursive-include docs *.*
-include *.ini
+recursive-include ios *.*
-# Include important top-level information
+recursive-include third_party *
-include *.md
+recursive-include test *.*
-# Include technical text files
+recursive-include benchmarks *.*
-include *.txt
+recursive-include scripts *.*
-
+recursive-include mypy_plugins *.*
-# Include ctags configuration
+recursive-include modules *.*
-include .ctags.d/*.ctags
+recursive-include functorch *.*
-
-# Include subfolders completely
-graft .devcontainer
-graft .vscode
-graft android
-# The following folder (assets) is empty except for a .gitignore file, which
-# will not be included in the sdist, hence we include the directory explicitly.
-include android/test_app/app/src/main/assets
-graft aten
-graft binaries
-graft c10
-graft caffe2
-graft cmake
-graft torch
-graft tools
-graft test
-graft docs
-graft ios
-graft third_party
-graft test
-graft benchmarks
-graft scripts
-graft mypy_plugins
-graft modules
-graft functorch
-graft torchgen
-
-# The following exclusions omit parts from third-party dependencies that
-# contain invalid symlinks[1] and that are not needed for pytorch, such as
-# bindings for unused languages
-prune third_party/ittapi/rust
-prune third_party/flatbuffers/java
-prune third_party/flatbuffers/kotlin
-prune third_party/nccl/pkg/debian
-prune third_party/opentelemetry-cpp/third_party/prometheus-cpp/cmake/project-import-*
-
-# The following document is also an invalid symlink[1] and superfluous
-exclude third_party/flatbuffers/docs/source/CONTRIBUTING.md
-
-# Omit autogenerated code
-prune torchgen/packaged
-
-# Omit caches, compiled, and scm related content
 prune */__pycache__
-prune **/.github
+global-exclude *.o *.so *.dylib *.a .git *.pyc *.swp
-prune **/.gitlab
-global-exclude *.o *.so *.dylib *.a
-global-exclude *.pyc *.swp
-global-exclude .git .git-blame-ignore-revs .gitattributes .gitignore .gitmodules
-global-exclude .gitlab-ci.yml
-
-# [1] Invalid symlinks for the purposes of Python source distributions are,
-# according to the source distribution format[2] links pointing outside the
-# destination directory or links with a `..` component, which is those of
-# concern here.
-
-# [2] https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-archive-features

docs/requirements.txt

@@ -1,61 +0,0 @@
-sphinx==5.3.0
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 5.3.0
--e git+https://github.com/pytorch/pytorch_sphinx_theme.git@pytorch_sphinx_theme2#egg=pytorch_sphinx_theme2
-
-# TODO: sphinxcontrib.katex 0.9.0 adds a local KaTeX server to speed up pre-rendering
-# but it doesn't seem to work and hangs around idly. The initial thought is probably
-# something related to Docker setup. We can investigate this later
-
-sphinxcontrib.katex==0.8.6
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 0.8.6
-
-sphinxext-opengraph==0.9.1
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 0.9.1
-
-sphinx_sitemap==2.6.0
-#Description: This is used to generate sitemap for PyTorch docs
-#Pinned versions: 2.6.0
-
-matplotlib==3.5.3 ; python_version < "3.13"
-matplotlib==3.6.3 ; python_version >= "3.13"
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 3.6.3 if python > 3.12. Otherwise 3.5.3.
-
-tensorboard==2.13.0 ; python_version < "3.13"
-tensorboard==2.18.0 ; python_version >= "3.13"
-#Description: This is used to generate PyTorch docs
-#Pinned versions: 2.13.0
-
-breathe==4.34.0
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 4.34.0
-
-exhale==0.2.3
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 0.2.3
-
-docutils==0.16
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 0.16
-
-bs4==0.0.1
-#Description: This is used to generate PyTorch C++ docs
-#Pinned versions: 0.0.1
-
-IPython==8.12.0
-#Description: This is used to generate PyTorch functorch docs
-#Pinned versions: 8.12.0
-
-myst-nb==0.17.2
-#Description: This is used to generate PyTorch functorch docs
-#Pinned versions: 0.13.2
-
-# The following are required to build torch.distributed.elastic.rendezvous.etcd* docs
-python-etcd==0.4.5
-sphinx-copybutton==0.5.0
-sphinx-design==0.4.0
-sphinxcontrib-mermaid==1.0.0
-myst-parser==0.18.1

docs/requirements.txt

@@ -0,0 +1 @@
+../.ci/docker/requirements-docs.txt

functorch/docs/source/notebooks

@@ -0,0 +1 @@
+../../notebooks/

functorch/notebooks

@@ -1 +0,0 @@
-docs/source/notebooks