From 023887fc5af5a7273d7eb0388fdf1f0e87eafb28 Mon Sep 17 00:00:00 2001 From: PyTorch MergeBot Date: Tue, 1 Jul 2025 14:14:52 +0000 Subject: [PATCH] Revert "Switch to standard pep517 sdist generation (#152098)" This reverts commit f16053f0c9a09fa337fbf85aaf64f88712b8dcdb. Reverted https://github.com/pytorch/pytorch/pull/152098 on behalf of https://github.com/malfet due to IMO this PR needs to be split into few helper ones, with better test plan ([comment](https://github.com/pytorch/pytorch/pull/152098#issuecomment-3024223880)) --- .ci/docker/build.sh | 4 +- .ci/docker/requirements-docs.txt | 62 +++++++++- .github/workflows/create_release.yml | 57 +++------ MANIFEST.in | 112 ++++-------------- docs/requirements.txt | 62 +--------- functorch/docs/source/notebooks | 1 + functorch/notebooks | 1 - .../notebooks/_src/plot_ensembling.py | 0 .../_src/plot_jacobians_and_hessians.py | 0 .../_src/plot_per_sample_gradients.py | 0 .../aot_autograd_optimizations.ipynb | 0 .../source => }/notebooks/ensembling.ipynb | 0 .../notebooks/jacobians_hessians.ipynb | 0 .../source => }/notebooks/minifier.ipynb | 0 .../notebooks/neural_tangent_kernels.ipynb | 0 .../notebooks/per_sample_grads.ipynb | 0 .../notebooks/whirlwind_tour.ipynb | 0 17 files changed, 106 insertions(+), 193 deletions(-) mode change 120000 => 100644 .ci/docker/requirements-docs.txt mode change 100644 => 120000 docs/requirements.txt create mode 120000 functorch/docs/source/notebooks delete mode 120000 functorch/notebooks rename functorch/{docs/source => }/notebooks/_src/plot_ensembling.py (100%) rename functorch/{docs/source => }/notebooks/_src/plot_jacobians_and_hessians.py (100%) rename functorch/{docs/source => }/notebooks/_src/plot_per_sample_gradients.py (100%) rename functorch/{docs/source => }/notebooks/aot_autograd_optimizations.ipynb (100%) rename functorch/{docs/source => }/notebooks/ensembling.ipynb (100%) rename functorch/{docs/source => }/notebooks/jacobians_hessians.ipynb (100%) rename functorch/{docs/source => }/notebooks/minifier.ipynb (100%) rename functorch/{docs/source => }/notebooks/neural_tangent_kernels.ipynb (100%) rename functorch/{docs/source => }/notebooks/per_sample_grads.ipynb (100%) rename functorch/{docs/source => }/notebooks/whirlwind_tour.ipynb (100%) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 33095950ca8c..2f7a85939e01 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -383,7 +383,7 @@ if [[ -n "${CI:-}" ]]; then fi # Build image -tar ch . | docker build \ +docker build \ ${no_cache_flag} \ ${progress_flag} \ --build-arg "BUILD_ENVIRONMENT=${image}" \ @@ -422,7 +422,7 @@ tar ch . | docker build \ -f $(dirname ${DOCKERFILE})/Dockerfile \ -t "$tmp_tag" \ "$@" \ - - + . # NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`, # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could diff --git a/.ci/docker/requirements-docs.txt b/.ci/docker/requirements-docs.txt deleted file mode 120000 index 22e439d953bb..000000000000 --- a/.ci/docker/requirements-docs.txt +++ /dev/null @@ -1 +0,0 @@ -../../docs/requirements.txt \ No newline at end of file diff --git a/.ci/docker/requirements-docs.txt b/.ci/docker/requirements-docs.txt new file mode 100644 index 000000000000..0d80c4d49372 --- /dev/null +++ b/.ci/docker/requirements-docs.txt @@ -0,0 +1,61 @@ +sphinx==5.3.0 +#Description: This is used to generate PyTorch docs +#Pinned versions: 5.3.0 +-e git+https://github.com/pytorch/pytorch_sphinx_theme.git@pytorch_sphinx_theme2#egg=pytorch_sphinx_theme2 + +# TODO: sphinxcontrib.katex 0.9.0 adds a local KaTeX server to speed up pre-rendering +# but it doesn't seem to work and hangs around idly. The initial thought is probably +# something related to Docker setup. We can investigate this later + +sphinxcontrib.katex==0.8.6 +#Description: This is used to generate PyTorch docs +#Pinned versions: 0.8.6 + +sphinxext-opengraph==0.9.1 +#Description: This is used to generate PyTorch docs +#Pinned versions: 0.9.1 + +sphinx_sitemap==2.6.0 +#Description: This is used to generate sitemap for PyTorch docs +#Pinned versions: 2.6.0 + +matplotlib==3.5.3 ; python_version < "3.13" +matplotlib==3.6.3 ; python_version >= "3.13" +#Description: This is used to generate PyTorch docs +#Pinned versions: 3.6.3 if python > 3.12. Otherwise 3.5.3. + +tensorboard==2.13.0 ; python_version < "3.13" +tensorboard==2.18.0 ; python_version >= "3.13" +#Description: This is used to generate PyTorch docs +#Pinned versions: 2.13.0 + +breathe==4.34.0 +#Description: This is used to generate PyTorch C++ docs +#Pinned versions: 4.34.0 + +exhale==0.2.3 +#Description: This is used to generate PyTorch C++ docs +#Pinned versions: 0.2.3 + +docutils==0.16 +#Description: This is used to generate PyTorch C++ docs +#Pinned versions: 0.16 + +bs4==0.0.1 +#Description: This is used to generate PyTorch C++ docs +#Pinned versions: 0.0.1 + +IPython==8.12.0 +#Description: This is used to generate PyTorch functorch docs +#Pinned versions: 8.12.0 + +myst-nb==0.17.2 +#Description: This is used to generate PyTorch functorch docs +#Pinned versions: 0.13.2 + +# The following are required to build torch.distributed.elastic.rendezvous.etcd* docs +python-etcd==0.4.5 +sphinx-copybutton==0.5.0 +sphinx-design==0.4.0 +sphinxcontrib-mermaid==1.0.0 +myst-parser==0.18.1 diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 8e3bef319b0d..2bcb841a715a 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -35,7 +35,6 @@ jobs: contents: write outputs: pt_release_name: ${{ steps.release_name.outputs.pt_release_name }} - pt_pep517_release_name: ${{ steps.release_name.outputs.pt_pep517_release_name }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -54,57 +53,36 @@ jobs: tag_or_branch="${tag_or_branch#refs/heads/}" # replace directory separators with _ in branch name tag_or_branch="${tag_or_branch//\//_}" - torch_version="$(python -c 'from tools.generate_torch_version import get_torch_version; print(get_torch_version())')" - { - echo "PT_RELEASE_NAME=pytorch-$tag_or_branch"; - echo "PT_RELEASE_FILE=pytorch-$tag_or_branch.tar.gz"; - echo "PT_PEP517_RELEASE_FILE=torch-${torch_version}.tar.gz"; - } >> "$GITHUB_ENV" + echo "PT_RELEASE_NAME=pytorch-$tag_or_branch" >> "$GITHUB_ENV" + echo "PT_RELEASE_FILE=pytorch-$tag_or_branch.tar.gz" >> "$GITHUB_ENV" - name: Checkout optional submodules run: python3 tools/optional_submodules.py - name: Create source distribution run: | - # Create new folder with specified name so extracting the archive yields that - rm -rf "/tmp/$PT_RELEASE_NAME" - cp -r "$PWD" "/tmp/$PT_RELEASE_NAME" - mv "/tmp/$PT_RELEASE_NAME" . - # Cleanup - rm -rf "$PT_RELEASE_NAME"/{.circleci,.ci} - find "$PT_RELEASE_NAME" -name '.git*' -exec rm -rv {} \; || true - # Create archive - tar -czf "$PT_RELEASE_FILE" "$PT_RELEASE_NAME" - echo "Created source archive $PT_RELEASE_FILE with content: $(ls -a "$PT_RELEASE_NAME")" - - name: Create PEP 517 compatible source distribution - run: | - pip install build==1.2.2.post1 || exit 1 - python -m build --sdist || exit 1 - cd dist || exit 1 + # Create new folder with specified name so extracting the archive yields that + rm -rf "/tmp/$PT_RELEASE_NAME" + cp -r "$PWD" "/tmp/$PT_RELEASE_NAME" + mv "/tmp/$PT_RELEASE_NAME" . + # Cleanup + rm -rf "$PT_RELEASE_NAME"/{.circleci,.ci} + find "$PT_RELEASE_NAME" -name '.git*' -exec rm -rv {} \; || true + # Create archive + tar -czf "$PT_RELEASE_FILE" "$PT_RELEASE_NAME" + echo "Created source archive $PT_RELEASE_FILE with content: $(ls -a "$PT_RELEASE_NAME")" - name: Upload source distribution for release if: ${{ github.event_name == 'release' }} uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 with: - files: | - ${{ env.PT_RELEASE_FILE }} - ${{ env.PT_PEP517_RELEASE_FILE }} + files: ${{env.PT_RELEASE_FILE}} - name: Upload source distribution to GHA artifacts for release tags if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }} uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: ${{ env.PT_RELEASE_FILE }} path: ${{ env.PT_RELEASE_FILE }} - - name: Upload PEP 517 source distribution to GHA artifacts for release tags - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }} - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: ${{ env.PT_PEP517_RELEASE_FILE }} - path: dist/${{ env.PT_PEP517_RELEASE_FILE }} - name: Set output id: release_name - run: | - { - echo "name=pt_release_name::${{ env.PT_RELEASE_FILE }}"; - echo "name=pt_pep517_release_name::${{ env.PT_PEP517_RELEASE_FILE }}"; - } >> "${GITHUB_OUTPUT}" + run: echo "name=pt_release_name::${{ env.PT_RELEASE_NAME }}.tar.gz" >> "${GITHUB_OUTPUT}" upload_source_code_to_s3: if: ${{ github.repository == 'pytorch/pytorch' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }} @@ -120,9 +98,6 @@ jobs: - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: name: ${{ needs.release.outputs.pt_release_name }} - - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 - with: - name: ${{ needs.release.outputs.pt_pep517_release_name }} - name: Configure AWS credentials(PyTorch account) uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: @@ -133,9 +108,7 @@ jobs: s3-bucket: pytorch s3-prefix: source_code/test if-no-files-found: warn - path: | - ${{ needs.release.outputs.pt_release_name }} - ${{ needs.release.outputs.pt_pep517_release_name }} + path: ${{ needs.release.outputs.pt_release_name }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name }} diff --git a/MANIFEST.in b/MANIFEST.in index d6c3a8f9f5c5..f6ffb4e02a8a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,91 +1,31 @@ -# Include individual top-level files include MANIFEST.in -include BUCK.oss -include BUILD.bazel +include CMakeLists.txt include CITATION.cff -include CODEOWNERS -include Dockerfile include LICENSE -include Makefile include NOTICE -include WORKSPACE -include .bazelignore .bazelrc .bazelversion -include .clang-format .clang-tidy -include .cmakelintrc -include .coveragerc -include .dockerignore -include .flake8 -include .gdbinit -include .lintrunner.toml -include .lldbinit -include docker.Makefile -include ubsan.supp - -# Include bazel related files -include *.bzl -# Include general configuration files -include *.ini -# Include important top-level information -include *.md -# Include technical text files -include *.txt - -# Include ctags configuration -include .ctags.d/*.ctags - -# Include subfolders completely -graft .devcontainer -graft .vscode -graft android -# The following folder (assets) is empty except for a .gitignore file, which -# will not be included in the sdist, hence we include the directory explicitly. -include android/test_app/app/src/main/assets -graft aten -graft binaries -graft c10 -graft caffe2 -graft cmake -graft torch -graft tools -graft test -graft docs -graft ios -graft third_party -graft test -graft benchmarks -graft scripts -graft mypy_plugins -graft modules -graft functorch -graft torchgen - -# The following exclusions omit parts from third-party dependencies that -# contain invalid symlinks[1] and that are not needed for pytorch, such as -# bindings for unused languages -prune third_party/ittapi/rust -prune third_party/flatbuffers/java -prune third_party/flatbuffers/kotlin -prune third_party/nccl/pkg/debian -prune third_party/opentelemetry-cpp/third_party/prometheus-cpp/cmake/project-import-* - -# The following document is also an invalid symlink[1] and superfluous -exclude third_party/flatbuffers/docs/source/CONTRIBUTING.md - -# Omit autogenerated code -prune torchgen/packaged - -# Omit caches, compiled, and scm related content +include .gitmodules +include build_variables.bzl +include mypy.ini +include requirements.txt +include ufunc_defs.bzl +include version.txt +recursive-include android *.* +recursive-include aten *.* +recursive-include binaries *.* +recursive-include c10 *.* +recursive-include caffe2 *.* +recursive-include cmake *.* +recursive-include torch *.* +recursive-include tools *.* +recursive-include test *.* +recursive-include docs *.* +recursive-include ios *.* +recursive-include third_party * +recursive-include test *.* +recursive-include benchmarks *.* +recursive-include scripts *.* +recursive-include mypy_plugins *.* +recursive-include modules *.* +recursive-include functorch *.* prune */__pycache__ -prune **/.github -prune **/.gitlab -global-exclude *.o *.so *.dylib *.a -global-exclude *.pyc *.swp -global-exclude .git .git-blame-ignore-revs .gitattributes .gitignore .gitmodules -global-exclude .gitlab-ci.yml - -# [1] Invalid symlinks for the purposes of Python source distributions are, -# according to the source distribution format[2] links pointing outside the -# destination directory or links with a `..` component, which is those of -# concern here. - -# [2] https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-archive-features \ No newline at end of file +global-exclude *.o *.so *.dylib *.a .git *.pyc *.swp diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 0d80c4d49372..000000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,61 +0,0 @@ -sphinx==5.3.0 -#Description: This is used to generate PyTorch docs -#Pinned versions: 5.3.0 --e git+https://github.com/pytorch/pytorch_sphinx_theme.git@pytorch_sphinx_theme2#egg=pytorch_sphinx_theme2 - -# TODO: sphinxcontrib.katex 0.9.0 adds a local KaTeX server to speed up pre-rendering -# but it doesn't seem to work and hangs around idly. The initial thought is probably -# something related to Docker setup. We can investigate this later - -sphinxcontrib.katex==0.8.6 -#Description: This is used to generate PyTorch docs -#Pinned versions: 0.8.6 - -sphinxext-opengraph==0.9.1 -#Description: This is used to generate PyTorch docs -#Pinned versions: 0.9.1 - -sphinx_sitemap==2.6.0 -#Description: This is used to generate sitemap for PyTorch docs -#Pinned versions: 2.6.0 - -matplotlib==3.5.3 ; python_version < "3.13" -matplotlib==3.6.3 ; python_version >= "3.13" -#Description: This is used to generate PyTorch docs -#Pinned versions: 3.6.3 if python > 3.12. Otherwise 3.5.3. - -tensorboard==2.13.0 ; python_version < "3.13" -tensorboard==2.18.0 ; python_version >= "3.13" -#Description: This is used to generate PyTorch docs -#Pinned versions: 2.13.0 - -breathe==4.34.0 -#Description: This is used to generate PyTorch C++ docs -#Pinned versions: 4.34.0 - -exhale==0.2.3 -#Description: This is used to generate PyTorch C++ docs -#Pinned versions: 0.2.3 - -docutils==0.16 -#Description: This is used to generate PyTorch C++ docs -#Pinned versions: 0.16 - -bs4==0.0.1 -#Description: This is used to generate PyTorch C++ docs -#Pinned versions: 0.0.1 - -IPython==8.12.0 -#Description: This is used to generate PyTorch functorch docs -#Pinned versions: 8.12.0 - -myst-nb==0.17.2 -#Description: This is used to generate PyTorch functorch docs -#Pinned versions: 0.13.2 - -# The following are required to build torch.distributed.elastic.rendezvous.etcd* docs -python-etcd==0.4.5 -sphinx-copybutton==0.5.0 -sphinx-design==0.4.0 -sphinxcontrib-mermaid==1.0.0 -myst-parser==0.18.1 diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 120000 index 000000000000..7176089000a2 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +../.ci/docker/requirements-docs.txt \ No newline at end of file diff --git a/functorch/docs/source/notebooks b/functorch/docs/source/notebooks new file mode 120000 index 000000000000..d4082256dcfe --- /dev/null +++ b/functorch/docs/source/notebooks @@ -0,0 +1 @@ +../../notebooks/ \ No newline at end of file diff --git a/functorch/notebooks b/functorch/notebooks deleted file mode 120000 index cacf787bdade..000000000000 --- a/functorch/notebooks +++ /dev/null @@ -1 +0,0 @@ -docs/source/notebooks \ No newline at end of file diff --git a/functorch/docs/source/notebooks/_src/plot_ensembling.py b/functorch/notebooks/_src/plot_ensembling.py similarity index 100% rename from functorch/docs/source/notebooks/_src/plot_ensembling.py rename to functorch/notebooks/_src/plot_ensembling.py diff --git a/functorch/docs/source/notebooks/_src/plot_jacobians_and_hessians.py b/functorch/notebooks/_src/plot_jacobians_and_hessians.py similarity index 100% rename from functorch/docs/source/notebooks/_src/plot_jacobians_and_hessians.py rename to functorch/notebooks/_src/plot_jacobians_and_hessians.py diff --git a/functorch/docs/source/notebooks/_src/plot_per_sample_gradients.py b/functorch/notebooks/_src/plot_per_sample_gradients.py similarity index 100% rename from functorch/docs/source/notebooks/_src/plot_per_sample_gradients.py rename to functorch/notebooks/_src/plot_per_sample_gradients.py diff --git a/functorch/docs/source/notebooks/aot_autograd_optimizations.ipynb b/functorch/notebooks/aot_autograd_optimizations.ipynb similarity index 100% rename from functorch/docs/source/notebooks/aot_autograd_optimizations.ipynb rename to functorch/notebooks/aot_autograd_optimizations.ipynb diff --git a/functorch/docs/source/notebooks/ensembling.ipynb b/functorch/notebooks/ensembling.ipynb similarity index 100% rename from functorch/docs/source/notebooks/ensembling.ipynb rename to functorch/notebooks/ensembling.ipynb diff --git a/functorch/docs/source/notebooks/jacobians_hessians.ipynb b/functorch/notebooks/jacobians_hessians.ipynb similarity index 100% rename from functorch/docs/source/notebooks/jacobians_hessians.ipynb rename to functorch/notebooks/jacobians_hessians.ipynb diff --git a/functorch/docs/source/notebooks/minifier.ipynb b/functorch/notebooks/minifier.ipynb similarity index 100% rename from functorch/docs/source/notebooks/minifier.ipynb rename to functorch/notebooks/minifier.ipynb diff --git a/functorch/docs/source/notebooks/neural_tangent_kernels.ipynb b/functorch/notebooks/neural_tangent_kernels.ipynb similarity index 100% rename from functorch/docs/source/notebooks/neural_tangent_kernels.ipynb rename to functorch/notebooks/neural_tangent_kernels.ipynb diff --git a/functorch/docs/source/notebooks/per_sample_grads.ipynb b/functorch/notebooks/per_sample_grads.ipynb similarity index 100% rename from functorch/docs/source/notebooks/per_sample_grads.ipynb rename to functorch/notebooks/per_sample_grads.ipynb diff --git a/functorch/docs/source/notebooks/whirlwind_tour.ipynb b/functorch/notebooks/whirlwind_tour.ipynb similarity index 100% rename from functorch/docs/source/notebooks/whirlwind_tour.ipynb rename to functorch/notebooks/whirlwind_tour.ipynb