Compare commits

...

6 Commits

Author SHA1 Message Date
6a84a8e8b9 tc 2025-10-05 14:04:39 -07:00
b68803025f tc 2025-10-03 15:56:06 -07:00
73d8f4b30a idk what im doing 2025-10-03 15:15:42 -07:00
6cac0e7404 tc 2025-10-03 11:43:56 -07:00
69501c64bc tc 2025-10-03 11:24:05 -07:00
e123f1803b tc 2025-10-03 11:23:08 -07:00
7 changed files with 136 additions and 283 deletions

View File

@ -365,20 +365,20 @@ def can_reuse_whl(args: argparse.Namespace) -> tuple[bool, str]:
if __name__ == "__main__":
args = parse_args()
reuse_whl, reason = can_reuse_whl(args)
# reuse_whl, reason = can_reuse_whl(args)
if reuse_whl:
if True:
print("Reusing old whl")
unzip_artifact_and_replace_files()
set_output()
emit_metric(
"reuse_old_whl",
{
"reuse_whl": reuse_whl,
"reason": reason,
"build_environment": args.build_environment,
"merge_base": get_merge_base(),
"head_sha": get_head_sha(),
},
)
# emit_metric(
# "reuse_old_whl",
# {
# "reuse_whl": reuse_whl,
# "reason": reason,
# "build_environment": args.build_environment,
# "merge_base": get_merge_base(),
# "head_sha": get_head_sha(),
# },
# )

View File

@ -92,271 +92,5 @@ jobs:
with:
build-environment: linux-jammy-py3.10-gcc11
docker-image: ${{ needs.linux-jammy-py3_10-gcc11-build.outputs.docker-image }}
secrets: inherit
linux-jammy-py3_10-gcc11-no-ops:
name: linux-jammy-py3.10-gcc11-no-ops
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-py3.10-gcc11-no-ops
docker-image-name: ci-image:pytorch-linux-jammy-py3.10-gcc11
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 1 },
]}
secrets: inherit
linux-jammy-py3_10-gcc11-pch:
name: linux-jammy-py3.10-gcc11-pch
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-py3.10-gcc11-pch
docker-image-name: ci-image:pytorch-linux-jammy-py3.10-gcc11
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 1 },
]}
secrets: inherit
linux-jammy-py3_10-clang18-asan-build:
name: linux-jammy-py3.10-clang18-asan
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
# More memory is needed to build with asan
runner: linux.2xlarge.memory
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-py3.10-clang18-asan
docker-image-name: ci-image:pytorch-linux-jammy-py3-clang18-asan
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 7, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 2, num_shards: 7, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 3, num_shards: 7, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 4, num_shards: 7, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 5, num_shards: 7, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 6, num_shards: 7, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 7, num_shards: 7, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
]}
sync-tag: asan-build
secrets: inherit
linux-jammy-py3_10-clang18-asan-test:
name: linux-jammy-py3.10-clang18-asan
uses: ./.github/workflows/_linux-test.yml
needs:
- linux-jammy-py3_10-clang18-asan-build
- target-determination
with:
build-environment: linux-jammy-py3.10-clang18-asan
docker-image: ${{ needs.linux-jammy-py3_10-clang18-asan-build.outputs.docker-image }}
test-matrix: ${{ needs.linux-jammy-py3_10-clang18-asan-build.outputs.test-matrix }}
sync-tag: asan-test
secrets: inherit
linux-jammy-py3_10-clang12-onnx-build:
name: linux-jammy-py3.10-clang12-onnx
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-py3.10-clang12-onnx
docker-image-name: ci-image:pytorch-linux-jammy-py3-clang12-onnx
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 2, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "default", shard: 2, num_shards: 2, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
]}
secrets: inherit
linux-jammy-py3_10-clang12-onnx-test:
name: linux-jammy-py3.10-clang12-onnx
uses: ./.github/workflows/_linux-test.yml
needs:
- linux-jammy-py3_10-clang12-onnx-build
- target-determination
with:
build-environment: linux-jammy-py3.10-clang12-onnx
docker-image: ${{ needs.linux-jammy-py3_10-clang12-onnx-build.outputs.docker-image }}
test-matrix: ${{ needs.linux-jammy-py3_10-clang12-onnx-build.outputs.test-matrix }}
secrets: inherit
linux-jammy-py3_10-clang12-build:
name: linux-jammy-py3.10-clang12
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-py3.10-clang12
docker-image-name: ci-image:pytorch-linux-jammy-py3.10-clang12
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 2, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 3, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 4, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 5, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "crossref", shard: 1, num_shards: 2, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "crossref", shard: 2, num_shards: 2, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "dynamo_wrapped", shard: 1, num_shards: 3, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "dynamo_wrapped", shard: 2, num_shards: 3, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "dynamo_wrapped", shard: 3, num_shards: 3, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "einops", shard: 1, num_shards: 1, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" }
]}
secrets: inherit
linux-jammy-py3_10-clang12-test:
name: linux-jammy-py3.10-clang12
uses: ./.github/workflows/_linux-test.yml
needs:
- linux-jammy-py3_10-clang12-build
- target-determination
with:
build-environment: linux-jammy-py3.10-clang12
docker-image: ${{ needs.linux-jammy-py3_10-clang12-build.outputs.docker-image }}
test-matrix: ${{ needs.linux-jammy-py3_10-clang12-build.outputs.test-matrix }}
secrets: inherit
linux-jammy-py3_13-clang12-build:
name: linux-jammy-py3.13-clang12
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-py3.13-clang12
docker-image-name: ci-image:pytorch-linux-jammy-py3.13-clang12
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 2, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 3, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 4, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "default", shard: 5, num_shards: 5, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
{ config: "crossref", shard: 1, num_shards: 2, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "crossref", shard: 2, num_shards: 2, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "dynamo_wrapped", shard: 1, num_shards: 3, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "dynamo_wrapped", shard: 2, num_shards: 3, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "dynamo_wrapped", shard: 3, num_shards: 3, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" },
{ config: "einops", shard: 1, num_shards: 1, runner: "${{ needs.get-label-type.outputs.label-type }}linux.2xlarge" }
]}
secrets: inherit
linux-jammy-py3_13-clang12-test:
name: linux-jammy-py3.13-clang12
uses: ./.github/workflows/_linux-test.yml
needs: linux-jammy-py3_13-clang12-build
with:
build-environment: linux-jammy-py3.13-clang12
docker-image: ${{ needs.linux-jammy-py3_13-clang12-build.outputs.docker-image }}
test-matrix: ${{ needs.linux-jammy-py3_13-clang12-build.outputs.test-matrix }}
secrets: inherit
linux-jammy-cuda12_8-cudnn9-py3_10-clang12-build:
name: linux-jammy-cuda12.8-cudnn9-py3.10-clang12
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-cuda12.8-cudnn9-py3.10-clang12
docker-image-name: ci-image:pytorch-linux-jammy-cuda12.8-cudnn9-py3.10-clang12
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 1 },
]}
secrets: inherit
linux-jammy-cpu-py3_10-gcc11-bazel-test:
name: linux-jammy-cpu-py3.10-gcc11-bazel-test
uses: ./.github/workflows/_bazel-build-test.yml
needs: get-label-type
with:
runner: "${{ needs.get-label-type.outputs.label-type }}linux.large"
build-environment: linux-jammy-cuda12.8-py3.10-gcc11-bazel-test
docker-image-name: ci-image:pytorch-linux-jammy-cuda12.8-cudnn9-py3-gcc11
cuda-version: cpu
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 1, runner: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" },
]}
secrets: inherit
linux-jammy-py3_10-gcc11-mobile-lightweight-dispatch-build:
name: linux-jammy-py3.10-gcc11-mobile-lightweight-dispatch-build
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-py3.10-gcc11-mobile-lightweight-dispatch-build
docker-image-name: ci-image:pytorch-linux-jammy-py3.10-gcc11
build-generates-artifacts: false
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 1 },
]}
secrets: inherit
linux-jammy-rocm-py3_10-build:
# don't run build twice on main
if: github.event_name == 'pull_request'
name: linux-jammy-rocm-py3.10
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-rocm-py3.10
docker-image-name: ci-image:pytorch-linux-jammy-rocm-n-py3
sync-tag: rocm-build
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 3, runner: "linux.rocm.gpu.2" },
{ config: "default", shard: 2, num_shards: 3, runner: "linux.rocm.gpu.2" },
{ config: "default", shard: 3, num_shards: 3, runner: "linux.rocm.gpu.2" },
]}
secrets: inherit
linux-jammy-cuda12_8-py3_10-gcc9-inductor-build:
name: cuda12.8-py3.10-gcc9-sm75
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
build-environment: linux-jammy-cuda12.8-py3.10-gcc9-sm75
docker-image-name: ci-image:pytorch-linux-jammy-cuda12.8-cudnn9-py3-gcc9-inductor-benchmarks
cuda-arch-list: '7.5'
test-matrix: |
{ include: [
{ config: "pr_time_benchmarks", shard: 1, num_shards: 1, runner: "linux.g4dn.metal.nvidia.gpu" },
]}
secrets: inherit
linux-jammy-cuda12_8-py3_10-gcc9-inductor-test:
name: cuda12.8-py3.10-gcc9-sm75
uses: ./.github/workflows/_linux-test.yml
needs: linux-jammy-cuda12_8-py3_10-gcc9-inductor-build
with:
build-environment: linux-jammy-cuda12.8-py3.10-gcc9-sm75
docker-image: ${{ needs.linux-jammy-cuda12_8-py3_10-gcc9-inductor-build.outputs.docker-image }}
test-matrix: ${{ needs.linux-jammy-cuda12_8-py3_10-gcc9-inductor-build.outputs.test-matrix }}
secrets: inherit
linux-jammy-xpu-n-py3_10-build:
name: linux-jammy-xpu-n-py3.10
uses: ./.github/workflows/_linux-build.yml
needs: get-label-type
with:
sync-tag: linux-xpu-n-build
runner_prefix: ${{ needs.get-label-type.outputs.label-type }}
build-environment: linux-jammy-xpu-n-py3.10
docker-image-name: ci-image:pytorch-linux-jammy-xpu-n-py3
test-matrix: |
{ include: [
{ config: "default", shard: 1, num_shards: 4, runner: "linux.idc.xpu" },
{ config: "default", shard: 2, num_shards: 4, runner: "linux.idc.xpu" },
{ config: "default", shard: 3, num_shards: 4, runner: "linux.idc.xpu" },
{ config: "default", shard: 4, num_shards: 4, runner: "linux.idc.xpu" },
]}
run-doxygen: true
secrets: inherit

View File

@ -24,4 +24,5 @@ clean:
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
bash source/pre_run_doxygen.sh
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

View File

@ -96,6 +96,13 @@ GENERATE_XML = YES
# NOTE: setting to NO may result in unrecovered file relationships
# (which file defined which compound)
XML_PROGRAMLISTING = YES
# Turn off heavy cross-reference and source browser features, which cost memory
# but add cross-linking data that Exhale doesnt use
REFERENCES_RELATION = NO
REFERENCED_BY_RELATION = NO
REFERENCES_LINK_SOURCE = NO
SOURCE_BROWSER = NO
################################################################################
# Doxygen preprocessor / parser control. #
################################################################################

View File

@ -86,8 +86,12 @@ repo_root = os.path.dirname( # {repo_root}
)
)
breathe_projects = {"PyTorch": doxygen_xml_dir}
breathe_default_project = "PyTorch"
breathe_projects = {
"ATen": os.path.join(doxygen_xml_dir, "aten"),
"C10": os.path.join(doxygen_xml_dir, "c10"),
"CSRC": os.path.join(doxygen_xml_dir, "csrc"),
}
breathe_default_project = "ATen"
# Setup the exhale extension
exhale_args = {
@ -102,7 +106,8 @@ exhale_args = {
# Suggested optional arguments. #
############################################################################
"createTreeView": True,
"exhaleExecutesDoxygen": True,
"exhaleExecutesDoxygen": False,
"exhaleUseBreatheProjects": True,
"exhaleUseDoxyfile": True,
"verboseBuild": True,
############################################################################
@ -145,6 +150,30 @@ exhale_args = {
"fullToctreeMaxDepth": 2,
}
exhale_args_chunks = [
{
**exhale_args,
"containmentFolder": "./api/aten",
"rootFileName": "aten_root.rst",
"rootFileTitle": "ATen API",
"breatheProjects": {"ATen": breathe_projects["ATen"]},
},
{
**exhale_args,
"containmentFolder": "./api/c10",
"rootFileName": "c10_root.rst",
"rootFileTitle": "C10 API",
"breatheProjects": {"C10": breathe_projects["C10"]},
},
{
**exhale_args,
"containmentFolder": "./api/csrc",
"rootFileName": "csrc_root.rst",
"rootFileTitle": "CSRC API",
"breatheProjects": {"CSRC": breathe_projects["CSRC"]},
},
]
# Tell sphinx what the primary language being documented is.
primary_domain = "cpp"
@ -329,3 +358,11 @@ texinfo_documents = [
"Miscellaneous",
),
]
# Function to run Exhale for each chunk
def setup(app):
from exhale import setup as exhale_setup
for chunk in exhale_args_chunks:
exhale_setup(app, chunk)

View File

@ -161,7 +161,9 @@ Contents
installing
frontend
api/library_root
api/aten/aten_root
api/c10/c10_root
api/csrc/csrc_root
.. toctree::
:glob:

View File

@ -0,0 +1,72 @@
#!/bin/bash
# Split up the doxygen run into multiple runs so that we can set different
# INPUT and XML_OUTPUT values for each run. This way we can have more control
# ATen
doxygen - <<EOF
$(cat Doxyfile)
INPUT = ../../../aten/src/ATen/ATen.h \
../../../aten/src/ATen/Backend.h \
../../../aten/src/ATen/core/ivalue.h \
../../../aten/src/ATen/core/ScalarType.h \
../../../aten/src/ATen/cuda/CUDAContext.h \
../../../aten/src/ATen/cudnn/Descriptors.h \
../../../aten/src/ATen/cudnn/Handles.h \
../../../aten/src/ATen/cudnn/Types.h \
../../../aten/src/ATen/cudnn/Utils.h \
../../../aten/src/ATen/DeviceGuard.h \
../../../aten/src/ATen/Layout.h \
../../../aten/src/ATen/mkl/Descriptors.h \
../../../aten/src/ATen/Scalar.h \
../../../aten/src/ATen/TensorOptions.h \
../../../aten/src/ATen/core/Tensor.h \
../../../aten/src/ATen/native/TensorShape.h \
../../../build/aten/src/ATen/Functions.h \
../../../build/aten/src/ATen/core/TensorBody.h
XML_OUTPUT = xml/aten
EOF
# c10
doxygen - <<EOF
$(cat Doxyfile)
INPUT = ../../../c10/core/Device.h \
../../../c10/core/DeviceType.h \
../../../c10/util/Half.h \
../../../c10/util/ArrayRef.h \
../../../c10/util/OptionalArrayRef.h \
../../../c10/util/Exception.h \
../../../c10/util/Optional.h \
../../../c10/cuda/CUDAGuard.h \
../../../c10/cuda/CUDAStream.h \
../../../c10/xpu/XPUStream.h
XML_OUTPUT = xml/c10
EOF
# csrc
doxygen - <<EOF
$(cat Doxyfile)
INPUT = ../../../torch/csrc/api/include \
../../../torch/csrc/api/src \
../../../torch/csrc/autograd/autograd.h \
../../../torch/csrc/autograd/custom_function.h \
../../../torch/csrc/autograd/function.h \
../../../torch/csrc/autograd/variable.h \
../../../torch/csrc/autograd/generated/variable_factories.h \
../../../torch/csrc/jit/runtime/custom_operator.h \
../../../torch/csrc/jit/serialization/import.h \
../../../torch/csrc/jit/api/module.h \
../../../torch/csrc/stable/library.h
XML_OUTPUT = xml/csrc
EOF
# other
doxygen - <<EOF
$(cat Doxyfile)
INPUT = ../../../torch/library.h \
../../../torch/custom_class.h
XML_OUTPUT = xml/csrc
EOF