mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 12:54:11 +08:00
b61bdc7cc4
Fixes #ISSUE_NUMBER Pull Request resolved: https://github.com/pytorch/pytorch/pull/162774 Approved by: https://github.com/malfet, https://github.com/atalman
213 lines
8.4 KiB
YAML
213 lines
8.4 KiB
YAML
name: build docs
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
build-environment:
|
|
required: true
|
|
type: string
|
|
description: Top-level label for what's being built/tested.
|
|
docker-image:
|
|
required: true
|
|
type: string
|
|
description: Docker image to run in.
|
|
push:
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
description: If set, push the docs to the docs website.
|
|
run-doxygen:
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
description: If set, will enable C++ API doc generation using doxygen / breathe / exhale.
|
|
sync-tag:
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
description: |
|
|
If this is set, our linter will use this to make sure that every other
|
|
job with the same `sync-tag` is identical.
|
|
s3-bucket:
|
|
description: S3 bucket to download artifact
|
|
required: false
|
|
type: string
|
|
default: "gha-artifacts"
|
|
aws-role-to-assume:
|
|
description: role to assume for downloading artifacts
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
upload-aws-role-to-assume:
|
|
description: role to assume for downloading artifacts
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
runner_prefix:
|
|
description: prefix for runner label
|
|
type: string
|
|
default: ""
|
|
secrets:
|
|
GH_PYTORCHBOT_TOKEN:
|
|
required: false
|
|
description: Permissions for pushing to the docs site.
|
|
|
|
jobs:
|
|
build-docs:
|
|
# Don't run on forked repos.
|
|
if: github.repository_owner == 'pytorch'
|
|
runs-on: ${{ matrix.runner }}
|
|
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'pytorchbot-env' || '' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- docs_type: cpp
|
|
# We recently seeing lots of exit code 137 running this in Docker indicating
|
|
# an OOM issue when running the job, so this upgrades the runner from 4xlarge
|
|
# to the next available tier of 12xlarge. So much memory just to generate cpp
|
|
# doc
|
|
runner: ${{ inputs.runner_prefix }}linux.12xlarge.memory
|
|
# TODO: Nightly cpp docs take longer and longer to finish (more than 3h now)
|
|
# Let's try to figure out how this can be improved
|
|
timeout-minutes: 360
|
|
- docs_type: python
|
|
runner: ${{ inputs.runner_prefix }}linux.2xlarge
|
|
# It takes less than 30m to finish python docs unless there are issues
|
|
timeout-minutes: 30
|
|
# Set a fixed name for this job instead of using the current matrix-generated name, i.e. build-docs (cpp, linux.12xlarge, 180)
|
|
# The current name requires updating the database last docs push query from test-infra every time the matrix is updated
|
|
name: build-docs-${{ matrix.docs_type }}-${{ inputs.push }}
|
|
steps:
|
|
- name: Setup SSH (Click me for login details)
|
|
uses: pytorch/test-infra/.github/actions/setup-ssh@main
|
|
with:
|
|
github-secret: ${{ secrets.GITHUB_TOKEN }}
|
|
instructions: |
|
|
All builds are done inside the container, to start an interactive session run:
|
|
docker exec -it $(docker container ps --format '{{.ID}}') bash
|
|
To start Python docs build type:
|
|
cd docs && make html && make coverage
|
|
|
|
# [see note: pytorch repo ref]
|
|
- name: Checkout PyTorch
|
|
uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
|
|
|
|
- name: Setup Linux
|
|
uses: ./.github/actions/setup-linux
|
|
|
|
- name: configure aws credentials
|
|
if : ${{ inputs.aws-role-to-assume != '' }}
|
|
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
|
|
with:
|
|
role-to-assume: ${{ inputs.aws-role-to-assume }}
|
|
role-session-name: gha-linux-test
|
|
aws-region: us-east-1
|
|
|
|
- name: Calculate docker image
|
|
id: calculate-docker-image
|
|
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
|
|
with:
|
|
docker-image-name: ${{ inputs.docker-image }}
|
|
|
|
- name: Pull docker image
|
|
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
|
|
with:
|
|
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
|
|
|
|
- name: Download build artifacts
|
|
uses: ./.github/actions/download-build-artifacts
|
|
with:
|
|
name: ${{ inputs.build-environment }}
|
|
s3-bucket: ${{ inputs.s3-bucket }}
|
|
|
|
- name: Generate netrc (only for docs-push)
|
|
if: inputs.push
|
|
env:
|
|
GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
|
|
run: |
|
|
# sometimes .netrc exists as a directory even though this is the temp folder
|
|
rm -rf "${RUNNER_TEMP}/.netrc"
|
|
# set credentials for https pushing
|
|
echo "machine github.com" > "${RUNNER_TEMP}/.netrc"
|
|
echo "login pytorchbot" >> "${RUNNER_TEMP}/.netrc"
|
|
echo "password ${GITHUB_PYTORCHBOT_TOKEN}" >> "${RUNNER_TEMP}/.netrc"
|
|
|
|
- name: Build ${{ matrix.docs_type }} docs
|
|
timeout-minutes: ${{ matrix.timeout-minutes }}
|
|
id: build-docs
|
|
env:
|
|
# After https://github.com/pytorch/pytorch/pull/88373, pull workflow can now be run periodically,
|
|
# so using a schedule event to determine if the docs should be pushed or not doesn't hold true
|
|
# anymore
|
|
WITH_PUSH: ${{ inputs.push }}
|
|
DOCKER_IMAGE: ${{ inputs.docker-image }}
|
|
DOCS_TYPE: ${{ matrix.docs_type }}
|
|
RUN_DOXYGEN: ${{ inputs.run-doxygen }}
|
|
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
|
|
run: |
|
|
set -ex
|
|
# Convert refs/tags/v1.12.0rc3 into 1.12
|
|
if [[ "${GITHUB_REF}" =~ ^refs/tags/v([0-9]+\.[0-9]+)\.* ]]; then
|
|
target="${BASH_REMATCH[1]}"
|
|
else
|
|
target="main"
|
|
fi
|
|
# detached container should get cleaned up by teardown_ec2_linux
|
|
container_name=$(docker run \
|
|
-e BUILD_ENVIRONMENT \
|
|
-e MAX_JOBS="$(nproc --ignore=2)" \
|
|
-e SHA1="$GITHUB_SHA" \
|
|
-e DOCS_VERSION="${target}" \
|
|
-e DOCS_TYPE \
|
|
-e RUN_DOXYGEN \
|
|
-e WITH_PUSH \
|
|
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
|
|
--security-opt seccomp=unconfined \
|
|
--cap-add=SYS_PTRACE \
|
|
--tty \
|
|
--detach \
|
|
--user jenkins \
|
|
-v "${RUNNER_TEMP}/.netrc":/var/lib/jenkins/.netrc \
|
|
-v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
|
|
-w /var/lib/jenkins/workspace \
|
|
"${DOCKER_IMAGE}"
|
|
)
|
|
docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install $(echo dist/*.whl)[opt-einsum] && ./.ci/pytorch/${DOCS_TYPE}_doc_push_script.sh"
|
|
|
|
- name: Chown workspace
|
|
uses: ./.github/actions/chown-workspace
|
|
if: always()
|
|
|
|
- name: configure aws credentials
|
|
if : ${{ inputs.upload-aws-role-to-assume != '' }}
|
|
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
|
|
with:
|
|
role-to-assume: ${{ inputs.upload-aws-role-to-assume }}
|
|
role-session-name: gha-linux-test
|
|
aws-region: us-east-1
|
|
|
|
- name: Upload Python Docs Preview
|
|
uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
|
|
if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' && steps.build-docs.outcome == 'success' }}
|
|
with:
|
|
retention-days: 14
|
|
s3-bucket: doc-previews
|
|
if-no-files-found: error
|
|
path: pytorch_docs/main/
|
|
s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}
|
|
|
|
- name: Upload C++ Docs Preview
|
|
uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
|
|
if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
|
|
with:
|
|
retention-days: 14
|
|
if-no-files-found: error
|
|
s3-bucket: doc-previews
|
|
path: cppdocs/
|
|
s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/cppdocs
|
|
|
|
- name: Teardown Linux
|
|
uses: pytorch/test-infra/.github/actions/teardown-linux@main
|
|
if: always()
|