pytorch/.github/templates/linux_binary_build_workflow.yml.j2

{% import 'common.yml.j2' as common %}
{% import 'upload.yml.j2' as upload %}

{%- block name -%}
# Template is at:    .github/templates/linux_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: !{{ build_environment }}
{%- endblock %}


on:
  push:
    {%- if branches == "nightly" %}
    # NOTE: Meta Employees can trigger new nightlies using: https://fburl.com/trigger_pytorch_nightly_build
    {%- endif %}
    branches:
      - !{{ branches }}
    {%- if branches == "nightly" %}
    tags:
      # NOTE: Binary build pipelines should only get triggered on release candidate builds
      # Release candidate tags look like: v1.11.0-rc1
      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
    {%- endif %}
{%- for label in ciflow_config.labels | sort %}
    {%- if loop.first and branches != "nightly" %}
    tags:
    {%- endif %}
      - '!{{ label }}/*'
{%- endfor %}
  workflow_dispatch:

permissions:
  id-token: write

env:
  # Needed for conda builds
  {%- if "aarch64" in build_environment %}
  ALPINE_IMAGE: "arm64v8/alpine"
  {%- elif "s390x" in build_environment %}
  ALPINE_IMAGE: "docker.io/s390x/alpine"
  {%- else %}
  ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
  {%- endif %}
  AWS_DEFAULT_REGION: us-east-1
  BINARY_ENV_FILE: /tmp/env
  BUILD_ENVIRONMENT: !{{ build_environment }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  PR_NUMBER: ${{ github.event.pull_request.number }}
  PYTORCH_FINAL_PACKAGE_DIR: /artifacts
  PYTORCH_ROOT: /pytorch
  SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
  SKIP_ALL_TESTS: 0
!{{ common.concurrency(build_environment) }}

jobs:
  get-label-type:
    if: github.repository_owner == 'pytorch'
    name: get-label-type
    uses: pytorch/pytorch/.github/workflows/_runner-determinator.yml@main
    with:
      triggering_actor: ${{ github.triggering_actor }}
      issue_owner: ${{ github.event.pull_request.user.login || github.event.issue.user.login }}
      curr_branch: ${{ github.head_ref || github.ref_name }}
      curr_ref_type: ${{ github.ref_type }}

{%- for config in build_configs %}
  !{{ config["build_name"] }}-build:
    if: ${{ github.repository_owner == 'pytorch' }}
    uses: ./.github/workflows/_binary-build-linux.yml
    needs: get-label-type
    with:!{{ upload.binary_env_as_input(config) }}
      {%- if "aarch64" in build_environment %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      runs_on: linux.arm64.r7g.12xlarge.memory
      ALPINE_IMAGE: "arm64v8/alpine"
      {%- elif "s390x" in build_environment %}
      runs_on: linux.s390x
      ALPINE_IMAGE: "docker.io/s390x/alpine"
      timeout-minutes: 420
      {%- elif config["gpu_arch_type"] == "rocm" %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      timeout-minutes: 300
      {%- elif "conda" in build_environment and config["gpu_arch_type"] == "cuda" %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      runs_on: linux.24xlarge.ephemeral
      {%- else %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      {%- endif %}
      build_name: !{{ config["build_name"] }}
      build_environment: !{{ build_environment }}
      {%- if config.pytorch_extra_install_requirements is defined and config.pytorch_extra_install_requirements|d('')|length > 0  %}
      PYTORCH_EXTRA_INSTALL_REQUIREMENTS: !{{ config.pytorch_extra_install_requirements }}
      {%- endif %}
      {%- if config["gpu_arch_type"] == "cuda-aarch64" %}
      timeout-minutes: 420
      {%- endif %}
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}

  {%- if config["gpu_arch_type"] != "cuda-aarch64" %}
  !{{ config["build_name"] }}-test:  # Testing
    if: ${{ github.repository_owner == 'pytorch' }}
    needs:
      - !{{ config["build_name"] }}-build
      - get-label-type
    {%- if config["gpu_arch_type"] not in ["rocm", "xpu"] %}
    uses: ./.github/workflows/_binary-test-linux.yml
    with:!{{ upload.binary_env_as_input(config) }}
      build_name: !{{ config["build_name"] }}
      build_environment: !{{ build_environment }}
      {%- if "aarch64" in build_environment %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      runs_on: linux.arm64.2xlarge
      ALPINE_IMAGE: "arm64v8/alpine"
      {%- elif "s390x" in build_environment %}
      runs_on: linux.s390x
      ALPINE_IMAGE: "docker.io/s390x/alpine"
      {%- elif config["gpu_arch_type"] == "rocm" %}
      runs_on: linux.rocm.gpu
      {%- elif config["gpu_arch_type"] == "cuda" and config["gpu_arch_version"] in ["12.6"] %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      runs_on: linux.4xlarge.nvidia.gpu  # 12.6 build can use maxwell (sm_50) runner
      {%- elif config["gpu_arch_type"] == "cuda" %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      runs_on: linux.g4dn.4xlarge.nvidia.gpu # 12.8+ builds need sm_70+ runner
      {%- else %}
      runner_prefix: "${{ needs.get-label-type.outputs.label-type }}"
      runs_on: linux.4xlarge
      {%- endif %}
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}
    {%- elif config["gpu_arch_type"] == "xpu" %}
    runs-on: linux.idc.xpu
    timeout-minutes: !{{ common.timeout_minutes }}
    !{{ upload.binary_env(config) }}
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Setup XPU
        uses: pytorch/pytorch/.github/actions/setup-xpu@main
      - name: configure aws credentials
        id: aws_creds
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_s3_and_ecr_read_only
          aws-region: us-east-1
      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2
      - uses: !{{ common.download_artifact_action }}
        name: Download Build Artifacts
        with:
          name: !{{ config["build_name"] }}
          path: "${{ runner.temp }}/artifacts/"
      !{{ common.checkout(deep_clone=False, directory="pytorch") }}
      - name: Calculate docker image
        id: calculate-docker-image
        uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
        with:
          docker-registry: ${{ startsWith(github.event.ref, 'refs/tags/ciflow/') && '308535385114.dkr.ecr.us-east-1.amazonaws.com' || 'docker.io' }}
          docker-image-name: !{{ config["container_image"] }}
          custom-tag-prefix: !{{ config["container_image_tag_prefix"] }}
          docker-build-dir: .ci/docker
          working-directory: pytorch
      - name: Pull Docker image
        uses: pytorch/test-infra/.github/actions/pull-docker-image@main
        with:
          docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
      - name: Test Pytorch binary
        uses: ./pytorch/.github/actions/test-pytorch-binary
        env:
          DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
      - name: Teardown XPU
        uses: ./.github/actions/teardown-xpu
    {%- else %}
    runs-on: linux.rocm.gpu.mi250
    timeout-minutes: !{{ common.timeout_minutes }}
    !{{ upload.binary_env(config) }}
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Setup ROCm
        uses: ./.github/actions/setup-rocm
      - uses: !{{ common.download_artifact_action }}
        name: Download Build Artifacts
        with:
          name: !{{ config["build_name"] }}
          path: "${{ runner.temp }}/artifacts/"
      !{{ common.checkout(deep_clone=False, directory="pytorch") }}
      - name: ROCm set GPU_FLAG
        run: |
          echo "GPU_FLAG=--device=/dev/mem --device=/dev/kfd --device=/dev/dri --group-add video --group-add daemon" >> "${GITHUB_ENV}"
      - name: configure aws credentials
        id: aws_creds
        if: ${{ startsWith(github.event.ref, 'refs/tags/ciflow/') }}
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_s3_and_ecr_read_only
          aws-region: us-east-1
          role-duration-seconds: 18000
      - name: Calculate docker image
        id: calculate-docker-image
        uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
        with:
          docker-registry: ${{ startsWith(github.event.ref, 'refs/tags/ciflow/') && '308535385114.dkr.ecr.us-east-1.amazonaws.com' || 'docker.io' }}
          docker-image-name: !{{ config["container_image"] }}
          custom-tag-prefix: !{{ config["container_image_tag_prefix"] }}
          docker-build-dir: .ci/docker
          working-directory: pytorch
      - name: Pull Docker image
        uses: pytorch/test-infra/.github/actions/pull-docker-image@main
        with:
          docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
      - name: Test Pytorch binary
        uses: ./pytorch/.github/actions/test-pytorch-binary
        env:
          DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
      - name: Teardown ROCm
        uses: ./.github/actions/teardown-rocm
    {%- endif %}
  {%- endif %}

{%- if branches == "nightly" %}
  !{{ upload.upload_binaries(config) }}
{%- endif %}
{% endfor %}