From d6048ecd6b172fcf0027c1a48839064a3f3abfe5 Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Tue, 3 Aug 2021 11:03:27 -0700 Subject: [PATCH] Enable bazel builds on `ciflow/default` (#62649) Summary: Add `regenerate.sh` convenience script Remove "TODO: Reenable on PR" label from workflows which are enabled on PRs Pull Request resolved: https://github.com/pytorch/pytorch/pull/62649 Reviewed By: seemethere Differential Revision: D30071905 Pulled By: malfet fbshipit-source-id: c82134cb676b273d23e225be21166588996a31d3 --- .github/regenerate.sh | 6 ++++++ .github/scripts/generate_ci_workflows.py | 8 +++++++- .github/templates/bazel_ci_workflow.yml.j2 | 15 +++++++++++++++ .github/templates/linux_ci_workflow.yml.j2 | 3 ++- ...generated-linux-bionic-py3.8-gcc9-coverage.yml | 1 - ...ed-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml | 1 - .../generated-linux-xenial-py3.6-gcc5.4.yml | 1 - ...nerated-linux-xenial-py3.6-gcc7-bazel-test.yml | 14 +++++++++++--- ...ch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml | 1 - ...ic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml | 1 - 10 files changed, 41 insertions(+), 10 deletions(-) create mode 100755 .github/regenerate.sh diff --git a/.github/regenerate.sh b/.github/regenerate.sh new file mode 100755 index 000000000000..f71707b0d09a --- /dev/null +++ b/.github/regenerate.sh @@ -0,0 +1,6 @@ +#!/bin/bash -e + +# Allows this script to be invoked from any directory: +cd "$(dirname "$0")" + +python3 scripts/generate_ci_workflows.py diff --git a/.github/scripts/generate_ci_workflows.py b/.github/scripts/generate_ci_workflows.py index 63f23915ec7a..8527d4e46d8b 100755 --- a/.github/scripts/generate_ci_workflows.py +++ b/.github/scripts/generate_ci_workflows.py @@ -269,7 +269,7 @@ LINUX_WORKFLOWS = [ enabled=True, trigger_action_only=True, labels=set(['ciflow/slow']), - ) + ), ), CIWorkflow( arch="linux", @@ -401,6 +401,12 @@ BAZEL_WORKFLOWS = [ build_environment="linux-xenial-py3.6-gcc7-bazel-test", docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc7", test_runner_type=LINUX_CPU_TEST_RUNNER, + on_pull_request=True, + ciflow_config=CIFlowConfig( + enabled=True, + trigger_action_only=True, + labels=set(['ciflow/default']), + ), ), ] diff --git a/.github/templates/bazel_ci_workflow.yml.j2 b/.github/templates/bazel_ci_workflow.yml.j2 index 4e85bab976bb..8634c625d864 100644 --- a/.github/templates/bazel_ci_workflow.yml.j2 +++ b/.github/templates/bazel_ci_workflow.yml.j2 @@ -5,6 +5,21 @@ # Generation script: .github/scripts/generate_ci_workflows.py name: !{{ build_environment }} {%- endblock %} + +on: +{%- if on_pull_request %} + pull_request: + {%- if ciflow_config.enabled %} + {%- if ciflow_config.trigger_action_only %} + types: [!{{ ciflow_config.trigger_action }}] + {%- else %} + types: [opened, synchronize, reopened, !{{ ciflow_config.trigger_action }}] + {%- endif %} + {%- endif %} +{%- else %} + # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers +{%- endif %} + {% block build +%} # building and testing in a single job since bazel runs only small subset of tests build-and-test: diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index a1ca02cda617..33e7fcb0147f 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -10,7 +10,6 @@ name: !{{ build_environment }} {%- endblock %} on: - # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers {%- if on_pull_request %} pull_request: {%- if ciflow_config.enabled %} @@ -20,6 +19,8 @@ on: types: [opened, synchronize, reopened, !{{ ciflow_config.trigger_action }}] {%- endif %} {%- endif %} +{%- else %} + # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers {%- endif %} {%- if is_scheduled %} diff --git a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml index 21076f66337e..0a892ffb632f 100644 --- a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml +++ b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml @@ -4,7 +4,6 @@ name: linux-bionic-py3.8-gcc9-coverage on: - # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers pull_request: types: [opened, synchronize, reopened, unassigned] push: diff --git a/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml index ed83d5d4a8d1..33beb21e03fb 100644 --- a/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml @@ -4,7 +4,6 @@ name: linux-xenial-cuda10.2-cudnn7-py3.6-gcc7 on: - # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers pull_request: types: [unassigned] push: diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml index 539131b37780..47ca1b7a59b1 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml @@ -4,7 +4,6 @@ name: linux-xenial-py3.6-gcc5.4 on: - # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers pull_request: push: branches: diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml index 9d140f10a65b..e1f9e542b820 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml @@ -4,7 +4,8 @@ name: linux-xenial-py3.6-gcc7-bazel-test on: - # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers + pull_request: + types: [unassigned] push: branches: - master @@ -28,9 +29,16 @@ concurrency: cancel-in-progress: true jobs: + ciflow_should_run: + runs-on: ubuntu-18.04 + if: ${{ (github.event_name != 'pull_request') || (github.event.action !='unassigned') || (github.event.action == 'unassigned' && contains(github.event.pull_request.labels.*.name, 'ciflow/default')) }} + steps: + - name: noop + run: echo running ciflow_should_run calculate-docker-image: if: ${{ github.repository_owner == 'pytorch' }} runs-on: linux.2xlarge + needs: [ciflow_should_run] env: DOCKER_BUILDKIT: 1 timeout-minutes: 90 @@ -104,7 +112,7 @@ jobs: # building and testing in a single job since bazel runs only small subset of tests build-and-test: runs-on: linux.2xlarge - needs: [calculate-docker-image, ] + needs: [calculate-docker-image, ciflow_should_run] env: DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }} JOB_BASE_NAME: linux-xenial-py3.6-gcc7-bazel-test-build-and-test @@ -249,7 +257,7 @@ jobs: # doesn't create the best experience render_test_results: if: always() - needs: [build-and-test, ] + needs: [build-and-test, ciflow_should_run] runs-on: linux.2xlarge steps: - name: Log in to ECR diff --git a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml index e8f203361d73..52c2f1cd331f 100644 --- a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml @@ -4,7 +4,6 @@ name: periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7 on: - # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers pull_request: types: [unassigned] schedule: diff --git a/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml b/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml index e734396ab6d2..74d2f66bd457 100644 --- a/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml @@ -4,7 +4,6 @@ name: periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7 on: - # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers pull_request: types: [unassigned] schedule: