.github: Add intial Windows CPU GHA workflow (#58199)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/58199

Signed-off-by: Eli Uriegas <eliuriegas@fb.com>

Test Plan: Imported from OSS

Reviewed By: malfet

Differential Revision: D28465272

Pulled By: seemethere

fbshipit-source-id: d221ad71d160088883896e018c58800dae85ff2c

.github/scripts/generate_ci_workflows.py

@@ -8,11 +8,44 @@ DOCKER_REGISTRY = "308535385114.dkr.ecr.us-east-1.amazonaws.com"
 
 GITHUB_DIR = Path(__file__).parent.parent
 
-CPU_TEST_RUNNER = "linux.2xlarge"
-CUDA_TEST_RUNNER = "linux.8xlarge.nvidia.gpu"
 
+class PyTorchWindowsWorkflow:
+    CPU_TEST_RUNNER = "windows.4xlarge"
+    CUDA_TEST_RUNNER = "windows.8xlarge.nvidia.gpu"
+
+    def __init__(
+            self,
+            build_environment: str,
+            on_pull_request: bool = False
+    ):
+        self.build_environment = build_environment
+        self.test_runner_type = self.CPU_TEST_RUNNER
+        self.on_pull_request = on_pull_request
+        if "cuda" in build_environment:
+            self.test_runner_type = self.CUDA_TEST_RUNNER
+
+    def generate_workflow_file(
+        self, workflow_template: jinja2.Template,
+    ) -> Path:
+        output_file_path = GITHUB_DIR.joinpath(
+            f"workflows/{self.build_environment}.yml"
+        )
+        with open(output_file_path, "w") as output_file:
+            output_file.writelines(["# @generated DO NOT EDIT MANUALLY\n"])
+            output_file.write(
+                workflow_template.render(
+                    build_environment=self.build_environment,
+                    test_runner_type=self.test_runner_type,
+                    on_pull_request=self.on_pull_request
+                )
+            )
+            output_file.write('\n')
+        return output_file_path
 
 class PyTorchLinuxWorkflow:
+    CPU_TEST_RUNNER = "linux.2xlarge"
+    CUDA_TEST_RUNNER = "linux.8xlarge.nvidia.gpu"
+
     def __init__(
             self,
             build_environment: str,
@@ -22,14 +55,14 @@ class PyTorchLinuxWorkflow:
     ):
         self.build_environment = build_environment
         self.docker_image_base = docker_image_base
-        self.test_runner_type = CPU_TEST_RUNNER
+        self.test_runner_type = self.CPU_TEST_RUNNER
         self.on_pull_request = on_pull_request
         self.enable_doc_jobs = enable_doc_jobs
         if "cuda" in build_environment:
-            self.test_runner_type = CUDA_TEST_RUNNER
+            self.test_runner_type = self.CUDA_TEST_RUNNER
 
     def generate_workflow_file(
-        self, workflow_template: jinja2.Template, jinja_env: jinja2.Environment
+        self, workflow_template: jinja2.Template,
     ) -> Path:
         output_file_path = GITHUB_DIR.joinpath(
             f"workflows/{self.build_environment}.yml"
@@ -49,7 +82,13 @@ class PyTorchLinuxWorkflow:
         return output_file_path
 
 
-WORKFLOWS = [
+WINDOWS_WORKFLOWS = [
+    PyTorchWindowsWorkflow(
+        build_environment="pytorch-win-vs2019-cpu-py3",
+    )
+]
+
+LINUX_WORKFLOWS = [
     PyTorchLinuxWorkflow(
         build_environment="pytorch-linux-xenial-py3.6-gcc5.4",
         docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
@@ -151,11 +190,10 @@ if __name__ == "__main__":
         variable_start_string="!{{",
         loader=jinja2.FileSystemLoader(str(GITHUB_DIR.joinpath("templates"))),
     )
-    workflow_template = jinja_env.get_template("linux_ci_workflow.yml.in")
+    template_and_workflows = [
-    for workflow in WORKFLOWS:
+        (jinja_env.get_template("linux_ci_workflow.yml.in"), LINUX_WORKFLOWS),
-        print(
+        (jinja_env.get_template("windows_ci_workflow.yml.in"), WINDOWS_WORKFLOWS)
-            workflow.generate_workflow_file(
+    ]
-                workflow_template=workflow_template,
+    for template, workflows in template_and_workflows:
-                jinja_env=jinja_env
+        for workflow in workflows:
-            )
+            print(workflow.generate_workflow_file(workflow_template=template))
-        )

.github/scripts/install_windows_sdk.ps1

@@ -0,0 +1,9 @@
+# Source install utils
+. $PSScriptRoot\windows_install_utils.ps1
+
+# Install Windows 10 SDK version 10.0.14393.795
+$sdkUrl = "https://go.microsoft.com/fwlink/p/?LinkId=838916"
+$sdkFileName = "sdksetup14393.exe"
+$argumentList = ("/q", "/norestart", "/ceip off", "/features OptionId.WindowsSoftwareDevelopmentKit")
+Write-Output "Installing $sdkUrl"
+Install-Binary -Url $sdkUrl -Name $sdkFileName -ArgumentList $argumentList

.github/scripts/windows_install_utils.ps1

@@ -0,0 +1,105 @@
+# Taken from actions/virtual-environments
+function Install-Binary
+{
+    <#
+    .SYNOPSIS
+        A helper function to install executables.
+    .DESCRIPTION
+        Download and install .exe or .msi binaries from specified URL.
+    .PARAMETER Url
+        The URL from which the binary will be downloaded. Required parameter.
+    .PARAMETER Name
+        The Name with which binary will be downloaded. Required parameter.
+    .PARAMETER ArgumentList
+        The list of arguments that will be passed to the installer. Required for .exe binaries.
+    .EXAMPLE
+        Install-Binary -Url "https://go.microsoft.com/fwlink/p/?linkid=2083338" -Name "winsdksetup.exe" -ArgumentList ("/features", "+", "/quiet")
+    #>
+
+    Param
+    (
+        [Parameter(Mandatory)]
+        [String] $Url,
+        [Parameter(Mandatory)]
+        [String] $Name,
+        [String[]] $ArgumentList
+    )
+
+    Write-Host "Downloading $Name..."
+    $filePath = Start-DownloadWithRetry -Url $Url -Name $Name
+
+    # MSI binaries should be installed via msiexec.exe
+    $fileExtension = ([System.IO.Path]::GetExtension($Name)).Replace(".", "")
+    if ($fileExtension -eq "msi")
+    {
+        $ArgumentList = ('/i', $filePath, '/QN', '/norestart')
+        $filePath = "msiexec.exe"
+    }
+
+    try
+    {
+        Write-Host "Starting Install $Name..."
+        $process = Start-Process -FilePath $filePath -ArgumentList $ArgumentList -Wait -PassThru
+
+        $exitCode = $process.ExitCode
+        if ($exitCode -eq 0 -or $exitCode -eq 3010)
+        {
+            Write-Host "Installation successful"
+        }
+        else
+        {
+            Write-Host "Non zero exit code returned by the installation process: $exitCode"
+            exit $exitCode
+        }
+    }
+    catch
+    {
+        Write-Host "Failed to install the $fileExtension ${Name}: $($_.Exception.Message)"
+        exit 1
+    }
+}
+
+function Start-DownloadWithRetry
+{
+    Param
+    (
+        [Parameter(Mandatory)]
+        [string] $Url,
+        [string] $Name,
+        [string] $DownloadPath = "${env:Temp}",
+        [int] $Retries = 20
+    )
+
+    if ([String]::IsNullOrEmpty($Name)) {
+        $Name = [IO.Path]::GetFileName($Url)
+    }
+
+    $filePath = Join-Path -Path $DownloadPath -ChildPath $Name
+
+    #Default retry logic for the package.
+    while ($Retries -gt 0)
+    {
+        try
+        {
+            Write-Host "Downloading package from: $Url to path $filePath ."
+            (New-Object System.Net.WebClient).DownloadFile($Url, $filePath)
+            break
+        }
+        catch
+        {
+            Write-Host "There is an error during package downloading:`n $_"
+            $Retries--
+
+            if ($Retries -eq 0)
+            {
+                Write-Host "File can't be downloaded. Please try later or check that file exists by url: $Url"
+                exit 1
+            }
+
+            Write-Host "Waiting 30 seconds before retrying. Retries left: $Retries"
+            Start-Sleep -Seconds 30
+        }
+    }
+
+    return $filePath
+}

.github/templates/linux_ci_workflow.yml.in

@@ -1,5 +1,5 @@
 # Template is at:    .github/templates/linux_ci_workflow.yml
-# Generation script: .github/scripts/generate_linux_ci_workflows.py
+# Generation script: .github/scripts/generate_ci_workflows.py
 name: Linux CI (!{{ build_environment }})
 
 on:

.github/templates/windows_ci_workflow.yml.in

@@ -0,0 +1,148 @@
+# Template is at:    .github/templates/windows_ci_workflow.yml
+# Generation script: .github/scripts/generate_ci_workflows.py
+name: Windows CI (!{{ build_environment }})
+
+on:
+{%- if on_pull_request %}
+  pull_request:
+{%- endif %}
+  push:
+    branches:
+      - master
+      - release/*
+  workflow_dispatch:
+
+env:
+  BUILD_ENVIRONMENT: !{{ build_environment }}
+  BUILD_WHEEL: 1
+  CUDA_VERSION: "cpu"
+  IN_CI: 1
+  JOB_BASE_NAME: test
+  PYTHON_VERSION: "3.6"
+  SCCACHE_BUCKET: "ossci-compiler-cache"
+  TORCH_CUDA_ARCH_LIST: "5.2;7.5"
+  VC_PRODUCT: "BuildTools"
+  VC_VERSION: ""
+  VC_YEAR: "2019"
+
+jobs:
+  build:
+    runs-on: "windows.4xlarge"
+    steps:
+      - name: Checkout PyTorch
+        uses: actions/checkout@v2
+      - name: Install Visual Studio 2019 toolchain
+        shell: powershell
+        run: |
+          choco install vswhere -y
+          .\.circleci\scripts\vs_install.ps1
+      - name: Install Windows 10 SDK
+        shell: powershell
+        run: |
+          .\.github\scripts\install_windows_sdk.ps1
+      - name: Build
+        shell: bash
+        run: |
+          .jenkins/pytorch/win-build.sh
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          retention-days: 30
+          if-no-files-found: error
+          name: ${{ env.BUILD_ENVIRONMENT }}
+          path: C:\w\build-results
+  test:
+    runs-on: !{{ test_runner_type }}
+    env:
+      JOB_BASE_NAME: !{{ build_environment }}-test
+    needs:
+      - build
+    steps:
+      - name: Checkout PyTorch
+        uses: actions/checkout@v2
+      - name: Install Visual Studio 2019 toolchain
+        shell: powershell
+        run: |
+          choco install vswhere -y
+          .\.circleci\scripts\vs_install.ps1
+      - name: Install Windows 10 SDK
+        shell: powershell
+        run: |
+          .\.github\scripts\install_windows_sdk.ps1
+      - uses: actions/download-artifact@v2
+        name: Download PyTorch Build Artifacts
+        with:
+          name: ${{ env.BUILD_ENVIRONMENT }}
+          path: C:\${{ github.run_id }}\build-results
+      - name: Check build-results folder
+        shell: powershell
+        run: |
+          tree /F C:\$Env:GITHUB_RUN_ID\build-results
+      # Needed for coverage in win-test.sh
+      - uses: actions/setup-python@v2
+        name: Setup Python3
+        with:
+          python-version: '3.x'
+      - name: Run test scripts
+        shell: bash
+        env:
+          PYTORCH_FINAL_PACKAGE_DIR: /c/${{ github.run_id }}/build-results/
+        run: |
+            .jenkins/pytorch/win-test.sh
+      - uses: actions/upload-artifact@v2
+        name: Store PyTorch Test Reports
+        if: always()
+        with:
+          name: test-reports
+          retention-days: 30
+          if-no-files-found: error
+          path:
+            test/**/*.xml
+  # TODO: Make this into a composite step
+  render_test_results:
+    if: always()
+    needs:
+      - test
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout PyTorch
+        uses: actions/checkout@v2
+        with:
+          # deep clone, to allow tools/print_test_stats.py to use Git commands
+          fetch-depth: 0
+      - uses: actions/download-artifact@v2
+        name: Download PyTorch Test Reports
+        with:
+          name: test-reports
+          path: test/test-reports
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        # boto3 version copied from .circleci/docker/common/install_conda.sh
+        run: |
+          pip install -r requirements.txt
+          pip install boto3==1.16.34 junitparser rich
+      - name: Output Test Results (Click Me)
+        run: |
+          python tools/render_junit.py test
+      - name: Parse ref
+        id: parse-ref
+        run: .github/scripts/parse_ref.py
+      - name: Display and upload test statistics (Click Me)
+        # temporary hack: set CIRCLE_* vars, until we update
+        # tools/print_test_stats.py to natively support GitHub Actions
+        env:
+          SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
+          CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
+          CIRCLE_JOB: !{{ build_environment }}
+          CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
+          CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
+          CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
+          CIRCLE_WORKFLOW_ID: ${{ github.run_id }} # dunno if this corresponds
+        run: |
+          export PYTHONPATH=$PWD
+          python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test

.github/workflows/lint.yml

@@ -142,7 +142,7 @@ jobs:
       - name: Checkout PyTorch
         uses: actions/checkout@v2
       - name: Regenerate workflows
-        run: .github/scripts/generate_linux_ci_workflows.py
+        run: .github/scripts/generate_ci_workflows.py
       - name: Assert that regenerating the workflows didn't change them
         run: .github/scripts/report_git_status.sh
 

.github/workflows/pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml

@@ -1,6 +1,6 @@
 # @generated DO NOT EDIT MANUALLY
 # Template is at:    .github/templates/linux_ci_workflow.yml
-# Generation script: .github/scripts/generate_linux_ci_workflows.py
+# Generation script: .github/scripts/generate_ci_workflows.py
 name: Linux CI (pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7)
 
 on:

.github/workflows/pytorch-linux-xenial-py3.6-gcc5.4.yml

@@ -1,6 +1,6 @@
 # @generated DO NOT EDIT MANUALLY
 # Template is at:    .github/templates/linux_ci_workflow.yml
-# Generation script: .github/scripts/generate_linux_ci_workflows.py
+# Generation script: .github/scripts/generate_ci_workflows.py
 name: Linux CI (pytorch-linux-xenial-py3.6-gcc5.4)
 
 on:

.github/workflows/pytorch-win-vs2019-cpu-py3.yml

@@ -0,0 +1,146 @@
+# @generated DO NOT EDIT MANUALLY
+# Template is at:    .github/templates/windows_ci_workflow.yml
+# Generation script: .github/scripts/generate_ci_workflows.py
+name: Windows CI (pytorch-win-vs2019-cpu-py3)
+
+on:
+  push:
+    branches:
+      - master
+      - release/*
+  workflow_dispatch:
+
+env:
+  BUILD_ENVIRONMENT: pytorch-win-vs2019-cpu-py3
+  BUILD_WHEEL: 1
+  CUDA_VERSION: "cpu"
+  IN_CI: 1
+  JOB_BASE_NAME: test
+  PYTHON_VERSION: "3.6"
+  SCCACHE_BUCKET: "ossci-compiler-cache"
+  TORCH_CUDA_ARCH_LIST: "5.2;7.5"
+  VC_PRODUCT: "BuildTools"
+  VC_VERSION: ""
+  VC_YEAR: "2019"
+
+jobs:
+  build:
+    runs-on: "windows.4xlarge"
+    steps:
+      - name: Checkout PyTorch
+        uses: actions/checkout@v2
+      - name: Install Visual Studio 2019 toolchain
+        shell: powershell
+        run: |
+          choco install vswhere -y
+          .\.circleci\scripts\vs_install.ps1
+      - name: Install Windows 10 SDK
+        shell: powershell
+        run: |
+          .\.github\scripts\install_windows_sdk.ps1
+      - name: Build
+        shell: bash
+        run: |
+          .jenkins/pytorch/win-build.sh
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          retention-days: 30
+          if-no-files-found: error
+          name: ${{ env.BUILD_ENVIRONMENT }}
+          path: C:\w\build-results
+  test:
+    runs-on: windows.4xlarge
+    env:
+      JOB_BASE_NAME: pytorch-win-vs2019-cpu-py3-test
+    needs:
+      - build
+    steps:
+      - name: Checkout PyTorch
+        uses: actions/checkout@v2
+      - name: Install Visual Studio 2019 toolchain
+        shell: powershell
+        run: |
+          choco install vswhere -y
+          .\.circleci\scripts\vs_install.ps1
+      - name: Install Windows 10 SDK
+        shell: powershell
+        run: |
+          .\.github\scripts\install_windows_sdk.ps1
+      - uses: actions/download-artifact@v2
+        name: Download PyTorch Build Artifacts
+        with:
+          name: ${{ env.BUILD_ENVIRONMENT }}
+          path: C:\${{ github.run_id }}\build-results
+      - name: Check build-results folder
+        shell: powershell
+        run: |
+          tree /F C:\$Env:GITHUB_RUN_ID\build-results
+      # Needed for coverage in win-test.sh
+      - uses: actions/setup-python@v2
+        name: Setup Python3
+        with:
+          python-version: '3.x'
+      - name: Run test scripts
+        shell: bash
+        env:
+          PYTORCH_FINAL_PACKAGE_DIR: /c/${{ github.run_id }}/build-results/
+        run: |
+            .jenkins/pytorch/win-test.sh
+      - uses: actions/upload-artifact@v2
+        name: Store PyTorch Test Reports
+        if: always()
+        with:
+          name: test-reports
+          retention-days: 30
+          if-no-files-found: error
+          path:
+            test/**/*.xml
+  # TODO: Make this into a composite step
+  render_test_results:
+    if: always()
+    needs:
+      - test
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout PyTorch
+        uses: actions/checkout@v2
+        with:
+          # deep clone, to allow tools/print_test_stats.py to use Git commands
+          fetch-depth: 0
+      - uses: actions/download-artifact@v2
+        name: Download PyTorch Test Reports
+        with:
+          name: test-reports
+          path: test/test-reports
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        # boto3 version copied from .circleci/docker/common/install_conda.sh
+        run: |
+          pip install -r requirements.txt
+          pip install boto3==1.16.34 junitparser rich
+      - name: Output Test Results (Click Me)
+        run: |
+          python tools/render_junit.py test
+      - name: Parse ref
+        id: parse-ref
+        run: .github/scripts/parse_ref.py
+      - name: Display and upload test statistics (Click Me)
+        # temporary hack: set CIRCLE_* vars, until we update
+        # tools/print_test_stats.py to natively support GitHub Actions
+        env:
+          SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
+          CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
+          CIRCLE_JOB: pytorch-win-vs2019-cpu-py3
+          CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
+          CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
+          CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
+          CIRCLE_WORKFLOW_ID: ${{ github.run_id }} # dunno if this corresponds
+        run: |
+          export PYTHONPATH=$PWD
+          python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test

.jenkins/pytorch/win-test.sh

@@ -23,7 +23,7 @@ export PROJECT_DIR_WIN
 export TEST_DIR="${PWD}/test"
 TEST_DIR_WIN=$(cygpath -w "${TEST_DIR}")
 export TEST_DIR_WIN
-export PYTORCH_FINAL_PACKAGE_DIR="/c/users/circleci/workspace/build-results"
+export PYTORCH_FINAL_PACKAGE_DIR="${PYTORCH_FINAL_PACKAGE_DIR:-/c/users/circleci/workspace/build-results}"
 PYTORCH_FINAL_PACKAGE_DIR_WIN=$(cygpath -w "${PYTORCH_FINAL_PACKAGE_DIR}")
 export PYTORCH_FINAL_PACKAGE_DIR_WIN
 export PYTORCH_TEST_SKIP_NOARCH=1
@@ -42,10 +42,8 @@ fi
 
 export SCRIPT_HELPERS_DIR=$SCRIPT_PARENT_DIR/win-test-helpers
 
-# Try to pull value from CIRCLE_PULL_REQUEST first then GITHUB_HEAD_REF second
+# Try to pull value from CIRCLE_PULL_REQUEST
-# CIRCLE_PULL_REQUEST comes from CircleCI
+IN_PULL_REQUEST=${CIRCLE_PULL_REQUEST:-}
-# GITHUB_HEAD_REF comes from Github Actions
-IN_PULL_REQUEST=${CIRCLE_PULL_REQUEST:-${GITHUB_HEAD_REF:-}}
 if [ -n "$IN_PULL_REQUEST" ]; then
   DETERMINE_FROM="${TMP_DIR}/determine_from"
   file_diff_from_base "$DETERMINE_FROM"

Makefile

@@ -28,7 +28,7 @@ shellcheck-gha:
 	tools/run_shellcheck.sh $(SHELLCHECK_GHA_GENERATED_FOLDER)
 
 generate-gha-workflows:
-	./.github/scripts/generate_linux_ci_workflows.py
+	.github/scripts/generate_ci_workflows.py
 	$(MAKE) shellcheck-gha
 
 setup_lint: