[CI] add new test config label ci-test-showlocals to control test log verbosity (#131981)

Add a new label `ci-test-showlocals` and add it to test config filter.
If the PR is labeled with `ci-test-showlocals` or "ci-test-showlocals"
present in the PR comment, the test config filter will set a environment
variable `TEST_SHOWLOCALS`. Then `pytest` will show local variables on
failures for better debugging.

Pull Request resolved: https://github.com/pytorch/pytorch/pull/131981
Approved by: https://github.com/malfet
ghstack dependencies: #131151

.github/actions/filter-test-configs/action.yml

@@ -41,6 +41,9 @@ outputs:
   ci-verbose-test-logs:
     description: True if ci-verbose-test-logs label was on PR or [ci-verbose-test-logs] in PR body.
     value: ${{ steps.filter.outputs.ci-verbose-test-logs }}
+  ci-test-showlocals:
+    description: True if ci-test-showlocals label was on PR or [ci-test-showlocals] in PR body.
+    value: ${{ steps.filter.outputs.ci-test-showlocals }}
   ci-no-test-timeout:
     description: True if ci-no-test-timeout label was on PR or [ci-no-test-timeout] in PR body.
     value: ${{ steps.filter.outputs.ci-no-test-timeout }}

.github/actions/linux-test/action.yml

@@ -167,6 +167,7 @@ runs:
         REENABLED_ISSUES: ${{ steps.keep-going.outputs.reenabled-issues }}
         CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }}
         VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }}
+        TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }}
         NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }}
         NO_TD: ${{ steps.keep-going.outputs.ci-no-td }}
         TD_DISTRIBUTED: ${{ steps.keep-going.outputs.ci-td-distributed }}

.github/scripts/filter_test_configs.py

@@ -505,6 +505,9 @@ def perform_misc_tasks(
         "ci-verbose-test-logs",
         check_for_setting(labels, pr_body, "ci-verbose-test-logs"),
     )
+    set_output(
+        "ci-test-showlocals", check_for_setting(labels, pr_body, "ci-test-showlocals")
+    )
     set_output(
         "ci-no-test-timeout", check_for_setting(labels, pr_body, "ci-no-test-timeout")
     )

.github/scripts/test_filter_test_configs.py

@@ -683,6 +683,7 @@ class TestConfigFilter(TestCase):
         def _gen_expected_string(
             keep_going: bool = False,
             ci_verbose_test_logs: bool = False,
+            ci_test_showlocals: bool = False,
             ci_no_test_timeout: bool = False,
             ci_no_td: bool = False,
             ci_td_distributed: bool = False,
@@ -692,6 +693,7 @@ class TestConfigFilter(TestCase):
             return (
                 f"keep-going={keep_going}\n"
                 f"ci-verbose-test-logs={ci_verbose_test_logs}\n"
+                f"ci-test-showlocals={ci_test_showlocals}\n"
                 f"ci-no-test-timeout={ci_no_test_timeout}\n"
                 f"ci-no-td={ci_no_td}\n"
                 f"ci-td-distributed={ci_td_distributed}\n"
@@ -733,6 +735,21 @@ class TestConfigFilter(TestCase):
                 ),
                 "description": "No pipe logs label and no test timeout in PR body",
             },
+            {
+                "labels": {"ci-test-showlocals"},
+                "test_matrix": '{include: [{config: "default"}]}',
+                "job_name": "A job name",
+                "expected": _gen_expected_string(ci_test_showlocals=True),
+                "description": "Has ci-test-showlocals",
+            },
+            {
+                "labels": {},
+                "test_matrix": '{include: [{config: "default"}]}',
+                "job_name": "A job name",
+                "pr_body": "[ci-test-showlocals]",
+                "expected": _gen_expected_string(ci_test_showlocals=True),
+                "description": "ci-test-showlocals in body",
+            },
             {
                 "labels": {"ci-no-test-timeout"},
                 "test_matrix": '{include: [{config: "default"}]}',

.github/workflows/_linux-test.yml

@@ -198,6 +198,7 @@ jobs:
           REENABLED_ISSUES: ${{ steps.keep-going.outputs.reenabled-issues }}
           CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }}
           VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }}
+          TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }}
           NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }}
           NO_TD: ${{ steps.keep-going.outputs.ci-no-td }}
           TD_DISTRIBUTED: ${{ steps.keep-going.outputs.ci-td-distributed }}
@@ -251,6 +252,7 @@ jobs:
             -e REENABLED_ISSUES \
             -e CONTINUE_THROUGH_ERROR \
             -e VERBOSE_TEST_LOGS \
+            -e TEST_SHOWLOCALS \
             -e NO_TEST_TIMEOUT \
             -e NO_TD \
             -e TD_DISTRIBUTED \

.github/workflows/_mac-test-mps.yml

@@ -35,6 +35,7 @@ jobs:
       is-test-matrix-empty: ${{ steps.filter.outputs.is-test-matrix-empty }}
       keep-going: ${{ steps.filter.outputs.keep-going }}
       ci-verbose-test-logs: ${{ steps.filter.outputs.ci-verbose-test-logs }}
+      ci-test-showlocals: ${{ steps.filter.outputs.ci-test-showlocals }}
       ci-no-test-timeout: ${{ steps.filter.outputs.ci-no-test-timeout }}
       ci-no-td: ${{ steps.filter.outputs.ci-no-td }}
       reenabled-issues: ${{ steps.filter.outputs.reenabled-issues }}
@@ -98,6 +99,7 @@ jobs:
           PR_BODY: ${{ github.event.pull_request.body }}
           CONTINUE_THROUGH_ERROR: ${{ needs.filter.outputs.keep-going }}
           VERBOSE_TEST_LOGS: ${{ needs.filter.outputs.ci-verbose-test-logs }}
+          TEST_SHOWLOCALS: ${{ needs.filter.outputs.ci-test-showlocals }}
           NO_TEST_TIMEOUT: ${{ needs.filter.outputs.ci-no-test-timeout }}
           NO_TD: ${{ needs.filter.outputs.ci-no-td }}
           PIP_REQUIREMENTS_FILE: .github/requirements/pip-requirements-${{ runner.os }}.txt

.github/workflows/_mac-test.yml

@@ -144,6 +144,7 @@ jobs:
           PYTORCH_TEST_RERUN_DISABLED_TESTS: ${{ matrix.rerun_disabled_tests && '1' || '0' }}
           CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }}
           VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }}
+          TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }}
           NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }}
           NO_TD: ${{ steps.keep-going.outputs.ci-no-td }}
           PIP_REQUIREMENTS_FILE: .github/requirements/pip-requirements-${{ runner.os }}.txt

.github/workflows/_rocm-test.yml

@@ -154,6 +154,7 @@ jobs:
           SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
           CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }}
           VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }}
+          TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }}
           NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }}
           NO_TD: ${{ steps.keep-going.outputs.ci-no-td }}
           TEST_CONFIG: ${{ matrix.config }}
@@ -205,6 +206,7 @@ jobs:
             -e REENABLED_ISSUES \
             -e CONTINUE_THROUGH_ERROR \
             -e VERBOSE_TEST_LOGS \
+            -e TEST_SHOWLOCALS \
             -e NO_TEST_TIMEOUT \
             -e NO_TD \
             -e MAX_JOBS="$(nproc --ignore=2)" \

.github/workflows/_win-test.yml

@@ -157,6 +157,7 @@ jobs:
           PYTHON_VERSION: 3.8
           CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }}
           VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }}
+          TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }}
           NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }}
           NO_TD: ${{ steps.keep-going.outputs.ci-no-td }}
           VC_PRODUCT: "BuildTools"

.github/workflows/_xpu-test.yml

@@ -143,6 +143,7 @@ jobs:
           PYTORCH_RETRY_TEST_CASES: 1
           PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1
           CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }}
+          TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }}
           VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }}
           NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }}
           NO_TD: ${{ steps.keep-going.outputs.ci-no-td }}
@@ -189,6 +190,7 @@ jobs:
             -e PYTORCH_OVERRIDE_FLAKY_SIGNAL \
             -e CONTINUE_THROUGH_ERROR \
             -e VERBOSE_TEST_LOGS \
+            -e TEST_SHOWLOCALS \
             -e NO_TEST_TIMEOUT \
             -e NO_TD \
             -e MAX_JOBS="$(nproc --ignore=2)" \

test/run_test.py

@@ -1129,14 +1129,14 @@ def parse_args():
         parser.add_argument(
             "--showlocals",
             action=argparse.BooleanOptionalAction,
-            default=False,
+            default=strtobool(os.environ.get("TEST_SHOWLOCALS", "False")),
             help="Show local variables in tracebacks (default: True)",
         )
     else:
         parser.add_argument(
             "--showlocals",
             action="store_true",
-            default=False,
+            default=strtobool(os.environ.get("TEST_SHOWLOCALS", "False")),
             help="Show local variables in tracebacks (default: True)",
         )
         parser.add_argument("--no-showlocals", dest="showlocals", action="store_false")

torch/testing/_internal/common_utils.py

@@ -4143,8 +4143,8 @@ class TestCase(expecttest.TestCase):
         env["PYTORCH_API_USAGE_STDERR"] = "1"
         # remove CI flag since this is a wrapped test process.
         # CI flag should be set in the parent process only.
-        if "CI" in env.keys():
-            del env["CI"]
+        env.pop("CI", None)
+        env.pop("TEST_SHOWLOCALS", None)
         (stdout, stderr) = TestCase.run_process_no_exception(code, env=env)
         return stderr.decode('ascii')