Add helpers for running tests and then putting them in a CSV (#92642)

Signed-off-by: Edward Z. Yang <ezyang@meta.com>
Pull Request resolved: https://github.com/pytorch/pytorch/pull/92642
Approved by: https://github.com/albanD

scripts/analysis/format_test_csv.py

@@ -0,0 +1,50 @@
+"""
+This script takes a pytest CSV file produced by pytest --csv foo.csv
+and summarizes it into a more minimal CSV that is good for uploading
+to Google Sheets.  We have been using this with dynamic shapes to
+understand how many tests fail when we turn on dynamic shapes.  If
+you have a test suite with a lot of skips or xfails, if force the
+tests to run anyway, this can help you understand what the actual
+errors things are failing with are.
+
+The resulting csv is written to stdout.  An easy way to get the csv
+onto your local file system is to send it to GitHub Gist:
+
+    $ python scripts/analysis/format_test_csv.py foo.csv | gh gist create -
+
+See also scripts/analysis/run_test_csv.sh
+"""
+
+import csv
+import subprocess
+import sys
+import argparse
+
+parser = argparse.ArgumentParser(description=__doc__,
+    formatter_class=argparse.RawDescriptionHelpFormatter)
+parser.add_argument("--log-url", type=str, default="", help="URL of raw logs")
+parser.add_argument("file", help="pytest CSV file to format")
+args = parser.parse_args()
+
+out = csv.writer(sys.stdout, dialect="excel")
+hash = subprocess.check_output(
+    "git rev-parse HEAD".split(" "), encoding="utf-8"
+).rstrip()
+
+out.writerow([hash, args.log_url, ""])
+
+with open(args.file, "r") as f:
+    reader = csv.DictReader(f)
+    for row in reader:
+        if row["status"] not in {"failed", "error"}:
+            continue
+        msg = row["message"].split("\n")[0]
+        msg.replace(
+            " - erroring out! It's likely that this is caused by data-dependent control flow or similar.",
+            "",
+        )
+        msg.replace("\t", " ")
+        # Feel free to edit this; the idea is to remove prefixes that are
+        # just gooping up the resulting spreadsheet outpu
+        name = row["name"].replace("test_make_fx_symbolic_exhaustive_", "")
+        out.writerow([name, msg, ""])

scripts/analysis/run_test_csv.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Typical usage:
+#
+#   scripts/analysis/run_test_csv.sh test/inductor/test_torchinductor.py
+
+set -x
+
+if getent hosts fwdproxy; then
+    export https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 no_proxy=.fbcdn.net,.facebook.com,.thefacebook.com,.tfbnw.net,.fb.com,.fburl.com,.facebook.net,.sb.fbsbx.com,localhost
+fi
+TEST_FILE="$1"
+TEST_ARGS="$*"  # includes file name
+shift
+pytest --csv "$TEST_FILE.csv" -v "$TEST_FILE" "$@" 2>&1 | tee "$TEST_FILE.log"
+LOG_URL="$(gh gist create -d "Test logs for $TEST_ARGS" "$TEST_FILE.log")"
+python "$(dirname "$BASH_SOURCE")"/format_test_csv.py --log-url "$LOG_URL" "$TEST_FILE.csv" | gh gist create -