more path stuff

test/run_test.py

@@ -73,7 +73,10 @@ from tools.testing.test_selections import (
     ShardedTest,
     THRESHOLD,
 )
-from tools.testing.upload_artifacts import zip_and_upload_artifacts
+from tools.testing.upload_artifacts import (
+    parse_xml_and_upload_json,
+    zip_and_upload_artifacts,
+)
 
 
 # Make sure to remove REPO_ROOT after import is done
@@ -1881,6 +1884,7 @@ def run_tests(
 
     def handle_complete(failure: Optional[TestFailure]):
         failed = failure is not None
+        parse_xml_and_upload_json()
         if IS_CI and options.upload_artifacts_while_running:
             zip_and_upload_artifacts(failed)
         if not failed:

tools/stats/upload_test_stats.py

@@ -24,12 +24,14 @@ def parse_xml_report(
     report: Path,
     workflow_id: int,
     workflow_run_attempt: int,
+    job_id: int | None = None,
 ) -> list[dict[str, Any]]:
     """Convert a test report xml file into a JSON-serializable list of test cases."""
     print(f"Parsing {tag}s for test report: {report}")
 
-    job_id = get_job_id(report)
-    print(f"Found job id: {job_id}")
+    if job_id is None:
+        job_id = get_job_id(report)
+        print(f"Found job id: {job_id}")
 
     test_cases: list[dict[str, Any]] = []
 

tools/testing/upload_artifacts.py

@@ -1,11 +1,14 @@
 import glob
 import gzip
+import json
 import os
 import time
 import zipfile
 from functools import lru_cache
 from pathlib import Path
-from typing import Any
+from typing import Any, Optional
+
+from tools.stats.upload_test_stats import parse_xml_report
 
 
 REPO_ROOT = Path(__file__).resolve().parent.parent.parent
@@ -140,3 +143,59 @@ def trigger_upload_test_stats_intermediate_workflow() -> None:
         },
     )
     print(x.text)
+
+
+def parse_xml_and_upload_json() -> None:
+    """
+    Parse xml test reports that do not yet have a corresponding json report
+    uploaded to s3, and upload the json reports to s3.
+    """
+    json_files = [
+        Path(f).with_suffix("")
+        for f in glob.glob(f"{REPO_ROOT}/test/test-reports/**/*.json", recursive=True)
+    ]
+
+    try:
+        job_id: Optional[int] = int(os.environ.get("JOB_ID", 0))
+        if job_id == 0:
+            job_id = None
+    except (ValueError, TypeError):
+        job_id = None
+
+    for xml_file in glob.glob(
+        f"{REPO_ROOT}/test/test-reports/**/*.xml", recursive=True
+    ):
+        xml_path = Path(xml_file)
+        json_file = xml_path.with_suffix(".json")
+        if json_file in json_files:
+            # It has already been parsed and uploaded
+            continue
+
+        test_cases = parse_xml_report(
+            "testcase",
+            xml_path,
+            int(os.environ.get("GITHUB_RUN_ID", "0")),
+            int(os.environ.get("GITHUB_RUN_ATTEMPT", "0")),
+            job_id,
+        )
+        line_by_line_jsons = "\n".join([json.dumps(tc) for tc in test_cases])
+
+        gzipped = gzip.compress(line_by_line_jsons.encode("utf-8"))
+        s3_key = (
+            json_file.relative_to(REPO_ROOT / "test/test-reports")
+            .as_posix()
+            .replace("/", "_")
+        )
+        get_s3_resource().put_object(
+            Body=gzipped,
+            Bucket="gha-artifacts",
+            Key=f"test_jsons_while_running/{os.environ.get('GITHUB_RUN_ID')}/{job_id}/{s3_key}",
+            ContentType="application/json",
+            ContentEncoding="gzip",
+        )
+
+        # We don't need to save the json file locally, but doing so lets us
+        # track which ones have been uploaded already. We could probably also
+        # check S3
+        with open(json_file, "w") as f:
+            f.write(line_by_line_jsons)