fix

.github/workflows/check_failed_tests.yml

@@ -44,7 +44,7 @@ jobs:
     name: "Find commits for new failing tests"
     strategy:
       matrix:
-        run_idx: [1, 2, 3]
+        run_idx: [1]
     runs-on:
       group: aws-g5-4xlarge-cache
     container:
@@ -169,68 +169,76 @@ jobs:
           ls -la /transformers
           ls -la /transformers/new_failures_with_bad_commit_${{ inputs.job }}
 
+      # Currently, we only run with a single runner by using `run_idx: [1]`. We might try to run with multiple runners
+      # to further reduce the false positive caused by flaky tests, which requires further processing to merge reports.
+      - name: Merge files
+        shell: bash
+        working-directory: /transformers
+        if: ${{ env.process == 'true' }}
+        run: |
+          cp /transformers/new_failures_with_bad_commit_${{ inputs.job }}/new_failures_with_bad_commit_${{ inputs.job }}_1.json new_failures_with_bad_commit.json
 
-#      - name: Process report
-#        shell: bash
-#        working-directory: /transformers
-#        if: ${{ env.process == 'true' }}
-#        env:
-#          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
-#          TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}
-#          JOB_NAME: ${{ inputs.job }}
-#          REPORT_REPO_ID: ${{ inputs.report_repo_id }}
-#        run: |
-#          python3 utils/process_bad_commit_report.py
+      - name: Process report
+        shell: bash
+        working-directory: /transformers
+        if: ${{ env.process == 'true' }}
+        env:
+          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
+          TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}
+          JOB_NAME: ${{ inputs.job }}
+          REPORT_REPO_ID: ${{ inputs.report_repo_id }}
+        run: |
+          python3 utils/process_bad_commit_report.py
 
-#      - name: Process report
-#        shell: bash
-#        working-directory: /transformers
-#        if: ${{ env.process == 'true' }}
-#        env:
-#          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
-#          TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}
-#          JOB_NAME: ${{ inputs.job }}
-#          REPORT_REPO_ID: ${{ inputs.report_repo_id }}
-#        run: |
-#          {
-#            echo 'REPORT_TEXT<<EOF'
-#            python3 utils/process_bad_commit_report.py
-#            echo EOF
-#          } >> "$GITHUB_ENV"
-#
-#      - name: Prepare Slack report title
-#        working-directory: /transformers
-#        if: ${{ env.process == 'true' }}
-#        run: |
-#          pip install slack_sdk
-#          echo "title=$(python3 -c 'import sys; sys.path.append("utils"); from utils.notification_service import job_to_test_map; ci_event = "${{ inputs.ci_event }}"; job = "${{ inputs.job }}"; test_name = job_to_test_map[job]; title = f"New failed tests of {ci_event}" + ":" + f" {test_name}"; print(title)')" >> $GITHUB_ENV
-#
-#      - name: Send processed report
-#        if: ${{ env.process == 'true' && !endsWith(env.REPORT_TEXT, '{}') }}
-#        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001
-#        with:
-#          # Slack channel id, channel name, or user id to post message.
-#          # See also: https://api.slack.com/methods/chat.postMessage#channels
-#          channel-id: '#${{ inputs.slack_report_channel }}'
-#          # For posting a rich message using Block Kit
-#          payload: |
-#            {
-#              "blocks": [
-#                {
-#                  "type": "header",
-#                  "text": {
-#                    "type": "plain_text",
-#                    "text": "${{ env.title }}"
-#                  }
-#                },
-#                {
-#                  "type": "section",
-#                  "text": {
-#                    "type": "mrkdwn",
-#                    "text": "${{ env.REPORT_TEXT }}"
-#                  }
-#                }
-#              ]
-#            }
-#        env:
-#          SLACK_BOT_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
+      - name: Process report
+        shell: bash
+        working-directory: /transformers
+        if: ${{ env.process == 'true' }}
+        env:
+          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
+          TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}
+          JOB_NAME: ${{ inputs.job }}
+          REPORT_REPO_ID: ${{ inputs.report_repo_id }}
+        run: |
+          {
+            echo 'REPORT_TEXT<<EOF'
+            python3 utils/process_bad_commit_report.py
+            echo EOF
+          } >> "$GITHUB_ENV"
+
+      - name: Prepare Slack report title
+        working-directory: /transformers
+        if: ${{ env.process == 'true' }}
+        run: |
+          pip install slack_sdk
+          echo "title=$(python3 -c 'import sys; sys.path.append("utils"); from utils.notification_service import job_to_test_map; ci_event = "${{ inputs.ci_event }}"; job = "${{ inputs.job }}"; test_name = job_to_test_map[job]; title = f"New failed tests of {ci_event}" + ":" + f" {test_name}"; print(title)')" >> $GITHUB_ENV
+
+      - name: Send processed report
+        if: ${{ env.process == 'true' && !endsWith(env.REPORT_TEXT, '{}') }}
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001
+        with:
+          # Slack channel id, channel name, or user id to post message.
+          # See also: https://api.slack.com/methods/chat.postMessage#channels
+          channel-id: '#${{ inputs.slack_report_channel }}'
+          # For posting a rich message using Block Kit
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": "${{ env.title }}"
+                  }
+                },
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "${{ env.REPORT_TEXT }}"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

utils/check_bad_commit.py

@@ -15,12 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import argparse
-import git
 import json
 import os
 import re
 import subprocess
 
+import git
 import requests
 
 
@@ -58,13 +58,16 @@ if f"FAILED {target_test}" in result.stdout:
 elif result.returncode != 0:
     if "ERROR: file or directory not found: " in result.stderr:
         print("test file or directory not found in this commit")
-        exit(125)
+        # git bisect treats exit code 125 as `test not found`. But this causes it not be able to make the conclusion
+        # if a test is added between the `good commit` (exclusive) and `bad commit` (inclusive) (in git bisect terminology).
+        # So we return 0 here in order to allow the process being able to identify the first commit that fails the test.
+        exit(0)
     elif "ERROR: not found: " in result.stderr:
         print("test not found in this commit")
-        exit(125)
+        exit(0)
     else:
         print(f"pytest gets unknown error: {{result.stderr}}")
-        exit(-1)
+        exit(1)
 
 print(f"pytest runs successfully.")
 exit(0)
@@ -75,7 +78,7 @@ exit(0)
 
 
 def is_bad_commit(target_test, commit):
-    repo = git.Repo('.')  # or specify path to your repo
+    repo = git.Repo(".")  # or specify path to your repo
 
     # Save the current HEAD reference
     original_head = repo.head.commit
@@ -112,19 +115,25 @@ def find_bad_commit(target_test, start_commit, end_commit):
     # check if `end_commit` fails the test
     failed_before = is_bad_commit(target_test, end_commit)
     if failed_before:
-        return None
+        return (
+            None,
+            f"flaky: test passed in the previous run (commit: {end_commit}) but failed (on the same commit) during the check of the current run.",
+        )
 
     # if there is no new commit (e.g. 2 different CI runs on the same commit):
     #   - failed once on `start_commit` but passed on `end_commit`, which are the same commit --> flaky (or something change externally) --> don't report
     if start_commit == end_commit:
-        return None
+        return (
+            None,
+            f"flaky: test fails on the current CI run but passed in the previous run which is running on the same commit {end_commit}.",
+        )
 
     # Now, we are (almost) sure `target_test` is not failing at `end_commit`
     # check if `start_commit` fail the test
     failed_now = is_bad_commit(target_test, start_commit)
     if not failed_now:
         # failed on CI run, but not reproducible here --> don't report
-        return None
+        return None, f"flaky: test fails on the current CI run (commit: {start_commit}) but passes during the check."
 
     create_script(target_test=target_test)
 
@@ -149,7 +158,7 @@ git bisect run python3 target_script.py
     if "error: bisect run failed" in result.stderr:
         error_msg = f"Error when running git bisect:\nbash error: {result.stderr}\nbash output:\n{result.stdout}\nset `bad_commit` to `None`."
         print(error_msg)
-        return None
+        return None, "git bisect failed"
 
     pattern = r"(.+) is the first bad commit"
     commits = re.findall(pattern, result.stdout)
@@ -161,7 +170,7 @@ git bisect run python3 target_script.py
     print(f"Between `start_commit` {start_commit} and `end_commit` {end_commit}")
     print(f"bad_commit: {bad_commit}\n")
 
-    return bad_commit
+    return bad_commit, "git bisect find the bad commit."
 
 
 def get_commit_info(commit):
@@ -215,9 +224,11 @@ if __name__ == "__main__":
         raise ValueError("Exactly one argument `test` or `file` must be specified.")
 
     if args.test is not None:
-        commit = find_bad_commit(target_test=args.test, start_commit=args.start_commit, end_commit=args.end_commit)
+        commit, status = find_bad_commit(
+            target_test=args.test, start_commit=args.start_commit, end_commit=args.end_commit
+        )
         with open(args.output_file, "w", encoding="UTF-8") as fp:
-            fp.write(f"{args.test}\n{commit}")
+            fp.write(f"{args.test}\n{commit}\n{status}")
     elif os.path.isfile(args.file):
         with open(args.file, "r", encoding="UTF-8") as fp:
             reports = json.load(fp)
@@ -229,8 +240,10 @@ if __name__ == "__main__":
 
             failed_tests_with_bad_commits = []
             for test in failed_tests:
-                commit = find_bad_commit(target_test=test, start_commit=args.start_commit, end_commit=args.end_commit)
-                info = {"test": test, "commit": commit}
+                commit, status = find_bad_commit(
+                    target_test=test, start_commit=args.start_commit, end_commit=args.end_commit
+                )
+                info = {"test": test, "commit": commit, "status": status}
 
                 if commit in commit_info_cache:
                     commit_info = commit_info_cache[commit]

utils/process_bad_commit_report.py

@@ -26,6 +26,33 @@ if __name__ == "__main__":
 
     job_name = os.environ.get("JOB_NAME")
 
+    # Upload to Hub and get the url
+    # if it is not a scheduled run, upload the reports to a subfolder under `report_repo_folder`
+    report_repo_subfolder = ""
+    if os.getenv("GITHUB_EVENT_NAME") != "schedule":
+        report_repo_subfolder = f"{os.getenv('GITHUB_RUN_NUMBER')}-{os.getenv('GITHUB_RUN_ID')}"
+        report_repo_subfolder = f"runs/{report_repo_subfolder}"
+
+    workflow_run = get_last_daily_ci_run(
+        token=os.environ["ACCESS_REPO_INFO_TOKEN"], workflow_run_id=os.getenv("GITHUB_RUN_ID")
+    )
+    workflow_run_created_time = workflow_run["created_at"]
+
+    report_repo_folder = workflow_run_created_time.split("T")[0]
+
+    if report_repo_subfolder:
+        report_repo_folder = f"{report_repo_folder}/{report_repo_subfolder}"
+
+    report_repo_id = os.getenv("REPORT_REPO_ID")
+
+    commit_info = api.upload_file(
+        path_or_fileobj="new_failures_with_bad_commit.json",
+        path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/new_failures_with_bad_commit.json",
+        repo_id=report_repo_id,
+        repo_type="dataset",
+        token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
+    )
+
     with open("new_failures_with_bad_commit.json") as fp:
         data = json.load(fp)
 
@@ -88,25 +115,6 @@ if __name__ == "__main__":
             _data[model] = {k: v for k, v in model_result.items() if len(v) > 0}
         new_data_full[author] = {k: v for k, v in _data.items() if len(v) > 0}
 
-    # Upload to Hub and get the url
-    # if it is not a scheduled run, upload the reports to a subfolder under `report_repo_folder`
-    report_repo_subfolder = ""
-    if os.getenv("GITHUB_EVENT_NAME") != "schedule":
-        report_repo_subfolder = f"{os.getenv('GITHUB_RUN_NUMBER')}-{os.getenv('GITHUB_RUN_ID')}"
-        report_repo_subfolder = f"runs/{report_repo_subfolder}"
-
-    workflow_run = get_last_daily_ci_run(
-        token=os.environ["ACCESS_REPO_INFO_TOKEN"], workflow_run_id=os.getenv("GITHUB_RUN_ID")
-    )
-    workflow_run_created_time = workflow_run["created_at"]
-
-    report_repo_folder = workflow_run_created_time.split("T")[0]
-
-    if report_repo_subfolder:
-        report_repo_folder = f"{report_repo_folder}/{report_repo_subfolder}"
-
-    report_repo_id = os.getenv("REPORT_REPO_ID")
-
     with open("new_failures_with_bad_commit_grouped_by_authors.json", "w") as fp:
         json.dump(new_data_full, fp, ensure_ascii=False, indent=4)
     commit_info = api.upload_file(