ci: aarch64: update wiki from ci (#3943)

.github/actions/update-wiki/action.yml

@@ -0,0 +1,60 @@
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+
+name: "Update Wiki"
+description: "Update wiki"
+inputs:
+  command:
+    description: "add-unit or add-perf"
+    required: true
+  title:
+    description: "title of section to add"
+    required: true
+  in-file:
+    description: "test results file to read from"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout oneDNN wiki
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      with:
+        repository: "${{ github.repository }}.wiki"
+        ref: master
+        path: .wiki
+
+    - name: Generate the Wiki Page
+      shell: bash
+      run: python oneDNN/.github/automation/aarch64/wiki_utils.py ${{ inputs.command }} --title ${{ inputs.title }} --in-file ${{ inputs.in-file }} --out-file .wiki/AArch64-status.md
+
+    - name: Configure the GitHub wiki identity
+      shell: bash
+      working-directory: .wiki
+      # "${GITHUB_ACTOR}@users.noreply.github.com" is associated to the actor without real emails
+      run: |
+        set -xe
+        git config user.name  "[bot] ${GITHUB_ACTOR}"
+        git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+
+    - name: Update the wiki page
+      shell: bash
+      working-directory: .wiki
+      run: |
+        set -xe
+        git add .
+        git commit --allow-empty -m "update the wiki page by bot (${GITHUB_WORKFLOW})"
+        git push origin master

.github/automation/aarch64/bisect_ctest.py

@@ -17,13 +17,12 @@
 # limitations under the License.
 # *******************************************************************************
 import argparse
-from collections import defaultdict
+import ctest_utils
 import os
 import pathlib
 import subprocess
 
 F_PATH = pathlib.Path(__file__).parent.resolve()
-CI_JSON_PATH = F_PATH / "ci.json"
 
 
 def print_to_github_message(message):
@@ -33,9 +32,6 @@ def print_to_github_message(message):
 
 
 def create_github_message(results_dict):
-    if not len(results_dict):
-        return "### :white_check_mark: All unit tests passed"
-
     message = (
         "### :x: Benchdnn Test Failures\n"
         "| Benchdnn Test | Bad Hash |\n"
@@ -48,23 +44,6 @@ def create_github_message(results_dict):
     return message
 
 
-def parse_ctest(args):
-    with open(args.file) as f:
-        r = f.readlines()
-
-    failed_cases = defaultdict(list)
-    for l in r:
-        if ":FAILED" in l:
-            l = l.split("__REPRO: ")[1]
-            op = l.split(" ")[0]
-            failed_cases[op].append(l.replace("\n", ""))
-
-    if args.unique:
-        return [x[0] for x in failed_cases.values()]
-
-    return [x for xs in failed_cases.values() for x in xs]  # Flatten list
-
-
 def main():
     args_parser = argparse.ArgumentParser(
         description="oneDNN log converter",
@@ -78,7 +57,7 @@ def main():
         help="whether to return only one test case per unique op",
     )
     args = args_parser.parse_args()
-    cases = parse_ctest(args)
+    cases = ctest_utils.failed_benchdnn_tests(args.file, args.unique)
 
     results_dict = {}
     for case in cases:
@@ -99,7 +78,8 @@ def main():
         print(f"First bad hash for {case}: {bad_hash}")
         results_dict[case] = bad_hash
 
-    print_to_github_message(create_github_message(results_dict))
+    if results_dict:
+        print_to_github_message(create_github_message(results_dict))
 
 
 if __name__ == "__main__":

.github/automation/aarch64/ctest_utils.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+from collections import defaultdict
+import xml.etree.ElementTree as ET
+
+
+def failed_benchdnn_tests(file, unique):
+    with open(file) as f:
+        r = f.readlines()
+
+    failed_cases = defaultdict(list)
+    for l in r:
+        if ":FAILED" in l:
+            l = l.split("__REPRO: ")[1]
+            op = l.split(" ")[0]
+            failed_cases[op].append(l.replace("\n", ""))
+
+    if unique:
+        return [x[0] for x in failed_cases.values()]
+
+    return [x for xs in failed_cases.values() for x in xs]  # Flatten list
+
+
+def get_failed_tests(file):
+    tree = ET.parse(file)
+    root = tree.getroot()
+    failed_tests = [
+        child.attrib["name"]
+        for child in root
+        if child.attrib["status"] == "fail"
+    ]
+    return failed_tests

.github/automation/aarch64/test.sh

@@ -32,5 +32,5 @@ if [[ "$ONEDNN_THREADING" == "SEQ" ]]; then
 fi
 
 set -x
-ctest --no-tests=error --output-on-failure -E $("${SCRIPT_DIR}"/skipped-tests.sh)
+ctest --no-tests=error --output-on-failure -E $("${SCRIPT_DIR}"/skipped-tests.sh) --output-junit $1
 set +x

.github/automation/aarch64/wiki_utils.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+import argparse
+import ctest_utils
+import os
+
+PAGE_TITLE = "AArch64 Testing Status"
+
+
+class MdConverter:
+    # Recursive function to convert dict to md sections
+    def __unpack_md_dict(self, data, out_str="", level=1):
+        # base case
+        if not isinstance(data, dict):
+            return out_str + data + "\n"
+
+        for k, v in data.items():
+            out_str += "#" * level + " " + k + "\n"
+            out_str = self.__unpack_md_dict(v, out_str, level + 1)
+        return out_str
+
+    # Recursive function to convert md to dict
+    def __unpack_md(self, data, level=1):
+        prev = ""
+        inner = []
+        out = {}
+        for l in data:
+            if l[:level] == "#" * level and l[level] != "#":
+                if prev:
+                    out[prev[level + 1 :]] = self.__unpack_md(inner, level + 1)
+                    inner = []
+                prev = l.strip()
+            else:
+                inner.append(l)
+        if prev:
+            out[prev[level + 1 :]] = self.__unpack_md(inner, level + 1)
+        elif inner:
+            # base case
+            return "".join(inner)
+        return out
+
+    def dict2md(self, in_dict, out_file):
+        output = self.__unpack_md_dict(in_dict)
+        with open(out_file, "w") as f:
+            f.write(output)
+
+    def md2dict(self, in_file):
+        if not os.path.isfile(in_file):
+            return {}
+
+        with open(in_file) as f:
+            r = f.readlines()
+
+        out = self.__unpack_md(r)
+
+        return out
+
+
+def parse(file, title, subtitle, body):
+    """
+    Add a new section/subsection or an existing section/subsection
+    without overwriting existing section/subsections
+    """
+    converter = MdConverter()
+    d = converter.md2dict(file)
+    if PAGE_TITLE not in d:
+        d[PAGE_TITLE] = {}
+    k0 = {}
+    if title in d[PAGE_TITLE]:
+        k0 = d[PAGE_TITLE][title]
+    k0[subtitle] = body
+    d[PAGE_TITLE][title] = k0
+    converter.dict2md(d, file)
+
+
+def parse_unit(args):
+    failed_tests = ctest_utils.get_failed_tests(args.in_file)
+
+    body = ""
+    if failed_tests:
+        body = "| :x: | Failed Test |\n" "| :-----------: | :------: |\n"
+        for test in failed_tests:
+            body += f"| :x: | {test} |\n"
+    else:
+        body = ":white_check_mark: unit tests passed\n"
+
+    parse(args.out_file, "Unit test results", args.title, body)
+
+
+def parse_perf(args):
+    with open(args.in_file) as f:
+        body = f.read()
+    parse(args.out_file, "Performance test results", args.title, body)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="oneDNN wiki update tools",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    subparsers = parser.add_subparsers()
+
+    unit_parser = subparsers.add_parser("add-unit", help="add unit test result")
+    unit_parser.add_argument(
+        "--title", required=True, help="title of unit test run"
+    )
+    # xml is the only machine-readable output format from ctest
+    unit_parser.add_argument(
+        "--in-file", required=True, help="xml file storing test results"
+    )
+    # md format required for github wiki
+    unit_parser.add_argument(
+        "--out-file", required=True, help="md file to write to"
+    )
+    unit_parser.set_defaults(func=parse_unit)
+
+    perf_parser = subparsers.add_parser(
+        "add-perf", help="add performance test result"
+    )
+    perf_parser.add_argument(
+        "--title", required=True, help="title of perf test run"
+    )
+    perf_parser.add_argument(
+        "--in-file",
+        required=True,
+        help="md file storing performance test results",
+    )
+    perf_parser.add_argument(
+        "--out-file", required=True, help="md file to write to"
+    )
+    perf_parser.set_defaults(func=parse_perf)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

.github/automation/performance/benchdnn_comparison.py

@@ -21,15 +21,22 @@
 Compare two benchdnn runs.
 
 Usage:
-    python benchdnn_comparison.py baseline.csv new.csv --check
+    python benchdnn_comparison.py baseline.txt new.txt --out-file out.md
 """
 
-import os
-from collections import defaultdict
-from scipy.stats import ttest_ind
-import warnings
-import statistics
 import argparse
+from collections import defaultdict
+import git
+import json
+import os
+import pathlib
+from scipy.stats import ttest_ind
+import statistics
+import warnings
+
+
+F_PATH = pathlib.Path(__file__).parent.resolve()
+CI_JSON_PATH = F_PATH / "../aarch64/ci.json"
 
 
 def print_to_github_out(message):
@@ -38,7 +45,7 @@ def print_to_github_out(message):
             print(message.replace("\n", "%0A"), file=f)
 
 
-def compare_two_benchdnn(file1, file2, check=False):
+def compare_two_benchdnn(file1, file2, out_file=None):
     """
     Compare two benchdnn output files
     """
@@ -76,10 +83,15 @@ def compare_two_benchdnn(file1, file2, check=False):
         r2_ctime[key].append(float(ctime))
 
     exec_failures, ctime_failures = [], []
-    if not check:
+    if out_file is not None:
-        print(
+        with open(CI_JSON_PATH) as f:
-            "primitive,exec_base,exec_new,ctime_base,ctime_new,exec_diff,ctime_diff"
+            ci_json = json.load(f)
-        )
+
+        repo = git.Repo(F_PATH / "../../..", search_parent_directories=True)
+        head_sha = repo.git.rev_parse(repo.head.object.hexsha, short=4)
+        headers = f"| problem | oneDNN ({ci_json['dependencies']['onednn-base']}) time(ms) | oneDNN ({head_sha}) time(ms) | speedup (>1 is faster) |\n"
+        with open(out_file, "w") as f:
+            f.write(headers + "| :---: | :---: | :---: | :---:|\n")
 
     for prb in r1_exec:
         if prb not in r2_exec:
@@ -128,38 +140,57 @@ def compare_two_benchdnn(file1, file2, check=False):
                 f"(p={ctime_ttest.pvalue:.3g})"
             )
 
-        if not check:
+        if (
-            print(
+            out_file is not None
-                f"{prb},{r1_med_exec:.3g},{r2_med_exec:.3g},"
+            and abs((r2_med_exec - r1_med_exec) / r1_med_exec) >= 0.05
-                f"{r1_med_ctime:.3g},{r2_med_ctime:.3g},"
+        ):
-                f"{(r2_med_exec - r1_med_exec)/r1_med_exec:.1%},"
+            prb_params = [x.replace("--", "") for x in prb.split(" ")]
-                f"{(r2_med_ctime - r1_med_ctime)/r1_med_ctime:.1%}"
+            prb_params = [prb_params[1]] + [
+                x for x in prb_params if ("dt=" in x) or ("alg=" in x)
+            ]  # filter out the problem and data types
+            prb_str = (
+                "<details>"
+                + f"<summary>{' '.join(prb_params)}</summary>"
+                + prb
+                + "</details>"
             )
-
+            colour = "green" if r1_med_exec >= r2_med_exec * 1.05 else "red"
-    if check:
+            speedup_str = (
-        print_to_github_out(f"pass={not exec_failures}")
+                "$${\\color{"
-
+                + colour
-        message = ""
+                + "}"
-        if ctime_failures:
+                + f"{(r1_med_exec)/r2_med_exec:.3g}\\times"
-            message += (
+                + "}$$"
-                "\n----The following ctime regression tests failed:----\n"
-                + "\n".join(ctime_failures)
-                + "\n"
             )
+            with open(out_file, "a") as f:
+                f.write(
+                    f"|{prb_str}|{r1_med_exec:.3g}|{r2_med_exec:.3g}|{speedup_str}|\n"
+                )
+
+    print_to_github_out(f"pass={not exec_failures}")
+
+    message = ""
+    if ctime_failures:
+        message += (
+            "\n----The following ctime regression tests failed:----\n"
+            + "\n".join(ctime_failures)
+            + "\n"
+        )
+
+    if not exec_failures:
+        print_to_github_out(f"message={message}")
+        print(message)
+        print("Execution Time regression tests passed")
+    else:
+        message += (
+            "\n----The following exec time regression tests failed:----\n"
+            + "\n".join(exec_failures)
+            + "\n"
+        )
+        print_to_github_out(f"message={message}")
+        print(message)
+        raise Exception("Some regression tests failed")
 
-        if not exec_failures:
-            print_to_github_out(f"message={message}")
-            print(message)
-            print("Execution Time regression tests passed")
-        else:
-            message += (
-                "\n----The following exec time regression tests failed:----\n"
-                + "\n".join(exec_failures)
-                + "\n"
-            )
-            print_to_github_out(f"message={message}")
-            print(message)
-            raise Exception("Some regression tests failed")
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -168,8 +199,8 @@ if __name__ == "__main__":
     parser.add_argument("file1", help="Path to baseline result file")
     parser.add_argument("file2", help="Path to new result file")
     parser.add_argument(
-        "--check", action="store_true", help="Enable regression checks"
+        "--out-file", help="md file to output performance results to"
     )
     args = parser.parse_args()
 
-    compare_two_benchdnn(args.file1, args.file2, check=args.check)
+    compare_two_benchdnn(args.file1, args.file2, args.out_file)

.github/workflows/ci-aarch64.yml

@@ -172,7 +172,7 @@ jobs:
           ONEDNN_ACTION: build
 
       - name: Run oneDNN tests
-        run: ${{ github.workspace }}/oneDNN/.github/automation/aarch64/test.sh
+        run: ${{ github.workspace }}/oneDNN/.github/automation/aarch64/test.sh ${{ github.workspace }}/test_results.xml
         working-directory: ${{ github.workspace }}/oneDNN/build
         env:
           CTEST_PARALLEL_LEVEL: 6
@@ -223,9 +223,9 @@ jobs:
         continue-on-error: true
         run: |
           echo "4 threads:"
-          python ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base_4.txt new_4.txt --check
+          python ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base_4.txt new_4.txt
           echo "16 threads:"
-          python ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base_16.txt new_16.txt --check
+          python ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base_16.txt new_16.txt
 
       - name: Check performance test failure
         if: ${{ steps.performance-test.outputs.pass != 'True' && github.event_name == 'pull_request' && matrix.config.build == 'Release' && matrix.config.name != 'cb100' && matrix.config.name != 'MacOS' }}

.github/workflows/nightly-aarch64.yml

@@ -32,7 +32,7 @@ concurrency:
   cancel-in-progress: true
 
 # Declare default permissions as read only.
-permissions: read-all
+permissions: write-all
 
 jobs:
   build-acl-cache:
@@ -134,7 +134,7 @@ jobs:
       - name: Run oneDNN tests
         run: |
           set -o pipefail
-          ${{ github.workspace }}/oneDNN/.github/automation/aarch64/test.sh | tee ${{ github.workspace }}/oneDNN/test_results.txt
+          ${{ github.workspace }}/oneDNN/.github/automation/aarch64/test.sh ${{ github.workspace }}/test_results.xml
         working-directory: ${{ github.workspace }}/oneDNN/build
         env:
           BUILD_TOOLSET: ${{ matrix.config.toolset }}
@@ -173,8 +173,15 @@ jobs:
       - name: Run git bisect
         if: failure()
         working-directory: ${{ github.workspace }}/oneDNN
-        run: python .github/automation/aarch64/bisect_ctest.py --unique ${{ steps.get-stable.outputs.stable-hash }} test_results.txt
+        run: python .github/automation/aarch64/bisect_ctest.py --unique ${{ steps.get-stable.outputs.stable-hash }} ${{ github.workspace }}/test_results.xml
 
+      - name: Update wiki
+        if: ${{ (success() || failure()) && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ryo-not-rio/wiki') }}
+        uses: ./oneDNN/.github/actions/update-wiki
+        with:
+          command: add-unit
+          title: ${{ matrix.config.name }}
+          in-file: ${{ github.workspace }}/test_results.xml
 
   #* This job adds a check named "Nightly AArch64" that represents overall
   #* workflow status and can be used in branch rulesets

.github/workflows/performance-aarch64.yml

@@ -53,7 +53,7 @@ concurrency:
   cancel-in-progress: true
 
 # Declare default permissions as read only.
-permissions: read-all
+permissions: write-all
 
 jobs:
   build-acl-base:
@@ -115,7 +115,7 @@ jobs:
 
       - name: Install scipy
         if: ${{ matrix.config.build == 'Release' }}
-        run: pip install scipy statistics
+        run: pip install scipy statistics GitPython
 
       - name: Clone base ACL
         run: ${{ github.workspace }}/oneDNN/.github/automation/aarch64/build_acl.sh
@@ -223,7 +223,15 @@ jobs:
         shell: bash
         run: |
           OMP_NUM_THREADS=${{ inputs.num_threads || 16 }} bash ${{ github.workspace }}/oneDNN/.github/automation/performance/bench_nightly_performance.sh ${{ github.workspace }}/oneDNN_base/build/tests/benchdnn/benchdnn ${{ github.workspace }}/oneDNN_new/build/tests/benchdnn/benchdnn base.txt new.txt
-          python ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base.txt new.txt --check
+          python ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base.txt new.txt --out-file perf_table.md
+
+      - name: Update wiki
+        if: ${{ (success() || failure()) && inputs.benchdnn_command == '' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ryo-not-rio/wiki') }}
+        uses: ./oneDNN/.github/actions/update-wiki
+        with:
+          command: add-perf
+          title: ${{ matrix.config.name }}
+          in-file: perf_table.md
 
       - name: Run custom performance tests
         if: ${{ inputs.benchdnn_command != '' }}
@@ -236,9 +244,8 @@ jobs:
             base.txt new.txt ${{ inputs.benchdnn_command }}
 
       - name: Print speed comparisons
-        if: ${{ inputs.benchdnn_command != '' }}
+        if: ${{ (success() || failure()) && inputs.benchdnn_command != '' }}
-        run: |
+        run: python3 ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base.txt new.txt --out-file perf_results.md; cat perf_results.md >> $GITHUB_STEP_SUMMARY
-          python3 ${{ github.workspace }}/oneDNN/.github/automation/performance/benchdnn_comparison.py base.txt new.txt
 
   #* This job adds a check named "Nightly Performance AArch64" that represents overall
   #* workflow status and can be used in branch rulesets