gitlab CI: parse through the YAML file to list failed keymap compilations

libxkbcommon (commit 1cae25005211) now provides the output of the layout
tester format in YAML, with successful compilations on stdout and failed ones
on stderr. This makes it easy to collect the results, extract and print the
failures with yq but also parse the yaml file and leave a JUnit XML in place
that will then show up as result on a MR page.

Signed-off-by: Peter Hutterer <peter.hutterer@who-t.net>

2 files changed, 73 insertions, 9 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1edc3e94..3dc25db7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,7 +23,7 @@ variables:
 
   # Changing the tag will rebuild the container images. The value is just a
   # string, but we use the date for human benefits.
-  FDO_DISTRIBUTION_TAG: '2021-01-08.0'
+  FDO_DISTRIBUTION_TAG: '2021-04-16.0'
 
 
 stages:
@@ -43,7 +43,7 @@ container-prep:
     # minimal set of packages required to build xkeyboard-config.
     BASE_PACKAGES: 'xorg-util-macros autoconf automake make gettext pkg-config gcc grep m4 python meson ninja git'
     # extra packages we need for various tests
-    EXTRA_PACKAGES: 'tree libxml2 bison xorg-xkbcomp python-pytest python-libevdev'
+    EXTRA_PACKAGES: 'tree libxml2 bison xorg-xkbcomp python-pytest python-libevdev python-yaml yq'
     FDO_DISTRIBUTION_PACKAGES: $BASE_PACKAGES $EXTRA_PACKAGES
 
 
@@ -229,21 +229,26 @@ layout_tests:
     - meson builddir -Denable-wayland=false -Denable-x11=false -Denable-docs=false -Dxkb-config-root="$INSTDIR/share/X11/xkb"
     - ninja -C builddir
     - echo Running test script - this will take several minutes
-    - ./builddir/xkeyboard-config-test "$INSTDIR/share/X11/xkb/rules/evdev.xml" > $INSTDIR/xkeyboard-config-test.stdout 2> $INSTDIR/xkeyboard-config-test.stderr
-    - ./builddir/xkeyboard-config-test "$INSTDIR/share/X11/xkb/rules/evdev.extras.xml" >> $INSTDIR/xkeyboard-config-test.stdout 2>> $INSTDIR/xkeyboard-config-test.stderr
+    - ./builddir/xkeyboard-config-test --verbose "$INSTDIR/share/X11/xkb/rules/evdev.xml" > $INSTDIR/keymaps-success.yaml 2> $INSTDIR/keymaps-failed.yaml
+    - ./builddir/xkeyboard-config-test --verbose "$INSTDIR/share/X11/xkb/rules/evdev.extras.xml" >> $INSTDIR/keymaps-success.yaml 2>> $INSTDIR/keymaps-failed.yaml
     - popd > /dev/null
   after_script:
-    - xz -z "$INSTDIR/xkeyboard-config-test.stdout"
-    - xz -z "$INSTDIR/xkeyboard-config-test.stderr"
+    - echo "Failed keymap compilations:"
+    - yq -c ".[] | select(.status != 0) | .cmd, .error" $INSTDIR/keymaps-failed.yaml
+    - .gitlab-ci/yaml-to-junit-xml.py $INSTDIR/keymaps-failed.yaml > junit-results.xml
+    - xz -z "$INSTDIR/keymaps-success.yaml"
+    - xz -z "$INSTDIR/keymaps-failed.yaml"
   variables:
-    GIT_STRATEGY: none
+    GIT_DEPTH: 1
   artifacts:
     when: on_failure
     name: xkeyboard-config test output
     expire_in: 2 weeks
     paths:
-    - $INSTDIR/xkeyboard-config-test.stdout.xz
-    - $INSTDIR/xkeyboard-config-test.stderr.xz
+    - $INSTDIR/keymaps-success.yaml.xz
+    - $INSTDIR/keymaps-failed.yaml.xz
+    reports:
+      junit: junit-results.xml
 
 keymap_tests:
   extends: .default_setup
diff --git a/.gitlab-ci/yaml-to-junit-xml.py b/.gitlab-ci/yaml-to-junit-xml.py
new file mode 100755
index 00000000..b2972900
--- /dev/null
+++ b/.gitlab-ci/yaml-to-junit-xml.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+#
+# Converts the YAML format from the layout tester into JUnit XML
+#
+# This file is formatted with Python Black
+
+import yaml
+import sys
+from xml.dom import minidom
+
+with open(sys.argv[1]) as fd:
+    yml = yaml.safe_load(open(sys.argv[1]))
+
+    doc = minidom.Document()
+    suite = doc.createElement("testsuite")
+    suite.setAttribute("name", "XKB layout compilation tests")
+    doc.appendChild(suite)
+
+    # JUnit differs between test case failures
+    # and errors (something else blew up)
+    # We use failures for unrecognized keysyms and errors
+    # for everything else (i.e. keymap compilation errors)
+    ntests, nfailures, nerrors = 0, 0, 0
+
+    for testcase in yml:
+        ntests += 1
+        node = doc.createElement("testcase")
+        node.setAttribute("classname", f"{testcase['rmlvo'][0]} rules layout test")
+        # We don't care about rules and model here, LVO is enough
+        r, m, l, v, o = testcase["rmlvo"]
+        if v:
+            name = f"{l}({v})"
+        else:
+            name = l
+        if o:
+            name += f", {o}"
+        node.setAttribute("name", f"keymap compilation: {name}")
+        suite.appendChild(node)
+
+        if testcase["status"] != 0:
+            f = None
+            if testcase["status"] == 99:  # missing keysym
+                nfailures += 1
+                f = doc.createElement("failure")
+            else:  # everything else is an error
+                nerrors += 1
+                f = doc.createElement("error")
+            f.setAttribute("message", testcase["error"])
+            cdata = doc.createCDATASection(
+                f"Error message: {testcase['error']} in command {testcase['cmd']}"
+            )
+            f.appendChild(cdata)
+            node.appendChild(f)
+
+    suite.setAttribute("tests", str(ntests))
+    suite.setAttribute("errors", str(nerrors))
+    suite.setAttribute("failures", str(nfailures))
+
+    print(doc.toprettyxml(indent="  "))