framework: remove forbidden XML chars for text in the JUnit backend

Text in XML cannot include certain characters. The XML 1.0
Recommendation defines a valid Char falling in this range:

"

  [2] Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] |
               [#x10000-#x10FFFF] /* any Unicode character, excluding
                                     the surrogate blocks, FFFE, and
                                     FFFF. */
"

This solves this kind of error:

"
  Traceback (most recent call last):
    File "./piglit", line 170, in <module>
      main()
    File "./piglit", line 166, in main
      sys.exit(runner(args))
    File "/piglit/framework/exceptions.py", line 49, in _inner
      return func(*args, **kwargs)
    File "/piglit/framework/programs/summary.py", line 271, in aggregate
      use_compression = backends.write(results, outfile)
    File "/piglit/framework/backends/__init__.py", line 201, in write
      return writer(results, file_path)
    File "/piglit/framework/backends/junit.py", line 483, in write_results
      writer(f, k, v)
    File "/piglit/framework/backends/junit.py", line 200, in __call__
      out.text = mola
    File "src/lxml/etree.pyx", line 1018, in lxml.etree._Element.text.__set__
    File "src/lxml/apihelpers.pxi", line 710, in lxml.etree._setNodeText
    File "src/lxml/apihelpers.pxi", line 698, in lxml.etree._createTextNode
    File "src/lxml/apihelpers.pxi", line 1493, in lxml.etree._utf8
  ValueError: All strings must be XML compatible: Unicode or ASCII, no NULL bytes or control characters
"

v2:
  - Speed up by compiling the regex before hand and move the logic to
    a specific function.

Signed-off-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Dylan Baker <dylan@pnwbakers.com>
Part-of: <https://gitlab.freedesktop.org/mesa/piglit/-/merge_requests/425>

1 files changed, 34 insertions, 3 deletions

diff --git a/framework/backends/junit.py b/framework/backends/junit.py
index 7090a6afe..9d919d4b7 100644
--- a/framework/backends/junit.py
+++ b/framework/backends/junit.py
@@ -1,5 +1,6 @@
 # coding=utf-8
 # Copyright (c) 2014-2016, 2019 Intel Corporation
+# Copyright © 2020 Valve Corporation.
 
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -22,6 +23,7 @@
 """ Module implementing a JUnitBackend for piglit """
 
 import os.path
+import re
 import shutil
 try:
     from lxml import etree
@@ -49,6 +51,29 @@ _PID_STR = "pid: "
 _START_TIME_STR = "start time: "
 _END_TIME_STR = "end time: "
 
+# XML cannot include certain characters. This regex matches the "invalid XML
+# text character range".
+_FORBIDDEN_XML_TEXT_CHARS_RE = re.compile(
+    u'[^\u0009\u000A\u000D\u0020-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]+'
+)
+
+
+def escape_forbidden_xml_text_chars(val, replacement=''):
+    """
+    Strip invalid XML text characters.
+    """
+    # See: https://github.com/html5lib/html5lib-python/issues/96
+    #
+    # The XML 1.0 spec defines the valid character range as:
+    # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
+    #
+    # Sources:
+    # https://www.w3.org/TR/REC-xml/#charsets,
+    # https://lsimons.wordpress.com/2011/03/17/stripping-illegal-characters-out-of-xml-in-python/
+
+    return _FORBIDDEN_XML_TEXT_CHARS_RE.sub(replacement, val)
+
+
 def junit_escape(name):
     name = name.replace('.', '_')
     if name in _JUNIT_SPECIAL_NAMES:
@@ -87,7 +112,9 @@ class JUnitWriter(object):
     def _set_xml_err(element, data, expected_result):
         """Adds the 'system-err' element."""
         err = etree.SubElement(element, 'system-err')
-        err.text = data.err
+        # We cannot control what is in the error output. Let's escape the
+        # forbidden XML characters.
+        err.text = escape_forbidden_xml_text_chars(data.err)
         err.text += '\n\n{}{}\n{}{}\n{}{}\n'.format(
             _PID_STR, data.pid,
             _START_TIME_STR, data.time.start,
@@ -195,7 +222,9 @@ class JUnitWriter(object):
 
             # Add stdout
             out = etree.SubElement(element, 'system-out')
-            out.text = data.out
+            # We cannot control what is in the output. Let's escape the
+            # forbidden XML characters.
+            out.text = escape_forbidden_xml_text_chars(data.out)
 
             # Prepend command line to stdout
             out.text = data.command + '\n' + out.text
@@ -244,7 +273,9 @@ class JUnitSubtestWriter(JUnitWriter):
 
             # Add stdout
             out = etree.SubElement(element, 'system-out')
-            out.text = data.out
+            # We cannot control what is in the output. Let's escape the
+            # forbidden XML characters.
+            out.text = escape_forbidden_xml_text_chars(data.out)
             # Prepend command line to stdout
             out.text = data.command + '\n' + out.text