framework: add support for xz compression via backports.lzma

This adds the option of using a python module, rather than calling out
to the shell for xz support.

v4: - add this patch

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>

2 files changed, 87 insertions, 72 deletions

diff --git a/README b/README
index 19682215c..773a35060 100644
--- a/README
+++ b/README
@@ -45,6 +45,9 @@ Optionally, you can install the following:
   - lxml. An accelerated python xml library using libxml2 (http://lxml.de/)
   - simplejson. A fast C based implementation of the python json library.
     (https://simplejson.readthedocs.org/en/latest/)
+  - backports.lzma. A packport of python3's lzma module to python2,
+    this enables fast native xz (de)compression in piglit for results files
+    (https://github.com/peterjc/backports.lzma)
 
 Now configure the build system:
 
diff --git a/framework/backends/compression.py b/framework/backends/compression.py
index d3a33b957..7bc488e9c 100644
--- a/framework/backends/compression.py
+++ b/framework/backends/compression.py
@@ -71,80 +71,92 @@ DECOMPRESSORS = {
 }
 
 # TODO: in python3 there is builtin xz support, and doesn't need this madness
-# If there is an xz binary then try calling out to that
+# First try to use backports.lzma, that's the easiest solution. If that fails
+# then go to trying the shell. If that fails then piglit won't have xz support,
+# and will raise an error if xz is used
 try:
-    with open(os.devnull, 'w') as d:
-        subprocess.check_call(['xz'], stderr=d)
-except subprocess.CalledProcessError as e:
-    if e.returncode == 1:
-        import contextlib
-        try:
-            import cStringIO as StringIO
-        except ImportError:
-            import StringIO
-
-        @contextlib.contextmanager
-        def _compress_xz(filename):
-            """Emulates an open function in write mode for xz.
-
-            Python 2.x doesn't support xz, but it's dang useful. This function calls
-            out to the shell and tries to use xz from the environment to get xz
-            compression.
-
-            This obviously won't work without a working xz binary.
-
-            This function tries to emulate the default values of the lzma module in
-            python3 as much as possible
-
-            """
-            if filename.endswith('.xz'):
-                filename = filename[:-2]
-
-            with open(filename, 'w') as f:
-                yield f
-
-            try:
-                subprocess.check_call(['xz', '--compress', '-9', filename])
-            except OSError as e:
-                if e.errno == errno.ENOENT:
-                    raise exceptions.PiglitFatalError('No xz binary available')
-                raise
-
-        @contextlib.contextmanager
-        def _decompress_xz(filename):
-            """Eumlates an option function in read mode for xz.
-
-            See the comment in _compress_xz for more information.
-
-            This module tries to emulate the lzma module as much as possible
-
-            """
-            if not filename.endswith('.xz'):
-                filename = '{}.xz'.format(filename)
-
+    import backports.lzma
+
+    COMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='w')
+    DECOMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='r')
+    COMPRESSION_SUFFIXES += ['.xz']
+except ImportError:
+    try:
+        with open(os.devnull, 'w') as d:
+            subprocess.check_call(['xz'], stderr=d)
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            import contextlib
             try:
-                string = subprocess.check_output(
-                    ['xz', '--decompress', '--stdout', filename])
-            except OSError as e:
-                if e.errno == errno.ENOENT:
-                    raise exceptions.PiglitFatalError('No xz binary available')
-                raise
-
-            # We need a file-like object, so the contents must be placed in a StringIO
-            # object.
-            io = StringIO.StringIO()
-            io.write(string)
-            io.seek(0)
-
-            yield io
-
-            io.close()
-
-        COMPRESSORS['xz'] = _compress_xz
-        DECOMPRESSORS['xz'] = _decompress_xz
-        COMPRESSION_SUFFIXES += ['.xz']
-except OSError:
-    pass
+                import cStringIO as StringIO
+            except ImportError:
+                import StringIO
+
+            @contextlib.contextmanager
+            def _compress_xz(filename):
+                """Emulates an open function in write mode for xz.
+
+                Python 2.x doesn't support xz, but it's dang useful. This
+                function calls out to the shell and tries to use xz from the
+                environment to get xz compression.
+
+                This obviously won't work without a working xz binary.
+
+                This function tries to emulate the default values of the lzma
+                module in python3 as much as possible
+
+                """
+                if filename.endswith('.xz'):
+                    filename = filename[:-2]
+
+                with open(filename, 'w') as f:
+                    yield f
+
+                try:
+                    subprocess.check_call(['xz', '--compress', '-9', filename])
+                except OSError as e:
+                    if e.errno == errno.ENOENT:
+                        raise exceptions.PiglitFatalError(
+                            'No xz binary available')
+                    raise
+
+            @contextlib.contextmanager
+            def _decompress_xz(filename):
+                """Eumlates an option function in read mode for xz.
+
+                See the comment in _compress_xz for more information.
+
+                This function tries to emulate the lzma module as much as
+                possible
+
+                """
+                if not filename.endswith('.xz'):
+                    filename = '{}.xz'.format(filename)
+
+                try:
+                    string = subprocess.check_output(
+                        ['xz', '--decompress', '--stdout', filename])
+                except OSError as e:
+                    if e.errno == errno.ENOENT:
+                        raise exceptions.PiglitFatalError(
+                            'No xz binary available')
+                    raise
+
+                # We need a file-like object, so the contents must be placed in
+                # a StringIO object.
+                io = StringIO.StringIO()
+                io.write(string)
+                io.seek(0)
+
+                yield io
+
+                io.close()
+
+            COMPRESSORS['xz'] = _compress_xz
+            DECOMPRESSORS['xz'] = _decompress_xz
+            COMPRESSION_SUFFIXES += ['.xz']
+    except OSError:
+        pass
 
 
 def _set_mode():