validate: launcher: Add a way to simply run SSIM checks on rendered files

We will run a simple pipeline with the IQA element to run ssim (dssim)
tests on the rendered files, comparing it with a reference file.

For now we use the very empiric 1.0 value as a ssim error threshold and
the goal is basically to detect completely broken renderings.

3 files changed, 59 insertions, 5 deletions

diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py
index c5d509c..00544ef 100644
--- a/validate/launcher/apps/gstvalidate.py
+++ b/validate/launcher/apps/gstvalidate.py
@@ -530,6 +530,11 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa
 
         self.uri = uri
 
+    def run_external_checks(self):
+        if self.media_descriptor.get_num_tracks("video") == 1 and \
+                self.options.validate_enable_iqa_tests:
+            self.run_iqa_test(self.uri)
+
     def set_rendering_info(self):
         self.dest_file = os.path.join(self.options.dest,
                                       self.classname.replace(".transcode.", os.sep).
@@ -741,6 +746,9 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
                            help="Run the server in GDB.")
         group.add_argument("--validate-disable-rtsp", dest="disable_rtsp",
                            help="Disable RTSP tests.")
+        group.add_argument("--validate-enable-iqa-tests", dest="validate_enable_iqa_tests",
+                           help="Enable Image Quality Assessment validation tests.",
+                           default=False, action='store_true')
 
     def print_valgrind_bugs(self):
         # Look for all the 'pending' bugs in our supp file
diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py
index 340d91a..30cdca2 100644
--- a/validate/launcher/baseclasses.py
+++ b/validate/launcher/baseclasses.py
@@ -24,6 +24,7 @@ import os
 import sys
 import re
 import copy
+import shlex
 import socketserver
 import struct
 import time
@@ -354,6 +355,9 @@ class Test(Loggable):
     def kill_subprocess(self):
         utils.kill_subprocess(self, self.process, DEFAULT_TIMEOUT)
 
+    def run_external_checks(self):
+        pass
+
     def thread_wrapper(self):
         self.process = subprocess.Popen(self.command,
                                         stderr=self.out,
@@ -362,6 +366,8 @@ class Test(Loggable):
                                         cwd=self.workdir)
         self.process.wait()
         if self.result is not Result.TIMEOUT:
+            if self.process.returncode == 0:
+                self.run_external_checks()
             self.queue.put(None)
 
     def get_valgrind_suppression_file(self, subdir, name):
@@ -417,7 +423,8 @@ class Test(Loggable):
         self.timeout *= VALGRIND_TIMEOUT_FACTOR
 
         # Enable 'valgrind.config'
-        self.add_validate_config(get_data_file('data', 'valgrind.config'), subenv)
+        self.add_validate_config(get_data_file(
+            'data', 'valgrind.config'), subenv)
         if subenv == self.proc_env:
             self.add_env_variable('G_DEBUG', 'gc-friendly')
             self.add_env_variable('G_SLICE', 'always-malloc')
@@ -982,6 +989,7 @@ class GstValidateEncodingTestInterface(object):
         """
         return re.sub(r"\(.+?\)\s*| |;", '', caps).split(',')
 
+    # pylint: disable=E1101
     def _has_caps_type_variant(self, c, ccaps):
         """
         Handle situations where we can have application/ogg or video/ogg or
@@ -1002,6 +1010,42 @@ class GstValidateEncodingTestInterface(object):
 
         return has_variant
 
+    # pylint: disable=E1101
+    def run_iqa_test(self, reference_file_uri):
+        """
+        Runs IQA test if @reference_file_path exists
+        @test: The test to run tests on
+        """
+        pipeline_desc = """
+            uridecodebin uri=%s !
+                iqa name=iqa do-dssim=true dssim-error-threshold=1.0 ! fakesink
+            uridecodebin uri=%s ! iqa.
+        """ % (reference_file_uri, self.dest_file)
+        pipeline_desc = pipeline_desc.replace("\n", "")
+
+        command = [ScenarioManager.GST_VALIDATE_COMMAND] + \
+            shlex.split(pipeline_desc)
+        if not self.options.redirect_logs:
+            self.out.write(
+                "=================\n"
+                "Running IQA tests on results of: %s\n"
+                "Command: '%s'\n"
+                "=================\n\n" % (
+                    self.classname, ' '.join(command)))
+            self.out.flush()
+        else:
+            message = "Running IQA tests on results of:%s %s\n" \
+                "    Command: %s\n" % (
+                    Colors.ENDC, self.classname, ' '.join(command))
+            printc(message, Colors.OKBLUE)
+
+        self.process = subprocess.Popen(command,
+                                        stderr=self.out,
+                                        stdout=self.out,
+                                        env=self.proc_env,
+                                        cwd=self.workdir)
+        self.process.wait()
+
     def check_encoded_file(self):
         result_descriptor = GstValidateMediaDescriptor.new_from_uri(
             self.dest_file)
@@ -1674,7 +1718,8 @@ class _TestsLauncher(Loggable):
             while jobs_running != 0:
                 test = self.tests_wait()
                 jobs_running -= 1
-                test.number = "[%d / %d] " % (current_test_num, self.total_num_tests)
+                test.number = "[%d / %d] " % (current_test_num,
+                                              self.total_num_tests)
                 current_test_num += 1
                 res = test.test_end()
                 self.reporter.after_test(test)
diff --git a/validate/launcher/utils.py b/validate/launcher/utils.py
index b19fc11..45420cf 100644
--- a/validate/launcher/utils.py
+++ b/validate/launcher/utils.py
@@ -357,6 +357,7 @@ class BackTraceGenerator(Loggable):
                 # yet.
                 time.sleep(1)
 
+            application = test.process.args[0]
             try:
                 info = subprocess.check_output(['coredumpctl', 'info',
                                                 str(test.process.pid)],
@@ -375,9 +376,9 @@ class BackTraceGenerator(Loggable):
                 # The trace might not be ready yet
                 continue
 
-            if executable != test.application:
+            if executable != application:
                 self.debug("PID: %s -- executable %s != test application: %s" % (
-                    test.process.pid, executable, test.application))
+                    test.process.pid, executable, application))
                 # The trace might not be ready yet
                 continue
 
@@ -396,7 +397,7 @@ class BackTraceGenerator(Loggable):
                                              tf.name], stderr=subprocess.STDOUT)
 
                     gdb = ['gdb', '-ex', 't a a bt', '-ex', 'quit',
-                           test.application, tf.name]
+                           application, tf.name]
                     bt_all = subprocess.check_output(
                         gdb, stderr=subprocess.STDOUT).decode()