functionaltests: move replayer functional test to a general location

Signed-off-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Dylan Baker <dylan@pnwbakers.com>
Part-of: <https://gitlab.freedesktop.org/mesa/piglit/-/merge_requests/353>

6 files changed, 305 insertions, 0 deletions

diff --git a/functionaltests/framework/replay/backends/testtrace.py b/functionaltests/framework/replay/backends/testtrace.py
new file mode 100644
index 000000000..08f8b717e
--- /dev/null
+++ b/functionaltests/framework/replay/backends/testtrace.py
@@ -0,0 +1,78 @@
+# coding=utf-8
+#
+# Copyright (c) 2014, 2016-2017, 2019-2020 Intel Corporation
+# Copyright (c) 2019 Collabora Ltd
+# Copyright © 2019-2020 Valve Corporation.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# SPDX-License-Identifier: MIT
+
+
+""" Module providing a trace testing dump backend for replayer """
+
+import os
+import subprocess
+
+from os import path
+
+from framework import core
+from .abstract import DumpBackend, dump_handler
+from .register import Registry
+
+
+__all__ = [
+    'REGISTRY',
+    'TestTraceBackend',
+]
+
+
+class TestTraceBackend(DumpBackend):
+    """ replayer's dump backend for trace testing
+
+    This is a simple dump backend which reads a RGBA color from a text file and
+    dumps a square image with that color.
+
+    """
+    _get_last_frame_call = None  # this silences the abstract-not-subclassed warning
+
+    def __init__(self, trace_path, output_dir=None, calls=[], **kwargs):
+        super(TestTraceBackend, self).__init__(trace_path, output_dir, calls,
+                                               **kwargs)
+        if len(self._calls) == 0: self._calls = ['0']
+
+    @dump_handler
+    def dump(self):
+        from PIL import Image
+        outputprefix = path.join(self._output_dir, path.basename(self._trace_path))
+        with open(self._trace_path) as f:
+            rgba = f.read()
+        color = [int(rgba[0:2], 16), int(rgba[2:4], 16),
+                 int(rgba[4:6], 16), int(rgba[6:8], 16)]
+        for c in self._calls:
+            outputfile = outputprefix + '-' + c + '.png'
+            print('Writing RGBA: {} to {}'.format(rgba, outputfile))
+            Image.frombytes('RGBA', (32, 32),
+                            bytes(color * 32 * 32)).save(outputfile)
+
+
+REGISTRY = Registry(
+    extensions=['.testtrace'],
+    backend=TestTraceBackend,
+)
diff --git a/functionaltests/framework/replay/programs/__init__.py b/functionaltests/framework/replay/programs/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/functionaltests/framework/replay/programs/__init__.py
diff --git a/functionaltests/framework/replay/programs/test_compare.py b/functionaltests/framework/replay/programs/test_compare.py
new file mode 100644
index 000000000..d2f8cec7e
--- /dev/null
+++ b/functionaltests/framework/replay/programs/test_compare.py
@@ -0,0 +1,216 @@
+import logging
+import os
+import pytest
+import re
+import shutil
+
+from os import path
+
+import tracie
+
+
+RESULTS_YAML = "results/results.yml"
+TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
+TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
+TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
+TRACE_PNG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace-0.png"
+TRACIE_DIR = path.dirname(path.realpath(__file__)) + "/.."
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def write_to(content, filename):
+    with open(filename, 'w') as f:
+        f.write(content)
+
+
+def read_from(filename):
+    with open(filename) as f:
+        content = f.read()
+    return content
+
+
+def run_tracie():
+    '''
+    Run tests for the .testtrace types, using the "gl-test-device" and
+    "vk-test-device" device names.
+    '''
+    result = tracie.main(["yaml",
+                          "--device-name", "gl-test-device",
+                          "--file", "./tests/traces.yml"])
+    if not result:
+        return False
+    result = tracie.main(["yaml",
+                          "--device-name", "vk-test-device",
+                          "--file", "./tests/traces.yml"])
+    return result
+
+
+def prepare_for_run(tmp_path):
+    '''
+    Copy all the tracie scripts to the test dir for the unit tests.
+    This avoids polluting the normal working dir with test result artifacts.
+    '''
+    test_dir = str(tmp_path) + "/run"
+    shutil.copytree(TRACIE_DIR, test_dir)
+    # Change the working dir to the test_dir
+    os.chdir(test_dir)
+    # Set the traces-db
+    shutil.move("./tests/test-data", "./traces-db")
+    # Disable trace storing
+    os.environ["TRACIE_STORE_IMAGES"] = "0"
+
+
+def cleanup(tmp_path):
+    '''
+    Performs the clean up of the test dir.
+    '''
+    if path.exists(tmp_path):
+        shutil.rmtree(tmp_path)
+
+
+@pytest.fixture(autouse=True)
+def run_test(tmp_path):
+    '''
+    Wraps the execution of each test as follows:
+
+      prepare_for_run()
+      test()
+      cleanup()
+    '''
+    logger.debug("Working dir: %s", tmp_path)
+    prepare_for_run(tmp_path)
+    yield
+    cleanup(tmp_path)
+
+
+def check_results_yaml_content(filename, expectations):
+    '''
+    Checks the content of the filename with the list of expectations
+    passed as parameter.
+
+    Arguments:
+        filename (str): The path of the file to check
+        expectations (list): A list with the content to find in the file
+
+    Returns:
+        bool: The return value. True if the content of the filename satisfies
+              the expectations, False otherwise.
+    '''
+    content = read_from(filename)
+    for e in expectations:
+        ocurrencies = re.findall(e, content)
+        if not len(ocurrencies):
+            logger.error("Expectation not found in %s: %s", filename, e)
+            return False
+    return True
+
+
+def test_tracie_succeeds_if_all_images_match():
+    assert run_tracie()
+    expectations = [
+        "actual: 5efda83854befe0155ff8517a58d5b51",
+        "expected: 5efda83854befe0155ff8517a58d5b51",
+    ]
+    assert check_results_yaml_content(RESULTS_YAML, expectations)
+
+
+def test_tracie_fails_on_image_mismatch():
+    filename = "./tests/traces.yml"
+    content = read_from(filename)
+    content = content.replace("5efda83854befe0155ff8517a58d5b51",
+                              "8e0a801367e1714463475a824dab363b")
+    write_to(content, filename)
+    assert not run_tracie()
+    expectations = [
+        "actual: 5efda83854befe0155ff8517a58d5b51",
+        "expected: 8e0a801367e1714463475a824dab363b",
+        "trace2/test/vk-test-device/olive.testtrace-0.png"
+    ]
+    assert check_results_yaml_content(RESULTS_YAML, expectations)
+
+
+def test_tracie_traces_with_and_without_checksum():
+    filename = "./tests/traces.yml"
+    content = read_from(filename)
+    content += '''  - path: trace1/red.testtrace
+    expectations:
+    - device: bla
+      checksum: 000000000000000'''
+    write_to(content, filename)
+
+    # red.testtrace should be skipped, since it doesn't
+    # have any checksums for our device
+    filename = "./traces-db/trace1/red.testtrace"
+    content = "ff0000ff"
+    write_to(content, filename)
+    assert run_tracie()
+
+
+def test_tracie_only_traces_without_checksum():
+    filename = "./tests/traces.yml"
+    content = '''traces:
+  - path: trace1/red.testtrace
+    expectations:
+    - device: bla
+      checksum: 000000000000000'''
+    write_to(content, filename)
+
+    # red.testtrace should be skipped, since it doesn't
+    # have any checksums for our device
+    filename = "./traces-db/trace1/red.testtrace"
+    content = "ff0000ff"
+    write_to(content, filename)
+    assert run_tracie()
+
+
+def test_tracie_with_no_traces():
+    filename = "./tests/traces.yml"
+    content = 'traces:'
+    write_to(content, filename)
+    assert run_tracie()
+    expectations = [
+        "{}",
+    ]
+    assert check_results_yaml_content(RESULTS_YAML, expectations)
+
+
+def test_tracie_fails_on_dump_image_error():
+    # "invalid" should fail to parse as rgba and
+    # cause an error
+    filename = "./traces-db/trace1/magenta.testtrace"
+    write_to("invalid\n", filename)
+    run_tracie()
+    expectations = [
+        "actual: error",
+        "expected: 8e0a801367e1714463475a824dab363b",
+        "trace1/magenta.testtrace",
+    ]
+    assert check_results_yaml_content(RESULTS_YAML, expectations)
+
+
+def test_tracie_stores_only_logs_on_checksum_match():
+    assert run_tracie()
+    assert path.exists(TRACE_LOG_TEST1)
+    assert path.exists(TRACE_LOG_TEST2)
+    assert not path.exists(TRACE_PNG_TEST1)
+    assert not path.exists(TRACE_PNG_TEST2)
+
+
+def test_tracie_stores_images_on_checksum_mismatch():
+    filename = "./tests/traces.yml"
+    content = read_from(filename)
+    content = content.replace("5efda83854befe0155ff8517a58d5b51",
+                              "8e0a801367e1714463475a824dab363b")
+    write_to(content, filename)
+    assert not run_tracie()
+    assert not path.exists(TRACE_PNG_TEST1)
+    assert path.exists(TRACE_PNG_TEST2)
+
+
+def test_tracie_stores_images_on_request():
+    os.environ["TRACIE_STORE_IMAGES"] = "1"
+    assert run_tracie()
+    assert path.exists(TRACE_PNG_TEST1)
+    assert path.exists(TRACE_PNG_TEST2)
diff --git a/functionaltests/framework/replay/programs/tests/test-data/trace1/magenta.testtrace b/functionaltests/framework/replay/programs/tests/test-data/trace1/magenta.testtrace
new file mode 100644
index 000000000..2354cb56d
--- /dev/null
+++ b/functionaltests/framework/replay/programs/tests/test-data/trace1/magenta.testtrace
@@ -0,0 +1 @@
+ff00ffff
diff --git a/functionaltests/framework/replay/programs/tests/test-data/trace2/olive.testtrace b/functionaltests/framework/replay/programs/tests/test-data/trace2/olive.testtrace
new file mode 100644
index 000000000..825890f66
--- /dev/null
+++ b/functionaltests/framework/replay/programs/tests/test-data/trace2/olive.testtrace
@@ -0,0 +1 @@
+80800080
diff --git a/functionaltests/framework/replay/programs/tests/traces.yml b/functionaltests/framework/replay/programs/tests/traces.yml
new file mode 100644
index 000000000..64e3e4fce
--- /dev/null
+++ b/functionaltests/framework/replay/programs/tests/traces.yml
@@ -0,0 +1,9 @@
+traces:
+  - path: trace1/magenta.testtrace
+    expectations:
+      - device: gl-test-device
+        checksum: 8e0a801367e1714463475a824dab363b
+  - path: trace2/olive.testtrace
+    expectations:
+      - device: vk-test-device
+        checksum: 5efda83854befe0155ff8517a58d5b51