summaryrefslogtreecommitdiff
path: root/unittests
diff options
context:
space:
mode:
authorDylan Baker <dylan@pnwbakers.com>2016-10-10 15:37:20 -0700
committerDylan Baker <dylan@pnwbakers.com>2016-10-24 11:12:38 -0700
commitef57fdd74368e7747100f37bc2fe638781602b4c (patch)
tree4bcee0df6dd1c17613d89c2cf89baf4efca18bb3 /unittests
parent6c8ddc3b8615f7f9da3b8c9ec1b59f87ac63050d (diff)
framework: Drop support for JSON formats < 7
Most of these are pretty old at this point (over a year), and it seems unlikely anyone still has results in those formats. The biggest problem with them is that some of them (especially the early ones) are complex transformations with a lot of validation needed, and some of the middle ones rely on the class representations to do the transformations, which is what this series is trying to change, since the tight coupling makes changes to the schema difficult, or nearly impossible in other cases. If there is compelling reason I could push support back a bit more, but I really want to drop some of the versions we support since it simplifies the code considerably. This also drops support for the older "main" file, and only supports results.json and results.json.<compression> Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>
Diffstat (limited to 'unittests')
-rw-r--r--unittests/framework/backends/test_json.py9
-rw-r--r--unittests/framework/backends/test_json_update.py738
2 files changed, 0 insertions, 747 deletions
diff --git a/unittests/framework/backends/test_json.py b/unittests/framework/backends/test_json.py
index 0c49aff8c..056e2c16c 100644
--- a/unittests/framework/backends/test_json.py
+++ b/unittests/framework/backends/test_json.py
@@ -234,15 +234,6 @@ class TestResume(object):
class TestLoadResults(object):
"""Tests for the load_results function."""
- def test_folder_with_main(self, tmpdir):
- """backends.json.load_results: takes a folder with a file named main in
- it.
- """
- p = tmpdir.join('main')
- with p.open('w') as f:
- f.write(json.dumps(shared.JSON))
- backends.json.load_results(six.text_type(tmpdir), 'none')
-
def test_folder_with_results_json(self, tmpdir):
"""backends.json.load_results: takes a folder with a file named
results.json.
diff --git a/unittests/framework/backends/test_json_update.py b/unittests/framework/backends/test_json_update.py
index 501a895a4..db45c1b1e 100644
--- a/unittests/framework/backends/test_json_update.py
+++ b/unittests/framework/backends/test_json_update.py
@@ -24,7 +24,6 @@ from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import os
-import copy
try:
import simplejson as json
except ImportError:
@@ -36,13 +35,10 @@ except ImportError:
import jsonschema
import pytest
-import six
from framework import backends
from framework import results
-from . import shared
-
# pylint: disable=protected-access,no-self-use
@@ -53,740 +49,6 @@ def setup_module():
yield
-class TestV0toV1(object):
- """Tests for version 0 -> version 1 of json results format."""
-
- # NOTE: It is very important to NOT use grouptools in this class.
- # The grouptools module changes and is updated from time to time, but this
- # file must remain static. It tests the update from one static version to
- # another static version.
- data = {
- 'options': {
- 'profile': "tests/fake.py",
- 'filter': [],
- 'exclude_filter': [],
- },
- 'name': 'fake-tests',
- 'lspci': 'fake',
- 'glxinfo': 'fake',
- 'tests': {
- 'sometest': {
- 'result': 'pass',
- 'time': 0.01,
- 'dmesg': ['this', 'is', 'dmesg'],
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n',
- },
- 'group1/groupA/test/subtest 1': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- 'subtest 2': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group1/groupA/test/subtest 2': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- 'subtest 2': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'single/test/thing': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group2/groupA/test/subtest 1/depth': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1/depth': 'pass',
- 'subtest 2/float': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group2/groupA/test/subtest 2/float': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1/depth': 'pass',
- 'subtest 2/float': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group3/groupA/test': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- 'subtest 2': 'pass',
- 'subtest 3': 'pass',
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('results.json')
- p.write(json.dumps(self.data))
- with p.open('r') as f:
- return backends.json._update_zero_to_one(backends.json._load(f))
-
- def test_dmesg(self, result):
- """backends.json.update_results (0 -> 1): dmesg is converted from a
- list to a string.
- """
- assert result.tests['sometest']['dmesg'] == 'this\nis\ndmesg'
-
- def test_subtests_remove_duplicates(self, result):
- """backends.json.update_results (0 -> 1): Removes duplicate entries"""
- assert 'group1/groupA/test/subtest 1' not in result.tests
- assert 'group1/groupA/test/subtest 2' not in result.tests
-
- def test_subtests_add_test(self, result):
- """backends.json.update_results (0 -> 1): Add an entry for the actual
- test.
- """
- assert result.tests.get('group1/groupA/test')
-
- def test_subtests_is_testresult(self, result):
- """backends.json.update_results (0 -> 1): The result of the new test is
- a dict Instance.
- """
- assert isinstance(result.tests['group1/groupA/test'], dict)
-
- def test_info_delete(self, result):
- """backends.json.update_results (0 -> 1): Remove the info name from
- results.
- """
- for value in six.itervalues(result.tests):
- assert 'info' not in value
-
- def test_returncode_from_info(self, result):
- """backends.json.update_results (0 -> 1): Use the returncode from info
- if there is no returncode.
- """
- assert result.tests['sometest']['returncode'] == 1
-
- def test_returncode_no_override(self, result):
- """backends.json.update_results (0 -> 1): Do not clobber returncode
- with info.
-
- The returncode from info should not overwrite an existing returcnode
- attribute, this test only tests that the value in info isn't used when
- there is a value in returncode already.
- """
- assert result.tests['group1/groupA/test']['returncode'] != 1
-
- def test_err_from_info(self, result):
- """backends.json.update_results (0 -> 1): add an err attribute from
- info.
- """
- assert result.tests['group1/groupA/test']['err'] == 'stderr'
-
- def test_out_from_info(self, result):
- """backends.json.update_results (0 -> 1): add an out attribute from
- info.
- """
- assert result.tests['group1/groupA/test']['out'] == 'stdout'
-
- def test_set_version(self, result):
- """backends.json.update_results (0 -> 1): Set the version to 1"""
- assert result.results_version == 1
-
- def test_dont_break_single_subtest(self, result):
- """backends.json.update_results (0 -> 1): Don't break single subtest
- entries.
-
- A test with a single subtest was written correctly before, don't break
- it by removing the name of the test. ex:
- test/foo/bar: {
- ...
- subtest: {
- 1x1: pass
- }
- }
-
- should remain test/foo/bar since bar is the name of the test not a
- subtest
- """
- assert result.tests.get('single/test/thing')
-
- def test_subtests_with_slash(self, result):
- """backends.json.update_results (0 -> 1): Subtest names with /'s are
- handled correctly.
- """
- expected = 'group2/groupA/test/subtest 1'
- assert expected not in result.tests
-
- def test_handle_fixed_subtests(self, result):
- """backends.json.update_results (0 -> 1): Correctly handle new single
- entry subtests correctly.
- """
- assert 'group3/groupA/test' in result.tests
-
- def test_load_results_unversioned(self, tmpdir):
- """backends.json.load_results: Loads unversioned results and updates .
- correctly.
-
- This test pickes on attribute (dmesg) to test, with the assumption taht
- if the other tests work then once the update path starts it runs
- correctly.
- """
- p = tmpdir.join('results.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- result = backends.json.load_results(six.text_type(p), 'none')
-
- assert result.tests['sometest'].dmesg == 'this\nis\ndmesg'
-
- def test_load_results_v0(self, tmpdir):
- """backends.json.load_results: Loads results v0 and updates correctly.
-
- This test pickes on attribute (dmesg) to test, with the assumption taht
- if the other tests work then once the update path starts it runs
- correctly.
- """
- data = copy.deepcopy(self.data)
- data['results_version'] = 0
-
- p = tmpdir.join('results.json')
- p.write(json.dumps(data, default=backends.json.piglit_encoder))
- result = backends.json.load_results(six.text_type(p), 'none')
-
- assert result.tests['sometest'].dmesg == 'this\nis\ndmesg'
-
- def test_info_split(self, tmpdir):
- """backends.json.update_results (0 -> 1): info can split into any
- number of elements.
- """
- data = copy.copy(self.data)
- data['tests']['sometest']['info'] = \
- 'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n\nmore\n\nstuff'
-
- p = tmpdir.join('results.json')
- p.write(json.dumps(data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- backends.json._update_zero_to_one(backends.json._load(f))
-
- def test_load_results(self, tmpdir):
- """backends.json.update_results (1 -> current): load_results properly
- updates.
- """
- p = tmpdir.join('results.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- result = backends.json.load_results(six.text_type(p), 'none')
- assert result.results_version == backends.json.CURRENT_JSON_VERSION # pylint: disable=no-member
-
-
-class TestV1toV2(object):
- """Tests version 1 to version 2."""
-
- class TestWithChanges(object):
- """Test V1 to V2 of results."""
- data = {
- "results_version": 1,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "test/is/a/test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_one_to_two(backends.json._load(f))
-
- def test_version_is_two(self, result):
- """backends.json.update_results (1 -> 2): The result version is updated
- to 2.
- """
- assert result.results_version == 2
-
- def test_no_env(self, result):
- """backends.json.update_results (1 -> 2): Removes options['env']."""
- assert 'env' not in result.options
-
- def test_glxinfo(self, result):
- """backends.json.update_results (1 -> 2): puts glxinfo in the root."""
- assert result.glxinfo == 'and stuff'
-
- def test_lspci(self, result):
- """backends.json.update_results (1 -> 2): puts lspci in the root."""
- assert result.lspci == 'stuff'
-
- def test_uname(self, result):
- """backends.json.update_results (1 -> 2): puts uname in the root."""
- assert result.uname == 'more stuff'
-
- def test_wglinfo(self, result):
- """backends.json.update_results (1 -> 2): puts wglinfo in the root."""
- assert result.wglinfo == 'stuff'
-
- class TestWithoutChanges(object):
- """Test a version 1 to 2 update when version 1 was correct"""
- data = {
- "results_version": 1,
- "name": "test",
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- },
- "tests": {
- "test/is/a/test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_one_to_two(backends.json._load(f))
-
- def test_version_is_two(self, result):
- """backends.json.update_results (1 -> 2) no change: The result version
- is updated to 2.
- """
- assert result.results_version == 2
-
- def test_glxinfo(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- glxinfo.
- """
- assert result.glxinfo == 'and stuff'
-
- def test_lspci(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- lspci.
- """
- assert result.lspci == 'stuff'
-
- def test_uname(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- uname.
- """
- assert result.uname == 'more stuff'
-
- def test_wglinfo(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- wglinfo.
- """
- assert result.wglinfo == 'stuff'
-
-
-class TestV2toV3(object):
- """Tests for version 2 -> version 3 of json results"""
- # NOTE: do NOT use grouptools in this class, see v0 tests for explanation
- data = {
- "results_version": 2,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "test/is/a/test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- },
- "Test/Is/SomE/Other1/Test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_two_to_three(backends.json._load(f))
-
- def test_unchanged(self, result):
- """backends.json.update_results (2 -> 3): results with no caps are not
- mangled.
- """
- assert 'test/is/a/test' in result.tests
-
- def test_lower(self, result):
- """backends.json.update_results (2 -> 3): results with caps are
- lowered.
- """
- assert 'test/is/some/other1/test' in result.tests
-
- def test_removed(self, result):
- """backends.json.update_results (2 -> 3): results with caps are
- removed.
- """
- assert 'Test/Is/SomE/Other1/Test' not in result.tests
-
-
-class TestV3toV4(object):
- """Tests for version 3 to version 4."""
-
- # NOTE: do NOT use grouptools in this module, see v0 tests for explanation
- test_data = {
- 'returncode': 0,
- 'err': None,
- 'environment': None,
- 'command': 'foo',
- 'result': 'skip',
- 'time': 0.123,
- 'out': None,
- }
-
- data = {
- "results_version": 3,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "spec/arb_texture_rg/fs-shadow2d-red-01": test_data,
- "spec/arb_texture_rg/fs-shadow2d-red-02": test_data,
- "spec/arb_texture_rg/fs-shadow2d-red-03": test_data,
- "spec/arb_draw_instanced/draw-non-instanced": test_data,
- "spec/arb_draw_instanced/instance-array-dereference": test_data,
- "glslparsertest/foo": test_data,
- }
- }
-
- old = list(data['tests'].keys())
- new = [
- "spec/arb_texture_rg/execution/fs-shadow2d-red-01",
- "spec/arb_texture_rg/execution/fs-shadow2d-red-02",
- "spec/arb_texture_rg/execution/fs-shadow2d-red-03",
- "spec/arb_draw_instanced/execution/draw-non-instanced",
- "spec/arb_draw_instanced/execution/instance-array-dereference",
- ]
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_three_to_four(backends.json._load(f))
-
- def test_old_removed(self, result):
- """backends.json.update_results (3 -> 4): All old test names are
- removed.
- """
- for old in self.old:
- assert old not in result.tests
-
- def test_new_added(self, result):
- """backends.json.update_results (3 -> 4): All new test names are added.
- """
- for new in self.new:
- assert new in result.tests
-
- def test_new_has_data(self, result):
- """backends.json.update_results (3 -> 4): All new tests have expected
- data.
- """
- for new in self.new:
- assert result.tests[new] == self.test_data
-
- def test_missing(self, tmpdir):
- """backends.json.update_results (3 -> 4): updates successfully when
- tests to rename are not present.
- """
- data = copy.copy(self.data)
- del data['tests']['spec/arb_draw_instanced/instance-array-dereference']
-
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
-
- with p.open('r') as f:
- backends.json._update_three_to_four(backends.json._load(f))
-
-
-class TestV4toV5(object):
- test_data = {
- 'returncode': 0,
- 'err': None,
- 'environment': None,
- 'command': 'foo',
- 'result': 'skip',
- 'time': 0.123,
- 'out': None,
- }
-
- data = {
- "results_version": 4,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "a/test/group/of/great/length": test_data,
- "has\\windows": test_data,
- }
- }
-
- old = list(data['tests'].keys())
- new = [
- 'a@test@group@of@great@length',
- 'has@windows',
- ]
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_four_to_five(backends.json._load(f))
-
- def test_posix_removed(self, result):
- """backends.json.update_results (4 -> 5): / is replaced with @."""
- assert 'a/test/of/great/length' not in result.tests
-
- def test_win_removed(self, result):
- """backends.json.update_results (4 -> 5): \\ is replaced with @."""
- assert 'has\\windows' not in result.tests
-
- def test_new_added(self, result):
- """backends.json.update_results (4 -> 5): All new test names are added.
- """
- for new in self.new:
- assert new in result.tests
-
- def test_new_has_data(self, result):
- """backends.json.update_results (4 -> 5): All new tests have expected
- data.
- """
- for new in self.new:
- assert result.tests[new] == self.test_data
-
-
-class TestV5toV6(object):
- test_data = {
- 'returncode': 0,
- 'err': '',
- 'environment': None,
- 'command': 'foo',
- 'result': 'skip',
- 'time': 0.123,
- 'out': '',
- }
-
- data = {
- "results_version": 4,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- 'a@test': test_data,
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_five_to_six(backends.json._load(f))
-
- def test_result_is_testresult_instance(self, result):
- """backends.json.update_results (5 -> 6): A test result is converted to
- a TestResult instance.
- """
- assert isinstance(result.tests['a@test'], results.TestResult)
-
-
-class TestV6toV7(object):
- """Tests for version 6 to version 7."""
-
- data = {
- "results_version": 6,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- 'a@test': results.TestResult('pass'),
- 'a@nother@test': results.TestResult('fail'),
- 'a@nother@thing': results.TestResult('crash'),
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_six_to_seven(backends.json._load(f))
-
- def test_is_TestrunResult(self, result):
- """backends.json.update_results (6 -> 7): makes TestrunResult."""
- assert isinstance(result, results.TestrunResult)
-
- def test_totals(self, result):
- """backends.json.update_results (6 -> 7): Totals are populated."""
- assert result.totals != {}
-
-
class TestV7toV8(object):
"""Tests for Version 7 to version 8."""