diff options
Diffstat (limited to 'unittests')
| -rw-r--r-- | unittests/base_tests.py | 2 | ||||
| -rw-r--r-- | unittests/framework/__init__.py | 0 | ||||
| -rw-r--r-- | unittests/framework/test_status.py | 139 | ||||
| -rw-r--r-- | unittests/status_tests.py | 300 |
4 files changed, 140 insertions, 301 deletions
diff --git a/unittests/base_tests.py b/unittests/base_tests.py index 0f89139c2..28e40ade0 100644 --- a/unittests/base_tests.py +++ b/unittests/base_tests.py @@ -46,7 +46,7 @@ except ImportError: pass from . import utils -from .status_tests import PROBLEMS, STATUSES +from .framework.test_status import PROBLEMS, STATUSES from framework.test.base import ( Test, TestRunError, diff --git a/unittests/framework/__init__.py b/unittests/framework/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/unittests/framework/__init__.py diff --git a/unittests/framework/test_status.py b/unittests/framework/test_status.py new file mode 100644 index 000000000..94f805a23 --- /dev/null +++ b/unittests/framework/test_status.py @@ -0,0 +1,139 @@ +# encoding=utf-8 +# Copyright © 2014, 2016 Intel Corporation + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Tests for framework.status. + +This module does not have the comprehensive tests for all of the various +combinations of comparisons between the various kinds of functions. Instead, it +just asserts that the regressions, fixes, etc lists are as expected. + +""" + +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) +import itertools + +import pytest +import six + +from framework import status + + +# Statuses from worst to last. NotRun is intentionally not in this list and +# tested separately because of upcoming features for it +STATUSES = [ + status.PASS, + status.WARN, + status.DMESG_WARN, + status.FAIL, + status.DMESG_FAIL, + status.TIMEOUT, + status.CRASH, + status.INCOMPLETE, +] + +# all statuses except pass are problems +PROBLEMS = STATUSES[1:] + +# Create lists of fixes and regressions programmatically based on the STATUSES +# list. This means less code, and easier expansion changes. +REGRESSIONS = list(itertools.combinations(STATUSES, 2)) + \ + list(itertools.combinations([status.SKIP] + PROBLEMS, 2)) +FIXES = list(itertools.combinations(reversed(STATUSES), 2)) + \ + list(itertools.combinations(list(reversed(PROBLEMS)) + [status.SKIP], 2)) + +# The statuses that don't cause changes when transitioning from one another +NO_OPS = [status.SKIP, status.NOTRUN] + + +@pytest.mark.raises(exception=status.StatusException) +def test_bad_lookup(): + """status.status_lookup: An unexepcted value raises a StatusException""" + status.status_lookup('foobar') + + +@pytest.mark.raises(exception=TypeError) +def test_status_eq_raises(): + """status.Status: eq comparison to uncomparable object results in TypeError""" + status.PASS == dict() + + +@pytest.mark.raises(exception=TypeError) +def test_nochangestatus_eq_raises(): + """status.NoChangeStatus: eq comparison to uncomparable type results in TypeError""" + status.NOTRUN == dict() + + +@pytest.mark.raises(exception=TypeError) +def test_nochangestatus_ne_raises(): + """status.NoChangeStatus: ne comparison to uncomparable type results in TypeError""" + status.NOTRUN != dict() + + +def test_status_in(): + """status.Status: A status can be looked up with 'x in y' synatx""" + stat = status.PASS + slist = ['pass'] + + assert stat in slist + + +@pytest.mark.parametrize( + 'stat', itertools.chain(STATUSES, [status.SKIP, status.NOTRUN])) +def test_lookup(stat): + status.status_lookup(stat) + + +@pytest.mark.parametrize('new,old', REGRESSIONS) +def test_regression(new, old): + assert status.status_lookup(new) < status.status_lookup(old) + + +@pytest.mark.parametrize('new,old', FIXES) +def test_fixes(new, old): + assert status.status_lookup(new) > status.status_lookup(old) + + +@pytest.mark.parametrize('new,old', itertools.permutations(STATUSES, 2)) +def test_changes(new, old): + assert status.status_lookup(new) != status.status_lookup(old) + + +@pytest.mark.parametrize('new,old', itertools.permutations(NO_OPS, 2)) +def test_no_change(new, old): + new = status.status_lookup(new) + old = status.status_lookup(old) + assert not new < old + assert not new > old + + +@pytest.mark.parametrize("stat,op,expected", [ + (status.Status('Test', 0, (0, 0)), six.text_type, 'Test'), + (status.Status('Test', 0, (0, 0)), six.binary_type, b'Test'), + (status.Status('Test', 0, (0, 0)), int, 0), + (status.Status('Test', 0, (0, 0)), repr, 'Status("Test", 0, (0, 0))'), + (status.Status('Test', 0, (0, 0)), hash, hash('Test')), + (status.NoChangeStatus('No'), hash, hash('No')), +]) +def test_status_comparisons(stat, op, expected): + """Test status.Status equality protocol.""" + assert op(stat) == expected diff --git a/unittests/status_tests.py b/unittests/status_tests.py deleted file mode 100644 index c2d62f8bf..000000000 --- a/unittests/status_tests.py +++ /dev/null @@ -1,300 +0,0 @@ -# Copyright (c) 2014 Intel Corperation - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" Tests for the Status module - -Note: see framework/status.py for the authoritative list of fixes, regression, -etc - -""" - -from __future__ import ( - absolute_import, division, print_function, unicode_literals -) -import itertools - -import nose.tools as nt -import six - -import framework.status as status -from . import utils - -# pylint: disable=expression-not-assigned,invalid-name,line-too-long - -# Statuses from worst to last. NotRun is intentionally not in this list and -# tested separately because of upcoming features for it -STATUSES = ["pass", "warn", "dmesg-warn", "fail", "dmesg-fail", "timeout", - "crash", 'incomplete'] - -# all statuses except pass are problems -PROBLEMS = STATUSES[1:] - -# Create lists of fixes and regressions programmatically based on the STATUSES -# list. This means less code, and easier expansion changes. -REGRESSIONS = list(itertools.combinations(STATUSES, 2)) + \ - list(itertools.combinations(["skip"] + PROBLEMS, 2)) -FIXES = list(itertools.combinations(reversed(STATUSES), 2)) + \ - list(itertools.combinations(list(reversed(PROBLEMS)) + ["skip"], 2)) - -# The statuses that don't cause changes when transitioning from one another -NO_OPS = ('skip', 'notrun') - - -@utils.nose.no_error -def initialize_status(): - """status.Status: class inializes""" - status.Status('test', 1) - - -@utils.nose.no_error -def initialize_nochangestatus(): - """status.NoChangeStatus: class initializes""" - status.NoChangeStatus('test') - - -@utils.nose.generator -def test_gen_lookup(): - """ Generator that attempts to do a lookup on all statuses """ - @utils.nose.no_error - def test(status_): - status.status_lookup(status_) - - for stat in STATUSES + ['skip', 'notrun']: - test.description = \ - "status.status_lookup: can lookup '{}' as expected".format(stat) - yield test, stat - - -@nt.raises(status.StatusException) -def test_bad_lookup(): - """status.status_lookup: An unexepcted value raises a StatusException""" - status.status_lookup('foobar') - - -def test_status_in(): - """status.Status: A status can be looked up with 'x in y' synatx""" - stat = status.PASS - slist = ['pass'] - - nt.ok_(stat in slist) - - -@utils.nose.generator -def test_is_regression(): - """ Generate all tests for regressions """ - def is_regression(new, old): - """ Test that old -> new is a regression """ - nt.ok_(status.status_lookup(new) < status.status_lookup(old)) - - for new, old in REGRESSIONS: - is_regression.description = \ - "status.Status: '{}' -> '{}' is a regression as expected".format( - old, new) - yield is_regression, new, old - - -@utils.nose.generator -def test_is_fix(): - """ Generates all tests for fixes """ - def is_fix(new, old): - """ Test that new -> old is a fix """ - nt.ok_(status.status_lookup(new) > status.status_lookup(old)) - - for new, old in FIXES: - is_fix.description = \ - "status.Status: '{}' -> '{}' is a fix as expected".format( - new, old) - yield is_fix, new, old - - -@utils.nose.generator -def test_is_change(): - """ Test that status -> !status is a change """ - def is_not_equivalent(new, old): - """ Test that new != old """ - nt.ok_(status.status_lookup(new) != status.status_lookup(old)) - - for new, old in itertools.permutations(STATUSES, 2): - is_not_equivalent.description = \ - "status.Status: '{}' -> '{}' is a change as expected".format( - new, old) - yield is_not_equivalent, new, old - - -@utils.nose.generator -def test_not_change(): - """ Skip and NotRun should not count as changes """ - def check_not_change(new, old): - """ Check that a status doesn't count as a change - - This checks that new < old and old < new do not return true. This is meant - for checking skip and notrun, which we don't want to show up as regressions - and fixes, but to go in their own special categories. - - """ - nt.assert_false(new < old, - msg="{new} -> {old}, is a change " - "but shouldn't be".format(**locals())) - nt.assert_false(new > old, - msg="{new} <- {old}, is a change " - "but shouldn't be".format(**locals())) - - for nochange, stat in itertools.permutations(NO_OPS, 2): - check_not_change.description = \ - "status.Status: {0} -> {1} is not a change".format(nochange, stat) - yield (check_not_change, status.status_lookup(nochange), - status.status_lookup(stat)) - - -@utils.nose.generator -def test_max_statuses(): - """ Verify that max() works between skip and non-skip statuses """ - def _max_nochange_stat(nochange, stat): - """ max(nochange, stat) should = stat """ - nt.assert_equal( - stat, max(nochange, stat), - msg="max({nochange}, {stat}) = {stat}".format(**locals())) - - def _max_stat_nochange(nochange, stat): - """ max(stat, nochange) should = stat """ - nt.assert_equal( - stat, max(stat, nochange), - msg="max({stat}, {nochange}) = {stat}".format(**locals())) - - for nochange, stat in itertools.product(NO_OPS, STATUSES): - nochange = status.status_lookup(nochange) - stat = status.status_lookup(stat) - _max_nochange_stat.description = \ - "status.Status: max({nochange}, {stat}) = {stat}".format(**locals()) - yield _max_nochange_stat, nochange, stat - - _max_stat_nochange.description = \ - "status.Status: max({stat}, {nochange}) = {stat}".format(**locals()) - yield _max_stat_nochange, nochange, stat - - -def check_operator(obj, op, result): - """ Test that the result of running an operator on an object is expected - - Arguments: - obj -- an instance to test - operator -- the operator to test on the object - result -- the expected result - - """ - nt.assert_equal(op(obj), result) - - -def check_operator_equal(obj, comp, op, result): - """ Test that the result of running an operator on an object is expected - - Arguments: - obj -- an instance to test - operator -- the operator to test on the object - result -- the expected result - - """ - nt.assert_equal(op(obj, comp), result) - - -def check_operator_not_equal(obj, comp, op, result): - """ Test that the result of running an operator on an object is expected - - Arguments: - obj -- an instance to test - operator -- the operator to test on the object - result -- the expected result - - """ - nt.assert_not_equal(op(obj, comp), result) - - -@utils.nose.generator -def test_nochangestatus_magic(): - """ Test that operators unique to NoChangeStatus work """ - obj = status.NoChangeStatus('Test') - stat = status.Status('Test', 0, (0, 0)) - - # generator equality tests - for comp, type_ in [(obj, 'status.NoChangeStatus'), - (stat, 'status.Status'), - ('Test', 'unicode')]: - check_operator_equal.description = ( - 'Status.NoChangeStatus: Operator eq works with type: {}'.format(type_) - ) - yield check_operator_equal, obj, comp, lambda x, y: x.__eq__(y), True - - check_operator_not_equal.description = ( - 'status.NoChangeStatus: Operator ne works with type: {}'.format(type_) - ) - yield check_operator_not_equal, obj, comp, lambda x, y: x.__ne__(y), True - - -@utils.nose.generator -def test_status_magic(): - """ Generator for testing magic methods in the Status class """ - obj = status.Status('foo', 0, (0, 0)) - comparitor = status.Status('bar', 10, (0, 0)) - - for func, name, result in [ - (six.text_type, 'unicode', 'foo'), - (repr, 'repr', 'foo'), - (int, 'int', 0)]: - check_operator.description = \ - 'status.Status: Operator {} works'.format(name) - yield check_operator, obj, func, result - - for func, name in [ - (lambda x, y: x.__lt__(y), 'lt'), - (lambda x, y: y.__gt__(x), 'gt')]: - - check_operator_equal.description = \ - 'status.Status: Operator {0} works when True'.format(name) - yield check_operator_equal, obj, comparitor, func, True - - for func, name in [ - (lambda x, y: x.__le__(x), 'le, when ='), - (lambda x, y: x.__le__(y), 'le, when !='), - (lambda x, y: x.__eq__(x), 'eq'), - (lambda x, y: x.__ge__(x), 'ge, when ='), - (lambda x, y: y.__ge__(x), 'ge, when !='), - (lambda x, y: x.__ne__(y), 'ne'), - (lambda x, y: x.__eq__(x), 'eq')]: - check_operator_not_equal.description = \ - 'status.Status: Operator {0} works when False'.format(name) - yield check_operator_not_equal, obj, comparitor, func, False - - -@nt.raises(TypeError) -def test_status_eq_raises(): - """status.Status: eq comparison to uncomparable object results in TypeError""" - status.PASS == dict() - - -@nt.raises(TypeError) -def test_nochangestatus_eq_raises(): - """status.NoChangeStatus: eq comparison to uncomparable type results in TypeError""" - status.NOTRUN == dict() - - -@nt.raises(TypeError) -def test_nochangestatus_ne_raises(): - """status.NoChangeStatus: ne comparison to uncomparable type results in TypeError""" - status.NOTRUN != dict() |