# Copyright 2013-2016 Intel Corporation # Copyright 2013, 2014 Advanced Micro Devices # Copyright 2014 VMWare # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """Shared functions for summary generation.""" from __future__ import ( absolute_import, division, print_function, unicode_literals ) import re import operator import six from six.moves import zip # a local variable status exists, prevent accidental overloading by renaming # the module import framework.status as so from framework.core import lazy_property from framework import grouptools class Results(object): # pylint: disable=too-few-public-methods """Container object for results. Has the results, the names of status, and the counts of statuses. """ def __init__(self, results): self.results = results self.names = Names(self) self.counts = Counts(self) def get_result(self, name): """Get all results for a single test. Replace any missing vaules with status.NOTRUN, correctly handles subtests. """ results = [] for res in self.results: try: results.append(res.get_result(name)) except KeyError: results.append(so.NOTRUN) return results class Names(object): """Class containing names of tests for various statuses. Members contain lists of sets of names that have a status. Each status is lazily evaluated and cached. """ def __init__(self, tests): self.__results = tests.results def __diff(self, comparator, handler=None): """Helper for simplifying comparators using find_diffs.""" ret = [''] if handler is None: ret.extend(find_diffs(self.__results, self.all, comparator)) else: ret.extend(find_diffs(self.__results, self.all, comparator, handler=handler)) return ret def __single(self, comparator): """Helper for simplifying comparators using find_single.""" return find_single(self.__results, self.all, comparator) @lazy_property def all(self): """A set of all tests in all runs.""" all_ = set() for res in self.__results: for key, value in six.iteritems(res.tests): if not value.subtests: all_.add(key) else: for subt in six.iterkeys(value.subtests): all_.add(grouptools.join(key, subt)) return all_ @lazy_property def changes(self): def handler(names, name, prev, cur): """Handle missing tests. For changes we want literally anything where the first result isn't the same as the second result. """ def _get(res): try: return res.get_result(name) except KeyError: return so.NOTRUN # Add any case of a != b except skip <-> notrun cur = _get(cur) prev = _get(prev) if cur != prev and {cur, prev} != {so.SKIP, so.NOTRUN}: names.add(name) return self.__diff(operator.ne, handler=handler) @lazy_property def problems(self): return self.__single(lambda x: x > so.PASS) @lazy_property def skips(self): # It is critical to use is not == here, otherwise so.NOTRUN will also # be added to this list return self.__single(lambda x: x is so.SKIP) @lazy_property def regressions(self): # By ensureing tha min(x, y) is >= so.PASS we eleminate NOTRUN and SKIP # from these pages return self.__diff(lambda x, y: x < y and min(x, y) >= so.PASS) @lazy_property def fixes(self): # By ensureing tha min(x, y) is >= so.PASS we eleminate NOTRUN and SKIP # from these pages return self.__diff(lambda x, y: x > y and min(x, y) >= so.PASS) @lazy_property def enabled(self): def handler(names, name, prev, cur): if _result_in(name, cur) and not _result_in(name, prev): names.add(name) return self.__diff( lambda x, y: x is so.NOTRUN and y is not so.NOTRUN, handler=handler) @lazy_property def disabled(self): def handler(names, name, prev, cur): if _result_in(name, prev) and not _result_in(name, cur): names.add(name) return self.__diff( lambda x, y: x is not so.NOTRUN and y is so.NOTRUN, handler=handler) @lazy_property def incomplete(self): return self.__single(lambda x: x is so.INCOMPLETE) @lazy_property def all_changes(self): if len(self.changes) > 1: return set.union(*self.changes[1:]) else: return set() @lazy_property def all_disabled(self): if len(self.disabled) > 1: return set.union(*self.disabled[1:]) else: return set() @lazy_property def all_enabled(self): if len(self.enabled) > 1: return set.union(*self.enabled[1:]) else: return set() @lazy_property def all_fixes(self): if len(self.fixes) > 1: return set.union(*self.fixes[1:]) else: return set() @lazy_property def all_regressions(self): if len(self.regressions) > 1: return set.union(*self.regressions[1:]) else: return set() @lazy_property def all_incomplete(self): if len(self.incomplete) > 1: return set.union(*self.incomplete) else: return self.incomplete[0] @lazy_property def all_problems(self): if len(self.problems) > 1: return set.union(*self.problems) else: return self.problems[0] @lazy_property def all_skips(self): if len(self.skips) > 1: return set.union(*self.skips) else: return self.skips[0] class Counts(object): """Number of tests in each category.""" def __init__(self, tests): self.__names = tests.names @lazy_property def all(self): return len(self.__names.all) @lazy_property def changes(self): return [len(x) for x in self.__names.changes] @lazy_property def problems(self): return [len(x) for x in self.__names.problems] @lazy_property def skips(self): return [len(x) for x in self.__names.skips] @lazy_property def regressions(self): return [len(x) for x in self.__names.regressions] @lazy_property def fixes(self): return [len(x) for x in self.__names.fixes] @lazy_property def enabled(self): return [len(x) for x in self.__names.enabled] @lazy_property def disabled(self): return [len(x) for x in self.__names.disabled] @lazy_property def incomplete(self): return [len(x) for x in self.__names.incomplete] def escape_filename(key): """Avoid reserved characters in filenames.""" return re.sub(r'[<>:"|?*#]', '_', key) def escape_pathname(key): """ Remove / and \\ from names """ return re.sub(r'[/\\]', '_', key) def _result_in(name, result): """If a result (or a subtest result) exists return True, else False.""" try: # This is a little hacky, but I don't know of a better way where we # ensure the value is truthy _ = result.get_result(name) return True except KeyError: return False def find_diffs(results, tests, comparator, handler=lambda *a: None): """Generate diffs between two or more sets of results. Arguments: results -- a list of results.TestrunResult instances tests -- an iterable of test names. Must be iterable more than once comparator -- a function with the signautre f(x, y), that returns True when the test should be added to the set of diffs Keyword Arguemnts: handler -- a function with the signature f(names, name, prev, cur). in the event of a KeyError while comparing the results with comparator, handler will be passed the (, , , ). This can be used to add name even when a KeyError is expected (ie, enabled tests). Default: pass """ diffs = [] # There can't be changes from nil -> 0 for prev, cur in zip(results[:-1], results[1:]): names = set() for name in tests: try: if comparator(prev.get_result(name), cur.get_result(name)): names.add(name) except KeyError: handler(names, name, prev, cur) diffs.append(names) return diffs def find_single(results, tests, func): """Find statuses in a single run.""" statuses = [] for res in results: names = set() for name in tests: try: if func(res.get_result(name)): names.add(name) except KeyError: pass statuses.append(names) return statuses