#!/usr/bin/env python3 # # Copyright © 2012 Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), # to deal in the Software without restriction, including without limitation # the rights to use, copy, modify, merge, publish, distribute, sublicense, # and/or sell copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice (including the next # paragraph) shall be included in all copies or substantial portions of the # Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from argparse import ArgumentParser import os import os.path as path import re import sys from framework.database import ResultDatabase def readfile(filename): with open(filename) as f: return f.read() def writefile(filename, text): with open(filename, "w") as f: f.write(text) templateDir = path.join(path.dirname(path.realpath(__file__)), 'templates') templates = { 'index': readfile(path.join(templateDir, 'index.html')), 'detail': readfile(path.join(templateDir, 'detail.html')), } ############################################################################# ##### Vector indicating the number of subtests that have passed/failed/etc. ############################################################################# class PassVector: def __init__(self, p, f, s, c, t, h): self.passnr = p self.failnr = f self.skipnr = s self.crashnr = c self.timeoutnr = t self.hangnr = h def add(self, o): self.passnr += o.passnr self.failnr += o.failnr self.skipnr += o.skipnr self.crashnr += o.crashnr self.timeoutnr += o.timeoutnr self.hangnr += o.hangnr # Do not count skips def totalRun(self): return self.passnr + self.failnr + self.crashnr + self.timeoutnr + self.hangnr def toPassVector(status): vectormap = { 'pass': PassVector(1,0,0,0,0,0), 'fail': PassVector(0,1,0,0,0,0), 'skip': PassVector(0,0,1,0,0,0), 'crash': PassVector(0,0,0,1,0,0), 'timeout': PassVector(0,0,0,0,1,0), 'hang': PassVector(0,0,0,0,0,1) } return vectormap[status if status else 'skip'] ############################################################################# ##### Helper functions ############################################################################# filename_char_re = re.compile(r'[^a-zA-Z0-9_]+') def escape(s): return filename_char_re.sub('', s.replace('/', '__')) ############################################################################# ##### Test filtering predicates ##### ##### These take a list of statuses (i.e. ['pass', 'fail']. ############################################################################# def broken(rs): return not all(r == 'pass' or r == 'skip' or r is None for r in rs) def changed(rs): return any(rs[0] != r for r in rs) def skipped(rs): return any(r == 'skip' for r in rs) def regressed(rs): return any(worseThan(a, b) for (a, b) in zip(rs, rs[1:])) def fixed(rs): return regressed(list(reversed(rs))) # Helper function: return True if status a is more severe than status b. def worseThan(a, b): statuses = ['hang', 'timeout', 'crash', 'fail', 'skip', 'pass'] if not a or a == 'skip' or not b or b == 'skip': return False return statuses.index(b) < statuses.index(a) def todo(rs): return any(r is None for r in rs) def actually_run(rs): return any(r is not None for r in rs) ############################################################################# ##### Summary page generation ############################################################################# pages = [ ('All', 'index.html', actually_run), ('Changes', 'changes.html', changed), ('Fixes', 'fixes.html', fixed), ('Problems', 'problems.html', broken), ('Skipped', 'skipped.html', skipped), ('Regressions', 'regressions.html', regressed), ('Scheduled', 'notrun.html', todo), ] def build_navbox(current_page): def link(to, filename): if to == current_page: return '%s' % to return '%s' % (filename, to) return ''.join([link(p[0], p[1]) for p in pages]) def testResult(run_name, full_name, status): if interesting(status): html = '%(status)s' % { 'status': status if status else 'todo', 'link': path.join(escape(run_name), detailFile(full_name)) } else: html = '%(status)s' % { 'status': status if status else 'todo', } return html class StackEntry: def __init__(self, num_runs, group_name): self.name = group_name self.results = [PassVector(0,0,0,0,0,0) for i in range(num_runs)] self.name_html = '' self.column_html = ['' for i in range(num_runs)] def buildGroupResultHeader(p): if p.hangnr > 0: status = 'hang' elif p.timeoutnr > 0: status = 'timeout' elif p.crashnr > 0: status = 'crash' elif p.failnr > 0: status = 'fail' elif p.passnr > 0: status = 'pass' else: status = 'skip' totalnr = p.totalRun() passnr = p.passnr return '
%(pass)d/%(total)d
' % { 'status': status, 'total': p.totalRun(), 'pass': p.passnr } def buildTable(run_names, results): # If the test list is empty, just return now. if not results: return ('', ['']) num_runs = len(run_names) last_group = '' stack = [] def openGroup(name): stack.append(StackEntry(num_runs, name)) def closeGroup(): group = stack.pop() stack[-1].name_html += ''.join(['
', group.name, '
', group.name_html, '
']) for i in range(num_runs): stack[-1].results[i].add(group.results[i]) stack[-1].column_html[i] += ''.join(['
', buildGroupResultHeader(group.results[i]), group.column_html[i], '
']) openGroup('fake') openGroup('All') for full_test in sorted(results.keys()): group, test = path.split(full_test) # or full_test.rpartition('/') if group != last_group: # We're in a different group now. Close the old ones # and open the new ones. for x in path.relpath(group, last_group).split('/'): if x == '..': closeGroup() else: openGroup(x) last_group = group # Add the current test stack[-1].name_html += '
' + test + '
\n'; for i in range(num_runs): passv = toPassVector(results[full_test][i]['result']) html = testResult(run_names[i], full_test, results[full_test][i]['result']) stack[-1].results[i].add(passv) stack[-1].column_html[i] += html # Close any remaining groups while len(stack) > 1: closeGroup() assert(len(stack) == 1) return (stack[0].name_html, stack[0].column_html) def writeSummaryHtml(run_names, results, resultsDir, page, filename): names, columns = buildTable(run_names, results) def makeColumn(name, contents): return ''.join(['
%s' % (escape(name), name), contents, '
']) column_html = ''.join([makeColumn(name, contents) for name, contents in zip(run_names, columns)]) group = '
%(name)s' + names + '
' filename = path.join(resultsDir, filename) writefile(filename, templates['index'] % { 'page': page.title(), 'showlinks': build_navbox(page), 'group': group, 'columns': column_html }) ############################################################################# ##### Detail page generation ############################################################################# def detailFile(test): return 'detail_' + escape(test) + '.html' def interesting(r): return r is not None # Create result.html pages containing the test result details for a # single testrun. def writeDetailPages(dirname, run_name, results, col): for test in results.keys(): r = results[test][col] if r and interesting(r['result']): writefile(path.join(dirname, detailFile(test)), templates['detail'] % dict(r, run=run_name, esc_test=escape(test), test=test)) ############################################################################# ##### Main program ############################################################################# def getCombinedResults(db, run_names, intersect): # Sadly, SQLite can't do full outer joins, so we have to query the results # for each run individually and reassemble them here. individual_results = dict([(run, db.getResults(run)) for run in run_names]) # Get a set containing all test names (in slash-separated form). # Some runs may have more tests than others; take the union. if intersect: test_names = set.intersection(*[set(individual_results[run].keys()) for run in run_names]) else: test_names = set.union(*[set(individual_results[run].keys()) for run in run_names]) results = {} for test in test_names: results[test] = [] for run in run_names: results[test].append(individual_results[run].get(test, None)) return results def parseArguments(argv, config): p = ArgumentParser(prog='robyn report', description='A GPU test runner') p.add_argument('-o', '--output', default='summary', metavar='') p.add_argument('-i', '--intersect', default=False, action='store_true', help='Take the intersection of the test sets rather than the union.') p.add_argument('runs', nargs='+', metavar='') # XXX: alternate database (pending refactoring) return p.parse_args(argv) def main(argv, config): args = parseArguments(argv, config) db = ResultDatabase(config) reportDir = args.output if not path.exists(reportDir): os.makedirs(reportDir) run_names = list(args.runs) results = getCombinedResults(db, run_names, args.intersect) for col in range(len(run_names)): dirname = path.join(reportDir, escape(run_names[col])) os.mkdir(dirname) # XXX: write system info writeDetailPages(dirname, run_names[col], results, col) def writeSummaryPage(page, filename, filterFunc = None): cut_results = {} for t in results.keys(): if filterFunc is None or filterFunc([r['result'] for r in results[t]]): cut_results[t] = results[t] writeSummaryHtml(run_names, cut_results, reportDir, page, filename) os.link(path.join(templateDir, 'index.css'), path.join(reportDir, 'index.css')) os.link(path.join(templateDir, 'detail.css'), path.join(reportDir, 'detail.css')) for p in pages: writeSummaryPage(p[0], p[1], p[2]) if __name__ == "__main__": main()