diff options
| -rw-r--r-- | framework/programs/summary.py | 37 | ||||
| -rw-r--r-- | framework/summary/__init__.py | 2 | ||||
| -rw-r--r-- | framework/summary/feature.py | 99 | ||||
| -rw-r--r-- | framework/summary/html_.py | 23 | ||||
| -rwxr-xr-x | piglit | 4 | ||||
| -rw-r--r-- | templates/feature.mako | 73 |
6 files changed, 236 insertions, 2 deletions
diff --git a/framework/programs/summary.py b/framework/programs/summary.py index 8711ee54c..b23f1ef31 100644 --- a/framework/programs/summary.py +++ b/framework/programs/summary.py @@ -35,6 +35,7 @@ __all__ = [ 'console', 'csv', 'html', + 'feature' ] @@ -224,3 +225,39 @@ def aggregate(input_): print("Aggregated file written to: {}.{}".format( outfile, backends.compression.get_mode())) + + +@exceptions.handler +def feature(input_): + parser = argparse.ArgumentParser() + parser.add_argument("-o", "--overwrite", + action="store_true", + help="Overwrite existing directories") + parser.add_argument("featureFile", + metavar="<Feature json file>", + help="Json file containing the features description") + parser.add_argument("summaryDir", + metavar="<Summary Directory>", + help="Directory to put HTML files in") + parser.add_argument("resultsFiles", + metavar="<Results Files>", + nargs="*", + help="Results files to include in HTML") + args = parser.parse_args(input_) + + # If args.list and args.resultsFiles are empty, then raise an error + if not args.featureFile and not args.resultsFiles: + raise parser.error("Missing required option -l or <resultsFiles>") + + # If args.list and args.resultsFiles are empty, then raise an error + if not args.resultsFiles or not path.exists(args.featureFile): + raise parser.error("Missing json file") + + # if overwrite is requested delete the output directory + if path.exists(args.summaryDir) and args.overwrite: + shutil.rmtree(args.summaryDir) + + # If the requested directory doesn't exist, create it or throw an error + core.checkDir(args.summaryDir, not args.overwrite) + + summary.feat(args.resultsFiles, args.summaryDir, args.featureFile) diff --git a/framework/summary/__init__.py b/framework/summary/__init__.py index 8a1ff8a79..0f0f144f0 100644 --- a/framework/summary/__init__.py +++ b/framework/summary/__init__.py @@ -25,5 +25,5 @@ # public parts here, so that we have a nice interface to work with. from __future__ import absolute_import, division, print_function -from .html_ import html +from .html_ import html, feat from .console_ import console diff --git a/framework/summary/feature.py b/framework/summary/feature.py new file mode 100644 index 000000000..2fb8f50e3 --- /dev/null +++ b/framework/summary/feature.py @@ -0,0 +1,99 @@ +# Copyright (c) 2015 Intel Corporation + +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# This permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from __future__ import print_function, division, absolute_import + +import copy + +try: + import simplejson as json +except ImportError: + import json + +from framework import core, exceptions, profile, status + + +class FeatResults(object): # pylint: disable=too-few-public-methods + """Container object for results. + + Has the results, feature profiles and feature computed results. + + """ + def __init__(self, results, json_file): + + with open(json_file) as data: + feature_data = json.load(data) + + self.feat_fractions = {} + self.feat_status = {} + self.features = set() + self.results = results + + profiles = {} + + # we expect all the result sets to be for the same profile + profile_orig = profile.load_test_profile(results[0].options['profile'][0]) + + for feature in feature_data: + self.features.add(feature) + + incl_str = feature_data[feature]["include_tests"] + excl_str = feature_data[feature]["exclude_tests"] + + include_filter = [incl_str] if incl_str and not incl_str.isspace() else [] + exclude_filter = [excl_str] if excl_str and not excl_str.isspace() else [] + + opts = core.Options(include_filter=include_filter, + exclude_filter=exclude_filter) + + profiles[feature] = copy.deepcopy(profile_orig) + + # An empty list will raise PiglitFatalError exception + # But for reporting we need to handle this situation + try: + profiles[feature]._prepare_test_list(opts) + except exceptions.PiglitFatalError: + pass + + for results in self.results: + self.feat_fractions[results.name] = {} + self.feat_status[results.name] = {} + + for feature in feature_data: + result_set = set(results.tests) + profile_set = set(profiles[feature].test_list) + + common_set = profile_set & result_set + passed_list = [x for x in common_set if results.tests[x].result == status.PASS] + + total = len(common_set) + passed = len(passed_list) + + self.feat_fractions[results.name][feature] = (passed, total) + if total == 0: + self.feat_status[results.name][feature] = status.NOTRUN + else: + if 100 * passed // total >= feature_data[feature]["target_rate"]: + self.feat_status[results.name][feature] = status.PASS + else: + self.feat_status[results.name][feature] = status.FAIL diff --git a/framework/summary/html_.py b/framework/summary/html_.py index 12172b75c..60d7f5e60 100644 --- a/framework/summary/html_.py +++ b/framework/summary/html_.py @@ -37,9 +37,11 @@ from mako.lookup import TemplateLookup from framework import backends, exceptions from .common import Results, escape_filename, escape_pathname +from .feature import FeatResults __all__ = [ 'html', + 'feat' ] _TEMP_DIR = os.path.join( @@ -62,8 +64,9 @@ def _copy_static_files(destination): os.path.join(destination, "result.css")) -def _make_testrun_info(results, destination, exclude): +def _make_testrun_info(results, destination, exclude=None): """Create the pages for each results file.""" + exclude = exclude or {} result_css = os.path.join(destination, "result.css") index = os.path.join(destination, "index.html") @@ -146,6 +149,14 @@ def _make_comparison_pages(results, destination, exclude): page=page, pages=pages)) +def _make_feature_info(results, destination): + """Create the feature readiness page.""" + + with open(os.path.join(destination, "feature.html"), 'w') as out: + out.write(_TEMPLATES.get_template('feature.mako').render( + results=results)) + + def html(results, destination, exclude): """ Produce HTML summaries. @@ -161,3 +172,13 @@ def html(results, destination, exclude): _copy_static_files(destination) _make_testrun_info(results, destination, exclude) _make_comparison_pages(results, destination, exclude) + + +def feat(results, destination, feat_desc): + """Produce HTML feature readiness summary.""" + + feat_res = FeatResults([backends.load(i) for i in results], feat_desc) + + _copy_static_files(destination) + _make_testrun_info(feat_res, destination) + _make_feature_info(feat_res, destination) @@ -139,6 +139,10 @@ def main(): add_help=False, help="Aggregate incomplete piglit run.") aggregate.set_defaults(func=summary.aggregate) + feature = summary_parser.add_parser('feature', + add_help=False, + help="generate feature readiness html report.") + feature.set_defaults(func=summary.feature) # Parse the known arguments (piglit run or piglit summary html for # example), and then pass the arguments that this parser doesn't know about diff --git a/templates/feature.mako b/templates/feature.mako new file mode 100644 index 000000000..ac9bc8677 --- /dev/null +++ b/templates/feature.mako @@ -0,0 +1,73 @@ +<%! + import posixpath # this must be posixpath, since we want /'s not \'s + import re + + + def feat_result(result): + """Percentage result string""" + return '{}/{}'.format(result[0], result[1]) + + + def escape_filename(key): + """Avoid reserved characters in filenames.""" + return re.sub(r'[<>:"|?*#]', '_', key) + + + def escape_pathname(key): + """ Remove / and \\ from names """ + return re.sub(r'[/\\]', '_', key) + + + def normalize_href(href): + """Force backward slashes in URLs.""" + return href.replace('\\', '/') +%> + +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <title>Result summary</title> + <link rel="stylesheet" href="index.css" type="text/css" /> + </head> + <body> + <h1>Feature readiness</h1> + <table> + <colgroup> + ## Name Column + <col /> + + ## Status columns + ## Create an additional column for each summary + % for _ in xrange(len(results.results)): + <col /> + % endfor + </colgroup> + <tr> + <th/> + % for res in results.results: + <th class="head"><b>${res.name}</b><br />\ + (<a href="${normalize_href(posixpath.join(escape_pathname(res.name), 'index.html'))}">info</a>)</th> + % endfor + </tr> + % for feature in results.features: + <tr> + ## Add the left most column, the feature name + <td> + <div class="group"> + <b>${feature}</b> + </div> + </td> + ## add the feature totals + % for res in results.results: + <td class="${results.feat_status[res.name][feature]}"> + <b>${feat_result(results.feat_fractions[res.name][feature])}</b> + </td> + % endfor + </tr> + % endfor + </table> + </body> +</html> |