diff options
Diffstat (limited to 'stats/gen_report.py')
| -rwxr-xr-x | stats/gen_report.py | 287 |
1 files changed, 287 insertions, 0 deletions
diff --git a/stats/gen_report.py b/stats/gen_report.py new file mode 100755 index 0000000..38a7c8d --- /dev/null +++ b/stats/gen_report.py @@ -0,0 +1,287 @@ +#!/usr/bin/python3 + +import matplotlib.pyplot as plt +from numpy import * +import subprocess +import argparse +import glob +import csv +import sys +import os + +# constants +html_name="index.html" +report_folder="ezbench_report/" + +class Benchmark: + def __init__(self, full_name): + self.full_name = full_name + self.prevValue = -1 + +class BenchResult: + def __init__(self, commit, benchmark, data_raw_file, img_src_name): + self.commit = commit + self.benchmark = benchmark + self.data_raw_file = data_raw_file + self.img_src_name = img_src_name + self.data = [] + +class Commit: + def __init__(self, sha1, full_name, compile_log): + self.sha1 = sha1 + self.full_name = full_name + self.compile_log = compile_log + self.results = [] + +benchmarks = [] +commits = [] + +# parse the options +parser = argparse.ArgumentParser() +parser.add_argument("log_folder") +args = parser.parse_args() + +# Look for the commit_list file +os.chdir(args.log_folder) + +try: + f = open( "commit_list", "r") + try: + commitsLines = f.readlines() + finally: + f.close() +except IOError: + sys.stderr.write("The log folder '{0}' does not contain a commit_list file\n".format(args.log_folder)) + sys.exit(1) + +# Check that there are commits +if (len(commitsLines) == 0): + sys.stderr.write("The commit_list file is empty\n") + sys.exit(2) + +# Gather all the information from the commits and generate the images +commits_txt = "" +table_entries_txt = "" +for commitLine in commitsLines: + full_name = commitLine.strip(' \t\n\r') + sha1 = commitLine.split()[0] + compile_log = sha1 + "_compile_log" + commit = Commit(sha1, full_name, compile_log) + + # find all the benchmarks + benchFiles = glob.glob("{sha1}_bench_*".format(sha1=commit.sha1)); + benchs_txt = "" + for benchFile in benchFiles: + # Get the bench name + bench_name = benchFile.replace("{sha1}_bench_".format(sha1=commit.sha1), "") + if bench_name.endswith(".png"): + continue + + # Find the right Benchmark or create one if none are found + try: + benchmark = next(b for b in benchmarks if b.full_name == bench_name) + except StopIteration: + benchmark = Benchmark(bench_name) + benchmarks.append(benchmark) + + # Create the result object + result = BenchResult(commit, benchmark, benchFile, report_folder + benchFile + ".png") + + # Read the data + with open(benchFile, 'rt') as f: + if (csv.Sniffer().has_header(f.read(1024))): + f.seek(0) + next(f) + else: + f.seek(0) + reader = csv.reader(f) + try: + for row in reader: + result.data.append(float(row[0])) + except csv.Error as e: + sys.stderr.write('file %s, line %d: %s' % (benchFile, reader.line_num, e)) + sys.exit(3) + + # Add the result to the commit's results + commit.results.append(result) + + # Add the commit to the list of commits + commit.results = sorted(commit.results, key=lambda res: res.benchmark.full_name) + commits.append(commit) + +# Sort the list of benchmarks +benchmarks = sorted(benchmarks, key=lambda bench: bench.full_name) + +def getResultsBenchmarkDiffs(benchmark): + prevValue = -1 + results = [] + + # Compute a report per application + i = 0 + for commit in commits: + for result in commit.results: + if result.benchmark != benchmark: + continue + + value = array(result.data).mean() + + if prevValue >= 0: + diff = (value * 100.0 / prevValue) - 100.0 + else: + diff = 0 + prevValue = value + + results.append([i, diff]) + i = i + 1 + + return results + +# Create a folder for the results +try: + os.mkdir(report_folder) +except OSError: + print ("Error while creating the report folder") + +# Generate the trend graph +plt.figure(figsize=(15,3)) +plt.xlabel('Commit #') +plt.ylabel('Perf. diff. with the prev. commit (%)') +plt.grid(True) +for i in range(len(benchmarks)): + data = getResultsBenchmarkDiffs(benchmarks[i]) + + x_val = [x[0] for x in data] + y_val = [x[1] for x in data] + + plt.plot(x_val, y_val, label=benchmarks[i].full_name) + +#plt.xticks(range(len(x)), x_val, rotation='vertical') +plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, + ncol=3, mode="expand", borderaxespad=0.) +plt.savefig(report_folder + 'overview.svg', bbox_inches='tight') + +# Generate the images (HACK, do that in python!) +for commit in commits: + for result in commit.results: + subprocess.call(['../../stats/test_report.R', result.data_raw_file, result.img_src_name]) + + +# Generate the report +html_template=""" +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> + +<html xmlns="http://www.w3.org/1999/xhtml"> + + <head> + <title>Performace report on the run named '{run_name}'</title> + </head> + + <body> + <h1>Performace report on the run named '{run_name}'</h1> + + <h2>Trends</h2> + + <center><img src="{report_folder}/overview.svg" alt="Trends"/></center> + + <h2>Stats</h2> + + <table border="1" style=""> + <tr> + <th>Commit SHA1</th> + <th>Geometric mean</th> + {tbl_hdr_benchmarks} + </tr> + {tbl_entries} + </table> + + <h2>Commits</h2> + {commits} + </body> + +</html> +""" + +table_commit_template=""" + <tr> + <td><a href="#commit_{sha1}">{sha1}</a></td> + <td>{geom_mean}</td> + {tbl_res_benchmarks} + </tr> +""" + +table_entry_template=""" +<td bgcolor="{color}"> + <a href="#commit_{sha1}_bench_{bench_name}">{value:.2f} ({diff:.2f} %)<a/> +</td>""" + +commit_template=""" + <h3 id="commit_{sha1}">{commit}</h3> + Here is the <a href="{compile_log}">compilation logs</a> and list of benchmarks found for commit {sha1}: + {benchs}""" + +bench_template=""" + <h4 id="commit_{sha1}_bench_{bench_name}">{bench_name}</h4> + + <a href="{raw_data_file}">Original data</a> + + <img src="{img_src}" alt="Test's time series and density of probability" />""" + +# For all commits +commits_txt = "" +tbl_entries_txt = "" +for commit in commits: + benchs_txt = "" + tbl_res_benchmarks = "" + for result in commit.results: + value = array(result.data).mean() + + if result.benchmark.prevValue > 0: + diff = (value * 100.0 / result.benchmark.prevValue) - 100.0 + else: + diff = 0 + result.benchmark.prevValue = value + + if diff < -1.5: + color = "#FF0000" + elif diff > 1.5: + color = "#00FF00" + else: + color = "#FFFFFF" + + # Generate the html + benchs_txt += bench_template.format(sha1=commit.sha1, + bench_name=result.benchmark.full_name, + img_src=result.img_src_name, + raw_data_file=result.data_raw_file) + + + tbl_res_benchmarks += table_entry_template.format(sha1=commit.sha1, + bench_name=result.benchmark.full_name, + value=value, + diff=diff, + color=color) + + # generate the html + tbl_entries_txt += table_commit_template.format(sha1=commit.sha1, geom_mean=0, + tbl_res_benchmarks=tbl_res_benchmarks) + commits_txt += commit_template.format(commit=commit.full_name, + sha1=commit.sha1, + benchs=benchs_txt, + compile_log=commit.compile_log) + +# generate the table's header +tbl_hdr_benchmarks = "" +for benchmark in benchmarks: + tbl_hdr_benchmarks += "<th>{benchmark}</th>\n".format(benchmark=benchmark.full_name) + +# Generate the final html file +html = html_template.format(run_name=args.log_folder, + commits=commits_txt, + tbl_entries=tbl_entries_txt, + tbl_hdr_benchmarks=tbl_hdr_benchmarks, + report_folder=report_folder); + +with open(html_name, 'w') as f: + f.write(html) + print("Output HTML generated at: {0}/{1}".format(os.getcwd(), html_name)) |