diff options
| -rw-r--r-- | tko/nightly.py | 1086 |
1 files changed, 748 insertions, 338 deletions
diff --git a/tko/nightly.py b/tko/nightly.py index c431829a..5a08608b 100644 --- a/tko/nightly.py +++ b/tko/nightly.py @@ -1,4 +1,4 @@ -# tko/nightly.py code shared by various *_nightly.cgi scripts +# tko/nightly.py code shared by various tko/*.cgi graphing scripts import cgi, cgitb import os, sys @@ -7,370 +7,780 @@ from autotest_lib.tko import db, plotgraph, perf from autotest_lib.client.common_lib import kernel_versions -def nightly_views(suite_notes, kernel_legend, benchmarks, - released_kernel_series, - nightly_kernel_series, - smp = 'smp', - test_group='Kernel_Qual_Containers', - tko_mysql_server='autotest', results_server='autotest', - max_rel_kernels=8, max_dev_kernels=5): - - test_runs = {} # kernel --> (platform --> list of test runs) - job_table = {} # kernel id --> list of job idxs - kernel_dates = {} # Kernel id --> date of nightly test - - - def add_kernel_jobs(label_pattern): - cmd = "select job_idx from jobs where label like '%s'" % label_pattern - nrows = perf.db_cur.execute(cmd) - return [row[0] for row in perf.db_cur.fetchall()] - - - def is_filtered_platform(platform): - for p in platforms_filter.split(','): - if platform.startswith(p): - return True - return False - - - def collect_testruns(jobs, regressed_platforms, test): - # get test_runs run #s for 1 test on 1 kernel and some platforms - # TODO: Is jobs list short enough to use directly in 1 sql cmd? - runs = {} # platform --> list of test runs - for jobx in jobs: - cmd = ( "select test_idx, machine_idx from tests" - " where job_idx = %s and test = %s" ) - args = [jobx, test] - nrows = perf.db_cur.execute(cmd, args) - for testrunx, machx in perf.db_cur.fetchall(): - platform, host = perf.machine_idx_to_platform_host(machx) - if by_hosts: - platform += '.'+host - if ( is_filtered_platform(platform) and - (regressed_platforms is None - or platform in regressed_platforms) ): - runs.setdefault(platform, []).append(testrunx) - return runs - - - def collect_all_testruns(trimmed_kernels, regressed_platforms, test): - # get test_runs run #s for 1 test on some kernels and platforms +def add_kernel_jobs(label_pattern): + cmd = "select job_idx from jobs where label like '%s'" % label_pattern + nrows = perf.db_cur.execute(cmd) + return [row[0] for row in perf.db_cur.fetchall()] + + +def is_filtered_platform(platform, platforms_filter): + if not platforms_filter: + return True + for p in platforms_filter: + if platform.startswith(p): + return True + return False + + +def get_test_attributes(testrunx): + cmd = ( "select attribute, value from test_attributes" + " where test_idx = %d" % testrunx ) + nrows = perf.db_cur.execute(cmd) + return dict(perf.db_cur.fetchall()) + + +def get_antag(testrunx): + attrs = get_test_attributes(testrunx) + return attrs.get('antag', None) + + +def matching_test_attributes(attrs, required_test_attributes): + if not required_test_attributes: + return True + matches = [attrs[key] == required_test_attributes[key] + for key in attrs if key in required_test_attributes] + return min(matches+[True]) # True if all jointly-existing keys matched + + +def collect_testruns(jobs, test, test_attributes, + platforms_filter, by_hosts, no_antag): + # get test_runs run #s for 1 test on 1 kernel and some platforms + # TODO: Is jobs list short enough to use directly in 1 sql cmd? + # TODO: add filtering on test series? + runs = {} # platform --> list of test runs + for jobx in jobs: + cmd = ( "select test_idx, machine_idx from tests" + " where job_idx = %s and test = %s" ) + args = [jobx, test] + nrows = perf.db_cur.execute(cmd, args) + for testrunx, machx in perf.db_cur.fetchall(): + platform, host = perf.machine_idx_to_platform_host(machx) + if by_hosts: + platform += '.'+host + if ( is_filtered_platform(platform, platforms_filter) and + matching_test_attributes(get_test_attributes(testrunx), + test_attributes) and + (not no_antag or get_antag(testrunx) == '') ): + runs.setdefault(platform, []).append(testrunx) + return runs + + +def all_tested_platforms(test_runs): + # extract list of all tested platforms from test_runs table + platforms = set() + for kernel in test_runs: + platforms.update(set(test_runs[kernel].keys())) + return sorted(platforms) + + +def divide_twoway_testruns(test_runs, platform): + # partition all twoway runs based on name of antagonist progs + twoway_runs = {} + antagonists = set() + for kernel in test_runs: + runs = {} + for testrunx in test_runs[kernel].get(platform, []): + antag = get_antag(testrunx) + if antag is not None: + runs.setdefault(antag, []).append(testrunx) + antagonists.add(antag) + twoway_runs[kernel] = runs + return twoway_runs, sorted(antagonists) + + +def collect_raw_scores(runs, metric): + # get unscaled scores of test runs for 1 test on certain jobs + # arrange them by platform type + platform_scores = {} # platform --> list of perf scores + for platform in runs: + vals = perf.get_metric_at_point(runs[platform], metric) + if vals: + platform_scores[platform] = vals + return platform_scores + + +def collect_scaled_scores(metric, test_runs, regressed_platforms, relative): + # get scores of test runs for 1 test on some kernels and platforms + # optionally make relative to oldest (?) kernel on that platform + # arrange by plotline (ie platform) for gnuplot + plot_data = {} # platform --> (kernel --> list of perf scores) + baseline = {} + for kernel in sorted(test_runs.keys()): + for platform in test_runs[kernel]: + if not (regressed_platforms is None or + platform in regressed_platforms): + continue # delete results for uninteresting platforms + vals = perf.get_metric_at_point(test_runs[kernel][platform], + metric) + if vals: + if relative: + if platform not in baseline: + baseline[platform], std = plotgraph.avg_dev(vals) + vals = [v/baseline[platform] for v in vals] + pdp = plot_data.setdefault(platform, {}) + pdp.setdefault(kernel, []).extend(vals) + return plot_data + + +def collect_twoway_scores(metric, antagonists, twoway_runs, relative): + alone = '' + plot_data = {} + for kernel in twoway_runs: + for test2 in antagonists: + runs = twoway_runs[kernel].get(test2, []) + vals = perf.get_metric_at_point(runs, metric) + plot_data.setdefault(test2, {}) + if vals: + plot_data[test2][kernel] = vals + if relative: + vals = plot_data[alone].get(kernel, []) + if vals: + baseline = perf.average(vals) + for test2 in antagonists: + vals = plot_data[test2].get(kernel, []) + vals = [val/baseline for val in vals] + if vals: + plot_data[test2][kernel] = vals + else: + for test2 in antagonists: + if kernel in plot_data[test2]: + del plot_data[test2][kernel] + return plot_data + + +def find_regressions(kernels, test_runs, metric): + # A test is regressed on some platform if its latest results are + # definitely lower than on the reference kernel. + # Runs for the latest kernel may be underway and incomplete. + # In that case, selectively use next-latest kernel. + # TODO: the next-latest method hurts if latest run is not sorted last, + # or if there are several dev threads + ref = kernels[0] + latest = kernels[-1] + prev = kernels[-2:][0] + scores = {} # kernel --> (platform --> list of perf scores) + for k in [ref, prev, latest]: + if k in test_runs: + scores[k] = collect_raw_scores(test_runs[k], metric) + regressed_platforms = [] + for platform in scores[ref]: + if latest in scores and platform in scores[latest]: + k = latest + elif prev in scores and platform in scores[prev]: + k = prev + else: # perhaps due to decay of test machines + k = ref # no regression info avail + ref_avg, ref_std = plotgraph.avg_dev(scores[ref][platform]) + avg, std = plotgraph.avg_dev(scores[ k ][platform]) + if avg+std < ref_avg-ref_std: + regressed_platforms.append(platform) + return sorted(regressed_platforms) + + +def get_testrun_context(testrun): + cmd = ( 'select jobs.label, jobs.tag, tests.subdir,' + ' tests.started_time' + ' from jobs, tests' + ' where jobs.job_idx = tests.job_idx' + ' and tests.test_idx = %d' % testrun ) + nrows = perf.db_cur.execute(cmd) + assert nrows == 1 + row = perf.db_cur.fetchone() + row = [row[0], row[1], row[2], row[3].strftime('%m/%d/%y %H:%M')] + return row + + +def html_top(): + print "Content-Type: text/html\n\n<html><body>" + + +def abs_rel_link(myurl, passthru): + # link redraws current page with opposite absolute/relative choice + mod_passthru = passthru[:] + if 'absolute' in passthru: + mod_passthru.remove('absolute') + opposite = 'relative' + else: + mod_passthru.append('absolute') + opposite = 'absolute' + url = '%s?%s' % (myurl, '&'.join(mod_passthru)) + return "<a href='%s'> %s </a>" % (url, opposite) + + +def table_1_metric_all_kernels(plot_data, columns, column_argname, + kernels, kernel_dates, + myurl, filtered_passthru): + # generate html table of graph's numbers + # for 1 benchmark metric over all kernels (rows), + # over various platforms or various antagonists etc (cols). + ref_thresholds = {} + print "<table border=1 cellpadding=3 cellspacing=0>" + print "<tr> <td><b> Kernel </b></td>", + for label in columns: + if not label and column_argname == 'antag': + label = 'no antag' + print "<td><b>", label.replace('_', '<br>_'), "</b></td>" + print "</tr>" + for kernel in kernels: + print "<tr> <td><b>", kernel, "</b>", + if kernel in kernel_dates: + print "<br><small>", kernel_dates[kernel], "</small>" + print "</td>" + for col in columns: + print "<td", + vals = plot_data[col].get(kernel, []) + if not vals: + print "> ?", + else: + (avg, std_dev) = plotgraph.avg_dev(vals) + if col not in ref_thresholds: + ref_thresholds[col] = avg - std_dev + if avg+std_dev < ref_thresholds[col]: + print "bgcolor=pink", + print "> ", + args = filtered_passthru[:] + perf.append_cgi_args(args, + {column_argname:col, 'kernel':kernel}) + print "<a href='%s?%s&runs&attrs'>" % (myurl, + '&'.join(args)) + print "<b>%.4g</b>" % avg, "</a><br>", + print " <small> %dr </small>" % len(vals), + print " <small> %.3g </small>" % std_dev, + print "</td>" + print "</tr>\n" + print "</table>" + print "<p> <b>Bold value:</b> Average of this metric, then <br>" + print "number of good test runs, then standard deviation of those runs" + print "<br> Pink if regressed from reference kernel" + + +def table_all_metrics_1_platform(test_runs, platform, relative): + # TODO: show std dev in cells + # can't mark regressions, since some metrics improve downwards + kernels = perf.sort_kernels(test_runs.keys()) + scores = {} + attrs = set() + for kernel in kernels: + testruns = test_runs[kernel].get(platform, []) + if testruns: + d = perf.collect_all_metrics_scores(testruns) + scores[kernel] = d + attrs.update(set(d.keys())) + else: + print "No runs completed on", kernel, "<br>" + attrs = sorted(list(attrs))[:100] + + print "<table border=1 cellpadding=4 cellspacing=0>" + print "<tr><td> Metric </td>" + for kernel in kernels: + kernel = kernel.replace("_", "_<br>") + print "<td>", kernel, "</td>" + print "</tr>" + for attr in attrs: + print "<tr>" + print "<td>", attr, "</td>" + baseline = None + for kernel in kernels: + print "<td>", + if kernel in scores and attr in scores[kernel]: + (avg, dev) = plotgraph.avg_dev(scores[kernel][attr]) + if baseline and relative: + percent = (avg/baseline - 1)*100 + print "%+.1f%%" % percent, + else: + baseline = avg + print "%.4g" % avg, + else: + print "?" + print "</td>" + print "</tr>" + print "</table>" + + +def table_variants_all_tests(plot_data, columns, colkeys, benchmarks, + myurl, filtered_passthru): + # generate html table of graph's numbers + # for primary metric over all benchmarks (rows), + # on one platform and one kernel, + # over various combos of test run attribute constraints (cols). + ref_thresholds = {} + print "<table border=1 cellpadding=3 cellspacing=0>" + print "<tr> <td><b> Benchmark </b></td>", + for col in columns: + print "<td><b>", colkeys[col].replace(',', ',<br>'), "</b></td>" + print "</tr>" + for benchmark in benchmarks: + print "<tr> <td><b>", benchmark, "</b></td>" + for col in columns: + print "<td>", + vals = plot_data[col].get(benchmark, []) + if not vals: + print "?", + else: + (avg, std_dev) = plotgraph.avg_dev(vals) + args = filtered_passthru[:] + perf.append_cgi_args(args, {'test':benchmark}) + for keyval in colkeys[col].split(','): + key, val = keyval.split('=', 1) + perf.append_cgi_args(args, {key:val}) + print "<a href='%s?%s&runs&attrs'>" % (myurl, + '&'.join(args)) + print "<b>%.4g</b>" % avg, "</a><br>", + print " <small> %dr </small>" % len(vals), + print " <small> %.3g </small>" % std_dev, + print "</td>" + print "</tr>\n" + print "</table>" + print "<p> <b>Bold value:</b> Average of this metric, then <br>" + print "number of good test runs, then standard deviation of those runs" + + +def table_testrun_details(runs, metric, tko_server, show_attrs): + print "<table border=1 cellpadding=4 cellspacing=0>" + print "<tr><td> %s metric </td>" % metric + print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>" + print "<td> Started_time </td>" + if show_attrs: + print "<td> Test attributes </td>" + print "</tr>\n" + + for testrunx in runs: + print "<tr> <td>", + vals = perf.get_metric_at_point([testrunx], metric) + for v in vals: + print "%.4g " % v, + print "</td>" + row = get_testrun_context(testrunx) + row[2] = ( "<a href='//%s/results/%s/%s/results/keyval'> %s </a>" + % (tko_server, row[1], row[2], row[2]) ) + for v in row: + print "<td> %s </td>" % v + if show_attrs: + attrs = get_test_attributes(testrunx) + print "<td>", + for attr in sorted(attrs.keys()): + if attr == "sysinfo-cmdline": continue + if attr[:4] == "svs-": continue + val = attrs[attr] + if len(val) > 40: + val = val[:40-3] + "..." + print "%s=%s " % (attr, val) + print "</td>" + print "</tr>\n" + print "</table>" + + +def overview_thumb(test, metric, myurl, passthru): + pass_ = passthru + ['test=%s' % test] + if metric: + pass_ += ['metric=%s' % metric] + pass_ = '&'.join(pass_) + print "<a href='%s?%s&table'>" % (myurl, pass_) + print " <img src='%s?%s&size=450,500'> </a>" % (myurl, pass_) + # embedded graphs fit 3 across on 1400x1050 laptop + + +def graph_1_test(title, metric, plot_data, line_argname, lines, + kernel_legend, relative, size, dark=False): + # generate graph image for one benchmark, showing avg and + # std dev of one metric, over various kernels (X columns), + # over various platforms or antagonists etc (graphed lines) + xlegend = kernel_legend + ylegend = metric.capitalize() + if relative: + ylegend += ', Relative' + ymin = 0.8 + else: + ymin = None + if len(lines) > 1: + keytitle = line_argname.capitalize() + ':' + else: + keytitle = '' + graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size, + xsort=perf.sort_kernels, keytitle=keytitle) + for line in lines: + label = line + if not label and line_argname == 'antag': + label = 'no antag' + graph.add_dataset(label, plot_data[line]) + graph.plot(cgi_header=True, ymin=ymin, dark=dark) + + +def graph_variants_all_tests(title, plot_data, linekeys, size, dark): + # generate graph image showing all benchmarks + # on one platform and one kernel, + # over various combos of test run attribute constraints (lines). + xlegend = "Benchmark" + ylegend = "Relative Perf" + graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size) + for i in linekeys: + graph.add_dataset(linekeys[i], plot_data[i]) + graph.plot(cgi_header=True, dark=dark, ymin=0.8) + + +class generate_views(object): + + + def __init__(self, kernel_legend, benchmarks, test_group, + site_benchmark_metrics, tko_server, + jobs_selector, no_antag): + self.kernel_legend = kernel_legend + self.benchmarks = benchmarks + self.test_group = test_group + self.tko_server = tko_server + self.jobs_selector = jobs_selector + self.no_antag = no_antag + + cgitb.enable() + test, antagonists = self.parse_most_cgi_args() + + perf.init(tko_server=tko_server) + for b in site_benchmark_metrics: + perf.add_benchmark_main_metric(b, site_benchmark_metrics[b]) + + self.test_runs = {} # kernel --> (platform --> list of test runs) + self.job_table = {} # kernel id --> list of job idxs + self.kernel_dates = {} # kernel id --> date of nightly test + + vary = self.cgiform.getlist('vary') + if vary: + platform = self.platforms_filter[0] + self.analyze_variants_all_tests_1_platform(platform, vary) + elif test: + self.analyze_1_test(test, antagonists) + else: + self.overview_page_all_tests(self.benchmarks, antagonists) + + + def collect_all_testruns(self, trimmed_kernels, test): + # get test_runs run #s for 1 test on some kernels and platforms for kernel in trimmed_kernels: - runs = collect_testruns(job_table[kernel], regressed_platforms, test) + runs = collect_testruns(self.job_table[kernel], test, + self.test_attributes, self.platforms_filter, + 'by_hosts' in self.toggles, self.no_antag) if runs: - test_runs[kernel] = runs + self.test_runs[kernel] = runs - def collect_raw_scores(runs, metric): - # get unscaled scores of test runs for 1 test on certain jobs - # arrange them by platform type - platform_scores = {} # platform --> list of perf scores - for platform in runs: - vals = perf.get_metric_at_point(runs[platform], metric) - if vals: - platform_scores[platform] = vals - return platform_scores - - - def collect_scaled_scores(metric): - # get scores of test runs for 1 test on some kernels and platforms - # optionally make relative to first kernel on that platform - # arrange by plotline (ie platform) for gnuplot - plot_data = {} # platform --> (kernel --> list of perf scores) - baseline = {} - for kernel in sorted(test_runs.keys()): - for platform in test_runs[kernel]: - vals = perf.get_metric_at_point(test_runs[kernel][platform], metric) - if vals: - if relative: - if platform not in baseline: - baseline[platform], std = plotgraph.avg_dev(vals) - vals = [v/baseline[platform] for v in vals] - pdp = plot_data.setdefault(platform, {}) - pdp.setdefault(kernel, []).extend(vals) - return plot_data - - - def find_regressions(kernels, test, metric): - # A test is regressed on some platform if its latest results are - # definitely lower than on the reference kernel. - # Runs for the latest kernel may be underway and incomplete. - # In that case, selectively use next-latest kernel. - if not regress: - return None - ref = kernels[0] - latest = kernels[-1] - prev = kernels[-2:][0] - scores = {} # kernel --> (platform --> list of perf scores) - for k in [ref, prev, latest]: - runs = collect_testruns(job_table[k], None, test) - scores[k] = collect_raw_scores(runs, metric) - regressed_platforms = [] - for platform in scores[ref]: - k = latest - if platform not in scores[k]: - k = prev - if platform not in scores[k]: - continue # perhaps due to decay of test machines - ref_avg, ref_std = plotgraph.avg_dev(scores[ref][platform]) - avg, std = plotgraph.avg_dev(scores[ k ][platform]) - if avg+std < ref_avg-ref_std: - regressed_platforms.append(platform) - return sorted(regressed_platforms) - - - def select_dev_kernels(): - # collect table of all tested kernels' test runs - kernels = [] - for series in released_kernel_series: - kernels += survey_all_kernels_tested(perf.db_cur, series+'.', - '', smp, test_group, - max_rel_kernels, - job_table, kernel_dates) - for series in nightly_kernel_series: - kernels += survey_all_kernels_tested(perf.db_cur, - '2.6.26-%s-' % series, - series, smp, test_group, - max_dev_kernels, - job_table, kernel_dates) - kernels = sort_kernels(kernels) - return kernels # sorted subset of kernels in job_table - - - def graph_1_test(test, metric, size): - # generate graph image for one benchmark, showing avg and - # std dev of one metric, over various kernels (X columns) - # and various platform types (graphed lines) - title = test.capitalize() + suite_notes - if regress: - title += ', Regressions Only' - if relative: - ylegend = 'Relative ' - ymin = 0.9 - else: - ylegend = '' - ymin = None - ylegend += metric.capitalize() - graph = plotgraph.gnuplot(title, kernel_legend, ylegend, size=size, - xsort=sort_kernels) - for platform in platforms: - graph.add_dataset(platform, plot_data[platform]) - graph.plot(cgi_header=True, ymin=ymin, dark=dark) - - - def table_for_1_test(test, metric): - # generate detailed html page with graph plus numeric table for 1 benchmark - print "Content-Type: text/html\n\n<html><body>" - heading = "%s %s:  %s%s" % (test_group, kernel_legend, - test.capitalize(), suite_notes) - if regress: - heading += ", Regressions Only" - print "<h2> %s </h2>" % heading - print "<img src='%s?%s'>" % (myself, '&'.join(passthru)) + def table_for_graph_1_test(self, title, metric, plot_data, + column_argname, columns, filtered_passthru): + # generate detailed html page with 1 graph and corresp numbers + # for 1 benchmark metric over all kernels (rows), + # over various platforms or various antagonists etc (cols). + html_top() + print '<h3> %s </h3>' % title + print ('%s, machine group %s on //%s server <br>' % + (self.kernel_legend, self.test_group, self.tko_server)) + if self.test_tag: + print '%s test script series <br>' % self.test_tag[1:] + + print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru)) + + link = abs_rel_link(self.myurl, self.passthru+['table']) + print "<p><p> <h4> Redraw this with %s performance? </h4>" % link - heading = "%s %s metric" % (test.capitalize(), metric) - if relative: heading += ", relative" + heading = "%s, %s metric" % (title, metric) + if self.relative: + heading += ", relative" print "<p><p> <h3> %s: </h3>" % heading + table_1_metric_all_kernels(plot_data, columns, column_argname, + self.kernels, self.kernel_dates, + self.myurl, filtered_passthru) + print "</body></html>" - ref_thresholds = {} - print "<table border=1, cellpadding=3>" - print "<tr> <td><b> Kernel </b></td>", - for platform in platforms: - p = platform.replace("_", "_<br>").replace(".", "<br>") - print "<td><b>", p, "</b></td>" - print "</tr>" - for kernel in kernels: - print "<tr> <td><b>", kernel, "</b><br><small>", - print kernel_dates[kernel], "</small></td>" - for platform in platforms: - print "<td", - vals = plot_data[platform].get(kernel, []) - if vals: - (avg, std_dev) = plotgraph.avg_dev(vals) - if platform not in ref_thresholds: - ref_thresholds[platform] = avg - std_dev - if avg+std_dev < ref_thresholds[platform]: - print "bgcolor=pink", - print ( "> <a href='%s?test=%s&metric=%s" - "&platforms=%s&runs&kernel=%s'>" - % (myself, test, metric, platform, kernel) ) - print "<b>%.4g</b>" % avg, "</a><br>", - print " <small> %dr </small>" % len(vals), - print " <small> %.3g </small>" % std_dev, - else: - print "> ?", - print "</td>" - print "</tr>\n" - print "</table>" - print "<p> <b>Bold value:</b> Average of this metric, then <br>" - print "number of good test runs, then standard deviation of those runs" - print "<br> Pink if regressed from reference kernel" + + def graph_1_test_all_platforms(self, test, metric, platforms, plot_data): + # generate graph image for one benchmark + title = test.capitalize() + if 'regress' in self.toggles: + title += ' Regressions' + if 'table' in self.cgiform: + self.table_for_graph_1_test(title, metric, plot_data, + 'platforms', platforms, + filtered_passthru=self.passthru) + else: + graph_1_test(title, metric, plot_data, 'platforms', platforms, + self.kernel_legend, self.relative, + self.size, 'dark' in self.toggles) + + + def testrun_details(self, title, runs, metric): + html_top() + print '<h3> %s </h3>' % title + print ('%s, machine group %s on //%s server' % + (self.kernel_legend, self.test_group, self.tko_server)) + if self.test_tag: + print '<br> %s test script series' % self.test_tag[1:] + print '<p>' + table_testrun_details(runs, metric, + self.tko_server, 'attrs' in self.cgiform) print "</body></html>" - def get_testrun_context(testrun): - cmd = ( 'select jobs.tag, tests.subdir,' - ' jobs.label, tests.started_time' - ' from jobs, tests' - ' where jobs.job_idx = tests.job_idx' - ' and tests.test_idx = %d' % testrun ) - nrows = perf.db_cur.execute(cmd) - assert nrows == 1 - row = perf.db_cur.fetchone() - row = (row[0], row[1], row[2], row[3].strftime('%m/%d/%y %H:%M')) - return row - - - def testrun_details_for_1_test_kernel_platform(test, metric, platform): - kernel = form.getvalue('kernel') - show_attrs = 'attrs' in form - print "Content-Type: text/html\n\n<html><body>" - print "<h3> %s %s: %s%s, Kernel %s on %s </h3>" % ( - test_group, kernel_legend, test.capitalize(), suite_notes, kernel, platform) - print "<table border=1 cellpadding=4>" - print "<tr><td> Test_idx </td> <td> %s metric </td>" % metric - print "<td> Job tag </td> <td> Subdir </td> <td> Job label </td>" - print "<td> Started_time </td>" - if show_attrs: - print "<td> Test attributes </td>" - print "</tr>\n" + def testrun_details_for_1_test_kernel_platform(self, test, + metric, platform): + default_kernel = min(self.test_runs.keys()) + kernel = self.cgiform.getvalue('kernel', default_kernel) + title = '%s on %s using %s' % (test.capitalize(), platform, kernel) + runs = self.test_runs[kernel].get(platform, []) + self.testrun_details(title, runs, metric) - for testrunx in test_runs[kernel][platform]: - print "<tr> <td>", testrunx, "</td>" - print "<td>", - vals = perf.get_metric_at_point([testrunx], metric) - for v in vals: - print "%.4g" % v, - print "</td>" - row = get_testrun_context(testrunx) - print ( "<td> <a href='//%s/results/%s/%s/results'> %s </a></td>" - % (results_server, row[0], row[1], row[0]) ) - for v in row[1:]: - print "<td> %s </td>" % v - if show_attrs: - attrs = perf.get_test_attributes(testrunx) - print "<td>", - for attr in attrs: - # if attr == "sysinfo-cmdline": continue - # if attr[:4] == "svs-": continue - val = attrs[attr] - if len(val) > 40: - val = val[:40-3] + "..." - print "%s=%s" % (attr, val) - print "</td>" - print "</tr>\n" - print "</table>" + + def analyze_1_metric_all_platforms(self, test, metric): + if 'regress' in self.toggles: + regressed_platforms = find_regressions(self.kernels, self.test_runs, + metric) + else: + regressed_platforms = None + plot_data = collect_scaled_scores(metric, self.test_runs, + regressed_platforms, self.relative) + platforms = sorted(plot_data.keys()) + if not plot_data: + html_top() + print 'No runs' + elif 'runs' in self.cgiform: + self.testrun_details_for_1_test_kernel_platform(test, metric, + platforms[0]) + else: + self.graph_1_test_all_platforms(test, metric, platforms, plot_data) + + + def analyze_all_metrics_1_platform(self, test, platform): + # TODO: show #runs in header + html_top() + heading = "%s %s:  %s" % (self.test_group, self.kernel_legend, + test.capitalize()) + print "<h2> %s </h2>" % heading + print "platform=%s <br>" % platform + for attr in self.test_attributes: + print "%s=%s " % (attr, self.test_attributes[attr]) + print "<p>" + table_all_metrics_1_platform(self.test_runs, platform, self.relative) print "</body></html>" - def overview_thumb(test, metric=[]): - pass_ = passthru + ['test=%s' % test] - if metric: - pass_ += ['metric=%s' % metric] - pass_ = '&'.join(pass_) - print "<a href='%s?%s&table'>" % (myself, pass_) - print " <img src='%s?%s&size=650,600'> </a>" % (myself, pass_) - # embedded graphs fit 2 across on 1400x1050 laptop + def table_for_variants_all_tests(self, title, plot_data, colkeys, columns, + filtered_passthru, test_tag): + # generate detailed html page with 1 graph and corresp numbers + # for primary metric over all benchmarks (rows), + # on one platform and one kernel, + # over various combos of test run attribute constraints (cols). + html_top() + print '<h3> %s </h3>' % title + print ('%s, machine group %s on //%s server <br>' % + (self.kernel_legend, self.test_group, self.tko_server)) + if test_tag: + print '%s test script series <br>' % test_tag[1:] + + varies = ['vary='+colkeys[col] for col in columns] + print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru+varies)) + + print "<p><p> <h3> %s: </h3>" % title + table_variants_all_tests(plot_data, columns, colkeys, self.benchmarks, + self.myurl, filtered_passthru) + print "</body></html>" + + + def analyze_variants_all_tests_1_platform(self, platform, vary): + # generate one graph image for results of all benchmarks + # on one platform and one kernel, comparing effects of + # two or more combos of kernel options (test run attributes) + # (numa_fake,stale_page,kswapd_merge,sched_idle, etc) + kernel = self.cgiform.getvalue('kernel', 'some_kernel') + self.passthru.append('kernel=%s' % kernel) + + # two or more vary_groups, one for each plotted line, + # each group begins with vary= and ends with next & + # each group has comma-separated list of test attribute key=val pairs + # eg vary=keyval1,keyval2&vary=keyval3,keyval4 + vary_groups = [dict(pair.split('=',1) for pair + in vary_group.split(',')) + for vary_group in vary] + + test = self.benchmarks[0] # pick any test in all jobs + kernels, test_tag = self.jobs_selector(test, self.job_table, + self.kernel_dates) + + linekeys = {} + plot_data = {} + baselines = {} + for i, vary_group in enumerate(vary_groups): + group_attributes = self.test_attributes.copy() + group_attributes.update(vary_group) + linekey = ','.join('%s=%s' % (attr, vary_group[attr]) + for attr in vary_group) + linekeys[i] = linekey + data = {} + for benchmark in self.benchmarks: + metric = perf.benchmark_main_metric(benchmark) + runs = collect_testruns(self.job_table[kernel], + benchmark+test_tag, + group_attributes, + self.platforms_filter, + 'by_hosts' in self.toggles, + self.no_antag) + vals = [] + for testrunx in runs[platform]: + vals += perf.get_metric_at_point([testrunx], metric) + if vals: + if benchmark not in baselines: + baselines[benchmark], stddev = plotgraph.avg_dev(vals) + vals = [val/baselines[benchmark] for val in vals] + data[benchmark] = vals + plot_data[i] = data + + title = "%s on %s" % (kernel, platform) + for attr in self.test_attributes: + title += ', %s=%s' % (attr, self.test_attributes[attr]) + if 'table' in self.cgiform: + self.table_for_variants_all_tests(title, plot_data, linekeys, + range(len(linekeys)), + filtered_passthru=self.passthru, + test_tag=test_tag) + else: + graph_variants_all_tests(title, plot_data, linekeys, + self.size, 'dark' in self.toggles) + + + def graph_twoway_antagonists_1_test_1_platform( + self, test, metric, platform, antagonists, twoway_runs): + # generate graph of one benchmark's performance paired with + # various antagonists, with one plotted line per antagonist, + # over most kernels (X axis), all on one machine type + # performance is relative to the no-antag baseline case + plot_data = collect_twoway_scores(metric, antagonists, + twoway_runs, self.relative) + title = "%s vs. an Antagonist on %s:" % (test.capitalize(), platform) + if 'table' in self.cgiform: + filtered_passthru = [arg for arg in self.passthru + if not arg.startswith('antag=')] + self.table_for_graph_1_test(title, metric, plot_data, + 'antag', antagonists, + filtered_passthru=filtered_passthru) + else: + graph_1_test(title, metric, plot_data, 'antag', antagonists, + self.kernel_legend, self.relative, + self.size, 'dark' in self.toggles) + + + def testrun_details_for_twoway_test(self, test, metric, platform, + antagonist, twoway_runs): + default_kernel = min(twoway_runs.keys()) + kernel = self.cgiform.getvalue('kernel', default_kernel) + title = '%s vs. Antagonist %s on %s using %s' % ( + test.capitalize(), antagonist.capitalize(), platform, kernel) + runs = twoway_runs[kernel].get(antagonist, []) + self.testrun_details(title, runs, metric) + + + def analyze_twoway_antagonists_1_test_1_platform( + self, test, metric, platform, antagonists): + twoway_runs, all_antagonists = divide_twoway_testruns(self.test_runs, + platform) + if antagonists == ['*']: + antagonists = all_antagonists + if not twoway_runs: + html_top() + print 'No runs' + elif 'runs' in self.cgiform: + self.testrun_details_for_twoway_test( + test, metric, platform, antagonists[0], twoway_runs) + else: + self.graph_twoway_antagonists_1_test_1_platform( + test, metric, platform, antagonists, twoway_runs) + + def get_twoway_default_platform(self): + if self.platforms_filter: + return self.platforms_filter[0] + test = 'unixbench' + kernels, test_tag = self.jobs_selector(test, self.job_table, + self.kernel_dates) + self.collect_all_testruns(kernels, test+test_tag) + return all_tested_platforms(self.test_runs)[0] - def overview_page(benchmarks): + + def overview_page_all_tests(self, benchmarks, antagonists): # generate overview html page with small graphs for each benchmark # linking to detailed html page for that benchmark # recursively link to this same cgi to generate each image - print "Content-Type: text/html\n\n<html><body>" - heading = "%s %s" % (test_group, kernel_legend) - if regress: + html_top() + if antagonists is not None: + heading = ('Twoway Container Isolation using %s on %s' % + (self.kernel_legend, self.get_twoway_default_platform())) + else: + heading = '%s, %s Benchmarks' % (self.kernel_legend, + self.test_group) + if 'regress' in self.toggles: heading += ", Regressions Only" - print "<h2> %s </h2>" % heading + print "<h3> %s </h3>" % heading for test in benchmarks: - overview_thumb(test) + overview_thumb(test, '', self.myurl, self.passthru) if test == 'unixbench': - overview_thumb('unixbench', 'Process_creation') + overview_thumb('unixbench', 'Process_creation', + self.myurl, self.passthru) + + link = abs_rel_link(self.myurl, self.passthru) + print "<p><p> <h4> Redraw this with %s performance? </h4>" % link print "</body></html>" - # body of nightly_views(): - cgitb.enable() - form = cgi.FieldStorage(keep_blank_values=True) - test = form.getvalue('test', '') - relative = 'relative' in form - regress = 'regress' in form - dark = 'dark' in form - platforms_filter = form.getvalue('platforms', '') - by_hosts = 'by_hosts' in form or '.' in platforms_filter - passthru = [] - if relative: - passthru += ['relative'] - if regress: - passthru += ['regress'] - if dark: - passthru += ['dark'] - if by_hosts: - passthru += ['by_hosts'] - if platforms_filter: - passthru += ['platforms=%s' % platforms_filter] - myself = os.path.basename(sys.argv[0]) - if test: - passthru += ['test=%s' % test] - metric = form.getvalue('metric', '') + def analyze_1_test(self, test, antagonists): + self.passthru.append('test=%s' % test) + metric = self.cgiform.getvalue('metric', '') if metric: - passthru += ['metric=%s' % metric] + self.passthru.append('metric=%s' % metric) else: metric = perf.benchmark_main_metric(test) assert metric, "no default metric for test %s" % test - # perf.init() - perf.db_cur = db.db(host=tko_mysql_server, - user='nobody', password='').cur - kernels = select_dev_kernels() - regressed_platforms = find_regressions(kernels, test, metric) - collect_all_testruns(kernels, regressed_platforms, test) - plot_data = collect_scaled_scores(metric) - platforms = sorted(plot_data.keys()) - if 'runs' in form: - testrun_details_for_1_test_kernel_platform(test, metric, - platforms[0]) - elif 'table' in form: - table_for_1_test(test, metric) + self.kernels, self.test_tag = self.jobs_selector(test, self.job_table, + self.kernel_dates) + self.collect_all_testruns(self.kernels, test+self.test_tag) + if not self.platforms_filter and (metric == '*' or + antagonists is not None): + # choose default platform + self.platforms_filter = all_tested_platforms(self.test_runs)[0:1] + self.passthru.append('platforms=%s' % + ','.join(self.platforms_filter)) + if antagonists is not None: + antagonists = antagonists.split(',') + if len(antagonists) == 1 and antagonists != ['*']: + self.relative = False + self.analyze_twoway_antagonists_1_test_1_platform( + test, metric, self.platforms_filter[0], antagonists) + elif metric == '*': + platform = self.platforms_filter[0] + self.analyze_all_metrics_1_platform(test, platform) else: - size = form.getvalue('size', '1200,850' ) - graph_1_test(test, metric, size) - else: - overview_page(benchmarks) - - -def sort_kernels(kernels): - return sorted(kernels, key=kernel_versions.version_encode) - - -def survey_all_kernels_tested(db_cur, kernel_series, kname_prefix, smp, - test_group, maxkernels, - kernel_jobs, kernel_dates): - kernels = set() - # script run's job label has form - # 'Kernel_Qual_Containers : 2.6.26-300.8-jilee : smp : 2009-05-15' - # or 'Kernel_Qual_Containers : 2.6.26-DEV-4099999 : smp : 2009-05-15' - job_label = ('%s : %s%% : %s : %%' - % (test_group, kernel_series, smp)) - # find names and job#s of all matching perf runs - cmd = ( "select job_idx, label, tag from jobs" - " where label like '%s' order by label desc" % job_label ) - nrows = db_cur.execute(cmd) - for jobx, joblabel, tag in db_cur.fetchall(): - cols = joblabel.split(' : ') - kernvers = cols[1].split('-') # 2.6.26 300.8 jilee - # or 2.6.26 DEV 4099999 - if kname_prefix: # nightly build, eg 'DEV' or '300' - changelist = kernvers[2] # build's CL number - testdate = cols[3] - kernel = '%s_%s' % (kname_prefix, changelist) - else: # release candidates - if len(kernvers) > 2: # reject jobs with -qual suffix - continue - kernel = kernvers[1] # 300.8 - testdate = '' - kernel_jobs.setdefault(kernel, []) - kernel_jobs[kernel].append(jobx) - kernel_dates[kernel] = testdate - kernels.add(kernel) - kernels = sort_kernels(kernels)[-maxkernels:] - return kernels + self.analyze_1_metric_all_platforms(test, metric) + + + def parse_most_cgi_args(self): + self.myurl = os.path.basename(sys.argv[0]) + self.cgiform = cgi.FieldStorage(keep_blank_values=True) + self.size = self.cgiform.getvalue('size', '1200,850') + all_toggles = set(('absolute', 'regress', 'dark', 'by_hosts')) + self.toggles = set(tog for tog in all_toggles if tog in self.cgiform) + platforms = self.cgiform.getvalue('platforms', '') + if '.' in platforms: + self.toggles.add('by_hosts') + self.passthru = list(self.toggles) + self.relative = 'absolute' not in self.toggles + if platforms: + self.passthru.append('platforms=%s' % platforms) + self.platforms_filter = platforms.split(',') + else: + self.platforms_filter = [] + self.test_attributes = perf.parse_test_attr_args(self.cgiform) + perf.append_cgi_args(self.passthru, self.test_attributes) + test = self.cgiform.getvalue('test', '') + if 'antag' in self.cgiform: + antagonists = ','.join(self.cgiform.getlist('antag')) + # antag=* + # or antag=test1,test2,test3,... + # or antag=test1&antag=test2&... + # testN is empty for solo case of no antagonist + self.passthru.append('antag=%s' % antagonists) + else: + antagonists = None # not same as '' + return test, antagonists |