From 11224976b21524bcef3fcd2dbaa1069ce7233cf2 Mon Sep 17 00:00:00 2001 From: Martin Peres Date: Wed, 15 Mar 2017 15:00:06 +0200 Subject: smartezbench: allow specifying if a task is user- or auto-requested We will first run user-requested tasks, then handle auto-requested ones. We also reset the auto tasks after every call to run(), to make sure the decision to run something is always up to date! --- python-modules/ezbench/smartezbench.py | 178 ++++++++++++++++++++++----------- 1 file changed, 122 insertions(+), 56 deletions(-) diff --git a/python-modules/ezbench/smartezbench.py b/python-modules/ezbench/smartezbench.py index 1a0e91c..9f4e623 100644 --- a/python-modules/ezbench/smartezbench.py +++ b/python-modules/ezbench/smartezbench.py @@ -92,7 +92,8 @@ def list_smart_ezbench_report_names(ezbench_dir, updatedSince = 0): return reports class TaskEntry: - def __init__(self, commit, test, rounds, resumeResultFile = None): + def __init__(self, commit, test, rounds, resumeResultFile = None, + user_requested = True): self.commit = commit self.test = test self.rounds = rounds @@ -100,6 +101,7 @@ class TaskEntry: self.start_date = None self.exec_time = None self.build_time = None + self.user_requested = user_requested self.cur_round = 1 self.last_round_completed_date = None @@ -186,6 +188,9 @@ class TaskEntry: else: string += "(no estimation available)" + if not self.user_requested: + string += " (autogenerated)" + return string GitCommit = namedtuple('GitCommit', 'sha1 timestamp') @@ -309,7 +314,14 @@ class SmartEzbench: del self.state['commits'] upgraded = True - latest_version = 2 + if self.state.get("version", 0) == 2: + self.__log(Criticality.II, "state: v2 -> v3: create a new 'auto' sections for tasks") + self.state['version'] = 3 + self.state['tasks']['auto'] = dict() + self.state['tasks']['auto']['commits'] = dict() + upgraded = True + + latest_version = 3 if self.state.get("version", 0) > latest_version: msg = "The state's version is higher than the latest supported version: {} vs {}" raise ValueError(msg.format(self.state.get("version", 0), latest_version)) @@ -530,15 +542,16 @@ class SmartEzbench: return total_rounds_before, total_rounds_after - def __add_test_unlocked__(self, commit, test, rounds): + def __add_test_unlocked__(self, commit, test, rounds, user_requested=True): scm = self.repo() if scm is not None: commit = scm.full_version_name(commit) - if rounds is None: - rounds = 0 + if user_requested: + commits = self.state['tasks']['user']['commits'] + else: + commits = self.state['tasks']['auto']['commits'] - commits = self.state['tasks']['user']['commits'] rounds_old, rounds_new = self.__task_tree_add_test__(commits, commit, test, rounds) # If we added rounds and the state was DONE, set it back to RUN @@ -546,16 +559,21 @@ class SmartEzbench: self.__running_mode_unlocked__(check_running=False) == RunningMode.DONE): self.__set_running_mode_unlocked__(RunningMode.RUN) + # If this is a user request, then remove all the "auto" tests, as + # additional user-requested data may render these auto tests useless + if user_requested: + self.state['tasks']['auto']['commits'] = dict() + return rounds_new - def add_test(self, commit, test, rounds = None): + def add_test(self, commit, test, rounds = None, user_requested=True): self.__reload_state(keep_lock=True) total_rounds = self.__add_test_unlocked__(commit, test, rounds) self.__save_state() self.__release_lock() return total_rounds - def add_testset(self, commit, testset, rounds = 1, ensure=False): + def add_testset(self, commit, testset, rounds = 1, ensure=False, user_requested=True): self.__reload_state(keep_lock=True) self.__log(Criticality.II, "Add the testset {} ({} tests)".format(testset.name, @@ -563,14 +581,14 @@ class SmartEzbench: for test in sorted(testset.keys()): if not ensure: - self.__add_test_unlocked__(commit, test, testset[test] * rounds) + self.__add_test_unlocked__(commit, test, testset[test] * rounds, user_requested) else: - self.__force_test_rounds_unlocked__(commit, test, testset[test] * rounds) + self.__force_test_rounds_unlocked__(commit, test, testset[test] * rounds, user_requested) self.__save_state() self.__release_lock() - def __force_test_rounds_unlocked__(self, commit, test, at_least): + def __force_test_rounds_unlocked__(self, commit, test, at_least, user_requested=True): scm = self.repo() if scm is not None: commit = scm.full_version_name(commit) @@ -580,7 +598,11 @@ class SmartEzbench: else: at_least = int(at_least) - commits = self.state['tasks']['user']['commits'] + if user_requested: + commits = self.state['tasks']['user']['commits'] + else: + commits = self.state['tasks']['auto']['commits'] + if commit not in commits: commits[commit] = dict() commits[commit]["tests"] = dict() @@ -597,9 +619,9 @@ class SmartEzbench: else: return 0 - def force_test_rounds(self, commit, test, at_least): + def force_test_rounds(self, commit, test, at_least, user_requested=True): self.__reload_state(keep_lock=True) - ret = self.__force_test_rounds_unlocked__(commit, test, at_least) + ret = self.__force_test_rounds_unlocked__(commit, test, at_least, user_requested) self.__save_state() self.__release_lock() @@ -632,10 +654,7 @@ class SmartEzbench: return c, tl, self._events_str - def __prioritize_runs(self, task_tree, deployed_version, resumable_tasks): - task_list = deque() - - # Aggregate all the subtests + def __aggregate_subtests__(self, task_tree): for commit in task_tree: test_subtests = dict() test_rounds = dict() @@ -656,6 +675,28 @@ class SmartEzbench: task_tree[commit]["tests"][full_name] = dict() task_tree[commit]["tests"][full_name]["rounds"] = test_rounds[basename] + def __prioritize_runs_add_by_commit__(self, task_list, task_tree, user_requested=True): + # Add all the remaining tasks in whatever order! + for commit in task_tree: + for test in task_tree[commit]["tests"]: + rounds = task_tree[commit]["tests"][test]["rounds"] + task_list.append(TaskEntry(commit, test, rounds, user_requested=user_requested)) + + def __prioritize_runs_add_deployed_first__(self, task_list, deployed_version, task_tree, user_requested=True): + if deployed_version is not None and deployed_version in task_tree: + for test in task_tree[deployed_version]["tests"]: + rounds = task_tree[deployed_version]["tests"][test]["rounds"] + task_list.append(TaskEntry(deployed_version, test, rounds, + user_requested=True)) + del task_tree[deployed_version] + + def __prioritize_runs(self, task_tree_user, task_tree_auto, deployed_version, resumable_tasks): + task_list = deque() + + # Aggregate all the subtests + self.__aggregate_subtests__(task_tree_user) + self.__aggregate_subtests__(task_tree_auto) + # Schedule resumable tasks. First the already-deployed # versions, other versions later for task in resumable_tasks: @@ -671,21 +712,17 @@ class SmartEzbench: task_list.append(entry) # Get rid of the task - self.__task_tree_add_test__(task_tree, entry.commit, entry.test, -1) + before, after = self.__task_tree_add_test__(task_tree_user, entry.commit, entry.test, -1) + if before == after: + self.__task_tree_add_test__(task_tree_auto, entry.commit, entry.test, -1) - # Schedule the tests using the already-deployed version after - # all resumable tasks - if deployed_version is not None and deployed_version in task_tree: - for test in task_tree[deployed_version]["tests"]: - rounds = task_tree[deployed_version]["tests"][test]["rounds"] - task_list.append(TaskEntry(deployed_version, test, rounds)) - del task_tree[deployed_version] + # Priority order: User-first, then Auto. Tests on the currently-deployed + # version first + self.__prioritize_runs_add_deployed_first__(task_list, deployed_version, task_tree_user, user_requested=True) + self.__prioritize_runs_add_by_commit__(task_list, task_tree_user, user_requested=True) - # Add all the remaining tasks in whatever order! - for commit in task_tree: - for test in task_tree[commit]["tests"]: - rounds = task_tree[commit]["tests"][test]["rounds"] - task_list.append(TaskEntry(commit, test, rounds)) + self.__prioritize_runs_add_deployed_first__(task_list, deployed_version, task_tree_auto, user_requested=False) + self.__prioritize_runs_add_by_commit__(task_list, task_tree_auto, user_requested=False) return task_list @@ -721,10 +758,11 @@ class SmartEzbench: def __remove_task_from_tasktree__(self, task_tree, commit, full_name, rounds): # Verify both that the short and long version names are used if commit not in task_tree: - return False + return 0 if full_name not in task_tree[commit]["tests"]: - return False + return 0 + rounds = task_tree[commit]["tests"][full_name]['rounds'] task_tree[commit]["tests"][full_name]['rounds'] -= rounds if task_tree[commit]["tests"][full_name]['rounds'] <= 0: @@ -733,12 +771,26 @@ class SmartEzbench: if len(task_tree[commit]["tests"]) == 0: del task_tree[commit] - return True + return rounds + + @classmethod + def __remove_existing_tasks_from_tree(cls, report, task_tree_user, task_tree_auto): + for commit in report.commits: + for result in commit.results.values(): + for key in result.results(): + full_name = Test.partial_name(result.test.full_name, [key]) + + rounds_found = len(result.result(key)) + user_rounds = SmartEzbench.__remove_task_from_tasktree__(task_tree_user, commit.full_sha1, full_name, rounds_found) + + SmartEzbench.__remove_task_from_tasktree__(task_tree_auto, commit.full_sha1, full_name, rounds_found - user_rounds) + @classmethod def __generate_task_and_events_list__(cls, q, state, log_folder, scm): exit_code = 1 - task_tree = list() + task_tree_user = list() + task_tree_auto = list() events_str = [] resumable_tasks = [] @@ -759,23 +811,24 @@ class SmartEzbench: events_str.append(str(event)) # Walk down the report and get rid of every run that has already been made! - task_tree = copy.deepcopy(self.state['tasks']['user']['commits']) - - for commit in report.commits: - for result in commit.results.values(): - for key in result.results(): - full_name = Test.partial_name(result.test.full_name, [key]) - SmartEzbench.__remove_task_from_tasktree__(task_tree, commit.full_sha1, full_name, len(result.result(key))) + task_tree_user = copy.deepcopy(state['tasks']['user']['commits']) + task_tree_auto = copy.deepcopy(state['tasks']['auto']['commits']) + cls.__remove_existing_tasks_from_tree(report, task_tree_user, task_tree_auto) resumable_tasks = report.journal.incomplete_tests() # Delete the tests on commits that do not compile for commit in report.commits: if commit.build_broken(): - if commit.full_sha1 in task_tree: - del task_tree[commit.full_sha1] - elif commit.sha1 in task_tree: - del task_tree[commit.sha1] + if commit.full_sha1 in task_tree_user: + del task_tree_user[commit.full_sha1] + elif commit.sha1 in task_tree_user: + del task_tree_user[commit.sha1] + + if commit.full_sha1 in task_tree_auto: + del task_tree_auto[commit.full_sha1] + elif commit.sha1 in task_tree_auto: + del task_tree_auto[commit.sha1] exit_code = 0 except Exception as e: @@ -784,7 +837,7 @@ class SmartEzbench: pass # Return the result - q.put((exit_code, task_tree, events_str, resumable_tasks)) + q.put((exit_code, task_tree_user, task_tree_auto, events_str, resumable_tasks)) def run(self): self.__log(Criticality.II, "----------------------") @@ -819,23 +872,30 @@ class SmartEzbench: p = multiprocessing.Process(target=SmartEzbench.__generate_task_and_events_list__, args=(q, self.state, self.log_folder, self.repo())) p.start() - exit_code, task_tree, self._events_str, resumable_tasks = q.get() + exit_code, task_tree_user, task_tree_auto, self._events_str, resumable_tasks = q.get() p.join() - if len(task_tree) == 0: + if len(task_tree_user) == 0 and len(task_tree_auto) == 0: self.__log(Criticality.II, "Nothing left to do, exit") return False - task_tree_str = pprint.pformat(task_tree) - self.__log(Criticality.II, "Task list: {tsk_str}".format(tsk_str=task_tree_str)) + task_tree_user_str = pprint.pformat(task_tree_user) + task_tree_auto_str = pprint.pformat(task_tree_auto) + self.__log(Criticality.II, "Task list (user): {tsk_str}".format(tsk_str=task_tree_user_str)) + self.__log(Criticality.II, "Task list (auto): {tsk_str}".format(tsk_str=task_tree_auto_str)) self.__log(Criticality.II, "Incomplete runs: {}".format([r['result_file'] for r in resumable_tasks])) - # Lock the report for further changes (like for profiles) - self.__write_attribute__('beenRunBefore', True) + # Write all the changes to the state + self.__reload_state(keep_lock=True) + try: + self.__write_attribute_unlocked__('beenRunBefore', True) + self.__save_state() + finally: + self.__release_lock() # Prioritize --> return a list of commits to do in order self._task_lock.acquire() - self._task_list = self.__prioritize_runs(task_tree, deployed_commit, resumable_tasks) + self._task_list = self.__prioritize_runs(task_tree_user, task_tree_auto, deployed_commit, resumable_tasks) # Call the hook file, telling we started running self.__call_hook__('start_running_tests') @@ -927,7 +987,13 @@ class SmartEzbench: self._task_current.round_done() - self._task_current = None + # Now that we have run everything, we can delete the "auto" tests + self.__reload_state(keep_lock=True) + try: + self.state['tasks']['auto']['commits'] = dict() + self.__save_state() + finally: + self.__release_lock() self.__done_running__(runner) self.__log(Criticality.II, "Done") @@ -1151,7 +1217,7 @@ class SmartEzbench: self.__log(Criticality.DD, "Add all the tasks using commit {}".format(commit)) for t in tasks_sorted: if t[1] == commit: - added = self.__force_test_rounds_unlocked__(t[1], t[2], t[3]) + added = self.__force_test_rounds_unlocked__(t[1], t[2], t[3], user_requested=False) if added > 0: self.__log(Criticality.II, "Scheduled {} more runs for the test {} on commit {}".format(added, t[2], commit)) -- cgit v1.2.3