smartezbench: allow specifying if a task is user- or auto-requested

We will first run user-requested tasks, then handle auto-requested
ones. We also reset the auto tasks after every call to run(), to
make sure the decision to run something is always up to date!

1 files changed, 122 insertions, 56 deletions

diff --git a/python-modules/ezbench/smartezbench.py b/python-modules/ezbench/smartezbench.py
index 1a0e91c..9f4e623 100644
--- a/python-modules/ezbench/smartezbench.py
+++ b/python-modules/ezbench/smartezbench.py
@@ -92,7 +92,8 @@ def list_smart_ezbench_report_names(ezbench_dir, updatedSince = 0):
     return reports
 
 class TaskEntry:
-    def __init__(self, commit, test, rounds, resumeResultFile = None):
+    def __init__(self, commit, test, rounds, resumeResultFile = None,
+                 user_requested = True):
         self.commit = commit
         self.test = test
         self.rounds = rounds
@@ -100,6 +101,7 @@ class TaskEntry:
         self.start_date = None
         self.exec_time = None
         self.build_time = None
+        self.user_requested = user_requested
 
         self.cur_round = 1
         self.last_round_completed_date = None
@@ -186,6 +188,9 @@ class TaskEntry:
             else:
                 string += "(no estimation available)"
 
+        if not self.user_requested:
+            string += " (autogenerated)"
+
         return string
 
 GitCommit = namedtuple('GitCommit', 'sha1 timestamp')
@@ -309,7 +314,14 @@ class SmartEzbench:
             del self.state['commits']
             upgraded = True
 
-        latest_version = 2
+        if self.state.get("version", 0) == 2:
+            self.__log(Criticality.II, "state: v2 -> v3: create a new 'auto' sections for tasks")
+            self.state['version'] = 3
+            self.state['tasks']['auto'] = dict()
+            self.state['tasks']['auto']['commits'] = dict()
+            upgraded = True
+
+        latest_version = 3
         if self.state.get("version", 0) > latest_version:
             msg = "The state's version is higher than the latest supported version: {} vs {}"
             raise ValueError(msg.format(self.state.get("version", 0), latest_version))
@@ -530,15 +542,16 @@ class SmartEzbench:
 
         return total_rounds_before, total_rounds_after
 
-    def __add_test_unlocked__(self, commit, test, rounds):
+    def __add_test_unlocked__(self, commit, test, rounds, user_requested=True):
         scm = self.repo()
         if scm is not None:
             commit = scm.full_version_name(commit)
 
-        if rounds is None:
-            rounds = 0
+        if user_requested:
+            commits = self.state['tasks']['user']['commits']
+        else:
+            commits = self.state['tasks']['auto']['commits']
 
-        commits = self.state['tasks']['user']['commits']
         rounds_old, rounds_new = self.__task_tree_add_test__(commits, commit, test, rounds)
 
         # If we added rounds and the state was DONE, set it back to RUN
@@ -546,16 +559,21 @@ class SmartEzbench:
             self.__running_mode_unlocked__(check_running=False) == RunningMode.DONE):
             self.__set_running_mode_unlocked__(RunningMode.RUN)
 
+            # If this is a user request, then remove all the "auto" tests, as
+            # additional user-requested data may render these auto tests useless
+            if user_requested:
+                self.state['tasks']['auto']['commits'] = dict()
+
         return rounds_new
 
-    def add_test(self, commit, test, rounds = None):
+    def add_test(self, commit, test, rounds = None, user_requested=True):
         self.__reload_state(keep_lock=True)
         total_rounds = self.__add_test_unlocked__(commit, test, rounds)
         self.__save_state()
         self.__release_lock()
         return total_rounds
 
-    def add_testset(self, commit, testset, rounds = 1, ensure=False):
+    def add_testset(self, commit, testset, rounds = 1, ensure=False, user_requested=True):
         self.__reload_state(keep_lock=True)
 
         self.__log(Criticality.II, "Add the testset {} ({} tests)".format(testset.name,
@@ -563,14 +581,14 @@ class SmartEzbench:
 
         for test in sorted(testset.keys()):
             if not ensure:
-                self.__add_test_unlocked__(commit, test, testset[test] * rounds)
+                self.__add_test_unlocked__(commit, test, testset[test] * rounds, user_requested)
             else:
-                self.__force_test_rounds_unlocked__(commit, test, testset[test] * rounds)
+                self.__force_test_rounds_unlocked__(commit, test, testset[test] * rounds, user_requested)
 
         self.__save_state()
         self.__release_lock()
 
-    def __force_test_rounds_unlocked__(self, commit, test, at_least):
+    def __force_test_rounds_unlocked__(self, commit, test, at_least, user_requested=True):
         scm = self.repo()
         if scm is not None:
             commit = scm.full_version_name(commit)
@@ -580,7 +598,11 @@ class SmartEzbench:
         else:
             at_least = int(at_least)
 
-        commits = self.state['tasks']['user']['commits']
+        if user_requested:
+            commits = self.state['tasks']['user']['commits']
+        else:
+            commits = self.state['tasks']['auto']['commits']
+
         if commit not in commits:
             commits[commit] = dict()
             commits[commit]["tests"] = dict()
@@ -597,9 +619,9 @@ class SmartEzbench:
         else:
             return 0
 
-    def force_test_rounds(self, commit, test, at_least):
+    def force_test_rounds(self, commit, test, at_least, user_requested=True):
         self.__reload_state(keep_lock=True)
-        ret = self.__force_test_rounds_unlocked__(commit, test, at_least)
+        ret = self.__force_test_rounds_unlocked__(commit, test, at_least, user_requested)
         self.__save_state()
         self.__release_lock()
 
@@ -632,10 +654,7 @@ class SmartEzbench:
 
         return c, tl, self._events_str
 
-    def __prioritize_runs(self, task_tree, deployed_version, resumable_tasks):
-        task_list = deque()
-
-        # Aggregate all the subtests
+    def __aggregate_subtests__(self, task_tree):
         for commit in task_tree:
             test_subtests = dict()
             test_rounds = dict()
@@ -656,6 +675,28 @@ class SmartEzbench:
                 task_tree[commit]["tests"][full_name] = dict()
                 task_tree[commit]["tests"][full_name]["rounds"] = test_rounds[basename]
 
+    def __prioritize_runs_add_by_commit__(self, task_list, task_tree, user_requested=True):
+        # Add all the remaining tasks in whatever order!
+        for commit in task_tree:
+            for test in task_tree[commit]["tests"]:
+                rounds = task_tree[commit]["tests"][test]["rounds"]
+                task_list.append(TaskEntry(commit, test, rounds, user_requested=user_requested))
+
+    def __prioritize_runs_add_deployed_first__(self, task_list, deployed_version, task_tree, user_requested=True):
+        if deployed_version is not None and deployed_version in task_tree:
+            for test in task_tree[deployed_version]["tests"]:
+                rounds = task_tree[deployed_version]["tests"][test]["rounds"]
+                task_list.append(TaskEntry(deployed_version, test, rounds,
+                                           user_requested=True))
+            del task_tree[deployed_version]
+
+    def __prioritize_runs(self, task_tree_user, task_tree_auto, deployed_version, resumable_tasks):
+        task_list = deque()
+
+        # Aggregate all the subtests
+        self.__aggregate_subtests__(task_tree_user)
+        self.__aggregate_subtests__(task_tree_auto)
+
         # Schedule resumable tasks. First the already-deployed
         # versions, other versions later
         for task in resumable_tasks:
@@ -671,21 +712,17 @@ class SmartEzbench:
                 task_list.append(entry)
 
             # Get rid of the task
-            self.__task_tree_add_test__(task_tree, entry.commit, entry.test, -1)
+            before, after = self.__task_tree_add_test__(task_tree_user, entry.commit, entry.test, -1)
+            if before == after:
+                self.__task_tree_add_test__(task_tree_auto, entry.commit, entry.test, -1)
 
-        # Schedule the tests using the already-deployed version after
-        # all resumable tasks
-        if deployed_version is not None and deployed_version in task_tree:
-            for test in task_tree[deployed_version]["tests"]:
-                rounds = task_tree[deployed_version]["tests"][test]["rounds"]
-                task_list.append(TaskEntry(deployed_version, test, rounds))
-            del task_tree[deployed_version]
+        # Priority order: User-first, then Auto. Tests on the currently-deployed
+        # version first
+        self.__prioritize_runs_add_deployed_first__(task_list, deployed_version, task_tree_user, user_requested=True)
+        self.__prioritize_runs_add_by_commit__(task_list, task_tree_user, user_requested=True)
 
-        # Add all the remaining tasks in whatever order!
-        for commit in task_tree:
-            for test in task_tree[commit]["tests"]:
-                rounds = task_tree[commit]["tests"][test]["rounds"]
-                task_list.append(TaskEntry(commit, test, rounds))
+        self.__prioritize_runs_add_deployed_first__(task_list, deployed_version, task_tree_auto, user_requested=False)
+        self.__prioritize_runs_add_by_commit__(task_list, task_tree_auto, user_requested=False)
 
         return task_list
 
@@ -721,10 +758,11 @@ class SmartEzbench:
     def __remove_task_from_tasktree__(self, task_tree, commit, full_name, rounds):
         # Verify both that the short and long version names are used
         if commit not in task_tree:
-            return False
+            return 0
         if full_name not in task_tree[commit]["tests"]:
-            return False
+            return 0
 
+        rounds =  task_tree[commit]["tests"][full_name]['rounds']
         task_tree[commit]["tests"][full_name]['rounds'] -= rounds
 
         if task_tree[commit]["tests"][full_name]['rounds'] <= 0:
@@ -733,12 +771,26 @@ class SmartEzbench:
         if len(task_tree[commit]["tests"]) == 0:
             del task_tree[commit]
 
-        return True
+        return rounds
+
+    @classmethod
+    def __remove_existing_tasks_from_tree(cls, report, task_tree_user, task_tree_auto):
+        for commit in report.commits:
+            for result in commit.results.values():
+                for key in result.results():
+                    full_name = Test.partial_name(result.test.full_name, [key])
+
+                    rounds_found = len(result.result(key))
+                    user_rounds = SmartEzbench.__remove_task_from_tasktree__(task_tree_user, commit.full_sha1, full_name, rounds_found)
+
+                    SmartEzbench.__remove_task_from_tasktree__(task_tree_auto, commit.full_sha1, full_name, rounds_found - user_rounds)
+
 
     @classmethod
     def __generate_task_and_events_list__(cls, q, state, log_folder, scm):
         exit_code = 1
-        task_tree = list()
+        task_tree_user = list()
+        task_tree_auto = list()
         events_str = []
         resumable_tasks = []
 
@@ -759,23 +811,24 @@ class SmartEzbench:
                 events_str.append(str(event))
 
             # Walk down the report and get rid of every run that has already been made!
-            task_tree = copy.deepcopy(self.state['tasks']['user']['commits'])
-
-            for commit in report.commits:
-                for result in commit.results.values():
-                    for key in result.results():
-                        full_name = Test.partial_name(result.test.full_name, [key])
-                        SmartEzbench.__remove_task_from_tasktree__(task_tree, commit.full_sha1, full_name, len(result.result(key)))
+            task_tree_user = copy.deepcopy(state['tasks']['user']['commits'])
+            task_tree_auto = copy.deepcopy(state['tasks']['auto']['commits'])
+            cls.__remove_existing_tasks_from_tree(report, task_tree_user, task_tree_auto)
 
             resumable_tasks = report.journal.incomplete_tests()
 
             # Delete the tests on commits that do not compile
             for commit in report.commits:
                 if commit.build_broken():
-                    if commit.full_sha1 in task_tree:
-                        del task_tree[commit.full_sha1]
-                    elif commit.sha1 in task_tree:
-                        del task_tree[commit.sha1]
+                    if commit.full_sha1 in task_tree_user:
+                        del task_tree_user[commit.full_sha1]
+                    elif commit.sha1 in task_tree_user:
+                        del task_tree_user[commit.sha1]
+
+                    if commit.full_sha1 in task_tree_auto:
+                        del task_tree_auto[commit.full_sha1]
+                    elif commit.sha1 in task_tree_auto:
+                        del task_tree_auto[commit.sha1]
 
             exit_code = 0
         except Exception as e:
@@ -784,7 +837,7 @@ class SmartEzbench:
             pass
 
         # Return the result
-        q.put((exit_code, task_tree, events_str, resumable_tasks))
+        q.put((exit_code, task_tree_user, task_tree_auto, events_str, resumable_tasks))
 
     def run(self):
         self.__log(Criticality.II, "----------------------")
@@ -819,23 +872,30 @@ class SmartEzbench:
         p = multiprocessing.Process(target=SmartEzbench.__generate_task_and_events_list__,
                                     args=(q, self.state, self.log_folder, self.repo()))
         p.start()
-        exit_code, task_tree, self._events_str, resumable_tasks = q.get()
+        exit_code, task_tree_user, task_tree_auto, self._events_str, resumable_tasks = q.get()
         p.join()
 
-        if len(task_tree) == 0:
+        if len(task_tree_user) == 0 and len(task_tree_auto) == 0:
             self.__log(Criticality.II, "Nothing left to do, exit")
             return False
 
-        task_tree_str = pprint.pformat(task_tree)
-        self.__log(Criticality.II, "Task list: {tsk_str}".format(tsk_str=task_tree_str))
+        task_tree_user_str = pprint.pformat(task_tree_user)
+        task_tree_auto_str = pprint.pformat(task_tree_auto)
+        self.__log(Criticality.II, "Task list (user): {tsk_str}".format(tsk_str=task_tree_user_str))
+        self.__log(Criticality.II, "Task list (auto): {tsk_str}".format(tsk_str=task_tree_auto_str))
         self.__log(Criticality.II, "Incomplete runs: {}".format([r['result_file'] for r in resumable_tasks]))
 
-        # Lock the report for further changes (like for profiles)
-        self.__write_attribute__('beenRunBefore', True)
+        # Write all the changes to the state
+        self.__reload_state(keep_lock=True)
+        try:
+            self.__write_attribute_unlocked__('beenRunBefore', True)
+            self.__save_state()
+        finally:
+            self.__release_lock()
 
         # Prioritize --> return a list of commits to do in order
         self._task_lock.acquire()
-        self._task_list = self.__prioritize_runs(task_tree, deployed_commit, resumable_tasks)
+        self._task_list = self.__prioritize_runs(task_tree_user, task_tree_auto, deployed_commit, resumable_tasks)
 
         # Call the hook file, telling we started running
         self.__call_hook__('start_running_tests')
@@ -927,7 +987,13 @@ class SmartEzbench:
 
                 self._task_current.round_done()
 
-        self._task_current = None
+        # Now that we have run everything, we can delete the "auto" tests
+        self.__reload_state(keep_lock=True)
+        try:
+            self.state['tasks']['auto']['commits'] = dict()
+            self.__save_state()
+        finally:
+            self.__release_lock()
 
         self.__done_running__(runner)
         self.__log(Criticality.II, "Done")
@@ -1151,7 +1217,7 @@ class SmartEzbench:
             self.__log(Criticality.DD, "Add all the tasks using commit {}".format(commit))
             for t in tasks_sorted:
                 if t[1] == commit:
-                    added = self.__force_test_rounds_unlocked__(t[1], t[2], t[3])
+                    added = self.__force_test_rounds_unlocked__(t[1], t[2], t[3], user_requested=False)
                     if added > 0:
                         self.__log(Criticality.II,
                                 "Scheduled {} more runs for the test {} on commit {}".format(added, t[2], commit))