Add an optional warning in case the system doesn't shutdown in a

timely manner.

Signed-off-by: Jean-Marc Eurin <jmeurin@google.com>


git-svn-id: svn://test.kernel.org/autotest/trunk@2975 592f7852-d20e-0410-864c-8624ca9c26a4

2 files changed, 23 insertions, 7 deletions

diff --git a/server/hosts/abstract_ssh.py b/server/hosts/abstract_ssh.py
index 093838ab..063e7cf8 100644
--- a/server/hosts/abstract_ssh.py
+++ b/server/hosts/abstract_ssh.py
@@ -282,7 +282,7 @@ class AbstractSSHHost(SiteHost):
         return False
 
 
-    def wait_down(self, timeout=None):
+    def wait_down(self, timeout=None, warning_timer=None):
         """
         Wait until the remote host is down or the timeout expires.
 
@@ -290,19 +290,33 @@ class AbstractSSHHost(SiteHost):
         host fails.
 
         Args:
-                timeout: time limit in seconds before returning even
-                        if the host is not up.
+            timeout: time limit in seconds before returning even
+                     if the host is still up.
+            warning_timer: time limit in seconds that will generate
+                     a warning if the host is not down yet.
 
         Returns:
                 True if the host was found to be down, False otherwise
         """
+        current_time = time.time()
         if timeout:
-            end_time = time.time() + timeout
+            end_time = current_time + timeout
 
-        while not timeout or time.time() < end_time:
+        if warning_timer:
+            warn_time = current_time + warning_timer
+
+        while not timeout or current_time < end_time:
             if not self.is_up():
                 return True
+
+            if warning_timer and current_time > warn_time:
+                self.record("WARN", None, "shutdown",
+                            "Shutdown took longer than %ds" % warning_timer)
+                # Print the warning only once.
+                warning_timer = None
+
             time.sleep(1)
+            current_time = time.time()
 
         return False
 
diff --git a/server/hosts/base_classes.py b/server/hosts/base_classes.py
index 918c550b..dae0a414 100644
--- a/server/hosts/base_classes.py
+++ b/server/hosts/base_classes.py
@@ -58,6 +58,7 @@ class Host(object):
     job = None
     DEFAULT_REBOOT_TIMEOUT = 1800
     WAIT_DOWN_REBOOT_TIMEOUT = 600
+    WAIT_DOWN_REBOOT_WARNING = 600
     HOURS_TO_WAIT_FOR_RECOVERY = 2.5
 
 
@@ -146,14 +147,15 @@ class Host(object):
         raise NotImplementedError('Wait up not implemented!')
 
 
-    def wait_down(self, timeout=None):
+    def wait_down(self, timeout=None, warning_timer=None):
         raise NotImplementedError('Wait down not implemented!')
 
 
     def wait_for_restart(self, timeout=DEFAULT_REBOOT_TIMEOUT, **dargs):
         """ Wait for the host to come back from a reboot. This is a generic
         implementation based entirely on wait_up and wait_down. """
-        if not self.wait_down(self.WAIT_DOWN_REBOOT_TIMEOUT):
+        if not self.wait_down(timeout=self.WAIT_DOWN_REBOOT_TIMEOUT,
+                              warning_timer=self.WAIT_DOWN_REBOOT_WARNING):
             self.record("ABORT", None, "reboot.verify", "shut down failed")
             raise error.AutoservRebootError("Host did not shut down")
         self.wait_up(timeout)