diff options
| -rw-r--r-- | client/common_lib/packages.py | 3 | ||||
| -rw-r--r-- | client/common_lib/utils.py | 46 | ||||
| -rw-r--r-- | server/hosts/abstract_ssh.py | 8 | ||||
| -rw-r--r-- | server/hosts/paramiko_host.py | 7 | ||||
| -rw-r--r-- | server/hosts/ssh_host.py | 14 |
5 files changed, 44 insertions, 34 deletions
diff --git a/client/common_lib/packages.py b/client/common_lib/packages.py index ac3cb94e..def8cccc 100644 --- a/client/common_lib/packages.py +++ b/client/common_lib/packages.py @@ -165,7 +165,8 @@ class BasePackageManager(object): ''' new_dargs = dict(run_function_dargs) new_dargs.update(_run_command_dargs) - new_dargs.update({'verbose' : False}) + # avoid polluting logs with extremely verbose packaging output + new_dargs.update({'stdout_tee' : None}) return run_function(command, *_run_command_args, **new_dargs) diff --git a/client/common_lib/utils.py b/client/common_lib/utils.py index 93617edb..eca81f22 100644 --- a/client/common_lib/utils.py +++ b/client/common_lib/utils.py @@ -5,7 +5,7 @@ import os, pickle, random, re, resource, select, shutil, signal, StringIO import socket, struct, subprocess, sys, time, textwrap, urllib, urlparse import warnings, smtplib, logging, urllib2 -from autotest_lib.client.common_lib import error, barrier +from autotest_lib.client.common_lib import error, barrier, logging_manager def deprecated(func): """This is a decorator which can be used to mark functions as deprecated. @@ -20,12 +20,32 @@ def deprecated(func): return new_func +class _NullStream(object): + def write(self, data): + pass + + + def flush(self): + pass + + +TEE_TO_LOGS = object() +_the_null_stream = _NullStream() + +def get_stream_tee_file(stream, level): + if stream is None: + return _the_null_stream + if stream is TEE_TO_LOGS: + return logging_manager.LoggingFile(level=level) + return stream + + class BgJob(object): def __init__(self, command, stdout_tee=None, stderr_tee=None, verbose=True, stdin=None): self.command = command - self.stdout_tee = stdout_tee - self.stderr_tee = stderr_tee + self.stdout_tee = get_stream_tee_file(stdout_tee, logging.DEBUG) + self.stderr_tee = get_stream_tee_file(stderr_tee, logging.ERROR) self.result = CmdResult(command) if verbose: logging.debug("Running '%s'" % command) @@ -60,12 +80,12 @@ class BgJob(object): # perform a single read data = os.read(pipe.fileno(), 1024) buf.write(data) - if tee: - tee.write(data) - tee.flush() + tee.write(data) def cleanup(self): + self.stdout_tee.flush() + self.stderr_tee.flush() self.sp.stdout.close() self.sp.stderr.close() self.result.stdout = self.stdout_file.getvalue() @@ -325,6 +345,7 @@ def run(command, timeout=None, ignore_status=False, will be written as it is generated (data will still be stored in result.stdout) stderr_tee: likewise for stderr + verbose: if True, log the command being run stdin: stdin to pass to the executed process Returns: @@ -527,7 +548,7 @@ def pid_is_alive(pid): def system(command, timeout=None, ignore_status=False): """This function returns the exit status of command.""" return run(command, timeout=timeout, ignore_status=ignore_status, - stdout_tee=sys.stdout, stderr_tee=sys.stderr).exit_status + stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS).exit_status def system_parallel(commands, timeout=None, ignore_status=False): @@ -535,14 +556,14 @@ def system_parallel(commands, timeout=None, ignore_status=False): list of commands.""" return [bg_jobs.exit_status for bg_jobs in run_parallel(commands, timeout=timeout, ignore_status=ignore_status, - stdout_tee=sys.stdout, stderr_tee=sys.stderr)] + stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] def system_output(command, timeout=None, ignore_status=False, retain_output=False): if retain_output: out = run(command, timeout=timeout, ignore_status=ignore_status, - stdout_tee=sys.stdout, stderr_tee=sys.stderr).stdout + stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS).stdout else: out = run(command, timeout=timeout, ignore_status=ignore_status).stdout if out[-1:] == '\n': out = out[:-1] @@ -552,9 +573,10 @@ def system_output(command, timeout=None, ignore_status=False, def system_output_parallel(commands, timeout=None, ignore_status=False, retain_output=False): if retain_output: - out = [bg_job.stdout for bg_job in run_parallel(commands, - timeout=timeout, ignore_status=ignore_status, - stdout_tee=sys.stdout, stderr_tee=sys.stderr)] + out = [bg_job.stdout for bg_job + in run_parallel(commands, timeout=timeout, + ignore_status=ignore_status, + stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] else: out = [bg_job.stdout for bg_job in run_parallel(commands, timeout=timeout, ignore_status=ignore_status)] diff --git a/server/hosts/abstract_ssh.py b/server/hosts/abstract_ssh.py index ee6556d7..9548264e 100644 --- a/server/hosts/abstract_ssh.py +++ b/server/hosts/abstract_ssh.py @@ -418,11 +418,3 @@ class AbstractSSHHost(SiteHost): raise # only want to raise if it's a space issue except Exception: pass # autotest dir may not exist, etc. ignore - - - def _get_stream_tee_file(self, stream, level, verbose): - if stream is not TEE_TO_LOGS: - return stream - if not verbose: - return None - return logging_manager.LoggingFile(level=level) diff --git a/server/hosts/paramiko_host.py b/server/hosts/paramiko_host.py index 2cc1b372..3b80cdf6 100644 --- a/server/hosts/paramiko_host.py +++ b/server/hosts/paramiko_host.py @@ -194,8 +194,7 @@ class ParamikoHost(abstract_ssh.AbstractSSHHost): def run(self, command, timeout=3600, ignore_status=False, - stdout_tee=abstract_ssh.TEE_TO_LOGS, - stderr_tee=abstract_ssh.TEE_TO_LOGS, + stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, connect_timeout=30, verbose=True): """ Run a command on the remote host. @@ -218,8 +217,8 @@ class ParamikoHost(abstract_ssh.AbstractSSHHost): AutoservSSHTimeout: ssh connection has timed out """ - stdout = self._get_stream_tee_file(stdout_tee, logging.DEBUG, verbose) - stderr = self._get_stream_tee_file(stderr_tee, logging.ERROR, verbose) + stdout = utils.get_stream_tee_file(stdout_tee, logging.DEBUG) + stderr = utils.get_stream_tee_file(stderr_tee, logging.ERROR) if verbose: logging.debug("ssh-paramiko: %s" % command) diff --git a/server/hosts/ssh_host.py b/server/hosts/ssh_host.py index 5bfe1685..36baabe7 100644 --- a/server/hosts/ssh_host.py +++ b/server/hosts/ssh_host.py @@ -89,8 +89,7 @@ class SSHHost(abstract_ssh.AbstractSSHHost): def run(self, command, timeout=3600, ignore_status=False, - stdout_tee=abstract_ssh.TEE_TO_LOGS, - stderr_tee=abstract_ssh.TEE_TO_LOGS, + stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, connect_timeout=30, options='', stdin=None, verbose=True): """ Run a command on the remote host. @@ -113,23 +112,20 @@ class SSHHost(abstract_ssh.AbstractSSHHost): execution was not 0 AutoservSSHTimeout: ssh connection has timed out """ - stdout = self._get_stream_tee_file(stdout_tee, logging.DEBUG, verbose) - stderr = self._get_stream_tee_file(stderr_tee, logging.ERROR, verbose) - if verbose: logging.debug("ssh: %s" % command) env = " ".join("=".join(pair) for pair in self.env.iteritems()) try: try: - return self._run(command, timeout, ignore_status, stdout, - stderr, connect_timeout, env, options, + return self._run(command, timeout, ignore_status, stdout_tee, + stderr_tee, connect_timeout, env, options, stdin=stdin) except error.AutoservSshPermissionDeniedError: logging.error("Permission denied to ssh; re-running with " "increased logging:") try: - self._run(command, timeout, ignore_status, stdout, - stderr, connect_timeout, env, '-v -v -v', + self._run(command, timeout, ignore_status, stdout_tee, + stderr_tee, connect_timeout, env, '-v -v -v', stdin=stdin) except Exception: pass |