I have attached the patch for the packaging system.

The documentation is available at : http://test.kernel.org/autotest/PackagingSystem

Signed-off-by: Ashwin Ganti <aganti@google.com>



git-svn-id: svn://test.kernel.org/autotest/trunk@1953 592f7852-d20e-0410-864c-8624ca9c26a4

51 files changed, 419 insertions, 90 deletions

diff --git a/client/bin/job.py b/client/bin/job.py
index 694b0a13..40f12b70 100755
--- a/client/bin/job.py
+++ b/client/bin/job.py
@@ -13,6 +13,7 @@ from autotest_lib.client.bin import autotest_utils, parallel, kernel, xen
 from autotest_lib.client.bin import profilers, fd_stack, boottool, harness
 from autotest_lib.client.bin import config, sysinfo, cpuset, test, filesystem
 from autotest_lib.client.common_lib import error, barrier, logging, utils
+from autotest_lib.client.common_lib import packages
 
 JOB_PREAMBLE = """
 from autotest_lib.client.common_lib.error import *
@@ -23,6 +24,9 @@ from autotest_lib.client.bin.autotest_utils import *
 class StepError(error.AutotestError):
     pass
 
+class NotAvailableError(error.AutotestError):
+    pass
+
 
 class base_job(object):
     """The actual job against which we do everything.
@@ -43,6 +47,8 @@ class base_job(object):
                     <autodir>/profilers/
             tmpdir
                     <autodir>/tmp/
+            pkgdir
+                    <autodir>/packages/
             resultdir
                     <autodir>/results/<jobtag>
             stdout
@@ -85,6 +91,9 @@ class base_job(object):
         self.current_step_ancestry = []
         self.next_step_index = 0
         self._load_state()
+        self.pkgmgr = packages.PackageManager(
+            self.autodir, run_function_dargs={'timeout':600})
+        self.pkgdir = os.path.join(self.autodir, 'packages')
 
         if not cont:
             """
@@ -97,6 +106,9 @@ class base_job(object):
             if not os.path.exists(self.tmpdir):
                 os.mkdir(self.tmpdir)
 
+            if not os.path.exists(self.pkgdir):
+                os.mkdir(self.pkgdir)
+
             results = os.path.join(self.autodir, 'results')
             if not os.path.exists(results):
                 os.mkdir(results)
@@ -226,7 +238,8 @@ class base_job(object):
         """Summon a xen object"""
         (results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
         build_dir = 'xen'
-        return xen.xen(self, base_tree, results_dir, tmp_dir, build_dir, leave, kjob)
+        return xen.xen(self, base_tree, results_dir, tmp_dir, build_dir,
+                       leave, kjob)
 
 
     def kernel(self, base_tree, results_dir = '', tmp_dir = '', leave = False):
@@ -242,17 +255,83 @@ class base_job(object):
         return barrier.barrier(*args, **kwds)
 
 
+    def install_pkg(self, name, pkg_type, install_dir):
+        '''
+        This method is a simple wrapper around the actual package
+        installation method in the Packager class. This is used
+        internally by the profilers, deps and tests code.
+        name : name of the package (ex: sleeptest, dbench etc.)
+        pkg_type : Type of the package (ex: test, dep etc.)
+        install_dir : The directory in which the source is actually
+                      untarred into. (ex: client/profilers/<name> for profilers)
+        '''
+        if len(self.pkgmgr.repo_urls) > 0:
+            self.pkgmgr.install_pkg(name, pkg_type,
+                                    self.pkgdir, install_dir)
+
+
+    def add_repository(self, repo_urls):
+        '''
+        Adds the repository locations to the job so that packages
+        can be fetched from them when needed. The repository list
+        needs to be a string list
+        Ex: job.add_repository(['http://blah1','http://blah2'])
+        '''
+        # TODO(aganti): Validate the list of the repository URLs.
+        repositories = repo_urls + self.pkgmgr.repo_urls
+        self.pkgmgr = packages.PackageManager(
+            self.autodir, repo_urls=repositories,
+            run_function_dargs={'timeout':600})
+        # Fetch the packages' checksum file that contains the checksums
+        # of all the packages if it is not already fetched. The checksum
+        # is always fetched whenever a job is first started. This
+        # is not done in the job's constructor as we don't have the list of
+        # the repositories there (and obviously don't care about this file
+        # if we are not using the repos)
+        try:
+            checksum_file_path = os.path.join(self.pkgmgr.pkgmgr_dir,
+                                              packages.CHECKSUM_FILE)
+            self.pkgmgr.fetch_pkg(packages.CHECKSUM_FILE, checksum_file_path,
+                                  use_checksum=False)
+        except packages.PackageFetchError, e:
+            # packaging system might not be working in this case
+            # Silently fall back to the normal case
+            pass
+
+
+    def require_gcc(self):
+        """
+        Test whether gcc is installed on the machine.
+        """
+        # check if gcc is installed on the system.
+        try:
+            utils.system('which gcc')
+        except error.CmdError, e:
+            raise NotAvailableError('gcc is required by this job and is '
+                                    'not available on the system')
+
+
     def setup_dep(self, deps):
         """Set up the dependencies for this test.
-
         deps is a list of libraries required for this test.
         """
+        # Fetch the deps from the repositories and set them up.
         for dep in deps:
+            dep_dir = os.path.join(self.autodir, 'deps', dep)
+            # Search for the dependency in the repositories if specified,
+            # else check locally.
             try:
-                os.chdir(os.path.join(self.autodir, 'deps', dep))
-                utils.system('./' + dep + '.py')
-            except Exception, e:
-                raise error.UnhandledTestError(e)
+                self.install_pkg(dep, 'dep', dep_dir)
+            except packages.PackageInstallError:
+                # see if the dep is there locally
+                pass
+
+            # dep_dir might not exist if it is not fetched from the repos
+            if not os.path.exists(dep_dir):
+                raise error.TestError("Dependency %s does not exist" % dep)
+
+            os.chdir(dep_dir)
+            utils.system('./' + dep + '.py')
 
 
     def _runtest(self, url, tag, args, dargs):
@@ -278,7 +357,7 @@ class base_job(object):
         if not url:
             raise TypeError("Test name is invalid. "
                             "Switched arguments?")
-        (group, testname) = test.testname(url)
+        (group, testname) = self.pkgmgr.get_package_name(url, 'test')
         namelen = len(testname)
         dargs = dargs.copy()
         tntag = dargs.pop('tag', None)
diff --git a/client/bin/job_unittest.py b/client/bin/job_unittest.py
index 6bab521a..bbfdffcd 100644
--- a/client/bin/job_unittest.py
+++ b/client/bin/job_unittest.py
@@ -5,7 +5,7 @@ import common
 
 from autotest_lib.client.bin import job, boottool, config, sysinfo, harness
 from autotest_lib.client.bin import test, xen, kernel, autotest_utils, cpuset
-from autotest_lib.client.common_lib import utils, error, logging
+from autotest_lib.client.common_lib import packages, utils, error, logging
 from autotest_lib.client.common_lib.test_utils import mock
 
 
@@ -70,12 +70,15 @@ class TestBaseJob(unittest.TestCase):
         download = os.path.join(self.autodir, 'tests', 'download')
         resultdir = os.path.join(self.autodir, 'results', self.jobtag)
         sysinfodir = os.path.join(resultdir, 'sysinfo')
+        pkgdir = os.path.join(self.autodir, 'packages')
 
         # record
         self.job._load_state.expect_call()
         if not cont:
             os.path.exists.expect_call(tmpdir).and_return(False)
             os.mkdir.expect_call(tmpdir)
+            os.path.exists.expect_call(pkgdir).and_return(False)
+            os.mkdir.expect_call(pkgdir)
             os.path.exists.expect_call(results).and_return(False)
             os.mkdir.expect_call(results)
             os.path.exists.expect_call(download).and_return(False)
@@ -259,16 +262,20 @@ class TestBaseJob(unittest.TestCase):
         self.god.stub_function(self.job, "setup_dirs")
         self.god.stub_class(kernel, "rpm_kernel")
         self.god.stub_function(kernel, "preprocess_path")
+        self.god.stub_function(self.job.pkgmgr, "fetch_pkg")
         results = 'results_dir'
         tmp = 'tmp'
         build = 'xen'
         path = "somepath.rpm"
+        packages_dir = os.path.join("autodir/packages", path)
 
         # record
         self.job.setup_dirs.expect_call(results,
                                         tmp).and_return((results, tmp))
         kernel.preprocess_path.expect_call(path).and_return(path)
-        mykernel = kernel.rpm_kernel.expect_new(self.job, path, results)
+        self.job.pkgmgr.fetch_pkg.expect_call(path, packages_dir, repo_url='')
+        mykernel = kernel.rpm_kernel.expect_new(self.job, packages_dir,
+                                                results)
 
         # check
         akernel = self.job.kernel(path, results, tmp)
@@ -305,7 +312,7 @@ class TestBaseJob(unittest.TestCase):
         self.construct_job(True)
 
         # set up stubs
-        self.god.stub_function(test, "testname")
+        self.god.stub_function(self.job.pkgmgr, 'get_package_name')
         self.god.stub_function(self.job, "_runtest")
 
         # create an unhandled error object
@@ -317,7 +324,8 @@ class TestBaseJob(unittest.TestCase):
         # set up the recording
         testname = "error_test"
         outputdir = os.path.join(self.job.resultdir, testname)
-        test.testname.expect_call(testname).and_return(("", testname))
+        self.job.pkgmgr.get_package_name.expect_call(
+            testname, 'test').and_return(("", testname))
         os.path.exists.expect_call(outputdir).and_return(False)
         os.mkdir.expect_call(outputdir)
         self.job.record.expect_call("START", testname, testname)
@@ -339,7 +347,7 @@ class TestBaseJob(unittest.TestCase):
         self.construct_job(True)
 
         # set up stubs
-        self.god.stub_function(test, "testname")
+        self.god.stub_function(self.job.pkgmgr, 'get_package_name')
         self.god.stub_function(self.job, "_runtest")
 
         # create an unhandled error object
@@ -352,7 +360,8 @@ class TestBaseJob(unittest.TestCase):
         # set up the recording
         testname = "error_test"
         outputdir = os.path.join(self.job.resultdir, testname)
-        test.testname.expect_call(testname).and_return(("", testname))
+        self.job.pkgmgr.get_package_name.expect_call(
+            testname, 'test').and_return(("", testname))
         os.path.exists.expect_call(outputdir).and_return(False)
         os.mkdir.expect_call(outputdir)
         self.job.record.expect_call("START", testname, testname)
diff --git a/client/bin/kernel.py b/client/bin/kernel.py
index e8dfbb93..6571bf92 100755
--- a/client/bin/kernel.py
+++ b/client/bin/kernel.py
@@ -4,7 +4,7 @@ import os, shutil, copy, pickle, re, glob, time
 from autotest_lib.client.bin.fd_stack import tee_output_logdir_mark
 from autotest_lib.client.bin import kernel_config, os_dep, kernelexpand, test
 from autotest_lib.client.bin import autotest_utils
-from autotest_lib.client.common_lib import logging, utils, error
+from autotest_lib.client.common_lib import logging, utils, error, packages
 
 
 class kernel(object):
@@ -720,13 +720,27 @@ except ImportError:
     def preprocess_path(path):
         return path
 
+
 def auto_kernel(job, path, subdir, tmp_dir, build_dir, leave=False):
     """\
     Create a kernel object, dynamically selecting the appropriate class to use
     based on the path provided.
     """
-    path = preprocess_path(path)
-    if path.endswith('.rpm'):
-        return rpm_kernel(job, path, subdir)
+    kernel_path = preprocess_path(path)
+    if kernel_path.endswith('.rpm'):
+        # Fetch the rpm into the job's packages directory and pass it to
+        # rpm_kernel
+        rpm_name = os.path.basename(kernel_path)
+
+        # If the preprocessed path (kernel_path) is only a name then
+        # search for the kernel in all the repositories, else fetch the kernel
+        # from that specific path.
+        job.pkgmgr.fetch_pkg(rpm_name, os.path.join(job.pkgdir, rpm_name),
+                             repo_url=os.path.dirname(kernel_path))
+
+        # TODO: rpm accepts http paths directly. It might be an optimization
+        # to pass in the http path directly to it. Fetching the file locally
+        # though.
+        return rpm_kernel(job, os.path.join(job.pkgdir, rpm_name), subdir)
     else:
-        return kernel(job, path, subdir, tmp_dir, build_dir, leave)
+        return kernel(job,kernel_path, subdir, tmp_dir, build_dir, leave)
diff --git a/client/bin/profilers.py b/client/bin/profilers.py
index b4ac8165..bac610d1 100755
--- a/client/bin/profilers.py
+++ b/client/bin/profilers.py
@@ -1,6 +1,6 @@
 import os, sys
 import common
-from autotest_lib.client.common_lib import error, utils
+from autotest_lib.client.common_lib import error, utils, packages
 
 class profilers:
 
@@ -13,13 +13,27 @@ class profilers:
 
     # add a profiler
     def add(self, profiler, *args, **dargs):
+        prof_dir = os.path.join(self.profdir, profiler)
+
+        try:
+            self.job.install_pkg(profiler, 'prof', prof_dir)
+        except packages.PackageInstallError:
+            pass
+
+        # prof_dir might not be present locally in the case where it is not
+        # fetched from the repositoryr
+        if not os.path.exists(prof_dir):
+            raise error.JobError('profiler %s not present' % profiler)
+
         profiler_module = common.setup_modules.import_module(profiler,
                                                              'autotest_lib.client.profilers.%s' % profiler)
         newprofiler = getattr(profiler_module, profiler)(self)
+
+
         newprofiler.name = profiler
-        newprofiler.bindir = self.profdir + '/' + profiler
-        newprofiler.srcdir = newprofiler.bindir + '/src'
-        newprofiler.tmpdir = self.tmpdir + '/' + profiler
+        newprofiler.bindir = os.path.join(self.profdir, profiler)
+        newprofiler.srcdir = os.path.join(newprofiler.bindir, 'src')
+        newprofiler.tmpdir = os.path.join(self.tmpdir, profiler)
         utils.update_version(newprofiler.srcdir, newprofiler.preserve_srcdir,
                              newprofiler.version, newprofiler.setup,
                              *args, **dargs)
diff --git a/client/bin/test.py b/client/bin/test.py
index eae1fc61..98ec9e1a 100755
--- a/client/bin/test.py
+++ b/client/bin/test.py
@@ -29,9 +29,6 @@ class test(common_test.base_test):
     pass
 
 
-testname = common_test.testname
-
-
 def _grab_sysinfo(mytest):
     try:
         sysinfo_dir = os.path.join(mytest.outputdir, 'sysinfo')
diff --git a/client/common_lib/global_config.py b/client/common_lib/global_config.py
index 8db02e18..6ca0d2a7 100644
--- a/client/common_lib/global_config.py
+++ b/client/common_lib/global_config.py
@@ -102,6 +102,8 @@ class global_config(object):
                 return 0
             elif type == float:
                 return 0.0
+            elif type == list:
+                return []
             else:
                 return None
 
@@ -111,6 +113,10 @@ class global_config(object):
             else:
                 return True
 
+        if type == list:
+            # Split the string using ',' and return a list
+            return [val.strip() for val in sval.split(',')]
+
         try:
             conv_val = type(sval)
             return conv_val
diff --git a/client/common_lib/test.py b/client/common_lib/test.py
index cc343f58..a527c96c 100644
--- a/client/common_lib/test.py
+++ b/client/common_lib/test.py
@@ -18,7 +18,8 @@
 
 import os, sys, re, fcntl, shutil, tarfile, warnings
 
-from autotest_lib.client.common_lib import error, utils
+from autotest_lib.client.common_lib import error, utils, packages
+from autotest_lib.client.bin import autotest_utils
 
 
 class base_test:
@@ -146,11 +147,17 @@ class base_test:
                 p_args, p_dargs = _cherry_pick_args(self.initialize,args,dargs)
                 self.initialize(*p_args, **p_dargs)
 
-                # Setup: (compile and install the test, if needed)
-                p_args, p_dargs = _cherry_pick_args(self.setup,args,dargs)
-                utils.update_version(self.srcdir, self.preserve_srcdir,
-                                     self.version, self.setup,
-                                     *p_args, **p_dargs)
+                lockfile = open(os.path.join(self.job.tmpdir, '.testlock'), 'w')
+                try:
+                    fcntl.flock(lockfile, fcntl.LOCK_EX)
+                    # Setup: (compile and install the test, if needed)
+                    p_args, p_dargs = _cherry_pick_args(self.setup,args,dargs)
+                    utils.update_version(self.srcdir, self.preserve_srcdir,
+                                         self.version, self.setup,
+                                         *p_args, **p_dargs)
+                finally:
+                    fcntl.flock(lockfile, fcntl.LOCK_UN)
+                    lockfile.close()
 
                 # Execute:
                 os.chdir(self.outputdir)
@@ -222,24 +229,8 @@ def _validate_args(args, dargs, *funcs):
                     raise error.AutotestError('Unknown parameter: %s' % param)
 
 
-def testname(url):
-    # Extract the testname from the test url.
-    match = re.match('[^:]+://(.*)/([^/]*)$', url)
-    if not match:
-        return ('', url)
-    (group, filename) = match.groups()
-
-    # Generate the group prefix.
-    group = re.sub(r'\W', '_', group)
-
-    # Drop the extension to get the raw test name.
-    testname = re.sub(r'\.tgz', '', filename)
-
-    return (group, testname)
-
-
 def _installtest(job, url):
-    (group, name) = testname(url)
+    (group, name) = job.pkgmgr.get_package_name(url, 'test')
 
     # Bail if the test is already installed
     group_dir = os.path.join(job.testdir, "download", group)
@@ -255,15 +246,19 @@ def _installtest(job, url):
         f.close()
 
     print name + ": installing test url=" + url
-    utils.get_file(url, os.path.join(group_dir, 'test.tgz'))
-    old_wd = os.getcwd()
-    os.chdir(group_dir)
-    tar = tarfile.open('test.tgz')
-    for member in tar.getmembers():
-        tar.extract(member)
-    tar.close()
-    os.chdir(old_wd)
-    os.remove(os.path.join(group_dir, 'test.tgz'))
+    tarball = os.path.basename(url)
+    tarball_path = os.path.join(group_dir, tarball)
+    test_dir = os.path.join(group_dir, name)
+    job.pkgmgr.fetch_pkg(tarball, tarball_path,
+                         repo_url = os.path.dirname(url))
+
+    # Create the directory for the test
+    if not os.path.exists(test_dir):
+        os.mkdir(os.path.join(group_dir, name))
+
+    job.pkgmgr.untar_pkg(tarball_path, test_dir)
+
+    os.remove(tarball_path)
 
     # For this 'sub-object' to be importable via the name
     # 'group.name' we need to provide an __init__.py,
@@ -282,7 +277,7 @@ def runtest(job, url, tag, args, dargs,
 
     # if this is not a plain test name then download and install the
     # specified test
-    if utils.is_url(url):
+    if url.endswith('.tar.bz2'):
         (group, testname) = _installtest(job, url)
         bindir = os.path.join(job.testdir, 'download', group, testname)
         site_bindir = None
@@ -298,6 +293,18 @@ def runtest(job, url, tag, args, dargs,
         else:
             site_bindir = None
 
+        # The job object here can be that of a server side job or a client
+        # side job. 'install_pkg' method won't be present for server side
+        # jobs, so do the fetch only if that method is present in the job
+        # obj.
+        if hasattr(job, 'install_pkg'):
+            try:
+                job.install_pkg(testname, 'test', bindir)
+            except packages.PackageInstallError, e:
+                # continue as a fall back mechanism and see if the test code
+                # already exists on the machine
+                pass
+
     outputdir = os.path.join(job.resultdir, testname)
     if tag:
         outputdir += '.' + tag
@@ -308,30 +315,26 @@ def runtest(job, url, tag, args, dargs,
         testdir = job.site_testdir
     elif os.path.exists(bindir):
         testdir = job.testdir
-    elif not os.path.exists(bindir):
+    else:
         raise error.TestError(testname + ': test does not exist')
 
+    local_namespace['job'] = job
+    local_namespace['bindir'] = bindir
+    local_namespace['outputdir'] = outputdir
+
     if group:
         sys.path.insert(0, os.path.join(testdir, 'download'))
         group += '.'
     else:
         sys.path.insert(0, os.path.join(testdir, testname))
 
-    local_namespace['job'] = job
-    local_namespace['bindir'] = bindir
-    local_namespace['outputdir'] = outputdir
-
-    lockfile = open(os.path.join(job.tmpdir, '.testlock'), 'w')
     try:
-        fcntl.flock(lockfile, fcntl.LOCK_EX)
         exec ("import %s%s" % (group, testname),
               local_namespace, global_namespace)
         exec ("mytest = %s%s.%s(job, bindir, outputdir)" %
               (group, testname, testname),
               local_namespace, global_namespace)
     finally:
-        fcntl.flock(lockfile, fcntl.LOCK_UN)
-        lockfile.close()
         sys.path.pop(0)
 
     pwd = os.getcwd()
diff --git a/client/common_lib/utils.py b/client/common_lib/utils.py
index c29d7e41..e54e1346 100644
--- a/client/common_lib/utils.py
+++ b/client/common_lib/utils.py
@@ -463,6 +463,20 @@ def get_sync_control_file(control, host_name, host_num,
     return "\n".join(control_new)
 
 
+def get_arch(run_function=run):
+    """
+    Get the hardware architecture of the machine.
+    run_function is used to execute the commands. It defaults to
+    utils.run() but a custom method (if provided) should be of the
+    same schema as utils.run. It should return a CmdResult object and
+    throw a CmdError exception.
+    """
+    arch = run_function('/bin/uname -m').stdout.rstrip()
+    if re.match(r'i\d86$', arch):
+        arch = 'i386'
+    return arch
+
+
 class CmdResult(object):
     """
     Command execution result.
diff --git a/client/tests/aio_dio_bugs/aio_dio_bugs.py b/client/tests/aio_dio_bugs/aio_dio_bugs.py
index 5cd4c2f0..9db1ffd8 100644
--- a/client/tests/aio_dio_bugs/aio_dio_bugs.py
+++ b/client/tests/aio_dio_bugs/aio_dio_bugs.py
@@ -18,11 +18,13 @@ class aio_dio_bugs(test.test):
     preserve_srcdir = True
 
     def initialize(self):
+        self.job.require_gcc()
         self.job.setup_dep(['libaio'])
         ldflags = '-L ' + self.autodir + '/deps/libaio/lib'
         cflags = '-I ' + self.autodir + '/deps/libaio/include'
         self.gcc_flags = ldflags + ' ' + cflags
 
+
     def setup(self):
         os.chdir(self.srcdir)
         utils.system('make ' + '"CFLAGS=' + self.gcc_flags + '"')
diff --git a/client/tests/aiostress/aiostress.py b/client/tests/aiostress/aiostress.py
index 5adb9fb4..85985ef4 100755
--- a/client/tests/aiostress/aiostress.py
+++ b/client/tests/aiostress/aiostress.py
@@ -12,6 +12,7 @@ class aiostress(test.test):
     version = 2
 
     def initialize(self):
+        self.job.require_gcc()
         self.job.setup_dep(['libaio'])
         ldflags = '-L ' + self.autodir + '/deps/libaio/lib'
         cflags = '-I ' + self.autodir + '/deps/libaio/include'
diff --git a/client/tests/bonnie/bonnie.py b/client/tests/bonnie/bonnie.py
index f9bac64c..6ca12d7a 100755
--- a/client/tests/bonnie/bonnie.py
+++ b/client/tests/bonnie/bonnie.py
@@ -29,6 +29,7 @@ class bonnie(test.test):
 
     def initialize(self):
         self.results = []
+        self.job.require_gcc()
 
     # http://www.coker.com.au/bonnie++/bonnie++-1.03a.tgz
     def setup(self, tarball = 'bonnie++-1.03a.tgz'):
diff --git a/client/tests/btreplay/btreplay.py b/client/tests/btreplay/btreplay.py
index 91b7092a..6e807a70 100644
--- a/client/tests/btreplay/btreplay.py
+++ b/client/tests/btreplay/btreplay.py
@@ -25,6 +25,7 @@ class btreplay(test.test):
 
 
     def initialize(self):
+        self.job.require_gcc()
         self.ldlib = 'LD_LIBRARY_PATH=%s/deps/libaio/lib'%(self.autodir)
         self.results = []
 
diff --git a/client/tests/cyclictest/cyclictest.py b/client/tests/cyclictest/cyclictest.py
index 5c9189d9..a4399d5c 100755
--- a/client/tests/cyclictest/cyclictest.py
+++ b/client/tests/cyclictest/cyclictest.py
@@ -8,10 +8,14 @@ class cyclictest(test.test):
     preserve_srcdir = True
 
     # git://git.kernel.org/pub/scm/linux/kernel/git/tglx/rt-tests.git
+    def initialize(self):
+        self.job.require_gcc()
+
 
     def setup(self):
         os.chdir(self.srcdir)
         utils.system('make')
 
+
     def execute(self, args = '-t 10 -l 100000'):
         utils.system(self.srcdir + '/cyclictest ' + args)
diff --git a/client/tests/dbench/dbench.py b/client/tests/dbench/dbench.py
index 83829d7a..d49d2c4e 100755
--- a/client/tests/dbench/dbench.py
+++ b/client/tests/dbench/dbench.py
@@ -5,6 +5,10 @@ from autotest_lib.client.common_lib import utils
 class dbench(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://samba.org/ftp/tridge/dbench/dbench-3.04.tar.gz
     def setup(self, tarball = 'dbench-3.04.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/dbt2/dbt2.py b/client/tests/dbt2/dbt2.py
index fef19c19..d0fd92bf 100644
--- a/client/tests/dbt2/dbt2.py
+++ b/client/tests/dbt2/dbt2.py
@@ -9,6 +9,10 @@ from autotest_lib.client.common_lib import utils
 class dbt2(test.test):
     version = 2
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://osdn.dl.sourceforge.net/sourceforge/osdldbt/dbt2-0.39.tar.gz
     def setup(self, tarball = 'dbt2-0.39.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -40,6 +44,7 @@ class dbt2(test.test):
         utils.system('ln -s %s %s' %
                      (self.resultsdir, self.srcdir + '.pgsql/scripts/output'))
 
+
     def execute(self, db_type, args = ''):
         logfile = self.resultsdir + '/dbt2.log'
 
@@ -50,11 +55,13 @@ class dbt2(test.test):
         elif (db_type == "pgsql"):
             self.execute_pgsql(args)
 
+
     def execute_mysql(self, args = ''):
         args = args
         utils.system(self.srcdir + '.mysql/scripts/mysql/build_db.sh -g -w 1')
         utils.system(self.srcdir + '.mysql/scripts/run_workload.sh ' + args)
 
+
     def execute_pgpool(self, args = ''):
         utils.system('%s/deps/pgpool/pgpool/bin/pgpool -f %s/../pgpool.conf' \
                         % (self.autodir, self.srcdir))
diff --git a/client/tests/fio/fio.py b/client/tests/fio/fio.py
index 29a2d3c2..ca580b4b 100644
--- a/client/tests/fio/fio.py
+++ b/client/tests/fio/fio.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class fio(test.test):
     version = 2
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://brick.kernel.dk/snaps/fio-1.16.5.tar.bz2
     def setup(self, tarball = 'fio-1.16.5.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -21,6 +25,7 @@ class fio(test.test):
         utils.system('patch -p1 < ../Makefile.patch')
         utils.system('%s %s make' % (var_ldflags, var_cflags))
 
+
     def execute(self, args = '', user = 'root'):
         os.chdir(self.srcdir)
         ##vars = 'TMPDIR=\"%s\" RESULTDIR=\"%s\"' % (self.tmpdir, self.resultsdir)
diff --git a/client/tests/fs_mark/fs_mark.py b/client/tests/fs_mark/fs_mark.py
index dfde3cfb..e08ce9be 100644
--- a/client/tests/fs_mark/fs_mark.py
+++ b/client/tests/fs_mark/fs_mark.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class fs_mark(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://developer.osdl.org/dev/doubt/fs_mark/archive/fs_mark-3.2.tgz
     def setup(self, tarball = 'fs_mark-3.2.tgz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -14,6 +18,7 @@ class fs_mark(test.test):
 
         utils.system('make')
 
+
     def execute(self, dir, iterations = 2, args = None):
         os.chdir(self.srcdir)
         if not args:
diff --git a/client/tests/fsfuzzer/fsfuzzer.py b/client/tests/fsfuzzer/fsfuzzer.py
index 050f4fcb..d58e514d 100755
--- a/client/tests/fsfuzzer/fsfuzzer.py
+++ b/client/tests/fsfuzzer/fsfuzzer.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class fsfuzzer(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://people.redhat.com/sgrubb/files/fsfuzzer-0.6.tar.gz
     def setup(self, tarball = 'fsfuzzer-0.6.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -14,6 +18,7 @@ class fsfuzzer(test.test):
 
         utils.system('make')
 
+
     def execute(self, iterations = 1, fstype = 'iso9660'):
         profilers = self.job.profilers
         args = fstype + ' 1'
diff --git a/client/tests/fsstress/fsstress.py b/client/tests/fsstress/fsstress.py
index 6521842b..eb925704 100644
--- a/client/tests/fsstress/fsstress.py
+++ b/client/tests/fsstress/fsstress.py
@@ -7,6 +7,10 @@ from autotest_lib.client.common_lib import utils
 class fsstress(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
     def setup(self, tarball = 'ext3-tools.tar.gz'):
         self.tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/fsx/fsx.py b/client/tests/fsx/fsx.py
index 8ed98d2e..307448cc 100755
--- a/client/tests/fsx/fsx.py
+++ b/client/tests/fsx/fsx.py
@@ -11,6 +11,10 @@ from autotest_lib.client.common_lib import utils
 class fsx(test.test):
     version = 3
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
     def setup(self, tarball = 'ext3-tools.tar.gz'):
         self.tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/interbench/interbench.py b/client/tests/interbench/interbench.py
index fa8411e6..900819da 100644
--- a/client/tests/interbench/interbench.py
+++ b/client/tests/interbench/interbench.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class interbench(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://www.kernel.org/pub/linux/kernel/people/ck/apps/interbench/interbench-0.30.tar.bz2
     def setup(self, tarball = 'interbench-0.30.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -14,6 +18,7 @@ class interbench(test.test):
 
         utils.system('make')
 
+
     def execute(self, iterations = 1, args = ''):
         os.chdir(self.tmpdir)
         args += " -c"
diff --git a/client/tests/iosched_bugs/iosched_bugs.py b/client/tests/iosched_bugs/iosched_bugs.py
index 1fe1928e..2b2b3048 100755
--- a/client/tests/iosched_bugs/iosched_bugs.py
+++ b/client/tests/iosched_bugs/iosched_bugs.py
@@ -8,6 +8,9 @@ class iosched_bugs(test.test):
     version = 1
     preserve_srcdir = True
 
+    def initialize(self):
+        self.job.require_gcc()
+
 
     def setup(self):
         os.chdir(self.srcdir)
diff --git a/client/tests/iozone/iozone.py b/client/tests/iozone/iozone.py
index f1e7e6d1..ee656e65 100644
--- a/client/tests/iozone/iozone.py
+++ b/client/tests/iozone/iozone.py
@@ -7,6 +7,10 @@ from autotest_lib.client.common_lib import utils
 class iozone(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://www.iozone.org/src/current/iozone3_283.tar
     def setup(self, tarball = 'iozone3_283.tar'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/isic/isic.py b/client/tests/isic/isic.py
index b7f6c7b2..af59af7c 100644
--- a/client/tests/isic/isic.py
+++ b/client/tests/isic/isic.py
@@ -10,8 +10,10 @@ class isic(test.test):
     # + http://www.stardust.webpages.pl/files/crap/isic-gcc41-fix.patch
 
     def initialize(self):
+        self.job.require_gcc()
         self.job.setup_dep(['libnet'])
 
+
     def setup(self, tarball = 'isic-0.06.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
         autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
diff --git a/client/tests/kernbench/kernbench.py b/client/tests/kernbench/kernbench.py
index 490ab3b7..974245cd 100755
--- a/client/tests/kernbench/kernbench.py
+++ b/client/tests/kernbench/kernbench.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class kernbench(test.test):
     version = 2
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     def setup(self, build_dir = None):
         if not build_dir:
             build_dir = self.srcdir
diff --git a/client/tests/kvmtest/kvmtest.py b/client/tests/kvmtest/kvmtest.py
index b738e3a8..666f23b0 100644
--- a/client/tests/kvmtest/kvmtest.py
+++ b/client/tests/kvmtest/kvmtest.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils, error
 class kvmtest(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     def setup(self, tarball = 'kvm-test.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
         autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
diff --git a/client/tests/libhugetlbfs/libhugetlbfs.py b/client/tests/libhugetlbfs/libhugetlbfs.py
index 2306ce20..80520e5a 100644
--- a/client/tests/libhugetlbfs/libhugetlbfs.py
+++ b/client/tests/libhugetlbfs/libhugetlbfs.py
@@ -5,6 +5,9 @@ from autotest_lib.client.common_lib import utils, error
 class libhugetlbfs(test.test):
     version = 4
 
+    def initialize(self):
+        self.job.require_gcc()
+
     # http://libhugetlbfs.ozlabs.org/releases/libhugetlbfs-1.3-pre1.tar.gz
     def setup(self, tarball = 'libhugetlbfs-1.3-pre1.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -18,6 +21,7 @@ class libhugetlbfs(test.test):
         except:
             utils.system('make OBJDIRS=obj64')
 
+
     def execute(self, dir = None, pages_requested = 20):
         autotest_utils.check_kernel_ver("2.6.16")
 
diff --git a/client/tests/lmbench/lmbench.py b/client/tests/lmbench/lmbench.py
index e31734b1..72cab690 100755
--- a/client/tests/lmbench/lmbench.py
+++ b/client/tests/lmbench/lmbench.py
@@ -7,6 +7,10 @@ from autotest_lib.client.common_lib import utils
 class lmbench(test.test):
     version = 2
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     def setup(self, tarball = 'lmbench3.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
         # http://www.bitmover.com/lm/lmbench/lmbench3.tar.gz
diff --git a/client/tests/ltp/ltp.py b/client/tests/ltp/ltp.py
index 434c18e6..28bc684d 100755
--- a/client/tests/ltp/ltp.py
+++ b/client/tests/ltp/ltp.py
@@ -5,6 +5,10 @@ from autotest_lib.client.common_lib import utils, error
 class ltp(test.test):
     version = 4
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://prdownloads.sourceforge.net/ltp/ltp-full-20080229.tgz
     def setup(self, tarball = 'ltp-full-20080229.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/netperf2/netperf2.py b/client/tests/netperf2/netperf2.py
index e10cfa2e..4b2d93a1 100755
--- a/client/tests/netperf2/netperf2.py
+++ b/client/tests/netperf2/netperf2.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils, error
 class netperf2(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # ftp://ftp.netperf.org/netperf/netperf-2.4.1.tar.gz
     def setup(self, tarball = 'netperf-2.4.1.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/pi_tests/pi_tests.py b/client/tests/pi_tests/pi_tests.py
index c3c79de6..fef4dc80 100644
--- a/client/tests/pi_tests/pi_tests.py
+++ b/client/tests/pi_tests/pi_tests.py
@@ -6,16 +6,19 @@ from autotest_lib.client.common_lib import utils
 class pi_tests(test.test):
     version = 1
 
-    # http://www.stardust.webpages.pl/files/patches/autotest/pi_tests.tar.bz2
+    def initialize(self):
+        self.job.require_gcc()
+
 
+    # http://www.stardust.webpages.pl/files/patches/autotest/pi_tests.tar.bz2
     def setup(self, tarball = 'pi_tests.tar.bz2'):
         autotest_utils.check_glibc_ver('2.5')
         tarball = autotest_utils.unmap_url(self.bindir, tarball, self.tmpdir)
         autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
         os.chdir(self.srcdir)
-
         utils.system('make')
 
+
     def execute(self, args = '1 300'):
         os.chdir(self.srcdir)
         utils.system('./start.sh ' + args)
diff --git a/client/tests/posixtest/posixtest.py b/client/tests/posixtest/posixtest.py
index d26ce0a4..469cf49c 100755
--- a/client/tests/posixtest/posixtest.py
+++ b/client/tests/posixtest/posixtest.py
@@ -9,6 +9,11 @@ __author__ = '''mohd.omar@in.ibm.com (Mohammed Omar)'''
 
 class posixtest(test.test):
     version = 1
+
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://ufpr.dl.sourceforge.net/sourceforge/posixtest/posixtestsuite-1.5.2.tar.gz
     def setup(self, tarball = 'posixtestsuite-1.5.2.tar.gz'):
         self.posix_tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/reaim/reaim.py b/client/tests/reaim/reaim.py
index 72e668f6..a456a4e1 100755
--- a/client/tests/reaim/reaim.py
+++ b/client/tests/reaim/reaim.py
@@ -34,6 +34,7 @@ class reaim(test.test):
 
 
     def initialize(self):
+        self.job.require_gcc()
         self.ldlib = 'LD_LIBRARY_PATH=%s/deps/libaio/lib'%(self.autodir)
 
 
diff --git a/client/tests/rmaptest/rmaptest.py b/client/tests/rmaptest/rmaptest.py
index e6afdef2..ee4befb1 100644
--- a/client/tests/rmaptest/rmaptest.py
+++ b/client/tests/rmaptest/rmaptest.py
@@ -15,6 +15,10 @@ class rmaptest(test.test):
     version = 1
     preserve_srcdir = True
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     def setup(self):
         os.chdir(self.srcdir)
         utils.system('gcc -Wall -o rmaptest rmap-test.c')
diff --git a/client/tests/scrashme/scrashme.py b/client/tests/scrashme/scrashme.py
index 77f8a1b2..e0a8f24a 100644
--- a/client/tests/scrashme/scrashme.py
+++ b/client/tests/scrashme/scrashme.py
@@ -6,14 +6,18 @@ from autotest_lib.client.common_lib import utils
 class scrashme(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://www.codemonkey.org.uk/projects/git-snapshots/scrashme/scrashme-2007-07-08.tar.gz
     def setup(self, tarball = 'scrashme-2007-07-08.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
         autotest_utils.extract_tarball_to_dir(tarball, self.srcdir)
         os.chdir(self.srcdir)
-
         utils.system('make')
 
+
     def execute(self, iterations = 1, args_list = ''):
         if len(args_list) != 0:
             args = '' + args_list
diff --git a/client/tests/signaltest/signaltest.py b/client/tests/signaltest/signaltest.py
index be4e095c..39303a18 100644
--- a/client/tests/signaltest/signaltest.py
+++ b/client/tests/signaltest/signaltest.py
@@ -7,11 +7,15 @@ class signaltest(test.test):
     version = 1
     preserve_srcdir = True
 
-    # git://git.kernel.org/pub/scm/linux/kernel/git/tglx/rt-tests.git
+    def initialize(self):
+        self.job.require_gcc()
+
 
+    # git://git.kernel.org/pub/scm/linux/kernel/git/tglx/rt-tests.git
     def setup(self):
         os.chdir(self.srcdir)
         utils.system('make')
 
+
     def execute(self, args = '-t 10 -l 100000'):
         utils.system(self.srcdir + '/signaltest ' + args)
diff --git a/client/tests/sparse/sparse.py b/client/tests/sparse/sparse.py
index 253675ce..1cf427b7 100755
--- a/client/tests/sparse/sparse.py
+++ b/client/tests/sparse/sparse.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class sparse(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://www.codemonkey.org.uk/projects/git-snapshots/sparse/sparse-2006-04-28.tar.gz
     def setup(self, tarball = 'sparse-2006-04-28.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -17,6 +21,7 @@ class sparse(test.test):
 
         self.top_dir = self.job.tmpdir+'/sparse'
 
+
     def execute(self, base_tree, patches, config, config_list = None):
         kernel = self.job.kernel(base_tree, self.resultsdir)
         kernel.patch(patches)
diff --git a/client/tests/spew/spew.py b/client/tests/spew/spew.py
index 77a657ea..1fcb502a 100755
--- a/client/tests/spew/spew.py
+++ b/client/tests/spew/spew.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class spew(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # ftp://ftp.berlios.de/pub/spew/1.0.5/spew-1.0.5.tgz
     def setup(self, tarball = 'spew-1.0.5.tgz'):
         self.tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/stress/stress.py b/client/tests/stress/stress.py
index c7f3341f..cb213921 100644
--- a/client/tests/stress/stress.py
+++ b/client/tests/stress/stress.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class stress(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://weather.ou.edu/~apw/projects/stress/stress-0.18.8.tar.gz
     def setup(self, tarball = 'stress-0.18.8.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/sysbench/sysbench.py b/client/tests/sysbench/sysbench.py
index 3845b383..f17f8cf8 100644
--- a/client/tests/sysbench/sysbench.py
+++ b/client/tests/sysbench/sysbench.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils, error
 class sysbench(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://osdn.dl.sourceforge.net/sourceforge/sysbench/sysbench-0.4.8.tar.gz
     def setup(self, tarball = 'sysbench-0.4.8.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/tbench/tbench.py b/client/tests/tbench/tbench.py
index 32cc3f0c..7fb8f248 100755
--- a/client/tests/tbench/tbench.py
+++ b/client/tests/tbench/tbench.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils
 class tbench(test.test):
     version = 2
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://samba.org/ftp/tridge/dbench/dbench-3.04.tar.gz
     def setup(self, tarball = 'dbench-3.04.tar.gz'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
@@ -15,6 +19,7 @@ class tbench(test.test):
         utils.system('./configure')
         utils.system('make')
 
+
     def execute(self, iterations = 1, nprocs = None, args = ''):
         # only supports combined server+client model at the moment
         # should support separate I suppose, but nobody uses it
diff --git a/client/tests/tsc/tsc.py b/client/tests/tsc/tsc.py
index 2dbdcf5a..31306034 100755
--- a/client/tests/tsc/tsc.py
+++ b/client/tests/tsc/tsc.py
@@ -12,5 +12,9 @@ class tsc(test.test):
         utils.system('make')
 
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     def run_once(self, args = ''):
         utils.system(self.srcdir + '/checktsc ' + args)
diff --git a/client/tests/unixbench/unixbench.py b/client/tests/unixbench/unixbench.py
index abdfc4f5..86fe6bbc 100755
--- a/client/tests/unixbench/unixbench.py
+++ b/client/tests/unixbench/unixbench.py
@@ -6,6 +6,10 @@ from autotest_lib.client.common_lib import utils, error
 class unixbench(test.test):
     version = 2
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # http://www.tux.org/pub/tux/niemi/unixbench/unixbench-4.1.0.tgz
     def setup(self, tarball = 'unixbench-4.1.0.tar.bz2'):
         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
diff --git a/client/tests/xmtest/xmtest.py b/client/tests/xmtest/xmtest.py
index 1f1c08d2..787b85e6 100644
--- a/client/tests/xmtest/xmtest.py
+++ b/client/tests/xmtest/xmtest.py
@@ -11,6 +11,10 @@ from autotest_lib.client.common_lib import utils
 class xmtest(test.test):
     version = 1
 
+    def initialize(self):
+        self.job.require_gcc()
+
+
     # This test expects just the xm-test directory, as a tarball
     # from the Xen source tree
     # hg clone http://xenbits.xensource.com/xen-unstable.hg
@@ -26,6 +30,7 @@ class xmtest(test.test):
         utils.system('./configure')
         utils.system('make existing')
 
+
     def execute(self, args = ''):
         os.chdir(self.srcdir)
         utils.system('./runtest.sh ' + args)
diff --git a/frontend/afe/control_file.py b/frontend/afe/control_file.py
index 3a76e266..49572d70 100644
--- a/frontend/afe/control_file.py
+++ b/frontend/afe/control_file.py
@@ -86,6 +86,10 @@ def format_step(item, lines):
 
 def get_tests_stanza(tests, is_server, prepend=[], append=[]):
     raw_control_files = [read_control_file(test) for test in tests]
+    return _get_tests_stanza(raw_control_files, is_server, prepend, append)
+
+
+def _get_tests_stanza(raw_control_files, is_server, prepend, append):
     if is_server:
         return '\n'.join(raw_control_files)
     raw_steps = prepend + [add_boilerplate_to_nested_steps(step)
diff --git a/frontend/afe/rpc_utils.py b/frontend/afe/rpc_utils.py
index 27738866..ac175f0f 100644
--- a/frontend/afe/rpc_utils.py
+++ b/frontend/afe/rpc_utils.py
@@ -102,6 +102,10 @@ class InconsistencyException(Exception):
 
 
 def get_consistent_value(objects, field):
+    if not objects:
+        # well a list of nothing is consistent
+        return None
+
     value = getattr(objects[0], field)
     for obj in objects:
         this_value = getattr(obj, field)
diff --git a/server/autotest.py b/server/autotest.py
index b338988b..da94ae8a 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -18,7 +18,7 @@ import re, os, sys, traceback, subprocess, tempfile, shutil, time
 
 from autotest_lib.server import installable_object, utils, server_job
 from autotest_lib.client.common_lib import logging
-from autotest_lib.client.common_lib import error
+from autotest_lib.client.common_lib import error, global_config, packages
 
 
 
@@ -94,10 +94,31 @@ class BaseAutotest(installable_object.InstallableObject):
 
         host.run('mkdir -p "%s"' % utils.sh_escape(autodir))
 
-        if getattr(host, 'site_install_autotest', None):
-            if host.site_install_autotest():
-                self.installed = True
-                return
+        # Fetch the autotest client from the nearest repository
+        try:
+            c = global_config.global_config
+            repos = c.get_config_value("PACKAGES", 'fetch_location', type=list)
+            pkgmgr = packages.PackageManager(
+                autodir, repo_urls=repos, do_locking=False,
+                run_function=host.run,
+                run_function_dargs=dict(timeout=600))
+            # The packages dir is used to store all the packages that
+            # are fetched on that client. (for the tests,deps etc.
+            # too apart from the client)
+            pkg_dir = os.path.join(autodir, 'packages')
+            # clean up the autodir except for the packages directory
+            host.run('cd %s && ls | grep -v "^packages$"'
+                     ' | xargs rm -rf && rm -rf .[^.]*' % autodir)
+            pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir,
+                               preserve_install_dir=True)
+            self.installed = True
+            return
+        except global_config.ConfigError, e:
+            print ("Could not install autotest using the"
+                   " packaging system %s" %  e)
+        except (packages.PackageInstallError, error.AutoservRunError), e:
+            print "Could not install autotest from %s : %s " % (repos, e)
+
 
         # try to install from file or directory
         if self.source_material:
@@ -249,8 +270,29 @@ class BaseAutotest(installable_object.InstallableObject):
                         atrun.manual_control_file + '.state']:
             host.run('rm -f ' + control)
 
-        # Copy control_file to remote_control_file on the host
         tmppath = utils.get(control_file)
+
+        # Insert the job.add_repository() lines in the control file
+        # if there are any repos defined in global_config.ini
+        try:
+            cfile = open(tmppath, 'r')
+            cfile_orig = cfile.read()
+            cfile.close()
+            c = global_config.global_config
+            repos = c.get_config_value("PACKAGES", 'fetch_location', type=list)
+            control_file_new = []
+            control_file_new.append('job.add_repository(%s)\n' % repos)
+            control_file_new.append(cfile_orig)
+
+            # Overwrite the control file with the new one
+            cfile = open(tmppath, 'w')
+            cfile.write('\n'.join(control_file_new))
+            cfile.close()
+        except global_config.ConfigError, e:
+            pass
+
+
+        # Copy control_file to remote_control_file on the host
         host.send_file(tmppath, atrun.remote_control_file)
         if os.path.abspath(tmppath) != os.path.abspath(control_file):
             os.remove(tmppath)
diff --git a/server/autotest_unittest.py b/server/autotest_unittest.py
index e63d0b35..573e19f1 100644
--- a/server/autotest_unittest.py
+++ b/server/autotest_unittest.py
@@ -92,10 +92,12 @@ class TestBaseAutotest(unittest.TestCase):
         c.get_config_value.expect_call("PACKAGES",
             'fetch_location', type=list).and_return('repos')
         pkgmgr = packages.PackageManager.expect_new('autodir',
-            repo_urls='repos', run_function=self.host.run,
+            repo_urls='repos', do_locking=False, run_function=self.host.run,
             run_function_dargs=dict(timeout=600))
         pkg_dir = os.path.join('autodir', 'packages')
-        self.host.run.expect_call('ls | grep -v packages | xargs rm -rf')
+        cmd = ('cd autodir && ls | grep -v "^packages$"'
+               ' | xargs rm -rf && rm -rf .[^.]*')
+        self.host.run.expect_call(cmd)
         pkgmgr.install_pkg.expect_call('autotest', 'client', pkg_dir,
                                        'autodir', preserve_install_dir=True)
 
diff --git a/server/base_server_job.py b/server/base_server_job.py
index 231036b7..03342a86 100644
--- a/server/base_server_job.py
+++ b/server/base_server_job.py
@@ -17,7 +17,7 @@ from autotest_lib.client.bin import fd_stack
 from autotest_lib.client.common_lib import error, logging
 from autotest_lib.server import test, subcommand
 from autotest_lib.tko import db as tko_db, status_lib, utils as tko_utils
-from autotest_lib.client.common_lib import utils
+from autotest_lib.client.common_lib import utils, packages
 
 
 # load up a control segment
@@ -208,6 +208,9 @@ class base_server_job:
             self.init_parser(resultdir)
         else:
             self.using_parser = False
+        self.pkgmgr = packages.PackageManager(
+            self.autodir, run_function_dargs={'timeout':600})
+        self.pkgdir = os.path.join(self.autodir, 'packages')
 
 
     def init_parser(self, resultdir):
@@ -390,7 +393,7 @@ class base_server_job:
                 url of the test to run
         """
 
-        (group, testname) = test.testname(url)
+        (group, testname) = self.pkgmgr.get_package_name(url, 'test')
         tag = None
         subdir = testname
 
diff --git a/server/base_server_job_unittest.py b/server/base_server_job_unittest.py
index 5dbb8ceb..6e259cfc 100644
--- a/server/base_server_job_unittest.py
+++ b/server/base_server_job_unittest.py
@@ -4,6 +4,7 @@ import unittest, os, time
 import common
 from autotest_lib.server import base_server_job, test, subcommand
 from autotest_lib.client.common_lib import utils, error, host_protections
+from autotest_lib.client.common_lib import packages
 from autotest_lib.tko import db as tko_db, status_lib, utils as tko_utils
 from autotest_lib.client.common_lib.test_utils import mock
 from autotest_lib.tko.parsers import version_1 as parser_mod
@@ -215,7 +216,7 @@ class BaseServerJobTest(unittest.TestCase):
         self.construct_server_job()
 
         # setup
-        self.god.stub_function(test, 'testname')
+        self.god.stub_function(self.job.pkgmgr, 'get_package_name')
         self.god.stub_function(test, 'runtest')
         self.god.stub_function(self.job, 'record')
 
@@ -223,7 +224,8 @@ class BaseServerJobTest(unittest.TestCase):
         url = "my.test.url"
         group = "group"
         testname = "testname"
-        test.testname.expect_call(url).and_return((group, testname))
+        self.job.pkgmgr.get_package_name.expect_call(
+            url, 'test').and_return((group, testname))
         outputdir = os.path.join(self.resultdir, testname)
         os.path.exists.expect_call(outputdir).and_return(False)
         os.mkdir.expect_call(outputdir)
@@ -242,7 +244,7 @@ class BaseServerJobTest(unittest.TestCase):
         self.construct_server_job()
 
         # setup
-        self.god.stub_function(test, 'testname')
+        self.god.stub_function(self.job.pkgmgr, 'get_package_name')
         self.god.stub_function(test, 'runtest')
         self.god.stub_function(self.job, 'record')
 
@@ -251,7 +253,8 @@ class BaseServerJobTest(unittest.TestCase):
         group = "group"
         testname = "testname"
         e = error.TestError("Unexpected error")
-        test.testname.expect_call(url).and_return((group, testname))
+        self.job.pkgmgr.get_package_name.expect_call(
+            url, 'test').and_return((group, testname))
         outputdir = os.path.join(self.resultdir, testname)
         os.path.exists.expect_call(outputdir).and_return(False)
         os.mkdir.expect_call(outputdir)
@@ -271,7 +274,7 @@ class BaseServerJobTest(unittest.TestCase):
         self.construct_server_job()
 
         # setup
-        self.god.stub_function(test, 'testname')
+        self.god.stub_function(self.job.pkgmgr, 'get_package_name')
         self.god.stub_function(test, 'runtest')
         self.god.stub_function(self.job, 'record')
 
@@ -280,7 +283,8 @@ class BaseServerJobTest(unittest.TestCase):
         group = "group"
         testname = "testname"
         e = error.TestFail("The test failed!")
-        test.testname.expect_call(url).and_return((group, testname))
+        self.job.pkgmgr.get_package_name.expect_call(
+            url, 'test').and_return((group, testname))
         outputdir = os.path.join(self.resultdir, testname)
         os.path.exists.expect_call(outputdir).and_return(False)
         os.mkdir.expect_call(outputdir)
diff --git a/server/test.py b/server/test.py
index 31768579..084cdbf9 100755
--- a/server/test.py
+++ b/server/test.py
@@ -12,9 +12,6 @@ class test(common_test.base_test):
     pass
 
 
-testname = common_test.testname
-
-
 def runtest(job, url, tag, args, dargs):
     common_test.runtest(job, url, tag, args, dargs,
                         locals(), globals())