Refactored build_externals to support site-specific packages. Package declarations are now in external_packages.py and site_external_packages.py; build_externals.py is just the base script.

Also added functions to support building packages using make.

Signed-off-by: Steve Howard <showard@google.com>


git-svn-id: svn://test.kernel.org/autotest/trunk@4238 592f7852-d20e-0410-864c-8624ca9c26a4

2 files changed, 733 insertions, 671 deletions

diff --git a/utils/build_externals.py b/utils/build_externals.py
index aea88791..268eadfc 100755
--- a/utils/build_externals.py
+++ b/utils/build_externals.py
@@ -12,11 +12,15 @@ Usage?  Just run it.
     utils/build_externals.py
 """
 
-import compileall, logging, os, shutil, sys, tempfile, time, urllib2
+import compileall, logging, os, sha, shutil, sys, tempfile, time, urllib2
 import subprocess, re
 import common
 from autotest_lib.client.common_lib import logging_config, logging_manager
 from autotest_lib.client.common_lib import utils
+from autotest_lib.utils import external_packages
+
+# bring in site packages as well
+utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages')
 
 # Where package source be fetched to relative to the top of the autotest tree.
 PACKAGE_DIR = 'ExternalSource'
@@ -29,11 +33,7 @@ INSTALL_ALL = False
 
 
 # Want to add more packages to fetch, build and install?  See the class
-# definitions at the end of this file for examples of how to do it.
-
-
-_READ_SIZE = 64*1024
-_MAX_PACKAGE_SIZE = 100*1024*1024
+# definitions at the end of external_packages.py for examples of how to do it.
 
 
 class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
@@ -42,12 +42,6 @@ class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
                                                                use_console=True,
                                                                verbose=verbose)
 
-class Error(Exception):
-    """Local exception to be raised by code in this file."""
-
-class FetchError(Error):
-    """Failed to fetch a package from any of its listed URLs."""
-
 
 def main():
     """
@@ -58,7 +52,7 @@ def main():
                                       verbose=True)
     os.umask(022)
 
-    top_of_tree = _find_top_of_autotest_tree()
+    top_of_tree = external_packages.find_top_of_autotest_tree()
     package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
     install_dir = os.path.join(top_of_tree, INSTALL_DIR)
 
@@ -86,7 +80,7 @@ def main():
     compileall.compile_dir(install_dir, quiet=True)
 
     # Some things install with whacky permissions, fix that.
-    system("chmod -R a+rX '%s'" % install_dir)
+    external_packages.system("chmod -R a+rX '%s'" % install_dir)
 
     errors = fetch_errors + install_errors
     for error_msg in errors:
@@ -110,7 +104,7 @@ def fetch_necessary_packages(dest_dir, install_dir):
     names_to_check = sys.argv[1:]
     errors = []
     fetched_packages = []
-    for package_class in ExternalPackage.subclasses:
+    for package_class in external_packages.ExternalPackage.subclasses:
         package = package_class()
         if names_to_check and package.name.lower() not in names_to_check:
             continue
@@ -149,661 +143,5 @@ def build_and_install_packages(packages, install_dir):
     return errors
 
 
-def _checksum_file(full_path):
-    """@returns The hex checksum of a file given its pathname."""
-    inputfile = open(full_path, 'rb')
-    try:
-        hex_sum = utils.hash('sha1', inputfile.read()).hexdigest()
-    finally:
-        inputfile.close()
-    return hex_sum
-
-
-def _find_top_of_autotest_tree():
-    """@returns The full path to the top of the autotest directory tree."""
-    dirname = os.path.dirname(__file__)
-    autotest_dir = os.path.abspath(os.path.join(dirname, '..'))
-    return autotest_dir
-
-
-def system(commandline):
-    """Same as os.system(commandline) but logs the command first."""
-    logging.info(commandline)
-    return os.system(commandline)
-
-
-class ExternalPackage(object):
-    """
-    Defines an external package with URLs to fetch its sources from and
-    a build_and_install() method to unpack it, build it and install it
-    beneath our own autotest/site-packages directory.
-
-    Base Class.  Subclass this to define packages.
-
-    Attributes:
-      @attribute urls - A tuple of URLs to try fetching the package from.
-      @attribute local_filename - A local filename to use when saving the
-              fetched package.
-      @attribute hex_sum - The hex digest (currently SHA1) of this package
-              to be used to verify its contents.
-      @attribute module_name - The installed python module name to be used for
-              for a version check.  Defaults to the lower case class name with
-              the word Package stripped off.
-      @attribute version - The desired minimum package version.
-      @attribute os_requirements - A dictionary mapping a file pathname on the
-              the OS distribution to a likely name of a package the user
-              needs to install on their system in order to get this file.
-      @attribute name - Read only, the printable name of the package.
-      @attribute subclasses - This class attribute holds a list of all defined
-              subclasses.  It is constructed dynamically using the metaclass.
-    """
-    subclasses = []
-    urls = ()
-    local_filename = None
-    hex_sum = None
-    module_name = None
-    version = None
-    os_requirements = None
-
-
-    class __metaclass__(type):
-        """Any time a subclass is defined, add it to our list."""
-        def __init__(mcs, name, bases, dict):
-            if name != 'ExternalPackage':
-                mcs.subclasses.append(mcs)
-
-
-    def __init__(self):
-        self.verified_package = ''
-        if not self.module_name:
-            self.module_name = self.name.lower()
-        self.installed_version = ''
-
-
-    @property
-    def name(self):
-        """Return the class name with any trailing 'Package' stripped off."""
-        class_name = self.__class__.__name__
-        if class_name.endswith('Package'):
-            return class_name[:-len('Package')]
-        return class_name
-
-
-    def is_needed(self, unused_install_dir):
-        """@returns True if self.module_name needs to be built and installed."""
-        if not self.module_name or not self.version:
-            logging.warning('version and module_name required for '
-                            'is_needed() check to work.')
-            return True
-        try:
-            module = __import__(self.module_name)
-        except ImportError, e:
-            logging.info('Could not import %s.', self.module_name)
-            return True
-        self.installed_version = self._get_installed_version_from_module(module)
-        logging.info('imported %s version %s.', self.module_name,
-                     self.installed_version)
-        return self.version > self.installed_version
-
-
-    def _get_installed_version_from_module(self, module):
-        """Ask our module its version string and return it or '' if unknown."""
-        try:
-            return module.__version__
-        except AttributeError:
-            logging.error('could not get version from %s', module)
-            return ''
-
-
-    def _build_and_install(self, install_dir):
-        """Subclasses MUST provide their own implementation."""
-        raise NotImplementedError
-
-
-    def _build_and_install_current_dir(self, install_dir):
-        """
-        Subclasses that use _build_and_install_from_package() MUST provide
-        their own implementation of this method.
-        """
-        raise NotImplementedError
-
-
-    def build_and_install(self, install_dir):
-        """
-        Builds and installs the package.  It must have been fetched already.
-
-        @param install_dir - The package installation directory.  If it does
-            not exist it will be created.
-        """
-        if not self.verified_package:
-            raise Error('Must call fetch() first.  - %s' % self.name)
-        self._check_os_requirements()
-        return self._build_and_install(install_dir)
-
-
-    def _check_os_requirements(self):
-        if not self.os_requirements:
-            return
-        failed = False
-        for file_name, package_name in self.os_requirements.iteritems():
-            if not os.path.exists(file_name):
-                failed = True
-                logging.error('File %s not found, %s needs it.',
-                              file_name, self.name)
-                logging.error('Perhaps you need to install something similar '
-                              'to the %s package for OS first.', package_name)
-        if failed:
-            raise Error('Missing OS requirements for %s.  (see above)' %
-                        self.name)
-
-
-    def _build_and_install_current_dir_setup_py(self, install_dir):
-        """For use as a _build_and_install_current_dir implementation."""
-        egg_path = self._build_egg_using_setup_py(setup_py='setup.py')
-        if not egg_path:
-            return False
-        return self._install_from_egg(install_dir, egg_path)
-
-
-    def _build_and_install_current_dir_setupegg_py(self, install_dir):
-        """For use as a _build_and_install_current_dir implementation."""
-        egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py')
-        if not egg_path:
-            return False
-        return self._install_from_egg(install_dir, egg_path)
-
-
-    def _build_and_install_current_dir_noegg(self, install_dir):
-        if not self._build_using_setup_py():
-            return False
-        return self._install_using_setup_py_and_rsync(install_dir)
-
-
-    def _build_and_install_from_package(self, install_dir):
-        """
-        This method may be used as a _build_and_install() implementation
-        for subclasses if they implement _build_and_install_current_dir().
-
-        Extracts the .tar.gz file, chdirs into the extracted directory
-        (which is assumed to match the tar filename) and calls
-        _build_and_isntall_current_dir from there.
-
-        Afterwards the build (regardless of failure) extracted .tar.gz
-        directory is cleaned up.
-
-        @returns True on success, False otherwise.
-
-        @raises OSError If the expected extraction directory does not exist.
-        """
-        self._extract_compressed_package()
-        if self.verified_package.endswith('.tar.gz'):
-            extension = '.tar.gz'
-        elif self.verified_package.endswith('.tar.bz2'):
-            extension = '.tar.bz2'
-        elif self.verified_package.endswith('.zip'):
-            extension = '.zip'
-        else:
-            raise Error('Unexpected package file extension on %s' %
-                        self.verified_package)
-        os.chdir(os.path.dirname(self.verified_package))
-        os.chdir(self.local_filename[:-len(extension)])
-        extracted_dir = os.getcwd()
-        try:
-            return self._build_and_install_current_dir(install_dir)
-        finally:
-            os.chdir(os.path.join(extracted_dir, '..'))
-            shutil.rmtree(extracted_dir)
-
-
-    def _extract_compressed_package(self):
-        """Extract the fetched compressed .tar or .zip within its directory."""
-        if not self.verified_package:
-            raise Error('Package must have been fetched first.')
-        os.chdir(os.path.dirname(self.verified_package))
-        if self.verified_package.endswith('gz'):
-            status = system("tar -xzf '%s'" % self.verified_package)
-        elif self.verified_package.endswith('bz2'):
-            status = system("tar -xjf '%s'" % self.verified_package)
-        elif self.verified_package.endswith('zip'):
-            status = system("unzip '%s'" % self.verified_package)
-        else:
-            raise Error('Unknown compression suffix on %s.' %
-                        self.verified_package)
-        if status:
-            raise Error('tar failed with %s' % (status,))
-
-
-    def _build_using_setup_py(self, setup_py='setup.py'):
-        """
-        Assuming the cwd is the extracted python package, execute a simple
-        python setup.py build.
-
-        @param setup_py - The name of the setup.py file to execute.
-
-        @returns True on success, False otherwise.
-        """
-        if not os.path.exists(setup_py):
-            raise Error('%sdoes not exist in %s' % (setup_py, os.getcwd()))
-        status = system("'%s' %s build" % (sys.executable, setup_py))
-        if status:
-            logging.error('%s build failed.' % self.name)
-            return False
-        return True
-
-
-    def _build_egg_using_setup_py(self, setup_py='setup.py'):
-        """
-        Assuming the cwd is the extracted python package, execute a simple
-        python setup.py bdist_egg.
-
-        @param setup_py - The name of the setup.py file to execute.
-
-        @returns The relative path to the resulting egg file or '' on failure.
-        """
-        if not os.path.exists(setup_py):
-            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
-        egg_subdir = 'dist'
-        if os.path.isdir(egg_subdir):
-            shutil.rmtree(egg_subdir)
-        status = system("'%s' %s bdist_egg" % (sys.executable, setup_py))
-        if status:
-            logging.error('bdist_egg of setuptools failed.')
-            return ''
-        # I've never seen a bdist_egg lay multiple .egg files.
-        for filename in os.listdir(egg_subdir):
-            if filename.endswith('.egg'):
-                return os.path.join(egg_subdir, filename)
-
-
-    def _install_from_egg(self, install_dir, egg_path):
-        """
-        Install a module from an egg file by unzipping the necessary parts
-        into install_dir.
-
-        @param install_dir - The installation directory.
-        @param egg_path - The pathname of the egg file.
-        """
-        status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path))
-        if status:
-            logging.error('unzip of %s failed', egg_path)
-            return False
-        egg_info = os.path.join(install_dir, 'EGG-INFO')
-        if os.path.isdir(egg_info):
-            shutil.rmtree(egg_info)
-        return True
-
-
-    def _install_using_setup_py_and_rsync(self, install_dir,
-                                          setup_py='setup.py'):
-        """
-        Assuming the cwd is the extracted python package, execute a simple:
-
-          python setup.py install --prefix=BLA
-
-        BLA will be a temporary directory that everything installed will
-        be picked out of and rsynced to the appropriate place under
-        install_dir afterwards.
-
-        Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/
-        directory tree that setuptools created and moves all installed
-        site-packages directly up into install_dir itself.
-
-        @param install_dir the directory for the install to happen under.
-        @param setup_py - The name of the setup.py file to execute.
-
-        @returns True on success, False otherwise.
-        """
-        if not os.path.exists(setup_py):
-            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
-
-        temp_dir = tempfile.mkdtemp(dir='/var/tmp')
-        try:
-            status = system("'%s' %s install --no-compile --prefix='%s'"
-                            % (sys.executable, setup_py, temp_dir))
-            if status:
-                logging.error('%s install failed.' % self.name)
-                return False
-
-            # This makes assumptions about what python setup.py install
-            # does when given a prefix.  Is this always correct?
-            python_xy = 'python%s' % sys.version[:3]
-            if os.path.isdir(os.path.join(temp_dir, 'lib')):
-                # NOTE: This ignores anything outside of the lib/ dir that
-                # was installed.
-                temp_site_dir = os.path.join(
-                        temp_dir, 'lib', python_xy, 'site-packages')
-            else:
-                temp_site_dir = temp_dir
-
-            status = system("rsync -r '%s/' '%s/'" %
-                            (temp_site_dir, install_dir))
-            if status:
-                logging.error('%s rsync to install_dir failed.' % self.name)
-                return False
-            return True
-        finally:
-            shutil.rmtree(temp_dir)
-
-
-
-    def fetch(self, dest_dir):
-        """
-        Fetch the package from one its URLs and save it in dest_dir.
-
-        If the the package already exists in dest_dir and the checksum
-        matches this code will not fetch it again.
-
-        Sets the 'verified_package' attribute with the destination pathname.
-
-        @param dest_dir - The destination directory to save the local file.
-            If it does not exist it will be created.
-
-        @returns A boolean indicating if we the package is now in dest_dir.
-        @raises FetchError - When something unexpected happens.
-        """
-        if not os.path.exists(dest_dir):
-            os.makedirs(dest_dir)
-        local_path = os.path.join(dest_dir, self.local_filename)
-
-        # If the package exists, verify its checksum and be happy if it is good.
-        if os.path.exists(local_path):
-            actual_hex_sum = _checksum_file(local_path)
-            if self.hex_sum == actual_hex_sum:
-                logging.info('Good checksum for existing %s package.',
-                             self.name)
-                self.verified_package = local_path
-                return True
-            logging.warning('Bad checksum for existing %s package.  '
-                            'Re-downloading', self.name)
-            os.rename(local_path, local_path + '.wrong-checksum')
-
-        # Download the package from one of its urls, rejecting any if the
-        # checksum does not match.
-        for url in self.urls:
-            logging.info('Fetching %s', url)
-            try:
-                url_file = urllib2.urlopen(url)
-            except (urllib2.URLError, EnvironmentError):
-                logging.warning('Could not fetch %s package from %s.',
-                                self.name, url)
-                continue
-            data_length = int(url_file.info().get('Content-Length',
-                                                  _MAX_PACKAGE_SIZE))
-            if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE:
-                raise FetchError('%s from %s fails Content-Length %d '
-                                 'sanity check.' % (self.name, url,
-                                                    data_length))
-            checksum = utils.hash('sha1')
-            total_read = 0
-            output = open(local_path, 'wb')
-            try:
-                while total_read < data_length:
-                    data = url_file.read(_READ_SIZE)
-                    if not data:
-                        break
-                    output.write(data)
-                    checksum.update(data)
-                    total_read += len(data)
-            finally:
-                output.close()
-            if self.hex_sum != checksum.hexdigest():
-                logging.warning('Bad checksum for %s fetched from %s.',
-                                self.name, url)
-                logging.warning('Got %s', checksum.hexdigest())
-                logging.warning('Expected %s', self.hex_sum)
-                os.unlink(local_path)
-                continue
-            logging.info('Good checksum.')
-            self.verified_package = local_path
-            return True
-        else:
-            return False
-
-
-# NOTE: This class definition must come -before- all other ExternalPackage
-# classes that need to use this version of setuptools so that is is inserted
-# into the ExternalPackage.subclasses list before them.
-class SetuptoolsPackage(ExternalPackage):
-    # For all known setuptools releases a string compare works for the
-    # version string.  Hopefully they never release a 0.10.  (Their own
-    # version comparison code would break if they did.)
-    version = '0.6c9'
-    urls = ('http://pypi.python.org/packages/source/s/setuptools/'
-            'setuptools-%s.tar.gz' % (version,),)
-    local_filename = 'setuptools-%s.tar.gz' % version
-    hex_sum = '79086433b341f0c1df45e10d586a7d3cc25089f1'
-
-    SUDO_SLEEP_DELAY = 15
-
-
-    def _build_and_install(self, install_dir):
-        """Install setuptools on the system."""
-        logging.info('NOTE: setuptools install does not use install_dir.')
-        return self._build_and_install_from_package(install_dir)
-
-
-    def _build_and_install_current_dir(self, install_dir):
-        egg_path = self._build_egg_using_setup_py()
-        if not egg_path:
-            return False
-
-        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'
-        print 'About to run sudo to install setuptools', self.version
-        print 'on your system for use by', sys.executable, '\n'
-        print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n'
-        time.sleep(self.SUDO_SLEEP_DELAY)
-
-        # Copy the egg to the local filesystem /var/tmp so that root can
-        # access it properly (avoid NFS squashroot issues).
-        temp_dir = tempfile.mkdtemp(dir='/var/tmp')
-        try:
-            shutil.copy(egg_path, temp_dir)
-            egg_name = os.path.split(egg_path)[1]
-            temp_egg = os.path.join(temp_dir, egg_name)
-            p = subprocess.Popen(['sudo', '/bin/sh', temp_egg],
-                                 stdout=subprocess.PIPE)
-            regex = re.compile('Copying (.*?) to (.*?)\n')
-            match = regex.search(p.communicate()[0])
-            status = p.wait()
-
-            if match:
-                compiled = os.path.join(match.group(2), match.group(1))
-                os.system("sudo chmod a+r '%s'" % compiled)
-        finally:
-            shutil.rmtree(temp_dir)
-
-        if status:
-            logging.error('install of setuptools from egg failed.')
-            return False
-        return True
-
-
-class MySQLdbPackage(ExternalPackage):
-    module_name = 'MySQLdb'
-    version = '1.2.2'
-    urls = ('http://downloads.sourceforge.net/project/mysql-python/'
-            'mysql-python/%(version)s/MySQL-python-%(version)s.tar.gz'
-            % dict(version=version),)
-    local_filename = 'MySQL-python-%s.tar.gz' % version
-    hex_sum = '945a04773f30091ad81743f9eb0329a3ee3de383'
-
-    _build_and_install_current_dir = (
-            ExternalPackage._build_and_install_current_dir_setup_py)
-
-
-    def _build_and_install(self, install_dir):
-        if not os.path.exists('/usr/bin/mysql_config'):
-            logging.error('You need to install /usr/bin/mysql_config')
-            logging.error('On Ubuntu or Debian based systems use this: '
-                          'sudo apt-get install libmysqlclient15-dev')
-            return False
-        return self._build_and_install_from_package(install_dir)
-
-
-class DjangoPackage(ExternalPackage):
-    version = '1.1.1'
-    local_filename = 'Django-%s.tar.gz' % version
-    urls = ('http://www.djangoproject.com/download/%s/tarball/' % version,)
-    hex_sum = '441c54f0e90730bf4a55432b64519169b1e6ef20'
-
-    _build_and_install = ExternalPackage._build_and_install_from_package
-    _build_and_install_current_dir = (
-            ExternalPackage._build_and_install_current_dir_noegg)
-
-
-    def _get_installed_version_from_module(self, module):
-        try:
-            return module.get_version().split()[0]
-        except AttributeError:
-            return '0.9.6'
-
-
-
-class NumpyPackage(ExternalPackage):
-    version = '1.2.1'
-    local_filename = 'numpy-%s.tar.gz' % version
-    urls = ('http://downloads.sourceforge.net/project/numpy/NumPy/%(version)s/'
-            'numpy-%(version)s.tar.gz' % dict(version=version),)
-    hex_sum = '1aa706e733aea18eaffa70d93c0105718acb66c5'
-
-    _build_and_install = ExternalPackage._build_and_install_from_package
-    _build_and_install_current_dir = (
-            ExternalPackage._build_and_install_current_dir_setupegg_py)
-
-
-# This requires numpy so it must be declared after numpy to guarantee that it
-# is already installed.
-class MatplotlibPackage(ExternalPackage):
-    version = '0.98.5.3'
-    short_version = '0.98.5'
-    local_filename = 'matplotlib-%s.tar.gz' % version
-    urls = ('http://downloads.sourceforge.net/project/matplotlib/matplotlib/'
-            'matplotlib-%s/matplotlib-%s.tar.gz' % (short_version, version),)
-    hex_sum = '2f6c894cf407192b3b60351bcc6468c0385d47b6'
-    os_requirements = {'/usr/include/ft2build.h': 'libfreetype6-dev',
-                       '/usr/include/png.h': 'libpng12-dev'}
-
-    _build_and_install = ExternalPackage._build_and_install_from_package
-    _build_and_install_current_dir = (
-            ExternalPackage._build_and_install_current_dir_setupegg_py)
-
-
-class AtForkPackage(ExternalPackage):
-    version = '0.1.2'
-    local_filename = 'atfork-%s.zip' % version
-    urls = ('http://python-atfork.googlecode.com/files/' + local_filename,)
-    hex_sum = '5baa64c73e966b57fa797040585c760c502dc70b'
-
-    _build_and_install = ExternalPackage._build_and_install_from_package
-    _build_and_install_current_dir = (
-            ExternalPackage._build_and_install_current_dir_noegg)
-
-
-class ParamikoPackage(ExternalPackage):
-    version = '1.7.5'
-    local_filename = 'paramiko-%s.tar.gz' % version
-    urls = ('http://www.lag.net/paramiko/download/' + local_filename,)
-    hex_sum = '592be7a08290070b71da63a8e6f28a803399e5c5'
-
-
-    _build_and_install = ExternalPackage._build_and_install_from_package
-
-
-    def _check_for_pycrypto(self):
-        # NOTE(gps): Linux distros have better python-crypto packages than we
-        # can easily get today via a wget due to the library's age and staleness
-        # yet many security and behavior bugs are fixed by patches that distros
-        # already apply.  PyCrypto has a new active maintainer in 2009.  Once a
-        # new release is made (http://pycrypto.org/) we should add an installer.
-        try:
-            import Crypto
-        except ImportError:
-            logging.error('Please run "sudo apt-get install python-crypto" '
-                          'or your Linux distro\'s equivalent.')
-            return False
-        return True
-
-
-    def _build_and_install_current_dir(self, install_dir):
-        if not self._check_for_pycrypto():
-            return False
-        # paramiko 1.7.4 doesn't require building, it is just a module directory
-        # that we can rsync into place directly.
-        if not os.path.isdir('paramiko'):
-            raise Error('no paramiko directory in %s.' % os.getcwd())
-        status = system("rsync -r 'paramiko' '%s/'" % install_dir)
-        if status:
-            logging.error('%s rsync to install_dir failed.' % self.name)
-            return False
-        return True
-
-
-class SimplejsonPackage(ExternalPackage):
-    version = '2.0.9'
-    local_filename = 'simplejson-%s.tar.gz' % version
-    urls = ('http://pypi.python.org/packages/source/s/simplejson/' +
-            local_filename,)
-    hex_sum = 'b5b26059adbe677b06c299bed30557fcb0c7df8c'
-
-    _build_and_install = ExternalPackage._build_and_install_from_package
-    _build_and_install_current_dir = (
-                        ExternalPackage._build_and_install_current_dir_setup_py)
-
-
-class Httplib2Package(ExternalPackage):
-    version = '0.6.0'
-    local_filename = 'httplib2-%s.tar.gz' % version
-    urls = ('http://httplib2.googlecode.com/files/' + local_filename,)
-    hex_sum = '995344b2704826cc0d61a266e995b328d92445a5'
-
-    _build_and_install = ExternalPackage._build_and_install_from_package
-    _build_and_install_current_dir = (
-                        ExternalPackage._build_and_install_current_dir_noegg)
-
-
-class GwtPackage(ExternalPackage):
-    """Fetch and extract a local copy of GWT used to build the frontend."""
-
-    version = '1.7.0'
-    local_filename = 'gwt-linux-%s.tar.bz2' % version
-    urls = ('http://google-web-toolkit.googlecode.com/files/' + local_filename,)
-    hex_sum = 'accb39506e1fa719ba166cf54451c91dafd9d456'
-    name = 'gwt'
-    about_filename = 'about.txt'
-    module_name = None  # Not a Python module.
-
-
-    def is_needed(self, install_dir):
-        gwt_dir = os.path.join(install_dir, self.name)
-        about_file = os.path.join(install_dir, self.name, self.about_filename)
-
-        if not os.path.exists(gwt_dir) or not os.path.exists(about_file):
-            logging.info('gwt not installed for autotest')
-            return True
-
-        f = open(about_file, 'r')
-        version_line = f.readline()
-        f.close()
-
-        match = re.match(r'Google Web Toolkit (.*)', version_line)
-        if not match:
-            logging.info('did not find gwt version')
-            return True
-
-        logging.info('found gwt version %s', match.group(1))
-        return match.group(1) != self.version
-
-
-    def build_and_install(self, install_dir):
-        os.chdir(install_dir)
-        self._extract_compressed_package()
-        extracted_dir = self.local_filename[:-len('.tar.bz2')]
-        target_dir = os.path.join(install_dir, self.name)
-        if os.path.exists(target_dir):
-            shutil.rmtree(target_dir)
-        os.rename(extracted_dir, target_dir)
-        return True
-
-
 if __name__ == '__main__':
     sys.exit(main())
diff --git a/utils/external_packages.py b/utils/external_packages.py
new file mode 100644
index 00000000..0db307dd
--- /dev/null
+++ b/utils/external_packages.py
@@ -0,0 +1,724 @@
+#!/usr/bin/python
+#
+# Please keep this code python 2.4 compatible and stand alone.
+
+import logging, os, shutil, sys, tempfile, time, urllib2
+import subprocess, re
+from autotest_lib.client.common_lib import utils
+
+_READ_SIZE = 64*1024
+_MAX_PACKAGE_SIZE = 100*1024*1024
+
+
+class Error(Exception):
+    """Local exception to be raised by code in this file."""
+
+class FetchError(Error):
+    """Failed to fetch a package from any of its listed URLs."""
+
+
+def _checksum_file(full_path):
+    """@returns The hex checksum of a file given its pathname."""
+    inputfile = open(full_path, 'rb')
+    try:
+        hex_sum = utils.hash('sha1', inputfile.read()).hexdigest()
+    finally:
+        inputfile.close()
+    return hex_sum
+
+
+def system(commandline):
+    """Same as os.system(commandline) but logs the command first."""
+    logging.info(commandline)
+    return os.system(commandline)
+
+
+def find_top_of_autotest_tree():
+    """@returns The full path to the top of the autotest directory tree."""
+    dirname = os.path.dirname(__file__)
+    autotest_dir = os.path.abspath(os.path.join(dirname, '..'))
+    return autotest_dir
+
+
+class ExternalPackage(object):
+    """
+    Defines an external package with URLs to fetch its sources from and
+    a build_and_install() method to unpack it, build it and install it
+    beneath our own autotest/site-packages directory.
+
+    Base Class.  Subclass this to define packages.
+
+    Attributes:
+      @attribute urls - A tuple of URLs to try fetching the package from.
+      @attribute local_filename - A local filename to use when saving the
+              fetched package.
+      @attribute hex_sum - The hex digest (currently SHA1) of this package
+              to be used to verify its contents.
+      @attribute module_name - The installed python module name to be used for
+              for a version check.  Defaults to the lower case class name with
+              the word Package stripped off.
+      @attribute version - The desired minimum package version.
+      @attribute os_requirements - A dictionary mapping a file pathname on the
+              the OS distribution to a likely name of a package the user
+              needs to install on their system in order to get this file.
+      @attribute name - Read only, the printable name of the package.
+      @attribute subclasses - This class attribute holds a list of all defined
+              subclasses.  It is constructed dynamically using the metaclass.
+    """
+    subclasses = []
+    urls = ()
+    local_filename = None
+    hex_sum = None
+    module_name = None
+    version = None
+    os_requirements = None
+
+
+    class __metaclass__(type):
+        """Any time a subclass is defined, add it to our list."""
+        def __init__(mcs, name, bases, dict):
+            if name != 'ExternalPackage':
+                mcs.subclasses.append(mcs)
+
+
+    def __init__(self):
+        self.verified_package = ''
+        if not self.module_name:
+            self.module_name = self.name.lower()
+        self.installed_version = ''
+
+
+    @property
+    def name(self):
+        """Return the class name with any trailing 'Package' stripped off."""
+        class_name = self.__class__.__name__
+        if class_name.endswith('Package'):
+            return class_name[:-len('Package')]
+        return class_name
+
+
+    def is_needed(self, unused_install_dir):
+        """@returns True if self.module_name needs to be built and installed."""
+        if not self.module_name or not self.version:
+            logging.warning('version and module_name required for '
+                            'is_needed() check to work.')
+            return True
+        try:
+            module = __import__(self.module_name)
+        except ImportError, e:
+            logging.info('Could not import %s.', self.module_name)
+            return True
+        self.installed_version = self._get_installed_version_from_module(module)
+        logging.info('imported %s version %s.', self.module_name,
+                     self.installed_version)
+        return self.version > self.installed_version
+
+
+    def _get_installed_version_from_module(self, module):
+        """Ask our module its version string and return it or '' if unknown."""
+        try:
+            return module.__version__
+        except AttributeError:
+            logging.error('could not get version from %s', module)
+            return ''
+
+
+    def _build_and_install(self, install_dir):
+        """Subclasses MUST provide their own implementation."""
+        raise NotImplementedError
+
+
+    def _build_and_install_current_dir(self, install_dir):
+        """
+        Subclasses that use _build_and_install_from_package() MUST provide
+        their own implementation of this method.
+        """
+        raise NotImplementedError
+
+
+    def build_and_install(self, install_dir):
+        """
+        Builds and installs the package.  It must have been fetched already.
+
+        @param install_dir - The package installation directory.  If it does
+            not exist it will be created.
+        """
+        if not self.verified_package:
+            raise Error('Must call fetch() first.  - %s' % self.name)
+        self._check_os_requirements()
+        return self._build_and_install(install_dir)
+
+
+    def _check_os_requirements(self):
+        if not self.os_requirements:
+            return
+        failed = False
+        for file_name, package_name in self.os_requirements.iteritems():
+            if not os.path.exists(file_name):
+                failed = True
+                logging.error('File %s not found, %s needs it.',
+                              file_name, self.name)
+                logging.error('Perhaps you need to install something similar '
+                              'to the %s package for OS first.', package_name)
+        if failed:
+            raise Error('Missing OS requirements for %s.  (see above)' %
+                        self.name)
+
+
+    def _build_and_install_current_dir_setup_py(self, install_dir):
+        """For use as a _build_and_install_current_dir implementation."""
+        egg_path = self._build_egg_using_setup_py(setup_py='setup.py')
+        if not egg_path:
+            return False
+        return self._install_from_egg(install_dir, egg_path)
+
+
+    def _build_and_install_current_dir_setupegg_py(self, install_dir):
+        """For use as a _build_and_install_current_dir implementation."""
+        egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py')
+        if not egg_path:
+            return False
+        return self._install_from_egg(install_dir, egg_path)
+
+
+    def _build_and_install_current_dir_noegg(self, install_dir):
+        if not self._build_using_setup_py():
+            return False
+        return self._install_using_setup_py_and_rsync(install_dir)
+
+
+    def _build_and_install_from_package(self, install_dir):
+        """
+        This method may be used as a _build_and_install() implementation
+        for subclasses if they implement _build_and_install_current_dir().
+
+        Extracts the .tar.gz file, chdirs into the extracted directory
+        (which is assumed to match the tar filename) and calls
+        _build_and_isntall_current_dir from there.
+
+        Afterwards the build (regardless of failure) extracted .tar.gz
+        directory is cleaned up.
+
+        @returns True on success, False otherwise.
+
+        @raises OSError If the expected extraction directory does not exist.
+        """
+        self._extract_compressed_package()
+        if self.verified_package.endswith('.tar.gz'):
+            extension = '.tar.gz'
+        elif self.verified_package.endswith('.tar.bz2'):
+            extension = '.tar.bz2'
+        elif self.verified_package.endswith('.zip'):
+            extension = '.zip'
+        else:
+            raise Error('Unexpected package file extension on %s' %
+                        self.verified_package)
+        os.chdir(os.path.dirname(self.verified_package))
+        os.chdir(self.local_filename[:-len(extension)])
+        extracted_dir = os.getcwd()
+        try:
+            return self._build_and_install_current_dir(install_dir)
+        finally:
+            os.chdir(os.path.join(extracted_dir, '..'))
+            shutil.rmtree(extracted_dir)
+
+
+    def _extract_compressed_package(self):
+        """Extract the fetched compressed .tar or .zip within its directory."""
+        if not self.verified_package:
+            raise Error('Package must have been fetched first.')
+        os.chdir(os.path.dirname(self.verified_package))
+        if self.verified_package.endswith('gz'):
+            status = system("tar -xzf '%s'" % self.verified_package)
+        elif self.verified_package.endswith('bz2'):
+            status = system("tar -xjf '%s'" % self.verified_package)
+        elif self.verified_package.endswith('zip'):
+            status = system("unzip '%s'" % self.verified_package)
+        else:
+            raise Error('Unknown compression suffix on %s.' %
+                        self.verified_package)
+        if status:
+            raise Error('tar failed with %s' % (status,))
+
+
+    def _build_using_setup_py(self, setup_py='setup.py'):
+        """
+        Assuming the cwd is the extracted python package, execute a simple
+        python setup.py build.
+
+        @param setup_py - The name of the setup.py file to execute.
+
+        @returns True on success, False otherwise.
+        """
+        if not os.path.exists(setup_py):
+            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
+        status = system("'%s' %s build" % (sys.executable, setup_py))
+        if status:
+            logging.error('%s build failed.' % self.name)
+            return False
+        return True
+
+
+    def _build_egg_using_setup_py(self, setup_py='setup.py'):
+        """
+        Assuming the cwd is the extracted python package, execute a simple
+        python setup.py bdist_egg.
+
+        @param setup_py - The name of the setup.py file to execute.
+
+        @returns The relative path to the resulting egg file or '' on failure.
+        """
+        if not os.path.exists(setup_py):
+            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
+        egg_subdir = 'dist'
+        if os.path.isdir(egg_subdir):
+            shutil.rmtree(egg_subdir)
+        status = system("'%s' %s bdist_egg" % (sys.executable, setup_py))
+        if status:
+            logging.error('bdist_egg of setuptools failed.')
+            return ''
+        # I've never seen a bdist_egg lay multiple .egg files.
+        for filename in os.listdir(egg_subdir):
+            if filename.endswith('.egg'):
+                return os.path.join(egg_subdir, filename)
+
+
+    def _install_from_egg(self, install_dir, egg_path):
+        """
+        Install a module from an egg file by unzipping the necessary parts
+        into install_dir.
+
+        @param install_dir - The installation directory.
+        @param egg_path - The pathname of the egg file.
+        """
+        status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path))
+        if status:
+            logging.error('unzip of %s failed', egg_path)
+            return False
+        egg_info = os.path.join(install_dir, 'EGG-INFO')
+        if os.path.isdir(egg_info):
+            shutil.rmtree(egg_info)
+        return True
+
+
+    def _get_temp_dir(self):
+        return tempfile.mkdtemp(dir='/var/tmp')
+
+
+    def _site_packages_path(self, temp_dir):
+        # This makes assumptions about what python setup.py install
+        # does when given a prefix.  Is this always correct?
+        python_xy = 'python%s' % sys.version[:3]
+        return os.path.join(temp_dir, 'lib', python_xy, 'site-packages')
+
+
+    def _install_using_setup_py_and_rsync(self, install_dir,
+                                          setup_py='setup.py',
+                                          temp_dir=None):
+        """
+        Assuming the cwd is the extracted python package, execute a simple:
+
+          python setup.py install --prefix=BLA
+
+        BLA will be a temporary directory that everything installed will
+        be picked out of and rsynced to the appropriate place under
+        install_dir afterwards.
+
+        Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/
+        directory tree that setuptools created and moves all installed
+        site-packages directly up into install_dir itself.
+
+        @param install_dir the directory for the install to happen under.
+        @param setup_py - The name of the setup.py file to execute.
+
+        @returns True on success, False otherwise.
+        """
+        if not os.path.exists(setup_py):
+            raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
+
+        if temp_dir is None:
+            temp_dir = self._get_temp_dir()
+
+        try:
+            status = system("'%s' %s install --no-compile --prefix='%s'"
+                            % (sys.executable, setup_py, temp_dir))
+            if status:
+                logging.error('%s install failed.' % self.name)
+                return False
+
+            if os.path.isdir(os.path.join(temp_dir, 'lib')):
+                # NOTE: This ignores anything outside of the lib/ dir that
+                # was installed.
+                temp_site_dir = self._site_packages_path(temp_dir)
+            else:
+                temp_site_dir = temp_dir
+
+            status = system("rsync -r '%s/' '%s/'" %
+                            (temp_site_dir, install_dir))
+            if status:
+                logging.error('%s rsync to install_dir failed.' % self.name)
+                return False
+            return True
+        finally:
+            shutil.rmtree(temp_dir)
+
+
+
+    def _build_using_make(self, install_dir):
+        """Build the current package using configure/make.
+
+        @returns True on success, False otherwise.
+        """
+        install_prefix = os.path.join(install_dir, 'usr', 'local')
+        status = system('./configure --prefix=%s' % install_prefix)
+        if status:
+            logging.error('./configure failed for %s', self.name)
+            return False
+        status = system('make')
+        if status:
+            logging.error('make failed for %s', self.name)
+            return False
+        status = system('make check')
+        if status:
+            logging.error('make check failed for %s', self.name)
+            return False
+        return True
+
+
+    def _install_using_make(self):
+        """Install the current package using make install.
+
+        Assumes the install path was set up while running ./configure (in
+        _build_using_make()).
+
+        @returns True on success, False otherwise.
+        """
+        status = system('make install')
+        return status == 0
+
+
+    def fetch(self, dest_dir):
+        """
+        Fetch the package from one its URLs and save it in dest_dir.
+
+        If the the package already exists in dest_dir and the checksum
+        matches this code will not fetch it again.
+
+        Sets the 'verified_package' attribute with the destination pathname.
+
+        @param dest_dir - The destination directory to save the local file.
+            If it does not exist it will be created.
+
+        @returns A boolean indicating if we the package is now in dest_dir.
+        @raises FetchError - When something unexpected happens.
+        """
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+        local_path = os.path.join(dest_dir, self.local_filename)
+
+        # If the package exists, verify its checksum and be happy if it is good.
+        if os.path.exists(local_path):
+            actual_hex_sum = _checksum_file(local_path)
+            if self.hex_sum == actual_hex_sum:
+                logging.info('Good checksum for existing %s package.',
+                             self.name)
+                self.verified_package = local_path
+                return True
+            logging.warning('Bad checksum for existing %s package.  '
+                            'Re-downloading', self.name)
+            os.rename(local_path, local_path + '.wrong-checksum')
+
+        # Download the package from one of its urls, rejecting any if the
+        # checksum does not match.
+        for url in self.urls:
+            logging.info('Fetching %s', url)
+            try:
+                url_file = urllib2.urlopen(url)
+            except (urllib2.URLError, EnvironmentError):
+                logging.warning('Could not fetch %s package from %s.',
+                                self.name, url)
+                continue
+            data_length = int(url_file.info().get('Content-Length',
+                                                  _MAX_PACKAGE_SIZE))
+            if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE:
+                raise FetchError('%s from %s fails Content-Length %d '
+                                 'sanity check.' % (self.name, url,
+                                                    data_length))
+            checksum = utils.hash('sha1')
+            total_read = 0
+            output = open(local_path, 'wb')
+            try:
+                while total_read < data_length:
+                    data = url_file.read(_READ_SIZE)
+                    if not data:
+                        break
+                    output.write(data)
+                    checksum.update(data)
+                    total_read += len(data)
+            finally:
+                output.close()
+            if self.hex_sum != checksum.hexdigest():
+                logging.warning('Bad checksum for %s fetched from %s.',
+                                self.name, url)
+                logging.warning('Got %s', checksum.hexdigest())
+                logging.warning('Expected %s', self.hex_sum)
+                os.unlink(local_path)
+                continue
+            logging.info('Good checksum.')
+            self.verified_package = local_path
+            return True
+        else:
+            return False
+
+
+# NOTE: This class definition must come -before- all other ExternalPackage
+# classes that need to use this version of setuptools so that is is inserted
+# into the ExternalPackage.subclasses list before them.
+class SetuptoolsPackage(ExternalPackage):
+    # For all known setuptools releases a string compare works for the
+    # version string.  Hopefully they never release a 0.10.  (Their own
+    # version comparison code would break if they did.)
+    version = '0.6c9'
+    urls = ('http://pypi.python.org/packages/source/s/setuptools/'
+            'setuptools-%s.tar.gz' % (version,),)
+    local_filename = 'setuptools-%s.tar.gz' % version
+    hex_sum = '79086433b341f0c1df45e10d586a7d3cc25089f1'
+
+    SUDO_SLEEP_DELAY = 15
+
+
+    def _build_and_install(self, install_dir):
+        """Install setuptools on the system."""
+        logging.info('NOTE: setuptools install does not use install_dir.')
+        return self._build_and_install_from_package(install_dir)
+
+
+    def _build_and_install_current_dir(self, install_dir):
+        egg_path = self._build_egg_using_setup_py()
+        if not egg_path:
+            return False
+
+        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'
+        print 'About to run sudo to install setuptools', self.version
+        print 'on your system for use by', sys.executable, '\n'
+        print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n'
+        time.sleep(self.SUDO_SLEEP_DELAY)
+
+        # Copy the egg to the local filesystem /var/tmp so that root can
+        # access it properly (avoid NFS squashroot issues).
+        temp_dir = self._get_temp_dir()
+        try:
+            shutil.copy(egg_path, temp_dir)
+            egg_name = os.path.split(egg_path)[1]
+            temp_egg = os.path.join(temp_dir, egg_name)
+            p = subprocess.Popen(['sudo', '/bin/sh', temp_egg],
+                                 stdout=subprocess.PIPE)
+            regex = re.compile('Copying (.*?) to (.*?)\n')
+            match = regex.search(p.communicate()[0])
+            status = p.wait()
+
+            if match:
+                compiled = os.path.join(match.group(2), match.group(1))
+                os.system("sudo chmod a+r '%s'" % compiled)
+        finally:
+            shutil.rmtree(temp_dir)
+
+        if status:
+            logging.error('install of setuptools from egg failed.')
+            return False
+        return True
+
+
+class MySQLdbPackage(ExternalPackage):
+    module_name = 'MySQLdb'
+    version = '1.2.2'
+    urls = ('http://downloads.sourceforge.net/project/mysql-python/'
+            'mysql-python/%(version)s/MySQL-python-%(version)s.tar.gz'
+            % dict(version=version),)
+    local_filename = 'MySQL-python-%s.tar.gz' % version
+    hex_sum = '945a04773f30091ad81743f9eb0329a3ee3de383'
+
+    _build_and_install_current_dir = (
+            ExternalPackage._build_and_install_current_dir_setup_py)
+
+
+    def _build_and_install(self, install_dir):
+        if not os.path.exists('/usr/bin/mysql_config'):
+            logging.error('You need to install /usr/bin/mysql_config')
+            logging.error('On Ubuntu or Debian based systems use this: '
+                          'sudo apt-get install libmysqlclient15-dev')
+            return False
+        return self._build_and_install_from_package(install_dir)
+
+
+class DjangoPackage(ExternalPackage):
+    version = '1.1.1'
+    local_filename = 'Django-%s.tar.gz' % version
+    urls = ('http://www.djangoproject.com/download/%s/tarball/' % version,)
+    hex_sum = '441c54f0e90730bf4a55432b64519169b1e6ef20'
+
+    _build_and_install = ExternalPackage._build_and_install_from_package
+    _build_and_install_current_dir = (
+            ExternalPackage._build_and_install_current_dir_noegg)
+
+
+    def _get_installed_version_from_module(self, module):
+        try:
+            return module.get_version().split()[0]
+        except AttributeError:
+            return '0.9.6'
+
+
+
+class NumpyPackage(ExternalPackage):
+    version = '1.2.1'
+    local_filename = 'numpy-%s.tar.gz' % version
+    urls = ('http://downloads.sourceforge.net/project/numpy/NumPy/%(version)s/'
+            'numpy-%(version)s.tar.gz' % dict(version=version),)
+    hex_sum = '1aa706e733aea18eaffa70d93c0105718acb66c5'
+
+    _build_and_install = ExternalPackage._build_and_install_from_package
+    _build_and_install_current_dir = (
+            ExternalPackage._build_and_install_current_dir_setupegg_py)
+
+
+# This requires numpy so it must be declared after numpy to guarantee that it
+# is already installed.
+class MatplotlibPackage(ExternalPackage):
+    version = '0.98.5.3'
+    short_version = '0.98.5'
+    local_filename = 'matplotlib-%s.tar.gz' % version
+    urls = ('http://downloads.sourceforge.net/project/matplotlib/matplotlib/'
+            'matplotlib-%s/matplotlib-%s.tar.gz' % (short_version, version),)
+    hex_sum = '2f6c894cf407192b3b60351bcc6468c0385d47b6'
+    os_requirements = {'/usr/include/ft2build.h': 'libfreetype6-dev',
+                       '/usr/include/png.h': 'libpng12-dev'}
+
+    _build_and_install = ExternalPackage._build_and_install_from_package
+    _build_and_install_current_dir = (
+            ExternalPackage._build_and_install_current_dir_setupegg_py)
+
+
+class AtForkPackage(ExternalPackage):
+    version = '0.1.2'
+    local_filename = 'atfork-%s.zip' % version
+    urls = ('http://python-atfork.googlecode.com/files/' + local_filename,)
+    hex_sum = '5baa64c73e966b57fa797040585c760c502dc70b'
+
+    _build_and_install = ExternalPackage._build_and_install_from_package
+    _build_and_install_current_dir = (
+            ExternalPackage._build_and_install_current_dir_noegg)
+
+
+class ParamikoPackage(ExternalPackage):
+    version = '1.7.5'
+    local_filename = 'paramiko-%s.tar.gz' % version
+    urls = ('http://www.lag.net/paramiko/download/' + local_filename,)
+    hex_sum = '592be7a08290070b71da63a8e6f28a803399e5c5'
+
+
+    _build_and_install = ExternalPackage._build_and_install_from_package
+
+
+    def _check_for_pycrypto(self):
+        # NOTE(gps): Linux distros have better python-crypto packages than we
+        # can easily get today via a wget due to the library's age and staleness
+        # yet many security and behavior bugs are fixed by patches that distros
+        # already apply.  PyCrypto has a new active maintainer in 2009.  Once a
+        # new release is made (http://pycrypto.org/) we should add an installer.
+        try:
+            import Crypto
+        except ImportError:
+            logging.error('Please run "sudo apt-get install python-crypto" '
+                          'or your Linux distro\'s equivalent.')
+            return False
+        return True
+
+
+    def _build_and_install_current_dir(self, install_dir):
+        if not self._check_for_pycrypto():
+            return False
+        # paramiko 1.7.4 doesn't require building, it is just a module directory
+        # that we can rsync into place directly.
+        if not os.path.isdir('paramiko'):
+            raise Error('no paramiko directory in %s.' % os.getcwd())
+        status = system("rsync -r 'paramiko' '%s/'" % install_dir)
+        if status:
+            logging.error('%s rsync to install_dir failed.' % self.name)
+            return False
+        return True
+
+
+class SimplejsonPackage(ExternalPackage):
+    version = '2.0.9'
+    local_filename = 'simplejson-%s.tar.gz' % version
+    urls = ('http://pypi.python.org/packages/source/s/simplejson/' +
+            local_filename,)
+    hex_sum = 'b5b26059adbe677b06c299bed30557fcb0c7df8c'
+
+    _build_and_install = ExternalPackage._build_and_install_from_package
+    _build_and_install_current_dir = (
+                        ExternalPackage._build_and_install_current_dir_setup_py)
+
+
+class Httplib2Package(ExternalPackage):
+    version = '0.6.0'
+    local_filename = 'httplib2-%s.tar.gz' % version
+    urls = ('http://httplib2.googlecode.com/files/' + local_filename,)
+    hex_sum = '995344b2704826cc0d61a266e995b328d92445a5'
+
+    def _get_installed_version_from_module(self, module):
+        # httplib2 doesn't contain a proper version
+        return self.version
+
+    _build_and_install = ExternalPackage._build_and_install_from_package
+    _build_and_install_current_dir = (
+                        ExternalPackage._build_and_install_current_dir_noegg)
+
+
+class GwtPackage(ExternalPackage):
+    """Fetch and extract a local copy of GWT used to build the frontend."""
+
+    version = '1.7.0'
+    local_filename = 'gwt-linux-%s.tar.bz2' % version
+    urls = ('http://google-web-toolkit.googlecode.com/files/' + local_filename,)
+    hex_sum = 'accb39506e1fa719ba166cf54451c91dafd9d456'
+    name = 'gwt'
+    about_filename = 'about.txt'
+    module_name = None  # Not a Python module.
+
+
+    def is_needed(self, install_dir):
+        gwt_dir = os.path.join(install_dir, self.name)
+        about_file = os.path.join(install_dir, self.name, self.about_filename)
+
+        if not os.path.exists(gwt_dir) or not os.path.exists(about_file):
+            logging.info('gwt not installed for autotest')
+            return True
+
+        f = open(about_file, 'r')
+        version_line = f.readline()
+        f.close()
+
+        match = re.match(r'Google Web Toolkit (.*)', version_line)
+        if not match:
+            logging.info('did not find gwt version')
+            return True
+
+        logging.info('found gwt version %s', match.group(1))
+        return match.group(1) != self.version
+
+
+    def build_and_install(self, install_dir):
+        os.chdir(install_dir)
+        self._extract_compressed_package()
+        extracted_dir = self.local_filename[:-len('.tar.bz2')]
+        target_dir = os.path.join(install_dir, self.name)
+        if os.path.exists(target_dir):
+            shutil.rmtree(target_dir)
+        os.rename(extracted_dir, target_dir)
+        return True
+
+
+if __name__ == '__main__':
+    sys.exit(main())