Make the parser read and use the machine platform from host_keyvals, if available. Goodbye machine_transfer! Also made it handle multimachine jobs (use a comma-separated, sorted list of distinct platforms from the individual machines) and atomic group jobs (use the same atomic group override value that was used for machine name, when applicable).

Signed-off-by: Steve Howard <showard@google.com>


git-svn-id: svn://test.kernel.org/autotest/trunk@3583 592f7852-d20e-0410-864c-8624ca9c26a4

4 files changed, 125 insertions, 40 deletions

diff --git a/tko/db.py b/tko/db.py
index ff939b4f..ec833553 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -314,8 +314,10 @@ class db_sql(object):
     def insert_job(self, tag, job, commit = None):
         job.machine_idx = self.lookup_machine(job.machine)
         if not job.machine_idx:
-            job.machine_idx = self.insert_machine(job,
-                                                  commit=commit)
+            job.machine_idx = self.insert_machine(job, commit=commit)
+        else:
+            self.update_machine_information(job, commit=commit)
+
         self.insert('jobs', {'tag':tag,
                              'label': job.label,
                              'username': job.user,
@@ -375,30 +377,40 @@ class db_sql(object):
 
 
     def read_machine_map(self):
-        self.machine_group = {}
+        if self.machine_group or not self.machine_map:
+            return
         for line in open(self.machine_map, 'r').readlines():
             (machine, group) = line.split()
             self.machine_group[machine] = group
 
 
-    def insert_machine(self, job, group = None, commit = None):
+    def machine_info_dict(self, job):
         hostname = job.machine
-        if self.machine_map and not self.machine_group:
-            self.read_machine_map()
+        group = job.machine_group
+        owner = job.machine_owner
 
         if not group:
+            self.read_machine_map()
             group = self.machine_group.get(hostname, hostname)
-            if group == hostname and job.machine_owner:
-                group = job.machine_owner + '/' + hostname
+            if group == hostname and owner:
+                group = owner + '/' + hostname
 
-        self.insert('machines',
-                    { 'hostname' : hostname ,
-                      'machine_group' : group ,
-                      'owner' : job.machine_owner },
-                    commit=commit)
+        return {'hostname': hostname, 'machine_group': group, 'owner': owner}
+
+
+    def insert_machine(self, job, commit = None):
+        machine_info = self.machine_info_dict(job)
+        self.insert('machines', machine_info, commit=commit)
         return self.get_last_autonumber_value()
 
 
+    def update_machine_information(self, job, commit = None):
+        machine_info = self.machine_info_dict(job)
+        self.update('machines', machine_info,
+                    where={'hostname': machine_info['hostname']},
+                    commit=commit)
+
+
     def lookup_machine(self, hostname):
         where = { 'hostname' : hostname }
         rows = self.select('machine_idx', 'machines', where)
diff --git a/tko/models.py b/tko/models.py
index 85166123..0842e18a 100644
--- a/tko/models.py
+++ b/tko/models.py
@@ -6,7 +6,8 @@ from autotest_lib.tko import utils as tko_utils
 
 class job(object):
     def __init__(self, dir, user, label, machine, queued_time, started_time,
-                 finished_time, machine_owner, aborted_by, aborted_on):
+                 finished_time, machine_owner, machine_group, aborted_by,
+                 aborted_on):
         self.dir = dir
         self.tests = []
         self.user = user
@@ -16,6 +17,7 @@ class job(object):
         self.started_time = started_time
         self.finished_time = finished_time
         self.machine_owner = machine_owner
+        self.machine_group = machine_group
         self.aborted_by = aborted_by
         self.aborted_on = aborted_on
 
@@ -110,7 +112,9 @@ class test(object):
             attributes = {}
 
         # grab test+host attributes from the host keyval
-        attributes.update(cls.parse_host_keyval(job.dir, job.machine))
+        host_keyval = cls.parse_host_keyval(job.dir, job.machine)
+        attributes.update(dict(("host-%s" % k, v)
+                               for k, v in host_keyval.iteritems()))
 
         if existing_instance:
             def constructor(*args, **dargs):
@@ -155,8 +159,7 @@ class test(object):
         # the keyval is <job_dir>/host_keyvals/<hostname> if it exists
         keyval_path = os.path.join(job_dir, "host_keyvals", hostname)
         if os.path.isfile(keyval_path):
-            keyval = utils.read_keyval(keyval_path)
-            return dict(("host-%s" % k, v) for k, v in keyval.iteritems())
+            return utils.read_keyval(keyval_path)
         else:
             return {}
 
diff --git a/tko/parsers/version_0.py b/tko/parsers/version_0.py
index 881b8d96..f1be7b48 100644
--- a/tko/parsers/version_0.py
+++ b/tko/parsers/version_0.py
@@ -22,31 +22,59 @@ class job(models.job):
 
         user = keyval.get("user", None)
         label = keyval.get("label", None)
-        host_group_name = keyval.get("host_group_name", None)
-        machine = keyval.get("hostname", None)
-        if not host_group_name and machine and "," in machine:
-            try:
-                machine = job.find_hostname(dir) # find a unique hostname
-            except NoHostnameError:
-                pass  # just use the comma-separated name
         queued_time = tko_utils.get_timestamp(keyval, "job_queued")
         started_time = tko_utils.get_timestamp(keyval, "job_started")
         finished_time = tko_utils.get_timestamp(keyval, "job_finished")
+        machine = cls.determine_hostname(keyval, dir)
+        machine_group = cls.determine_machine_group(machine, dir)
         machine_owner = keyval.get("owner", None)
 
         aborted_by = keyval.get("aborted_by", None)
         aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
 
-        tko_utils.dprint("MACHINE NAME: %s" % machine)
-        if host_group_name and ((machine and "," in machine) or not machine):
-            tko_utils.dprint("Using host_group_name %r instead of "
-                             "machine name." % host_group_name)
-            machine = host_group_name
-
         return {"user": user, "label": label, "machine": machine,
                 "queued_time": queued_time, "started_time": started_time,
                 "finished_time": finished_time, "machine_owner": machine_owner,
-                "aborted_by": aborted_by, "aborted_on": aborted_at}
+                "machine_group": machine_group, "aborted_by": aborted_by,
+                "aborted_on": aborted_at}
+
+
+    @classmethod
+    def determine_hostname(cls, keyval, job_dir):
+        host_group_name = keyval.get("host_group_name", None)
+        machine = keyval.get("hostname", "")
+        is_multimachine = "," in machine
+
+        # determine what hostname to use
+        if host_group_name:
+            if is_multimachine or not machine:
+                tko_utils.dprint("Using host_group_name %r instead of "
+                                 "machine name." % host_group_name)
+                machine = host_group_name
+        elif is_multimachine:
+            try:
+                machine = job.find_hostname(job_dir) # find a unique hostname
+            except NoHostnameError:
+                pass  # just use the comma-separated name
+
+        tko_utils.dprint("MACHINE NAME: %s" % machine)
+        return machine
+
+
+    @classmethod
+    def determine_machine_group(cls, hostname, job_dir):
+        machine_groups = set()
+        for individual_hostname in hostname.split(","):
+            host_keyval = models.test.parse_host_keyval(job_dir,
+                                                        individual_hostname)
+            if not host_keyval:
+                tko_utils.dprint('Unable to parse host keyval for %s'
+                                 % individual_hostname)
+            elif "platform" in host_keyval:
+                machine_groups.add(host_keyval["platform"])
+        machine_group = ",".join(sorted(machine_groups))
+        tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
+        return machine_group
 
 
     @staticmethod
diff --git a/tko/parsers/version_0_unittest.py b/tko/parsers/version_0_unittest.py
index 8c076ff7..4807beec 100644
--- a/tko/parsers/version_0_unittest.py
+++ b/tko/parsers/version_0_unittest.py
@@ -9,55 +9,95 @@ from autotest_lib.tko.parsers import version_0
 
 
 class test_job_load_from_dir(unittest.TestCase):
+    keyval_return = {'job_queued': 1234567890,
+                     'job_started': 1234567891,
+                     'job_finished': 1234567892,
+                     'user': 'janet',
+                     'label': 'steeltown',
+                     'hostname': 'abc123'}
+
+
     def setUp(self):
         self.god = mock.mock_god()
         self.god.stub_function(models.job, 'read_keyval')
         self.god.stub_function(version_0.job, 'find_hostname')
+        self.god.stub_function(models.test, 'parse_host_keyval')
 
 
     def tearDown(self):
         self.god.unstub_all()
 
 
-    keyval_return = {'job_queued': 1234567890,
-                     'job_started': 1234567891,
-                     'job_finished': 1234567892,
-                     'user': 'janet',
-                     'label': 'steeltown',
-                     'hostname': 'abc123'}
+    def _expect_host_keyval(self, hostname, platform=None):
+        return_dict = {}
+        if platform:
+            return_dict['platform'] = platform
+            return_dict['labels'] = platform + ',other_label'
+        (models.test.parse_host_keyval.expect_call('.', hostname)
+                .and_return(return_dict))
 
 
     def test_load_from_dir_simple(self):
         models.job.read_keyval.expect_call('.').and_return(
                 dict(self.keyval_return))
+        self._expect_host_keyval('abc123', 'my_platform')
         job = version_0.job.load_from_dir('.')
         self.assertEqual('janet', job['user'])
         self.assertEqual('steeltown', job['label'])
         self.assertEqual('abc123', job['machine'])
+        self.assertEqual('my_platform', job['machine_group'])
         self.god.check_playback()
 
 
-    def test_load_from_dir_two_machine(self):
+    def _setup_two_machines(self):
         raw_keyval = dict(self.keyval_return)
         raw_keyval['hostname'] = 'easyas,abc123'
         models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+
+
+    def test_load_from_dir_two_machines(self):
+        self._setup_two_machines()
         version_0.job.find_hostname.expect_call('.').and_raises(
-                version_0.NoHostnameError('find_hostname stubbed out'))
+                    version_0.NoHostnameError('find_hostname stubbed out'))
+        self._expect_host_keyval('easyas', 'platform')
+        self._expect_host_keyval('abc123', 'platform')
+
         job = version_0.job.load_from_dir('.')
         self.assertEqual('easyas,abc123', job['machine'])
+        self.assertEqual('platform', job['machine_group'])
 
-        models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+        self.god.check_playback()
+
+
+    def test_load_from_dir_two_machines_with_find_hostname(self):
+        self._setup_two_machines()
         version_0.job.find_hostname.expect_call('.').and_return('foo')
+        self._expect_host_keyval('foo')
+
         job = version_0.job.load_from_dir('.')
         self.assertEqual('foo', job['machine'])
 
         self.god.check_playback()
 
 
+    def test_load_from_dir_two_machines_different_platforms(self):
+        self._setup_two_machines()
+        version_0.job.find_hostname.expect_call('.').and_raises(
+                    version_0.NoHostnameError('find_hostname stubbed out'))
+        self._expect_host_keyval('easyas', 'platformZ')
+        self._expect_host_keyval('abc123', 'platformA')
+
+        job = version_0.job.load_from_dir('.')
+        self.assertEqual('easyas,abc123', job['machine'])
+        self.assertEqual('platformA,platformZ', job['machine_group'])
+
+        self.god.check_playback()
+
     def test_load_from_dir_one_machine_group_name(self):
         raw_keyval = dict(self.keyval_return)
         raw_keyval['host_group_name'] = 'jackson five'
         models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+        self._expect_host_keyval('abc123')
         job = version_0.job.load_from_dir('.')
         self.assertEqual('janet', job['user'])
         self.assertEqual('abc123', job['machine'])
@@ -70,6 +110,7 @@ class test_job_load_from_dir(unittest.TestCase):
         raw_keyval['hostname'] = 'abc123,dancingmachine'
         raw_keyval['host_group_name'] = 'jackson five'
         models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+        self._expect_host_keyval('jackson five')
         job = version_0.job.load_from_dir('.')
         self.assertEqual('michael', job['user'])
         # The host_group_name is used instead because machine appeared to be
@@ -83,6 +124,7 @@ class test_job_load_from_dir(unittest.TestCase):
         del raw_keyval['hostname']
         raw_keyval['host_group_name'] = 'jackson five'
         models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+        self._expect_host_keyval('jackson five')
         job = version_0.job.load_from_dir('.')
         # The host_group_name is used because there is no machine.
         self.assertEqual('jackson five', job['machine'])