diff options
| -rwxr-xr-x | tko/compose_query.cgi | 106 | ||||
| -rwxr-xr-x | tko/frontend.py | 179 | ||||
| -rwxr-xr-x | tko/group_test.cgi | 61 | ||||
| -rw-r--r-- | tko/index.html | 71 | ||||
| -rwxr-xr-x | tko/machine_benchmark.cgi | 70 | ||||
| -rwxr-xr-x | tko/machine_kernel.cgi | 59 | ||||
| -rwxr-xr-x | tko/machine_kernel_test.cgi | 95 | ||||
| -rwxr-xr-x | tko/machine_kernel_test_jobs.cgi | 130 | ||||
| -rw-r--r-- | tko/query_lib.py | 99 | ||||
| -rw-r--r-- | tko/report.py | 256 |
10 files changed, 106 insertions, 1020 deletions
diff --git a/tko/compose_query.cgi b/tko/compose_query.cgi index e2862306..976dafef 100755 --- a/tko/compose_query.cgi +++ b/tko/compose_query.cgi @@ -53,19 +53,6 @@ html_header = """\ """ -# dictionary used simply for fast lookups -field_dict = { - 'kernel': 'kernel_printable', - 'hostname': 'machine_hostname', - 'test': 'test', - 'label': 'job_label', - 'machine_group': 'machine_group', - 'reason': 'reason', - 'tag': 'job_tag', - 'user': 'job_username', - 'status': 'status_word', -} - next_field = { 'machine_group': 'hostname', 'hostname': 'tag', @@ -86,7 +73,7 @@ def parse_field(form, form_field, field_default): if not form_field in form: return field_default field_input = form[form_field].value.lower() - if field_input and field_input in field_dict: + if field_input and field_input in frontend.test_view_field_dict: return field_input return field_default @@ -98,42 +85,26 @@ def parse_condition(form, form_field, field_default): form = cgi.FieldStorage() -row_field = parse_field(form, 'rows', 'kernel') -column_field = parse_field(form, 'columns', 'machine_group') +row = parse_field(form, 'rows', 'kernel') +column = parse_field(form, 'columns', 'machine_group') condition_field = parse_condition(form, 'condition', '') cgitb.enable() db = db.db() -def get_value(test, field): - if field == 'kernel': - return test.kernel_printable - if field == 'hostname': - return test.machine_hostname - if field == 'test': - return test.testname - if field == 'label': - return test.job_label - if field == 'machine_group': - return test.machine_group - if field == 'reason': - return test.reason - raise "Unknown field" - - -def construct_link(row_val, column_val): - next_row = row_field - next_column = column_field +def construct_link(x, y): + next_row = row + next_column = column condition_list = [] if condition_field != '': condition_list.append(condition_field) - if row_val: - next_row = next_field[row_field] - condition_list.append("%s='%s'" % (row_field, row_val)) - if column_val: - next_column = next_field[column_field] - condition_list.append("%s='%s'" % (column_field, column_val)) + if y: + next_row = next_field[row] + condition_list.append("%s='%s'" % (row, y)) + if x: + next_column = next_field[column] + condition_list.append("%s='%s'" % (column, x)) next_condition = '&'.join(condition_list) return 'compose_query.cgi?' + urllib.urlencode({'columns': next_column, 'rows': next_row, 'condition': next_condition}) @@ -142,69 +113,54 @@ def construct_link(row_val, column_val): def create_select_options(selected_val): ret = "" - for option in sorted(field_dict.keys()): + for option in sorted(frontend.test_view_field_dict.keys()): if selected_val == option: selected = " SELECTED" else: selected = "" - ret += '<OPTION VALUE="%s"%s>%s</OPTION>\n' % (option, - selected, - option) + ret += '<OPTION VALUE="%s"%s>%s</OPTION>\n' % \ + (option, selected, option) return ret -def smart_sort(list, field): - if field == 'kernel': - def kernel_encode(kernel): - return kernel_versions.version_encode(kernel) - list.sort(key = kernel_encode, reverse = True) - else: - list.sort() - - def gen_matrix(): where = None if condition_field.strip() != '': where = query_lib.parse_scrub_and_gen_condition( - condition_field, field_dict) + condition_field, frontend.test_view_field_dict) print "<!-- where clause: %s -->" % (where,) - ret = frontend.get_matrix_data(db, field_dict[column_field], - field_dict[row_field], where) - (data, column_list, row_list, stat_list, job_tags) = ret + test_data = frontend.get_matrix_data(db, column, row, where) - if not row_list: + if not test_data.y_values: msg = "There are no results for this query (yet?)." return [[display.box(msg)]] - smart_sort(row_list, row_field) - smart_sort(column_list, column_field) - link = 'compose_query.cgi?columns=%s&rows=%s&condition=%s' % ( - row_field, column_field, condition_field) + row, column, condition_field) header_row = [display.box("<center>(Flip Axis)</center>", link=link)] - for column in column_list: - link = construct_link(None, column) - header_row.append(display.box(column, header=True, link=link)) + for x in test_data.x_values: + link = construct_link(x, None) + header_row.append(display.box(x, header=True, link=link)) matrix = [header_row] - for row in row_list: - link = construct_link(row, None) - cur_row = [display.box(row, header=True, link=link)] - for column in column_list: + for y in test_data.y_values: + link = construct_link(None, y) + cur_row = [display.box(y, header=True, link=link)] + for x in test_data.x_values: try: - box_data = data[column][row] + box_data = test_data.data[x][y].status_count except: cur_row.append(display.box(None, None)) continue - job_tag = job_tags[column][row] + job_tag = test_data.data[x][y].job_tag if job_tag: link = '/results/%s/' % job_tag else: - link = construct_link(row, column) + link = construct_link(x, y) cur_row.append(display.status_precounted_box(db, box_data, link)) @@ -219,8 +175,8 @@ def main(): print 'Filtered Autotest Results' print '</title></head><body>' display.print_main_header() - print html_header % (create_select_options(column_field), - create_select_options(row_field), + print html_header % (create_select_options(column), + create_select_options(row), condition_field) display.print_table(gen_matrix()) print '</body></html>' diff --git a/tko/frontend.py b/tko/frontend.py index 5f431f35..85f67ec9 100755 --- a/tko/frontend.py +++ b/tko/frontend.py @@ -1,125 +1,97 @@ #!/usr/bin/python import os, re, db, sys -tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) +tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__))) +client_bin = os.path.abspath(os.path.join(tko, '../client/bin')) +sys.path.insert(0, client_bin) +import kernel_versions + root_url_file = os.path.join(tko, '.root_url') if os.path.exists(root_url_file): html_root = open(root_url_file, 'r').readline().rstrip() else: html_root = '/results/' -def select(db, field, value=None, distinct=False): - """ returns the relevant index values where the field value matches the - input value to the function. - If there is no value passed, then it returns the index values and the - field values corresponsing to them. """ - - fields = { 'kernel': ['printable', 'kernel_idx', 'kernel_idx'], - 'machine_group': ['machine_group', 'machine_idx', 'machine_idx'], - 'hostname': ['hostname', 'machine_idx', 'machine_idx'], - 'label': ['label', 'job_idx', 'job_idx'], - 'tag': ['tag', 'job_idx', 'job_idx'], - 'job': ['job_idx', 'job_idx', 'job_idx'], - 'user': ['username', 'job_idx', 'job_idx'], - 'test': ['test', 'test', 'test'], - 'status': ['word', 'status_idx', 'status'], - 'reason': ['reason', 'test_idx', 'test_idx'] } - table = { 'kernel': 'kernels', - 'machine_group': 'machines', - 'hostname': 'machines', - 'label': 'jobs', - 'tag': 'jobs', - 'job': 'jobs', - 'user': 'jobs', - 'test': 'tests', - 'status': 'status', - 'reason': 'tests' } - - lookup_field, idx_field, field_name_in_main_table = fields[field] - tablename = table[field] - # select all the index values that match the given field name. - sql = "" - if distinct: - sql += " distinct " - if not value: - sql += " %s , %s " % (lookup_field, idx_field) - where = " %s is not null " % lookup_field - else: - sql += "%s " % idx_field - if field == 'tag': - where = " %s LIKE %s " % (lookup_field, value) - else: - where = " %s = %s " % (lookup_field, value) - match = db.select(sql, tablename, where) - # returns the value and its field name - return match, field_name_in_main_table +class status_cell: + # One cell in the matrix of status data. + def __init__(self): + # Count is a dictionary: status -> count of tests with status + self.status_count = {} + self.job_tag = None + self.job_tag_count = 0 + + + def add(self, status, count, job_tags): + assert not self.status_count.has_key(status) + assert count > 0 + + self.job_tag = job_tags + self.job_tag_count += count + if self.job_tag_count > 1: + self.job_tag = None + + self.status_count[status] = count + +class status_data: + def __init__(self, sql_rows, x_field, y_field): + data = {} + y_values = set() -def get_axis_data(axis): - rows = db.select(axis , 'test_view', distinct = True) - # Need to do a magic sort here if axis == 'kernel_printable' - return sorted([row[0] for row in rows]) + # Walk through the query, filing all results by x, y info + for (x, y, status, count, job_tags) in sql_rows: + if not data.has_key(x): + data[x] = {} + if not data[x].has_key(y): + y_values.add(y) + data[x][y] = status_cell() + data[x][y].add(status, count, job_tags) + + # 2-d hash of data - [x-value][y-value] + self.data = data + # List of possible columns (x-values) + self.x_values = smart_sort(data.keys(), x_field) + # List of rows columns (y-values) + self.y_values = smart_sort(list(y_values), y_field) def get_matrix_data(db, x_axis, y_axis, where = None): - # Return a 3-d hash of data - [x-value][y-value][status_word] # Searches on the test_view table - x_axis and y_axis must both be # column names in that table. - fields = ('%s, %s, status, COUNT(status_word), ' + - 'LEFT(GROUP_CONCAT(job_tag), 100)' # limit what's returned - ) % (x_axis, y_axis) - group_by = '%s, %s, status' % (x_axis, y_axis) + x_field = test_view_field_dict[x_axis] + y_field = test_view_field_dict[y_axis] + fields = ('%s, %s, status, COUNT(status), ' + + 'LEFT(GROUP_CONCAT(job_tag), 100)' # limit what's returned + ) % (x_field, y_field) + group_by = '%s, %s, status' % (x_field, y_field) rows = db.select(fields, 'test_view', where=where, group_by=group_by) - data = {} - job_tags = {} - x_set = set() - y_set = set() - status_set = set() - for (x, y, status, count, job_tag) in rows: - if not data.has_key(x): - data[x] = {} - job_tags[x] = {} - if not data[x].has_key(y): - data[x][y] = {} - data[x][y][status] = count - if job_tags[x].has_key(y) or count != 1: - job_tags[x][y] = None - else: - job_tags[x][y] = job_tag - x_set.add(x) - y_set.add(y) - status_set.add(status) - return (data, list(x_set), list(y_set), list(status_set), job_tags) - - -class anygroup: - @classmethod - def selectunique(klass, db, field): - """Return unique values for all possible groups within - the table.""" - rows, field_name_in_main_table = select(db, field, value=None, distinct=True) - groupnames = sorted([row for row in rows]) - - # collapse duplicates where records have the same name but - # multiple index values - headers = {} - for field_name, idx_value in groupnames: - if headers.has_key(field_name): - headers[field_name].append(idx_value) - else: - headers[field_name] = [idx_value] - headers = headers.items() - headers.sort() - return [klass(db, field_name_in_main_table, groupname) for groupname in headers] - - - def __init__(self, db, idx_name, name): - self.db = db - self.name = name[0] - self.idx_name = idx_name - self.idx_value = name[1] + return status_data(rows, x_field, y_field) + + +# Dictionary used simply for fast lookups from short reference names for users +# to fieldnames in test_view +test_view_field_dict = { + 'kernel' : 'kernel_printable', + 'hostname' : 'machine_hostname', + 'test' : 'test', + 'label' : 'job_label', + 'machine_group' : 'machine_group', + 'reason' : 'reason', + 'tag' : 'job_tag', + 'user' : 'job_username', + 'status' : 'status_word', +} + +def smart_sort(list, field): + if field == 'kernel_printable': + def kernel_encode(kernel): + return kernel_versions.version_encode(kernel) + list.sort(key = kernel_encode, reverse = True) + else: + list.sort() + return list class group: @@ -225,7 +197,6 @@ class test: self.url = None - def iterations(self): """ Caching function for iterations diff --git a/tko/group_test.cgi b/tko/group_test.cgi index 82e311b6..e69de29b 100755 --- a/tko/group_test.cgi +++ b/tko/group_test.cgi @@ -1,61 +0,0 @@ -#!/usr/bin/python -print "Content-type: text/html\n" -import cgi, cgitb, os, sys, re -sys.stdout.flush() -cgitb.enable() - -tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) -sys.path.insert(0, tko) -import db, display, frontend - -db = db.db() - -def main(): - display.print_main_header() - - form = cgi.FieldStorage() - kernel_idx = form["kernel"].value - kernel = frontend.kernel.select(db, {'kernel_idx' : kernel_idx })[0] - groups = frontend.group.select(db) - - print_kernel_groups_vs_tests(kernel, groups) - - -def print_kernel_groups_vs_tests(kernel, groups): - # first we have to get a list of all run tests across all machines - all_tests = set() - present_groups = [] - for group in groups: - tests = group.tests({ 'kernel_idx' : kernel.idx }) - if tests: - present_groups.append(group) - for test in tests: - all_tests.add(test.testname) - all_tests = list(all_tests) - all_tests.sort() - - print '<h1>%s</h1>' % kernel.printable - - header_row = [ display.box('Test', header=True) ] - for group in present_groups: - group_name = display.group_name(group) - header_row.append( display.box(group_name, header=True) ) - - matrix = [header_row] - for testname in all_tests: - shortname = re.sub(r'kernel.', r'kernel<br>', testname) - row = [display.box(shortname)] - for group in present_groups: - tests = group.tests({ 'kernel_idx' : kernel.idx , - 'test' : testname }) - link = 'machine_kernel_test.cgi?' - link += 'group=%s&kernel=%s&test=%s' % \ - (group.name, kernel.idx, testname) - box = display.status_count_box(db, tests, link=link) - row.append(box) - matrix.append(row) - matrix.append(header_row) - - display.print_table(matrix) - -main() diff --git a/tko/index.html b/tko/index.html index 8afb8439..e69de29b 100644 --- a/tko/index.html +++ b/tko/index.html @@ -1,71 +0,0 @@ -<html> -<head> -</head> -<body> -<table border="0"> - <form action="compose_query.cgi" method="get"> -<tr> - <td>Column: </td> - <td> - <SELECT NAME="columns"> - <OPTION VALUE="kernel">kernel - <OPTION VALUE="hostname">hostname - <OPTION VALUE="test">test - <OPTION VALUE="label">label - <OPTION VALUE="machine_group">machine_group - <OPTION VALUE="reason">reason - <OPTION VALUE="tag">tag - <OPTION VALUE="user">user - <OPTION VALUE="status">status - </SELECT> - </td> -</tr> -<tr> - <td>Row: </td> - <td> - <SELECT NAME="rows"> - <OPTION VALUE="kernel">kernel - <OPTION VALUE="hostname">hostname - <OPTION VALUE="test" SELECTED>test - <OPTION VALUE="label">label - <OPTION VALUE="machine_group">machine_group - <OPTION VALUE="reason">reason - <OPTION VALUE="tag">tag - <OPTION VALUE="user">user - <OPTION VALUE="status">status - </SELECT> - </td> -</tr> -<tr> - <td>Condition: </td> - <td> - <input type="text" name="condition" size="80" maxlength="80"><br /> - <input type="hidden" name="title" value="Report"> - </td> -</tr> -<tr> - <td colspan="100%" align="center"><input type="submit" value="Submit"> - </td> -</tr> -<tr> - <td colspan="100%"> -<p> -Conditions of the form <column>=<value> & <column>=<value> & ... & <column>=<value> -</p> -<p> -Textual values must be quoted. -</p> -<p> -<div>Examples:</div> -<ul> -<li>user='johnmacdonald' & test='burnin'</li> -<li>hostname='bdpk1' & user='yinghan'</li> -<li>tag~'134-jorlow%' to search for job '134-jorlow'</li> -</ul> -</p> - </td> -</tr> -</form> -</table> -</body> -</html> diff --git a/tko/machine_benchmark.cgi b/tko/machine_benchmark.cgi index d0b1aef0..e69de29b 100755 --- a/tko/machine_benchmark.cgi +++ b/tko/machine_benchmark.cgi @@ -1,70 +0,0 @@ -#!/usr/bin/python -print "Content-type: text/html\n" -import cgi, cgitb, os, sys, re -sys.stdout.flush() -cgitb.enable() - -tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) -sys.path.insert(0, tko) -import db, display, frontend - -db = db.db() - -benchmark_key = { -'kernbench' : 'elapsed', -'dbench' : 'throughput', -'tbench' : 'throughput', -} - -def main(): - - display.print_main_header() - - rows = db.select('test', 'tests', {}, distinct = True) - benchmarks = [] - for row in rows: - benchmark = row[0] - testname = re.sub(r'\..*', '', benchmark) - if not benchmark_key.has_key(testname): - continue - benchmarks.append(benchmark) - benchmarks = display.sort_tests(benchmarks) - - machine_idx = {} - benchmark_data = {} - for benchmark in benchmarks: - fields = 'machine_idx,machine_hostname,count(status_word)' - where = { 'subdir': benchmark, 'status_word' : 'GOOD' } - data = {} - for (idx, machine, count) in db.select(fields, 'test_view', - where, group_by = 'machine_hostname'): - data[machine] = count - machine_idx[machine] = idx - benchmark_data[benchmark] = data - - print '<h1>Performance</h1>' - - header_row = [ display.box('Benchmark', header=True) ] - header_row += [ display.box(re.sub(r'\.', '<br>', benchmark), header=True) for benchmark in benchmarks ] - - matrix = [header_row] - for machine in machine_idx: - row = [display.box(machine)] - for benchmark in benchmarks: - count = benchmark_data[benchmark].get(machine, None) - if not count: - row.append(display.box(None)) - continue - key = benchmark_key[re.sub(r'\..*', '', benchmark)] - url = 'machine_test_attribute_graph.cgi' - url += '?machine=' + str(machine_idx[machine]) - url += '&benchmark=' + benchmark - url += '&key=' + key - html = '<a href="%s">%d</a>' % (url, count) - row.append(display.box(html)) - matrix.append(row) - matrix.append(header_row) - - display.print_table(matrix) - -main() diff --git a/tko/machine_kernel.cgi b/tko/machine_kernel.cgi index dbd52bb3..e69de29b 100755 --- a/tko/machine_kernel.cgi +++ b/tko/machine_kernel.cgi @@ -1,59 +0,0 @@ -#!/usr/bin/python -print "Content-type: text/html\n" -import cgi, cgitb, os, sys, re -sys.stdout.flush() -cgitb.enable() - -tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) -sys.path.insert(0, tko) -import db, frontend, display -client_bin = os.path.abspath(os.path.join(tko, '../client/bin')) -sys.path.insert(0, client_bin) -import kernel_versions - -db = db.db() - - -def kernel_group_box(kernel, group, box_data): - machine_idxs = ['%d' % machine.idx for machine in group.machines()] - link = 'machine_kernel_test.cgi?machine=%s&kernel=%s' % \ - (','.join(machine_idxs), kernel.idx) - return display.status_precounted_box(db, box_data, link) - - -def kernel_encode(kernel): - return kernel_versions.version_encode(kernel.printable) - - -def main(): - display.print_main_header() - - ret = frontend.get_matrix_data(db, 'machine_group', 'kernel_printable') - (data, group_list, kernel_list, status_list, job_tags) = ret - - groups = frontend.group.select(db) - group_names = [display.group_name(g) for g in groups] - headers = ['Version'] + group_names - header_row = [ display.box(x, header=True) for x in headers ] - - kernels = frontend.kernel.select(db) - kernels.sort(key = kernel_encode, reverse = True) - - matrix = [header_row] - for kernel in kernels: - link = 'group_test.cgi?kernel=%s' % kernel.idx - row = [display.box(kernel.printable, link=link)] - for group in groups: - try: - box_data = data[group.name][kernel.printable] - except: - row.append(display.box(None, None)) - continue - row.append(kernel_group_box(kernel, group, box_data)) - matrix.append(row) - matrix.append(header_row) - - display.print_table(matrix) - - -main() diff --git a/tko/machine_kernel_test.cgi b/tko/machine_kernel_test.cgi index 25f6a38e..e69de29b 100755 --- a/tko/machine_kernel_test.cgi +++ b/tko/machine_kernel_test.cgi @@ -1,95 +0,0 @@ -#!/usr/bin/python -print "Content-type: text/html\n" -import cgi, cgitb, os, sys, re -sys.stdout.flush() -cgitb.enable() - -tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) -sys.path.insert(0, tko) -import db, display, frontend - -db = db.db() - -def main(): - display.print_main_header() - - form = cgi.FieldStorage() - - kernel_idx = form['kernel'].value - - if form.has_key('machine'): - mlist = form['machine'].value - machines = [] - for idx in mlist.split(','): - where = { 'machine_idx' : idx } - machines.append(frontend.machine.select(db, where)[0]) - elif form.has_key('group'): - where = { 'machine_group' : form['group'].value } - machines = frontend.machine.select(db, where) - mlist = ','.join(['%s'%machine.idx for machine in machines]) - if form.has_key('test'): - test = form['test'].value - else: - test = None - - kernel = frontend.kernel.select(db, {'kernel_idx' : kernel_idx })[0] - print_kernel_machines_vs_test(machines, kernel, test, mlist) - - -def print_kernel_machines_vs_test(machines, kernel, only_test, mlist): - # first we have to get a list of all run tests across all machines - all_tests = [] - results = {} # will be a 2d hash [machine][testname] - for machine in machines: - where = { 'kernel_idx':kernel.idx , 'machine_idx':machine.idx } - if only_test: - where['subdir'] = only_test - tests = frontend.test.select(db, where) - if not tests: - continue - dict = {} - for test in tests: - testname = test.testname - all_tests.append(testname) - dict[testname] = dict.get(testname, []) + [test] - results[machine.idx] = dict - test_list = display.sort_tests(list(set(all_tests))) - - print '<h1>%s</h1>' % kernel.printable - - header_row = [ display.box('Version', header=True) ] - for test in [re.sub(r'kernel.', r'kernel<br>', x) for x in test_list]: - url='machine_kernel_test_jobs.cgi?machine=%s&kernel=%s&test=%s'\ - % (mlist, kernel.idx, test) - header_row.append( display.box(test, link=url, header=True) ) - - matrix = [header_row] - for machine in machines: - if not results.has_key(machine.idx): - continue - if machine.owner: - hostname = machine.owner + ' ' + machine.hostname - else: - hostname = machine.hostname - url = 'machine_kernel_test_jobs.cgi?machine=%s&kernel=%s' % \ - (machine.idx, kernel.idx) - row = [display.box(hostname, link = url)] - for testname in test_list: - if results[machine.idx].has_key(testname): - tests = results[machine.idx][testname] - if len(tests) == 1: - link = tests[0].url - else: - link = 'machine_kernel_test_jobs.cgi?machine=%s&kernel=%s&test=%s' % (machine.idx, kernel.idx, testname) - else: - tests = [] - link = None - - box = display.status_count_box(db, tests, link = link) - row.append(box) - matrix.append(row) - matrix.append(header_row) - - display.print_table(matrix) - -main() diff --git a/tko/machine_kernel_test_jobs.cgi b/tko/machine_kernel_test_jobs.cgi index a8a16252..e69de29b 100755 --- a/tko/machine_kernel_test_jobs.cgi +++ b/tko/machine_kernel_test_jobs.cgi @@ -1,130 +0,0 @@ -#!/usr/bin/python -print "Content-type: text/html\n" -import cgi, cgitb, os, sys, re -sys.stdout.flush() -cgitb.enable() - -tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) -sys.path.insert(0, tko) -import db, display, frontend - -db = db.db() - -def main(): - display.print_main_header() - form = cgi.FieldStorage() - - kernel_idx = form['kernel'].value - kernel = frontend.kernel.select(db, {'kernel_idx' : kernel_idx })[0] - - if form.has_key('machine'): - machines = [] - for idx in form['machine'].value.split(','): - where = { 'machine_idx' : idx } - machines.append(frontend.machine.select(db, where)[0]) - elif form.has_key('group'): - where = { 'machine_group' : form['group'].value } - machines = frontend.machine.select(db, where) - - if form.has_key('test'): - test = form['test'].value - else: - test = None - - if len(machines) == 1: - print_jobs_vs_tests(machines[0], kernel, test) - else: - print_jobs_vs_machines(machines, kernel, test) - - -def print_jobs_vs_tests(machine, kernel, only_test): - # first we have to get a list of all run tests - results = {} # will be a 2d hash on [testname][jobname] - all_tests = set([]) - all_jobs = set([]) - where = { 'kernel_idx':kernel.idx , 'machine_idx':machine.idx } - if only_test: - where['subdir'] = only_test - tests = frontend.test.select(db, where) - for test in tests: - all_tests.add(test.testname) - all_jobs.add(test.job.tag) - if not results.has_key(test.testname): - results[test.testname] = {} - results[test.testname][test.job.tag] = test - test_list = sorted(list(all_tests)) - job_list = sorted(list(all_jobs)) - - print '<h1>Kernel: %s</h1>\n' % kernel.printable - print '<h1>Machine: %s</h1>\n' % machine.hostname - - header_row = [ display.box('Job', header=True) ] - for jobname in job_list: - header_row.append( display.box(jobname, header=True) ) - - matrix = [header_row] - for testname in test_list: - if not results.has_key(testname): - continue - row = [display.box(testname)] - for jobname in job_list: - test = results[testname].get(jobname, None) - if test: - box = display.box(test.status_word, - color_key = test.status_word, - link = test.url) - else: - box = display.box(None) - row.append(box) - matrix.append(row) - matrix.append(header_row) - - display.print_table(matrix) - - -def print_jobs_vs_machines(machines, kernel, only_test): - if not only_test: - raise "No test specified!" - results = {} # will be a 2d hash [machine][jobname] - all_jobs = set([]) - for machine in machines: - where = { 'kernel_idx':kernel.idx , 'machine_idx':machine.idx, - 'subdir':only_test } - tests = frontend.test.select(db, where) - if tests: - results[machine] = {} - for test in tests: - results[machine][test.job.tag] = test - all_jobs.add(test.job.tag) - results[machine.idx] = tests - job_list = sorted(list(all_jobs)) - - print '<h1>Kernel: %s</h1>\n' % kernel.printable - print '<h1>Test: %s</h1>\n' % only_test - - header_row = [ display.box('Machine', header=True) ] - for jobname in job_list: - header_row.append( display.box(jobname, header=True) ) - - matrix = [header_row] - for machine in machines: - if not results.has_key(machine): - continue - tests = results[machine] - row = [display.box(machine.hostname)] - for jobname in job_list: - test = results[machine].get(jobname, None) - if test: - box = display.box(test.status_word, - color_key = test.status_word, - link = test.url) - else: - box = display.box(None) - row.append(box) - matrix.append(row) - matrix.append(header_row) - - display.print_table(matrix) - - -main() diff --git a/tko/query_lib.py b/tko/query_lib.py index e20a6671..9f492b9d 100644 --- a/tko/query_lib.py +++ b/tko/query_lib.py @@ -88,102 +88,3 @@ def parse_scrub_and_gen_condition(condition, valid_field_dict): raise "Could not parse '%s' (%s)" % (condition, regex) - -### -### Everything past here is depricated. -### - -def generate_sql_condition(condition_list): - """ generate the sql for the condition list.""" - sql = '' - value = [] - for field, operator, values in condition_list: - if len(values) == 1: - if sql != '': - sql += " and " - sql += " %s%s%%s" % (field, operator) - value.append(values[0][0]) - elif len(values) > 1: - expression = [" %s %s %%s" % (field, operator) for val in values] - for val in values: - value.append(val[0]) - if sql != '': - sql += " and " - sql += "(%s)" % " or ".join(expression) - return sql, value - - -def prune_list(thelist, condition_sql, condition_value): - """ keep track of which columns do not have any elements.""" - pruned_list = [] - for g in thelist: - # check for multiple index values in the db. - sql = "t where " - expr = [" %s = %%s" % (g.idx_name) for val in g.idx_value] - sql += " (%s) " % " or ".join(expr) - value = [] - value.extend(g.idx_value) - if condition_sql: - sql += " and " - sql += condition_sql - value.extend(condition_value) - tests = frontend.test.select_sql(db, sql, value) - if len(tests) > 0: - pruned_list.append(g) - return pruned_list - - -def parse_condition(condition): - """ parse the condition into independent clauses.""" - condition_list = [] - if not condition: - return condition_list - attribute_re = r"(\w+)" - op_re = r"(=|!=)" - value_re = r"('[^']*')" - # condition is clause & clause & .. - clause_re = r"%s\s*%s\s*%s" % (attribute_re, op_re, value_re) - condition_re = re.compile(r"^\s*%s(\s*&\s*%s)*\s*$" % (clause_re, clause_re)) - if not condition_re.match(condition): - print "Condition not in the correct format: %s" % condition - sys.exit(0) - triples = [] - for clause in [c.strip() for c in condition.split('&')]: - attribute, op, value = re.match(clause_re, clause).groups() - triples.append((attribute, op, value)) - for (field_name, operator, value) in triples: - match, field = frontend.select(db, field_name, value, distinct=True) - if len(match) > 0: - condition_list.append((field, operator, match)) - else: - print "No matching results found for condition %s." % \ - condition - sys.exit(0) - return condition_list - - -def get_value(test, field): - """ get specific field values from the given test object.""" - if field == 'test': - return test.testname - elif field == 'kernel_idx': - return test.kernel_idx - elif field == 'machine_idx': - return test.machine_idx - elif field == 'status': - return test.status_num - - -def get_tests(condition_sql, condition_value): - # get all the tests that satify the given condition. - if condition_sql: - sql = "t where " - sql += condition_sql - value = [str(val) for val in condition_value] - #print sql , value - tests = frontend.test.select_sql(db, sql, value) - else: - sql = None - value = None - tests = frontend.test.select_sql(db, " t ", None) - return tests diff --git a/tko/report.py b/tko/report.py index bd4921f2..e69de29b 100644 --- a/tko/report.py +++ b/tko/report.py @@ -1,256 +0,0 @@ -#!/usr/bin/python - -""" -CLI support to enable user to query the database. -""" - -import sys, os, getopt - -tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) -sys.path.insert(0, tko) - -import query_lib, db, frontend - -db = db.db() - -help_msg_header = """ -NAME -report.py - Print the results matching a given condition in the specified format. - -SYNOPSIS -report.py [options] - -OPTIONS -""" - -help_msg_trailer = """ -EXAMPLES -To see every job that has ever been run: - report.py - -To see all the jobs started by johnmacdonald: - report.py --condition="user='johnmacdonald'" - -To see all the jobs started by johnmandonald and on hostname arh22: - report.py --condition="user='johnmacdonald' & hostname='arh22'" - -To see only the test, hostname and user for the reports: - report.py --columns="test, hostname, user" - -You can use both the columns and condition options to generate the kind of report you want. -""" - -condition_desc = """Condition to filter the results with. - Supported fields are: test, hostname, user, label, machine_group, status, reason, kernel. - Supported operators are =, != and string values must be quoted within single quotes. -""" - -columns_desc = """Specific columns to display in the results. - Supported fields are: test, hostname, user, label, machine_group, status, reason, kernel. -""" - -help_desc = """Print command help. -""" - - -class CliError(Exception): - pass - - -class InvalidArgsError(CliError): - def __init__(self, error): - CliError.__init__(self, 'Unknown arguments: %r\nTry report.py --help' % error) - - -class InvalidColumnValue(CliError): - def __init__(self, error): - CliError.__init__(self, 'Unrecognized column value: %r\nTry report.py --help' % error) - - -class cli: - def __init__(self): - self.__options = {} - - - def add_option(self, name=None, short_name=None, type=None, - description=None, value=None): - """ Adds the options to the cli. - """ - if not name and not short_name: - raise Error("No name provided for the option.") - - short = False - - if not name and short_name: - short = True - name = short_name - - self.__options[name] = dict(name=name, type=type, - description=description, - value=value, short=short) - - - def list_options(self): - """ Return the options for this cli. - """ - return self.__options - - - def parse_options(self, args): - """ Parse the options and the values the cli is invoked with. - """ - short_opts = "" - long_opts = [] - for name,val in self.__options.items(): - if val['short']: - short_opts += val['name'] - if val['type'] != 'bool': - short_opts += ':' - else: - opt = val['name'] - if val['type'] != 'bool': - opt += '=' - long_opts.append(opt) - - opts, args = getopt.getopt(args[1:], short_opts, long_opts) - return opts, args - - - def usage(self): - """ Help for the cli. - """ - msg = help_msg_header - for opt,value in self.__options.items(): - if value['short']: - msg += '-' - else: - msg += '--' - msg += '%s \t: %s\n' % (value['name'], value['description']) - - msg += help_msg_trailer - return msg - - -def pretty_print(header, results): - """ pretty prints the result with all the proper space indentations. - """ - # add an extra column for the record number. - header.insert(0, ' # ') - - # add the record number to each result. - for j in xrange(len(results)): - results[j].insert(0, "[%d]" % (j+1)) - - # number of columns in the results table. - size = len(header) - - # list containing the max width of each column. - column_width = [len(col_name) for col_name in header] - - # update the column width based on the values in the table. - for record in results: - for i in xrange(size): - column_width[i] = max(column_width[i], len(record[i])) - - # Generates the header. - lines = [] - lines.append(' '.join([header[i].capitalize().ljust(column_width[i]) - for i in xrange(size)])) - lines.append(' '.join(['-' * c_size for c_size in column_width])) - - # Generates the table with the appropriate space indent. - for record in results: - lines.append(' '.join([record[i].ljust(column_width[i]) - for i in xrange(size)])) - - return '\n'.join(lines) - - -def main(args): - cli_obj = cli() - - # Add all the known and acceptable options. - cli_obj.add_option(name='condition', type='string', - description=condition_desc) - cli_obj.add_option(name='columns', type='string', - description=columns_desc) - cli_obj.add_option(name='help', type='bool', - description=help_desc) - - # Parse the options. - opts,args = cli_obj.parse_options(args) - - # unexpected argument. - if args: - raise InvalidArgsError(args) - - sql = None - value = None - - # by default display these columns - requested_columns = ['test', 'hostname', 'status', 'reason'] - - for option, value in opts: - if option == '--help': - print cli_obj.usage() - return - elif option == '--condition': - condition_list = query_lib.parse_condition(value.strip('"')) - sql, value = query_lib.generate_sql_condition(condition_list) - elif option == '--columns': - supported_columns = ['test', 'hostname', 'user', 'label', - 'machine_group', 'status', 'reason', 'kernel'] - requested_columns = [x.strip() for x in value.split(',')] - for col in requested_columns: - if col not in supported_columns: - raise InvalidColumnValue, 'Unknown field %s specified in the columns option' % col - - # get the values corresponding to the index fields. - col_values = {} - for col in requested_columns: - if col != 'test' and col != 'status' and col != 'reason': - # the rest of the columns need the index values. - col_group = frontend.anygroup.selectunique(db, col) - col_value, field_name = frontend.select(db, col) - col_values[col] = list(col_value) - - # get all the tests that satisfy the given conditions. - tests = query_lib.get_tests(sql, value) - - # accumulate the fields that are of interest to the user. - result = [] - - for test in tests: - record = [] - - test_values = {} - test_values['hostname'] = test.machine_idx - test_values['user'] = test.job.job_idx - test_values['label'] = test.job.job_idx - test_values['machine_group'] = test.machine_idx - test_values['kernel'] = test.kernel_idx - - for col in requested_columns: - if col == 'test': - record.append(test.testname) - elif col == 'status': - record.append(test.status_word) - elif col == 'reason': - record.append(test.reason.strip()) - else: - column = col_values[col] - found = False - for idx_name, idx_value in column: - if idx_value == test_values[col]: - record.append(idx_name) - found = True - break - if not found: - record.append('') - result.append(record) - - # generate the pretty table. - print pretty_print(requested_columns, result) - - -main(sys.argv) |