#!/usr/bin/env python3 # # This file is part of the LibreOffice project. # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. # import common import math from datetime import datetime, timedelta import matplotlib import matplotlib.pyplot as plt lKeywords = ['havebacktrace', 'regression', 'bisected'] oldBugsYears = 4 def util_create_basic_schema(): return { 'id': [], 'author': {}, 'day': {}, 'difftime': [] } def util_create_statList(): return { 'created': util_create_basic_schema(), 'confirmed': util_create_basic_schema(), 'verified': util_create_basic_schema(), 'wfm': util_create_basic_schema(), 'duplicate': util_create_basic_schema(), 'fixed': util_create_basic_schema(), 'resolvedStatuses' : {}, 'criticalFixed': {}, 'highFixed': {}, 'crashFixed': {}, 'perfFixed': {}, 'oldBugsFixed': {}, 'metabug': util_create_basic_schema(), 'keywords': { k : util_create_basic_schema() for k in lKeywords}, 'people' : {}, 'unconfirmedCount' : {}, 'regressionCount' : {}, 'bibisectRequestCount' : {}, 'highestCount' : {}, 'highCount' : {}, 'stat': {'oldest': datetime.now(), 'newest': datetime(2001, 1, 1)} } def util_increase_action(value, rowId, creatorMail, day, difftime=-1): value['id'].append(rowId) if creatorMail not in value['author']: value['author'][creatorMail] = 0 value['author'][creatorMail] += 1 if day not in value['day']: value['day'][day] = 0 value['day'][day] += 1 if difftime >= 0: value['difftime'].append(difftime) def util_decrease_action(value, creatorMail, day): value['id'].pop() value['author'][creatorMail] -= 1 value['day'][day] -= 1 if value['difftime']: value['difftime'].pop() def daterange(cfg): for n in range(int ((cfg.Date[1] - cfg.Date[0]).days)): yield cfg.Date[0] + timedelta(n) def analyze_bugzilla_data(statList, bugzillaData, cfg): print("Analyzing bugzilla\n", end="", flush=True) unconfirmedCountPerDay = {} regressionsCountPerDay = {} bibisectRequestCountPerDay = {} highestCountPerDay = {} highCountPerDay = {} fixedBugs = {} for key, row in bugzillaData['bugs'].items(): rowId = row['id'] #Ignore META bugs and deletionrequest bugs. if not row['summary'].lower().startswith('[meta]') and row['component'].lower() != 'deletionrequest': creationDate = datetime.strptime(row['creation_time'], "%Y-%m-%dT%H:%M:%SZ") #Some old bugs were directly created as NEW, skipping the UNCONFIRMED status #Use the oldest bug ID in the unconfirmed list if rowId >= 89589: actionDay = creationDate.strftime("%Y-%m-%d") if actionDay not in unconfirmedCountPerDay: unconfirmedCountPerDay[actionDay] = 0 unconfirmedCountPerDay[actionDay] += 1 rowStatus = row['status'] rowResolution = row['resolution'] rowKeywords = row['keywords'] creatorMail = row['creator'] #get information about created bugs in the period of time if common.util_check_range_time(creationDate, cfg): creationDay = str(creationDate.strftime("%Y-%m-%d")) util_increase_action(statList['created'], rowId, creatorMail, creationDay) if row['severity'] == 'enhancement': if 'enhancement' not in statList['created']: statList['created']['enhancement'] = 0 statList['created']['enhancement'] += 1 common.util_check_bugzilla_mail( statList, creatorMail, row['creator_detail']['real_name'], creationDate, rowId) isFixed = False isWFM = False isDuplicate = False isResolved = False isConfirmed = False isVerified = False dayConfirmed = None dayVerified = None dayWFM = None dayDuplicate = None authorConfirmed = None authorVerified = None authorWFM = None authorDuplicate = None isRegression = False isRegressionClosed = False isBibisectRequest = False isBibisectRequestClosed = False isHighest = False isHighestClosed = False isHigh = False isHighClosed = False isThisBugClosed = False for action in row['history']: actionMail = action['who'] actionDate = datetime.strptime(action['when'], "%Y-%m-%dT%H:%M:%SZ") common.util_check_bugzilla_mail( statList, actionMail, '', actionDate, rowId) actionDay = str(actionDate.strftime("%Y-%m-%d")) diffTime = (actionDate - creationDate).days for change in action['changes']: if change['field_name'] == 'priority': addedPriority = change['added'] removedPriority = change['removed'] # Sometimes the priority is increased to highest after the bug is fixed # Ignore those cases if not isThisBugClosed and not isHighestClosed: if not isHighest and addedPriority == "highest": if actionDay not in highestCountPerDay: highestCountPerDay[actionDay] = 0 highestCountPerDay[actionDay] += 1 isHighest = True if isHighest and removedPriority == "highest": if actionDay not in highestCountPerDay: highestCountPerDay[actionDay] = 0 highestCountPerDay[actionDay] -= 1 isHighest = False # TODO: IsThisBugClosed should be check here, but the result is not accurate if not isHighClosed: if not isHigh and addedPriority == "high": if actionDay not in highCountPerDay: highCountPerDay[actionDay] = 0 highCountPerDay[actionDay] += 1 isHigh = True if isHigh and removedPriority == "high": if actionDay not in highCountPerDay: highCountPerDay[actionDay] = 0 highCountPerDay[actionDay] -= 1 isHigh = False if change['field_name'] == 'status': addedStatus = change['added'] removedStatus = change['removed'] if common.isOpen(addedStatus): isThisBugClosed = False else: isThisBugClosed = True #See above if rowId >= 89589: if removedStatus == "UNCONFIRMED": if actionDay not in unconfirmedCountPerDay: unconfirmedCountPerDay[actionDay] = 0 unconfirmedCountPerDay[actionDay] -= 1 elif addedStatus == 'UNCONFIRMED': if actionDay not in unconfirmedCountPerDay: unconfirmedCountPerDay[actionDay] = 0 unconfirmedCountPerDay[actionDay] += 1 if isRegression: # the regression is being reopened if isRegressionClosed and not isThisBugClosed: if actionDay not in regressionsCountPerDay: regressionsCountPerDay[actionDay] = 0 regressionsCountPerDay[actionDay] += 1 isRegressionClosed = False # the regression is being closed if not isRegressionClosed and isThisBugClosed: if actionDay not in regressionsCountPerDay: regressionsCountPerDay[actionDay] = 0 regressionsCountPerDay[actionDay] -= 1 isRegressionClosed = True if isBibisectRequest: # the bibisectRequest is being reopened if isBibisectRequestClosed and not isThisBugClosed: if actionDay not in bibisectRequestCountPerDay: bibisectRequestCountPerDay[actionDay] = 0 bibisectRequestCountPerDay[actionDay] += 1 isBibisectRequestClosed = False # the bibisectRequest is being closed if not isBibisectRequestClosed and isThisBugClosed: if actionDay not in bibisectRequestCountPerDay: bibisectRequestCountPerDay[actionDay] = 0 bibisectRequestCountPerDay[actionDay] -= 1 isBibisectRequestClosed = True if isHighest: # the Highest priority bug is being reopened if isHighestClosed and not isThisBugClosed: if actionDay not in highestCountPerDay: highestCountPerDay[actionDay] = 0 highestCountPerDay[actionDay] += 1 isHighestClosed = False # the Highest priority bug is being closed if not isHighestClosed and isThisBugClosed: if actionDay not in highestCountPerDay: highestCountPerDay[actionDay] = 0 highestCountPerDay[actionDay] -= 1 isHighestClosed = True if isHigh: # the High priority bug is being reopened if isHighClosed and not isThisBugClosed: if actionDay not in highCountPerDay: highCountPerDay[actionDay] = 0 highCountPerDay[actionDay] += 1 isHighClosed = False # the High priority bug is being closed if not isHighClosed and isThisBugClosed: if actionDay not in highCountPerDay: highCountPerDay[actionDay] = 0 highCountPerDay[actionDay] -= 1 isHighClosed = True if common.util_check_range_time(actionDate, cfg): if removedStatus == "UNCONFIRMED": util_increase_action(statList['confirmed'], rowId, actionMail, actionDay, diffTime) dayConfirmed = actionDay authorConfirmed = actionMail isConfirmed = True elif addedStatus == 'UNCONFIRMED' and isConfirmed: util_decrease_action(statList['confirmed'], authorConfirmed, dayConfirmed) isConfirmed = False if addedStatus == 'VERIFIED': util_increase_action(statList['verified'], rowId, actionMail, actionDay, diffTime) dayVerified = actionDay authorVerified = actionMail isVerified = True elif removedStatus == 'VERIFIED' and isVerified and common.isOpen(addedStatus): util_decrease_action(statList['verified'], authorVerified, dayVerified) isVerified = False elif change['field_name'] == 'resolution': if common.util_check_range_time(actionDate, cfg): addedResolution = change['added'] removedResolution = change['removed'] if isResolved and removedResolution: statList['resolvedStatuses'][removedResolution] -= 1 isResolved = False if addedResolution: if addedResolution not in statList['resolvedStatuses']: statList['resolvedStatuses'][addedResolution] = 0 statList['resolvedStatuses'][addedResolution] += 1 isResolved = True if addedResolution == 'FIXED': fixedBugs[rowId] = actionDate isFixed = True elif removedResolution == 'FIXED' and isFixed: del fixedBugs[rowId] isFixed = False if addedResolution == 'WORKSFORME': isWFM = True dayWFM = actionDay authorWFM = actionMail util_increase_action(statList['wfm'], rowId, actionMail, actionDay, diffTime) elif removedResolution == 'WORKSFORME' and isWFM: util_decrease_action(statList['wfm'], authorWFM, dayWFM) isWFM = False if addedResolution == 'DUPLICATE': isDuplicate = True dayDuplicate = actionDay authorDuplicate = actionMail util_increase_action(statList['duplicate'], rowId, actionMail, actionDay, diffTime) elif removedResolution == 'DUPLICATE' and isDuplicate: util_decrease_action(statList['duplicate'], authorDuplicate, dayDuplicate) isDuplicate = False elif change['field_name'] == 'keywords': keywordsAdded = change['added'].lower().split(", ") keywordsRemoved = change['removed'].lower().split(", ") if common.util_check_range_time(actionDate, cfg): for keyword in keywordsAdded: if keyword in lKeywords: util_increase_action(statList['keywords'][keyword], rowId, actionMail, actionDay, diffTime) # TODO: IsThisBugClosed should be check here, but the result is not accurate if not isRegressionClosed: if not isRegression and 'regression' in keywordsAdded: if actionDay not in regressionsCountPerDay: regressionsCountPerDay[actionDay] = 0 regressionsCountPerDay[actionDay] += 1 isRegression = True if isRegression and 'regression' in keywordsRemoved: if actionDay not in regressionsCountPerDay: regressionsCountPerDay[actionDay] = 0 regressionsCountPerDay[actionDay] -= 1 isRegression = False # In the past, 'bibisectRequest' was added after the bug got fixed # to find the commit fixing it. Ignore them if not isThisBugClosed and not isBibisectRequestClosed: if not isBibisectRequest and 'bibisectrequest' in keywordsAdded: if actionDay not in bibisectRequestCountPerDay: bibisectRequestCountPerDay[actionDay] = 0 bibisectRequestCountPerDay[actionDay] += 1 isBibisectRequest = True if isBibisectRequest and 'bibisectrequest' in keywordsRemoved: if actionDay not in bibisectRequestCountPerDay: bibisectRequestCountPerDay[actionDay] = 0 bibisectRequestCountPerDay[actionDay] -= 1 isBibisectRequest = False elif change['field_name'] == 'blocks': if common.util_check_range_time(actionDate, cfg): if change['added']: for metabug in change['added'].split(', '): if int(metabug) in row['blocks']: util_increase_action(statList['metabug'], rowId, actionMail, actionDay, diffTime) commentMail = None comments = row['comments'][1:] bugFixers = [] commitNoticiation = False for idx, comment in enumerate(comments): commentMail = comment['creator'] commentDate = datetime.strptime(comment['time'], "%Y-%m-%dT%H:%M:%SZ") common.util_check_bugzilla_mail( statList, commentMail, '', commentDate, rowId) if common.util_check_range_time(commentDate, cfg) and rowId in fixedBugs: if commentMail == "libreoffice-commits@lists.freedesktop.org": commentText = comment['text'] author = commentText.split(' committed a patch related')[0] if author not in bugFixers and 'uitest' not in commentText.lower() and\ 'unittest' not in commentText.lower(): bugFixers.append(author) diffTime = (commentDate - creationDate).days commentDay = commentDate.strftime("%Y-%m-%d") util_increase_action(statList['fixed'], rowId, author, commentDay, diffTime) commitNoticiation = True if row['priority'] == "highest": statList['criticalFixed'][rowId]= {'summary': row['summary'], 'author': author} if row['priority'] == "high": statList['highFixed'][rowId]= {'summary': row['summary'], 'author': author} if 'crash' in row['summary'].lower(): statList['crashFixed'][rowId]= {'summary': row['summary'], 'author': author} if 'perf' in row['keywords']: statList['perfFixed'][rowId]= {'summary': row['summary'], 'author': author} if creationDate < common.util_convert_days_to_datetime(oldBugsYears * 365): statList['oldBugsFixed'][rowId]= {'summary': row['summary'], 'author': author} if rowId in fixedBugs and not commitNoticiation: actionDate = fixedBugs[rowId] actionDay = actionDate.strftime("%Y-%m-%d") diffTime = (actionDate - creationDate).days util_increase_action(statList['fixed'], rowId, 'UNKNOWN', actionDay, diffTime) for person in row['cc_detail']: email = person['email'] if commentMail == email or actionMail == email: common.util_check_bugzilla_mail(statList, email, person['real_name']) for k, v in statList['people'].items(): if not statList['people'][k]['name']: statList['people'][k]['name'] = statList['people'][k]['email'].split('@')[0] statList['people'][k]['oldest'] = statList['people'][k]['oldest'].strftime("%Y-%m-%d") statList['people'][k]['newest'] = statList['people'][k]['newest'].strftime("%Y-%m-%d") for single_date in daterange(cfg): single_day = single_date.strftime("%Y-%m-%d") #Fill empty days to be displayed on the charts for k0, v0 in statList.items(): if k0 == 'keywords': for k1, v1 in statList['keywords'].items(): if single_day not in statList['keywords'][k1]['day']: statList['keywords'][k1]['day'][single_day] = 0 else: if 'day' in statList[k0]: if single_day not in statList[k0]['day']: statList[k0]['day'][single_day] = 0 totalCount1 = 0 for k, v in unconfirmedCountPerDay.items(): xDay = datetime.strptime( k, "%Y-%m-%d") if xDay < single_date: totalCount1 += v statList['unconfirmedCount'][single_day] = totalCount1 totalCount2 = 0 for k, v in regressionsCountPerDay.items(): xDay = datetime.strptime( k, "%Y-%m-%d") if xDay < single_date: totalCount2 += v statList['regressionCount'][single_day] = totalCount2 totalCount3 = 0 for k, v in highestCountPerDay.items(): xDay = datetime.strptime( k, "%Y-%m-%d") if xDay < single_date: totalCount3 += v statList['highestCount'][single_day] = totalCount3 totalCount4 = 0 for k, v in highCountPerDay.items(): xDay = datetime.strptime( k, "%Y-%m-%d") if xDay < single_date: totalCount4 += v statList['highCount'][single_day] = totalCount4 totalCount5 = 0 for k, v in bibisectRequestCountPerDay.items(): xDay = datetime.strptime( k, "%Y-%m-%d") if xDay < single_date: totalCount5 += v statList['bibisectRequestCount'][single_day] = totalCount5 def makeStrong(text): return "" + str(text) + "" def makeLI(text): return "