#!/usr/bin/env python3 # This file is part of the LibreOffice project. # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Use this script to update # https://wiki.documentfoundation.org/MissingUnitTests import os import datetime import subprocess import sys import re import json import requests def splitList(lst, n): for i in range(0, len(lst), n): yield lst[i:i + n] def whiteboardNotes(whiteboard): if not whiteboard: return '' if ' ' in whiteboard: whiteboardList = reversed(whiteboard.split(' ')) for w in whiteboardList: if w.startswith("unitTestNotes"): return w.split(':')[1] elif whiteboard.startswith("unitTestNotes"): return whiteboard.split(':')[1] return '' def linesModified(commit_hash): repoPath = os.path.dirname(os.path.abspath(__file__)) + '/..' commits = subprocess.check_output( ['git', '-C', repoPath, 'show', commit_hash, '--shortstat'], stderr=subprocess.DEVNULL) linesmodified = commits.decode('utf-8', 'ignore').split('\n\n') stats = linesmodified[-1].strip("\n").split(",") insertions = 0 deletions = 0 # Make sure we have the file stats, a file stat must have # file(s) changed in it if "files changed" in linesmodified[-1] or "file changed" in linesmodified[-1]: fileschanged = re.findall(r'\b\d+\b', stats[0])[0] for element in stats: if "+" in element: insertions = re.findall(r'\b\d+\b', element)[0] elif "-" in element: deletions = re.findall(r'\b\d+\b', element)[0] return [fileschanged, insertions, deletions] else: return ["not found", "not found", "not found"] def main(ignoredBugs): results = { 'export': { 'docx': {}, 'doc': {}, 'pptx': {}, 'xlsx': {}, 'xhtml': {}, 'html': {}, }, 'writer': { 'undo': {}, 'autoformat': {}, 'autocorrect': {}, 'others': {}, }, 'calc': { 'import': {}, 'format': {}, 'others': {}, }, 'impress': { 'drawingml': {}, 'slidesorter': {}, 'others': {}, }, } hasTestSet = set() repoPath = os.path.dirname(os.path.abspath(__file__)) + '/..' branch = subprocess.check_output( ['git', '-C', repoPath, 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=subprocess.DEVNULL) last_hash = subprocess.check_output( ['git', '-C', repoPath, 'rev-parse', 'HEAD'], stderr=subprocess.DEVNULL) output = subprocess.check_output( ['git', '-C', repoPath, 'log', '--since="2012-01-01', '--name-only' ,'--pretty=format:"%s%n%ad%n%H"', '--date=format:"%Y/%m/%d"'], stderr=subprocess.DEVNULL) commits = output.decode('utf-8', 'ignore').split('\n\n') for commit in reversed(commits): commitInfo = commit.split('\n') summary = commitInfo[0].strip('"').lower() # Check for bugIds in the summary. Ignore those with a '-' after the digits. # Those are used for file names ( e.g. tdf129410-1.ods ) bugIds = re.findall("\\b(?:bug|fdo|tdf|lo)[#:]?(\\d+)(?!-)\\b", summary) if bugIds is None or len(bugIds) == 0: continue for bugId in bugIds: isIgnored = False for i in ignoredBugs: if i in summary: isIgnored = True if isIgnored: continue if bugId in hasTestSet: continue commitHash = commitInfo[2].strip('"') date = commitInfo[1].strip('"') infoList = [date, summary, commitHash] changedFiles = "".join(commitInfo[3:]) if 'qa' in changedFiles: hasTestSet.add(bugId) continue elif 'sw/source/filter/ww8/docx' in changedFiles or \ 'writerfilter/source/dmapper' in changedFiles or \ 'starmath/source/ooxmlimport' in changedFiles: results['export']['docx'][bugId] = infoList elif 'sw/source/filter/ww8/ww8' in changedFiles: results['export']['doc'][bugId] = infoList elif 'sc/source/filter/excel/xe' in changedFiles: results['export']['xlsx'][bugId] = infoList elif 'oox/source/export/' in changedFiles: results['export']['pptx'][bugId] = infoList elif 'filter/source/xslt/odf2xhtml/export' in changedFiles: results['export']['xhtml'][bugId] = infoList elif 'sw/source/filter/html/' in changedFiles: results['export']['html'][bugId] = infoList elif 'sw/source/core/undo/' in changedFiles: results['writer']['undo'][bugId] = infoList elif 'sw/source/core/edit/autofmt' in changedFiles: results['writer']['autoformat'][bugId] = infoList elif 'sw/source/core/edit/acorrect' in changedFiles: results['writer']['autocorrect'][bugId] = infoList elif 'drawingml' in changedFiles: results['impress']['drawingml'][bugId] = infoList elif 'sd/source/ui/slidesorter/' in changedFiles: results['impress']['slidesorter'][bugId] = infoList elif 'sc/source/core/tool/interpr' in changedFiles: results['calc']['import'][bugId] = infoList elif 'svl/source/numbers/' in changedFiles: results['calc']['format'][bugId] = infoList # Keep the following if statements at the end elif 'sc/source/core/' in changedFiles: results['calc']['others'][bugId] = infoList elif 'sw/source/core/' in changedFiles: results['writer']['others'][bugId] = infoList elif 'sd/source/core/' in changedFiles: results['impress']['others'][bugId] = infoList listOfBugIdsWithoutTest = [] for k,v in results.items(): for k1, v1 in v.items(): for bugId, info in v1.items(): if bugId not in hasTestSet: listOfBugIdsWithoutTest.append(bugId) bugzillaJson = [] resultList = [] fixList = [] #Split the list into different chunks for the requests, otherwise it fails for chunk in splitList(listOfBugIdsWithoutTest, 50): urlGet = 'https://bugs.documentfoundation.org/rest/bug?id=' + ','.join(chunk) rGet = requests.get(urlGet) rawData = json.loads(rGet.text) rGet.close() bugzillaJson.extend(rawData['bugs']) for k,v in results.items(): for k1, v1 in v.items(): for bugId, info in v1.items(): resolution = '' keywords = [] priority = '' notes = '' for bug in bugzillaJson: if str(bug['id']) == str(bugId): resolution = bug['resolution'] keywords = bug['keywords'] priority = bug['priority'] notes = whiteboardNotes(bug['whiteboard']) break # Only care about FIXED bugs # Ignore performance bugs and accessibility bugs if resolution and resolution == 'FIXED' and 'perf' not in keywords \ and 'accessibility' not in keywords: stats = linesModified(info[2]) fixList.append({ "id": bugId, "date": info[0], "priority": priority.upper(), "summary": info[1], "maintopic":k, "subtopic":k1, "fileschanged": stats[0], "insertions": stats[1], "deletions": stats[2], "notes": notes }) resultList.append([{ "Generator": os.path.basename(sys.argv[0]), "Date": str(datetime.datetime.now()), "Commits": str(len(commits)), "Branch": branch.decode().strip(), "Hash": str(last_hash.decode().strip()), }]) resultList.append(fixList) print(json.dumps(resultList)) def usage(): message = """usage: {program} [bugs to ignore (each one is one argument)] Sample: {program} 10000 10001 10002""" print(message.format(program = os.path.basename(sys.argv[0]))) if __name__ == '__main__': args = set() if len(sys.argv) > 1: arg1 = sys.argv[1] if arg1 == '-h' or arg1 == "--help": usage() sys.exit(1) for i in sys.argv: if i.isdigit(): args.add(i) main(sorted(args))