tools: New tool to view and compare xunit resultsanalyze-tool

1 files changed, 154 insertions, 0 deletions

diff --git a/validate/tools/gst-validate-analyze b/validate/tools/gst-validate-analyze
new file mode 100755
index 0000000..3c6d114
--- /dev/null
+++ b/validate/tools/gst-validate-analyze
@@ -0,0 +1,154 @@
+#!/usr/bin/env python2
+#
+# Copyright (c) 2015, Edward Hervey <edward@centricular.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+
+import os
+import sys
+import xml.etree.cElementTree
+
+def extract_info(xmlfile):
+    e = xml.etree.cElementTree.parse(xmlfile).getroot()
+    r = {}
+    for i in e:
+        r[(i.get("classname"), i.get("name"))] = i
+    return r
+
+if "__main__" == __name__:
+    if len(sys.argv) < 2:
+        print "Usage : %s [<old run xml>] <new run xml>" % sys.argv[0]
+    if len(sys.argv) == 3:
+        oldfile = extract_info(sys.argv[1])
+        newfile = extract_info(sys.argv[2])
+    else:
+        oldfile = []
+        newfile = extract_info(sys.argv[1])
+
+    # new failures (pass in old run, fail in new run)
+    newfail = []
+    # new fixes (fail in old run, pass in new run)
+    newfix = []
+    # tests that are still failing
+    stillfail = []
+    # tests that are still failling but for a different reason
+    failchange = []
+
+    # failed tests sorted by reason
+    reasons = {}
+
+    # all files
+    allfiles = []
+
+    # failed tests sorted by file
+    failedfiles = {}
+
+    if oldfile:
+        # tests that weren't present in old run
+        newtests = [x for x in newfile.keys() if not oldfile.has_key(x)]
+        # tests that are no longer present in new run
+        gonetests = [x for x in oldfile.keys() if not newfile.has_key(x)]
+
+    # go over new tests
+    for k,v in newfile.iteritems():
+        tn, fn = k
+        if not fn in allfiles:
+            allfiles.append(fn)
+
+        newf = v.findall("failure")
+        if newf:
+            # extract the failure reason
+            r = newf[0].get("message")
+            if "Application returned 18" in r or "Application returned -5" in r:
+                rs = r.split('[')[1].split(']')[0].split(',')
+                for la in rs:
+                    la = la.strip()
+                    if not reasons.has_key(la):
+                        reasons[la] = []
+                    reasons[la].append(k)
+            if not failedfiles.has_key(fn):
+                failedfiles[fn] = []
+            failedfiles[fn].append((tn, r))
+
+        if k in oldfile:
+            oldone = oldfile.get(k)
+
+            # compare failures
+            oldf = oldone.findall("failure")
+            if newf and not oldf:
+                newfail.append(k)
+            if oldf and not newf:
+                newfix.append(k)
+            if oldf and newf:
+                stillfail.append(k)
+                a = oldf[0]
+                b = newf[0]
+                # check if the failure reasons are the same
+                if a.get("type") != b.get("type"):
+                    failchange.append(k)
+                elif a.get("message") != b.get("message"):
+                    failchange.append(k)
+
+
+    if newfail:
+        print "New failures", len(newfail)
+        newfail.sort()
+        for i in newfail:
+            print "   %s : %s" % (i[0], i[1])
+            f = newfile[i].find("failure")
+            print "     ", f.get("type"), f.get("message")
+        print
+
+    if newfix:
+        print "New fixes", len(newfix)
+        newfix.sort()
+        for i in newfix:
+            print "   %s : %s" % (i[0], i[1])
+        print
+
+    if failchange:
+        print "Failure changes", len(failchange)
+        failchange.sort()
+        for i in failchange:
+            print "   %s : %s" % (i[0], i[1])
+            oldt = oldfile[i].find("failure").get("type")
+            newt = newfile[i].find("failure").get("type")
+            if oldt != newt:
+                print "       Went from '%s' to '%s'" % (oldt, newt)
+            print "       Previous message :", oldfile[i].find("failure").get("message")
+            print "       New message      :", newfile[i].find("failure").get("message")
+        print
+
+    for k,v in reasons.iteritems():
+        print "Failure type : ", k, len(v)
+        v.sort()
+        for i in v:
+            print "   %s : %s" % (i[0], i[1])
+        print
+
+    nofailfiles = [fn for fn in allfiles if not failedfiles.has_key(fn)]
+    nofailfiles.sort()
+    if nofailfiles:
+        print "Files without failures", len(nofailfiles)
+        for f in nofailfiles:
+            print "    ", f
+        print
+
+    for k,v in failedfiles.iteritems():
+        print "Failed File :", k
+        for i in v:
+            print "    %s : %s" % (i[0], i[1])
+