update patches to work with the latest gitdm, fixing misc. bugs

4 files changed, 212 insertions, 163 deletions

diff --git a/analyse_commits b/analyse_commits
index 27780d05..ba482acb 100755
--- a/analyse_commits
+++ b/analyse_commits
@@ -276,6 +276,7 @@ unlink ("$output_dir/git-hackers-data.csv.bak");
 # Analyze the commit logs
 print "analyse output using gitdm\n";
 my $gitdmfh;
+chdir ($config_dir);
 my $cmd = "cat $log | $gitdm_path -u -X '\.(sdf|po|dic)\$'" .
                                 " -b $config_dir" .
 	                        " -x $output_dir/git-hackers-data.csv" .
diff --git a/gitdm-patches/0001-Export-hackers-individual-raw-data-as-CSV.patch b/gitdm-patches/0001-Export-hackers-individual-raw-data-as-CSV.patch
index 8ea0a65b..7ce3f652 100644
--- a/gitdm-patches/0001-Export-hackers-individual-raw-data-as-CSV.patch
+++ b/gitdm-patches/0001-Export-hackers-individual-raw-data-as-CSV.patch
@@ -1,6 +1,6 @@
-From 6d538e7837dce89ba6a70996fd583272c8ad5290 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?C=C3=A9dric=20Bosdonnat?= <cedric.bosdonnat.ooo@free.fr>
-Date: Wed, 20 Apr 2011 11:42:46 +0200
+From 82cb8cc64e7e2d744bbf65673e61979160a5b767 Mon Sep 17 00:00:00 2001
+From: Cedric Bosdonnat <cedric.bosdonnat.ooo@free.fr>
+Date: Wed, 14 Sep 2011 00:55:30 +0200
 Subject: [PATCH 1/2] Export hackers individual raw data as CSV
 
 The data exported for each hacker are the date of the first commit,
@@ -8,16 +8,16 @@ the date of the last commit, the commits count. These help distinguish
 regular core hackers from one-shot contributors.
 ---
  README      |    3 +++
- csv.py      |   12 ++++++++++++
+ csvdump.py  |   16 +++++++++++++++-
  database.py |    8 +++++++-
- gitdm       |   12 ++++++++++--
- 4 files changed, 32 insertions(+), 3 deletions(-)
+ gitdm       |   15 +++++++++++----
+ 4 files changed, 36 insertions(+), 6 deletions(-)
 
 diff --git a/README b/README
-index 7226541..7d92b1a 100644
+index dab372e..e60285d 100644
 --- a/README
 +++ b/README
-@@ -42,6 +42,9 @@ be:
+@@ -46,6 +46,9 @@ be:
  
  	-h file	Generate HTML output to the given file
  
@@ -26,16 +26,16 @@ index 7226541..7d92b1a 100644
 +
  	-l num	Only list the top <num> entries in each report.
  
- 	-o file	Write text output to the given file (default is stdout).
-diff --git a/csv.py b/csv.py
-index cec1f06..13c3204 100644
---- a/csv.py
-+++ b/csv.py
-@@ -38,3 +38,15 @@ def OutputCSV (file):
-         author_name = stat.name.replace ("\"", ".").replace ("\\", ".")
-         file.write ("\"%s\"\t\"%s\"\t%s\t%d\t%d\n"%(author_name, empl_name, stat.date, \
-                                                     stat.added, stat.removed))
-+
+     -n  Use --numstat instead of generated patches to get the statistics.
+diff --git a/csvdump.py b/csvdump.py
+index b76a5f6..66b8a12 100644
+--- a/csvdump.py
++++ b/csvdump.py
+@@ -85,4 +85,18 @@ def OutputCSV (file):
+         writer.writerow ([author_name, stat.email, empl_name, stat.date,
+                           stat.added, stat.removed])
+ 
+-__all__ = [  'AccumulatePatch', 'OutputCSV', 'store_patch' ]
 +def OutputHackersCSV (file, hlist):
 +    if file is None:
 +        return
@@ -47,8 +47,12 @@ index cec1f06..13c3204 100644
 +                        hacker.activity_start, hacker.activity_end, \
 +                        len(hacker.patches)))
 +
++__all__ = [  'AccumulatePatch', 'OutputCSV', 'OutputHackersCSV', 'store_patch' ]
++
++
++
 diff --git a/database.py b/database.py
-index b5d9382..1c9691a 100644
+index d8c8095..a8677e0 100644
 --- a/database.py
 +++ b/database.py
 @@ -25,6 +25,8 @@ class Hacker:
@@ -81,38 +85,40 @@ index b5d9382..1c9691a 100644
      #
      # There's got to be a better way.
 diff --git a/gitdm b/gitdm
-index 8133c22..b93cf4a 100755
+index 41634e6..774b059 100755
 --- a/gitdm
 +++ b/gitdm
-@@ -37,6 +37,7 @@ DumpDB = 0
- CFName = 'gitdm.config'
- DirName = ''
- Aggregate = 'month'
+@@ -36,6 +36,7 @@ AuthorSOBs = 1
+ FileFilter = None
+ CSVFile = None
+ CSVPrefix = None
 +HackersCSV = None
- 
- #
- # Options:
-@@ -47,6 +48,7 @@ Aggregate = 'month'
+ AkpmOverLt = 0
+ DumpDB = 0
+ CFName = 'gitdm.config'
+@@ -53,6 +54,7 @@ ReportByFileType = 0
  # -d		Output individual developer stats
  # -D		Output date statistics
  # -h hfile	HTML output to hfile
 +# -H file   Export individual developer raw data as CSV
  # -l count	Maximum length for output lists
+ # -n        Use numstats instead of generated patch from git log
  # -o file	File for text output
- # -r pattern	Restrict to files matching pattern
-@@ -59,9 +61,9 @@ Aggregate = 'month'
+@@ -67,10 +69,10 @@ ReportByFileType = 0
  def ParseOpts ():
      global MapUnknown, DevReports
      global DateStats, AuthorSOBs, FileFilter, AkpmOverLt, DumpDB
--    global CFName, CSVFile, DirName, Aggregate
-+    global CFName, CSVFile, DirName, Aggregate, HackersCSV
+-    global CFName, CSVFile, CSVPrefix,DirName, Aggregate, Numstat
+-    global ReportByFileType
++    global CFName, CSVFile, CSVPrefix, HackersCSV, DirName,
++    global Aggregate, Numstat, ReportByFileType
  
--    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:l:o:r:suwx:z')
-+    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:H:l:o:r:suwx:z')
+-    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:l:no:p:r:stuwx:z')
++    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:H:l:no:p:r:stuwx:z')
      for opt in opts:
          if opt[0] == '-a':
              AkpmOverLt = 1
-@@ -75,6 +77,8 @@ def ParseOpts ():
+@@ -84,6 +86,8 @@ def ParseOpts ():
              DateStats = 1
          elif opt[0] == '-h':
              reports.SetHTMLOutput (open (opt[1], 'w'))
@@ -120,18 +126,26 @@ index 8133c22..b93cf4a 100755
 +            HackersCSV = open (opt[1], 'w')
          elif opt[0] == '-l':
              reports.SetMaxList (int (opt[1]))
-         elif opt[0] == '-o':
-@@ -403,6 +407,10 @@ if TotalChanged == 0:
+         elif opt[0] == '-n':
+@@ -108,7 +112,6 @@ def ParseOpts ():
+             Aggregate = 'week'
+         elif opt[0] == '-z':
+             DumpDB = 1
+-        
+ 
+ 
+ def LookupStoreHacker (name, email):
+@@ -480,7 +483,10 @@ if TotalChanged == 0:
  if DateStats:
      PrintDateStats ()
  
-+csv.OutputHackersCSV (HackersCSV, hlist);
-+if HackersCSV is not None:
++if HackersCSV:
++    csvdump.OutputHackersCSV (HackersCSV, hlist);
 +    HackersCSV.close ()
 +
- csv.OutputCSV (CSVFile)
- if CSVFile is not None:
-         CSVFile.close ()
+ if CSVPrefix:
+     csvdump.save_csv (CSVPrefix)
+ 
 -- 
 1.7.3.4
 
diff --git a/gitdm-patches/0002-Add-X-option-to-exclude-files.patch b/gitdm-patches/0002-Add-X-option-to-exclude-files.patch
deleted file mode 100644
index caafef9b..00000000
--- a/gitdm-patches/0002-Add-X-option-to-exclude-files.patch
+++ /dev/null
@@ -1,121 +0,0 @@
-From 9519f102756af1515868f913a51d426837317499 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?C=C3=A9dric=20Bosdonnat?= <cedric.bosdonnat.ooo@free.fr>
-Date: Wed, 20 Apr 2011 11:50:02 +0200
-Subject: [PATCH 2/2] Add -X option to exclude files
-
-This option does the opposite of the -r option. It happens to be useful
-when there are less patterns to exclude than patterns to include.
----
- README |    2 ++
- gitdm  |   33 ++++++++++++++++++++++++++++++---
- 2 files changed, 32 insertions(+), 3 deletions(-)
-
-diff --git a/README b/README
-index 7d92b1a..3394443 100644
---- a/README
-+++ b/README
-@@ -60,6 +60,8 @@ be:
- 
-     -x file  Export raw statistics as CSV.
- 
-+    -X pattern Exclude matching files
-+
-     -w  Aggregate the data by weeks instead of months in the
-         CSV file when -x is used.
- 
-diff --git a/gitdm b/gitdm
-index b93cf4a..b84c210 100755
---- a/gitdm
-+++ b/gitdm
-@@ -38,6 +38,7 @@ CFName = 'gitdm.config'
- DirName = ''
- Aggregate = 'month'
- HackersCSV = None
-+ExcludeFilter = None
- 
- #
- # Options:
-@@ -55,6 +56,7 @@ HackersCSV = None
- # -s		Ignore author SOB lines
- # -u		Map unknown employers to '(Unknown)'
- # -x file.csv   Export raw statistics as CSV
-+# -X pattern    Exclude matching files
- # -w        Aggregrate the raw statistics by weeks instead of months
- # -z		Dump out the hacker database at completion
- 
-@@ -62,8 +64,9 @@ def ParseOpts ():
-     global MapUnknown, DevReports
-     global DateStats, AuthorSOBs, FileFilter, AkpmOverLt, DumpDB
-     global CFName, CSVFile, DirName, Aggregate, HackersCSV
-+    global ExcludeFilter
- 
--    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:H:l:o:r:suwx:z')
-+    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:H:l:o:r:suwx:X:z')
-     for opt in opts:
-         if opt[0] == '-a':
-             AkpmOverLt = 1
-@@ -93,6 +96,8 @@ def ParseOpts ():
-         elif opt[0] == '-x':
-             CSVFile = open (opt[1], 'w')
-             print "open output file " + opt[1] + "\n"
-+        elif opt[0] == '-X':
-+            ExcludeFilter = re.compile (opt[1])
-         elif opt [0] == '-w':
-             Aggregate = 'week'
-         elif opt[0] == '-z':
-@@ -178,6 +183,7 @@ def grabpatch():
-     p = patch(m.group (1))
-     NextLine = sys.stdin.readline ()
-     ignore = (FileFilter is not None)
-+    exclude = False
-     while NextLine:
-         Line = NextLine
-         #
-@@ -253,14 +259,19 @@ def grabpatch():
-                 p.date = Today
-             continue
-         #
-+        # First apply the ignore filter
-+        #
-+        if ExcludeFilter:
-+            exclude = ApplyExcludeFilter (Line, exclude)
-+        #
-         # If we have a file filter, check for file lines.
-         #
--        if FileFilter:
-+        if not exclude and FileFilter:
-             ignore = ApplyFileFilter (Line, ignore)
-         #
-         # OK, maybe it's part of the diff itself.
-         #
--        if not ignore:
-+        if not ignore and not exclude:
-             if Padd.match (Line):
-                 p.added += 1
-                 continue
-@@ -299,6 +310,22 @@ def ApplyFileFilter (line, ignore):
-             return 0
-     return ignore
- 
-+def ApplyExcludeFilter (line, exclude):
-+    if ExcludeFilter is not None:
-+        m = Pfilea.match (line)
-+        file = None
-+        if m:
-+            file = m.group (1)
-+        m = Pfileb.match (line)
-+        if m:
-+            file = m.group (1)
-+
-+        if file is None:
-+            return exclude
-+        if ExcludeFilter.search (file):
-+            return True
-+    return False
-+
- #
- # If this patch is signed off by both Andrew Morton and Linus Torvalds,
- # remove the (redundant) Linus signoff.
--- 
-1.7.3.4
-
diff --git a/gitdm-patches/0002-Add-exclude-functionality.patch b/gitdm-patches/0002-Add-exclude-functionality.patch
new file mode 100644
index 00000000..51799fe4
--- /dev/null
+++ b/gitdm-patches/0002-Add-exclude-functionality.patch
@@ -0,0 +1,155 @@
+From 39a692eaa885629a9a8ed7b7f7e38989591958d4 Mon Sep 17 00:00:00 2001
+From: Michael Meeks <michael.meeks@novell.com>
+Date: Wed, 14 Sep 2011 01:12:00 +0200
+Subject: [PATCH 2/2] Add exclude functionality
+
+---
+ README |    2 ++
+ gitdm  |   37 ++++++++++++++++++++++---------------
+ 2 files changed, 24 insertions(+), 15 deletions(-)
+
+diff --git a/README b/README
+index e60285d..44bd4b5 100644
+--- a/README
++++ b/README
+@@ -73,6 +73,8 @@ be:
+ 
+     -x file  Export raw statistics as CSV.
+ 
++    -X pat   Exclude matching files (the opposite of -r)
++
+     -w  Aggregate the data by weeks instead of months in the
+         CSV file when -x is used.
+ 
+diff --git a/gitdm b/gitdm
+index 774b059..62a5a57 100755
+--- a/gitdm
++++ b/gitdm
+@@ -1,4 +1,4 @@
+-#!/usr/bin/pypy
++#!/usr/bin/python
+ #-*- coding:utf-8 -*-
+ #
+ 
+@@ -34,6 +34,7 @@ DevReports = 1
+ DateStats = 0
+ AuthorSOBs = 1
+ FileFilter = None
++ExcludeFilter = None
+ CSVFile = None
+ CSVPrefix = None
+ HackersCSV = None
+@@ -63,16 +64,17 @@ ReportByFileType = 0
+ # -s		Ignore author SOB lines
+ # -u		Map unknown employers to '(Unknown)'
+ # -x file.csv   Export raw statistics as CSV
++# -X pattern    Exclude matching files
+ # -w        Aggregrate the raw statistics by weeks instead of months
+ # -z		Dump out the hacker database at completion
+ 
+ def ParseOpts ():
+     global MapUnknown, DevReports
+     global DateStats, AuthorSOBs, FileFilter, AkpmOverLt, DumpDB
+-    global CFName, CSVFile, CSVPrefix, HackersCSV, DirName,
+-    global Aggregate, Numstat, ReportByFileType
++    global CFName, CSVFile, CSVPrefix, HackersCSV, DirName
++    global Aggregate, Numstat, ReportByFileType, ExcludeFilter
+ 
+-    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:H:l:no:p:r:stuwx:z')
++    opts, rest = getopt.getopt (sys.argv[1:], 'ab:dc:Dh:H:l:no:p:r:stuwx:X:z')
+     for opt in opts:
+         if opt[0] == '-a':
+             AkpmOverLt = 1
+@@ -108,6 +110,8 @@ def ParseOpts ():
+         elif opt[0] == '-x':
+             CSVFile = open (opt[1], 'w')
+             print "open output file " + opt[1] + "\n"
++        elif opt[0] == '-X':
++            ExcludeFilter = re.compile (opt[1])
+         elif opt [0] == '-w':
+             Aggregate = 'week'
+         elif opt[0] == '-z':
+@@ -223,7 +227,7 @@ def grabpatch(logpatch):
+         return None
+ 
+     p = patch(m.group (1))
+-    ignore = (FileFilter is not None)
++    ignore = False;
+     for Line in logpatch[1:]:
+         #
+         # Maybe it's an author line?
+@@ -274,6 +278,7 @@ def grabpatch(logpatch):
+             p.author.reportcredit (patch)
+             p.author.testcredit (patch)
+             continue
++
+         #
+         # If this one is a merge, make note of the fact.
+         #
+@@ -294,10 +299,12 @@ def grabpatch(logpatch):
+             continue
+         if not Numstat:
+             #
+-            # If we have a file filter, check for file lines.
++            # If we have file filters, check for file lines.
+             #
+             if FileFilter:
+-                ignore = ApplyFileFilter (Line, ignore)
++                ignore = ApplyFilter (Line, ignore, FileFilter, False)
++            if ExcludeFilter:
++                ignore = ApplyFilter (Line, ignore, ExcludeFilter, True)
+             #
+             # OK, maybe it's part of the diff itself.
+             #
+@@ -327,7 +334,7 @@ def GripeAboutAuthorName (name):
+     GripedAuthorNames.append (name)
+     print '%s is an author name, probably not what you want' % (name)
+ 
+-def ApplyFileFilter (line, ignore):
++def ApplyFilter (line, ignore, filter, exclude):
+     #
+     # If this is the first file line (--- a/), set ignore one way
+     # or the other.
+@@ -335,17 +342,17 @@ def ApplyFileFilter (line, ignore):
+     m = patterns['filea'].match (line)
+     if m:
+         file = m.group (1)
+-        if FileFilter.search (file):
+-            return 0
+-        return 1
++        if filter.search (file):
++            return exclude
++        return not exclude
+     #
+     # For the second line, we can turn ignore off, but not on
+     #
+     m = patterns['fileb'].match (line)
+     if m:
+         file = m.group (1)
+-        if FileFilter.search (file):
+-            return 0
++        if filter.search (file):
++            return exclude
+     return ignore
+ 
+ def is_svntag(logpatch):
+@@ -422,14 +429,14 @@ for logpatch in patches:
+ #    if p.added > 100000 or p.removed > 100000:
+ #        print 'Skipping massive add', p.commit
+ #        continue
+-    if FileFilter and p.added == 0 and p.removed == 0:
++    if (FileFilter or ExcludeFilter) and p.added == 0 and p.removed == 0:
+         continue
+ 
+     #
+     # Record some global information - but only if this patch had
+     # stuff which wasn't ignored.
+     #
+-    if ((p.added + p.removed) > 0 or not FileFilter) and not p.merge:
++    if ((p.added + p.removed) > 0 or (not FileFilter and not ExcludeFilter) ) and not p.merge:
+         TotalAdded += p.added
+         TotalRemoved += p.removed
+         TotalChanged += max (p.added, p.removed)
+-- 
+1.7.3.4
+