2006-02-28 Kristian Høgsberg <krh@redhat.com>poppler-0.4

        * goo/gmem.c: (gmalloc), (grealloc):
        * poppler/JBIG2Stream.cc:
        * poppler/Stream.cc:
        * poppler/Stream.h:
        * splash/SplashXPathScanner.cc:

        More integer overflow fixes from Derek Noonburg (#5922).

6 files changed, 49 insertions, 20 deletions

diff --git a/ChangeLog b/ChangeLog
index 894d757b..2c2e4edf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2006-02-28  Kristian Høgsberg  <krh@redhat.com>
+
+	* goo/gmem.c: (gmalloc), (grealloc):
+	* poppler/JBIG2Stream.cc:
+	* poppler/Stream.cc:
+	* poppler/Stream.h:
+	* splash/SplashXPathScanner.cc:
+
+	More integer overflow fixes from Derek Noonburg (#5922).
+
 2006-02-09  Kristian Høgsberg  <krh@redhat.com>
 
 	* configure.ac: Bump realease to 0.4.5.
diff --git a/goo/gmem.c b/goo/gmem.c
index 3a1b5911..0e8b780b 100644
--- a/goo/gmem.c
+++ b/goo/gmem.c
@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <stddef.h>
 #include <string.h>
+#include <limits.h>
 #include "gmem.h"
 
 #ifdef DEBUG_MEM
@@ -62,7 +63,7 @@ void *gmalloc(size_t size) {
   int lst;
   unsigned long *trl, *p;
 
-  if (size == 0)
+  if (size <= 0)
     return NULL;
   size1 = gMemDataSize(size);
   if (!(mem = (char *)malloc(size1 + gMemHdrSize + gMemTrlSize))) {
@@ -84,7 +85,7 @@ void *gmalloc(size_t size) {
 #else
   void *p;
 
-  if (size == 0)
+  if (size <= 0)
     return NULL;
   if (!(p = malloc(size))) {
     fprintf(stderr, "Out of memory\n");
@@ -100,7 +101,7 @@ void *grealloc(void *p, size_t size) {
   void *q;
   size_t oldSize;
 
-  if (size == 0) {
+  if (size <= 0) {
     if (p)
       gfree(p);
     return NULL;
@@ -118,7 +119,7 @@ void *grealloc(void *p, size_t size) {
 #else
   void *q;
 
-  if (size == 0) {
+  if (size <= 0) {
     if (p)
       free(p);
     return NULL;
diff --git a/poppler/JBIG2Stream.cc b/poppler/JBIG2Stream.cc
index 495272bb..007d9f01 100644
--- a/poppler/JBIG2Stream.cc
+++ b/poppler/JBIG2Stream.cc
@@ -683,7 +683,7 @@ JBIG2Bitmap::JBIG2Bitmap(Guint segNumA, int wA, int hA):
   h = hA;
   line = (wA + 7) >> 3;
 
-  if (h < 0 || line <= 0 || h >= (INT_MAX - 1) / line) {
+  if (w <= 0 || h <= 0 || line <= 0 || h >= (INT_MAX - 1) / line) {
     error(-1, "invalid width/height");
     data = NULL;
     return;
@@ -701,7 +701,7 @@ JBIG2Bitmap::JBIG2Bitmap(Guint segNumA, JBIG2Bitmap *bitmap):
   h = bitmap->h;
   line = bitmap->line;
 
-  if (h < 0 || line <= 0 || h >= (INT_MAX - 1) / line) {
+  if (w <= 0 || h <= 0 || line <= 0 || h >= (INT_MAX - 1) / line) {
     error(-1, "invalid width/height");
     data = NULL;
     return;
@@ -2268,6 +2268,14 @@ void JBIG2Stream::readHalftoneRegionSeg(Guint segNum, GBool imm,
       !readUWord(&stepX) || !readUWord(&stepY)) {
     goto eofError;
   }
+  if (w == 0 || h == 0 || w >= INT_MAX / h) {
+    error(getPos(), "Bad bitmap size in JBIG2 halftone segment");
+    return;
+  }
+  if (gridH == 0 || gridW >= INT_MAX / gridH) {
+    error(getPos(), "Bad grid size in JBIG2 halftone segment");
+    return;
+  }
 
   // get pattern dictionary
   if (nRefSegs != 1) {
diff --git a/poppler/Stream.cc b/poppler/Stream.cc
index 96b18705..37dcfd52 100644
--- a/poppler/Stream.cc
+++ b/poppler/Stream.cc
@@ -426,7 +426,8 @@ StreamPredictor::StreamPredictor(Stream *strA, int predictorA,
 
   if (width <= 0 || nComps <= 0 || nBits <= 0 ||
       nComps >= INT_MAX/nBits ||
-      width >= INT_MAX/nComps/nBits) {
+      width >= INT_MAX/nComps/nBits ||
+      nVals * nBits + 7 < 0) {
     return;
   }
   nVals = width * nComps;
@@ -3078,6 +3079,7 @@ GBool DCTStream::readHuffmanTables() {
 	numACHuffTables = index+1;
       tbl = &acHuffTables[index];
     } else {
+      index &= 0x0f;
       if (index >= numDCHuffTables)
 	numDCHuffTables = index+1;
       tbl = &dcHuffTables[index];
diff --git a/poppler/Stream.h b/poppler/Stream.h
index 982e5616..4dadfe89 100644
--- a/poppler/Stream.h
+++ b/poppler/Stream.h
@@ -532,7 +532,7 @@ private:
   short getWhiteCode();
   short getBlackCode();
   short lookBits(int n);
-  void eatBits(int n) { inputBits -= n; }
+  void eatBits(int n) { if ((inputBits -= n) < 0) inputBits = 0; }
 };
 
 #ifndef ENABLE_LIBJPEG
diff --git a/splash/SplashXPathScanner.cc b/splash/SplashXPathScanner.cc
index eba3c4f7..bc0474c4 100644
--- a/splash/SplashXPathScanner.cc
+++ b/splash/SplashXPathScanner.cc
@@ -182,7 +182,7 @@ GBool SplashXPathScanner::getNextSpan(int y, int *x0, int *x1) {
 }
 
 void SplashXPathScanner::computeIntersections(int y) {
-  SplashCoord ySegMin, ySegMax, xx0, xx1;
+  SplashCoord xSegMin, xSegMax, ySegMin, ySegMax, xx0, xx1;
   SplashXPathSeg *seg;
   int i, j;
 
@@ -232,19 +232,27 @@ void SplashXPathScanner::computeIntersections(int y) {
     } else if (seg->flags & splashXPathVert) {
       xx0 = xx1 = seg->x0;
     } else {
-      if (ySegMin <= y) {
-	// intersection with top edge
-	xx0 = seg->x0 + (y - seg->y0) * seg->dxdy;
+      if (seg->x0 < seg->x1) {
+	xSegMin = seg->x0;
+	xSegMax = seg->x1;
       } else {
-	// x coord of segment endpoint with min y coord
-	xx0 = (seg->flags & splashXPathFlip) ? seg->x1 : seg->x0;
+	xSegMin = seg->x1;
+	xSegMax = seg->x0;
       }
-      if (ySegMax >= y + 1) {
-	// intersection with bottom edge
-	xx1 = seg->x0 + (y + 1 - seg->y0) * seg->dxdy;
-      } else {
-	// x coord of segment endpoint with max y coord
-	xx1 = (seg->flags & splashXPathFlip) ? seg->x0 : seg->x1;
+      // intersection with top edge
+      xx0 = seg->x0 + ((SplashCoord)y - seg->y0) * seg->dxdy;
+      // intersection with bottom edge
+      xx1 = seg->x0 + ((SplashCoord)y + 1 - seg->y0) * seg->dxdy;
+      // the segment may not actually extend to the top and/or bottom edges
+      if (xx0 < xSegMin) {
+	xx0 = xSegMin;
+      } else if (xx0 > xSegMax) {
+	xx0 = xSegMax;
+      }
+      if (xx1 < xSegMin) {
+	xx1 = xSegMin;
+      } else if (xx1 > xSegMax) {
+	xx1 = xSegMax;
       }
     }
     if (xx0 < xx1) {