Fix signed-unsigned semantics in reduce_32

2 files changed, 34 insertions, 21 deletions

diff --git a/pixman/pixman-bits-image.c b/pixman/pixman-bits-image.c
index f050f35..6f65420 100644
--- a/pixman/pixman-bits-image.c
+++ b/pixman/pixman-bits-image.c
@@ -209,15 +209,15 @@ static force_inline void reduce_32(unsigned int satot, unsigned int srtot,
 {
     uint32_t *ret = p;
 
-    satot = (satot + 0x8000) >> 16;
-    srtot = (srtot + 0x8000) >> 16;
-    sgtot = (sgtot + 0x8000) >> 16;
-    sbtot = (sbtot + 0x8000) >> 16;
+    satot = (int32_t)(satot + 0x8000) / 65536;
+    srtot = (int32_t)(srtot + 0x8000) / 65536;
+    sgtot = (int32_t)(sgtot + 0x8000) / 65536;
+    sbtot = (int32_t)(sbtot + 0x8000) / 65536;
 
-    satot = CLIP (satot, 0, 0xff);
-    srtot = CLIP (srtot, 0, 0xff);
-    sgtot = CLIP (sgtot, 0, 0xff);
-    sbtot = CLIP (sbtot, 0, 0xff);
+    satot = CLIP ((int32_t)satot, 0, 0xff);
+    srtot = CLIP ((int32_t)srtot, 0, 0xff);
+    sgtot = CLIP ((int32_t)sgtot, 0, 0xff);
+    sbtot = CLIP ((int32_t)sbtot, 0, 0xff);
 
     *ret = ((satot << 24) | (srtot << 16) | (sgtot <<  8) | (sbtot));
 }
@@ -240,10 +240,10 @@ static force_inline void reduce_float(unsigned int satot, unsigned int srtot,
 {
     argb_t *ret = p;
 
-    ret->a = CLIP (satot / 65536.f, 0.f, 1.f);
-    ret->r = CLIP (srtot / 65536.f, 0.f, 1.f);
-    ret->g = CLIP (sgtot / 65536.f, 0.f, 1.f);
-    ret->b = CLIP (sbtot / 65536.f, 0.f, 1.f);
+    ret->a = CLIP ((int32_t)satot / 65536.f, 0.f, 1.f);
+    ret->r = CLIP ((int32_t)srtot / 65536.f, 0.f, 1.f);
+    ret->g = CLIP ((int32_t)sgtot / 65536.f, 0.f, 1.f);
+    ret->b = CLIP ((int32_t)sbtot / 65536.f, 0.f, 1.f);
 }
 
 typedef void (* accumulate_pixel_t) (unsigned int *satot, unsigned int *srtot,
diff --git a/pixman/pixman-filter.c b/pixman/pixman-filter.c
index 5f3b752..fa94532 100644
--- a/pixman/pixman-filter.c
+++ b/pixman/pixman-filter.c
@@ -237,11 +237,14 @@ create_1d_filter (int              width,
 		  pixman_kernel_t  sample,
 		  double           scale,
 		  int              n_phases,
-		  pixman_fixed_t *p)
+		  pixman_fixed_t *pstart,
+		  pixman_fixed_t *pend
+		  )
 {
+    pixman_fixed_t *p = pstart;
     double step;
     int i;
-
+    if(width <= 0) return;
     step = 1.0 / n_phases;
 
     for (i = 0; i < n_phases; ++i)
@@ -258,7 +261,7 @@ create_1d_filter (int              width,
 
 	x1 = ceil (frac - width / 2.0 - 0.5);
 	x2 = x1 + width;
-
+    assert( p >= pstart && p + (x2 - x1) <= pend ); /* assert validity of the following loop */
 	total = 0;
         for (x = x1; x < x2; ++x)
         {
@@ -287,8 +290,10 @@ create_1d_filter (int              width,
 
 	/* Normalize, with error diffusion */
 	p -= width;
-        total = 65536.0 / total;
-        new_total = 0;
+	assert(p >= pstart && p + (x2 - x1) <= pend); /* assert validity of the following loop */
+
+    total = 65536.0 / total;
+    new_total = 0;
 	e = 0.0;
 	for (x = x1; x < x2; ++x)
 	{
@@ -304,6 +309,8 @@ create_1d_filter (int              width,
 	 * at the first sample, since that is the only one that
 	 * hasn't had any error diffused into it.
 	 */
+
+	assert(p - width >= pstart && p - width < pend); /* assert... */
 	*(p - width) += pixman_fixed_1 - new_total;
     }
 }
@@ -465,10 +472,16 @@ pixman_filter_create_separable_convolution (int             *n_values,
     params[2] = pixman_int_to_fixed (subsample_bits_x);
     params[3] = pixman_int_to_fixed (subsample_bits_y);
 
-    create_1d_filter (width, reconstruct_x, sample_x, sx, subsample_x,
-		      params + 4);
-    create_1d_filter (height, reconstruct_y, sample_y, sy, subsample_y,
-		      params + 4 + width * subsample_x);
+    {
+        pixman_fixed_t
+            *xparams = params+4,
+            *yparams = xparams + width*subsample_x,
+            *endparams = params + *n_values;
+        create_1d_filter(width, reconstruct_x, sample_x, sx, subsample_x,
+                         xparams, yparams);
+        create_1d_filter(height, reconstruct_y, sample_y, sy, subsample_y,
+                         yparams, endparams);
+    }
 
 #ifdef PIXMAN_GNUPLOT
     gnuplot_filter(width, subsample_x, params + 4);