Handle transform failure when computing shadow damage area.

PictureTransformBounds can fail, when this happens, damage the entire screen
so that the shadow gets repainted correctly.

3 files changed, 35 insertions, 23 deletions

diff --git a/hw/xfree86/modes/xf86Rotate.c b/hw/xfree86/modes/xf86Rotate.c
index 75f4a55bb..865d59c41 100644
--- a/hw/xfree86/modes/xf86Rotate.c
+++ b/hw/xfree86/modes/xf86Rotate.c
@@ -135,25 +135,6 @@ xf86RotateCrtcRedisplay (xf86CrtcPtr crtc, RegionPtr region)
 }
 
 static void
-xf86CrtcDamageShadow (xf86CrtcPtr crtc)
-{
-    ScrnInfoPtr	pScrn = crtc->scrn;
-    BoxRec	damage_box;
-    RegionRec   damage_region;
-    ScreenPtr	pScreen = pScrn->pScreen;
-
-    damage_box.x1 = 0;
-    damage_box.x2 = crtc->mode.HDisplay;
-    damage_box.y1 = 0;
-    damage_box.y2 = crtc->mode.VDisplay;
-    PictureTransformBounds (&damage_box, &crtc->crtc_to_framebuffer);
-    REGION_INIT (pScreen, &damage_region, &damage_box, 1);
-    DamageRegionAppend(&(*pScreen->GetScreenPixmap)(pScreen)->drawable,
-			&damage_region);
-    REGION_UNINIT (pScreen, &damage_region);
-}
-
-static void
 xf86CrtcShadowClear (xf86CrtcPtr crtc)
 {
     PixmapPtr		dst_pixmap = crtc->rotatedPixmap;
@@ -185,6 +166,36 @@ xf86CrtcShadowClear (xf86CrtcPtr crtc)
 }
 
 static void
+xf86CrtcDamageShadow (xf86CrtcPtr crtc)
+{
+    ScrnInfoPtr	pScrn = crtc->scrn;
+    BoxRec	damage_box;
+    RegionRec   damage_region;
+    ScreenPtr	pScreen = pScrn->pScreen;
+
+    damage_box.x1 = 0;
+    damage_box.x2 = crtc->mode.HDisplay;
+    damage_box.y1 = 0;
+    damage_box.y2 = crtc->mode.VDisplay;
+    if (!PictureTransformBounds (&damage_box, &crtc->crtc_to_framebuffer))
+    {
+	damage_box.x1 = 0;
+	damage_box.y1 = 0;
+	damage_box.x2 = pScreen->width;
+	damage_box.y2 = pScreen->height;
+    }
+    if (damage_box.x1 < 0) damage_box.x1 = 0;
+    if (damage_box.y1 < 0) damage_box.y1 = 0;
+    if (damage_box.x2 > pScreen->width) damage_box.x2 = pScreen->width;
+    if (damage_box.y2 > pScreen->height) damage_box.y2 = pScreen->height;
+    REGION_INIT (pScreen, &damage_region, &damage_box, 1);
+    DamageRegionAppend (&(*pScreen->GetScreenPixmap)(pScreen)->drawable,
+			&damage_region);
+    REGION_UNINIT (pScreen, &damage_region);
+    xf86CrtcShadowClear (crtc);
+}
+
+static void
 xf86RotatePrepare (ScreenPtr pScreen)
 {
     ScrnInfoPtr		pScrn = xf86Screens[pScreen->myNum];
diff --git a/render/matrix.c b/render/matrix.c
index 603281ed6..560ee943e 100644
--- a/render/matrix.c
+++ b/render/matrix.c
@@ -84,7 +84,6 @@ _X_EXPORT Bool
 PictureTransformPoint (PictTransformPtr transform,
 		       PictVectorPtr	vector)
 {
-    PictVector	    result;
     int		    i, j;
     xFixed_32_32    partial;
     xFixed_34_30    v[3];
@@ -227,7 +226,7 @@ PictureTransformTranslate (PictTransformPtr forward,
     return TRUE;
 }
 
-_X_EXPORT void
+_X_EXPORT Bool
 PictureTransformBounds (BoxPtr b, PictTransformPtr matrix)
 {
     PictVector	v[4];
@@ -240,7 +239,8 @@ PictureTransformBounds (BoxPtr b, PictTransformPtr matrix)
     v[3].vector[0] = F (b->x1);    v[3].vector[1] = F (b->y2);	v[3].vector[2] = F(1);
     for (i = 0; i < 4; i++)
     {
-	PictureTransformPoint (matrix, &v[i]);
+	if (!PictureTransformPoint (matrix, &v[i]))
+	    return FALSE;
 	x1 = xFixedToInt (v[i].vector[0]);
 	y1 = xFixedToInt (v[i].vector[1]);
 	x2 = xFixedToInt (xFixedCeil (v[i].vector[0]));
@@ -258,6 +258,7 @@ PictureTransformBounds (BoxPtr b, PictTransformPtr matrix)
 	    if (y2 > b->y2) b->y2 = y2;
 	}
     }
+    return TRUE;
 }
 
 _X_EXPORT Bool
diff --git a/render/picturestr.h b/render/picturestr.h
index 4a7b6e754..ab5dc162a 100644
--- a/render/picturestr.h
+++ b/render/picturestr.h
@@ -724,7 +724,7 @@ PictureTransformTranslate (PictTransformPtr forward,
 			   PictTransformPtr reverse,
 			   xFixed tx, xFixed ty);
 
-void
+Bool
 PictureTransformBounds (BoxPtr b, PictTransformPtr matrix);
 
 Bool