Handle RandR transform matrices in floating point.

RandR matrix computations lose too much precision in fixed point;
computations using the inverted matrix can be as much as 10 pixels off.
Convert them to double precision values and pass those around. These API
changes are fairly heavyweight; the official Render interface remains fixed
point, so the fixed point matrix comes along for the ride everywhere.

4 files changed, 64 insertions, 125 deletions

diff --git a/hw/xfree86/modes/xf86Crtc.c b/hw/xfree86/modes/xf86Crtc.c
index 65c003057..9f11037b7 100644
--- a/hw/xfree86/modes/xf86Crtc.c
+++ b/hw/xfree86/modes/xf86Crtc.c
@@ -104,7 +104,8 @@ xf86CrtcCreate (ScrnInfoPtr		scrn,
     crtc->rotation = RR_Rotate_0;
     crtc->desiredRotation = RR_Rotate_0;
     PictureTransformInitIdentity (&crtc->crtc_to_framebuffer);
-    PictureTransformInitIdentity (&crtc->framebuffer_to_crtc);
+    pict_f_transform_init_identity (&crtc->f_crtc_to_framebuffer);
+    pict_f_transform_init_identity (&crtc->f_framebuffer_to_crtc);
     crtc->filter = NULL;
     crtc->params = NULL;
     crtc->nparams = 0;
diff --git a/hw/xfree86/modes/xf86Crtc.h b/hw/xfree86/modes/xf86Crtc.h
index d093b1a55..026ec7803 100644
--- a/hw/xfree86/modes/xf86Crtc.h
+++ b/hw/xfree86/modes/xf86Crtc.h
@@ -305,7 +305,8 @@ struct _xf86Crtc {
      * Current transformation matrix
      */
     PictTransform   crtc_to_framebuffer;
-    PictTransform   framebuffer_to_crtc;
+    struct pict_f_transform f_crtc_to_framebuffer;
+    struct pict_f_transform f_framebuffer_to_crtc;
     PictFilterPtr   filter;
     xFixed	    *params;
     int		    nparams;
diff --git a/hw/xfree86/modes/xf86Cursors.c b/hw/xfree86/modes/xf86Cursors.c
index 7bf92651f..40657897d 100644
--- a/hw/xfree86/modes/xf86Cursors.c
+++ b/hw/xfree86/modes/xf86Cursors.c
@@ -326,15 +326,13 @@ xf86_crtc_set_cursor_position (xf86CrtcPtr crtc, int x, int y)
 	xf86CursorScreenPtr ScreenPriv =
 	    (xf86CursorScreenPtr)dixLookupPrivate(&screen->devPrivates,
 						  xf86CursorScreenKey);
-	PictVector	v;
-	x += ScreenPriv->HotX;
-	y += ScreenPriv->HotY;
-	v.vector[0] = IntToxFixed (x); v.vector[1] = IntToxFixed (y); v.vector[2] = IntToxFixed(1);
-	PictureTransformPoint (&crtc->framebuffer_to_crtc, &v);
-	x = xFixedToInt (v.vector[0]); y = xFixedToInt (v.vector[1]);
-	x -= ScreenPriv->HotX;
-	y -= ScreenPriv->HotY;
-    }
+	struct pict_f_vector   v;
+
+	v.v[0] = x + ScreenPriv->HotX; v.v[1] = y + ScreenPriv->HotY; v.v[2] = 1;
+	pict_f_transform_point (&crtc->f_framebuffer_to_crtc, &v);
+	x = floor (v.v[0] + 0.5) - ScreenPriv->HotX;
+	y = floor (v.v[1] + 0.5) - ScreenPriv->HotY;
+   }
     else
     {
 	x -= crtc->x;
diff --git a/hw/xfree86/modes/xf86Rotate.c b/hw/xfree86/modes/xf86Rotate.c
index 7b49afa4b..efbacec8e 100644
--- a/hw/xfree86/modes/xf86Rotate.c
+++ b/hw/xfree86/modes/xf86Rotate.c
@@ -122,7 +122,7 @@ xf86RotateCrtcRedisplay (xf86CrtcPtr crtc, RegionPtr region)
 	dst_box.x2 += crtc->filter_width >> 1;
 	dst_box.y1 -= crtc->filter_height >> 1;
 	dst_box.y2 += crtc->filter_height >> 1;
-	PictureTransformBounds (&dst_box, &crtc->framebuffer_to_crtc);
+	pict_f_transform_bounds (&crtc->f_framebuffer_to_crtc, &dst_box);
 	CompositePicture (PictOpSrc,
 			  src, NULL, dst,
 			  dst_box.x1, dst_box.y1, 0, 0, dst_box.x1, dst_box.y1,
@@ -165,6 +165,8 @@ xf86CrtcShadowClear (xf86CrtcPtr crtc)
     xRectangle		rect;
     int			error;
 
+    if (!dst_pixmap)
+	return;
     dst = CreatePicture (None,
 			 &dst_pixmap->drawable,
 			 format,
@@ -199,7 +201,6 @@ xf86RotatePrepare (ScreenPtr pScreen)
 							     crtc->rotatedData,
 							     crtc->mode.HDisplay,
 							     crtc->mode.VDisplay);
-	    xf86CrtcShadowClear (crtc);
 	    if (!xf86_config->rotation_damage_registered)
 	    {
 		/* Hook damage to screen pixmap */
@@ -355,109 +356,33 @@ xf86CrtcRotate (xf86CrtcPtr crtc, DisplayModePtr mode, Rotation rotation)
     xf86CrtcConfigPtr   xf86_config = XF86_CRTC_CONFIG_PTR(pScrn);
     /* if this is called during ScreenInit() we don't have pScrn->pScreen yet */
     ScreenPtr		pScreen = screenInfo.screens[pScrn->scrnIndex];
-    PictTransform	crtc_to_fb, fb_to_crtc;
+    PictTransform	crtc_to_fb;
+    struct pict_f_transform f_crtc_to_fb, f_fb_to_crtc;
     xFixed		*new_params = NULL;
     int			new_nparams = 0;
     PictFilterPtr	new_filter = NULL;
     int			new_width = 0;
     int			new_height = 0;
-    
-    PictureTransformInitIdentity (&crtc_to_fb);
-    PictureTransformInitIdentity (&fb_to_crtc);
-    if (rotation != RR_Rotate_0)
-    {
-	xFixed	rot_cos, rot_sin, rot_dx, rot_dy;
-	xFixed	scale_x, scale_y, scale_dx, scale_dy;
-	int	mode_w = crtc->mode.HDisplay;
-	int	mode_h = crtc->mode.VDisplay;
-	
-	/* rotation */
-	switch (rotation & 0xf) {
-	default:
-	case RR_Rotate_0:
-	    rot_cos = F ( 1);	    rot_sin = F ( 0);
-	    rot_dx  = F ( 0);	    rot_dy  = F ( 0);
-	    break;
-	case RR_Rotate_90:
-	    rot_cos = F ( 0);	    rot_sin = F ( 1);
-	    rot_dx =  F ( mode_h);  rot_dy  = F (0);
-	    break;
-	case RR_Rotate_180:
-	    rot_cos = F (-1);	    rot_sin = F ( 0);
-	    rot_dx  = F (mode_w);   rot_dy  = F ( mode_h);
-	    break;
-	case RR_Rotate_270:
-	    rot_cos = F ( 0);	    rot_sin = F (-1);
-	    rot_dx  = F ( 0);	    rot_dy  = F ( mode_w);
-	    break;
-	}
-	
-	PictureTransformRotate (&crtc_to_fb, &fb_to_crtc, rot_cos, rot_sin);
-	PictureTransformTranslate (&crtc_to_fb, &fb_to_crtc, rot_dx, rot_dy);
-
-	/* reflection */
-	scale_x = F (1);
-	scale_dx = 0;
-	scale_y = F (1);
-	scale_dy = 0;
-	if (rotation & RR_Reflect_X)
-	{
-	    scale_x = F(-1);
-	    if (rotation & (RR_Rotate_0|RR_Rotate_180))
-		scale_dx = F(mode_w);
-	    else
-		scale_dx = F(mode_h);
-	}
-	if (rotation & RR_Reflect_Y)
-	{
-	    scale_y = F(-1);
-	    if (rotation & (RR_Rotate_0|RR_Rotate_180))
-		scale_dy = F(mode_h);
-	    else
-		scale_dy = F(mode_w);
-	}
-	
-	PictureTransformScale (&crtc_to_fb, &fb_to_crtc, scale_x, scale_y);
-	PictureTransformTranslate (&crtc_to_fb, &fb_to_crtc, scale_dx, scale_dy);
-    }
+    RRTransformPtr	transform = NULL;
     
 #ifdef RANDR_12_INTERFACE
     if (crtc->randr_crtc)
-    {
-
-	RRTransformPtr	transform = RRCrtcGetTransform (crtc->randr_crtc);
-	if (transform)
-	{
-	    if (transform->nparams) {
-		new_params = xalloc (transform->nparams * sizeof (xFixed));
-		if (new_params) {
-		    memcpy (new_params, transform->params,
-			    transform->nparams * sizeof (xFixed));
-		    new_nparams = transform->nparams;
-		    new_filter = transform->filter;
-		}
-	    } else
-		new_filter = transform->filter;
-	    if (new_filter)
-	    {
-		new_width = new_filter->width;
-		new_height = new_filter->height;
-	    }
-	    PictureTransformMultiply (&crtc_to_fb, &transform->transform, &crtc_to_fb);
-	    PictureTransformMultiply (&fb_to_crtc, &fb_to_crtc, &transform->inverse);
-	}
-    }
+	transform = RRCrtcGetTransform (crtc->randr_crtc);
 #endif
-    /*
-     * If the untranslated transformation is the identity,
-     * disable the shadow buffer
-     */
-    if (PictureTransformIsIdentity (&crtc_to_fb))
+    if (!transform ||
+	!RRComputeTransform (crtc->x, crtc->y,
+			     crtc->mode.HDisplay, crtc->mode.VDisplay,
+			     rotation,
+			     transform,
+
+			     &crtc_to_fb,
+			     &f_crtc_to_fb,
+			     &f_fb_to_crtc))
     {
-	PictureTransformInitTranslate (&crtc_to_fb,
-				       F (-crtc->x), F (-crtc->y));
-	PictureTransformInitTranslate (&fb_to_crtc,
-				       F ( crtc->x), F ( crtc->y));
+	/*
+	 * If the untranslated transformation is the identity,
+	 * disable the shadow buffer
+	 */
 	xf86RotateDestroy (crtc);
 	crtc->transform_in_use = FALSE;
 	if (new_params)
@@ -470,24 +395,18 @@ xf86CrtcRotate (xf86CrtcPtr crtc, DisplayModePtr mode, Rotation rotation)
     }
     else
     {
-	int width, height, old_width, old_height;
-	void *shadowData;
-	PixmapPtr shadow;
-
-	PictureTransformTranslate (&crtc_to_fb, &fb_to_crtc, F(crtc->x), F(crtc->y));
-
-	/* 
+	/*
 	 * these are the size of the shadow pixmap, which
 	 * matches the mode, not the pre-rotated copy in the
 	 * frame buffer
 	 */
-	width = mode->HDisplay;
-	height = mode->VDisplay;
-	shadowData = crtc->rotatedData;
-	shadow = crtc->rotatedPixmap;
-	old_width = shadow ? shadow->drawable.width : 0;
-	old_height = shadow ? shadow->drawable.height : 0;
-	
+	int	    width = mode->HDisplay;
+	int	    height = mode->VDisplay;
+	void	    *shadowData = crtc->rotatedData;
+	PixmapPtr   shadow = crtc->rotatedPixmap;
+	int	    old_width = shadow ? shadow->drawable.width : 0;
+	int	    old_height = shadow ? shadow->drawable.height : 0;
+
 	/* Allocate memory for rotation */
 	if (old_width != width || old_height != height)
 	{
@@ -522,6 +441,27 @@ xf86CrtcRotate (xf86CrtcPtr crtc, DisplayModePtr mode, Rotation rotation)
 	    xf86_config->BlockHandler = pScreen->BlockHandler;
 	    pScreen->BlockHandler = xf86RotateBlockHandler;
 	}
+#ifdef RANDR_12_INTERFACE
+	if (transform)
+	{
+	    if (transform->nparams) {
+		new_params = xalloc (transform->nparams * sizeof (xFixed));
+		if (new_params) {
+		    memcpy (new_params, transform->params,
+			    transform->nparams * sizeof (xFixed));
+		    new_nparams = transform->nparams;
+		    new_filter = transform->filter;
+		}
+	    } else
+		new_filter = transform->filter;
+	    if (new_filter)
+	    {
+		new_width = new_filter->width;
+		new_height = new_filter->height;
+	    }
+	}
+#endif
+
 	if (0)
 	{
     bail2:
@@ -537,14 +477,13 @@ xf86CrtcRotate (xf86CrtcPtr crtc, DisplayModePtr mode, Rotation rotation)
 								  NULL,
 								  old_width,
 								  old_height);
-	    if (new_params)
-		xfree (new_params);
 	    return FALSE;
 	}
 	crtc->transform_in_use = TRUE;
     }
     crtc->crtc_to_framebuffer = crtc_to_fb;
-    crtc->framebuffer_to_crtc = fb_to_crtc;
+    crtc->f_crtc_to_framebuffer = f_crtc_to_fb;
+    crtc->f_framebuffer_to_crtc = f_fb_to_crtc;
     if (crtc->params)
 	xfree (crtc->params);
     crtc->params = new_params;
@@ -556,8 +495,8 @@ xf86CrtcRotate (xf86CrtcPtr crtc, DisplayModePtr mode, Rotation rotation)
     crtc->bounds.x2 = crtc->mode.HDisplay;
     crtc->bounds.y1 = 0;
     crtc->bounds.y2 = crtc->mode.VDisplay;
-    PictureTransformBounds (&crtc->bounds, &crtc_to_fb);
-    
+    pict_f_transform_bounds (&f_crtc_to_fb, &crtc->bounds);
+
     /* All done */
     return TRUE;
 }