Make the dedicated scanout mechanism work with arbitrary transforms v2

This makes TearFree work with arbitrary transforms, and makes transforms
work better even without TearFree, with xserver >= 1.12.

v2: Preserve clamping of transformed damage extents to CRTC boundaries.

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

2 files changed, 17 insertions, 35 deletions

diff --git a/src/amdgpu_kms.c b/src/amdgpu_kms.c
index b34a223..33dd903 100644
--- a/src/amdgpu_kms.c
+++ b/src/amdgpu_kms.c
@@ -256,19 +256,16 @@ static Bool
 amdgpu_scanout_extents_intersect(xf86CrtcPtr xf86_crtc, BoxPtr extents, int w,
 				 int h)
 {
-	extents->x1 = max(extents->x1 - xf86_crtc->x, 0);
-	extents->y1 = max(extents->y1 - xf86_crtc->y, 0);
-
-	switch (xf86_crtc->rotation & 0xf) {
-	case RR_Rotate_90:
-	case RR_Rotate_270:
-		extents->x2 = min(extents->x2 - xf86_crtc->x, h);
-		extents->y2 = min(extents->y2 - xf86_crtc->y, w);
-		break;
-	default:
-		extents->x2 = min(extents->x2 - xf86_crtc->x, w);
-		extents->y2 = min(extents->y2 - xf86_crtc->y, h);
-	}
+	extents->x1 -= xf86_crtc->filter_width >> 1;
+	extents->x2 += xf86_crtc->filter_width >> 1;
+	extents->y1 -= xf86_crtc->filter_height >> 1;
+	extents->y2 += xf86_crtc->filter_height >> 1;
+	pixman_f_transform_bounds(&xf86_crtc->f_framebuffer_to_crtc, extents);
+
+	extents->x1 = max(extents->x1, 0);
+	extents->y1 = max(extents->y1, 0);
+	extents->x2 = min(extents->x2, w);
+	extents->y2 = min(extents->y2, h);
 
 	return (extents->x1 < extents->x2 && extents->y1 < extents->y2);
 }
@@ -340,12 +337,6 @@ amdgpu_scanout_do_update(xf86CrtcPtr xf86_crtc, int scanout_id)
 			SetPicturePictFilter(src, xf86_crtc->filter, xf86_crtc->params,
 					     xf86_crtc->nparams);
 
-		extents.x1 += xf86_crtc->x - (xf86_crtc->filter_width >> 1);
-		extents.x2 += xf86_crtc->x + (xf86_crtc->filter_width >> 1);
-		extents.y1 += xf86_crtc->y - (xf86_crtc->filter_height >> 1);
-		extents.y2 += xf86_crtc->y + (xf86_crtc->filter_height >> 1);
-		pixman_f_transform_bounds(&xf86_crtc->f_framebuffer_to_crtc, &extents);
-
 		pScreen->SourceValidate = NULL;
 		CompositePicture(PictOpSrc,
 				 src, NULL, dst,
diff --git a/src/drmmode_display.c b/src/drmmode_display.c
index 5bddfd0..f5d73c4 100644
--- a/src/drmmode_display.c
+++ b/src/drmmode_display.c
@@ -606,15 +606,15 @@ drmmode_handle_transform(xf86CrtcPtr crtc)
 	Bool ret;
 
 #if XF86_CRTC_VERSION >= 7
-	if (!crtc->transformPresent && crtc->rotation != RR_Rotate_0)
+	if (crtc->transformPresent || crtc->rotation != RR_Rotate_0)
 	    crtc->driverIsPerformingTransform = XF86DriverTransformOutput;
 	else
 	    crtc->driverIsPerformingTransform = XF86DriverTransformNone;
 #else
 	AMDGPUInfoPtr info = AMDGPUPTR(crtc->scrn);
 
-	crtc->driverIsPerformingTransform = info->tear_free &&
-		!crtc->transformPresent && crtc->rotation != RR_Rotate_0;
+	crtc->driverIsPerformingTransform = crtc->transformPresent ||
+		(info->tear_free && crtc->rotation != RR_Rotate_0);
 #endif
 
 	ret = xf86CrtcRotate(crtc);
@@ -730,19 +730,10 @@ drmmode_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
 					RegionUninit(pRegion);
 					pRegion->data = NULL;
 					pBox = RegionExtents(pRegion);
-					pBox->x1 = min(pBox->x1, x);
-					pBox->y1 = min(pBox->y1, y);
-
-					switch (crtc->rotation & 0xf) {
-					case RR_Rotate_90:
-					case RR_Rotate_270:
-						pBox->x2 = max(pBox->x2, x + mode->VDisplay);
-						pBox->y2 = max(pBox->y2, y + mode->HDisplay);
-						break;
-					default:
-						pBox->x2 = max(pBox->x2, x + mode->HDisplay);
-						pBox->y2 = max(pBox->y2, y + mode->VDisplay);
-					}
+					pBox->x1 = 0;
+					pBox->y1 = 0;
+					pBox->x2 = max(pBox->x2, pScrn->virtualX);
+					pBox->y2 = max(pBox->y2, pScrn->virtualY);
 				}
 			}