vmware/vmwgfx: Don't change backing-store of active scanout surfaces

With option "HwPresents" on, the driver would sometimes change backing
store of active scanout surfaces, making the kernel module refuse to
present. This was caused by scanout surfaces not having the RENDERTARGET flag
on by default. So when rendered to, using copies or composites, they
would be reallocated. Fix this by adding the RENDERTARGET flag from start.

Also add code that prints out an error message when we change backing store
of active scanout surfaces

Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com>
Reviewed-by: Jakob Bornecrantz <jakob@vmware.com>

3 files changed, 70 insertions, 11 deletions

diff --git a/vmwgfx/vmwgfx_saa.c b/vmwgfx/vmwgfx_saa.c
index b56b05f..3da0222 100644
--- a/vmwgfx/vmwgfx_saa.c
+++ b/vmwgfx/vmwgfx_saa.c
@@ -610,9 +610,9 @@ vmwgfx_pix_resize(PixmapPtr pixmap, unsigned int old_pitch,
     }
 
     if (vpix->hw) {
-	if (xa_surface_redefine(vpix->hw, draw->width, draw->height,
-				draw->depth, xa_type_argb,
-				xa_format_unknown, vpix->xa_flags, 1) != 0)
+      if (!vmwgfx_xa_surface_redefine(vpix, vpix->hw, draw->width,
+				      draw->height, draw->depth, xa_type_argb,
+				      xa_format_unknown, vpix->xa_flags, 1))
 	    return FALSE;
     }
 
@@ -1504,7 +1504,8 @@ vmwgfx_scanout_ref(struct vmwgfx_screen_entry  *entry)
 	     * The KMS fb will be a HW surface. Create it, add damage
 	     * and get the handle.
 	     */
-	    if (!vmwgfx_hw_accel_validate(pixmap, 0, XA_FLAG_SCANOUT, 0, NULL))
+	    if (!vmwgfx_hw_accel_validate(pixmap, 0, XA_FLAG_SCANOUT |
+					  XA_FLAG_RENDER_TARGET, 0, NULL))
 		goto out_err;
 	    if (_xa_surface_handle(vpix->hw, &handle, &dummy) != 0)
 		goto out_err;
diff --git a/vmwgfx/vmwgfx_saa_priv.h b/vmwgfx/vmwgfx_saa_priv.h
index c84827b..4a0c302 100644
--- a/vmwgfx/vmwgfx_saa_priv.h
+++ b/vmwgfx/vmwgfx_saa_priv.h
@@ -100,6 +100,17 @@ vmwgfx_hw_composite_dst_stage(PixmapPtr pixmap,
 Bool
 vmwgfx_hw_commit(PixmapPtr pixmap);
 
+Bool
+vmwgfx_xa_surface_redefine(struct vmwgfx_saa_pixmap *vpix,
+			   struct xa_surface *srf,
+			   int width,
+			   int height,
+			   int depth,
+			   enum xa_surface_type stype,
+			   enum xa_formats rgb_format,
+			   unsigned int new_flags,
+			   int copy_contents);
+
 /*
  * vmwgfx_xa_composite.c
  */
diff --git a/vmwgfx/vmwgfx_xa_surface.c b/vmwgfx/vmwgfx_xa_surface.c
index 189bfdc..a30b41c 100644
--- a/vmwgfx/vmwgfx_xa_surface.c
+++ b/vmwgfx/vmwgfx_xa_surface.c
@@ -43,6 +43,52 @@ static const enum xa_surface_type vmwgfx_stype_map[] = {
 static const unsigned int vmwgfx_stype_map_size =
     sizeof(vmwgfx_stype_map) / sizeof(enum xa_surface_type);
 
+/**
+ * vmwgfx_xa_surface_redefine - wrapper around xa_surface_redefine
+ *
+ * @vpix: Pointer to the struct vmwgfx_saa_pixmap the surface is attached to.
+ * @srf: The surface.
+ * @width: New width.
+ * @height: New height.
+ * @depth: New pixel depth.
+ * @stype: New surface type.
+ * @rgb_format: New rgb format.
+ * @new_flags: New surface flags.
+ * @copy_contents: Copy contents if new backing store is allocated.
+ *
+ * This is a wrapper that prints out an error message if the backing store
+ * of an active scanout surface is changed.
+ */
+Bool
+vmwgfx_xa_surface_redefine(struct vmwgfx_saa_pixmap *vpix,
+			   struct xa_surface *srf,
+			   int width,
+			   int height,
+			   int depth,
+			   enum xa_surface_type stype,
+			   enum xa_formats rgb_format,
+			   unsigned int new_flags,
+			   int copy_contents)
+{
+    uint32_t handle, new_handle, dummy;
+    Bool have_handle = FALSE;
+
+    if (!WSBMLISTEMPTY(&vpix->scanout_list))
+	have_handle = (_xa_surface_handle(srf, &handle, &dummy) == XA_ERR_NONE);
+
+    if (xa_surface_redefine(srf, width, height, depth, stype, rgb_format,
+			    new_flags, copy_contents) != XA_ERR_NONE)
+	return FALSE;
+
+    if (!WSBMLISTEMPTY(&vpix->scanout_list) && have_handle &&
+	_xa_surface_handle(srf, &new_handle, &dummy) == XA_ERR_NONE &&
+	new_handle != handle) {
+	LogMessage(X_ERROR, "Changed active scanout surface handle.\n");
+    }
+
+    return TRUE;
+}
+
 
 /*
  * Create an xa format from a PICT format.
@@ -323,13 +369,14 @@ vmwgfx_hw_commit(PixmapPtr pixmap)
 	if (vpix->staging_format != xa_surface_format(vpix->hw))
 	    LogMessage(X_INFO, "Changing hardware format.\n");
 
-	if (xa_surface_redefine(vpix->hw,
-				pixmap->drawable.width,
-				pixmap->drawable.height,
-				0,
-				xa_type_other,
-				vpix->staging_format,
-				new_flags, 1) != XA_ERR_NONE)
+	if (!vmwgfx_xa_surface_redefine(vpix,
+					vpix->hw,
+					pixmap->drawable.width,
+					pixmap->drawable.height,
+					0,
+					xa_type_other,
+					vpix->staging_format,
+					new_flags, 1) != XA_ERR_NONE)
 	    return FALSE;
 	vpix->xa_flags = new_flags;
     } else if (!vmwgfx_create_hw(vsaa, pixmap))