compositor-drm: Make alpha-to-opaque handling common

Rather than a hardcoded ARGB8888 -> XRGB8888 translation inside a
GBM-specific helper, just determine whether or not the view is opaque,
and use the generic helpers to implement the format translation.

Signed-off-by: Daniel Stone <daniels@collabora.com>

Differential Revision: https://phabricator.freedesktop.org/D1515

1 files changed, 43 insertions, 71 deletions

diff --git a/libweston/compositor-drm.c b/libweston/compositor-drm.c
index 13cf6e3e..645ac17d 100644
--- a/libweston/compositor-drm.c
+++ b/libweston/compositor-drm.c
@@ -983,7 +983,7 @@ drm_fb_ref(struct drm_fb *fb)
 
 static struct drm_fb *
 drm_fb_get_from_bo(struct gbm_bo *bo, struct drm_backend *backend,
-		   uint32_t format, enum drm_fb_type type)
+		   bool is_opaque, enum drm_fb_type type)
 {
 	struct drm_fb *fb = gbm_bo_get_user_data(bo);
 	uint32_t handles[4] = { 0 }, pitches[4] = { 0 }, offsets[4] = { 0 };
@@ -1004,16 +1004,19 @@ drm_fb_get_from_bo(struct gbm_bo *bo, struct drm_backend *backend,
 	fb->height = gbm_bo_get_height(bo);
 	fb->stride = gbm_bo_get_stride(bo);
 	fb->handle = gbm_bo_get_handle(bo).u32;
-	fb->format = pixel_format_get_info(format);
+	fb->format = pixel_format_get_info(gbm_bo_get_format(bo));
 	fb->size = fb->stride * fb->height;
 	fb->fd = backend->drm.fd;
 
 	if (!fb->format) {
 		weston_log("couldn't look up format 0x%lx\n",
-			   (unsigned long) format);
+			   (unsigned long) gbm_bo_get_format(bo));
 		goto err_free;
 	}
 
+	if (is_opaque)
+		fb->format = pixel_format_get_opaque_substitute(fb->format);
+
 	if (backend->min_width > fb->width ||
 	    fb->width > backend->max_width ||
 	    backend->min_height > fb->height ||
@@ -1024,13 +1027,14 @@ drm_fb_get_from_bo(struct gbm_bo *bo, struct drm_backend *backend,
 
 	ret = -1;
 
-	if (format && !backend->no_addfb2) {
+	if (!backend->no_addfb2) {
 		handles[0] = fb->handle;
 		pitches[0] = fb->stride;
 		offsets[0] = 0;
 
 		ret = drmModeAddFB2(backend->drm.fd, fb->width, fb->height,
-				    format, handles, pitches, offsets,
+				    fb->format->format,
+				    handles, pitches, offsets,
 				    &fb->fb_id, 0);
 		if (ret) {
 			weston_log("addfb2 failed: %m\n");
@@ -1421,34 +1425,26 @@ drm_view_transform_supported(struct weston_view *ev)
 		(ev->transform.matrix.type < WESTON_MATRIX_TRANSFORM_ROTATE);
 }
 
-static uint32_t
-drm_output_check_scanout_format(struct drm_output *output,
-				struct weston_surface *es, struct gbm_bo *bo)
+static bool
+drm_view_is_opaque(struct weston_view *ev)
 {
-	uint32_t format;
 	pixman_region32_t r;
+	bool ret = false;
 
-	format = gbm_bo_get_format(bo);
-
-	if (format == GBM_FORMAT_ARGB8888) {
-		/* We can scanout an ARGB buffer if the surface's
-		 * opaque region covers the whole output, but we have
-		 * to use XRGB as the KMS format code. */
-		pixman_region32_init_rect(&r, 0, 0,
-					  output->base.width,
-					  output->base.height);
-		pixman_region32_subtract(&r, &r, &es->opaque);
+	/* We can scanout an ARGB buffer if the surface's
+	 * opaque region covers the whole output, but we have
+	 * to use XRGB as the KMS format code. */
+	pixman_region32_init_rect(&r, 0, 0,
+				  ev->surface->width,
+				  ev->surface->height);
+	pixman_region32_subtract(&r, &r, &ev->surface->opaque);
 
-		if (!pixman_region32_not_empty(&r))
-			format = GBM_FORMAT_XRGB8888;
-
-		pixman_region32_fini(&r);
-	}
+	if (!pixman_region32_not_empty(&r))
+		ret = true;
 
-	if (output->gbm_format == format)
-		return format;
+	pixman_region32_fini(&r);
 
-	return 0;
+	return ret;
 }
 
 static struct weston_plane *
@@ -1462,7 +1458,6 @@ drm_output_prepare_scanout_view(struct drm_output_state *output_state,
 	struct weston_buffer *buffer = ev->surface->buffer_ref.buffer;
 	struct weston_buffer_viewport *viewport = &ev->surface->buffer_viewport;
 	struct gbm_bo *bo;
-	uint32_t format;
 
 	/* Don't import buffers which span multiple outputs. */
 	if (ev->output_mask != (1u << output->base.id))
@@ -1506,17 +1501,20 @@ drm_output_prepare_scanout_view(struct drm_output_state *output_state,
 	if (!bo)
 		return NULL;
 
-	format = drm_output_check_scanout_format(output, ev->surface, bo);
-	if (format == 0) {
+	state = drm_output_state_get_plane(output_state, scanout_plane);
+	state->fb = drm_fb_get_from_bo(bo, b, drm_view_is_opaque(ev),
+				       BUFFER_CLIENT);
+	if (!state->fb) {
+		drm_plane_state_put_back(state);
 		gbm_bo_destroy(bo);
 		return NULL;
 	}
 
-	state = drm_output_state_get_plane(output_state, scanout_plane);
-	state->fb = drm_fb_get_from_bo(bo, b, format, BUFFER_CLIENT);
-	if (!state->fb) {
+	/* Can't change formats with just a pageflip */
+	if (state->fb->format->format != output->gbm_format) {
+		/* No need to destroy the GBM BO here, as it's now owned
+		 * by the FB. */
 		drm_plane_state_put_back(state);
-		gbm_bo_destroy(bo);
 		return NULL;
 	}
 
@@ -1553,7 +1551,7 @@ drm_output_render_gl(struct drm_output *output, pixman_region32_t *damage)
 		return NULL;
 	}
 
-	ret = drm_fb_get_from_bo(bo, b, output->gbm_format, BUFFER_GBM_SURFACE);
+	ret = drm_fb_get_from_bo(bo, b, false, BUFFER_GBM_SURFACE);
 	if (!ret) {
 		weston_log("failed to get drm_fb for bo\n");
 		gbm_surface_release_buffer(output->gbm_surface, bo);
@@ -2203,35 +2201,6 @@ atomic_flip_handler(int fd, unsigned int frame,
 	drm_output_update_complete(output, flags, sec, usec);
 }
 
-static uint32_t
-drm_output_check_plane_format(struct drm_plane *p,
-			       struct weston_view *ev, struct gbm_bo *bo)
-{
-	uint32_t i, format;
-
-	format = gbm_bo_get_format(bo);
-
-	if (format == GBM_FORMAT_ARGB8888) {
-		pixman_region32_t r;
-
-		pixman_region32_init_rect(&r, 0, 0,
-					  ev->surface->width,
-					  ev->surface->height);
-		pixman_region32_subtract(&r, &r, &ev->surface->opaque);
-
-		if (!pixman_region32_not_empty(&r))
-			format = GBM_FORMAT_XRGB8888;
-
-		pixman_region32_fini(&r);
-	}
-
-	for (i = 0; i < p->count_formats; i++)
-		if (p->formats[i] == format)
-			return format;
-
-	return 0;
-}
-
 static struct weston_plane *
 drm_output_prepare_overlay_view(struct drm_output_state *output_state,
 				struct weston_view *ev)
@@ -2247,8 +2216,8 @@ drm_output_prepare_overlay_view(struct drm_output_state *output_state,
 	struct gbm_bo *bo;
 	pixman_region32_t dest_rect, src_rect;
 	pixman_box32_t *box, tbox;
-	uint32_t format;
 	wl_fixed_t sx1, sy1, sx2, sy2;
+	unsigned int i;
 
 	if (b->sprites_are_broken)
 		return NULL;
@@ -2345,12 +2314,16 @@ drm_output_prepare_overlay_view(struct drm_output_state *output_state,
 	if (!bo)
 		goto err;
 
-	format = drm_output_check_plane_format(p, ev, bo);
-	if (format == 0)
+	state->fb = drm_fb_get_from_bo(bo, b, drm_view_is_opaque(ev),
+				       BUFFER_CLIENT);
+	if (!state->fb)
 		goto err;
 
-	state->fb = drm_fb_get_from_bo(bo, b, format, BUFFER_CLIENT);
-	if (!state->fb)
+	/* Check whether the format is supported */
+	for (i = 0; i < p->count_formats; i++)
+		if (p->formats[i] == state->fb->format->format)
+			break;
+	if (i == p->count_formats)
 		goto err;
 
 	drm_fb_set_buffer(state->fb, ev->surface->buffer_ref.buffer);
@@ -3494,8 +3467,7 @@ drm_output_init_cursor_egl(struct drm_output *output, struct drm_backend *b)
 			goto err;
 
 		output->gbm_cursor_fb[i] =
-			drm_fb_get_from_bo(bo, b, GBM_FORMAT_ARGB8888,
-					   BUFFER_CURSOR);
+			drm_fb_get_from_bo(bo, b, false, BUFFER_CURSOR);
 		if (!output->gbm_cursor_fb[i]) {
 			gbm_bo_destroy(bo);
 			goto err;