libs,plugin: break surface-bufferproxy circular reference

The bufferproxy may reference the surface and the surface may also
reference the bufferproxy, producing a circular reference, which might
lead to serious resource leak problems.

Now make the relationship clearer, the bufferproxy's references is
transfered to surface, while bufferproxy just keeps the surface's
address without increasing its reference count.

The surface can be created through a bufferproxy like in
gst_vaapi_surface_new_with_dma_buf_handle(), and the surface might
get its bufferproxy via gst_vaapi_surface_get_dma_buf_handle(). In
both cases the surface holds a bufferproxy's reference.

5 files changed, 52 insertions, 32 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapibufferproxy.c b/gst-libs/gst/vaapi/gstvaapibufferproxy.c
index 14dc5b49..41533cd8 100644
--- a/gst-libs/gst/vaapi/gstvaapibufferproxy.c
+++ b/gst-libs/gst/vaapi/gstvaapibufferproxy.c
@@ -94,7 +94,7 @@ gst_vaapi_buffer_proxy_finalize (GstVaapiBufferProxy * proxy)
   if (proxy->destroy_func)
     proxy->destroy_func (proxy->destroy_data);
 
-  gst_mini_object_replace ((GstMiniObject **) & proxy->surface, NULL);
+  proxy->surface = NULL;
 }
 
 static inline const GstVaapiMiniObjectClass *
@@ -157,7 +157,7 @@ gst_vaapi_buffer_proxy_new_from_surface (GstMiniObject * surface,
   if (!proxy)
     return NULL;
 
-  proxy->surface = gst_mini_object_ref (surface);
+  proxy->surface = surface;
   proxy->destroy_func = destroy_func;
   proxy->destroy_data = data;
   proxy->type = type;
diff --git a/gst-libs/gst/vaapi/gstvaapisurface.c b/gst-libs/gst/vaapi/gstvaapisurface.c
index 3e92e0ea..5c821f6b 100644
--- a/gst-libs/gst/vaapi/gstvaapisurface.c
+++ b/gst-libs/gst/vaapi/gstvaapisurface.c
@@ -492,6 +492,7 @@ gst_vaapi_surface_new_from_buffer_proxy (GstVaapiDisplay * display,
 
   g_return_val_if_fail (proxy != NULL, NULL);
   g_return_val_if_fail (info != NULL, NULL);
+  g_return_val_if_fail (!proxy->surface, NULL);
 
   surface = gst_vaapi_surface_create (display);
   if (!surface)
@@ -499,6 +500,8 @@ gst_vaapi_surface_new_from_buffer_proxy (GstVaapiDisplay * display,
 
   if (!gst_vaapi_surface_init_from_buffer_proxy (surface, proxy, info))
     goto error;
+
+  proxy->surface = GST_MINI_OBJECT_CAST (surface);
   return surface;
 
   /* ERRORS */
diff --git a/gst-libs/gst/vaapi/gstvaapisurface_drm.c b/gst-libs/gst/vaapi/gstvaapisurface_drm.c
index cbf12c67..833c6ac5 100644
--- a/gst-libs/gst/vaapi/gstvaapisurface_drm.c
+++ b/gst-libs/gst/vaapi/gstvaapisurface_drm.c
@@ -59,48 +59,69 @@ error_alloc_export_buffer:
 }
 
 /**
- * gst_vaapi_surface_get_dma_buf_handle:
+ * gst_vaapi_surface_peek_dma_buf_handle:
  * @surface: a #GstVaapiSurface
  *
  * If the underlying VA driver implementation supports it, this
  * function allows for returning a suitable dma_buf (DRM) buffer
- * handle as a #GstVaapiBufferProxy instance. The resulting buffer
- * handle is live until the last reference to the proxy gets
- * released. Besides, any further change to the parent VA @surface may
- * fail.
+ * handle as a #GstVaapiBufferProxy instance. The returned buffer
+ * proxy does not increase the ref of underlying buffer proxy.
  *
  * Return value: the underlying buffer as a #GstVaapiBufferProxy
  * instance.
  */
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_dma_buf_handle (GstVaapiSurface * surface)
+gst_vaapi_surface_peek_dma_buf_handle (GstVaapiSurface * surface)
 {
+  GstVaapiBufferProxy *buf_proxy;
+
   g_return_val_if_fail (surface != NULL, NULL);
 
-  return gst_vaapi_surface_get_drm_buf_handle (surface,
+  if (surface->extbuf_proxy)
+    return surface->extbuf_proxy;
+
+  buf_proxy = gst_vaapi_surface_get_drm_buf_handle (surface,
       GST_VAAPI_BUFFER_MEMORY_TYPE_DMA_BUF);
+
+  if (buf_proxy) {
+    gst_vaapi_surface_set_buffer_proxy (surface, buf_proxy);
+    gst_vaapi_buffer_proxy_unref (buf_proxy);
+  }
+
+  return buf_proxy;
 }
 
 /**
- * gst_vaapi_surface_get_gem_buf_handle:
+ * gst_vaapi_surface_peek_gem_buf_handle:
  * @surface: a #GstVaapiSurface
  *
  * If the underlying VA driver implementation supports it, this
  * function allows for returning a suitable GEM buffer handle as a
- * #GstVaapiBufferProxy instance. The resulting buffer handle is live
- * until the last reference to the proxy gets released. Besides, any
- * further change to the parent VA @surface may fail.
+ * #GstVaapiBufferProxy instance. The returned buffer proxy does
+ * not increase the ref of underlying buffer proxy.
  *
  * Return value: the underlying buffer as a #GstVaapiBufferProxy
  * instance.
  */
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_gem_buf_handle (GstVaapiSurface * surface)
+gst_vaapi_surface_peek_gem_buf_handle (GstVaapiSurface * surface)
 {
+  GstVaapiBufferProxy *buf_proxy;
+
   g_return_val_if_fail (surface != NULL, NULL);
 
-  return gst_vaapi_surface_get_drm_buf_handle (surface,
+  if (surface->extbuf_proxy)
+    return surface->extbuf_proxy;
+
+  buf_proxy = gst_vaapi_surface_get_drm_buf_handle (surface,
       GST_VAAPI_BUFFER_MEMORY_TYPE_GEM_BUF);
+
+  if (buf_proxy) {
+    gst_vaapi_surface_set_buffer_proxy (surface, buf_proxy);
+    gst_vaapi_buffer_proxy_unref (buf_proxy);
+  }
+
+  return buf_proxy;
 }
 
 static void
@@ -149,6 +170,7 @@ gst_vaapi_surface_new_with_dma_buf_handle (GstVaapiDisplay * display, gint fd,
     return NULL;
 
   surface = gst_vaapi_surface_new_from_buffer_proxy (display, proxy, vi);
+  /* Surface holds proxy's reference */
   gst_vaapi_buffer_proxy_unref (proxy);
   return surface;
 }
@@ -187,6 +209,7 @@ gst_vaapi_surface_new_with_gem_buf_handle (GstVaapiDisplay * display,
 
   fill_video_info (&vi, format, width, height, offset, stride);
   surface = gst_vaapi_surface_new_from_buffer_proxy (display, proxy, &vi);
+  /* Surface holds proxy's reference */
   gst_vaapi_buffer_proxy_unref (proxy);
   return surface;
 }
diff --git a/gst-libs/gst/vaapi/gstvaapisurface_drm.h b/gst-libs/gst/vaapi/gstvaapisurface_drm.h
index 1dd637d8..0bc5f846 100644
--- a/gst-libs/gst/vaapi/gstvaapisurface_drm.h
+++ b/gst-libs/gst/vaapi/gstvaapisurface_drm.h
@@ -29,10 +29,10 @@
 G_BEGIN_DECLS
 
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_dma_buf_handle (GstVaapiSurface * surface);
+gst_vaapi_surface_peek_dma_buf_handle (GstVaapiSurface * surface);
 
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_gem_buf_handle (GstVaapiSurface * surface);
+gst_vaapi_surface_peek_gem_buf_handle (GstVaapiSurface * surface);
 
 GstVaapiSurface *
 gst_vaapi_surface_new_with_dma_buf_handle (GstVaapiDisplay * display, gint fd,
diff --git a/gst/vaapi/gstvaapivideomemory.c b/gst/vaapi/gstvaapivideomemory.c
index f01a2ef4..48f42ab2 100644
--- a/gst/vaapi/gstvaapivideomemory.c
+++ b/gst/vaapi/gstvaapivideomemory.c
@@ -1019,8 +1019,9 @@ gst_vaapi_dmabuf_memory_holds_surface (GstMemory * mem)
 {
   g_return_val_if_fail (mem != NULL, FALSE);
 
-  return gst_mini_object_get_qdata (GST_MINI_OBJECT_CAST (mem),
-      GST_VAAPI_BUFFER_PROXY_QUARK) != NULL;
+  return
+      GPOINTER_TO_INT (gst_mini_object_get_qdata (GST_MINI_OBJECT_CAST (mem),
+          GST_VAAPI_BUFFER_PROXY_QUARK)) == TRUE;
 }
 
 GstMemory *
@@ -1065,20 +1066,21 @@ gst_vaapi_dmabuf_memory_new (GstAllocator * base_allocator,
     proxy = gst_vaapi_surface_proxy_new (surface);
     if (!proxy)
       goto error_create_surface_proxy;
+    /* The proxy has incremented the surface ref count. */
+    gst_vaapi_surface_unref (surface);
   } else {
     /* When exporting existing surfaces that come from decoder's
      * context. */
     surface = GST_VAAPI_SURFACE_PROXY_SURFACE (proxy);
   }
 
-  dmabuf_proxy = gst_vaapi_surface_get_dma_buf_handle (surface);
+  dmabuf_proxy = gst_vaapi_surface_peek_dma_buf_handle (surface);
   if (!dmabuf_proxy)
     goto error_create_dmabuf_proxy;
 
   if (needs_surface) {
-    /* The proxy has incremented the surface ref count.  */
-    gst_vaapi_surface_unref (surface);
     gst_vaapi_video_meta_set_surface_proxy (meta, proxy);
+    /* meta holds the proxy's reference */
     gst_vaapi_surface_proxy_unref (proxy);
   }
 
@@ -1095,17 +1097,9 @@ gst_vaapi_dmabuf_memory_new (GstAllocator * base_allocator,
     goto error_create_dmabuf_memory;
 
   if (needs_surface) {
-    /* Just set the GstVaapiBufferProxy (dmabuf_proxy) as qdata and
-     * forget about it. */
+    /* qdata express that memory has an associated surface. */
     gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
-        GST_VAAPI_BUFFER_PROXY_QUARK, dmabuf_proxy,
-        (GDestroyNotify) gst_vaapi_buffer_proxy_unref);
-  } else {
-    /* When not allocating the surface from this pool, so when
-     * exporting from the decoder's VA context, we need to know which
-     * GstMemory belongs to a provided surface. */
-    gst_vaapi_buffer_proxy_set_mem (dmabuf_proxy, mem);
-    gst_vaapi_surface_set_buffer_proxy (surface, dmabuf_proxy);
+        GST_VAAPI_BUFFER_PROXY_QUARK, GINT_TO_POINTER (TRUE), NULL);
   }
 
   /* When a VA surface is going to be filled by a VAAPI element