libs,plugin: break surface-bufferproxy circular reference

The bufferproxy may reference the surface and the surface may also
reference the bufferproxy, producing a circular reference, which might
lead to serious resource leak problems.

Now make the relationship clearer, the bufferproxy's references is
transfered to surface, while bufferproxy just keeps the surface's
address without increasing its reference count.

The surface can be created through a bufferproxy like in
gst_vaapi_surface_new_with_dma_buf_handle(), and the surface might
get its bufferproxy via gst_vaapi_surface_get_dma_buf_handle(). In
both cases the surface holds a bufferproxy's reference.

4 files changed, 43 insertions, 17 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapibufferproxy.c b/gst-libs/gst/vaapi/gstvaapibufferproxy.c
index 14dc5b49..41533cd8 100644
--- a/gst-libs/gst/vaapi/gstvaapibufferproxy.c
+++ b/gst-libs/gst/vaapi/gstvaapibufferproxy.c
@@ -94,7 +94,7 @@ gst_vaapi_buffer_proxy_finalize (GstVaapiBufferProxy * proxy)
   if (proxy->destroy_func)
     proxy->destroy_func (proxy->destroy_data);
 
-  gst_mini_object_replace ((GstMiniObject **) & proxy->surface, NULL);
+  proxy->surface = NULL;
 }
 
 static inline const GstVaapiMiniObjectClass *
@@ -157,7 +157,7 @@ gst_vaapi_buffer_proxy_new_from_surface (GstMiniObject * surface,
   if (!proxy)
     return NULL;
 
-  proxy->surface = gst_mini_object_ref (surface);
+  proxy->surface = surface;
   proxy->destroy_func = destroy_func;
   proxy->destroy_data = data;
   proxy->type = type;
diff --git a/gst-libs/gst/vaapi/gstvaapisurface.c b/gst-libs/gst/vaapi/gstvaapisurface.c
index 3e92e0ea..5c821f6b 100644
--- a/gst-libs/gst/vaapi/gstvaapisurface.c
+++ b/gst-libs/gst/vaapi/gstvaapisurface.c
@@ -492,6 +492,7 @@ gst_vaapi_surface_new_from_buffer_proxy (GstVaapiDisplay * display,
 
   g_return_val_if_fail (proxy != NULL, NULL);
   g_return_val_if_fail (info != NULL, NULL);
+  g_return_val_if_fail (!proxy->surface, NULL);
 
   surface = gst_vaapi_surface_create (display);
   if (!surface)
@@ -499,6 +500,8 @@ gst_vaapi_surface_new_from_buffer_proxy (GstVaapiDisplay * display,
 
   if (!gst_vaapi_surface_init_from_buffer_proxy (surface, proxy, info))
     goto error;
+
+  proxy->surface = GST_MINI_OBJECT_CAST (surface);
   return surface;
 
   /* ERRORS */
diff --git a/gst-libs/gst/vaapi/gstvaapisurface_drm.c b/gst-libs/gst/vaapi/gstvaapisurface_drm.c
index cbf12c67..833c6ac5 100644
--- a/gst-libs/gst/vaapi/gstvaapisurface_drm.c
+++ b/gst-libs/gst/vaapi/gstvaapisurface_drm.c
@@ -59,48 +59,69 @@ error_alloc_export_buffer:
 }
 
 /**
- * gst_vaapi_surface_get_dma_buf_handle:
+ * gst_vaapi_surface_peek_dma_buf_handle:
  * @surface: a #GstVaapiSurface
  *
  * If the underlying VA driver implementation supports it, this
  * function allows for returning a suitable dma_buf (DRM) buffer
- * handle as a #GstVaapiBufferProxy instance. The resulting buffer
- * handle is live until the last reference to the proxy gets
- * released. Besides, any further change to the parent VA @surface may
- * fail.
+ * handle as a #GstVaapiBufferProxy instance. The returned buffer
+ * proxy does not increase the ref of underlying buffer proxy.
  *
  * Return value: the underlying buffer as a #GstVaapiBufferProxy
  * instance.
  */
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_dma_buf_handle (GstVaapiSurface * surface)
+gst_vaapi_surface_peek_dma_buf_handle (GstVaapiSurface * surface)
 {
+  GstVaapiBufferProxy *buf_proxy;
+
   g_return_val_if_fail (surface != NULL, NULL);
 
-  return gst_vaapi_surface_get_drm_buf_handle (surface,
+  if (surface->extbuf_proxy)
+    return surface->extbuf_proxy;
+
+  buf_proxy = gst_vaapi_surface_get_drm_buf_handle (surface,
       GST_VAAPI_BUFFER_MEMORY_TYPE_DMA_BUF);
+
+  if (buf_proxy) {
+    gst_vaapi_surface_set_buffer_proxy (surface, buf_proxy);
+    gst_vaapi_buffer_proxy_unref (buf_proxy);
+  }
+
+  return buf_proxy;
 }
 
 /**
- * gst_vaapi_surface_get_gem_buf_handle:
+ * gst_vaapi_surface_peek_gem_buf_handle:
  * @surface: a #GstVaapiSurface
  *
  * If the underlying VA driver implementation supports it, this
  * function allows for returning a suitable GEM buffer handle as a
- * #GstVaapiBufferProxy instance. The resulting buffer handle is live
- * until the last reference to the proxy gets released. Besides, any
- * further change to the parent VA @surface may fail.
+ * #GstVaapiBufferProxy instance. The returned buffer proxy does
+ * not increase the ref of underlying buffer proxy.
  *
  * Return value: the underlying buffer as a #GstVaapiBufferProxy
  * instance.
  */
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_gem_buf_handle (GstVaapiSurface * surface)
+gst_vaapi_surface_peek_gem_buf_handle (GstVaapiSurface * surface)
 {
+  GstVaapiBufferProxy *buf_proxy;
+
   g_return_val_if_fail (surface != NULL, NULL);
 
-  return gst_vaapi_surface_get_drm_buf_handle (surface,
+  if (surface->extbuf_proxy)
+    return surface->extbuf_proxy;
+
+  buf_proxy = gst_vaapi_surface_get_drm_buf_handle (surface,
       GST_VAAPI_BUFFER_MEMORY_TYPE_GEM_BUF);
+
+  if (buf_proxy) {
+    gst_vaapi_surface_set_buffer_proxy (surface, buf_proxy);
+    gst_vaapi_buffer_proxy_unref (buf_proxy);
+  }
+
+  return buf_proxy;
 }
 
 static void
@@ -149,6 +170,7 @@ gst_vaapi_surface_new_with_dma_buf_handle (GstVaapiDisplay * display, gint fd,
     return NULL;
 
   surface = gst_vaapi_surface_new_from_buffer_proxy (display, proxy, vi);
+  /* Surface holds proxy's reference */
   gst_vaapi_buffer_proxy_unref (proxy);
   return surface;
 }
@@ -187,6 +209,7 @@ gst_vaapi_surface_new_with_gem_buf_handle (GstVaapiDisplay * display,
 
   fill_video_info (&vi, format, width, height, offset, stride);
   surface = gst_vaapi_surface_new_from_buffer_proxy (display, proxy, &vi);
+  /* Surface holds proxy's reference */
   gst_vaapi_buffer_proxy_unref (proxy);
   return surface;
 }
diff --git a/gst-libs/gst/vaapi/gstvaapisurface_drm.h b/gst-libs/gst/vaapi/gstvaapisurface_drm.h
index 1dd637d8..0bc5f846 100644
--- a/gst-libs/gst/vaapi/gstvaapisurface_drm.h
+++ b/gst-libs/gst/vaapi/gstvaapisurface_drm.h
@@ -29,10 +29,10 @@
 G_BEGIN_DECLS
 
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_dma_buf_handle (GstVaapiSurface * surface);
+gst_vaapi_surface_peek_dma_buf_handle (GstVaapiSurface * surface);
 
 GstVaapiBufferProxy *
-gst_vaapi_surface_get_gem_buf_handle (GstVaapiSurface * surface);
+gst_vaapi_surface_peek_gem_buf_handle (GstVaapiSurface * surface);
 
 GstVaapiSurface *
 gst_vaapi_surface_new_with_dma_buf_handle (GstVaapiDisplay * display, gint fd,