libs: encoder: h264/h265: flush pending ordered pictures

In order to flush the pending pictures, a new internal encoder vmethod
is used: get_pending_reordered()

This method follows an iterator pattern which will return the next
picture to encode and push.

The base encoder will call this function in a loop when flush() is called.

For now, only H.264 and H.265 encoders implement this flushing mechanism.

5 files changed, 174 insertions, 103 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder.c b/gst-libs/gst/vaapi/gstvaapiencoder.c
index 5c3caf02..d463d6e3 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder.c
@@ -419,7 +419,7 @@ _coded_buffer_proxy_released_notify (GstVaapiEncoder * encoder)
 }
 
 /* Creates a new VA coded buffer object proxy, backed from a pool */
-GstVaapiCodedBufferProxy *
+static GstVaapiCodedBufferProxy *
 gst_vaapi_encoder_create_coded_buffer (GstVaapiEncoder * encoder)
 {
   GstVaapiCodedBufferPool *const pool =
@@ -597,6 +597,17 @@ error_invalid_buffer:
   }
 }
 
+static inline gboolean
+_get_pending_reordered (GstVaapiEncoder * encoder,
+    GstVaapiEncPicture ** picture, gpointer * state)
+{
+  GstVaapiEncoderClass *const klass = GST_VAAPI_ENCODER_GET_CLASS (encoder);
+
+  if (!klass->get_pending_reordered)
+    return FALSE;
+  return klass->get_pending_reordered (encoder, picture, state);
+}
+
 /**
  * gst_vaapi_encoder_flush:
  * @encoder: a #GstVaapiEncoder
@@ -609,8 +620,47 @@ GstVaapiEncoderStatus
 gst_vaapi_encoder_flush (GstVaapiEncoder * encoder)
 {
   GstVaapiEncoderClass *const klass = GST_VAAPI_ENCODER_GET_CLASS (encoder);
+  GstVaapiCodedBufferProxy *codedbuf_proxy;
+  GstVaapiEncPicture *picture;
+  GstVaapiEncoderStatus status;
+  gpointer iter = NULL;
+
+  picture = NULL;
+  while (_get_pending_reordered (encoder, &picture, &iter)) {
+    if (!picture)
+      continue;
+
+    codedbuf_proxy = gst_vaapi_encoder_create_coded_buffer (encoder);
+    if (!codedbuf_proxy)
+      goto error_create_coded_buffer;
+
+    status = klass->encode (encoder, picture, codedbuf_proxy);
+    if (status != GST_VAAPI_ENCODER_STATUS_SUCCESS)
+      goto error_encode;
+
+    gst_vaapi_coded_buffer_proxy_set_user_data (codedbuf_proxy,
+        picture, (GDestroyNotify) gst_vaapi_mini_object_unref);
+    g_async_queue_push (encoder->codedbuf_queue, codedbuf_proxy);
+    encoder->num_codedbuf_queued++;
+  }
+  g_free (iter);
 
   return klass->flush (encoder);
+
+  /* ERRORS */
+error_create_coded_buffer:
+  {
+    GST_ERROR ("failed to allocate coded buffer");
+    gst_vaapi_enc_picture_unref (picture);
+    return GST_VAAPI_ENCODER_STATUS_ERROR_ALLOCATION_FAILED;
+  }
+error_encode:
+  {
+    GST_ERROR ("failed to encode frame (status = %d)", status);
+    gst_vaapi_enc_picture_unref (picture);
+    gst_vaapi_coded_buffer_proxy_unref (codedbuf_proxy);
+    return status;
+  }
 }
 
 /**
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder.h b/gst-libs/gst/vaapi/gstvaapiencoder.h
index 6be4f13e..974a70a9 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder.h
@@ -202,10 +202,6 @@ gst_vaapi_encoder_flush (GstVaapiEncoder * encoder);
 GArray *
 gst_vaapi_encoder_get_surface_formats (GstVaapiEncoder * encoder,
     GstVaapiProfile profile);
-
-GstVaapiCodedBufferProxy *
-gst_vaapi_encoder_create_coded_buffer (GstVaapiEncoder * encoder);
-
 G_END_DECLS
 
 #endif /* GST_VAAPI_ENCODER_H */
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index 32756bbe..42f99be4 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -2908,75 +2908,86 @@ error:
   }
 }
 
-static GstVaapiEncoderStatus
-gst_vaapi_encoder_h264_flush (GstVaapiEncoder * base_encoder)
+struct _PendingIterState
+{
+  guint cur_view;
+  GstVaapiPictureType pic_type;
+};
+
+static gboolean
+gst_vaapi_encoder_h264_get_pending_reordered (GstVaapiEncoder * base_encoder,
+    GstVaapiEncPicture ** picture, gpointer * state)
 {
   GstVaapiEncoderH264 *const encoder = GST_VAAPI_ENCODER_H264 (base_encoder);
   GstVaapiH264ViewReorderPool *reorder_pool;
   GstVaapiEncPicture *pic;
-  GstVaapiCodedBufferProxy *codedbuf_proxy;
-  guint i;
-  gboolean p_frame = TRUE;
-  GstVaapiEncoderStatus status;
+  struct _PendingIterState *iter;
 
-  for (i = 0; i < encoder->num_views; i++) {
-    reorder_pool = &encoder->reorder_pools[i];
+  g_return_val_if_fail (state, FALSE);
 
-    while (!g_queue_is_empty (&reorder_pool->reorder_frame_list)) {
-      pic = NULL;
-      if (p_frame) {
-        pic = (GstVaapiEncPicture *)
-            g_queue_pop_tail (&reorder_pool->reorder_frame_list);
-        set_p_frame (pic, encoder);
-        p_frame = FALSE;
-      } else {
-        pic = (GstVaapiEncPicture *)
-            g_queue_pop_head (&reorder_pool->reorder_frame_list);
-        set_b_frame (pic, encoder);
-      }
-      g_assert (pic);
+  if (!*state) {
+    iter = g_new0 (struct _PendingIterState, 1);
+    iter->pic_type = GST_VAAPI_PICTURE_TYPE_P;
+    *state = iter;
+  } else {
+    iter = *state;
+  }
+
+  *picture = NULL;
+
+  if (iter->cur_view >= encoder->num_views)
+    return FALSE;
+
+  reorder_pool = &encoder->reorder_pools[iter->cur_view];
+  if (g_queue_is_empty (&reorder_pool->reorder_frame_list)) {
+    iter->cur_view++;
+    return TRUE;                /* perhaps other views has pictures? */
+  }
 
-      set_frame_num (encoder, pic);
+  pic = g_queue_pop_tail (&reorder_pool->reorder_frame_list);
+  g_assert (pic);
+  if (iter->pic_type == GST_VAAPI_PICTURE_TYPE_P) {
+    set_p_frame (pic, encoder);
+    iter->pic_type = GST_VAAPI_PICTURE_TYPE_B;
+  } else if (iter->pic_type == GST_VAAPI_PICTURE_TYPE_B) {
+    set_b_frame (pic, encoder);
+  } else {
+    GST_WARNING ("Unhandled pending picture type");
+  }
 
-      if (GST_CLOCK_TIME_IS_VALID (pic->frame->pts))
-        pic->frame->pts += encoder->cts_offset;
+  set_frame_num (encoder, pic);
 
-      codedbuf_proxy = gst_vaapi_encoder_create_coded_buffer (base_encoder);
-      if (!codedbuf_proxy)
-        goto error_create_coded_buffer;
+  if (GST_CLOCK_TIME_IS_VALID (pic->frame->pts))
+    pic->frame->pts += encoder->cts_offset;
 
-      status =
-          gst_vaapi_encoder_h264_encode (base_encoder, pic, codedbuf_proxy);
-      if (status != GST_VAAPI_ENCODER_STATUS_SUCCESS)
-        goto error_encode;
+  *picture = pic;
+  return TRUE;
+}
 
-      gst_vaapi_coded_buffer_proxy_set_user_data (codedbuf_proxy,
-          pic, (GDestroyNotify) gst_vaapi_mini_object_unref);
-      g_async_queue_push (base_encoder->codedbuf_queue, codedbuf_proxy);
-      base_encoder->num_codedbuf_queued++;
-    }
+static GstVaapiEncoderStatus
+gst_vaapi_encoder_h264_flush (GstVaapiEncoder * base_encoder)
+{
+  GstVaapiEncoderH264 *const encoder = GST_VAAPI_ENCODER_H264 (base_encoder);
+  GstVaapiH264ViewReorderPool *reorder_pool;
+  GstVaapiEncPicture *pic;
+  guint i;
+
+  for (i = 0; i < encoder->num_views; i++) {
+    reorder_pool = &encoder->reorder_pools[i];
     reorder_pool->frame_index = 0;
     reorder_pool->cur_frame_num = 0;
     reorder_pool->cur_present_index = 0;
     reorder_pool->prev_frame_is_ref = FALSE;
+
+    while (!g_queue_is_empty (&reorder_pool->reorder_frame_list)) {
+      pic = (GstVaapiEncPicture *)
+          g_queue_pop_head (&reorder_pool->reorder_frame_list);
+      gst_vaapi_enc_picture_unref (pic);
+    }
+    g_queue_clear (&reorder_pool->reorder_frame_list);
   }
 
   return GST_VAAPI_ENCODER_STATUS_SUCCESS;
-
-/* ERRORS */
-error_create_coded_buffer:
-  {
-    GST_ERROR ("failed to allocate coded buffer");
-    gst_vaapi_enc_picture_unref (pic);
-    return GST_VAAPI_ENCODER_STATUS_ERROR_ALLOCATION_FAILED;
-  }
-error_encode:
-  {
-    GST_ERROR ("failed to encode frame (status = %d)", status);
-    gst_vaapi_enc_picture_unref (pic);
-    gst_vaapi_coded_buffer_proxy_unref (codedbuf_proxy);
-    return status;
-  }
 }
 
 /* Generate "codec-data" buffer */
@@ -3511,7 +3522,8 @@ gst_vaapi_encoder_h264_class (void)
   static const GstVaapiEncoderClass GstVaapiEncoderH264Class = {
     GST_VAAPI_ENCODER_CLASS_INIT (H264, h264),
     .set_property = gst_vaapi_encoder_h264_set_property,
-    .get_codec_data = gst_vaapi_encoder_h264_get_codec_data
+    .get_codec_data = gst_vaapi_encoder_h264_get_codec_data,
+    .get_pending_reordered = gst_vaapi_encoder_h264_get_pending_reordered,
   };
   return &GstVaapiEncoderH264Class;
 }
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
index aced90f9..ace2e56b 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
@@ -2134,67 +2134,73 @@ error:
   }
 }
 
-static GstVaapiEncoderStatus
-gst_vaapi_encoder_h265_flush (GstVaapiEncoder * base_encoder)
+struct _PendingIterState
+{
+  GstVaapiPictureType pic_type;
+};
+
+static gboolean
+gst_vaapi_encoder_h265_get_pending_reordered (GstVaapiEncoder * base_encoder,
+    GstVaapiEncPicture ** picture, gpointer * state)
 {
   GstVaapiEncoderH265 *const encoder = GST_VAAPI_ENCODER_H265 (base_encoder);
   GstVaapiH265ReorderPool *reorder_pool;
   GstVaapiEncPicture *pic;
-  GstVaapiCodedBufferProxy *codedbuf_proxy;
-  gboolean p_frame = TRUE;
-  GstVaapiEncoderStatus status;
+  struct _PendingIterState *iter;
+
+  g_return_val_if_fail (state, FALSE);
+
+  if (!*state) {
+    iter = g_new0 (struct _PendingIterState, 1);
+    iter->pic_type = GST_VAAPI_PICTURE_TYPE_P;
+    *state = iter;
+  } else {
+    iter = *state;
+  }
+
+  *picture = NULL;
 
   reorder_pool = &encoder->reorder_pool;
+  if (g_queue_is_empty (&reorder_pool->reorder_frame_list))
+    return FALSE;
 
-  while (!g_queue_is_empty (&reorder_pool->reorder_frame_list)) {
-    pic = NULL;
-    if (p_frame) {
-      pic = (GstVaapiEncPicture *)
-          g_queue_pop_tail (&reorder_pool->reorder_frame_list);
-      set_p_frame (pic, encoder);
-      p_frame = FALSE;
-    } else {
-      pic = (GstVaapiEncPicture *)
-          g_queue_pop_head (&reorder_pool->reorder_frame_list);
-      set_b_frame (pic, encoder);
-    }
-    g_assert (pic);
+  pic = g_queue_pop_tail (&reorder_pool->reorder_frame_list);
+  g_assert (pic);
+  if (iter->pic_type == GST_VAAPI_PICTURE_TYPE_P) {
+    set_p_frame (pic, encoder);
+    iter->pic_type = GST_VAAPI_PICTURE_TYPE_B;
+  } else if (iter->pic_type == GST_VAAPI_PICTURE_TYPE_B) {
+    set_b_frame (pic, encoder);
+  } else {
+    GST_WARNING ("Unhandled pending picture type");
+  }
 
-    if (GST_CLOCK_TIME_IS_VALID (pic->frame->pts))
-      pic->frame->pts += encoder->cts_offset;
+  if (GST_CLOCK_TIME_IS_VALID (pic->frame->pts))
+    pic->frame->pts += encoder->cts_offset;
 
-    codedbuf_proxy = gst_vaapi_encoder_create_coded_buffer (base_encoder);
-    if (!codedbuf_proxy)
-      goto error_create_coded_buffer;
+  *picture = pic;
+  return TRUE;
+}
 
-    status = gst_vaapi_encoder_h265_encode (base_encoder, pic, codedbuf_proxy);
-    if (status != GST_VAAPI_ENCODER_STATUS_SUCCESS)
-      goto error_encode;
+static GstVaapiEncoderStatus
+gst_vaapi_encoder_h265_flush (GstVaapiEncoder * base_encoder)
+{
+  GstVaapiEncoderH265 *const encoder = GST_VAAPI_ENCODER_H265 (base_encoder);
+  GstVaapiH265ReorderPool *reorder_pool;
+  GstVaapiEncPicture *pic;
 
-    gst_vaapi_coded_buffer_proxy_set_user_data (codedbuf_proxy,
-        pic, (GDestroyNotify) gst_vaapi_mini_object_unref);
-    g_async_queue_push (base_encoder->codedbuf_queue, codedbuf_proxy);
-    base_encoder->num_codedbuf_queued++;
-  }
+  reorder_pool = &encoder->reorder_pool;
   reorder_pool->frame_index = 0;
   reorder_pool->cur_present_index = 0;
 
-  return GST_VAAPI_ENCODER_STATUS_SUCCESS;
-
-  /* ERRORS */
-error_create_coded_buffer:
-  {
-    GST_ERROR ("failed to allocate coded buffer");
-    gst_vaapi_enc_picture_unref (pic);
-    return GST_VAAPI_ENCODER_STATUS_ERROR_ALLOCATION_FAILED;
-  }
-error_encode:
-  {
-    GST_ERROR ("failed to encode frame (status = %d)", status);
+  while (!g_queue_is_empty (&reorder_pool->reorder_frame_list)) {
+    pic = (GstVaapiEncPicture *)
+        g_queue_pop_head (&reorder_pool->reorder_frame_list);
     gst_vaapi_enc_picture_unref (pic);
-    gst_vaapi_coded_buffer_proxy_unref (codedbuf_proxy);
-    return status;
   }
+  g_queue_clear (&reorder_pool->reorder_frame_list);
+
+  return GST_VAAPI_ENCODER_STATUS_SUCCESS;
 }
 
 /* Generate "codec-data" buffer */
@@ -2654,7 +2660,8 @@ gst_vaapi_encoder_h265_class (void)
   static const GstVaapiEncoderClass GstVaapiEncoderH265Class = {
     GST_VAAPI_ENCODER_CLASS_INIT (H265, h265),
     .set_property = gst_vaapi_encoder_h265_set_property,
-    .get_codec_data = gst_vaapi_encoder_h265_get_codec_data
+    .get_codec_data = gst_vaapi_encoder_h265_get_codec_data,
+    .get_pending_reordered = gst_vaapi_encoder_h265_get_pending_reordered,
   };
   return &GstVaapiEncoderH265Class;
 }
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_priv.h b/gst-libs/gst/vaapi/gstvaapiencoder_priv.h
index d3253a23..843004e4 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_priv.h
@@ -362,6 +362,12 @@ struct _GstVaapiEncoderClass
 
   /* To create a secondary context for a single base encoder */
   gboolean              (*ensure_secondary_context) (GstVaapiEncoder * encoder);
+
+  /* Iterator that retrieves the pending pictures in the reordered
+   * list */
+  gboolean              (*get_pending_reordered) (GstVaapiEncoder * encoder,
+                                                  GstVaapiEncPicture ** picture,
+                                                  gpointer * state);
 };
 
 #define GST_VAAPI_ENCODER_CLASS_HOOK(codec, func) \