Implement decoder reset on flush, rather than recreating

Clear decoders out on a flush but keep the same instance,
rather than completely recreating them. That avoids
unecessarily freeing and recreating surface pools
and contexts, which can be quite expensive

https://bugzilla.gnome.org/show_bug.cgi?id=781142

7 files changed, 103 insertions, 14 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapidecoder.c b/gst-libs/gst/vaapi/gstvaapidecoder.c
index 6af1372f..ce9c065b 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder.c
@@ -41,6 +41,26 @@
 static void drop_frame (GstVaapiDecoder * decoder, GstVideoCodecFrame * frame);
 
 static void
+parser_state_reset (GstVaapiParserState * ps)
+{
+
+  if (ps->input_adapter)
+    gst_adapter_clear (ps->input_adapter);
+  if (ps->output_adapter)
+    gst_adapter_clear (ps->output_adapter);
+  ps->current_adapter = NULL;
+
+  if (ps->next_unit_pending) {
+    gst_vaapi_decoder_unit_clear (&ps->next_unit);
+    ps->next_unit_pending = FALSE;
+  }
+
+  ps->current_frame_number = 0;
+  ps->input_offset1 = ps->input_offset2 = 0;
+  ps->at_eos = FALSE;
+}
+
+static void
 parser_state_finalize (GstVaapiParserState * ps)
 {
   if (ps->input_adapter) {
@@ -266,16 +286,6 @@ do_decode (GstVaapiDecoder * decoder, GstVideoCodecFrame * base_frame)
   return status;
 }
 
-static inline GstVaapiDecoderStatus
-do_flush (GstVaapiDecoder * decoder)
-{
-  GstVaapiDecoderClass *const klass = GST_VAAPI_DECODER_GET_CLASS (decoder);
-
-  if (klass->flush)
-    return klass->flush (decoder);
-  return GST_VAAPI_DECODER_STATUS_SUCCESS;
-}
-
 static GstVaapiDecoderStatus
 decode_step (GstVaapiDecoder * decoder)
 {
@@ -1029,13 +1039,69 @@ gst_vaapi_decoder_decode (GstVaapiDecoder * decoder, GstVideoCodecFrame * frame)
   return do_decode (decoder, frame);
 }
 
+/* This function really marks the end of input,
+ * so that the decoder will drain out any pending
+ * frames on calls to gst_vaapi_decoder_get_frame_with_timeout() */
 GstVaapiDecoderStatus
 gst_vaapi_decoder_flush (GstVaapiDecoder * decoder)
 {
+  GstVaapiDecoderClass *klass;
+
+  g_return_val_if_fail (decoder != NULL,
+      GST_VAAPI_DECODER_STATUS_ERROR_INVALID_PARAMETER);
+
+  klass = GST_VAAPI_DECODER_GET_CLASS (decoder);
+
+  if (klass->flush)
+    return klass->flush (decoder);
+
+  return GST_VAAPI_DECODER_STATUS_SUCCESS;
+}
+
+/* Reset the decoder instance to a clean state,
+ * clearing any pending decode state, without
+ * reallocating the entire decoder */
+GstVaapiDecoderStatus
+gst_vaapi_decoder_reset (GstVaapiDecoder * decoder)
+{
+  GstVaapiDecoderClass *klass;
+  GstVaapiDecoderStatus ret = GST_VAAPI_DECODER_STATUS_SUCCESS;
+
   g_return_val_if_fail (decoder != NULL,
       GST_VAAPI_DECODER_STATUS_ERROR_INVALID_PARAMETER);
 
-  return do_flush (decoder);
+  klass = GST_VAAPI_DECODER_GET_CLASS (decoder);
+
+  GST_DEBUG ("Resetting decoder");
+
+  if (klass->reset) {
+    ret = klass->reset (decoder);
+  } else {
+    if (klass->destroy)
+      klass->destroy (decoder);
+    if (klass->create)
+      if (!klass->create (decoder))
+        ret = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
+  }
+
+  if (ret != GST_VAAPI_DECODER_STATUS_SUCCESS)
+    return ret;
+
+  /* Clear any buffers and frame in the queues */
+  {
+    GstVideoCodecFrame *frame;
+    GstBuffer *buffer;
+
+    while ((frame = g_async_queue_try_pop (decoder->frames)) != NULL)
+      gst_video_codec_frame_unref (frame);
+
+    while ((buffer = g_async_queue_try_pop (decoder->buffers)) != NULL)
+      gst_buffer_unref (buffer);
+  }
+
+  parser_state_reset (&decoder->parser_state);
+
+  return GST_VAAPI_DECODER_STATUS_SUCCESS;
 }
 
 GstVaapiDecoderStatus
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder.h b/gst-libs/gst/vaapi/gstvaapidecoder.h
index a8ca5aa9..8b442b9f 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder.h
+++ b/gst-libs/gst/vaapi/gstvaapidecoder.h
@@ -133,6 +133,9 @@ GstVaapiDecoderStatus
 gst_vaapi_decoder_flush (GstVaapiDecoder * decoder);
 
 GstVaapiDecoderStatus
+gst_vaapi_decoder_reset (GstVaapiDecoder * decoder);
+
+GstVaapiDecoderStatus
 gst_vaapi_decoder_check_status (GstVaapiDecoder * decoder);
 
 gboolean
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
index d4bd6dda..f87ac308 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
@@ -1221,10 +1221,11 @@ gst_vaapi_decoder_h264_destroy (GstVaapiDecoder * base_decoder)
   guint i;
 
   gst_vaapi_decoder_h264_close (decoder);
+  priv->is_opened = FALSE;
 
   g_free (priv->dpb);
   priv->dpb = NULL;
-  priv->dpb_size = 0;
+  priv->dpb_size_max = priv->dpb_size = 0;
 
   g_free (priv->prev_ref_frames);
   priv->prev_ref_frames = NULL;
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
index 2675f79a..15be632b 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
@@ -957,6 +957,8 @@ gst_vaapi_decoder_h265_close (GstVaapiDecoderH265 * decoder)
     gst_h265_parser_free (priv->parser);
     priv->parser = NULL;
   }
+
+  priv->is_opened = FALSE;
 }
 
 static gboolean
@@ -982,7 +984,8 @@ gst_vaapi_decoder_h265_destroy (GstVaapiDecoder * base_decoder)
   gst_vaapi_decoder_h265_close (decoder);
   g_free (priv->dpb);
   priv->dpb = NULL;
-  priv->dpb_size = 0;
+  priv->dpb_count = priv->dpb_size_max = priv->dpb_size = 0;
+
   for (i = 0; i < G_N_ELEMENTS (priv->pps); i++)
     gst_vaapi_parser_info_h265_replace (&priv->pps[i], NULL);
   gst_vaapi_parser_info_h265_replace (&priv->active_pps, NULL);
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_mpeg2.c b/gst-libs/gst/vaapi/gstvaapidecoder_mpeg2.c
index cb05b6e7..8d459102 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_mpeg2.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_mpeg2.c
@@ -317,6 +317,8 @@ gst_vaapi_decoder_mpeg2_close (GstVaapiDecoderMpeg2 * decoder)
   priv->state = 0;
 
   gst_vaapi_dpb_replace (&priv->dpb, NULL);
+
+  priv->is_opened = FALSE;
 }
 
 static gboolean
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_priv.h b/gst-libs/gst/vaapi/gstvaapidecoder_priv.h
index 3473315f..c80db0f9 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_priv.h
@@ -225,6 +225,7 @@ struct _GstVaapiDecoderClass
       struct _GstVaapiDecoderUnit * unit);
   GstVaapiDecoderStatus (*end_frame) (GstVaapiDecoder * decoder);
   GstVaapiDecoderStatus (*flush) (GstVaapiDecoder * decoder);
+  GstVaapiDecoderStatus (*reset) (GstVaapiDecoder * decoder);
   GstVaapiDecoderStatus (*decode_codec_data) (GstVaapiDecoder * decoder,
       const guchar * buf, guint buf_size);
 };
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_vc1.c b/gst-libs/gst/vaapi/gstvaapidecoder_vc1.c
index 256f017c..7fc0244b 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_vc1.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_vc1.c
@@ -67,7 +67,6 @@ struct _GstVaapiDecoderVC1Private
   guint8 rndctrl;
   guint rbdu_buffer_size;
   guint is_opened:1;
-  guint is_first_field:1;
   guint has_codec_data:1;
   guint has_entrypoint:1;
   guint size_changed:1;
@@ -134,6 +133,7 @@ gst_vaapi_decoder_vc1_close (GstVaapiDecoderVC1 * decoder)
     gst_vc1_bitplanes_free (priv->bitplanes);
     priv->bitplanes = NULL;
   }
+  priv->is_opened = FALSE;
 }
 
 static gboolean
@@ -150,6 +150,11 @@ gst_vaapi_decoder_vc1_open (GstVaapiDecoderVC1 * decoder)
   priv->bitplanes = gst_vc1_bitplanes_new ();
   if (!priv->bitplanes)
     return FALSE;
+
+  memset (&priv->seq_hdr, 0, sizeof (GstVC1SeqHdr));
+  memset (&priv->entrypoint_hdr, 0, sizeof (GstVC1EntryPointHdr));
+  memset (&priv->frame_hdr, 0, sizeof (GstVC1FrameHdr));
+
   return TRUE;
 }
 
@@ -174,8 +179,13 @@ gst_vaapi_decoder_vc1_create (GstVaapiDecoder * base_decoder)
   GstVaapiDecoderVC1 *const decoder = GST_VAAPI_DECODER_VC1_CAST (base_decoder);
   GstVaapiDecoderVC1Private *const priv = &decoder->priv;
 
+  priv->has_codec_data = priv->has_entrypoint =
+      priv->size_changed = priv->profile_changed =
+      priv->closed_entry = priv->broken_link = FALSE;
+
   priv->profile = (GstVaapiProfile) 0;
   priv->rndctrl = 0;
+  priv->width = priv->height = 0;
   return TRUE;
 }
 
@@ -1357,6 +1367,9 @@ gst_vaapi_decoder_vc1_start_frame (GstVaapiDecoder * base_decoder,
     GST_ERROR ("failed to reset context");
     return status;
   }
+  status = ensure_decoder (decoder);
+  if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
+    return status;
 
   picture = GST_VAAPI_PICTURE_NEW (VC1, decoder);
   if (!picture) {