msdk: Reorganize context preparation code

Split it out into a separate function with early exits to make the
flow clearer, and document what the function is doing clearly.
No functional changes.

5 files changed, 129 insertions, 71 deletions

diff --git a/sys/msdk/gstmsdkcontextutil.c b/sys/msdk/gstmsdkcontextutil.c
index 8e0d0cb88..0e25080ef 100644
--- a/sys/msdk/gstmsdkcontextutil.c
+++ b/sys/msdk/gstmsdkcontextutil.c
@@ -147,7 +147,7 @@ found:
 }
 
 gboolean
-gst_msdk_context_prepare (GstElement * element, GstMsdkContext ** context_ptr)
+gst_msdk_context_find (GstElement * element, GstMsdkContext ** context_ptr)
 {
   g_return_val_if_fail (element != NULL, FALSE);
   g_return_val_if_fail (context_ptr != NULL, FALSE);
diff --git a/sys/msdk/gstmsdkcontextutil.h b/sys/msdk/gstmsdkcontextutil.h
index 294622246..4792006da 100644
--- a/sys/msdk/gstmsdkcontextutil.h
+++ b/sys/msdk/gstmsdkcontextutil.h
@@ -44,7 +44,7 @@
 G_BEGIN_DECLS
 
 gboolean
-gst_msdk_context_prepare (GstElement * element, GstMsdkContext ** context_ptr);
+gst_msdk_context_find (GstElement * element, GstMsdkContext ** context_ptr);
 
 gboolean
 gst_msdk_context_get_context (GstContext * context, GstMsdkContext ** msdk_context);
diff --git a/sys/msdk/gstmsdkdec.c b/sys/msdk/gstmsdkdec.c
index 739233a38..03a4fbb27 100644
--- a/sys/msdk/gstmsdkdec.c
+++ b/sys/msdk/gstmsdkdec.c
@@ -642,39 +642,54 @@ gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task)
 }
 
 static gboolean
-gst_msdkdec_start (GstVideoDecoder * decoder)
+gst_msdkdec_context_prepare (GstMsdkDec * thiz)
 {
-  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
-
-  if (gst_msdk_context_prepare (GST_ELEMENT_CAST (thiz), &thiz->context)) {
-    GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
-        thiz->context);
-    thiz->use_video_memory = TRUE;
+  /* Try to find an existing context from the pipeline. This may (indirectly)
+   * invoke gst_msdkdec_set_context, which will set thiz->context. */
+  if (!gst_msdk_context_find (GST_ELEMENT_CAST (thiz), &thiz->context))
+    return FALSE;
 
-    if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_DECODER) {
-      GstMsdkContext *parent_context, *msdk_context;
+  thiz->use_video_memory = TRUE;
 
-      parent_context = thiz->context;
-      msdk_context = gst_msdk_context_new_with_parent (parent_context);
+  GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
+      thiz->context);
 
-      if (!msdk_context) {
-        GST_ERROR_OBJECT (thiz, "Context creation failed");
-        return FALSE;
-      }
+  if (!gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_DECODER) {
+    gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_DECODER);
+    return TRUE;
+  }
 
-      thiz->context = msdk_context;
+  /* Found an existing context that's already being used as a decoder, clone
+   * the MFX session inside it to create a new one */
+  {
+    GstMsdkContext *parent_context, *msdk_context;
 
-      gst_msdk_context_add_shared_async_depth (thiz->context,
-          gst_msdk_context_get_shared_async_depth (parent_context));
-      gst_object_unref (parent_context);
+    GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT " with "
+        "joined session", thiz->context);
+    parent_context = thiz->context;
+    msdk_context = gst_msdk_context_new_with_parent (parent_context);
 
-      GST_INFO_OBJECT (thiz,
-          "Creating new context %" GST_PTR_FORMAT " with joined session",
-          thiz->context);
-    } else {
-      gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_DECODER);
+    if (!msdk_context) {
+      GST_ERROR_OBJECT (thiz, "Failed to create a context with parent context "
+          "as %" GST_PTR_FORMAT, parent_context);
+      return FALSE;
     }
-  } else {
+
+    thiz->context = msdk_context;
+    gst_msdk_context_add_shared_async_depth (thiz->context,
+        gst_msdk_context_get_shared_async_depth (parent_context));
+    gst_object_unref (parent_context);
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_msdkdec_start (GstVideoDecoder * decoder)
+{
+  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
+
+  if (!gst_msdkdec_context_prepare (thiz)) {
     if (!gst_msdk_context_ensure_context (GST_ELEMENT_CAST (thiz),
             thiz->hardware, GST_MSDK_JOB_DECODER))
       return FALSE;
diff --git a/sys/msdk/gstmsdkenc.c b/sys/msdk/gstmsdkenc.c
index e4d9ba736..f97e35cee 100644
--- a/sys/msdk/gstmsdkenc.c
+++ b/sys/msdk/gstmsdkenc.c
@@ -1395,35 +1395,57 @@ invalid_frame:
 }
 
 static gboolean
-gst_msdkenc_start (GstVideoEncoder * encoder)
+gst_msdkenc_context_prepare (GstMsdkEnc * thiz)
 {
-  GstMsdkEnc *thiz = GST_MSDKENC (encoder);
-
-  if (gst_msdk_context_prepare (GST_ELEMENT_CAST (thiz), &thiz->context)) {
-    GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
-        thiz->context);
+  /* Try to find an existing context from the pipeline. This may (indirectly)
+   * invoke gst_msdkenc_set_context, which will set thiz->context. */
+  if (!gst_msdk_context_find (GST_ELEMENT_CAST (thiz), &thiz->context))
+    return FALSE;
 
-    if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_ENCODER) {
-      GstMsdkContext *parent_context, *msdk_context;
+  GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
+      thiz->context);
 
-      parent_context = thiz->context;
-      msdk_context = gst_msdk_context_new_with_parent (parent_context);
+  /* Check GST_MSDK_JOB_VPP and GST_MSDK_JOB_ENCODER together to avoid sharing context
+   * between VPP and ENCODER
+   * Example:
+   * gst-launch-1.0 videotestsrc ! video/x-raw,format=I420 ! msdkh264enc ! \
+   * msdkh264dec ! msdkvpp ! video/x-raw,format=YUY2 ! fakesink
+   */
+  if (!gst_msdk_context_get_job_type (thiz->context) & (GST_MSDK_JOB_VPP |
+          GST_MSDK_JOB_ENCODER)) {
+    gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_ENCODER);
+    return TRUE;
+  }
 
-      if (!msdk_context) {
-        GST_ERROR_OBJECT (thiz, "Context creation failed");
-        return FALSE;
-      }
+  /* Found an existing context that's already being used as an encoder, clone
+   * the MFX session inside it to create a new one */
+  {
+    GstMsdkContext *parent_context, *msdk_context;
 
-      thiz->context = msdk_context;
-      gst_object_unref (parent_context);
+    GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT " with "
+        "joined session", thiz->context);
+    parent_context = thiz->context;
+    msdk_context = gst_msdk_context_new_with_parent (parent_context);
 
-      GST_INFO_OBJECT (thiz,
-          "Creating new context %" GST_PTR_FORMAT " with joined session",
-          thiz->context);
-    } else {
-      gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_ENCODER);
+    if (!msdk_context) {
+      GST_ERROR_OBJECT (thiz, "Failed to create a context with parent context "
+          "as %" GST_PTR_FORMAT, parent_context);
+      return FALSE;
     }
-  } else {
+
+    thiz->context = msdk_context;
+    gst_object_unref (parent_context);
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_msdkenc_start (GstVideoEncoder * encoder)
+{
+  GstMsdkEnc *thiz = GST_MSDKENC (encoder);
+
+  if (!gst_msdkenc_context_prepare (thiz)) {
     if (!gst_msdk_context_ensure_context (GST_ELEMENT_CAST (thiz),
             thiz->hardware, GST_MSDK_JOB_ENCODER))
       return FALSE;
diff --git a/sys/msdk/gstmsdkvpp.c b/sys/msdk/gstmsdkvpp.c
index 125e67c42..2c7e07c40 100644
--- a/sys/msdk/gstmsdkvpp.c
+++ b/sys/msdk/gstmsdkvpp.c
@@ -154,35 +154,56 @@ gst_msdkvpp_add_extra_param (GstMsdkVPP * thiz, mfxExtBuffer * param)
 }
 
 static gboolean
-ensure_context (GstBaseTransform * trans)
+gst_msdkvpp_context_prepare (GstMsdkVPP * thiz)
 {
-  GstMsdkVPP *thiz = GST_MSDKVPP (trans);
-
-  if (gst_msdk_context_prepare (GST_ELEMENT_CAST (thiz), &thiz->context)) {
-    GST_INFO_OBJECT (thiz, "Found context from neighbour %" GST_PTR_FORMAT,
-        thiz->context);
+  /* Try to find an existing context from the pipeline. This may (indirectly)
+   * invoke gst_msdkvpp_set_context, which will set thiz->context. */
+  if (!gst_msdk_context_find (GST_ELEMENT_CAST (thiz), &thiz->context))
+    return FALSE;
 
-    if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_VPP) {
-      GstMsdkContext *parent_context, *msdk_context;
+  GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
+      thiz->context);
 
-      parent_context = thiz->context;
-      msdk_context = gst_msdk_context_new_with_parent (parent_context);
+  /* Check GST_MSDK_JOB_VPP and GST_MSDK_JOB_ENCODER together to avoid sharing context
+   * between VPP and ENCODER
+   * Example:
+   * gst-launch-1.0 videotestsrc ! msdkvpp ! video/x-raw,format=YUY2 ! msdkh264enc ! fakesink
+   */
+  if (!gst_msdk_context_get_job_type (thiz->context) & (GST_MSDK_JOB_VPP |
+          GST_MSDK_JOB_ENCODER)) {
+    gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_VPP);
+    return TRUE;
+  }
 
-      if (!msdk_context) {
-        GST_ERROR_OBJECT (thiz, "Context creation failed");
-        return FALSE;
-      }
+  /* Found an existing context that's already being used as VPP, so clone the
+   * MFX session inside it to create a new one */
+  {
+    GstMsdkContext *parent_context, *msdk_context;
 
-      thiz->context = msdk_context;
-      gst_object_unref (parent_context);
+    GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT " with "
+        "joined session", thiz->context);
+    parent_context = thiz->context;
+    msdk_context = gst_msdk_context_new_with_parent (parent_context);
 
-      GST_INFO_OBJECT (thiz,
-          "Creating new context %" GST_PTR_FORMAT " with joined session",
-          thiz->context);
-    } else {
-      gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_VPP);
+    if (!msdk_context) {
+      GST_ERROR_OBJECT (thiz, "Failed to create a context with parent context "
+          "as %" GST_PTR_FORMAT, parent_context);
+      return FALSE;
     }
-  } else {
+
+    thiz->context = msdk_context;
+    gst_object_unref (parent_context);
+  }
+
+  return TRUE;
+}
+
+static gboolean
+ensure_context (GstBaseTransform * trans)
+{
+  GstMsdkVPP *thiz = GST_MSDKVPP (trans);
+
+  if (!gst_msdkvpp_context_prepare (thiz)) {
     if (!gst_msdk_context_ensure_context (GST_ELEMENT_CAST (thiz),
             thiz->hardware, GST_MSDK_JOB_VPP))
       return FALSE;