libs: encode: merge all possible surface formats

When the function gst_vaapi_encoder_get_surface_formats() was added
it was under the assumption that any VA profile of the specific codec
supported the same format colors. But it is not, for example the
profiles that support 10bit formats.

In other words, different VA profiles of a same codec may support
different color formats in their upload surfaces.

In order to expose all the possible color formats, if no profile is
specified via source caps, or if the encoder doesn't have yet a
context, all the possible VA profiles for the specific codec are
iterated and their color formats are merged.

https://bugzilla.gnome.org/show_bug.cgi?id=771291

1 files changed, 70 insertions, 6 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder.c b/gst-libs/gst/vaapi/gstvaapiencoder.c
index cfbdfc84..327e2155 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder.c
@@ -1178,6 +1178,50 @@ create_test_context_config (GstVaapiEncoder * encoder, GstVaapiProfile profile)
   return ctxt;
 }
 
+static GArray *
+get_profile_surface_formats (GstVaapiEncoder * encoder, GstVaapiProfile profile)
+{
+  GstVaapiContext *ctxt;
+  GArray *formats;
+
+  ctxt = create_test_context_config (encoder, profile);
+  if (!ctxt)
+    return NULL;
+  formats = gst_vaapi_context_get_surface_formats (ctxt);
+  gst_vaapi_object_unref (ctxt);
+  return formats;
+}
+
+static gboolean
+merge_profile_surface_formats (GstVaapiEncoder * encoder,
+    GstVaapiProfile profile, GArray * formats)
+{
+  GArray *surface_fmts;
+  guint i, j;
+  GstVideoFormat fmt, sfmt;
+
+  if (profile == GST_VAAPI_PROFILE_UNKNOWN)
+    return FALSE;
+
+  surface_fmts = get_profile_surface_formats (encoder, profile);
+  if (!surface_fmts)
+    return FALSE;
+
+  for (i = 0; i < surface_fmts->len; i++) {
+    sfmt = g_array_index (surface_fmts, GstVideoFormat, i);
+    for (j = 0; j < formats->len; j++) {
+      fmt = g_array_index (formats, GstVideoFormat, j);
+      if (fmt == sfmt)
+        break;
+    }
+    if (j >= formats->len)
+      g_array_append_val (formats, sfmt);
+  }
+
+  g_array_unref (surface_fmts);
+  return TRUE;
+}
+
 /**
  * gst_vaapi_encoder_get_surface_formats:
  * @encoder: a #GstVaapiEncoder instances
@@ -1190,14 +1234,34 @@ GArray *
 gst_vaapi_encoder_get_surface_formats (GstVaapiEncoder * encoder,
     GstVaapiProfile profile)
 {
-  GstVaapiContext *ctxt;
-  GArray *formats;
+  const GstVaapiEncoderClassData *const cdata =
+      GST_VAAPI_ENCODER_GET_CLASS (encoder)->class_data;
+  GArray *profiles, *formats;
+  guint i;
 
-  ctxt = create_test_context_config (encoder, profile);
-  if (!ctxt)
+  if (profile || encoder->context)
+    return get_profile_surface_formats (encoder, profile);
+
+  /* no specific context neither specific profile, let's iterate among
+   * the codec's profiles */
+  profiles = gst_vaapi_display_get_encode_profiles (encoder->display);
+  if (!profiles)
     return NULL;
-  formats = gst_vaapi_context_get_surface_formats (ctxt);
-  gst_vaapi_object_unref (ctxt);
+
+  formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
+  for (i = 0; i < profiles->len; i++) {
+    profile = g_array_index (profiles, GstVaapiProfile, i);
+    if (gst_vaapi_profile_get_codec (profile) == cdata->codec) {
+      if (!merge_profile_surface_formats (encoder, profile, formats)) {
+        g_array_unref (formats);
+        formats = NULL;
+        break;
+      }
+    }
+  }
+
+  g_array_unref (profiles);
+
   return formats;
 }