alpha: Some preparations for supporting more color formats

2 files changed, 171 insertions, 58 deletions

diff --git a/gst/alpha/gstalpha.c b/gst/alpha/gstalpha.c
index e4dbe3a40..1c32cb895 100644
--- a/gst/alpha/gstalpha.c
+++ b/gst/alpha/gstalpha.c
@@ -34,6 +34,43 @@
 #define M_PI  3.14159265358979323846
 #endif
 
+/* Generated by -bad/ext/cog/generate_tables */
+static const int cog_ycbcr_to_rgb_matrix_8bit_hdtv[] = {
+  298, 0, 459, -63514,
+  298, -55, -136, 19681,
+  298, 541, 0, -73988,
+};
+
+static const int cog_ycbcr_to_rgb_matrix_8bit_sdtv[] = {
+  298, 0, 409, -57068,
+  298, -100, -208, 34707,
+  298, 516, 0, -70870,
+};
+
+static const gint cog_rgb_to_ycbcr_matrix_8bit_hdtv[] = {
+  47, 157, 16, 4096,
+  -26, -87, 112, 32768,
+  112, -102, -10, 32768,
+};
+
+static const gint cog_rgb_to_ycbcr_matrix_8bit_sdtv[] = {
+  66, 129, 25, 4096,
+  -38, -74, 112, 32768,
+  112, -94, -18, 32768,
+};
+
+static const gint cog_ycbcr_sdtv_to_ycbcr_hdtv_matrix_8bit[] = {
+  256, -30, -53, 10600,
+  0, 261, 29, -4367,
+  0, 19, 262, -3289,
+};
+
+static const gint cog_ycbcr_hdtv_to_ycbcr_sdtv_matrix_8bit[] = {
+  256, 25, 49, -9536,
+  0, 253, -28, 3958,
+  0, -19, 252, 2918,
+};
+
 /* Alpha signals and args */
 enum
 {
@@ -92,6 +129,7 @@ static GstFlowReturn gst_alpha_transform (GstBaseTransform * btrans,
     GstBuffer * in, GstBuffer * out);
 
 static void gst_alpha_init_params (GstAlpha * alpha);
+static gboolean gst_alpha_set_process_function (GstAlpha * alpha);
 
 static void gst_alpha_set_property (GObject * object, guint prop_id,
     const GValue * value, GParamSpec * pspec);
@@ -235,6 +273,7 @@ gst_alpha_set_property (GObject * object, guint prop_id,
         default:
           break;
       }
+      gst_alpha_set_process_function (alpha);
       gst_alpha_init_params (alpha);
       break;
     case PROP_ALPHA:
@@ -339,30 +378,36 @@ gst_alpha_transform_caps (GstBaseTransform * btrans,
   GstStructure *structure;
   gint i;
 
-  ret = gst_caps_copy (caps);
+  ret = gst_caps_new_empty ();
 
   /* When going from the SINK pad to the src, we just need to make sure the
    * format is AYUV */
   if (direction == GST_PAD_SINK) {
-    for (i = 0; i < gst_caps_get_size (ret); i++) {
-      structure = gst_caps_get_structure (ret, i);
+    for (i = 0; i < gst_caps_get_size (caps); i++) {
+      structure = gst_structure_copy (gst_caps_get_structure (caps, i));
       gst_structure_set (structure, "format",
           GST_TYPE_FOURCC, GST_MAKE_FOURCC ('A', 'Y', 'U', 'V'), NULL);
+      gst_structure_remove_field (structure, "chroma-site");
+      gst_structure_remove_field (structure, "color-matrix");
+      gst_caps_append_structure (ret, structure);
     }
   } else {
-    GstCaps *ayuv_caps;
-
     /* In the other direction, prepend a copy of the caps with format AYUV, 
      * and set the first to I420 */
-    ayuv_caps = gst_caps_copy (ret);
 
-    for (i = 0; i < gst_caps_get_size (ret); i++) {
-      structure = gst_caps_get_structure (ret, i);
+    for (i = 0; i < gst_caps_get_size (caps); i++) {
+      structure = gst_structure_copy (gst_caps_get_structure (caps, i));
+      gst_structure_remove_field (structure, "color-matrix");
+
       gst_structure_set (structure, "format",
           GST_TYPE_FOURCC, GST_MAKE_FOURCC ('I', '4', '2', '0'), NULL);
-    }
+      gst_caps_append_structure (ret, gst_structure_copy (structure));
 
-    gst_caps_append (ret, ayuv_caps);
+      gst_structure_set (structure, "format",
+          GST_TYPE_FOURCC, GST_MAKE_FOURCC ('A', 'Y', 'U', 'V'), NULL);
+      gst_structure_remove_field (structure, "chroma-site");
+      gst_caps_append_structure (ret, structure);
+    }
   }
 
   gst_caps_do_simplify (ret);
@@ -375,19 +420,40 @@ gst_alpha_set_caps (GstBaseTransform * btrans,
     GstCaps * incaps, GstCaps * outcaps)
 {
   GstAlpha *alpha = GST_ALPHA (btrans);
+  const gchar *matrix;
+
+  GST_OBJECT_LOCK (alpha);
 
-  if (!gst_video_format_parse_caps (incaps, &alpha->format,
-          &alpha->width, &alpha->height))
+  if (!gst_video_format_parse_caps (incaps, &alpha->in_format,
+          &alpha->width, &alpha->height) ||
+      !gst_video_format_parse_caps (incaps, &alpha->out_format,
+          &alpha->width, &alpha->height)) {
+    GST_OBJECT_UNLOCK (alpha);
     return FALSE;
+  }
+
+  if (!gst_alpha_set_process_function (alpha)) {
+    GST_OBJECT_UNLOCK (alpha);
+    return FALSE;
+  }
+
+  matrix = gst_video_parse_caps_color_matrix (incaps);
+  alpha->in_sdtv = matrix ? g_str_equal (matrix, "sdtv") : TRUE;
+
+  matrix = gst_video_parse_caps_color_matrix (outcaps);
+  alpha->out_sdtv = matrix ? g_str_equal (matrix, "sdtv") : TRUE;
+
+  gst_alpha_init_params (alpha);
+  GST_OBJECT_UNLOCK (alpha);
 
   return TRUE;
 }
 
 static void
-gst_alpha_set_ayuv (guint8 * src, guint8 * dest, gint width, gint height,
-    gdouble alpha)
+gst_alpha_set_ayuv (const guint8 * src, guint8 * dest, gint width, gint height,
+    GstAlpha * alpha)
 {
-  gint s_alpha = CLAMP ((gint) (alpha * 256), 0, 256);
+  gint s_alpha = CLAMP ((gint) (alpha->alpha * 256), 0, 256);
   gint y, x;
 
   for (y = 0; y < height; y++) {
@@ -401,13 +467,13 @@ gst_alpha_set_ayuv (guint8 * src, guint8 * dest, gint width, gint height,
 }
 
 static void
-gst_alpha_set_i420 (guint8 * src, guint8 * dest, gint width, gint height,
-    gdouble alpha)
+gst_alpha_set_i420 (const guint8 * src, guint8 * dest, gint width, gint height,
+    GstAlpha * alpha)
 {
-  gint b_alpha = CLAMP ((gint) (alpha * 255), 0, 255);
-  guint8 *srcY;
-  guint8 *srcU;
-  guint8 *srcV;
+  gint b_alpha = CLAMP ((gint) (alpha->alpha * 255), 0, 255);
+  const guint8 *srcY;
+  const guint8 *srcU;
+  const guint8 *srcV;
   gint i, j;
   gint src_wrap, src_uv_wrap;
   gint y_stride, uv_stride;
@@ -537,10 +603,10 @@ chroma_keying_yuv (gint a, gint * y, guint ny, gint * u,
 }
 
 static void
-gst_alpha_chroma_key_ayuv (guint8 * src, guint8 * dest, gint width, gint height,
-    GstAlpha * alpha)
+gst_alpha_chroma_key_ayuv (const guint8 * src, guint8 * dest, gint width,
+    gint height, GstAlpha * alpha)
 {
-  guint8 *src1;
+  const guint8 *src1;
   guint8 *dest1;
   gint i, j;
   gint a, y, u, v;
@@ -578,7 +644,8 @@ gst_alpha_chroma_key_ayuv (guint8 * src, guint8 * dest, gint width, gint height,
 
 static inline void
 gst_alpha_chromakey_row_i420 (GstAlpha * alpha, guint8 * dest1, guint8 * dest2,
-    guint8 * srcY1, guint8 * srcY2, guint8 * srcU, guint8 * srcV, gint width)
+    const guint8 * srcY1, const guint8 * srcY2, const guint8 * srcU,
+    const guint8 * srcV, gint width)
 {
   gint xpos;
   gint a, a2, y[4], u, v;
@@ -624,10 +691,10 @@ gst_alpha_chromakey_row_i420 (GstAlpha * alpha, guint8 * dest1, guint8 * dest2,
 }
 
 static void
-gst_alpha_chroma_key_i420 (guint8 * src, guint8 * dest, gint width, gint height,
-    GstAlpha * alpha)
+gst_alpha_chroma_key_i420 (const guint8 * src, guint8 * dest, gint width,
+    gint height, GstAlpha * alpha)
 {
-  guint8 *srcY1, *srcY2, *srcU, *srcV;
+  const guint8 *srcY1, *srcY2, *srcU, *srcV;
   guint8 *dest1, *dest2;
   gint ypos;
   gint dest_stride, src_y_stride, src_uv_stride;
@@ -673,16 +740,28 @@ gst_alpha_init_params (GstAlpha * alpha)
   float kgl;
   float tmp;
   float tmp1, tmp2;
+  const gint *matrix;
+
+  matrix =
+      (alpha->out_sdtv) ? cog_rgb_to_ycbcr_matrix_8bit_sdtv :
+      cog_rgb_to_ycbcr_matrix_8bit_hdtv;
 
   alpha->y =
-      0.257 * alpha->target_r + 0.504 * alpha->target_g +
-      0.098 * alpha->target_b;
+      (matrix[0] * ((gint) alpha->target_r) +
+      matrix[1] * ((gint) alpha->target_g) +
+      matrix[2] * ((gint) alpha->target_b) + matrix[3]) >> 8;
+  /* Cb,Cr without offset here because the chroma keying
+   * works with them being in range [-128,127]
+   */
   tmp1 =
-      -0.148 * alpha->target_r - 0.291 * alpha->target_g +
-      0.439 * alpha->target_b;
+      (matrix[4] * ((gint) alpha->target_r) +
+      matrix[5] * ((gint) alpha->target_g) +
+      matrix[6] * ((gint) alpha->target_b)) >> 8;
   tmp2 =
-      0.439 * alpha->target_r - 0.368 * alpha->target_g -
-      0.071 * alpha->target_b;
+      (matrix[8] * ((gint) alpha->target_r) +
+      matrix[9] * ((gint) alpha->target_g) +
+      matrix[10] * ((gint) alpha->target_b)) >> 8;
+
   kgl = sqrt (tmp1 * tmp1 + tmp2 * tmp2);
   alpha->cb = 127 * (tmp1 / kgl);
   alpha->cr = 127 * (tmp2 / kgl);
@@ -706,6 +785,53 @@ gst_alpha_init_params (GstAlpha * alpha)
 }
 
 static gboolean
+gst_alpha_set_process_function (GstAlpha * alpha)
+{
+  alpha->process = NULL;
+  switch (alpha->method) {
+    case ALPHA_METHOD_SET:
+      switch (alpha->out_format) {
+        case GST_VIDEO_FORMAT_AYUV:
+          switch (alpha->in_format) {
+            case GST_VIDEO_FORMAT_AYUV:
+              alpha->process = gst_alpha_set_ayuv;
+              break;
+            case GST_VIDEO_FORMAT_I420:
+              alpha->process = gst_alpha_set_i420;
+              break;
+            default:
+              break;
+          }
+        default:
+          break;
+      }
+      break;
+    case ALPHA_METHOD_GREEN:
+    case ALPHA_METHOD_BLUE:
+    case ALPHA_METHOD_CUSTOM:
+      switch (alpha->out_format) {
+        case GST_VIDEO_FORMAT_AYUV:
+          switch (alpha->in_format) {
+            case GST_VIDEO_FORMAT_AYUV:
+              alpha->process = gst_alpha_chroma_key_ayuv;
+              break;
+            case GST_VIDEO_FORMAT_I420:
+              alpha->process = gst_alpha_chroma_key_i420;
+              break;
+            default:
+              break;
+          }
+        default:
+          break;
+      }
+      break;
+    default:
+      break;
+  }
+  return alpha->process != NULL;
+}
+
+static gboolean
 gst_alpha_start (GstBaseTransform * btrans)
 {
   GstAlpha *alpha = GST_ALPHA (btrans);
@@ -734,30 +860,14 @@ gst_alpha_transform (GstBaseTransform * btrans, GstBuffer * in, GstBuffer * out)
     gst_object_sync_values (G_OBJECT (alpha), timestamp);
 
   GST_OBJECT_LOCK (alpha);
-  switch (alpha->method) {
-    case ALPHA_METHOD_SET:
-      if (alpha->format == GST_VIDEO_FORMAT_AYUV) {
-        gst_alpha_set_ayuv (GST_BUFFER_DATA (in),
-            GST_BUFFER_DATA (out), width, height, alpha->alpha);
-      } else {
-        gst_alpha_set_i420 (GST_BUFFER_DATA (in),
-            GST_BUFFER_DATA (out), width, height, alpha->alpha);
-      }
-      break;
-    case ALPHA_METHOD_GREEN:
-    case ALPHA_METHOD_BLUE:
-    case ALPHA_METHOD_CUSTOM:
-      if (alpha->format == GST_VIDEO_FORMAT_AYUV) {
-        gst_alpha_chroma_key_ayuv (GST_BUFFER_DATA (in),
-            GST_BUFFER_DATA (out), width, height, alpha);
-      } else {
-        gst_alpha_chroma_key_i420 (GST_BUFFER_DATA (in),
-            GST_BUFFER_DATA (out), width, height, alpha);
-      }
-      break;
-    default:
-      break;
+  if (G_UNLIKELY (!alpha->process)) {
+    GST_ERROR_OBJECT (alpha, "Not negotiated yet");
+    GST_OBJECT_UNLOCK (alpha);
+    return GST_FLOW_NOT_NEGOTIATED;
   }
+
+  alpha->process (GST_BUFFER_DATA (in),
+      GST_BUFFER_DATA (out), width, height, alpha);
   GST_OBJECT_UNLOCK (alpha);
 
   return GST_FLOW_OK;
diff --git a/gst/alpha/gstalpha.h b/gst/alpha/gstalpha.h
index 5cc99fd75..2b116dba7 100644
--- a/gst/alpha/gstalpha.h
+++ b/gst/alpha/gstalpha.h
@@ -61,8 +61,9 @@ struct _GstAlpha
   GstVideoFilter parent;
 
   /* caps */
-  GstVideoFormat format;
+  GstVideoFormat in_format, out_format;
   gint width, height;
+  gboolean in_sdtv, out_sdtv;
 
   gdouble alpha;
 
@@ -87,6 +88,8 @@ struct _GstAlpha
   guint8 accept_angle_ctg;
   guint8 one_over_kc;
   guint8 kfgy_scale;
+
+  void (*process) (const guint8 *src, guint8 *dest, gint width, gint height, GstAlpha *alpha);
 };
 
 struct _GstAlphaClass