videoscale: Try harder to keep the DAR if possible

Fixes bug #371108.

1 files changed, 354 insertions, 83 deletions

diff --git a/gst/videoscale/gstvideoscale.c b/gst/videoscale/gstvideoscale.c
index a6f1db880..06ea8b18f 100644
--- a/gst/videoscale/gstvideoscale.c
+++ b/gst/videoscale/gstvideoscale.c
@@ -54,6 +54,8 @@
 
 #include <string.h>
 
+#include <math.h>
+
 #include <gst/video/video.h>
 #include <liboil/liboil.h>
 
@@ -338,7 +340,8 @@ gst_video_scale_transform_caps (GstBaseTransform * trans,
   /* if pixel aspect ratio, make a range of it */
   if (gst_structure_has_field (structure, "pixel-aspect-ratio")) {
     gst_structure_set (structure,
-        "pixel-aspect-ratio", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
+        "pixel-aspect-ratio", GST_TYPE_FRACTION_RANGE, 1, G_MAXINT, G_MAXINT, 1,
+        NULL);
   }
   gst_caps_merge_structure (ret, gst_structure_copy (structure));
   gst_structure_free (structure);
@@ -491,7 +494,6 @@ gst_video_scale_set_caps (GstBaseTransform * trans, GstCaps * in, GstCaps * out)
   const GstStructure *ins, *outs;
   gint from_dar_n, from_dar_d, to_dar_n, to_dar_d;
   gint from_par_n, from_par_d, to_par_n, to_par_d;
-  gint gcd;
 
   ret = parse_caps (in, &videoscale->format, &videoscale->from_width,
       &videoscale->from_height, &videoscale->interlaced);
@@ -529,25 +531,20 @@ gst_video_scale_set_caps (GstBaseTransform * trans, GstCaps * in, GstCaps * out)
           &to_par_d))
     to_par_n = to_par_d = 1;
 
-  if (gst_util_fraction_multiply (videoscale->from_width,
+  if (!gst_util_fraction_multiply (videoscale->from_width,
           videoscale->from_height, from_par_n, from_par_d, &from_dar_n,
           &from_dar_d)) {
-    gcd = gst_util_greatest_common_divisor (from_dar_n, from_dar_d);
-    from_dar_n /= gcd;
-    from_dar_d /= gcd;
-  } else {
     from_dar_n = from_dar_d = -1;
   }
 
-  if (gst_util_fraction_multiply (videoscale->to_width, videoscale->to_height,
+  if (!gst_util_fraction_multiply (videoscale->to_width, videoscale->to_height,
           to_par_n, to_par_d, &to_dar_n, &to_dar_d)) {
-    gcd = gst_util_greatest_common_divisor (to_dar_n, to_dar_d);
-    to_dar_n /= gcd;
-    to_dar_d /= gcd;
-  } else {
     to_dar_n = to_dar_d = -1;
   }
 
+  if (to_dar_n != from_dar_n || to_dar_d != from_dar_d)
+    GST_WARNING_OBJECT (videoscale, "Can't keep DAR!");
+
   GST_DEBUG_OBJECT (videoscale, "from=%dx%d (par=%d/%d dar=%d/%d), size %d "
       "-> to=%dx%d (par=%d/%d dar=%d/%d), size %d",
       videoscale->from_width, videoscale->from_height, from_par_n, from_par_d,
@@ -585,6 +582,8 @@ gst_video_scale_fixate_caps (GstBaseTransform * base, GstPadDirection direction,
 {
   GstStructure *ins, *outs;
   const GValue *from_par, *to_par;
+  GValue fpar = { 0, }, tpar = {
+  0,};
 
   g_return_if_fail (gst_caps_is_fixed (caps));
 
@@ -597,11 +596,46 @@ gst_video_scale_fixate_caps (GstBaseTransform * base, GstPadDirection direction,
   from_par = gst_structure_get_value (ins, "pixel-aspect-ratio");
   to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
 
+  /* If we're fixating from the sinkpad we always set the PAR and
+   * assume that missing PAR on the sinkpad means 1/1 and
+   * missing PAR on the srcpad means undefined
+   */
+  if (direction == GST_PAD_SINK) {
+    if (!from_par) {
+      g_value_init (&fpar, GST_TYPE_FRACTION);
+      gst_value_set_fraction (&fpar, 1, 1);
+      from_par = &fpar;
+    }
+    if (!to_par) {
+      g_value_init (&tpar, GST_TYPE_FRACTION_RANGE);
+      gst_value_set_fraction_range_full (&tpar, 1, G_MAXINT, G_MAXINT, 1);
+      to_par = &tpar;
+
+      gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION_RANGE, 1,
+          G_MAXINT, G_MAXINT, 1, NULL);
+    }
+  } else {
+    if (!to_par) {
+      g_value_init (&tpar, GST_TYPE_FRACTION);
+      gst_value_set_fraction (&tpar, 1, 1);
+      to_par = &tpar;
+
+      gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
+          NULL);
+    }
+    if (!from_par) {
+      g_value_init (&fpar, GST_TYPE_FRACTION_RANGE);
+      gst_value_set_fraction_range_full (&fpar, 1, G_MAXINT, G_MAXINT, 1);
+      from_par = &fpar;
+    }
+  }
+
   /* we have both PAR but they might not be fixated */
-  if (from_par && to_par) {
+  {
     gint from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d;
-    gint count = 0, w = 0, h = 0;
-    guint num, den;
+    gint w = 0, h = 0;
+    gint from_dar_n, from_dar_d;
+    gint num, den;
 
     /* from_par should be fixed */
     g_return_if_fail (gst_value_is_fixed (from_par));
@@ -614,102 +648,339 @@ gst_video_scale_fixate_caps (GstBaseTransform * base, GstPadDirection direction,
 
     /* if both width and height are already fixed, we can't do anything
      * about it anymore */
-    if (gst_structure_get_int (outs, "width", &w))
-      ++count;
-    if (gst_structure_get_int (outs, "height", &h))
-      ++count;
-    if (count == 2) {
+    if (gst_structure_get_int (outs, "width", &w)
+        && gst_structure_get_int (outs, "height", &h)) {
+      guint n, d;
+
       GST_DEBUG_OBJECT (base, "dimensions already set to %dx%d, not fixating",
           w, h);
       if (!gst_value_is_fixed (to_par)) {
-        if (gst_video_calculate_display_ratio (&num, &den, from_w, from_h,
+        if (gst_video_calculate_display_ratio (&n, &d, from_w, from_h,
                 from_par_n, from_par_d, w, h)) {
           GST_DEBUG_OBJECT (base, "fixating to_par to %dx%d", num, den);
-          gst_structure_fixate_field_nearest_fraction (outs,
-              "pixel-aspect-ratio", num, den);
+          if (gst_structure_has_field (outs, "pixel-aspect-ratio"))
+            gst_structure_fixate_field_nearest_fraction (outs,
+                "pixel-aspect-ratio", n, d);
+          else
+            gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
+                n, d, NULL);
         }
       }
-      return;
-    }
-
-    /* fixate the out PAR */
-    if (!gst_value_is_fixed (to_par)) {
-      GST_DEBUG_OBJECT (base, "fixating to_par to %dx%d", from_par_n,
-          from_par_d);
-      gst_structure_fixate_field_nearest_fraction (outs, "pixel-aspect-ratio",
-          from_par_n, from_par_d);
+      goto done;
     }
 
-    to_par_n = gst_value_get_fraction_numerator (to_par);
-    to_par_d = gst_value_get_fraction_denominator (to_par);
-
-    if (!gst_video_calculate_display_ratio (&num, &den, from_w, from_h,
-            from_par_n, from_par_d, to_par_n, to_par_d)) {
+    /* Calculate input DAR */
+    if (!gst_util_fraction_multiply (from_w, from_h, from_par_n, from_par_d,
+            &from_dar_n, &from_dar_d)) {
       GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
           ("Error calculating the output scaled size - integer overflow"));
-      return;
+      goto done;
     }
 
-    GST_DEBUG_OBJECT (base,
-        "scaling input with %dx%d and PAR %d/%d to output PAR %d/%d",
-        from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d);
-    GST_DEBUG_OBJECT (base, "resulting output should respect ratio of %d/%d",
-        num, den);
+    GST_DEBUG_OBJECT (base, "Input DAR is %d/%d", from_dar_n, from_dar_d);
 
-    /* now find a width x height that respects this display ratio.
-     * prefer those that have one of w/h the same as the incoming video
-     * using wd / hd = num / den */
-
-    /* if one of the output width or height is fixed, we work from there */
+    /* If either width or height are fixed there's not much we
+     * can do either except choosing a height or width and PAR
+     * that matches the DAR as good as possible
+     */
     if (h) {
-      GST_DEBUG_OBJECT (base, "height is fixed,scaling width");
+      GstStructure *tmp;
+      gint set_w, set_par_n, set_par_d;
+
+      GST_DEBUG_OBJECT (base, "height is fixed (%d)", h);
+
+      /* If the PAR is fixed too, there's not much to do
+       * except choosing the width that is nearest to the
+       * width with the same DAR */
+      if (gst_value_is_fixed (to_par)) {
+        to_par_n = gst_value_get_fraction_numerator (to_par);
+        to_par_d = gst_value_get_fraction_denominator (to_par);
+
+        GST_DEBUG_OBJECT (base, "PAR is fixed %d/%d", to_par_n, to_par_d);
+
+        if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
+                to_par_n, &num, &den)) {
+          GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+              ("Error calculating the output scaled size - integer overflow"));
+          goto done;
+        }
+
+        w = (guint) gst_util_uint64_scale_int (h, num, den);
+        gst_structure_fixate_field_nearest_int (outs, "width", w);
+
+        goto done;
+      }
+
+      /* The PAR is not fixed and it's quite likely that we can set
+       * an arbitrary PAR. */
+
+      /* Check if we can keep the input width */
+      tmp = gst_structure_copy (outs);
+      gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
+      gst_structure_get_int (tmp, "width", &set_w);
+
+      /* Might have failed but try to keep the DAR nonetheless by
+       * adjusting the PAR */
+      if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, h, set_w,
+              &to_par_n, &to_par_d)) {
+        GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+            ("Error calculating the output scaled size - integer overflow"));
+        gst_structure_free (tmp);
+        goto done;
+      }
+      gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
+          to_par_n, to_par_d);
+      gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
+          &set_par_d);
+      gst_structure_free (tmp);
+
+      /* Check if the adjusted PAR is accepted */
+      if (set_par_n == to_par_n && set_par_d == to_par_d) {
+        gst_structure_set (outs, "width", G_TYPE_INT, set_w,
+            "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d,
+            NULL);
+        goto done;
+      }
+
+      /* Otherwise scale the width to the new PAR and check if the
+       * adjusted with is accepted. If all that fails we can't keep
+       * the DAR */
+      if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
+              set_par_n, &num, &den)) {
+        GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+            ("Error calculating the output scaled size - integer overflow"));
+        goto done;
+      }
+
       w = (guint) gst_util_uint64_scale_int (h, num, den);
+      gst_structure_fixate_field_nearest_int (outs, "width", w);
+      gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
+          set_par_n, set_par_d, NULL);
+
+      goto done;
     } else if (w) {
-      GST_DEBUG_OBJECT (base, "width is fixed, scaling height");
+      GstStructure *tmp;
+      gint set_h, set_par_n, set_par_d;
+
+      GST_DEBUG_OBJECT (base, "width is fixed (%d)", w);
+
+      /* If the PAR is fixed too, there's not much to do
+       * except choosing the height that is nearest to the
+       * height with the same DAR */
+      if (gst_value_is_fixed (to_par)) {
+        to_par_n = gst_value_get_fraction_numerator (to_par);
+        to_par_d = gst_value_get_fraction_denominator (to_par);
+
+        GST_DEBUG_OBJECT (base, "PAR is fixed %d/%d", to_par_n, to_par_d);
+
+        if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
+                to_par_n, &num, &den)) {
+          GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+              ("Error calculating the output scaled size - integer overflow"));
+          goto done;
+        }
+
+        h = (guint) gst_util_uint64_scale_int (w, den, num);
+        gst_structure_fixate_field_nearest_int (outs, "height", h);
+
+        goto done;
+      }
+
+      /* The PAR is not fixed and it's quite likely that we can set
+       * an arbitrary PAR. */
+
+      /* Check if we can keep the input height */
+      tmp = gst_structure_copy (outs);
+      gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
+      gst_structure_get_int (tmp, "height", &set_h);
+
+      /* Might have failed but try to keep the DAR nonetheless by
+       * adjusting the PAR */
+      if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, w,
+              &to_par_n, &to_par_d)) {
+        GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+            ("Error calculating the output scaled size - integer overflow"));
+        gst_structure_free (tmp);
+        goto done;
+      }
+      gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
+          to_par_n, to_par_d);
+      gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
+          &set_par_d);
+      gst_structure_free (tmp);
+
+      /* Check if the adjusted PAR is accepted */
+      if (set_par_n == to_par_n && set_par_d == to_par_d) {
+        gst_structure_set (outs, "height", G_TYPE_INT, set_h,
+            "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d,
+            NULL);
+        goto done;
+      }
+
+      /* Otherwise scale the height to the new PAR and check if the
+       * adjusted with is accepted. If all that fails we can't keep
+       * the DAR */
+      if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
+              set_par_n, &num, &den)) {
+        GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+            ("Error calculating the output scaled size - integer overflow"));
+        goto done;
+      }
+
       h = (guint) gst_util_uint64_scale_int (w, den, num);
+      gst_structure_fixate_field_nearest_int (outs, "height", h);
+      gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
+          set_par_n, set_par_d, NULL);
+
+      goto done;
+    } else if (gst_value_is_fixed (to_par)) {
+      GstStructure *tmp;
+      gint set_h, set_w, f_h, f_w;
+
+      to_par_n = gst_value_get_fraction_numerator (to_par);
+      to_par_d = gst_value_get_fraction_denominator (to_par);
+
+      /* Calculate scale factor for the PAR change */
+      if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_n,
+              to_par_d, &num, &den)) {
+        GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+            ("Error calculating the output scaled size - integer overflow"));
+        goto done;
+      }
+
+      /* Try to keep the input height (because of interlacing) */
+      tmp = gst_structure_copy (outs);
+      gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
+      gst_structure_get_int (tmp, "height", &set_h);
+
+      /* This might have failed but try to scale the width
+       * to keep the DAR nonetheless */
+      w = (guint) gst_util_uint64_scale_int (set_h, num, den);
+      gst_structure_fixate_field_nearest_int (tmp, "width", w);
+      gst_structure_get_int (tmp, "width", &set_w);
+      gst_structure_free (tmp);
+
+      /* We kept the DAR and the height is nearest to the original height */
+      if (set_w == w) {
+        gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
+            G_TYPE_INT, set_h, NULL);
+        goto done;
+      }
+
+      f_h = set_h;
+      f_w = set_w;
+
+      /* If the former failed, try to keep the input width at least */
+      tmp = gst_structure_copy (outs);
+      gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
+      gst_structure_get_int (tmp, "width", &set_w);
+
+      /* This might have failed but try to scale the width
+       * to keep the DAR nonetheless */
+      h = (guint) gst_util_uint64_scale_int (set_w, den, num);
+      gst_structure_fixate_field_nearest_int (tmp, "height", h);
+      gst_structure_get_int (tmp, "height", &set_h);
+      gst_structure_free (tmp);
+
+      /* We kept the DAR and the width is nearest to the original width */
+      if (set_h == h) {
+        gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
+            G_TYPE_INT, set_h, NULL);
+        goto done;
+      }
+
+      /* If all this failed, keep the height that was nearest to the orignal
+       * height and the nearest possible width. This changes the DAR but
+       * there's not much else to do here.
+       */
+      gst_structure_set (outs, "width", G_TYPE_INT, f_w, "height", G_TYPE_INT,
+          f_h, NULL);
+      goto done;
     } else {
-      /* none of width or height is fixed, figure out both of them based only on
-       * the input width and height */
-      /* check hd / den is an integer scale factor, and scale wd with the PAR */
-      if (from_h % den == 0) {
-        GST_DEBUG_OBJECT (base, "keeping video height");
-        h = from_h;
-        w = (guint) gst_util_uint64_scale_int (h, num, den);
-      } else if (from_w % num == 0) {
-        GST_DEBUG_OBJECT (base, "keeping video width");
-        w = from_w;
-        h = (guint) gst_util_uint64_scale_int (w, den, num);
-      } else {
-        GST_DEBUG_OBJECT (base, "approximating but keeping video height");
-        h = from_h;
-        w = (guint) gst_util_uint64_scale_int (h, num, den);
+      GstStructure *tmp;
+      gint set_h, set_w, set_par_n, set_par_d, tmp2;
+
+      /* width, height and PAR are not fixed but passthrough is not possible */
+
+      /* First try to keep the height and width as good as possible
+       * and scale PAR */
+      tmp = gst_structure_copy (outs);
+      gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
+      gst_structure_get_int (tmp, "height", &set_h);
+      gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
+      gst_structure_get_int (tmp, "width", &set_w);
+
+      if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, set_w,
+              &to_par_n, &to_par_d)) {
+        GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+            ("Error calculating the output scaled size - integer overflow"));
+        goto done;
       }
-    }
-    GST_DEBUG_OBJECT (base, "scaling to %dx%d", w, h);
 
-    /* now fixate */
-    gst_structure_fixate_field_nearest_int (outs, "width", w);
-    gst_structure_fixate_field_nearest_int (outs, "height", h);
-  } else {
-    gint width, height;
+      gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
+          to_par_n, to_par_d);
+      gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
+          &set_par_d);
+      gst_structure_free (tmp);
+
+      if (set_par_n == to_par_n && set_par_d == to_par_d) {
+        gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
+            G_TYPE_INT, set_h, "pixel-aspect-ratio", GST_TYPE_FRACTION,
+            set_par_n, set_par_d, NULL);
+        goto done;
+      }
 
-    if (gst_structure_get_int (ins, "width", &width)) {
-      if (gst_structure_has_field (outs, "width")) {
-        gst_structure_fixate_field_nearest_int (outs, "width", width);
-      } else {
-        gst_structure_set (outs, "width", G_TYPE_INT, width, NULL);
+      /* Otherwise try to scale width to keep the DAR with the set
+       * PAR and height */
+      if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
+              set_par_n, &num, &den)) {
+        GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
+            ("Error calculating the output scaled size - integer overflow"));
+        goto done;
       }
-    }
-    if (gst_structure_get_int (ins, "height", &height)) {
-      if (gst_structure_has_field (outs, "height")) {
-        gst_structure_fixate_field_nearest_int (outs, "height", height);
-      } else {
-        gst_structure_set (outs, "height", G_TYPE_INT, height, NULL);
+
+      w = (guint) gst_util_uint64_scale_int (set_h, num, den);
+      tmp = gst_structure_copy (outs);
+      gst_structure_fixate_field_nearest_int (tmp, "width", w);
+      gst_structure_get_int (tmp, "width", &tmp2);
+      gst_structure_free (tmp);
+
+      if (tmp2 == w) {
+        gst_structure_set (outs, "width", G_TYPE_INT, tmp2, "height",
+            G_TYPE_INT, set_h, "pixel-aspect-ratio", GST_TYPE_FRACTION,
+            set_par_n, set_par_d, NULL);
+        goto done;
+      }
+
+      /* ... or try the same with the height */
+      h = (guint) gst_util_uint64_scale_int (set_w, den, num);
+      tmp = gst_structure_copy (outs);
+      gst_structure_fixate_field_nearest_int (tmp, "height", h);
+      gst_structure_get_int (tmp, "height", &tmp2);
+      gst_structure_free (tmp);
+
+      if (tmp2 == h) {
+        gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
+            G_TYPE_INT, tmp2, "pixel-aspect-ratio", GST_TYPE_FRACTION,
+            set_par_n, set_par_d, NULL);
+        goto done;
       }
+
+      /* If all fails we can't keep the DAR and take the nearest values
+       * for everything from the first try */
+      gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
+          G_TYPE_INT, set_h, "pixel-aspect-ratio", GST_TYPE_FRACTION,
+          set_par_n, set_par_d, NULL);
     }
   }
 
+done:
   GST_DEBUG_OBJECT (base, "fixated othercaps to %" GST_PTR_FORMAT, othercaps);
+
+  if (from_par == &fpar)
+    g_value_unset (&fpar);
+  if (to_par == &tpar)
+    g_value_unset (&tpar);
 }
 
 static gboolean