vaapipostproc: rotate outbuf and crop meta if forwarding

When forwarding crop meta to downstream, the output
buffer and crop meta need to be rotated, too.

Test:
 for i in 90r 180 90l vert horiz ul-lr ur-ll
 do
  gst-launch-1.0 -vf videotestsrc num-buffers=500 \
   ! videocrop top=100 bottom=30 left=40 right=20 \
   ! vaapipostproc video-direction=$i \
   ! vaapisink & \
  gst-launch-1.0 -vf videotestsrc num-buffers=500 \
   ! videocrop top=100 bottom=30 left=40 right=20 \
   ! vaapipostproc video-direction=$i \
   ! identity drop-allocation=true \
   ! vaapisink
 done

1 files changed, 68 insertions, 1 deletions

diff --git a/gst/vaapi/gstvaapipostproc.c b/gst/vaapi/gstvaapipostproc.c
index 724cf978..73af3b9f 100644
--- a/gst/vaapi/gstvaapipostproc.c
+++ b/gst/vaapi/gstvaapipostproc.c
@@ -678,6 +678,56 @@ use_vpp_crop (GstVaapiPostproc * postproc)
   return !postproc->forward_crop;
 }
 
+static void
+rotate_crop_meta (GstVaapiPostproc * const postproc, const GstVideoMeta * vmeta,
+    GstVideoCropMeta * crop)
+{
+  guint tmp;
+
+  /* The video meta is required since the caps width/height are smaller,
+   * which would not result in a usable GstVideoInfo for mapping the
+   * buffer. */
+  if (!vmeta || !crop)
+    return;
+
+  switch (gst_vaapi_filter_get_video_direction (postproc->filter)) {
+    case GST_VIDEO_ORIENTATION_HORIZ:
+      crop->x = vmeta->width - crop->width - crop->x;
+      break;
+    case GST_VIDEO_ORIENTATION_VERT:
+      crop->y = vmeta->height - crop->height - crop->y;
+      break;
+    case GST_VIDEO_ORIENTATION_90R:
+      tmp = crop->x;
+      crop->x = vmeta->height - crop->height - crop->y;
+      crop->y = tmp;
+      G_PRIMITIVE_SWAP (guint, crop->width, crop->height);
+      break;
+    case GST_VIDEO_ORIENTATION_180:
+      crop->x = vmeta->width - crop->width - crop->x;
+      crop->y = vmeta->height - crop->height - crop->y;
+      break;
+    case GST_VIDEO_ORIENTATION_90L:
+      tmp = crop->x;
+      crop->x = crop->y;
+      crop->y = vmeta->width - crop->width - tmp;
+      G_PRIMITIVE_SWAP (guint, crop->width, crop->height);
+      break;
+    case GST_VIDEO_ORIENTATION_UR_LL:
+      tmp = crop->x;
+      crop->x = vmeta->height - crop->height - crop->y;
+      crop->y = vmeta->width - crop->width - tmp;
+      G_PRIMITIVE_SWAP (guint, crop->width, crop->height);
+      break;
+    case GST_VIDEO_ORIENTATION_UL_LR:
+      G_PRIMITIVE_SWAP (guint, crop->x, crop->y);
+      G_PRIMITIVE_SWAP (guint, crop->width, crop->height);
+      break;
+    default:
+      break;
+  }
+}
+
 static GstFlowReturn
 gst_vaapipostproc_process_vpp (GstBaseTransform * trans, GstBuffer * inbuf,
     GstBuffer * outbuf)
@@ -713,6 +763,7 @@ gst_vaapipostproc_process_vpp (GstBaseTransform * trans, GstBuffer * inbuf,
     crop_rect->width = crop_meta->width;
     crop_rect->height = crop_meta->height;
   }
+
   if (!crop_rect)
     crop_rect = (GstVaapiRectangle *)
         gst_vaapi_video_meta_get_render_rect (inbuf_meta);
@@ -871,6 +922,9 @@ gst_vaapipostproc_process_vpp (GstBaseTransform * trans, GstBuffer * inbuf,
 
   copy_metadata (postproc, outbuf, inbuf);
 
+  rotate_crop_meta (postproc, gst_buffer_get_video_meta (inbuf),
+      gst_buffer_get_video_crop_meta (outbuf));
+
   if (deint && deint_refs)
     ds_add_buffer (ds, inbuf);
   postproc->use_vpp = TRUE;
@@ -1452,7 +1506,8 @@ gst_vaapipostproc_prepare_output_buffer (GstBaseTransform * trans,
   }
 
   /* If we are not using vpp crop (i.e. forwarding crop meta to downstream)
-   * then, ensure our output buffer pool is sized for uncropped output */
+   * then, ensure our output buffer pool is sized and rotated for uncropped
+   * output */
   if (gst_buffer_get_video_crop_meta (inbuf) && !use_vpp_crop (postproc)) {
     /* The video meta is required since the caps width/height are smaller,
      * which would not result in a usable GstVideoInfo for mapping the
@@ -1464,6 +1519,18 @@ gst_vaapipostproc_prepare_output_buffer (GstBaseTransform * trans,
     info = postproc->srcpad_info;
     info.width = video_meta->width;
     info.height = video_meta->height;
+
+    /* compensate for rotation if needed */
+    switch (gst_vaapi_filter_get_video_direction (postproc->filter)) {
+      case GST_VIDEO_ORIENTATION_90R:
+      case GST_VIDEO_ORIENTATION_UL_LR:
+      case GST_VIDEO_ORIENTATION_90L:
+      case GST_VIDEO_ORIENTATION_UR_LL:
+        G_PRIMITIVE_SWAP (guint, info.width, info.height);
+      default:
+        break;
+    }
+
     ensure_buffer_pool (postproc, &info);
   }