msdk: use cached response for DMABuf when the frame size is same

User is seeing corrupted display when running `videotestsrc !
video/x-raw,format=NV12,width=xxx,height=xxx ! msdkh265enc ! msdkh265dec
! glimagesink` with changed frame size, e.g. from 1920x1080 to 1920x240

The root cause is a same dmabuf fd is used for frames with
different size, which causes some unexpected result. This patch requires
cached response is used for frames with same size only for DMABuf, so a
dmabuf fd can't be used for frames with different size any more.

1 files changed, 18 insertions, 4 deletions

diff --git a/sys/msdk/gstmsdkcontext.c b/sys/msdk/gstmsdkcontext.c
index b2341d3e6..7be6cb49e 100644
--- a/sys/msdk/gstmsdkcontext.c
+++ b/sys/msdk/gstmsdkcontext.c
@@ -349,6 +349,22 @@ _find_response (gconstpointer resp, gconstpointer comp_resp)
   return cached_resp ? cached_resp->response.mids != _resp->mids : -1;
 }
 
+static inline gboolean
+_requested_frame_size_is_equal_or_lower (mfxFrameAllocRequest * _req,
+    GstMsdkAllocResponse * cached_resp)
+{
+  if (((_req->Type & MFX_MEMTYPE_EXPORT_FRAME) &&
+          _req->Info.Width == cached_resp->request.Info.Width &&
+          _req->Info.Height == cached_resp->request.Info.Height) ||
+      (!(_req->Type & MFX_MEMTYPE_EXPORT_FRAME) &&
+          _req->Info.Width <= cached_resp->request.Info.Width &&
+          _req->Info.Height <= cached_resp->request.Info.Height))
+
+    return TRUE;
+
+  return FALSE;
+}
+
 static gint
 _find_request (gconstpointer resp, gconstpointer req)
 {
@@ -356,12 +372,10 @@ _find_request (gconstpointer resp, gconstpointer req)
   mfxFrameAllocRequest *_req = (mfxFrameAllocRequest *) req;
 
   /* Confirm if it's under the size of the cached response */
-  if (_req->Info.Width <= cached_resp->request.Info.Width &&
-      _req->Info.Height <= cached_resp->request.Info.Height &&
-      _req->NumFrameSuggested <= cached_resp->request.NumFrameSuggested) {
+  if (_req->NumFrameSuggested <= cached_resp->request.NumFrameSuggested &&
+      _requested_frame_size_is_equal_or_lower (_req, cached_resp))
     return _req->Type & cached_resp->
         request.Type & MFX_MEMTYPE_FROM_DECODE ? 0 : -1;
-  }
 
   return -1;
 }