st/vdpau: Implement VdpOutputSurfacePutBitsIndexed and VdpOutputSurfaceRenderOutputSurface

This gets mplayers menu overlay working.

Signed-off-by: Christian König <deathsimple@vodafone.de>

4 files changed, 307 insertions, 19 deletions

diff --git a/src/gallium/state_trackers/vdpau/device.c b/src/gallium/state_trackers/vdpau/device.c
index 4fcce9112aa..25fa8d88844 100644
--- a/src/gallium/state_trackers/vdpau/device.c
+++ b/src/gallium/state_trackers/vdpau/device.c
@@ -73,6 +73,8 @@ vdp_imp_device_create_x11(Display *display, int screen, VdpDevice *device,
       goto no_handle;
    }
 
+   vl_compositor_init(&dev->compositor, dev->context->pipe);
+
    *get_proc_address = &vlVdpGetProcAddress;
    VDPAU_MSG(VDPAU_TRACE, "[VDPAU] Device created succesfully\n");
 
@@ -152,6 +154,7 @@ vlVdpDeviceDestroy(VdpDevice device)
    if (!dev)
       return VDP_STATUS_INVALID_HANDLE;
       
+   vl_compositor_cleanup(&dev->compositor);
    vl_video_destroy(dev->context);
    vl_screen_destroy(dev->vscreen);
 
diff --git a/src/gallium/state_trackers/vdpau/mixer.c b/src/gallium/state_trackers/vdpau/mixer.c
index a80bffd7d3c..8530b7d4a1d 100644
--- a/src/gallium/state_trackers/vdpau/mixer.c
+++ b/src/gallium/state_trackers/vdpau/mixer.c
@@ -139,7 +139,7 @@ VdpStatus vlVdpVideoMixerRender(VdpVideoMixer mixer,
                                 uint32_t layer_count,
                                 VdpLayer const *layers)
 {
-   struct pipe_video_rect src_rect, *p_src_rect = NULL;
+   struct pipe_video_rect src_rect;
 
    vlVdpVideoMixer *vmixer;
    vlVdpSurface *surf;
@@ -157,16 +157,9 @@ VdpStatus vlVdpVideoMixerRender(VdpVideoMixer mixer,
    if (!dst)
       return VDP_STATUS_INVALID_HANDLE;
 
-   if (video_source_rect) {
-      src_rect.x = MIN2(video_source_rect->x1, video_source_rect->x0);
-      src_rect.y = MIN2(video_source_rect->y1, video_source_rect->y0);
-      src_rect.w = abs(video_source_rect->x1 - video_source_rect->x0);
-      src_rect.h = abs(video_source_rect->y1 - video_source_rect->y0);
-      p_src_rect = &src_rect;
-   }
-
    vl_compositor_clear_layers(&vmixer->compositor);
-   vl_compositor_set_buffer_layer(&vmixer->compositor, 0, surf->video_buffer, p_src_rect, NULL);
+   vl_compositor_set_buffer_layer(&vmixer->compositor, 0, surf->video_buffer,
+                                  RectToPipe(video_source_rect, &src_rect), NULL);
    vl_compositor_render(&vmixer->compositor, dst->surface, NULL, NULL, false);
 
    return VDP_STATUS_OK;
diff --git a/src/gallium/state_trackers/vdpau/output.c b/src/gallium/state_trackers/vdpau/output.c
index 332690f0ff7..1736eb942fb 100644
--- a/src/gallium/state_trackers/vdpau/output.c
+++ b/src/gallium/state_trackers/vdpau/output.c
@@ -31,6 +31,7 @@
 #include "util/u_debug.h"
 #include "util/u_memory.h"
 #include "util/u_sampler.h"
+#include "util/u_format.h"
 
 #include "vdpau_private.h"
 
@@ -63,6 +64,8 @@ vlVdpOutputSurfaceCreate(VdpDevice device,
    if (!vlsurface)
       return VDP_STATUS_RESOURCES;
 
+   vlsurface->device = dev;
+
    memset(&res_tmpl, 0, sizeof(res_tmpl));
 
    res_tmpl.target = PIPE_TEXTURE_2D;
@@ -82,10 +85,6 @@ vlVdpOutputSurfaceCreate(VdpDevice device,
 
    memset(&sv_templ, 0, sizeof(sv_templ));
    u_sampler_view_default_template(&sv_templ, res, res->format);
-
-   // as long as we don't have a background picture we don't want an alpha channel
-   sv_templ.swizzle_a = PIPE_SWIZZLE_ONE;
-
    vlsurface->sampler_view = pipe->create_sampler_view(pipe, res, &sv_templ);
    if (!vlsurface->sampler_view) {
       pipe_resource_reference(&res, NULL);
@@ -182,7 +181,127 @@ vlVdpOutputSurfacePutBitsIndexed(VdpOutputSurface surface,
                                  VdpColorTableFormat color_table_format,
                                  void const *color_table)
 {
-   return VDP_STATUS_NO_IMPLEMENTATION;
+   vlVdpOutputSurface *vlsurface;
+   struct pipe_context *context;
+   struct vl_compositor *compositor;
+
+   enum pipe_format index_format;
+   enum pipe_format colortbl_format;
+
+   struct pipe_resource *res, res_tmpl;
+   struct pipe_sampler_view sv_tmpl;
+   struct pipe_sampler_view *sv_idx = NULL, *sv_tbl = NULL;
+
+   struct pipe_box box;
+   struct pipe_video_rect dst_rect;
+
+   VDPAU_MSG(VDPAU_TRACE, "[VDPAU] uploading indexed output surface\n");
+
+   vlsurface = vlGetDataHTAB(surface);
+   if (!vlsurface)
+      return VDP_STATUS_INVALID_HANDLE;
+
+   context = vlsurface->device->context->pipe;
+   compositor = &vlsurface->device->compositor;
+
+   index_format = FormatIndexedToPipe(source_indexed_format);
+   if (index_format == PIPE_FORMAT_NONE)
+       return VDP_STATUS_INVALID_INDEXED_FORMAT;
+
+   if (!source_data || !source_pitch)
+       return VDP_STATUS_INVALID_POINTER;
+
+   colortbl_format = FormatColorTableToPipe(color_table_format);
+   if (colortbl_format == PIPE_FORMAT_NONE)
+       return VDP_STATUS_INVALID_COLOR_TABLE_FORMAT;
+
+   if (!color_table)
+       return VDP_STATUS_INVALID_POINTER;
+
+   memset(&res_tmpl, 0, sizeof(res_tmpl));
+   res_tmpl.target = PIPE_TEXTURE_2D;
+   res_tmpl.format = index_format;
+
+   if (destination_rect) {
+      res_tmpl.width0 = abs(destination_rect->x0-destination_rect->x1);
+      res_tmpl.height0 = abs(destination_rect->y0-destination_rect->y1);
+   } else {
+      res_tmpl.width0 = vlsurface->surface->texture->width0;
+      res_tmpl.height0 = vlsurface->surface->texture->height0;
+   }
+   res_tmpl.depth0 = 1;
+   res_tmpl.array_size = 1;
+   res_tmpl.usage = PIPE_USAGE_STAGING;
+   res_tmpl.bind = PIPE_BIND_SAMPLER_VIEW;
+
+   res = context->screen->resource_create(context->screen, &res_tmpl);
+   if (!res)
+      goto error_resource;
+
+   box.x = box.y = box.z = 0;
+   box.width = res->width0;
+   box.height = res->height0;
+   box.depth = res->depth0;
+
+   context->transfer_inline_write(context, res, 0, PIPE_TRANSFER_WRITE, &box,
+                                  source_data[0], source_pitch[0],
+                                  source_pitch[0] * res->height0);
+
+   memset(&sv_tmpl, 0, sizeof(sv_tmpl));
+   u_sampler_view_default_template(&sv_tmpl, res, res->format);
+
+   sv_idx = context->create_sampler_view(context, res, &sv_tmpl);
+   pipe_resource_reference(&res, NULL);
+
+   if (!sv_idx)
+      goto error_resource;
+
+   memset(&res_tmpl, 0, sizeof(res_tmpl));
+   res_tmpl.target = PIPE_TEXTURE_1D;
+   res_tmpl.format = colortbl_format;
+   res_tmpl.width0 = 1 << util_format_get_component_bits(
+      index_format, UTIL_FORMAT_COLORSPACE_RGB, 0);
+   res_tmpl.height0 = 1;
+   res_tmpl.depth0 = 1;
+   res_tmpl.array_size = 1;
+   res_tmpl.usage = PIPE_USAGE_STAGING;
+   res_tmpl.bind = PIPE_BIND_SAMPLER_VIEW;
+
+   res = context->screen->resource_create(context->screen, &res_tmpl);
+   if (!res)
+      goto error_resource;
+
+   box.x = box.y = box.z = 0;
+   box.width = res->width0;
+   box.height = res->height0;
+   box.depth = res->depth0;
+
+   context->transfer_inline_write(context, res, 0, PIPE_TRANSFER_WRITE, &box, color_table,
+                                  util_format_get_stride(colortbl_format, res->width0), 0);
+
+   memset(&sv_tmpl, 0, sizeof(sv_tmpl));
+   u_sampler_view_default_template(&sv_tmpl, res, res->format);
+
+   sv_tbl = context->create_sampler_view(context, res, &sv_tmpl);
+   pipe_resource_reference(&res, NULL);
+
+   if (!sv_tbl)
+      goto error_resource;
+
+   vl_compositor_clear_layers(compositor);
+   vl_compositor_set_palette_layer(compositor, 0, sv_idx, sv_tbl, NULL, NULL, false);
+   vl_compositor_render(compositor, vlsurface->surface,
+                        RectToPipe(destination_rect, &dst_rect), NULL, false);
+
+   pipe_sampler_view_reference(&sv_idx, NULL);
+   pipe_sampler_view_reference(&sv_tbl, NULL);
+
+   return VDP_STATUS_OK;
+
+error_resource:
+   pipe_sampler_view_reference(&sv_idx, NULL);
+   pipe_sampler_view_reference(&sv_tbl, NULL);
+   return VDP_STATUS_RESOURCES;
 }
 
 VdpStatus
@@ -196,6 +315,95 @@ vlVdpOutputSurfacePutBitsYCbCr(VdpOutputSurface surface,
    return VDP_STATUS_NO_IMPLEMENTATION;
 }
 
+static unsigned
+BlendFactorToPipe(VdpOutputSurfaceRenderBlendFactor factor)
+{
+   switch(factor) {
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ZERO:
+      return PIPE_BLENDFACTOR_ZERO;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE:
+      return PIPE_BLENDFACTOR_ONE;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_SRC_COLOR:
+      return PIPE_BLENDFACTOR_SRC_COLOR;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE_MINUS_SRC_COLOR:
+      return PIPE_BLENDFACTOR_INV_SRC_COLOR;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_SRC_ALPHA:
+      return PIPE_BLENDFACTOR_SRC_ALPHA;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA:
+      return PIPE_BLENDFACTOR_INV_SRC_ALPHA;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_DST_ALPHA:
+      return PIPE_BLENDFACTOR_DST_ALPHA;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE_MINUS_DST_ALPHA:
+      return PIPE_BLENDFACTOR_INV_DST_ALPHA;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_DST_COLOR:
+      return PIPE_BLENDFACTOR_DST_COLOR;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE_MINUS_DST_COLOR:
+      return PIPE_BLENDFACTOR_INV_DST_COLOR;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_SRC_ALPHA_SATURATE:
+      return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_CONSTANT_COLOR:
+      return PIPE_BLENDFACTOR_CONST_COLOR;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR:
+      return PIPE_BLENDFACTOR_INV_CONST_COLOR;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_CONSTANT_ALPHA:
+      return PIPE_BLENDFACTOR_CONST_ALPHA;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA:
+      return PIPE_BLENDFACTOR_INV_CONST_ALPHA;
+   default:
+      assert(0);
+      return PIPE_BLENDFACTOR_ONE;
+   }
+}
+
+static unsigned
+BlendEquationToPipe(VdpOutputSurfaceRenderBlendEquation equation)
+{
+   switch(equation) {
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_EQUATION_SUBTRACT:
+      return PIPE_BLEND_SUBTRACT;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_EQUATION_REVERSE_SUBTRACT:
+      return PIPE_BLEND_REVERSE_SUBTRACT;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_EQUATION_ADD:
+      return PIPE_BLEND_ADD;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_EQUATION_MIN:
+      return PIPE_BLEND_MIN;
+   case VDP_OUTPUT_SURFACE_RENDER_BLEND_EQUATION_MAX:
+      return PIPE_BLEND_MAX;
+   default:
+      assert(0);
+      return PIPE_BLEND_ADD;
+   }
+}
+
+static void *
+BlenderToPipe(struct pipe_context *context,
+              VdpOutputSurfaceRenderBlendState const *blend_state)
+{
+   struct pipe_blend_state blend;
+
+   memset(&blend, 0, sizeof blend);
+   blend.independent_blend_enable = 0;
+
+   if (blend_state) {
+      blend.rt[0].blend_enable = 1;
+      blend.rt[0].rgb_src_factor = BlendFactorToPipe(blend_state->blend_factor_source_color);
+      blend.rt[0].rgb_dst_factor = BlendFactorToPipe(blend_state->blend_factor_destination_color);
+      blend.rt[0].alpha_src_factor = BlendFactorToPipe(blend_state->blend_factor_source_alpha);
+      blend.rt[0].alpha_dst_factor = BlendFactorToPipe(blend_state->blend_factor_destination_alpha);
+      blend.rt[0].rgb_func = BlendEquationToPipe(blend_state->blend_equation_color);
+      blend.rt[0].alpha_func = BlendEquationToPipe(blend_state->blend_equation_alpha);
+   } else {
+      blend.rt[0].blend_enable = 0;
+   }
+
+   blend.logicop_enable = 0;
+   blend.logicop_func = PIPE_LOGICOP_CLEAR;
+   blend.rt[0].colormask = PIPE_MASK_RGBA;
+   blend.dither = 0;
+
+   return context->create_blend_state(context, &blend);
+}
+
 VdpStatus
 vlVdpOutputSurfaceRenderOutputSurface(VdpOutputSurface destination_surface,
                                       VdpRect const *destination_rect,
@@ -205,7 +413,45 @@ vlVdpOutputSurfaceRenderOutputSurface(VdpOutputSurface destination_surface,
                                       VdpOutputSurfaceRenderBlendState const *blend_state,
                                       uint32_t flags)
 {
-   return VDP_STATUS_NO_IMPLEMENTATION;
+   vlVdpOutputSurface *dst_vlsurface;
+   vlVdpOutputSurface *src_vlsurface;
+
+   struct pipe_context *context;
+   struct vl_compositor *compositor;
+
+   struct pipe_video_rect src_rect;
+   struct pipe_video_rect dst_rect;
+
+   void *blend;
+
+   VDPAU_MSG(VDPAU_TRACE, "[VDPAU] composing output surfaces\n");
+
+   dst_vlsurface = vlGetDataHTAB(destination_surface);
+   if (!dst_vlsurface)
+      return VDP_STATUS_INVALID_HANDLE;
+
+   src_vlsurface = vlGetDataHTAB(source_surface);
+   if (!src_vlsurface)
+      return VDP_STATUS_INVALID_HANDLE;
+
+   if (dst_vlsurface->device != src_vlsurface->device)
+      return VDP_STATUS_HANDLE_DEVICE_MISMATCH;
+
+   context = dst_vlsurface->device->context->pipe;
+   compositor = &dst_vlsurface->device->compositor;
+
+   blend = BlenderToPipe(context, blend_state);
+
+   vl_compositor_clear_layers(compositor);
+   vl_compositor_set_layer_blend(compositor, 0, blend, false);
+   vl_compositor_set_rgba_layer(compositor, 0, src_vlsurface->sampler_view,
+                                RectToPipe(source_rect, &src_rect), NULL);
+   vl_compositor_render(compositor, dst_vlsurface->surface,
+                        RectToPipe(destination_rect, &dst_rect), NULL, false);
+
+   context->delete_blend_state(context, blend);
+
+   return VDP_STATUS_OK;
 }
 
 VdpStatus
diff --git a/src/gallium/state_trackers/vdpau/vdpau_private.h b/src/gallium/state_trackers/vdpau/vdpau_private.h
index 263e2545a4e..705b57e72c0 100644
--- a/src/gallium/state_trackers/vdpau/vdpau_private.h
+++ b/src/gallium/state_trackers/vdpau/vdpau_private.h
@@ -95,14 +95,14 @@ FormatYCBCRToPipe(VdpYCbCrFormat vdpau_format)
       case VDP_YCBCR_FORMAT_YUYV:
          return PIPE_FORMAT_YUYV;
       case VDP_YCBCR_FORMAT_Y8U8V8A8: /* Not defined in p_format.h */
-         return 0;
+         return PIPE_FORMAT_NONE;
       case VDP_YCBCR_FORMAT_V8U8Y8A8:
 	     return PIPE_FORMAT_VUYA;
       default:
          assert(0);
    }
 
-   return -1;
+   return PIPE_FORMAT_NONE;
 }
 
 static inline VdpYCbCrFormat
@@ -146,7 +146,7 @@ FormatRGBAToPipe(VdpRGBAFormat vdpau_format)
          assert(0);
    }
 
-   return -1;
+   return PIPE_FORMAT_NONE;
 }
 
 static inline VdpRGBAFormat
@@ -170,6 +170,38 @@ PipeToFormatRGBA(enum pipe_format p_format)
    return -1;
 }
 
+static inline enum pipe_format
+FormatIndexedToPipe(VdpRGBAFormat vdpau_format)
+{
+   switch (vdpau_format) {
+      case VDP_INDEXED_FORMAT_A4I4:
+         return PIPE_FORMAT_A4R4_UNORM;
+      case VDP_INDEXED_FORMAT_I4A4:
+         return PIPE_FORMAT_R4A4_UNORM;
+      case VDP_INDEXED_FORMAT_A8I8:
+         return PIPE_FORMAT_A8R8_UNORM;
+      case VDP_INDEXED_FORMAT_I8A8:
+         return PIPE_FORMAT_R8A8_UNORM;
+      default:
+         assert(0);
+   }
+
+   return PIPE_FORMAT_NONE;
+}
+
+static inline enum pipe_format
+FormatColorTableToPipe(VdpColorTableFormat vdpau_format)
+{
+   switch(vdpau_format) {
+      case VDP_COLOR_TABLE_FORMAT_B8G8R8X8:
+         return PIPE_FORMAT_B8G8R8X8_UNORM;
+      default:
+         assert(0);
+   }
+   return PIPE_FORMAT_NONE;
+}
+
+
 static inline enum pipe_video_profile
 ProfileToPipe(VdpDecoderProfile vdpau_profile)
 {
@@ -213,10 +245,24 @@ PipeToProfile(enum pipe_video_profile p_profile)
    }
 }
 
+static inline struct pipe_video_rect *
+RectToPipe(const VdpRect *src, struct pipe_video_rect *dst)
+{
+   if (src) {
+      dst->x = MIN2(src->x1, src->x0);
+      dst->y = MIN2(src->y1, src->y0);
+      dst->w = abs(src->x1 - src->x0);
+      dst->h = abs(src->y1 - src->y0);
+      return dst;
+   }
+   return NULL;
+}
+
 typedef struct
 {
    struct vl_screen *vscreen;
    struct vl_context *context;
+   struct vl_compositor compositor;
 } vlVdpDevice;
 
 typedef struct