cog: hacking to improve matrix quality

2 files changed, 59 insertions, 3 deletions

diff --git a/ext/cog/cog.orc b/ext/cog/cog.orc
index 7aeea8eae..23a63c5cc 100644
--- a/ext/cog/cog.orc
+++ b/ext/cog/cog.orc
@@ -264,6 +264,29 @@ shrsw t1, t1, 6
 convsuswb d1, t1
 
 
+.function orc_matrix2_2_u8
+.dest 1 d1 uint8_t
+.source 1 s1 uint8_t
+.source 1 s2 uint8_t
+.param 2 p1
+.param 2 p2
+.param 2 p3
+.param 2 p4
+.temp 2 t1
+.temp 2 t2
+
+convubw t1, s1
+subw t1, t1, 16
+mullw t1, t1, p1
+convubw t2, s2
+subw t2, t2, 128
+mullw t2, t2, p2
+addw t1, t1, t2
+addw t1, t1, p3
+shrsw t1, t1, p4
+convsuswb d1, t1
+
+
 .function orc_matrix3_u8
 .dest 1 d1 uint8_t
 .source 1 s1 uint8_t
@@ -289,6 +312,35 @@ shrsw t1, t1, 6
 convsuswb d1, t1
 
 
+.function orc_matrix3_2_u8
+.dest 1 d1 uint8_t
+.source 1 s1 uint8_t
+.source 1 s2 uint8_t
+.source 1 s3 uint8_t
+.param 2 p1
+.param 2 p2
+.param 2 p3
+.param 2 p4
+.param 2 p5
+.temp 2 t1
+.temp 2 t2
+
+convubw t1, s1
+subw t1, t1, 16
+mullw t1, t1, p1
+convubw t2, s2
+subw t2, t2, 128
+mullw t2, t2, p2
+addw t1, t1, t2
+convubw t2, s3
+subw t2, t2, 128
+mullw t2, t2, p3
+addw t1, t1, t2
+addw t1, t1, p4
+shrsw t1, t1, p5
+convsuswb d1, t1
+
+
 
 .function orc_pack_123x
 .dest 4 d1 uint32_t
diff --git a/ext/cog/cogvirtframe.c b/ext/cog/cogvirtframe.c
index 850d6f0ee..fcab0f396 100644
--- a/ext/cog/cogvirtframe.c
+++ b/ext/cog/cogvirtframe.c
@@ -1327,14 +1327,17 @@ color_matrix_YCbCr_to_RGB (CogFrame * frame, void *_dest, int component, int i)
       m2 = 0;
       m3 = 1.596;
       offset = -222.92;
-      orc_matrix2_u8 (dest, src1, src3, 75, 102, -14269 + 32, frame->width);
+      //orc_matrix2_u8 (dest, src1, src3, 75, 102, -14269 + 32, frame->width);
+      orc_matrix2_2_u8 (dest, src1, src3, 75, 102, 32, 6, frame->width);
       break;
     case 1:
       m1 = 1.1644;
       m2 = -0.39176;
       m3 = -0.81297;
       offset = 135.58;
-      orc_matrix3_u8 (dest, src1, src2, src3, 75, -25, -52, 8677 + 32,
+      //orc_matrix3_u8 (dest, src1, src2, src3, 75, -25, -52, 8677 + 32,
+      //    frame->width);
+      orc_matrix3_2_u8 (dest, src1, src2, src3, 75, -25, -52, 32, 6,
           frame->width);
       break;
     case 2:
@@ -1342,7 +1345,8 @@ color_matrix_YCbCr_to_RGB (CogFrame * frame, void *_dest, int component, int i)
       m2 = 2.0172;
       m3 = 0;
       offset = -276.84;
-      orc_matrix2_u8 (dest, src1, src2, 75, 129, -17718 + 32, frame->width);
+      //orc_matrix2_u8 (dest, src1, src2, 75, 129, -17718 + 32, frame->width);
+      orc_matrix2_2_u8 (dest, src1, src2, 75, 129, 32, 6, frame->width);
       break;
     default:
       m1 = 0.0;