tdf#113197 Add MaskPrimitive (clip) to EMF/WMF if needed

Added code to quartz vcl implementation that takes care
when BitmapPalette.count != (depth^^2)-1 - which may
be the case anytime. If then a bitmap value exists that
goes beyond that count, a invalid access was executed

Change-Id: Iab332c91b8753aab85e9d365323f5c9e531efab2
Reviewed-on: https://gerrit.libreoffice.org/44058
Reviewed-by: Bartosz Kosiorek <gang65@poczta.onet.pl>
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-on: https://gerrit.libreoffice.org/53085
Reviewed-by: Armin Le Grand <Armin.Le.Grand@cib.de>
(cherry picked from commit 6e591f06e39c69c270c8c01e9ed138d315720624)

2 files changed, 52 insertions, 40 deletions

diff --git a/drawinglayer/source/primitive2d/metafileprimitive2d.cxx b/drawinglayer/source/primitive2d/metafileprimitive2d.cxx
index bb3d243b4beb..5ace807a264d 100644
--- a/drawinglayer/source/primitive2d/metafileprimitive2d.cxx
+++ b/drawinglayer/source/primitive2d/metafileprimitive2d.cxx
@@ -20,43 +20,10 @@
 #include <drawinglayer/primitive2d/metafileprimitive2d.hxx>
 #include <wmfemfhelper.hxx>
 
-//#include <basegfx/utils/canvastools.hxx>
 #include <drawinglayer/primitive2d/drawinglayer_primitivetypes2d.hxx>
-//#include <basegfx/color/bcolor.hxx>
-//#include <drawinglayer/primitive2d/pointarrayprimitive2d.hxx>
-//#include <vcl/lineinfo.hxx>
-//#include <drawinglayer/attribute/lineattribute.hxx>
-//#include <drawinglayer/attribute/strokeattribute.hxx>
-//#include <drawinglayer/primitive2d/polygonprimitive2d.hxx>
-//#include <vcl/metaact.hxx>
 #include <drawinglayer/primitive2d/transformprimitive2d.hxx>
-//#include <basegfx/matrix/b2dhommatrixtools.hxx>
-//#include <drawinglayer/primitive2d/polypolygonprimitive2d.hxx>
-//#include <basegfx/polygon/b2dpolygontools.hxx>
-//#include <drawinglayer/primitive2d/discretebitmapprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/bitmapprimitive2d.hxx>
-//#include <vcl/salbtype.hxx>
-//#include <drawinglayer/primitive2d/unifiedtransparenceprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/fillgradientprimitive2d.hxx>
-//#include <vcl/svapp.hxx>
-//#include <drawinglayer/primitive2d/transparenceprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/fillhatchprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/maskprimitive2d.hxx>
-//#include <basegfx/polygon/b2dpolygonclipper.hxx>
-//#include <drawinglayer/primitive2d/invertprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/modifiedcolorprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/fillgraphicprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/wallpaperprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/textprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/textlayoutdevice.hxx>
-//#include <drawinglayer/primitive2d/textdecoratedprimitive2d.hxx>
-//#include <i18nlangtag/languagetag.hxx>
-//#include <drawinglayer/primitive2d/textlineprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/textstrikeoutprimitive2d.hxx>
-//#include <drawinglayer/primitive2d/epsprimitive2d.hxx>
-//#include <tools/fract.hxx>
-//#include <numeric>
-//#include <emfplushelper.hxx>
+#include <basegfx/polygon/b2dpolygontools.hxx>
+#include <drawinglayer/primitive2d/maskprimitive2d.hxx>
 
 using namespace com::sun::star;
 
@@ -74,6 +41,29 @@ namespace drawinglayer
             {
                 // get target size
                 const ::tools::Rectangle aMtfTarget(getMetaFile().GetPrefMapMode().GetOrigin(), getMetaFile().GetPrefSize());
+                const basegfx::B2DRange aMtfRange(aMtfTarget.Left(), aMtfTarget.Top(), aMtfTarget.Right(), aMtfTarget.Bottom());
+
+                // tdf#113197 get content range and check if we have an overlap with
+                // defined target range (aMtfRange)
+                if (!aMtfRange.isEmpty())
+                {
+                    const basegfx::B2DRange aContentRange(xRetval.getB2DRange(rViewInformation));
+
+                    // also test equal since isInside gives also true for equal
+                    if (!aMtfRange.equal(aContentRange) && !aMtfRange.isInside(aContentRange))
+                    {
+                        // contentRange is partly larger than aMtfRange (stuff sticks
+                        // outside), clipping is needed
+                        const drawinglayer::primitive2d::Primitive2DReference xMask(
+                            new drawinglayer::primitive2d::MaskPrimitive2D(
+                                basegfx::B2DPolyPolygon(
+                                    basegfx::utils::createPolygonFromRect(
+                                        aMtfRange)),
+                                xRetval));
+
+                        xRetval = drawinglayer::primitive2d::Primitive2DContainer{ xMask };
+                    }
+                }
 
                 // create transformation
                 basegfx::B2DHomMatrix aAdaptedTransform;
diff --git a/vcl/quartz/salbmp.cxx b/vcl/quartz/salbmp.cxx
index b96e25421686..a74f6096bf5a 100644
--- a/vcl/quartz/salbmp.cxx
+++ b/vcl/quartz/salbmp.cxx
@@ -451,11 +451,13 @@ class ImplPixelFormat8 : public ImplPixelFormat
 private:
     sal_uInt8* pData;
     const BitmapPalette& mrPalette;
+    const sal_uInt8 mnPaletteCount;
 
 public:
     explicit ImplPixelFormat8( const BitmapPalette& rPalette )
         : pData(nullptr)
         , mrPalette(rPalette)
+        , mnPaletteCount(static_cast< sal_uInt8 >(rPalette.GetEntryCount()))
         {
         }
     virtual void StartLine( sal_uInt8* pLine ) override { pData = pLine; }
@@ -465,7 +467,13 @@ public:
         }
     virtual ColorData ReadPixel() override
         {
-            return mrPalette[ *pData++ ].operator Color().GetColor();
+            const sal_uInt8 nIndex(*pData++);
+
+            // Caution(!) rPalette.GetEntryCount() may be != (depth^^2)-1 (!)
+            if(nIndex < mnPaletteCount)
+                return mrPalette[nIndex].operator Color().GetColor();
+            else
+                return Color(COL_BLACK).GetColor();
         }
     virtual void WritePixel( ColorData nColor ) override
         {
@@ -481,6 +489,7 @@ class ImplPixelFormat4 : public ImplPixelFormat
 private:
     sal_uInt8* pData;
     const BitmapPalette& mrPalette;
+    const sal_uInt8 mnPaletteCount;
     sal_uInt32 mnX;
     sal_uInt32 mnShift;
 
@@ -488,6 +497,7 @@ public:
     explicit ImplPixelFormat4( const BitmapPalette& rPalette )
         : pData(nullptr)
         , mrPalette(rPalette)
+        , mnPaletteCount(static_cast< sal_uInt8 >(rPalette.GetEntryCount()))
         , mnX(0)
         , mnShift(0)
         {
@@ -508,10 +518,15 @@ public:
         }
     virtual ColorData ReadPixel() override
         {
-            const BitmapColor& rColor = mrPalette[( pData[mnX >> 1] >> mnShift) & 0x0f];
+            // Caution(!) rPalette.GetEntryCount() may be != (depth^^2)-1 (!)
+            const sal_uInt8 nIndex(( pData[mnX >> 1] >> mnShift) & 0x0f);
             mnX++;
             mnShift ^= 4;
-            return rColor.operator Color().GetColor();
+
+            if(nIndex < mnPaletteCount)
+                return mrPalette[nIndex].operator Color().GetColor();
+            else
+                return Color(COL_BLACK).GetColor();
         }
     virtual void WritePixel( ColorData nColor ) override
         {
@@ -530,12 +545,14 @@ class ImplPixelFormat1 : public ImplPixelFormat
 private:
     sal_uInt8* pData;
     const BitmapPalette& mrPalette;
+    const sal_uInt8 mnPaletteCount;
     sal_uInt32 mnX;
 
 public:
     explicit ImplPixelFormat1( const BitmapPalette& rPalette )
         : pData(nullptr)
         , mrPalette(rPalette)
+        , mnPaletteCount(static_cast< sal_uInt8 >(rPalette.GetEntryCount()))
         , mnX(0)
         {
         }
@@ -550,9 +567,14 @@ public:
         }
     virtual ColorData ReadPixel() override
         {
-            const BitmapColor& rColor = mrPalette[ (pData[mnX >> 3 ] >> ( 7 - ( mnX & 7 ) )) & 1];
+            // Caution(!) rPalette.GetEntryCount() may be != (depth^^2)-1 (!)
+            const sal_uInt8 nIndex( (pData[mnX >> 3 ] >> ( 7 - ( mnX & 7 ) )) & 1);
             mnX++;
-            return rColor.operator Color().GetColor();
+
+            if(nIndex < mnPaletteCount)
+                return mrPalette[nIndex].operator Color().GetColor();
+            else
+                return Color(COL_BLACK).GetColor();
         }
     virtual void WritePixel( ColorData nColor ) override
         {