ofz#574 sanitize png palette indexes

clarify we're not setting a pal index in the mask

no logic changed intended, SetPixelIndex expands
to this, just want to clarify that this is an alpha
value, not an index into a palette

(cherry picked from commit e5b23d924d60e7a0fb67c44c6dad3f4cb3bd5ddc)

Change-Id: I006ffdf5c8c751e7e07b133716022e2a9b154478
e9bb188b116adf3eab4e2bb1d6da8badbaead88b

crashtesting: fix tdf95481-1.odg reexport to odg failure

revealed since

commit 81e3ca4f60e6ac0823c1233841c22a759cfe937f
Author: Tor Lillqvist <tml@collabora.com>
Date:   Tue Jun 21 10:34:21 2016 +0300

    Use real assert() instead of DBG_ASSERT()

sanitize invalid palette entry indexes at the outer perimeter on initial load
to try and avoid having to do it in all sort of places in the interior.

png spec says that the palette has to appear before the first IDAT so we
should always know the palette size here

Change-Id: I6e04223adce1c88d037f9cf34862e6f54e381bb0
(cherry picked from commit 9d2fa0bfdc93fa9bfed7dea89d603b1611d7a6e0)
Reviewed-on: https://gerrit.libreoffice.org/34036
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Miklos Vajna <vmiklos@collabora.co.uk>

1 files changed, 38 insertions, 5 deletions

diff --git a/vcl/source/gdi/pngread.cxx b/vcl/source/gdi/pngread.cxx
index 377d41be6df6..e6c9abea5e1c 100644
--- a/vcl/source/gdi/pngread.cxx
+++ b/vcl/source/gdi/pngread.cxx
@@ -1122,6 +1122,37 @@ void PNGReaderImpl::ImplApplyFilter()
     memcpy( mpScanPrior, mpInflateInBuf, mnScansize );
 }
 
+namespace
+{
+    sal_uInt8 SanitizePaletteIndex(sal_uInt8 nIndex, sal_uInt16 nPaletteEntryCount)
+    {
+        if (nIndex >= nPaletteEntryCount)
+        {
+            auto nSanitizedIndex = nIndex % nPaletteEntryCount;
+            SAL_WARN_IF(nIndex != nSanitizedIndex, "vcl", "invalid colormap index: "
+                        << static_cast<unsigned int>(nIndex) << ", colormap len is: "
+                        << nPaletteEntryCount);
+            nIndex = nSanitizedIndex;
+        }
+        return nIndex;
+    }
+
+    void SanitizePaletteIndexes(sal_uInt8* pEntries, int nLen, BitmapWriteAccess* pAcc)
+    {
+        sal_uInt16 nPaletteEntryCount = pAcc->GetPaletteEntryCount();
+        for (int nX = 0; nX < nLen; ++nX)
+        {
+            if (pEntries[nX] >= nPaletteEntryCount)
+            {
+                SAL_WARN("vcl.gdi", "invalid colormap index: "
+                          << static_cast<unsigned int>(pEntries[nX]) << ", colormap len is: "
+                          << nPaletteEntryCount);
+                pEntries[nX] = pEntries[nX] % nPaletteEntryCount;
+            }
+        }
+    }
+}
+
 // ImplDrawScanlines draws the complete Scanline (nY) into the target bitmap
 // In interlace mode the parameter nXStart and nXAdd append to the currently used pass
 
@@ -1137,7 +1168,7 @@ void PNGReaderImpl::ImplDrawScanline( sal_uInt32 nXStart, sal_uInt32 nXAdd )
     // => TODO; also do this for nX here instead of in the ImplSet*Pixel() methods
     const sal_uInt32 nY = mnYpos >> mnPreviewShift;
 
-    const sal_uInt8* pTmp = mpInflateInBuf + 1;
+    sal_uInt8* pTmp = mpInflateInBuf + 1;
     if ( mpAcc->HasPalette() ) // alphachannel is not allowed by pictures including palette entries
     {
         switch ( mpAcc->GetBitCount() )
@@ -1304,6 +1335,8 @@ void PNGReaderImpl::ImplDrawScanline( sal_uInt32 nXStart, sal_uInt32 nXAdd )
                         if( nXAdd == 1 && mnPreviewShift == 0 )  // copy raw line data if possible
                         {
                             int nLineBytes = maOrigSize.Width();
+                            if (mbPalette)
+                                SanitizePaletteIndexes(pTmp, nLineBytes, mpAcc);
                             mpAcc->CopyScanline( nY, pTmp, ScanlineFormat::N8BitPal, nLineBytes );
                         }
                         else
@@ -1592,7 +1625,7 @@ void PNGReaderImpl::ImplSetPixel( sal_uInt32 nY, sal_uInt32 nX, sal_uInt8 nPalIn
         return;
     nX >>= mnPreviewShift;
 
-    mpAcc->SetPixelIndex( nY, nX, nPalIndex );
+    mpAcc->SetPixelIndex(nY, nX, SanitizePaletteIndex(nPalIndex, mpAcc->GetPaletteEntryCount()));
 }
 
 void PNGReaderImpl::ImplSetTranspPixel( sal_uInt32 nY, sal_uInt32 nX, const BitmapColor& rBitmapColor, bool bTrans )
@@ -1618,8 +1651,8 @@ void PNGReaderImpl::ImplSetAlphaPixel( sal_uInt32 nY, sal_uInt32 nX,
         return;
     nX >>= mnPreviewShift;
 
-    mpAcc->SetPixelIndex( nY, nX, nPalIndex );
-    mpMaskAcc->SetPixelIndex( nY, nX, ~nAlpha );
+    mpAcc->SetPixelIndex(nY, nX, SanitizePaletteIndex(nPalIndex, mpAcc->GetPaletteEntryCount()));
+    mpMaskAcc->SetPixel(nY, nX, BitmapColor(~nAlpha));
 }
 
 void PNGReaderImpl::ImplSetAlphaPixel( sal_uInt32 nY, sal_uInt32 nX,
@@ -1633,7 +1666,7 @@ void PNGReaderImpl::ImplSetAlphaPixel( sal_uInt32 nY, sal_uInt32 nX,
     mpAcc->SetPixel( nY, nX, rBitmapColor );
     if (!mpMaskAcc)
         return;
-    mpMaskAcc->SetPixelIndex( nY, nX, ~nAlpha );
+    mpMaskAcc->SetPixel(nY, nX, BitmapColor(~nAlpha));
 }
 
 sal_uInt32 PNGReaderImpl::ImplReadsal_uInt32()