tdf92789 fix reading of some PICT images

(cherry picked from commit 5fa73031aa42b62ccd167f193376565df2e635fc)

Conflicts:
	filter/source/graphicfilter/ipict/ipict.cxx

add a test image

(cherry picked from commit 3f0677b86f4831b011a2baece85cf93c68646cd5)

Change-Id: I6809ef52c462958eed2329fe2d32b5cbc691194c
Reviewed-on: https://gerrit.libreoffice.org/17203
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

2 files changed, 25 insertions, 37 deletions

diff --git a/filter/qa/cppunit/data/pict/pass/tdf92789.pct b/filter/qa/cppunit/data/pict/pass/tdf92789.pct
new file mode 100644
index 000000000000..2d6f0d8848bf
--- /dev/null
+++ b/filter/qa/cppunit/data/pict/pass/tdf92789.pct
diff --git a/filter/source/graphicfilter/ipict/ipict.cxx b/filter/source/graphicfilter/ipict/ipict.cxx
index 4f0c39b93e8a..53f4c3a7f478 100644
--- a/filter/source/graphicfilter/ipict/ipict.cxx
+++ b/filter/source/graphicfilter/ipict/ipict.cxx
@@ -879,7 +879,7 @@ sal_uLong PictReader::ReadPixMapEtc( Bitmap &rBitmap, bool bBaseAddr, bool bColo
                     }
                     else
                     {
-                        nCount = static_cast<sal_uInt16>( 1 - ( ( (sal_uInt16)nFlagCounterByte ) | 0xff00 ) );
+                        nCount = static_cast<sal_uInt16>( 1 - sal_Int16( ( (sal_uInt16)nFlagCounterByte ) | 0xff00 ) );
                         pPict->ReadUChar( nDat );
                         for ( i = 0; i < nCount; i++ )
                         {
@@ -901,21 +901,10 @@ sal_uLong PictReader::ReadPixMapEtc( Bitmap &rBitmap, bool bBaseAddr, bool bColo
         if (nWidth > nRowBytes / 2)
             BITMAPERROR;
 
-        size_t nMinRecordSize;
-        if ( nRowBytes < 8 || nPackType == 1 )
-            nMinRecordSize = sizeof(sal_uInt16);
-        else if ( nRowBytes > 250 )
-            nMinRecordSize = sizeof(sal_uInt16);
-        else
-            nMinRecordSize = 1;
-
-        const size_t nMinRowWidth = nWidth * nMinRecordSize;
-        const size_t nMaxRows = pPict->remainingSize() / nMinRowWidth;
-        if (nHeight > nMaxRows)
-            BITMAPERROR;
-        const size_t nMaxCols = pPict->remainingSize() / nHeight;
-        if (nWidth > nMaxCols)
-            BITMAPERROR;
+        if ( nRowBytes < 8 || nPackType == 1 ) {
+            if (pPict->remainingSize() < sizeof(sal_uInt16) * nHeight * nWidth)
+                BITMAPERROR;
+        }
 
         for ( ny = 0; ny < nHeight; ny++ )
         {
@@ -952,10 +941,17 @@ sal_uLong PictReader::ReadPixMapEtc( Bitmap &rBitmap, bool bBaseAddr, bool bColo
                     if ( (nFlagCounterByte & 0x80) == 0)
                     {
                         nCount=((sal_uInt16)nFlagCounterByte)+1;
-                        if ( nCount + nx > nWidth)              // SJ: the RLE decoding seems not to be correct here,
-                            nCount = nWidth - nx;               // I don't want to change this until I have a bugdoc for
-                        for (i=0; i<nCount; i++)                // this case. Have a look at 32bit, there I changed the
-                        {                                       // encoding, so that it is used a straight forward array
+                        if ( nCount + nx > nWidth)
+                            nCount = nWidth - nx;
+                        if (pPict->remainingSize() < sizeof(sal_uInt16) * nCount)
+                            BITMAPERROR;
+                        /* SJ: the RLE decoding seems not to be correct here,
+                           I don't want to change this until I have a bugdoc for
+                           this case. Have a look at 32bit, there I changed the
+                           encoding, so that it is used a straight forward array
+                         */
+                        for (i=0; i<nCount; i++)
+                        {
                             pPict->ReadUInt16( nD );
                             nRed = (sal_uInt8)( nD >> 7 );
                             nGreen = (sal_uInt8)( nD >> 2 );
@@ -965,7 +961,9 @@ sal_uLong PictReader::ReadPixMapEtc( Bitmap &rBitmap, bool bBaseAddr, bool bColo
                     }
                     else
                     {
-                        nCount=(1-(((sal_uInt16)nFlagCounterByte)|0xff00));
+                        if (pPict->remainingSize() < sizeof(sal_uInt16))
+                            BITMAPERROR;
+                        nCount=(1-sal_Int16(((sal_uInt16)nFlagCounterByte)|0xff00));
                         if ( nCount + nx > nWidth )
                             nCount = nWidth - nx;
                         pPict->ReadUInt16( nD );
@@ -1039,21 +1037,7 @@ sal_uLong PictReader::ReadPixMapEtc( Bitmap &rBitmap, bool bBaseAddr, bool bColo
         {
             if ( ( nCmpCount == 3 ) || ( nCmpCount == 4 ) )
             {
-                size_t nMinRecordSize;
-                if (nRowBytes > 250)
-                    nMinRecordSize = sizeof(sal_uInt16);
-                else
-                    nMinRecordSize = 1;
-
-                const size_t nMinRowWidth = nWidth * nMinRecordSize;
-                const size_t nMaxRows = pPict->remainingSize() / nMinRowWidth;
-                if (nHeight > nMaxRows)
-                    BITMAPERROR;
-                const size_t nMaxWidth = pPict->remainingSize() / nHeight;
-                if (nWidth > nMaxWidth)
-                    BITMAPERROR;
-
-                boost::scoped_array<sal_uInt8> pScanline(new sal_uInt8[static_cast<size_t>(nWidth) * nCmpCount]);
+                std::unique_ptr<sal_uInt8[]> pScanline(new sal_uInt8[static_cast<size_t>(nWidth) * nCmpCount]);
                 for ( ny = 0; ny < nHeight; ny++ )
                 {
                     nSrcBitsPos = pPict->Tell();
@@ -1077,6 +1061,8 @@ sal_uLong PictReader::ReadPixMapEtc( Bitmap &rBitmap, bool bBaseAddr, bool bColo
                             nCount = ( (sal_uInt16)nFlagCounterByte ) + 1;
                             if ( ( i + nCount ) > static_cast<size_t>(nWidth) * nCmpCount )
                                 nCount = static_cast<size_t>(nWidth) * nCmpCount - i;
+                            if (pPict->remainingSize() < nCount)
+                                BITMAPERROR;
                             while( nCount-- )
                             {
                                 pPict->ReadUChar( nDat );
@@ -1085,7 +1071,9 @@ sal_uLong PictReader::ReadPixMapEtc( Bitmap &rBitmap, bool bBaseAddr, bool bColo
                         }
                         else
                         {
-                            nCount = ( 1 - ( ( (sal_uInt16)nFlagCounterByte ) | 0xff00 ) );
+                            if (pPict->remainingSize() < 1)
+                                BITMAPERROR;
+                            nCount = ( 1 - sal_Int16( ( (sal_uInt16)nFlagCounterByte ) | 0xff00 ) );
                             if ( ( i + nCount ) > static_cast<size_t>(nWidth) * nCmpCount)
                                 nCount = static_cast<size_t>(nWidth) * nCmpCount - i;
                             pPict->ReadUChar( nDat );