valgrind+afl: bad rle

Change-Id: I0a9e5fc88ed1fcc7f1bd21218cabeb0adf65c9f4

1 files changed, 38 insertions, 3 deletions

diff --git a/vcl/source/gdi/dibtools.cxx b/vcl/source/gdi/dibtools.cxx
index 93f5dad69eb6..975051322bc0 100644
--- a/vcl/source/gdi/dibtools.cxx
+++ b/vcl/source/gdi/dibtools.cxx
@@ -283,9 +283,10 @@ bool ImplReadDIBPalette( SvStream& rIStm, BitmapWriteAccess& rAcc, bool bQuad )
     return( rIStm.GetError() == 0UL );
 }
 
-void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess& rAcc, bool bRLE4 )
+bool ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess& rAcc, bool bRLE4 )
 {
-    Scanline    pRLE = pBuffer;
+    Scanline pRLE = pBuffer;
+    Scanline pEndRLE = pBuffer + rHeader.nSizeImage;
     long        nY = rHeader.nHeight - 1L;
     const sal_uLong nWidth = rAcc.Width();
     sal_uLong       nCountByte;
@@ -296,8 +297,12 @@ void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
 
     do
     {
+        if (pRLE == pEndRLE)
+            return false;
         if( ( nCountByte = *pRLE++ ) == 0 )
         {
+            if (pRLE == pEndRLE)
+                return false;
             nRunByte = *pRLE++;
 
             if( nRunByte > 2 )
@@ -308,6 +313,9 @@ void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
 
                     for( sal_uLong i = 0UL; i < nCountByte; i++ )
                     {
+                        if (pRLE == pEndRLE)
+                            return false;
+
                         cTmp = *pRLE++;
 
                         if( nX < nWidth )
@@ -319,6 +327,9 @@ void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
 
                     if( nRunByte & 1 )
                     {
+                        if (pRLE == pEndRLE)
+                            return false;
+
                         if( nX < nWidth )
                             rAcc.SetPixelIndex( nY, nX++, *pRLE >> 4 );
 
@@ -326,12 +337,20 @@ void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
                     }
 
                     if( ( ( nRunByte + 1 ) >> 1 ) & 1 )
+                    {
+                        if (pRLE == pEndRLE)
+                            return false;
+
                         pRLE++;
+                    }
                 }
                 else
                 {
                     for( sal_uLong i = 0UL; i < nRunByte; i++ )
                     {
+                        if (pRLE == pEndRLE)
+                            return false;
+
                         if( nX < nWidth )
                             rAcc.SetPixelIndex( nY, nX++, *pRLE );
 
@@ -339,7 +358,12 @@ void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
                     }
 
                     if( nRunByte & 1 )
+                    {
+                        if (pRLE == pEndRLE)
+                            return false;
+
                         pRLE++;
+                    }
                 }
             }
             else if( !nRunByte )
@@ -351,12 +375,21 @@ void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
                 bEndDecoding = true;
             else
             {
+                if (pRLE == pEndRLE)
+                    return false;
+
                 nX += *pRLE++;
+
+                if (pRLE == pEndRLE)
+                    return false;
+
                 nY -= *pRLE++;
             }
         }
         else
         {
+            if (pRLE == pEndRLE)
+                return false;
             cTmp = *pRLE++;
 
             if( bRLE4 )
@@ -382,7 +415,9 @@ void ImplDecodeRLE( sal_uInt8* pBuffer, DIBV5Header& rHeader, BitmapWriteAccess&
             }
         }
     }
-    while ( !bEndDecoding && ( nY >= 0L ) );
+    while (!bEndDecoding && (nY >= 0L));
+
+    return true;
 }
 
 bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& rAcc, BitmapWriteAccess* pAccAlpha, bool bTopDown, bool& rAlphaUsed)