Avoid overflow in PBMReader::ImplReadHeader

...as found by UBSan in CppunitTest_filter_ppm_test on
filter/qa/cppunit/data/pbm/fail/crash-1.pbm

Change-Id: Ib7c50ef1f07aba6b78f79c608be69c3dac38ddfe
(cherry picked from commit 662498ab80833a2b671c247fb859603632e52105)
Reviewed-on: https://gerrit.libreoffice.org/17984
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

1 files changed, 24 insertions, 0 deletions

diff --git a/filter/source/graphicfilter/ipbm/ipbm.cxx b/filter/source/graphicfilter/ipbm/ipbm.cxx
index 18b32498eece..d7cf94145333 100644
--- a/filter/source/graphicfilter/ipbm/ipbm.cxx
+++ b/filter/source/graphicfilter/ipbm/ipbm.cxx
@@ -218,17 +218,41 @@ bool PBMReader::ImplReadHeader()
             nDat -= '0';
             if ( nCount == 0 )
             {
+                if (mnWidth > SAL_MAX_INT32 / 10)
+                {
+                    return false;
+                }
                 mnWidth *= 10;
+                if (nDat > SAL_MAX_INT32 - mnWidth)
+                {
+                    return false;
+                }
                 mnWidth += nDat;
             }
             else if ( nCount == 1 )
             {
+                if (mnHeight > SAL_MAX_INT32 / 10)
+                {
+                    return false;
+                }
                 mnHeight *= 10;
+                if (nDat > SAL_MAX_INT32 - mnHeight)
+                {
+                    return false;
+                }
                 mnHeight += nDat;
             }
             else if ( nCount == 2 )
             {
+                if (mnMaxVal > std::numeric_limits<sal_uLong>::max() / 10)
+                {
+                    return false;
+                }
                 mnMaxVal *= 10;
+                if (nDat > std::numeric_limits<sal_uLong>::max() - mnMaxVal)
+                {
+                    return false;
+                }
                 mnMaxVal += nDat;
             }
         }