various sft fixes

backport removing unneeded code

use ptr diff rather than int

(cherry picked from commit 5b426038a7befcaf0d05824ffb20200ff8833ad3)

fix mem leak

(cherry picked from commit 603cb6cf31a5212d03736a552770e5734b0e8066)

check more table sizes

(cherry picked from commit 1828490bb22f1c8273c4a9f5b1db819b173ca70d)

check cmap offset

(cherry picked from commit 683d9883ad8fd6568e6a7832e5bb347c1d043e4b)

check tableoffset against size

(cherry picked from commit ae73c3ff112e1ed38eb4678ac5745990661a2e66)

check ntables offset

(cherry picked from commit 75a171a405afd6eac236af93aa9d29a9c3ec9c64)

use safeint on calculations

(cherry picked from commit 139b6c6cf898467098f3a6f29fa84013a182285f)

simplify returns

(cherry picked from commit e8b2aad1cb2107304761e68aa380b5c29d8ef22f)

extend to cover the last byte needed

(cherry picked from commit c7b0117f26a386c98a721ff3897479c268103d74)

bounds check

(cherry picked from commit bb32616bdd6e3b327654bab0e1d790d8d50b893d)

move largest bounds check to start

(cherry picked from commit f832198e6a33052c7cc86b25843badfb962a2ae8)

android:update safeint header to use the else implementation with clang

Clang toolchain does not defines the __builtin_mul_overflow for 32-bit
ARM. So, fallback to else implementation of checked_multiply when
building Android with Clang

Reviewed-on: https://gerrit.libreoffice.org/39005
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Miklos Vajna <vmiklos@collabora.co.uk>
(cherry picked from commit 2149d4a88e9dc88c30e7475f8ea317e5c7b78529)

add checked_add

Reviewed-on: https://gerrit.libreoffice.org/43779
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>
(cherry picked from commit 9cfb27ae6cb94f0a853ff70e9ad9f3109d305a94)

Change-Id: I0b59f1b93d31a7cb5b8db2a21736db11aed46536
a174fd94c57cc3c899c10e1c0dc5968965a50427
0d329357ac282d4652b0f7ebc401cbd51963461b
004a6d322f60d916cc4635b362ce948e8a10d7c7
5aae26c38f3645020f0e1d6d7b6877c2727af1b4
a79be052dd3f6b6ed38fb326558924c853af5fff
d75f5f4b578fd176c17e5763569f1403260c6594
c063786ba41aa9a985f505e62b43d3d543a0d48f
1a67cb2f1c686032438852fec1267a59fbd04d7f
5177d42b47a4bca614878dce4a69ab16b5cfe163
6d32a6b6f1dd91db42a3f154700ea55603f0e4dd
03fe80f9568759b829fac4e9bcfd496efebe6a26
6120606f521ce121541a5b7f1150229258012d55
10cba898bba528f5f1bfbd583e27a6821c789ab9
Reviewed-on: https://gerrit.libreoffice.org/49485
Reviewed-by: Michael Stahl <mstahl@redhat.com>
Tested-by: Jenkins <ci@libreoffice.org>

(cherry picked from commit 9826e506a64260d577d463fe49fe46d1e10848e6)

3 files changed, 106 insertions, 51 deletions

diff --git a/include/o3tl/safeint.hxx b/include/o3tl/safeint.hxx
index ce144d22d9ea..5ebf353b6bd6 100644
--- a/include/o3tl/safeint.hxx
+++ b/include/o3tl/safeint.hxx
@@ -29,13 +29,23 @@ template<typename T> inline bool checked_multiply(T a, T b, T& result)
     return !msl::utilities::SafeMultiply(a, b, result);
 }
 
-#elif (defined __GNUC__ && __GNUC__ >= 5) || (__has_builtin(__builtin_mul_overflow))
+template<typename T> inline bool checked_add(T a, T b, T& result)
+{
+    return !msl::utilities::SafeAdd(a, b, result);
+}
+
+#elif (defined __GNUC__ && __GNUC__ >= 5) || (__has_builtin(__builtin_mul_overflow) && !(defined ANDROID && defined __clang__))
 
 template<typename T> inline bool checked_multiply(T a, T b, T& result)
 {
     return __builtin_mul_overflow(a, b, &result);
 }
 
+template<typename T> inline bool checked_add(T a, T b, T& result)
+{
+    return __builtin_add_overflow(a, b, &result);
+}
+
 #else
 
 //https://www.securecoding.cert.org/confluence/display/c/INT32-C.+Ensure+that+operations+on+signed+integers+do+not+result+in+overflow
@@ -80,6 +90,31 @@ template<typename T> inline typename std::enable_if<std::is_unsigned<T>::value,
     return false;
 }
 
+//https://www.securecoding.cert.org/confluence/display/c/INT32-C.+Ensure+that+operations+on+signed+integers+do+not+result+in+overflow
+template<typename T> inline typename std::enable_if<std::is_signed<T>::value, bool>::type checked_add(T a, T b, T& result)
+{
+    if (((b > 0) && (a > (std::numeric_limits<T>::max() - b))) ||
+        ((b < 0) && (a < (std::numeric_limits<T>::min() - b)))) {
+        return true;
+    }
+
+    result = a + b;
+
+    return false;
+}
+
+//https://www.securecoding.cert.org/confluence/display/c/INT30-C.+Ensure+that+unsigned+integer+operations+do+not+wrap
+template<typename T> inline typename std::enable_if<std::is_unsigned<T>::value, bool>::type checked_add(T a, T b, T& result)
+{
+    if (std::numeric_limits<T>::max() - a < b) {
+        return true;/* Handle error */
+    }
+
+    result = a + b;
+
+    return false;
+}
+
 #endif
 
 }
diff --git a/vcl/inc/sft.hxx b/vcl/inc/sft.hxx
index 2efe115c7ccd..a000c59443f8 100644
--- a/vcl/inc/sft.hxx
+++ b/vcl/inc/sft.hxx
@@ -190,19 +190,12 @@ namespace vcl
         int   descender;          /**< typographic descent.                                    */
         int   linegap;            /**< typographic line gap.\ Negative values are treated as
                                      zero in Win 3.1, System 6 and System 7.                 */
-        int   vascent;            /**< typographic ascent for vertical writing mode            */
-        int   vdescent;           /**< typographic descent for vertical writing mode           */
         int   typoAscender;       /**< OS/2 portable typographic ascender                      */
         int   typoDescender;      /**< OS/2 portable typographic descender                     */
         int   typoLineGap;        /**< OS/2 portable typographic line gap                       */
         int   winAscent;          /**< ascender metric for Windows                             */
         int   winDescent;         /**< descender metric for Windows                            */
         bool  symbolEncoded;      /**< true: MS symbol encoded */
-        int   rangeFlag;          /**< if set to 1 Unicode Range flags are applicable          */
-        sal_uInt32 ur1;           /**< bits 0 - 31 of Unicode Range flags                      */
-        sal_uInt32 ur2;           /**< bits 32 - 63 of Unicode Range flags                     */
-        sal_uInt32 ur3;           /**< bits 64 - 95 of Unicode Range flags                     */
-        sal_uInt32 ur4;           /**< bits 96 - 127 of Unicode Range flags                    */
         sal_uInt8  panose[10];    /**< PANOSE classification number                            */
         sal_uInt32 typeFlags;     /**< type flags (copyright bits + PS-OpenType flag)       */
         sal_uInt16 fsSelection;   /**< OS/2 fsSelection */
@@ -537,8 +530,6 @@ namespace vcl
 /*- private definitions */
 
     struct TrueTypeFont {
-        sal_uInt32 tag;
-
         char        *fname;
         sal_Int32   fsize;
         sal_uInt8   *ptr;
diff --git a/vcl/source/fontsubset/sft.cxx b/vcl/source/fontsubset/sft.cxx
index 52e9cf540fd3..a12ad83dccec 100644
--- a/vcl/source/fontsubset/sft.cxx
+++ b/vcl/source/fontsubset/sft.cxx
@@ -39,7 +39,7 @@
 #include "xlat.hxx"
 #include <rtl/crc.h>
 #include <rtl/ustring.hxx>
-
+#include <o3tl/safeint.hxx>
 #include <osl/endian.h>
 #include <algorithm>
 
@@ -104,9 +104,6 @@ typedef struct {
     sal_uInt32 *offs;             /* array of nGlyphs offsets */
 } GlyphOffsets;
 
-/* private tags */
-static const sal_uInt32 TTFontClassTag = 0x74746663;  /* 'ttfc' */
-
 static const sal_uInt32 T_true = 0x74727565;        /* 'true' */
 static const sal_uInt32 T_ttcf = 0x74746366;        /* 'ttcf' */
 static const sal_uInt32 T_otto = 0x4f54544f;        /* 'OTTO' */
@@ -1328,6 +1325,13 @@ static void FindCmap(TrueTypeFont *ttf)
     }
 
     if (ttf->cmapType != CMAP_NOT_USABLE) {
+        if( (ttf->cmap - ttf->ptr + 2U) > static_cast<sal_uInt32>(ttf->fsize) ) {
+            ttf->cmapType = CMAP_NOT_USABLE;
+            ttf->cmap = nullptr;
+        }
+    }
+
+    if (ttf->cmapType != CMAP_NOT_USABLE) {
         switch (GetUInt16(ttf->cmap, 0)) {
             case 0: ttf->mapper = getGlyph0; break;
             case 2: ttf->mapper = getGlyph2; break;
@@ -1448,7 +1452,6 @@ static void allocTrueTypeFont( TrueTypeFont** ttf )
     *ttf = static_cast<TrueTypeFont*>(calloc(1,sizeof(TrueTypeFont)));
     if( *ttf != nullptr )
     {
-        (*ttf)->tag = 0;
         (*ttf)->fname = nullptr;
         (*ttf)->fsize = -1;
         (*ttf)->ptr = nullptr;
@@ -1533,10 +1536,41 @@ int OpenTTFontBuffer(const void* pBuffer, sal_uInt32 nLen, sal_uInt32 facenum, T
     return doOpenTTFont( facenum, *ttf );
 }
 
+namespace {
+
+bool withinBounds(sal_uInt32 tdoffset, sal_uInt32 moreoffset, sal_uInt32 len, sal_uInt32 available)
+{
+    sal_uInt32 result;
+    if (o3tl::checked_add(tdoffset, moreoffset, result))
+        return false;
+    if (o3tl::checked_add(result, len, result))
+        return false;
+    return result <= available;
+}
+
+class TTFontCloser
+{
+    TrueTypeFont* m_font;
+public:
+    TTFontCloser(TrueTypeFont* t)
+        : m_font(t)
+    {
+    }
+    void clear() { m_font = nullptr; }
+    ~TTFontCloser()
+    {
+        if (m_font)
+            CloseTTFont(m_font);
+    }
+};
+
+}
+
 static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
 {
+    TTFontCloser aCloseGuard(t);
+
     if (t->fsize < 4) {
-        CloseTTFont(t);
         return SF_TTFORMAT;
     }
     int i;
@@ -1550,27 +1584,28 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
     } else if (TTCTag == T_otto) {                         /* PS-OpenType font */
         tdoffset = 0;
     } else if (TTCTag == T_ttcf) {                         /* TrueType collection */
+        if (!withinBounds(12, 4 * facenum, sizeof(sal_uInt32), t->fsize)) {
+            return SF_FONTNO;
+        }
         sal_uInt32 Version = GetUInt32(t->ptr, 4);
         if (Version != 0x00010000 && Version != 0x00020000) {
-            CloseTTFont(t);
             return SF_TTFORMAT;
         }
         if (facenum >= GetUInt32(t->ptr, 8)) {
-            CloseTTFont(t);
             return SF_FONTNO;
         }
         tdoffset = GetUInt32(t->ptr, 12 + 4 * facenum);
     } else {
-        CloseTTFont(t);
         return SF_TTFORMAT;
     }
 
-    /* magic number */
-    t->tag = TTFontClassTag;
+    if (withinBounds(tdoffset, 0, 4 + sizeof(sal_uInt16), t->fsize)) {
+        t->ntables = GetUInt16(t->ptr + tdoffset, 4);
+    }
 
-    t->ntables = GetUInt16(t->ptr + tdoffset, 4);
-    if( t->ntables >= 128 )
+    if (t->ntables >= 128 || t->ntables == 0) {
         return SF_TTFORMAT;
+    }
 
     t->tables = static_cast<const sal_uInt8**>(calloc(NUM_TAGS, sizeof(sal_uInt8 *)));
     assert(t->tables != nullptr);
@@ -1582,7 +1617,7 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
         int nIndex;
         const sal_uInt32 nStart = tdoffset + 12;
         const sal_uInt32 nOffset = 16 * i;
-        if (nStart + nOffset + sizeof(sal_uInt32) <=  static_cast<sal_uInt32>(t->fsize))
+        if (withinBounds(nStart, nOffset, sizeof(sal_uInt32), t->fsize))
             tag = GetUInt32(t->ptr + nStart, nOffset);
         else
             tag = static_cast<sal_uInt32>(-1);
@@ -1607,9 +1642,10 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
             case T_CFF:  nIndex = O_CFF; break;
             default: nIndex = -1; break;
         }
-        if( nIndex >= 0 ) {
-            sal_uInt32 nTableOffset = GetUInt32(t->ptr + tdoffset + 12, 16 * i + 8);
-            length = GetUInt32(t->ptr + tdoffset + 12, 16 * i + 12);
+
+        if ((nIndex >= 0) && withinBounds(nStart, nOffset, 12 + sizeof(sal_uInt32), t->fsize)) {
+            sal_uInt32 nTableOffset = GetUInt32(t->ptr + nStart, nOffset + 8);
+            length = GetUInt32(t->ptr + nStart, nOffset + 12);
             t->tables[nIndex] = t->ptr + nTableOffset;
             t->tlens[nIndex] = length;
         }
@@ -1618,8 +1654,9 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
     /* Fixup offsets when only a TTC extract was provided */
     if( facenum == (sal_uInt32)~0 ) {
         sal_uInt8* pHead = const_cast<sal_uInt8*>(t->tables[O_head]);
-        if( !pHead )
+        if (!pHead) {
             return SF_TTFORMAT;
+        }
         /* limit Head candidate to TTC extract's limits */
         if( pHead > t->ptr + (t->fsize - 54) )
             pHead = t->ptr + (t->fsize - 54);
@@ -1635,8 +1672,9 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
                 break;
             }
         }
-        if( p <= t->ptr )
+        if (p <= t->ptr) {
             return SF_TTFORMAT;
+        }
     }
 
     /* Check the table offsets after TTC correction */
@@ -1658,7 +1696,7 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
         }
         else if( const_cast<sal_uInt8*>(t->tables[i]) + t->tlens[i] > t->ptr + t->fsize )
         {
-            int nMaxLen = (t->ptr + t->fsize) - t->tables[i];
+            sal_PtrDiff nMaxLen = (t->ptr + t->fsize) - t->tables[i];
             if( nMaxLen < 0 )
                 nMaxLen = 0;
             t->tlens[i] = nMaxLen;
@@ -1676,7 +1714,6 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
      */
 
     if( !(getTable(t, O_maxp) && getTable(t, O_head) && getTable(t, O_name) && getTable(t, O_cmap)) ) {
-        CloseTTFont(t);
         return SF_TTFORMAT;
     }
 
@@ -1687,14 +1724,12 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
     table = getTable(t, O_head);
     table_size = getTableSize(t, O_head);
     if (table_size < 52) {
-        CloseTTFont(t);
         return SF_TTFORMAT;
     }
     t->unitsPerEm = GetUInt16(table, 18);
     int indexfmt = GetInt16(table, 50);
 
     if( ((indexfmt != 0) && (indexfmt != 1)) || (t->unitsPerEm <= 0) ) {
-        CloseTTFont(t);
         return SF_TTFORMAT;
     }
 
@@ -1718,7 +1753,6 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
         /* TODO: implement to get subsetting */
         assert(t->goffsets != nullptr);
     } else {
-        CloseTTFont(t);
         return SF_TTFORMAT;
     }
 
@@ -1735,6 +1769,8 @@ static int doOpenTTFont( sal_uInt32 facenum, TrueTypeFont* t )
     GetKern(t);
     ReadGSUB( t, 0, 0 );
 
+    aCloseGuard.clear();
+
     return SF_OK;
 }
 
@@ -2605,13 +2641,6 @@ void GetTTGlobalFontInfo(TrueTypeFont *ttf, TTGlobalFontInfo *info)
             if( info->winDescent > 5*UPEm )
                 info->winDescent = XUnits(UPEm, GetInt16(table, 76));
         }
-        if (ttf->cmapType == CMAP_MS_Unicode) {
-            info->rangeFlag = 1;
-            info->ur1 = GetUInt32(table, 42);
-            info->ur2 = GetUInt32(table, 46);
-            info->ur3 = GetUInt32(table, 50);
-            info->ur4 = GetUInt32(table, 54);
-        }
         memcpy(info->panose, table + 32, 10);
         info->typeFlags = GetUInt16( table, 8 );
         if( getTable(ttf, O_CFF) )
@@ -2625,24 +2654,24 @@ void GetTTGlobalFontInfo(TrueTypeFont *ttf, TTGlobalFontInfo *info)
     }
 
     table = getTable(ttf, O_head);      /* 'head' tables is always there */
-    info->xMin = XUnits(UPEm, GetInt16(table, 36));
-    info->yMin = XUnits(UPEm, GetInt16(table, 38));
-    info->xMax = XUnits(UPEm, GetInt16(table, 40));
-    info->yMax = XUnits(UPEm, GetInt16(table, 42));
-    info->macStyle = GetInt16(table, 44);
+    table_size = getTableSize(ttf, O_head);
+    if (table_size >= 46) {
+        info->xMin = XUnits(UPEm, GetInt16(table, 36));
+        info->yMin = XUnits(UPEm, GetInt16(table, 38));
+        info->xMax = XUnits(UPEm, GetInt16(table, 40));
+        info->yMax = XUnits(UPEm, GetInt16(table, 42));
+        info->macStyle = GetInt16(table, 44);
+    }
 
     table = getTable(ttf, O_hhea);
-    if (table) {
+    table_size = getTableSize(ttf, O_hhea);
+    if (table && table_size >= 10) {
         info->ascender  = XUnits(UPEm, GetInt16(table, 4));
         info->descender = XUnits(UPEm, GetInt16(table, 6));
         info->linegap   = XUnits(UPEm, GetInt16(table, 8));
     }
 
     table = getTable(ttf, O_vhea);
-    if (table) {
-        info->vascent  = XUnits(UPEm, GetInt16(table, 4));
-        info->vdescent = XUnits(UPEm, GetInt16(table, 6));
-    }
 }
 
 GlyphData *GetTTRawGlyphData(TrueTypeFont *ttf, sal_uInt32 glyphID)