tdf#99258 bail out if we fail to reserve the texture

Change-Id: I830e313352b69a7665bff953aadb1334be0dc847
Reviewed-on: https://gerrit.libreoffice.org/24509
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Tomaž Vajngerl <quikee@gmail.com>

3 files changed, 32 insertions, 8 deletions

diff --git a/vcl/inc/opengl/AccumulatedTextures.hxx b/vcl/inc/opengl/AccumulatedTextures.hxx
index e74c06535f69..bc40c48f3ab0 100644
--- a/vcl/inc/opengl/AccumulatedTextures.hxx
+++ b/vcl/inc/opengl/AccumulatedTextures.hxx
@@ -88,10 +88,13 @@ public:
         maEntries.clear();
     }
 
-    void insert(OpenGLTexture& rTexture, const SalColor& aColor, const SalTwoRect& r2Rect)
+    bool insert(OpenGLTexture& rTexture, const SalColor& aColor, const SalTwoRect& r2Rect)
     {
         GLuint nTextureId = rTexture.Id();
 
+        if (!rTexture)
+            return false;
+
         if (maEntries.find(nTextureId) == maEntries.end())
         {
             OpenGLTexture aWholeTexture(rTexture.GetWholeTexture());
@@ -100,6 +103,8 @@ public:
 
         std::unique_ptr<AccumulatedTexturesEntry>& rEntry = maEntries[nTextureId];
         rEntry->insert(rTexture, aColor, r2Rect);
+
+        return true;
     }
 
     AccumulatedTexturesMap& getAccumulatedTexturesMap()
diff --git a/vcl/opengl/texture.cxx b/vcl/opengl/texture.cxx
index aa796eb6cde2..0999484e0a60 100644
--- a/vcl/opengl/texture.cxx
+++ b/vcl/opengl/texture.cxx
@@ -457,7 +457,9 @@ void OpenGLTexture::GetWholeCoord( GLfloat* pCoord ) const
 
 OpenGLTexture OpenGLTexture::GetWholeTexture()
 {
-    return OpenGLTexture(mpImpl, Rectangle(Point(0, 0), Size(mpImpl->mnWidth, mpImpl->mnHeight)), -1);
+    if (mpImpl)
+        return OpenGLTexture(mpImpl, Rectangle(Point(0, 0), Size(mpImpl->mnWidth, mpImpl->mnHeight)), -1);
+    return OpenGLTexture();
 }
 
 GLenum OpenGLTexture::GetFilter() const
diff --git a/vcl/win/gdi/winlayout.cxx b/vcl/win/gdi/winlayout.cxx
index 391f7008c10b..be97cfe59a21 100644
--- a/vcl/win/gdi/winlayout.cxx
+++ b/vcl/win/gdi/winlayout.cxx
@@ -127,10 +127,12 @@ public:
         gGlobalGlyphCache.get()->maGlyphCaches.erase(this);
     }
 
-    void ReserveTextureSpace(OpenGLGlyphDrawElement& rElement, int nWidth, int nHeight)
+    bool ReserveTextureSpace(OpenGLGlyphDrawElement& rElement, int nWidth, int nHeight)
     {
         GlobalGlyphCache* pGlobalGlyphCache = gGlobalGlyphCache.get();
         rElement.maTexture = pGlobalGlyphCache->maPackedTextureAtlas.Reserve(nWidth, nHeight);
+        if (!rElement.maTexture)
+            return false;
         std::vector<GLuint> aTextureIDs = pGlobalGlyphCache->maPackedTextureAtlas.ReduceTextureNumber(8);
         if (!aTextureIDs.empty())
         {
@@ -139,6 +141,7 @@ public:
                 pGlyphCache->RemoveTextures(aTextureIDs);
             }
         }
+        return true;
     }
 
     void RemoveTextures(std::vector<GLuint>& rTextureIDs)
@@ -521,8 +524,11 @@ bool WinFontInstance::CacheGlyphToAtlas(bool bRealGlyphIndices, int nGlyphIndex,
 
     pTxt->ReleaseFont();
 
-    maGlyphCache.ReserveTextureSpace(aElement, nBitmapWidth, nBitmapHeight);
-    aDC.copyToTexture(aElement.maTexture);
+    if (!maGlyphCache.ReserveTextureSpace(aElement, nBitmapWidth, nBitmapHeight))
+        return false;
+    if (!aDC.copyToTexture(aElement.maTexture))
+        return false;
+
     maGlyphCache.PutDrawElementInCache(aElement, nGlyphIndex);
 
     SelectFont(aDC.getCompatibleHDC(), hOrigFont);
@@ -1461,8 +1467,10 @@ bool SimpleWinLayout::CacheGlyphs(SalGraphics& rGraphics) const
         }
 
         if (!mrWinFontEntry.GetGlyphCache().IsGlyphCached(nCodePoint))
-            assert(mrWinFontEntry.CacheGlyphToAtlas(false, nCodePoint, *this, rGraphics));
-
+        {
+            if (!mrWinFontEntry.CacheGlyphToAtlas(false, nCodePoint, *this, rGraphics))
+                return false;
+        }
     }
 
     return true;
@@ -1507,6 +1515,9 @@ bool SimpleWinLayout::DrawCachedGlyphs(SalGraphics& rGraphics) const
         OpenGLGlyphDrawElement& rElement(mrWinFontEntry.GetGlyphCache().GetDrawElement(nCodePoint));
         OpenGLTexture& rTexture = rElement.maTexture;
 
+        if (!rTexture)
+            return false;
+
         SalTwoRect a2Rects(0, 0,
                            rTexture.GetWidth(), rTexture.GetHeight(),
                            nAdvance + aPos.X() - rElement.getExtraOffset() + rElement.maLeftOverhangs,
@@ -2728,7 +2739,10 @@ bool UniscribeLayout::CacheGlyphs(SalGraphics& rGraphics) const
         {
             int nCodePoint = mpOutGlyphs[i];
             if (!mrWinFontEntry.GetGlyphCache().IsGlyphCached(nCodePoint))
-                assert(mrWinFontEntry.CacheGlyphToAtlas(true, nCodePoint, *this, rGraphics));
+            {
+                if (!mrWinFontEntry.CacheGlyphToAtlas(true, nCodePoint, *this, rGraphics))
+                    return false;
+            }
         }
     }
 
@@ -3019,6 +3033,9 @@ bool UniscribeLayout::DrawCachedGlyphsUsingTextures(SalGraphics& rGraphics) cons
             OpenGLGlyphDrawElement& rElement = mrWinFontEntry.GetGlyphCache().GetDrawElement(mpOutGlyphs[i]);
             OpenGLTexture& rTexture = rElement.maTexture;
 
+            if (!rTexture)
+                return false;
+
             if (rElement.mbVertical)
             {
                 SalTwoRect a2Rects(0, 0,