tdf#107166 improve AA mode selection, retry, more checks

Major problem when setting the render mode and the text antialias
mode is that when you set the render mode to something that isn't
compatible with the text antialias mode, then every next call will
cause an error (invalid parameters). So we need to be sure that we
never set incompatible modes. Additionally we just need to set it
one time when we create the surface and not every time we draw.

If we get the D2DERR_RECREATE_TARGET we can create a new render
target and retry the whole call. Somethimes this is not possible
so we try 3 times and the give up.

We need to add more checks where we exit early or not continue with
some calls as any additional calls could taint the draw state and
some things wouldn't be drawn. For example if we calculate the
sizes of 0 glyphs we shouldn't continue with binding the hDC with
an "empty" rectangle. This will fail and cause some text that is
called afterwards to not draw.

Change-Id: I8f802da2e7ece0bf16d665f2e5011ffaa5267fca
Reviewed-on: https://gerrit.libreoffice.org/40260
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>
Tested-by: Jenkins <ci@libreoffice.org>

2 files changed, 91 insertions, 42 deletions

diff --git a/vcl/inc/win/winlayout.hxx b/vcl/inc/win/winlayout.hxx
index 128ff1f96a47..e070ef5a7249 100644
--- a/vcl/inc/win/winlayout.hxx
+++ b/vcl/inc/win/winlayout.hxx
@@ -199,8 +199,9 @@ public:
 enum class D2DTextAntiAliasMode
 {
     Default,
-    ClearType,
+    Aliased,
     AntiAliased,
+    ClearType,
 };
 
 class D2DWriteTextOutRenderer : public TextOutRenderer
@@ -225,13 +226,7 @@ public:
         SalGraphics &rGraphics,
         HDC hDC) override;
 
-    bool BindDC(HDC hDC, tools::Rectangle const & rRect = tools::Rectangle(0, 0, 1, 1))
-    {
-        if (rRect.GetWidth() == 0 || rRect.GetHeight() == 0)
-            return false;
-        RECT const rc = { rRect.Left(), rRect.Top(), rRect.Right(), rRect.Bottom() };
-        return SUCCEEDED(mpRT->BindDC(hDC, &rc));
-    }
+    HRESULT BindDC(HDC hDC, tools::Rectangle const & rRect = tools::Rectangle(0, 0, 1, 1));
 
     bool BindFont(HDC hDC) /*override*/;
     bool ReleaseFont() /*override*/;
@@ -241,18 +236,13 @@ public:
     IDWriteFontFace   * GetFontFace() const { return mpFontFace; }
     float               GetEmHeight() const { return mlfEmHeight; }
 
-    HRESULT CreateRenderTarget() {
-        if (mpRT) mpRT->Release(); mpRT = nullptr;
-        return mpD2DFactory->CreateDCRenderTarget(&mRTProps, &mpRT);
-    }
+    HRESULT CreateRenderTarget();
 
     bool Ready() const { return mpGdiInterop && mpRT; }
 
     void applyTextAntiAliasMode();
-    void setTextAntiAliasMode(D2DTextAntiAliasMode eMode)
-    {
-        meTextAntiAliasMode = eMode;
-    }
+    void changeTextAntiAliasMode(D2DTextAntiAliasMode eMode);
+
 private:
     static void CleanupModules();
 
@@ -261,6 +251,7 @@ private:
     D2DWriteTextOutRenderer & operator = (const D2DWriteTextOutRenderer &) = delete;
 
     bool GetDWriteFaceFromHDC(HDC hDC, IDWriteFontFace ** ppFontFace, float * lfSize) const;
+    bool performRender(CommonSalLayout const &rLayout, SalGraphics &rGraphics, HDC hDC, bool& bRetry);
 
     ID2D1Factory        * mpD2DFactory;
     IDWriteFactory      * mpDWriteFactory;
diff --git a/vcl/win/gdi/winlayout.cxx b/vcl/win/gdi/winlayout.cxx
index 81acf53b7ba2..15c4b9730808 100644
--- a/vcl/win/gdi/winlayout.cxx
+++ b/vcl/win/gdi/winlayout.cxx
@@ -77,7 +77,7 @@ bool WinFontInstance::CacheGlyphToAtlas(HDC hDC, HFONT hFont, int nGlyphIndex, S
     if (!pTxt)
         return false;
 
-    pTxt->setTextAntiAliasMode(D2DTextAntiAliasMode::AntiAliased);
+    pTxt->changeTextAntiAliasMode(D2DTextAntiAliasMode::AntiAliased);
 
     if (!pTxt->BindFont(aHDC.get()))
     {
@@ -178,7 +178,6 @@ bool WinFontInstance::CacheGlyphToAtlas(HDC hDC, HFONT hFont, int nGlyphIndex, S
     };
 
     pRT->BeginDraw();
-    pTxt->applyTextAntiAliasMode();
     pRT->DrawGlyphRun(baseline, &glyphs, pBrush);
     HRESULT hResult = pRT->EndDraw();
 
@@ -334,12 +333,14 @@ D2DTextAntiAliasMode lclGetSystemTextAntiAliasMode()
 
     if (bFontSmoothing)
     {
+        eMode = D2DTextAntiAliasMode::AntiAliased;
         UINT nType;
-        if (!SystemParametersInfo(SPI_GETFONTSMOOTHINGTYPE, 0, &nType, 0))
-            return eMode;
-
-        eMode = (nType == FE_FONTSMOOTHINGCLEARTYPE) ? D2DTextAntiAliasMode::ClearType
-                                                     : D2DTextAntiAliasMode::AntiAliased;
+        if (SystemParametersInfo(SPI_GETFONTSMOOTHINGTYPE, 0, &nType, 0) && nType == FE_FONTSMOOTHINGCLEARTYPE)
+            eMode = D2DTextAntiAliasMode::ClearType;
+    }
+    else
+    {
+        eMode = D2DTextAntiAliasMode::Aliased;
     }
 
     return eMode;
@@ -385,7 +386,6 @@ D2DWriteTextOutRenderer::D2DWriteTextOutRenderer()
         hr = CreateRenderTarget();
     }
     meTextAntiAliasMode = lclGetSystemTextAntiAliasMode();
-    mpRenderingParameters = lclSetRenderingMode(mpDWriteFactory, DWRITE_RENDERING_MODE_GDI_CLASSIC);
 }
 
 D2DWriteTextOutRenderer::~D2DWriteTextOutRenderer()
@@ -404,51 +404,107 @@ D2DWriteTextOutRenderer::~D2DWriteTextOutRenderer()
 
 void D2DWriteTextOutRenderer::applyTextAntiAliasMode()
 {
-    D2D1_TEXT_ANTIALIAS_MODE eMode = D2D1_TEXT_ANTIALIAS_MODE_DEFAULT;
+    D2D1_TEXT_ANTIALIAS_MODE eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_DEFAULT;
+    DWRITE_RENDERING_MODE eRenderingMode = DWRITE_RENDERING_MODE_DEFAULT;
     switch (meTextAntiAliasMode)
     {
         case D2DTextAntiAliasMode::Default:
-            eMode = D2D1_TEXT_ANTIALIAS_MODE_ALIASED;
+            eRenderingMode = DWRITE_RENDERING_MODE_DEFAULT;
+            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_DEFAULT;
             break;
+        case D2DTextAntiAliasMode::Aliased:
+            eRenderingMode = DWRITE_RENDERING_MODE_ALIASED;
+            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_ALIASED;
         case D2DTextAntiAliasMode::AntiAliased:
-            eMode = D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE;
+            eRenderingMode = DWRITE_RENDERING_MODE_CLEARTYPE_GDI_CLASSIC;
+            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE;
             break;
         case D2DTextAntiAliasMode::ClearType:
-            eMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE;
+            eRenderingMode = DWRITE_RENDERING_MODE_CLEARTYPE_GDI_CLASSIC;
+            eTextAAMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE;
             break;
         default:
             break;
     }
-    mpRT->SetTextAntialiasMode(eMode);
-    mpRT->SetTextRenderingParams(mpRenderingParameters);
+    mpRT->SetTextRenderingParams(lclSetRenderingMode(mpDWriteFactory, eRenderingMode));
+    mpRT->SetTextAntialiasMode(eTextAAMode);
 }
 
-bool D2DWriteTextOutRenderer::operator ()(CommonSalLayout const &rLayout,
-    SalGraphics &rGraphics,
-    HDC hDC)
+HRESULT D2DWriteTextOutRenderer::CreateRenderTarget()
+{
+    HRESULT hr = mpD2DFactory->CreateDCRenderTarget(&mRTProps, &mpRT);
+    if (SUCCEEDED(hr))
+        applyTextAntiAliasMode();
+    return hr;
+}
+
+void D2DWriteTextOutRenderer::changeTextAntiAliasMode(D2DTextAntiAliasMode eMode)
+{
+    if (meTextAntiAliasMode != eMode)
+    {
+        meTextAntiAliasMode = eMode;
+        applyTextAntiAliasMode();
+    }
+}
+
+HRESULT D2DWriteTextOutRenderer::BindDC(HDC hDC, tools::Rectangle const & rRect)
+{
+     RECT const rc = { rRect.Left(), rRect.Top(), rRect.Right(), rRect.Bottom() };
+    return mpRT->BindDC(hDC, &rc);
+}
+
+bool D2DWriteTextOutRenderer::operator ()(CommonSalLayout const & rLayout, SalGraphics& rGraphics, HDC hDC)
+{
+    bool bRetry = false;
+    bool bResult = false;
+    int nCount = 0;
+    do
+    {
+       bRetry = false;
+       bResult = performRender(rLayout, rGraphics, hDC, bRetry);
+       nCount++;
+    } while (bRetry && nCount < 3);
+    return bResult;
+}
+
+bool D2DWriteTextOutRenderer::performRender(CommonSalLayout const & rLayout, SalGraphics& rGraphics, HDC hDC, bool& bRetry)
 {
     if (!Ready())
         return false;
 
-    if (!BindFont(hDC))
+    HRESULT hr = S_OK;
+    hr = BindDC(hDC);
+
+    if (hr == D2DERR_RECREATE_TARGET)
     {
-        // If for any reason we can't bind fallback to legacy APIs.
-        return ExTextOutRenderer()(rLayout, rGraphics, hDC);
+        CreateRenderTarget();
+        bRetry = true;
+        return false;
     }
 
+    mlfEmHeight = 0;
+    if (!GetDWriteFaceFromHDC(hDC, &mpFontFace, &mlfEmHeight))
+        return false;
+
     tools::Rectangle bounds;
     bool succeeded = rLayout.GetBoundRect(rGraphics, bounds);
-    succeeded &= BindDC(hDC, bounds);   // Update the bounding rect.
+    if (succeeded)
+    {
+        hr = BindDC(hDC, bounds);   // Update the bounding rect.
+        succeeded &= SUCCEEDED(hr);
+    }
 
     ID2D1SolidColorBrush* pBrush = nullptr;
-    COLORREF bgrTextColor = GetTextColor(mhDC);
-    succeeded &= SUCCEEDED(mpRT->CreateSolidColorBrush(D2D1::ColorF(GetRValue(bgrTextColor) / 255.0f, GetGValue(bgrTextColor) / 255.0f, GetBValue(bgrTextColor) / 255.0f), &pBrush));
+    if (succeeded)
+    {
+        COLORREF bgrTextColor = GetTextColor(hDC);
+        D2D1::ColorF aD2DColor(GetRValue(bgrTextColor) / 255.0f, GetGValue(bgrTextColor) / 255.0f, GetBValue(bgrTextColor) / 255.0f);
+        succeeded &= SUCCEEDED(mpRT->CreateSolidColorBrush(aD2DColor, &pBrush));
+    }
 
-    HRESULT hr = S_OK;
     if (succeeded)
     {
         mpRT->BeginDraw();
-        applyTextAntiAliasMode();
 
         int nStart = 0;
         Point aPos(0, 0);
@@ -482,8 +538,10 @@ bool D2DWriteTextOutRenderer::operator ()(CommonSalLayout const &rLayout,
     ReleaseFont();
 
     if (hr == D2DERR_RECREATE_TARGET)
+    {
         CreateRenderTarget();
-
+        bRetry = true;
+    }
     return succeeded;
 }