tdf#98960 fix OpenGL crash by optimized image scaling

using area scale shader - 2 passes impl - sqrt scale factor

(cherry-picked from the commit f11a228cfd326bc089d7ff1c11a1561cdf5ee986)

Change-Id: I973ae0a281735787b045ce8fd5df03f8caa8f189
Reviewed-on: https://gerrit.libreoffice.org/23613
Reviewed-by: László Németh <nemeth@numbertext.org>
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Tomaž Vajngerl <quikee@gmail.com>

2 files changed, 76 insertions, 9 deletions

diff --git a/vcl/opengl/areaScaleFragmentShader.glsl b/vcl/opengl/areaScaleFragmentShader.glsl
index b95b8698a26f..c83c5e0699f2 100644
--- a/vcl/opengl/areaScaleFragmentShader.glsl
+++ b/vcl/opengl/areaScaleFragmentShader.glsl
@@ -48,11 +48,11 @@ void main(void)
 
     // How much each column/row will contribute to the resulting pixel.
     // assert( xscale <= 100 ); assert( yscale <= 100 );
-    float xratio[ 100 + 2 ];
-    float yratio[ 100 + 2 ];
+    float xratio[ 16 + 2 ];
+    float yratio[ 16 + 2 ];
     // For finding the first and last source pixel.
-    int xpixel[ 100 + 2 ];
-    int ypixel[ 100 + 2 ];
+    int xpixel[ 16 + 2 ];
+    int ypixel[ 16 + 2 ];
 
     int xpos = 0;
     int ypos = 0;
diff --git a/vcl/opengl/scale.cxx b/vcl/opengl/scale.cxx
index 80bbdd6d521a..d2792295b0b8 100644
--- a/vcl/opengl/scale.cxx
+++ b/vcl/opengl/scale.cxx
@@ -208,11 +208,28 @@ bool OpenGLSalBitmap::ImplScaleArea( const rtl::Reference< OpenGLContext > &xCon
     bool fast = ( ixscale == int( ixscale ) && iyscale == int( iyscale )
         && int( nNewWidth * ixscale ) == mnWidth && int( nNewHeight * iyscale ) == mnHeight );
 
+    bool bTwoPasses = false;
+
     // The generic case has arrays only up to 100 ratio downscaling, which is hopefully enough
     // in practice, but protect against buffer overflows in case such an extreme case happens
     // (and in such case the precision of the generic algorithm probably doesn't matter anyway).
     if( ixscale > 100 || iyscale > 100 )
+    {
         fast = true;
+    }
+    else
+    {
+        if (ixscale > 16 || iyscale > 16)
+        {
+            ixscale = std::floor(std::sqrt(ixscale));
+            iyscale = std::floor(std::sqrt(iyscale));
+            nNewWidth = int(mnWidth / ixscale);
+            rScaleX *= ixscale; // second pass x-scale factor
+            nNewHeight = int(mnHeight / iyscale);
+            rScaleY *= iyscale; // second pass y-scale factor
+            bTwoPasses = true;
+        }
+    }
 
     // TODO Make sure the framebuffer is alright
 
@@ -251,13 +268,58 @@ bool OpenGLSalBitmap::ImplScaleArea( const rtl::Reference< OpenGLContext > &xCon
     pProgram->DrawTexture( maTexture );
     pProgram->Clean();
 
-    maTexture = aScratchTex;
-    OpenGLContext::ReleaseFramebuffer( pFramebuffer );
-
-    mnWidth = nNewWidth;
-    mnHeight = nNewHeight;
+    OpenGLContext::ReleaseFramebuffer(pFramebuffer);
 
     CHECK_GL_ERROR();
+
+    if (bTwoPasses)
+    {
+        mnWidth = nNewWidth;
+        mnHeight = nNewHeight;
+
+        nNewWidth = int(mnWidth * rScaleX);
+        nNewHeight = int (mnHeight * rScaleY);
+
+        ixscale = 1 / rScaleX;
+        iyscale = 1 / rScaleY;
+
+        pProgram = xContext->UseProgram("textureVertexShader", OUString("areaScaleFragmentShader"));
+        if (pProgram == nullptr)
+            return false;
+
+        OpenGLTexture aScratchTex2(nNewWidth, nNewHeight);
+
+        pFramebuffer = xContext->AcquireFramebuffer(aScratchTex2);
+
+        pProgram->SetUniform1f("xscale", ixscale);
+        pProgram->SetUniform1f("yscale", iyscale);
+        pProgram->SetUniform1i("swidth", mnWidth);
+        pProgram->SetUniform1i("sheight", mnHeight);
+        // For converting between <0,mnWidth-1> and <0.0,1.0> coordinate systems.
+        pProgram->SetUniform1f("xsrcconvert", 1.0 / (mnWidth - 1));
+        pProgram->SetUniform1f("ysrcconvert", 1.0 / (mnHeight - 1));
+        pProgram->SetUniform1f("xdestconvert", 1.0 * (nNewWidth - 1));
+        pProgram->SetUniform1f("ydestconvert", 1.0 * (nNewHeight - 1));
+
+        pProgram->SetTexture("sampler", aScratchTex);
+        pProgram->DrawTexture(aScratchTex);
+        pProgram->Clean();
+
+        OpenGLContext::ReleaseFramebuffer(pFramebuffer);
+
+        CHECK_GL_ERROR();
+
+        maTexture = aScratchTex2;
+        mnWidth = nNewWidth;
+        mnHeight = nNewHeight;
+    }
+    else
+    {
+        maTexture = aScratchTex;
+        mnWidth = nNewWidth;
+        mnHeight = nNewHeight;
+    }
+
     return true;
 }
 
@@ -269,6 +331,11 @@ bool OpenGLSalBitmap::ImplScale( const double& rScaleX, const double& rScaleY, B
     OpenGLVCLContextZone aContextZone;
     rtl::Reference<OpenGLContext> xContext = OpenGLContext::getVCLContext();
 
+    if (rScaleX <= 1 && rScaleY <= 1)
+    {
+        nScaleFlag = BmpScaleFlag::BestQuality;
+    }
+
     if( nScaleFlag == BmpScaleFlag::Fast )
     {
         return ImplScaleFilter( xContext, rScaleX, rScaleY, GL_NEAREST );