Fix Splash::arbitraryTransformImage causes bogus memory allocation size

Bug #49523

3 files changed, 48 insertions, 37 deletions

diff --git a/poppler/SplashOutputDev.cc b/poppler/SplashOutputDev.cc
index be35c259..abdcea47 100644
--- a/poppler/SplashOutputDev.cc
+++ b/poppler/SplashOutputDev.cc
@@ -4059,7 +4059,7 @@ GBool SplashOutputDev::tilingPatternFill(GfxState *state, Gfx *gfx1, Catalog *ca
   matc[1] = ctm[1];
   matc[2] = ctm[2];
   matc[3] = ctm[3];
-  splash->drawImage(&tilingBitmapSrc, &imgData, colorMode, gTrue, result_width, result_height, matc);
+  splash->drawImage(&tilingBitmapSrc, &imgData, colorMode, gTrue, result_width, result_height, matc, gTrue);
   delete tBitmap;
   delete gfx;
   return gTrue;
diff --git a/splash/Splash.cc b/splash/Splash.cc
index 0e07c705..b927e5e7 100644
--- a/splash/Splash.cc
+++ b/splash/Splash.cc
@@ -3375,7 +3375,8 @@ void Splash::blitMask(SplashBitmap *src, int xDest, int yDest,
 
 SplashError Splash::drawImage(SplashImageSource src, void *srcData,
 			      SplashColorMode srcMode, GBool srcAlpha,
-			      int w, int h, SplashCoord *mat) {
+			      int w, int h, SplashCoord *mat,
+			      GBool tilingPattern) {
   GBool ok;
   SplashBitmap *scaledImg;
   SplashClipResult clipRes;
@@ -3499,7 +3500,7 @@ SplashError Splash::drawImage(SplashImageSource src, void *srcData,
   // all other cases
   } else {
     return arbitraryTransformImage(src, srcData, srcMode, nComps, srcAlpha,
-			    w, h, mat);
+			    w, h, mat, tilingPattern);
   }
 
   return splashOk;
@@ -3509,7 +3510,8 @@ SplashError Splash::arbitraryTransformImage(SplashImageSource src, void *srcData
 				     SplashColorMode srcMode, int nComps,
 				     GBool srcAlpha,
 				     int srcWidth, int srcHeight,
-				     SplashCoord *mat) {
+				     SplashCoord *mat,
+				     GBool tilingPattern) {
   SplashBitmap *scaledImg;
   SplashClipResult clipRes, clipRes2;
   SplashPipe pipe;
@@ -3558,44 +3560,53 @@ SplashError Splash::arbitraryTransformImage(SplashImageSource src, void *srcData
   }
 
   // compute the scale factors
-  if (mat[0] >= 0) {
-    t0 = imgCoordMungeUpper(mat[0] + mat[4]) - imgCoordMungeLower(mat[4]);
+  if (splashAbs(mat[0]) >= splashAbs(mat[1])) {
+    scaledWidth = xMax - xMin;
+    scaledHeight = yMax - yMin;
   } else {
-    t0 = imgCoordMungeUpper(mat[4]) - imgCoordMungeLower(mat[0] + mat[4]);
+    scaledWidth = yMax - yMin;
+    scaledHeight = xMax - xMin;
   }
-  if (mat[1] >= 0) {
-    t1 = imgCoordMungeUpper(mat[1] + mat[5]) - imgCoordMungeLower(mat[5]);
-  } else {
-    t1 = imgCoordMungeUpper(mat[5]) - imgCoordMungeLower(mat[1] + mat[5]);
-  }
-  scaledWidth = t0 > t1 ? t0 : t1;
-  if (mat[2] >= 0) {
-    t0 = imgCoordMungeUpper(mat[2] + mat[4]) - imgCoordMungeLower(mat[4]);
-    if (splashAbs(mat[1]) >= 1) {
-      th = imgCoordMungeUpper(mat[2]) - imgCoordMungeLower(mat[0] * mat[3] / mat[1]);
-	  if (th > t0) t0 = th;
+  if (scaledHeight <= 1 || scaledHeight <= 1 || tilingPattern) {
+    if (mat[0] >= 0) {
+      t0 = imgCoordMungeUpper(mat[0] + mat[4]) - imgCoordMungeLower(mat[4]);
+    } else {
+      t0 = imgCoordMungeUpper(mat[4]) - imgCoordMungeLower(mat[0] + mat[4]);
     }
-  } else {
-    t0 = imgCoordMungeUpper(mat[4]) - imgCoordMungeLower(mat[2] + mat[4]);
-    if (splashAbs(mat[1]) >= 1) {
-      th = imgCoordMungeUpper(mat[0] * mat[3] / mat[1]) - imgCoordMungeLower(mat[2]);
-      if (th > t0) t0 = th;
+    if (mat[1] >= 0) {
+      t1 = imgCoordMungeUpper(mat[1] + mat[5]) - imgCoordMungeLower(mat[5]);
+    } else {
+      t1 = imgCoordMungeUpper(mat[5]) - imgCoordMungeLower(mat[1] + mat[5]);
     }
-  }
-  if (mat[3] >= 0) {
-    t1 = imgCoordMungeUpper(mat[3] + mat[5]) - imgCoordMungeLower(mat[5]);
-    if (splashAbs(mat[0]) >= 1) {
-      th = imgCoordMungeUpper(mat[3]) - imgCoordMungeLower(mat[1] * mat[2] / mat[0]);
-	  if (th > t1) t1 = th;
+    scaledWidth = t0 > t1 ? t0 : t1;
+    if (mat[2] >= 0) {
+      t0 = imgCoordMungeUpper(mat[2] + mat[4]) - imgCoordMungeLower(mat[4]);
+      if (splashAbs(mat[1]) >= 1) {
+        th = imgCoordMungeUpper(mat[2]) - imgCoordMungeLower(mat[0] * mat[3] / mat[1]);
+	    if (th > t0) t0 = th;
+      }
+    } else {
+      t0 = imgCoordMungeUpper(mat[4]) - imgCoordMungeLower(mat[2] + mat[4]);
+      if (splashAbs(mat[1]) >= 1) {
+        th = imgCoordMungeUpper(mat[0] * mat[3] / mat[1]) - imgCoordMungeLower(mat[2]);
+        if (th > t0) t0 = th;
+      }
     }
-  } else {
-    t1 = imgCoordMungeUpper(mat[5]) - imgCoordMungeLower(mat[3] + mat[5]);
-    if (splashAbs(mat[0]) >= 1) {
-      th = imgCoordMungeUpper(mat[1] * mat[2] / mat[0]) - imgCoordMungeLower(mat[3]);
-	  if (th > t1) t1 = th;
+    if (mat[3] >= 0) {
+      t1 = imgCoordMungeUpper(mat[3] + mat[5]) - imgCoordMungeLower(mat[5]);
+      if (splashAbs(mat[0]) >= 1) {
+        th = imgCoordMungeUpper(mat[3]) - imgCoordMungeLower(mat[1] * mat[2] / mat[0]);
+	    if (th > t1) t1 = th;
+      }
+    } else {
+      t1 = imgCoordMungeUpper(mat[5]) - imgCoordMungeLower(mat[3] + mat[5]);
+      if (splashAbs(mat[0]) >= 1) {
+        th = imgCoordMungeUpper(mat[1] * mat[2] / mat[0]) - imgCoordMungeLower(mat[3]);
+	    if (th > t1) t1 = th;
+      }
     }
+    scaledHeight = t0 > t1 ? t0 : t1;
   }
-  scaledHeight = t0 > t1 ? t0 : t1;
   if (scaledWidth == 0) {
     scaledWidth = 1;
   }
diff --git a/splash/Splash.h b/splash/Splash.h
index bc82faab..f4fb5429 100644
--- a/splash/Splash.h
+++ b/splash/Splash.h
@@ -209,7 +209,7 @@ public:
   // The matrix behaves as for fillImageMask.
   SplashError drawImage(SplashImageSource src, void *srcData,
 			SplashColorMode srcMode, GBool srcAlpha,
-			int w, int h, SplashCoord *mat);
+			int w, int h, SplashCoord *mat, GBool tilingPattern = gFalse);
 
   // Composite a rectangular region from <src> onto this Splash
   // object.
@@ -348,7 +348,7 @@ private:
 			       SplashColorMode srcMode, int nComps,
 			       GBool srcAlpha,
 			       int srcWidth, int srcHeight,
-			       SplashCoord *mat);
+			       SplashCoord *mat, GBool tilingPattern = gFalse);
   SplashBitmap *scaleImage(SplashImageSource src, void *srcData,
 			   SplashColorMode srcMode, int nComps,
 			   GBool srcAlpha, int srcWidth, int srcHeight,