Resolves: tdf#90419 diminish precision error in Series Fill

There may be more elegant ways to accomplish this, go and find one..

Change-Id: Iceaa0783db9cf3d3e1aa20f075fe7e0618a1feb6
(cherry picked from commit e89c0e4fb783bd36d5f5fea154ee8608e542dae4)
Reviewed-on: https://gerrit.libreoffice.org/26219
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Eike Rathke <erack@redhat.com>

1 files changed, 39 insertions, 4 deletions

diff --git a/sc/source/core/data/table4.cxx b/sc/source/core/data/table4.cxx
index 79cf4b4d85d7..016b8e63e1ce 100644
--- a/sc/source/core/data/table4.cxx
+++ b/sc/source/core/data/table4.cxx
@@ -186,6 +186,41 @@ void setSuffixCell(
 
 }
 
+namespace {
+/* TODO: move this to rtl::math::approxDiff() ? Though the name is funny, the
+ * approx is expected to be more correct than the raw diff. */
+/** Calculate a-b trying to diminish precision errors such as for 0.11-0.12
+    not return -0.009999999999999995 but -0.01 instead.
+ */
+double approxDiff( double a, double b )
+{
+    if (a == b)
+        return 0.0;
+    if (a == 0.0)
+        return -b;
+    if (b == 0.0)
+        return a;
+    const double c = a - b;
+    const double aa = fabs(a);
+    const double ab = fabs(b);
+    if (aa < 1e-16 || aa > 1e+16 || ab < 1e-16 || ab > 1e+16)
+        // This is going nowhere, live with the result.
+        return c;
+
+    const double q = aa < ab ? b / a : a / b;
+    const double d = (a * q - b * q) / q;
+    if (d == c)
+        // No differing error, live with the result.
+        return c;
+
+    // We now have two subtractions with a similar but not equal error. Obtain
+    // the exponent of the error magnitude and round accordingly.
+    const double e = fabs(d - c);
+    const double fExp = floor( log10( e));
+    return rtl::math::round( c, -(static_cast<int>(fExp))-1);
+}
+}
+
 void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2,
                             FillCmd& rCmd, FillDateCmd& rDateCmd,
                             double& rInc, sal_uInt16& rMinDigits,
@@ -312,7 +347,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2,
             {
                 double nVal1 = aFirstCell.mfValue;
                 double nVal2 = GetValue(nCol+nAddX, nRow+nAddY);
-                rInc = nVal2 - nVal1;
+                rInc = approxDiff( nVal2, nVal1);
                 nCol = sal::static_int_cast<SCCOL>( nCol + nAddX );
                 nRow = sal::static_int_cast<SCROW>( nRow + nAddY );
                 bool bVal = true;
@@ -322,7 +357,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2,
                     if (aCell.meType == CELLTYPE_VALUE)
                     {
                         nVal2 = aCell.mfValue;
-                        double nDiff = nVal2 - nVal1;
+                        double nDiff = approxDiff( nVal2, nVal1);
                         if ( !::rtl::math::approxEqual( nDiff, rInc, 13 ) )
                             bVal = false;
                         nVal1 = nVal2;
@@ -387,7 +422,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2,
                 short nFlag2 = lcl_DecompValueString( aStr, nVal2, &rMinDigits );
                 if ( nFlag1 == nFlag2 )
                 {
-                    rInc = (double)nVal2 - (double)nVal1;
+                    rInc = approxDiff( nVal2, nVal1);
                     nCol = sal::static_int_cast<SCCOL>( nCol + nAddX );
                     nRow = sal::static_int_cast<SCROW>( nRow + nAddY );
                     bool bVal = true;
@@ -401,7 +436,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2,
                             nFlag2 = lcl_DecompValueString( aStr, nVal2, &rMinDigits );
                             if ( nFlag1 == nFlag2 )
                             {
-                                double nDiff = (double)nVal2 - (double)nVal1;
+                                double nDiff = approxDiff( nVal2, nVal1);
                                 if ( !::rtl::math::approxEqual( nDiff, rInc, 13 ) )
                                     bVal = false;
                                 nVal1 = nVal2;