summaryrefslogtreecommitdiff
path: root/sc/source
diff options
context:
space:
mode:
authorMike Kaganski <mike.kaganski@collabora.com>2020-09-21 10:21:24 +0200
committerMike Kaganski <mike.kaganski@collabora.com>2020-09-21 11:12:07 +0200
commitd5ebe7c3089ab9f4d3fe0707169fc1ce024cdb70 (patch)
tree364d77a5d931be50bc3b9b00d5cc01e5e40cfcdf /sc/source
parentfdc8590032b292dcb8152b328401e591fea642a4 (diff)
tdf#129606: Round the mean of the two subtractions
This improves handling of cases when the two subtraction methods give these values: c = -0.10000000000000053 d = -0.10000000000000006 Rounding c would give -0.10000000000000100, while rounding the mean gives -0.10000000000000001. Change-Id: I9805bc78bd1dec8ed77b660df1e366d36348f68b Reviewed-on: https://gerrit.libreoffice.org/c/core/+/103048 Tested-by: Jenkins Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
Diffstat (limited to 'sc/source')
-rw-r--r--sc/source/core/data/table4.cxx3
1 files changed, 2 insertions, 1 deletions
diff --git a/sc/source/core/data/table4.cxx b/sc/source/core/data/table4.cxx
index 9403d1b913a7..09a17546c002 100644
--- a/sc/source/core/data/table4.cxx
+++ b/sc/source/core/data/table4.cxx
@@ -208,7 +208,8 @@ double approxDiff( double a, double b )
// tdf#129606: Limit precision to the 16th significant digit of the least precise argument.
// Cf. mnMaxGeneralPrecision in sc/source/core/data/column3.cxx.
const int nExpArg = static_cast<int>(floor(log10(std::max(aa, ab)))) - 15;
- return rtl::math::round(c, -std::max(nExp, nExpArg));
+ // Round the mean of the two subtractions
+ return rtl::math::round((c + d) / 2, -std::max(nExp, nExpArg));
}
}