summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTor Lillqvist <tml@collabora.com>2016-02-05 14:58:11 +0200
committerMichael Meeks <michael.meeks@collabora.com>2016-02-08 10:13:38 +0000
commitaa064e8209e57ac91f16305d38e657d12a42093f (patch)
tree2df86411cd30b5533a31f5c97e71bec0011bcfbb
parent7608c8740104617f52f6b625a4bccd3bdddee4cf (diff)
tdf#97587: Treat plain NaNs as zero in the software interpreter for SUM
The NaNs that have been stored in the arrays by ScColumn::FetchVectorRefArray() and other code correspond to empty cells. For the purpose of SUM they should be treated as zero. Change-Id: I8ac0c8afdf71da415ed120f9f8f6d51a8b5edb15 Reviewed-on: https://gerrit.libreoffice.org/22199 Tested-by: Jenkins <ci@libreoffice.org> Reviewed-by: Michael Meeks <michael.meeks@collabora.com> Tested-by: Michael Meeks <michael.meeks@collabora.com>
-rw-r--r--sc/source/core/inc/arraysumfunctor.hxx22
-rw-r--r--sc/source/core/tool/interpr6.cxx10
2 files changed, 32 insertions, 0 deletions
diff --git a/sc/source/core/inc/arraysumfunctor.hxx b/sc/source/core/inc/arraysumfunctor.hxx
index 200fdc6b16f0..3955fd9451f6 100644
--- a/sc/source/core/inc/arraysumfunctor.hxx
+++ b/sc/source/core/inc/arraysumfunctor.hxx
@@ -12,6 +12,7 @@
#define INCLUDED_SC_SOURCE_CORE_INC_ARRAYSUMFUNCTOR_HXX
#include <cstdint>
+#include <rtl/math.hxx>
#include <tools/cpuid.hxx>
#if defined(LO_SSE2_AVAILABLE)
@@ -65,6 +66,27 @@ public:
for (; i < mnSize; ++i)
fSum += mpArray[i];
+ // If the sum is a NaN, some of the terms were empty cells, probably.
+ // Re-calculate, carefully
+ if (!rtl::math::isFinite(fSum))
+ {
+ sal_uInt32 nErr = reinterpret_cast< sal_math_Double * >(&fSum)->nan_parts.fraction_lo;
+ if (nErr & 0xffff0000)
+ {
+ fSum = 0;
+ for (i = 0; i < mnSize; i++)
+ {
+ if (!rtl::math::isFinite(mpArray[i]))
+ {
+ nErr = reinterpret_cast< const sal_math_Double * >(&mpArray[i])->nan_parts.fraction_lo;
+ if (!(nErr & 0xffff0000))
+ fSum += mpArray[i]; // Let errors encoded as NaNs propagate ???
+ }
+ else
+ fSum += mpArray[i];
+ }
+ }
+ }
return fSum;
}
diff --git a/sc/source/core/tool/interpr6.cxx b/sc/source/core/tool/interpr6.cxx
index 5bf453300c81..e7e321be36a0 100644
--- a/sc/source/core/tool/interpr6.cxx
+++ b/sc/source/core/tool/interpr6.cxx
@@ -412,6 +412,16 @@ void IterateMatrix(
case ifSUM:
{
ScMatrix::IterateResult aRes = pMat->Sum(bTextAsZero);
+ // If the first value is a NaN, it probably means it was an empty cell,
+ // and should be treated as zero.
+ if ( !rtl::math::isFinite(aRes.mfFirst) )
+ {
+ sal_uInt32 nErr = reinterpret_cast< sal_math_Double * >(&aRes.mfFirst)->nan_parts.fraction_lo;
+ if (nErr & 0xffff0000)
+ {
+ aRes.mfFirst = 0;
+ }
+ }
if ( fMem )
fRes += aRes.mfFirst + aRes.mfRest;
else