diff options
| author | Werner Koerner <wk661lo@gmail.com> | 2012-12-11 22:23:39 +0100 |
|---|---|---|
| committer | Eike Rathke <erack@redhat.com> | 2012-12-14 11:15:02 +0000 |
| commit | a9a734a680dda8c177f92cf2f14061c9b785ec8d (patch) | |
| tree | cad1e95401a1a05f1913d233b4b0ed1024cb7bd4 /sc | |
| parent | c6f104e0c9a8d1953b74de4c4d13ba38a61e60d0 (diff) | |
calc: fix mathematical error in CritBinom
CRITBINOM(462,0.8,0.9), expected result 381, calculated value is 462.
Similar errors in BinomDist, B, CritBinom and NegBinomDist were
fixed by commit 5cf55f5b7800e443c4f087e72ae05abc8b7fef45.
Change-Id: I9b12a1c4410ec72258ae1fb68409ad00c922b94c
Reviewed-on: https://gerrit.libreoffice.org/1301
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Eike Rathke <erack@redhat.com>
Diffstat (limited to 'sc')
| -rw-r--r-- | sc/source/core/tool/interpr3.cxx | 24 |
1 files changed, 11 insertions, 13 deletions
diff --git a/sc/source/core/tool/interpr3.cxx b/sc/source/core/tool/interpr3.cxx index ada1cff0f1a6..638806d4bfc2 100644 --- a/sc/source/core/tool/interpr3.cxx +++ b/sc/source/core/tool/interpr3.cxx @@ -1348,26 +1348,25 @@ void ScInterpreter::ScCritBinom() RTL_LOGFILE_CONTEXT_AUTHOR( aLogger, "sc", "er", "ScInterpreter::ScCritBinom" ); if ( MustHaveParamCount( GetByte(), 3 ) ) { - double alpha = GetDouble(); // alpha - double p = GetDouble(); // p + double alpha = GetDouble(); + double p = GetDouble(); double n = ::rtl::math::approxFloor(GetDouble()); if (n < 0.0 || alpha <= 0.0 || alpha >= 1.0 || p < 0.0 || p > 1.0) PushIllegalArgument(); else { - double q = 1.0 - p; + double q = (0.5 - p) + 0.5; // get one bit more for p near 1.0 double fFactor = pow(q,n); - if (fFactor == 0.0) + if (fFactor <= ::std::numeric_limits<double>::min()) { fFactor = pow(p, n); - if (fFactor == 0.0) + if (fFactor <= ::std::numeric_limits<double>::min()) PushNoValue(); else { - double fSum = 1.0 - fFactor; sal_uLong max = (sal_uLong) n; - sal_uLong i; - - for ( i = 0; i < max && fSum >= alpha; i++) + double fSum = 1.0 - fFactor; + sal_uInt32 max = static_cast<sal_uInt32> (n), i; + for (i = 0; i < max && fSum >= alpha; i++) { fFactor *= (n-i)/(i+1)*q/p; fSum -= fFactor; @@ -1377,10 +1376,9 @@ void ScInterpreter::ScCritBinom() } else { - double fSum = fFactor; sal_uLong max = (sal_uLong) n; - sal_uLong i; - - for ( i = 0; i < max && fSum < alpha; i++) + double fSum = fFactor; + sal_uInt32 max = static_cast<sal_uInt32> (n), i; + for (i = 0; i < max && fSum < alpha; i++) { fFactor *= (n-i)/(i+1)*p/q; fSum += fFactor; |