unify the handling of string position arguments, fdo#75971 related

Only two text functions had the full set of checks as introduced by the
fix for fdo#75971. Let all text functions check their arguments in the
same way. Additionally this will ease a transition to accept string
lengths >64k in spreadsheet functions as now we have only one place that
checks for SAL_MAX_UINT16 instead of having that scattered all over the
place.

Change-Id: I454e617a59d0b3c2ca725047e7f8c7370bc0bb1f

2 files changed, 77 insertions, 23 deletions

diff --git a/sc/source/core/inc/interpre.hxx b/sc/source/core/inc/interpre.hxx
index 6022c8ea7f6b..642699a5a0be 100644
--- a/sc/source/core/inc/interpre.hxx
+++ b/sc/source/core/inc/interpre.hxx
@@ -368,6 +368,24 @@ void ScTableOp();                                       // repeated operations
 void SetMaxIterationCount(sal_uInt16 n);
 inline void CurFmtToFuncFmt()
     { nFuncFmtType = nCurFmtType; nFuncFmtIndex = nCurFmtIndex; }
+
+/** Check if a double is suitable as string position or length argument.
+
+    If fVal is Inf or NaN it is changed to -1, if it is less than 0 it is
+    sanitized to 0, if it is greater than some implementation defined max
+    string length it is sanitized to that max.
+
+    @return TRUE if double value fVal is suitable as string argument and was
+            not sanitized.
+            FALSE if not and fVal was adapted.
+ */
+inline bool CheckStringPositionArgument( double & fVal );
+
+/** Obtain a double suitable as string position or length argument.
+    Returns -1 if the number is Inf or NaN or less than 0 or greater than some
+    implementation defined max string length. */
+inline double GetStringPositionArgument();
+
 // Check for String overflow of rResult+rAdd and set error and erase rResult
 // if so. Return true if ok, false if overflow
 inline bool CheckStringResultLen( OUString& rResult, const OUString& rAdd );
@@ -921,6 +939,36 @@ inline bool ScInterpreter::MustHaveParamCountMin( short nAct, short nMin )
     return false;
 }
 
+inline bool ScInterpreter::CheckStringPositionArgument( double & fVal )
+{
+    if (!rtl::math::isFinite( fVal))
+    {
+        fVal = -1.0;
+        return false;
+    }
+    else if (fVal < 0.0)
+    {
+        fVal = 0.0;
+        return false;
+    }
+    else if (fVal > SAL_MAX_UINT16)
+    {
+        fVal = static_cast<double>(SAL_MAX_UINT16);
+        return false;
+    }
+    return true;
+}
+
+inline double ScInterpreter::GetStringPositionArgument()
+{
+    double fVal = rtl::math::approxFloor( GetDouble());
+    if (!CheckStringPositionArgument( fVal))
+    {
+        fVal = -1.0;
+    }
+    return fVal;
+}
+
 inline bool ScInterpreter::CheckStringResultLen( OUString& rResult, const OUString& rAdd )
 {
     if ( rResult.getLength() + rAdd.getLength() > SAL_MAX_UINT16 )
diff --git a/sc/source/core/tool/interpr1.cxx b/sc/source/core/tool/interpr1.cxx
index bae79627092b..83393ca22fc2 100644
--- a/sc/source/core/tool/interpr1.cxx
+++ b/sc/source/core/tool/interpr1.cxx
@@ -7761,11 +7761,10 @@ void ScInterpreter::ScReplace()
     if ( MustHaveParamCount( GetByte(), 4 ) )
     {
         OUString aNewStr = GetString().getString();
-        double fCount = ::rtl::math::approxFloor( GetDouble());
-        double fPos   = ::rtl::math::approxFloor( GetDouble());
+        double fCount = GetStringPositionArgument();
+        double fPos   = GetStringPositionArgument();
         OUString aOldStr = GetString().getString();
-        if (fPos < 1.0 || fPos > static_cast<double>(SAL_MAX_UINT16)
-                || fCount < 0.0 || fCount > static_cast<double>(SAL_MAX_UINT16))
+        if (fPos < 1.0 || fCount < 0.0)
             PushIllegalArgument();
         else
         {
@@ -7888,8 +7887,8 @@ void ScInterpreter::ScLeft()
         sal_Int32 n;
         if (nParamCount == 2)
         {
-            double nVal = ::rtl::math::approxFloor(GetDouble());
-            if (rtl::math::isNan(nVal) || nVal < 0.0 || nVal > SAL_MAX_UINT16)
+            double nVal = GetStringPositionArgument();
+            if (nVal < 0.0)
             {
                 PushIllegalArgument();
                 return ;
@@ -7996,8 +7995,8 @@ void ScInterpreter::ScRightB()
         sal_Int32 n;
         if (nParamCount == 2)
         {
-            double nVal = ::rtl::math::approxFloor(GetDouble());
-            if ( nVal < 0.0 || nVal > SAL_MAX_UINT16 )
+            double nVal = GetStringPositionArgument();
+            if ( nVal < 0.0 )
             {
                 PushIllegalArgument();
                 return ;
@@ -8047,8 +8046,8 @@ void ScInterpreter::ScLeftB()
         sal_Int32 n;
         if (nParamCount == 2)
         {
-            double nVal = ::rtl::math::approxFloor(GetDouble());
-            if ( nVal < 0.0 || nVal > SAL_MAX_UINT16 )
+            double nVal = GetStringPositionArgument();
+            if ( nVal < 0.0 )
             {
                 PushIllegalArgument();
                 return ;
@@ -8066,10 +8065,10 @@ void ScInterpreter::ScMidB()
 {
     if ( MustHaveParamCount( GetByte(), 3 ) )
     {
-        double fAnz    = ::rtl::math::approxFloor(GetDouble());
-        double fAnfang = ::rtl::math::approxFloor(GetDouble());
+        double fAnz    = GetStringPositionArgument();
+        double fAnfang = GetStringPositionArgument();
         OUString aStr = GetString().getString();
-        if (fAnfang < 1.0 || fAnz < 0.0 || fAnfang > double(SAL_MAX_UINT16) || fAnz > double(SAL_MAX_UINT16))
+        if (fAnfang < 1.0 || fAnz < 0.0)
             PushIllegalArgument();
         else
         {
@@ -8090,8 +8089,8 @@ void ScInterpreter::ScRight()
         sal_Int32 n;
         if (nParamCount == 2)
         {
-            double nVal = ::rtl::math::approxFloor(GetDouble());
-            if ( rtl::math::isNan(nVal) || nVal < 0.0 || nVal > SAL_MAX_UINT16 )
+            double nVal = GetStringPositionArgument();
+            if (nVal < 0.0)
             {
                 PushIllegalArgument();
                 return ;
@@ -8117,8 +8116,15 @@ void ScInterpreter::ScSearch()
         double fAnz;
         if (nParamCount == 3)
         {
-            fAnz = ::rtl::math::approxFloor(GetDouble());
-            if (fAnz > double(SAL_MAX_UINT16))
+            // This should use GetStringPositionArgument() but old versions up
+            // to LibreOffice 4.2.5 allowed and ignored 0 and negative values.
+            // It is unnecessary to break existing documents that "rely" on
+            // that behavior. Though ODFF constrains Start to be >=1.
+            /* TODO: fix this and possibly break those broken documents? */
+            fAnz = rtl::math::approxFloor( GetDouble());
+            if (fAnz < 1.0)
+                fAnz = 1.0;
+            else if (!CheckStringPositionArgument( fAnz))
             {
                 PushIllegalArgument();
                 return;
@@ -8152,10 +8158,10 @@ void ScInterpreter::ScMid()
 {
     if ( MustHaveParamCount( GetByte(), 3 ) )
     {
-        double fAnz    = ::rtl::math::approxFloor(GetDouble());
-        double fAnfang = ::rtl::math::approxFloor(GetDouble());
+        double fAnz    = GetStringPositionArgument();
+        double fAnfang = GetStringPositionArgument();
         OUString aStr = GetString().getString();
-        if (fAnfang < 1.0 || fAnz < 0.0 || fAnfang > double(SAL_MAX_UINT16) || fAnz > double(SAL_MAX_UINT16))
+        if (fAnfang < 1.0 || fAnz < 0.0)
             PushIllegalArgument();
         else
         {
@@ -8250,8 +8256,8 @@ void ScInterpreter::ScSubstitute()
         sal_Int32 nAnz;
         if (nParamCount == 4)
         {
-            double fAnz = ::rtl::math::approxFloor(GetDouble());
-            if( fAnz < 1 || fAnz > SAL_MAX_UINT16 )
+            double fAnz = GetStringPositionArgument();
+            if( fAnz < 1 )
             {
                 PushIllegalArgument();
                 return;
@@ -8300,7 +8306,7 @@ void ScInterpreter::ScRept()
 {
     if ( MustHaveParamCount( GetByte(), 2 ) )
     {
-        double fAnz = ::rtl::math::approxFloor(GetDouble());
+        double fAnz = GetStringPositionArgument();
         OUString aStr = GetString().getString();
         if ( fAnz < 0.0 )
             PushIllegalArgument();