don't try to use self-referencing ScSortedRangeCache (tdf#149752)

Both the documents from the bugreport use COUNTIF where the range
includes the cell itself. So trying to create ScSortedRangeCache
for the range would try to read the values of the cells in the range
including the cell itself, which would try to interpret COUNTIF,
which would recurse and deadlock on the mutex.

Change-Id: I95c33d0b75015dcd1d628740ef52c2d680864791
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/136581
Tested-by: Jenkins
Reviewed-by: Luboš Luňák <l.lunak@collabora.com>

1 files changed, 5 insertions, 1 deletions

diff --git a/sc/source/core/data/queryiter.cxx b/sc/source/core/data/queryiter.cxx
index e712b24d4443..36272b95bcbf 100644
--- a/sc/source/core/data/queryiter.cxx
+++ b/sc/source/core/data/queryiter.cxx
@@ -1259,7 +1259,9 @@ static bool CanBeUsedForSorterCache(ScDocument& rDoc, const ScQueryParam& rParam
         if(!inUnitTest)
             return false;
     }
-    if( !cell || !cell->GetCellGroup() || cell->GetCellGroup()->mnLength < 10 )
+    if( !cell )
+        return false;
+    if( !cell->GetCellGroup() || cell->GetCellGroup()->mnLength < 10 )
     {
         if(!inUnitTest)
             return false;
@@ -1269,6 +1271,8 @@ static bool CanBeUsedForSorterCache(ScDocument& rDoc, const ScQueryParam& rParam
     for(SCCOL col : rDoc.GetAllocatedColumnsRange(nTab, rParam.nCol1, rParam.nCol2))
     {
         ScRange aSortedRangeRange( col, rParam.nRow1, nTab, col, rParam.nRow2, nTab);
+        if( aSortedRangeRange.Contains( cell->aPos ))
+            return false; // self-referencing, can't create cache
         ScSortedRangeCache& cache = rDoc.GetSortedRangeCache( aSortedRangeRange, rParam, &context );
         if(!cache.isValid())
             return false;