don't fill the matrix cell by cell, tdf#67071

We are now at a place where filling one cell takes about a second with
the big document. We could in theory for the special case of this
document add some caching but that would fail in other cases. This
document is already quite large with an external reference to 70k cells
for each formula cell.

I consider it already quite nice and useable again.

Change-Id: I804094838471507e6bb7b0e0e3387e0c7fe53e4b
Reviewed-on: https://gerrit.libreoffice.org/22388
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>

7 files changed, 36 insertions, 67 deletions

diff --git a/sc/inc/column.hxx b/sc/inc/column.hxx
index 0aaaeaf641fd..0b560b9ef4f6 100644
--- a/sc/inc/column.hxx
+++ b/sc/inc/column.hxx
@@ -562,7 +562,7 @@ public:
     ScFormulaVectorState GetFormulaVectorState( SCROW nRow ) const;
     formula::FormulaTokenRef ResolveStaticReference( SCROW nRow );
     bool ResolveStaticReference( ScMatrix& rMat, SCCOL nMatCol, SCROW nRow1, SCROW nRow2 );
-    void FillMatrix( ScMatrix& rMat, size_t nMatCol, SCROW nRow1, SCROW nRow2 ) const;
+    void FillMatrix( ScMatrix& rMat, size_t nMatCol, SCROW nRow1, SCROW nRow2, svl::SharedStringPool* pPool = nullptr ) const;
     formula::VectorRefArray FetchVectorRefArray( SCROW nRow1, SCROW nRow2 );
     void SetFormulaResults( SCROW nRow, const double* pResults, size_t nLen );
     void SetFormulaResults( SCROW nRow, const formula::FormulaTokenRef* pResults, size_t nLen );
diff --git a/sc/inc/document.hxx b/sc/inc/document.hxx
index 9b8ab5e50f58..a50b0046553e 100644
--- a/sc/inc/document.hxx
+++ b/sc/inc/document.hxx
@@ -1847,7 +1847,7 @@ public:
 
     SC_DLLPUBLIC ScMacroManager* GetMacroManager();
 
-    void FillMatrix( ScMatrix& rMat, SCTAB nTab, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2 ) const;
+    void FillMatrix( ScMatrix& rMat, SCTAB nTab, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, svl::SharedStringPool* pPool = nullptr) const;
 
     /**
      * Set an array of numerical formula results to a group of contiguous
diff --git a/sc/inc/table.hxx b/sc/inc/table.hxx
index e1b07061e688..bcb61f4aa3e7 100644
--- a/sc/inc/table.hxx
+++ b/sc/inc/table.hxx
@@ -909,7 +909,7 @@ public:
     void DeleteBroadcasters( sc::ColumnBlockPosition& rBlockPos, SCCOL nCol, SCROW nRow1, SCROW nRow2 );
     bool HasBroadcaster( SCCOL nCol ) const;
 
-    void FillMatrix( ScMatrix& rMat, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2 ) const;
+    void FillMatrix( ScMatrix& rMat, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, svl::SharedStringPool* pPool = nullptr ) const;
 
     void InterpretDirtyCells( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2 );
 
diff --git a/sc/source/core/data/column2.cxx b/sc/source/core/data/column2.cxx
index 07bd2a647b7d..2823d6332c69 100644
--- a/sc/source/core/data/column2.cxx
+++ b/sc/source/core/data/column2.cxx
@@ -2150,10 +2150,12 @@ class FillMatrixHandler
     SCTAB mnTab;
     ScDocument* mpDoc;
     svl::SharedStringPool& mrPool;
+    svl::SharedStringPool* mpPool; // if matrix is not in the same document
 
 public:
-    FillMatrixHandler(ScMatrix& rMat, size_t nMatCol, size_t nTopRow, SCCOL nCol, SCTAB nTab, ScDocument* pDoc) :
-        mrMat(rMat), mnMatCol(nMatCol), mnTopRow(nTopRow), mnCol(nCol), mnTab(nTab), mpDoc(pDoc), mrPool(pDoc->GetSharedStringPool()) {}
+    FillMatrixHandler(ScMatrix& rMat, size_t nMatCol, size_t nTopRow, SCCOL nCol, SCTAB nTab, ScDocument* pDoc, svl::SharedStringPool* pPool) :
+        mrMat(rMat), mnMatCol(nMatCol), mnTopRow(nTopRow), mnCol(nCol), mnTab(nTab),
+        mpDoc(pDoc), mrPool(pDoc->GetSharedStringPool()), mpPool(pPool) {}
 
     void operator() (const sc::CellStoreType::value_type& node, size_t nOffset, size_t nDataSize)
     {
@@ -2169,8 +2171,22 @@ public:
             break;
             case sc::element_type_string:
             {
-                const svl::SharedString* p = &sc::string_block::at(*node.data, nOffset);
-                mrMat.PutString(p, nDataSize, mnMatCol, nMatRow);
+                if (!mpPool)
+                {
+                    const svl::SharedString* p = &sc::string_block::at(*node.data, nOffset);
+                    mrMat.PutString(p, nDataSize, mnMatCol, nMatRow);
+                }
+                else
+                {
+                    std::vector<svl::SharedString> aStrings;
+                    aStrings.reserve(nDataSize);
+                    const svl::SharedString* p = &sc::string_block::at(*node.data, nOffset);
+                    for (size_t i = 0; i < nDataSize; ++i)
+                    {
+                        aStrings.push_back(mpPool->intern(p[i].getString()));
+                    }
+                    mrMat.PutString(aStrings.data(), aStrings.size(), mnMatCol, nMatRow);
+                }
             }
             break;
             case sc::element_type_edittext:
@@ -2184,7 +2200,10 @@ public:
                 for (; it != itEnd; ++it)
                 {
                     OUString aStr = ScEditUtil::GetString(**it, mpDoc);
-                    aSSs.push_back(mrPool.intern(aStr));
+                    if (!mpPool)
+                        aSSs.push_back(mrPool.intern(aStr));
+                    else
+                        aSSs.push_back(mpPool->intern(aStr));
                 }
 
                 const svl::SharedString* p = &aSSs[0];
@@ -2246,6 +2265,8 @@ public:
                     }
 
                     svl::SharedString aStr = rCell.GetString();
+                    if (mpPool)
+                        aStr = mpPool->intern(aStr.getString());
                     if (!aBucket.maStrVals.empty() && nThisRow == nPrevRow + 1)
                     {
                         // Secondary strings.
@@ -2271,9 +2292,9 @@ public:
 
 }
 
-void ScColumn::FillMatrix( ScMatrix& rMat, size_t nMatCol, SCROW nRow1, SCROW nRow2 ) const
+void ScColumn::FillMatrix( ScMatrix& rMat, size_t nMatCol, SCROW nRow1, SCROW nRow2, svl::SharedStringPool* pPool ) const
 {
-    FillMatrixHandler aFunc(rMat, nMatCol, nRow1, nCol, nTab, pDocument);
+    FillMatrixHandler aFunc(rMat, nMatCol, nRow1, nCol, nTab, pDocument, pPool);
     sc::ParseBlock(maCells.begin(), maCells, aFunc, nRow1, nRow2);
 }
 
diff --git a/sc/source/core/data/documen8.cxx b/sc/source/core/data/documen8.cxx
index 15fbb88c2e64..cd284792b8a0 100644
--- a/sc/source/core/data/documen8.cxx
+++ b/sc/source/core/data/documen8.cxx
@@ -392,7 +392,7 @@ ScMacroManager* ScDocument::GetMacroManager()
 }
 
 void ScDocument::FillMatrix(
-    ScMatrix& rMat, SCTAB nTab, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2 ) const
+    ScMatrix& rMat, SCTAB nTab, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, svl::SharedStringPool* pPool ) const
 {
     const ScTable* pTab = FetchTable(nTab);
     if (!pTab)
@@ -406,7 +406,7 @@ void ScDocument::FillMatrix(
     if (static_cast<SCROW>(nR) != nRow2 - nRow1 + 1 || static_cast<SCCOL>(nC) != nCol2 - nCol1 + 1)
         return;
 
-    pTab->FillMatrix(rMat, nCol1, nRow1, nCol2, nRow2);
+    pTab->FillMatrix(rMat, nCol1, nRow1, nCol2, nRow2, pPool);
 }
 
 void ScDocument::SetFormulaResults( const ScAddress& rTopPos, const double* pResults, size_t nLen )
diff --git a/sc/source/core/data/table1.cxx b/sc/source/core/data/table1.cxx
index eeefa37db0cc..7761828d1359 100644
--- a/sc/source/core/data/table1.cxx
+++ b/sc/source/core/data/table1.cxx
@@ -2217,11 +2217,11 @@ bool ScTable::HasBroadcaster( SCCOL nCol ) const
     return aCol[nCol].HasBroadcaster();
 }
 
-void ScTable::FillMatrix( ScMatrix& rMat, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2 ) const
+void ScTable::FillMatrix( ScMatrix& rMat, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, svl::SharedStringPool* pPool ) const
 {
     size_t nMatCol = 0;
     for (SCCOL nCol = nCol1; nCol <= nCol2; ++nCol, ++nMatCol)
-        aCol[nCol].FillMatrix(rMat, nMatCol, nRow1, nRow2);
+        aCol[nCol].FillMatrix(rMat, nMatCol, nRow1, nRow2, pPool);
 }
 
 void ScTable::InterpretDirtyCells( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2 )
diff --git a/sc/source/ui/docshell/externalrefmgr.cxx b/sc/source/ui/docshell/externalrefmgr.cxx
index ee5efcc1448d..7f053dee341e 100644
--- a/sc/source/ui/docshell/externalrefmgr.cxx
+++ b/sc/source/ui/docshell/externalrefmgr.cxx
@@ -1491,59 +1491,7 @@ static std::unique_ptr<ScTokenArray> convertToTokenArray(
         ScMatrixRef xMat = new ScFullMatrix(
             static_cast<SCSIZE>(nCol2-nCol1+1), static_cast<SCSIZE>(nRow2-nRow1+1));
 
-        ColumnBatch<svl::SharedString> aStringBatch(pHostDoc, pSrcDoc, CELLTYPE_STRING, CELLTYPE_EDIT);
-        ColumnBatch<double> aDoubleBatch(pHostDoc, pSrcDoc, CELLTYPE_VALUE, CELLTYPE_VALUE);
-
-        for (SCCOL nCol = nDataCol1; nCol <= nDataCol2; ++nCol)
-        {
-            const SCSIZE nC = nCol - nCol1;
-            for (SCROW nRow = nDataRow1; nRow <= nDataRow2; ++nRow)
-            {
-                const SCSIZE nR = nRow - nRow1;
-
-                ScRefCellValue aCell(*pSrcDoc, ScAddress(nCol, nRow, nTab));
-
-                aStringBatch.update(aCell, nC, nR, xMat);
-                aDoubleBatch.update(aCell, nC, nR, xMat);
-
-                if (aCell.hasEmptyValue())
-                    // Skip empty cells.  Matrix's default values are empty elements.
-                    continue;
-
-                switch (aCell.meType)
-                {
-                    case CELLTYPE_FORMULA:
-                    {
-                        ScFormulaCell* pFCell = aCell.mpFormula;
-                        sal_uInt16 nError = pFCell->GetErrCode();
-                        if (nError)
-                            xMat->PutDouble( CreateDoubleError( nError), nC, nR);
-                        else if (pFCell->IsValue())
-                        {
-                            double fVal = pFCell->GetValue();
-                            xMat->PutDouble(fVal, nC, nR);
-                        }
-                        else
-                        {
-                            svl::SharedString aStr = pFCell->GetString();
-                            aStr = pHostDoc->GetSharedStringPool().intern(aStr.getString());
-                            xMat->PutString(aStr, nC, nR);
-                        }
-                    }
-                    break;
-                    // These are handled in batch:
-                    case CELLTYPE_VALUE:
-                    case CELLTYPE_STRING:
-                    case CELLTYPE_EDIT:
-                    break;
-                    default:
-                        OSL_FAIL("attempted to convert an unknown cell type.");
-                }
-            }
-
-            aStringBatch.flush(nC, xMat);
-            aDoubleBatch.flush(nC, xMat);
-        }
+        pSrcDoc->FillMatrix(*xMat, nTab, nCol1, nRow1, nCol2, nRow2, &pHostDoc->GetSharedStringPool());
         if (!bFirstTab)
             pArray->AddOpCode(ocSep);