fdo#51266: Sync dimension members in several places upon refresh.

Or else the pivot table would generate erroneous results or crash
after refresh.

Change-Id: Ia14a6e3d25112e6ecd62d21928639f75e6a8ba7c

13 files changed, 119 insertions, 0 deletions

diff --git a/sc/inc/dpcachetable.hxx b/sc/inc/dpcachetable.hxx
index f104d3b200ba..d83491089d16 100644
--- a/sc/inc/dpcachetable.hxx
+++ b/sc/inc/dpcachetable.hxx
@@ -56,6 +56,12 @@ class ScRange;
 struct ScDPValueData;
 struct ScQueryParam;
 
+/**
+ * Despite the name, this class is only a wrapper to the actual cache, to
+ * provide filtering on the raw data based on the query filter and/or page
+ * field filters. I will rename this class to a more appropriate name in the
+ * future.
+ */
 class SC_DLLPUBLIC ScDPCacheTable
 {
     struct RowFlag
diff --git a/sc/inc/dpgroup.hxx b/sc/inc/dpgroup.hxx
index fe5fd62cd980..219e2e3c35fc 100644
--- a/sc/inc/dpgroup.hxx
+++ b/sc/inc/dpgroup.hxx
@@ -183,6 +183,7 @@ public:
                                                      ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Any > >& rData);
     virtual void                    CalcResults(CalcInfo& rInfo, bool bAutoShow);
     virtual const ScDPCacheTable&   GetCacheTable() const;
+    virtual void ClearCacheTable();
 
     virtual sal_Bool                    IsBaseForGroup(long nDim) const;
     virtual long                    GetGroupBase(long nGroupDim) const;
diff --git a/sc/inc/dpobject.hxx b/sc/inc/dpobject.hxx
index 64525fdab2e1..86aa8aa26056 100644
--- a/sc/inc/dpobject.hxx
+++ b/sc/inc/dpobject.hxx
@@ -241,6 +241,12 @@ public:
 
     void                BuildAllDimensionMembers();
 
+    /**
+     * Remove in the save data entries for members that don't exist anymore.
+     * This is called during pivot table refresh.
+     */
+    void SyncAllDimensionMembers();
+
     static bool         HasRegisteredSources();
     static com::sun::star::uno::Sequence<rtl::OUString> GetRegisteredSources();
     static com::sun::star::uno::Reference<com::sun::star::sheet::XDimensionsSupplier>
diff --git a/sc/inc/dpsave.hxx b/sc/inc/dpsave.hxx
index c4df4eda7b76..28bfafdbe1f4 100644
--- a/sc/inc/dpsave.hxx
+++ b/sc/inc/dpsave.hxx
@@ -33,6 +33,7 @@
 
 #include <boost/ptr_container/ptr_vector.hpp>
 #include <boost/unordered_map.hpp>
+#include <boost/unordered_set.hpp>
 #include <boost/scoped_ptr.hpp>
 
 #include <com/sun/star/sheet/XDimensionsSupplier.hpp>
@@ -122,6 +123,7 @@ private:
     ::com::sun::star::sheet::DataPilotFieldLayoutInfo* pLayoutInfo; // (level)
 
 public:
+    typedef boost::unordered_set<rtl::OUString, rtl::OUStringHash> MemberSetType;
     typedef boost::unordered_map <rtl::OUString, ScDPSaveMember*, rtl::OUStringHash> MemberHash;
     typedef std::list <ScDPSaveMember*> MemberList;
 
@@ -230,6 +232,8 @@ public:
     void UpdateMemberVisibility(const ::boost::unordered_map< ::rtl::OUString, bool, ::rtl::OUStringHash>& rData);
 
     bool HasInvisibleMember() const;
+
+    void RemoveObsoleteMembers(const MemberSetType& rMembers);
 };
 
 
@@ -344,6 +348,7 @@ public:
     SC_DLLPUBLIC ScDPDimensionSaveData* GetDimensionData(); // create if not there
     void SetDimensionData( const ScDPDimensionSaveData* pNew ); // copied
     void BuildAllDimensionMembers(ScDPTableData* pData);
+    void SyncAllDimensionMembers(ScDPTableData* pData);
 
     /**
      * Check whether a dimension has one or more invisible members.
diff --git a/sc/inc/dpsdbtab.hxx b/sc/inc/dpsdbtab.hxx
index eba6da20d52d..a96b0115014b 100644
--- a/sc/inc/dpsdbtab.hxx
+++ b/sc/inc/dpsdbtab.hxx
@@ -88,6 +88,7 @@ public:
                                                      ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Any > >& rData);
     virtual void                    CalcResults(CalcInfo& rInfo, bool bAutoShow);
     virtual const ScDPCacheTable&   GetCacheTable() const;
+    virtual void ClearCacheTable();
 };
 
 
diff --git a/sc/inc/dpshttab.hxx b/sc/inc/dpshttab.hxx
index 38a0d589e235..f3cf2fd1b0d8 100644
--- a/sc/inc/dpshttab.hxx
+++ b/sc/inc/dpshttab.hxx
@@ -130,6 +130,7 @@ public:
                                                      ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Any > >& rData);
     virtual void                    CalcResults(CalcInfo& rInfo, bool bAutoShow);
     virtual const ScDPCacheTable&   GetCacheTable() const;
+    virtual void ClearCacheTable();
 };
 
 
diff --git a/sc/inc/dptabdat.hxx b/sc/inc/dptabdat.hxx
index b11e8927d34f..03107de7419b 100644
--- a/sc/inc/dptabdat.hxx
+++ b/sc/inc/dptabdat.hxx
@@ -156,6 +156,7 @@ public:
                                                      ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Any > >& rData) = 0;
     virtual void                    CalcResults(CalcInfo& rInfo, bool bAutoShow) = 0;
     virtual const ScDPCacheTable&   GetCacheTable() const = 0;
+    virtual void ClearCacheTable() = 0;
 
                                     // overloaded in ScDPGroupTableData:
     virtual sal_Bool                    IsBaseForGroup(long nDim) const;
diff --git a/sc/source/core/data/dpgroup.cxx b/sc/source/core/data/dpgroup.cxx
index 9b5bf9b5763c..2959a65621b6 100644
--- a/sc/source/core/data/dpgroup.cxx
+++ b/sc/source/core/data/dpgroup.cxx
@@ -789,6 +789,11 @@ const ScDPCacheTable& ScDPGroupTableData::GetCacheTable() const
     return pSourceData->GetCacheTable();
 }
 
+void ScDPGroupTableData::ClearCacheTable()
+{
+    pSourceData->ClearCacheTable();
+}
+
 void ScDPGroupTableData::FillGroupValues(vector<SCROW>& rItems, const vector<long>& rDims)
 {
     long nGroupedColumns = aGroups.size();
diff --git a/sc/source/core/data/dpobject.cxx b/sc/source/core/data/dpobject.cxx
index b4b62cfdc278..369307b012f2 100644
--- a/sc/source/core/data/dpobject.cxx
+++ b/sc/source/core/data/dpobject.cxx
@@ -699,6 +699,23 @@ void ScDPObject::BuildAllDimensionMembers()
     pSaveData->BuildAllDimensionMembers(GetTableData());
 }
 
+void ScDPObject::SyncAllDimensionMembers()
+{
+    if (!pSaveData)
+        return;
+
+    // #i111857# don't always create empty mpTableData for external service.
+    // Ideally, xSource should be used instead of mpTableData.
+    if (pServDesc)
+        return;
+
+    ScDPTableData* pData = GetTableData();
+    // Refresh the cache wrapper since the cache may have changed.
+    pData->ClearCacheTable();
+    pData->CreateCacheTable();
+    pSaveData->SyncAllDimensionMembers(pData);
+}
+
 bool ScDPObject::GetMemberNames( sal_Int32 nDim, Sequence<OUString>& rNames )
 {
     vector<ScDPLabelData::Member> aMembers;
diff --git a/sc/source/core/data/dpsave.cxx b/sc/source/core/data/dpsave.cxx
index 2bc011a8e548..fc289b93f2bf 100644
--- a/sc/source/core/data/dpsave.cxx
+++ b/sc/source/core/data/dpsave.cxx
@@ -37,6 +37,7 @@
 #include "global.hxx"
 #include "dptabsrc.hxx"
 #include "dputil.hxx"
+#include "stlalgorithm.hxx"
 
 #include <sal/types.h>
 #include "comphelper/string.hxx"
@@ -732,6 +733,30 @@ bool ScDPSaveDimension::HasInvisibleMember() const
     return false;
 }
 
+void ScDPSaveDimension::RemoveObsoleteMembers(const MemberSetType& rMembers)
+{
+    maMemberHash.clear();
+    MemberList aNew;
+    MemberList::iterator it = maMemberList.begin(), itEnd = maMemberList.end();
+    for (; it != itEnd; ++it)
+    {
+        ScDPSaveMember* pMem = *it;
+        if (rMembers.count(pMem->GetName()))
+        {
+            // This member still exists.
+            maMemberHash.insert(MemberHash::value_type(pMem->GetName(), pMem));
+            aNew.push_back(pMem);
+        }
+        else
+        {
+            // This member no longer exists.
+            delete pMem;
+        }
+    }
+
+    maMemberList.swap(aNew);
+}
+
 ScDPSaveData::ScDPSaveData() :
     pDimensionData( NULL ),
     nColumnGrandMode( SC_DPSAVEMODE_DONTKNOW ),
@@ -1260,6 +1285,46 @@ void ScDPSaveData::BuildAllDimensionMembers(ScDPTableData* pData)
     mbDimensionMembersBuilt = true;
 }
 
+void ScDPSaveData::SyncAllDimensionMembers(ScDPTableData* pData)
+{
+    typedef boost::unordered_map<rtl::OUString, long, rtl::OUStringHash> NameIndexMap;
+
+    // First, build a dimension name-to-index map.
+    NameIndexMap aMap;
+    long nColCount = pData->GetColumnCount();
+    for (long i = 0; i < nColCount; ++i)
+        aMap.insert(NameIndexMap::value_type(pData->getDimensionName(i), i));
+
+    NameIndexMap::const_iterator itMapEnd = aMap.end();
+
+    DimsType::iterator it = aDimList.begin(), itEnd = aDimList.end();
+    for (it = aDimList.begin(); it != itEnd; ++it)
+    {
+        const ::rtl::OUString& rDimName = it->GetName();
+        if (rDimName.isEmpty())
+            // empty dimension name. It must be data layout.
+            continue;
+
+        NameIndexMap::const_iterator itMap = aMap.find(rDimName);
+        if (itMap == itMapEnd)
+            // dimension name not in the data. This should never happen!
+            continue;
+
+        ScDPSaveDimension::MemberSetType aMemNames;
+        long nDimIndex = itMap->second;
+        const std::vector<SCROW>& rMembers = pData->GetColumnEntries(nDimIndex);
+        size_t nMemberCount = rMembers.size();
+        for (size_t j = 0; j < nMemberCount; ++j)
+        {
+            const ScDPItemData* pMemberData = pData->GetMemberById(nDimIndex, rMembers[j]);
+            rtl::OUString aMemName = pData->GetFormattedString(nDimIndex, *pMemberData);
+            aMemNames.insert(aMemName);
+        }
+
+        it->RemoveObsoleteMembers(aMemNames);
+    }
+}
+
 bool ScDPSaveData::HasInvisibleMember(const OUString& rDimName) const
 {
     ScDPSaveDimension* pDim = GetExistingDimensionByName(rDimName);
diff --git a/sc/source/core/data/dpsdbtab.cxx b/sc/source/core/data/dpsdbtab.cxx
index b085473b2408..27c2ef23834f 100644
--- a/sc/source/core/data/dpsdbtab.cxx
+++ b/sc/source/core/data/dpsdbtab.cxx
@@ -177,6 +177,11 @@ const ScDPCacheTable& ScDatabaseDPData::GetCacheTable() const
     return aCacheTable;
 }
 
+void ScDatabaseDPData::ClearCacheTable()
+{
+    aCacheTable.clear();
+}
+
 // -----------------------------------------------------------------------
 
 
diff --git a/sc/source/core/data/dpshttab.cxx b/sc/source/core/data/dpshttab.cxx
index 43911c8a5a9f..8e9a982bf5ec 100644
--- a/sc/source/core/data/dpshttab.cxx
+++ b/sc/source/core/data/dpshttab.cxx
@@ -228,6 +228,11 @@ const ScDPCacheTable& ScSheetDPData::GetCacheTable() const
     return aCacheTable;
 }
 
+void ScSheetDPData::ClearCacheTable()
+{
+    aCacheTable.clear();
+}
+
 ScSheetSourceDesc::ScSheetSourceDesc(ScDocument* pDoc) :
     mpDoc(pDoc) {}
 
diff --git a/sc/source/ui/docshell/dbdocfun.cxx b/sc/source/ui/docshell/dbdocfun.cxx
index 0e25ee0d38f0..ee642c75a8f8 100644
--- a/sc/source/ui/docshell/dbdocfun.cxx
+++ b/sc/source/ui/docshell/dbdocfun.cxx
@@ -1478,6 +1478,7 @@ sal_uLong ScDBDocFunc::RefreshPivotTables(ScDPObject* pDPObj, bool bApi)
     for (; it != itEnd; ++it)
     {
         ScDPObject* pObj = *it;
+        pObj->SyncAllDimensionMembers();
 
         // This action is intentionally not undoable since it modifies cache.
         DataPilotUpdate(pObj, pObj, false, bApi);