fix deadlock observed on jenkins

where we are locking SolarMutex and UndoManagerHelper_Impl::m_aMutex in
different orders in different code-paths, which is deadly when we call
into the same class recursively.

Thread 2 (Thread 0x7f5cf303b700 (LWP 22637)):
(this=0x7f5cf3037cd8) at include/vcl/svapp.hxx:1373
framework::UndoManagerHelper_Impl::impl_clear() (this=0x5f90d50) at
framework/source/fwe/helper/undomanagerhelper.cxx:691
  holds UndoManagerHelper_Impl::m_aMutex
  tries to acquire SolarMutex
framework::UndoManagerHelper_Impl::clear(framework::IMutexGuard&)::$_3::operator()()
const (this=0x5519040) at
framework/source/fwe/helper/undomanagerhelper.cxx:403
framework::UndoManagerHelper_Impl::clear(framework::IMutexGuard&)::$_3>::_M_invoke(std::_Any_data
const&) (__functor=...) at
/opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/../../../../include/c++/7/bits/std_function.h:316
(this=0x5519040) at
/opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/../../../../include/c++/7/bits/std_function.h:706
namespace)::UndoManagerRequest::execute() (this=0x5519030) at
framework/source/fwe/helper/undomanagerhelper.cxx:160
framework::UndoManagerHelper_Impl::impl_processRequest(std::function<void
()> const&, framework::IMutexGuard&) (this=0x5f90d50, i_request=...,
i_instanceLock=...) at
framework/source/fwe/helper/undomanagerhelper.cxx:490
framework::UndoManagerHelper_Impl::clear(framework::IMutexGuard&)
(this=0x5f90d50, i_instanceLock=...) at
framework/source/fwe/helper/undomanagerhelper.cxx:402
framework::UndoManagerHelper::clear(framework::IMutexGuard&)
(this=0x5f90c38, i_instanceLock=...) at
framework/source/fwe/helper/undomanagerhelper.cxx:999
at chart2/source/model/main/UndoManager.cxx:278
void*, _typelib_TypeDescriptionReference*, bool, unsigned long*,
unsigned int, unsigned long*, double*) (pThis=0x5f90bc8,
nVtableIndex=18, pRegisterReturn=0x0, pReturnTypeRef=0xa2dac0,
bSimpleReturn=true, pStack=0x7f5cf30380e0, nStack=0,
pGPR=0x7f5cf30383e0, pFPR=0x7f5cf30383a0) at
bridges/source/cpp_uno/gcc3_linux_x86-64/callvirtualmethod.cxx:77
cpp_call(bridges::cpp_uno::shared::UnoInterfaceProxy*,
bridges::cpp_uno::shared::VtableSlot,
_typelib_TypeDescriptionReference*, int, _typelib_MethodParameter*,
void*, void**, _uno_Any**) (pThis=0x5e40ac0, aVtableSlot=...,
pReturnTypeRef=0xa2dac0, nParams=0, pParams=0x0, pUnoReturn=0x0,
pUnoArgs=0x0, ppUnoExc=0x7f5cf30388e0) at
bridges/source/cpp_uno/gcc3_linux_x86-64/uno2cpp.cxx:233
pMemberDescr=0x551dd30, pReturn=0x0, pArgs=0x0,
ppException=0x7f5cf30388e0) at
bridges/source/cpp_uno/gcc3_linux_x86-64/uno2cpp.cxx:413
binaryurp::IncomingRequest::execute_throw(binaryurp::BinaryAny*,
std::__debug::vector<binaryurp::BinaryAny,
std::allocator<binaryurp::BinaryAny> >*) const (this=0x28a3cc0,
returnValue=0x7f5cf3039040, outArguments=0x7f5cf3039008) at
binaryurp/source/incomingrequest.cxx:236
(this=0x28a3cc0) at binaryurp/source/incomingrequest.cxx:79

Thread 1 (Thread 0x7f5cfd03eec0 (LWP 22302)):
(this=0x7ffcdf1b1720, t=...) at include/osl/mutex.hxx:144
comphelper::OInterfaceContainerHelper3<com::sun::star::util::XModifyListener>::addInterface(com::sun::star::uno::Reference<com::sun::star::util::XModifyListener>
const&) (this=0x5f90da0, rListener=...) at
include/comphelper/interfacecontainer3.hxx:313
    trying to acquire UndoManagerHelper_Impl::m_aMutex
framework::UndoManagerHelper_Impl::addModifyListener(com::sun::star::uno::Reference<com::sun::star::util::XModifyListener>
const&) (this=0x5f90d50, i_listener=...) at
framework/source/fwe/helper/undomanagerhelper.cxx:286
framework::UndoManagerHelper::addModifyListener(com::sun::star::uno::Reference<com::sun::star::util::XModifyListener>
const&) (this=0x5f90c38, i_listener=...) at
framework/source/fwe/helper/undomanagerhelper.cxx:1047
chart::UndoManager::addModifyListener(com::sun::star::uno::Reference<com::sun::star::util::XModifyListener>
const&) (this=0x5f90bc0, i_listener=...) at
chart2/source/model/main/UndoManager.cxx:338
(this=0x1d470d0) at
chart2/source/controller/main/UndoCommandDispatch.cxx:57
chart::CommandDispatchContainer::getDispatchForURL(com::sun::star::util::URL
const&) (this=0x5f66e58, rURL=...) at
chart2/source/controller/main/CommandDispatchContainer.cxx:93
chart::ChartController::queryDispatch(com::sun::star::util::URL const&,
rtl::OUString const&, int) (this=0x5f66c50, rURL=...,
rTargetFrameName=...) at
chart2/source/controller/main/ChartController.cxx:1055
    locks SolarMutex
chart::ChartController::queryDispatch(com::sun::star::util::URL const&,
rtl::OUString const&, int) () at
/home/tdf/lode/jenkins/workspace/lo_gerrit/Config/linux_clang_dbgutil_64/instdir/program/../program/libchartcontrollerlo.so
framework::DispatchProvider::implts_queryFrameDispatch(com::sun::star::uno::Reference<com::sun::star::frame::XFrame>
const&, com::sun::star::util::URL const&, rtl::OUString const&, int)
(this=0x5f5e6c0, xFrame=..., aURL=..., sTargetFrameName=...,
nSearchFlags=0) at framework/source/dispatch/dispatchprovider.cxx:352
framework::DispatchProvider::queryDispatch(com::sun::star::util::URL
const&, rtl::OUString const&, int) (this=0x5f5e6c0, aURL=...,
sTargetFrameName=..., nSearchFlags=0) at
framework/source/dispatch/dispatchprovider.cxx:106
framework::DispatchProvider::queryDispatch(com::sun::star::util::URL
const&, rtl::OUString const&, int) () at
/home/tdf/lode/jenkins/workspace/lo_gerrit/Config/linux_clang_dbgutil_64/instdir/program/libfwklo.so
framework::InterceptionHelper::queryDispatch(com::sun::star::util::URL
const&, rtl::OUString const&, int) (this=0x5f59ce0, aURL=...,
sTargetFrameName=..., nSearchFlags=0) at
framework/source/dispatch/interceptionhelper.cxx:87
framework::InterceptionHelper::queryDispatch(com::sun::star::util::URL
const&, rtl::OUString const&, int) () at
/home/tdf/lode/jenkins/workspace/lo_gerrit/Config/linux_clang_dbgutil_64/instdir/program/libfwklo.so
namespace)::XFrameImpl::queryDispatch(com::sun::star::util::URL const&,
rtl::OUString const&, int) (this=0x5f599c0, aURL=...,
sTargetFrameName=..., nSearchFlags=0) at
framework/source/services/frame.cxx:2329
namespace)::XFrameImpl::queryDispatch(com::sun::star::util::URL const&,
rtl::OUString const&, int) () at
/home/tdf/lode/jenkins/workspace/lo_gerrit/Config/linux_clang_dbgutil_64/instdir/program/libfwklo.so
(this=0x5fe8b00) at svtools/source/uno/toolboxcontroller.cxx:520
(this=0x5fe8b00) at svtools/source/uno/toolboxcontroller.cxx:242
(this=0x5fdd060) at framework/source/uielement/toolbarmanager.cxx:731
framework::ToolBarManager::AsyncUpdateControllersHdl(Timer*)
(this=0x5fdd060) at framework/source/uielement/toolbarmanager.cxx:2298
framework::ToolBarManager::LinkStubAsyncUpdateControllersHdl(void*,
Timer*) (instance=0x5fdd060, data=0x5fdd258) at
framework/source/uielement/toolbarmanager.cxx:2285
(this=0x5fdd278, data=0x5fdd258) at include/tools/link.hxx:111
vcl/source/app/timer.cxx:75
vcl/source/app/scheduler.cxx:474
vcl/inc/saltimer.hxx:54
(this=0xa1ab10, bExecuteTimers=true) at vcl/headless/svpinst.cxx:161

Change-Id: I1971d094513cc747eff44db8e2a131ad0aae1506
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/145316
Tested-by: Jenkins
Reviewed-by: Noel Grandin <noel.grandin@collabora.co.uk>
(cherry picked from commit 92d9888e79b768996db5526eb3965259b38ced76)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/145305
Reviewed-by: Xisco Fauli <xiscofauli@libreoffice.org>

1 files changed, 4 insertions, 2 deletions

diff --git a/framework/source/fwe/helper/undomanagerhelper.cxx b/framework/source/fwe/helper/undomanagerhelper.cxx
index 582335f3d71e..27780dc2719e 100644
--- a/framework/source/fwe/helper/undomanagerhelper.cxx
+++ b/framework/source/fwe/helper/undomanagerhelper.cxx
@@ -201,6 +201,8 @@ namespace framework
     {
     private:
         ::osl::Mutex                        m_aMutex;
+        /// Use different mutex for listeners to prevent ABBA deadlocks
+        ::osl::Mutex                        m_aListenerMutex;
         std::mutex                          m_aQueueMutex;
         bool                                m_bAPIActionRunning;
         bool                                m_bProcessingEvents;
@@ -224,8 +226,8 @@ namespace framework
             :m_bAPIActionRunning( false )
             ,m_bProcessingEvents( false )
             ,m_nLockCount( 0 )
-            ,m_aUndoListeners( m_aMutex )
-            ,m_aModifyListeners( m_aMutex )
+            ,m_aUndoListeners( m_aListenerMutex )
+            ,m_aModifyListeners( m_aListenerMutex )
             ,m_rUndoManagerImplementation( i_undoManagerImpl )
         {
             getUndoManager().AddUndoListener( *this );