tdf#118786 WIN just assert in Yield

This reminds me - again - that Jenkins doesn't run make check.

It turns out InSendMessage() also returns true, if you process a
nested SendMessage in the same thread. Therefore we have to
remove the SalComWndProc assert and just keep the one in the
Yield call.

Why? Because there seem to be no way to get the information
ReplyMessage has access to, so we could detect the caller /
origin of the send message and implement proper nested call
checks. The alternative would be to change all call sites of
SendMessage to:

if ( !pSalData->mpInstance->IsMainThread() )
    SendMessage(...)
else
    SalComWndProc(...)

which is the same SendMessage already does.

Change-Id: I991d68a64952dc5d47ba51edd8635c9e8c46614c
Reviewed-on: https://gerrit.libreoffice.org/59538
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>
(cherry picked from commit bf0a63dc93a746a008fb1656457c77de8df693ba)
Reviewed-on: https://gerrit.libreoffice.org/59565

2 files changed, 11 insertions, 13 deletions

diff --git a/vcl/inc/win/salinst.h b/vcl/inc/win/salinst.h
index 43902c065324..99ca44056641 100644
--- a/vcl/inc/win/salinst.h
+++ b/vcl/inc/win/salinst.h
@@ -39,7 +39,7 @@ public:
     SalYieldMutex*      mpSalYieldMutex;
 
     osl::Condition      maWaitingYieldCond;
-    bool                mbNoYieldLock;
+    unsigned            m_nNoYieldLock;
 
 public:
     WinSalInstance();
diff --git a/vcl/win/app/salinst.cxx b/vcl/win/app/salinst.cxx
index 4e57b8406217..1d9afa7b7265 100644
--- a/vcl/win/app/salinst.cxx
+++ b/vcl/win/app/salinst.cxx
@@ -129,7 +129,7 @@ void SalYieldMutex::doAcquire( sal_uInt32 nLockCount )
     WinSalInstance* pInst = GetSalData()->mpInstance;
     if ( pInst && pInst->IsMainThread() )
     {
-        if ( pInst->mbNoYieldLock )
+        if ( pInst->m_nNoYieldLock )
             return;
         // tdf#96887 If this is the main thread, then we must wait for two things:
         // - the mpSalYieldMutex being freed
@@ -160,7 +160,7 @@ void SalYieldMutex::doAcquire( sal_uInt32 nLockCount )
 sal_uInt32 SalYieldMutex::doRelease( const bool bUnlockAll )
 {
     WinSalInstance* pInst = GetSalData()->mpInstance;
-    if ( pInst && pInst->mbNoYieldLock && pInst->IsMainThread() )
+    if ( pInst && pInst->m_nNoYieldLock && pInst->IsMainThread() )
         return 1;
 
     sal_uInt32 nCount = comphelper::GenericSolarMutex::doRelease( bUnlockAll );
@@ -175,7 +175,7 @@ bool SalYieldMutex::tryToAcquire()
     WinSalInstance* pInst = GetSalData()->mpInstance;
     if ( pInst )
     {
-        if ( pInst->mbNoYieldLock && pInst->IsMainThread() )
+        if ( pInst->m_nNoYieldLock && pInst->IsMainThread() )
             return true;
         else
             return comphelper::GenericSolarMutex::tryToAcquire();
@@ -209,7 +209,7 @@ void ImplSalYieldMutexRelease()
 
 bool SalYieldMutex::IsCurrentThread() const
 {
-    if ( !GetSalData()->mpInstance->mbNoYieldLock )
+    if ( !GetSalData()->mpInstance->m_nNoYieldLock )
         // For the Windows backend, the LO identifier is the system thread ID
         return m_nThreadId == GetCurrentThreadId();
     else
@@ -422,7 +422,7 @@ void DestroySalInstance( SalInstance* pInst )
 
 WinSalInstance::WinSalInstance()
     : mhComWnd( nullptr )
-    , mbNoYieldLock( false )
+    , m_nNoYieldLock( 0 )
 {
     mpSalYieldMutex = new SalYieldMutex();
     mpSalYieldMutex->acquire();
@@ -468,7 +468,7 @@ bool ImplSalYield( bool bWait, bool bHandleAllCurrentEvents )
     bool bWasMsg = false, bOneEvent = false, bWasTimeoutMsg = false;
     ImplSVData *const pSVData = ImplGetSVData();
     WinSalTimer* pTimer = static_cast<WinSalTimer*>( pSVData->maSchedCtx.mpSalTimer );
-    const bool bNoYieldLock = GetSalData()->mpInstance->mbNoYieldLock;
+    const bool bNoYieldLock = (GetSalData()->mpInstance->m_nNoYieldLock > 0);
 
     assert( !bNoYieldLock );
     if ( bNoYieldLock )
@@ -569,10 +569,9 @@ bool WinSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
     case salmsg: \
         if (bIsOtherThreadMessage) \
         { \
-            assert( !pInst->mbNoYieldLock ); \
-            pInst->mbNoYieldLock = true; \
+            ++pInst->m_nNoYieldLock; \
             function; \
-            pInst->mbNoYieldLock = false; \
+            --pInst->m_nNoYieldLock; \
         } \
         else \
         { \
@@ -585,10 +584,9 @@ bool WinSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
     case salmsg: \
         if (bIsOtherThreadMessage) \
         { \
-            assert( !pInst->mbNoYieldLock ); \
-            pInst->mbNoYieldLock = true; \
+            ++pInst->m_nNoYieldLock; \
             nRet = reinterpret_cast<LRESULT>( function ); \
-            pInst->mbNoYieldLock = false; \
+            --pInst->m_nNoYieldLock; \
         } \
         else \
         { \