Lok: correct tile invalidation merging

Rectangles that are empty (i.e. IsEmpty() returns true)
were incorrectly considered to mean equivalent to "EMPTY".
The latter means full-area, while the former mean zero-area.

This fixes the issue by restrict full-area to rectangles
with 2 billion units on the each side (roughly INT_MAX) or
more, and using this new check rather than IsEmpty().

Change-Id: I12aca17267f5dd33b2932012d1d9db3545f9af6f
Reviewed-on: https://gerrit.libreoffice.org/31458
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Ashod Nakashian <ashnakash@gmail.com>
(cherry picked from commit 9c218858f1bd83ffdd72dd943a841cffa5a93b8c)

2 files changed, 42 insertions, 3 deletions

diff --git a/desktop/qa/desktop_lib/test_desktop_lib.cxx b/desktop/qa/desktop_lib/test_desktop_lib.cxx
index 005a740954fd..e79866dddc4e 100644
--- a/desktop/qa/desktop_lib/test_desktop_lib.cxx
+++ b/desktop/qa/desktop_lib/test_desktop_lib.cxx
@@ -98,6 +98,7 @@ public:
     void testContextMenuWriter();
     void testContextMenuImpress();
     void testNotificationCompression();
+    void testTileInvalidationCompression();
     void testPartInInvalidation();
     void testRedlineWriter();
     void testTrackChanges();
@@ -131,6 +132,7 @@ public:
     CPPUNIT_TEST(testContextMenuWriter);
     CPPUNIT_TEST(testContextMenuImpress);
     CPPUNIT_TEST(testNotificationCompression);
+    CPPUNIT_TEST(testTileInvalidationCompression);
     CPPUNIT_TEST(testPartInInvalidation);
     CPPUNIT_TEST(testRedlineWriter);
     CPPUNIT_TEST(testTrackChanges);
@@ -1376,6 +1378,33 @@ void DesktopLOKTest::testNotificationCompression()
     CPPUNIT_ASSERT_EQUAL(std::string(".uno:AssignLayout=1"), std::get<1>(notifs[i++]));
 }
 
+void DesktopLOKTest::testTileInvalidationCompression()
+{
+    LibLODocument_Impl* pDocument = loadDoc("blank_text.odt");
+    std::vector<std::tuple<int, std::string>> notifs;
+    std::unique_ptr<CallbackFlushHandler> handler(new CallbackFlushHandler(pDocument, callbackCompressionTest, &notifs));
+
+    comphelper::LibreOfficeKit::setPartInInvalidation(true);
+    comphelper::ScopeGuard aGuard([]()
+    {
+        comphelper::LibreOfficeKit::setPartInInvalidation(false);
+    });
+
+    handler->queue(LOK_CALLBACK_INVALIDATE_TILES, "0, 0, 239, 239, 0");
+    handler->queue(LOK_CALLBACK_INVALIDATE_TILES, "0, 0, 2147483767, 2147483767, 0");
+    handler->queue(LOK_CALLBACK_INVALIDATE_TILES, "0, 0, 239, 239, 0");
+    handler->queue(LOK_CALLBACK_INVALIDATE_TILES, "-121, -121, 300, 300, 0");
+    handler->queue(LOK_CALLBACK_INVALIDATE_TILES, "0, 0, -32767, -32767, 0");
+
+    Scheduler::ProcessEventsToIdle();
+
+    CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(1), notifs.size());
+
+    size_t i = 0;
+    CPPUNIT_ASSERT_EQUAL((int)LOK_CALLBACK_INVALIDATE_TILES, (int)std::get<0>(notifs[i]));
+    CPPUNIT_ASSERT_EQUAL(std::string("0, 0, 2147483767, 2147483767, 0"), std::get<1>(notifs[i++]));
+}
+
 void DesktopLOKTest::testPartInInvalidation()
 {
     LibLODocument_Impl* pDocument = loadDoc("blank_text.odt");
diff --git a/desktop/source/lib/init.cxx b/desktop/source/lib/init.cxx
index 362ee6194caf..0539e9766b14 100644
--- a/desktop/source/lib/init.cxx
+++ b/desktop/source/lib/init.cxx
@@ -356,11 +356,20 @@ struct RectangleAndPart
         return ss.str().c_str();
     }
 
+    /// Infinite Rectangle is when both dimensions are >= 2e7.
+    // ~2 billion twips is INT_MAX, which is full-area.
+    bool isInfinite() const
+    {
+        return m_aRectangle.GetWidth() >= 2e7 &&
+               m_aRectangle.GetHeight() >= 2e7;
+    }
+
     static RectangleAndPart Create(const std::string& rPayload)
     {
         RectangleAndPart aRet;
         if (rPayload.find("EMPTY") == 0) // payload starts with "EMPTY"
         {
+            aRet.m_aRectangle = Rectangle(0, 0, INT_MAX, INT_MAX);
             if (comphelper::LibreOfficeKit::isPartInInvalidation())
                 aRet.m_nPart = std::stol(rPayload.substr(6));
 
@@ -673,9 +682,9 @@ void CallbackFlushHandler::queue(const int type, const char* data)
         {
             RectangleAndPart rcOld = RectangleAndPart::Create(pos->second);
             RectangleAndPart rcNew = RectangleAndPart::Create(payload);
-            if (rcOld.m_aRectangle.IsEmpty() && rcOld.m_nPart == rcNew.m_nPart)
+            if (rcOld.isInfinite() && rcOld.m_nPart == rcNew.m_nPart)
             {
-                //SAL_WARN("lok", "Skipping queue [" + std::to_string(type) + "]: [" + payload + "] since all tiles need to be invalidated.");
+                SAL_WARN("lok", "Skipping queue [" << type << "]: [" << payload << "] since all tiles need to be invalidated.");
                 return;
             }
         }
@@ -762,8 +771,9 @@ void CallbackFlushHandler::queue(const int type, const char* data)
             {
                 RectangleAndPart rcNew = RectangleAndPart::Create(payload);
                 //SAL_WARN("lok", "New: " << rcNew.toString());
-                if (rcNew.m_aRectangle.IsEmpty())
+                if (rcNew.isInfinite())
                 {
+                    SAL_WARN("lok", "Have Empty [" << type << "]: [" << payload << "] so removing all.");
                     removeAll(
                         [type, &rcNew] (const queue_type::value_type& elem) {
                             if (elem.first == type)