Resolves: fdo49629 GotoEndOfWord fails with footnote at word end

a) remove special handling of 0x0002 in our custom icu rules.
   Which brings us a step closer to getting rid of at least
   some of them in favour of the defaults
b) expand the 0x02 in SwTxtNode::BuildConversionMap like we
   do for fields so

Good side effect is our word count and character count now take into account
the actual footnote indicator text, as does our cursor travelling. Both of
which are more word-alike.

Change-Id: I3b0024ac4b10934bee7a9e83b0fce08a18556c7b

14 files changed, 141 insertions, 31 deletions

diff --git a/i18npool/qa/cppunit/test_breakiterator.cxx b/i18npool/qa/cppunit/test_breakiterator.cxx
index 3c2ba50b8712..3ed44fcb9ff5 100644
--- a/i18npool/qa/cppunit/test_breakiterator.cxx
+++ b/i18npool/qa/cppunit/test_breakiterator.cxx
@@ -125,6 +125,39 @@ void TestBreakIterator::testLineBreaking()
             CPPUNIT_ASSERT_MESSAGE("Expected a break at the the start of the word", aResult.breakIndex == aWord.getLength()+1);
         }
     }
+
+    //See https://bugs.freedesktop.org/show_bug.cgi?id=49629
+    for (int mode = i18n::WordType::ANY_WORD; mode <= i18n::WordType::WORD_COUNT; ++mode)
+    {
+        //make sure that in all cases isBeginWord and isEndWord matches getWordBoundary
+        //
+        //test "Word", then "Word\x01" then "Word\x02"
+        for (sal_Unicode i = 0; i < 3; ++i)
+        {
+            ::rtl::OUString aTest("Word");
+            if (i > 0)
+                aTest += rtl::OUString(i) + rtl::OUString("Word");
+            i18n::Boundary aBounds = m_xBreak->getWordBoundary(aTest, 0, aLocale, mode, true);
+            switch (mode)
+            {
+                case i18n::WordType::ANY_WORD:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+                case i18n::WordType::ANYWORD_IGNOREWHITESPACES:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+                case i18n::WordType::DICTIONARY_WORD:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+                case i18n::WordType::WORD_COUNT:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+            }
+
+            CPPUNIT_ASSERT(m_xBreak->isBeginWord(aTest, aBounds.startPos, aLocale, mode));
+            CPPUNIT_ASSERT(m_xBreak->isEndWord(aTest, aBounds.endPos, aLocale, mode));
+        }
+    }
 }
 
 //See https://bugs.freedesktop.org/show_bug.cgi?id=49629
diff --git a/i18npool/source/breakiterator/data/dict_word.txt b/i18npool/source/breakiterator/data/dict_word.txt
index f69f038e7f7e..b1666f44daab 100644
--- a/i18npool/source/breakiterator/data/dict_word.txt
+++ b/i18npool/source/breakiterator/data/dict_word.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/dict_word_ca.txt b/i18npool/source/breakiterator/data/dict_word_ca.txt
index ca4a14abe724..6ad6a0bbb915 100644
--- a/i18npool/source/breakiterator/data/dict_word_ca.txt
+++ b/i18npool/source/breakiterator/data/dict_word_ca.txt
@@ -22,7 +22,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
                                    [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
 
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Katakana
                            - [:Script = Thai:]
                            - [:Script = Lao:]
diff --git a/i18npool/source/breakiterator/data/dict_word_fi.txt b/i18npool/source/breakiterator/data/dict_word_fi.txt
index 2d7617908576..7026c992f1f9 100644
--- a/i18npool/source/breakiterator/data/dict_word_fi.txt
+++ b/i18npool/source/breakiterator/data/dict_word_fi.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:] [:name = HYPHEN-MINUS:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:] [:name = HYPHEN-MINUS:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/dict_word_he.txt b/i18npool/source/breakiterator/data/dict_word_he.txt
index b98350e9d3fe..40197d92a431 100644
--- a/i18npool/source/breakiterator/data/dict_word_he.txt
+++ b/i18npool/source/breakiterator/data/dict_word_he.txt
@@ -22,7 +22,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
                                    [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
 
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Katakana
                            - [:Script = Thai:]
                            - [:Script = Lao:]
diff --git a/i18npool/source/breakiterator/data/dict_word_hu.txt b/i18npool/source/breakiterator/data/dict_word_hu.txt
index 49ae8b5bf407..33d014ec60ed 100644
--- a/i18npool/source/breakiterator/data/dict_word_hu.txt
+++ b/i18npool/source/breakiterator/data/dict_word_hu.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                 [:name = PERCENT SIGN:] [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
                 [:name = SECTION SIGN:] [:name = DEGREE SIGN:] [:name = EURO SIGN:]
                 [:name = HYPHEN-MINUS:] [:name = EN DASH:] [:name = EM DASH:]
diff --git a/i18npool/source/breakiterator/data/dict_word_nodash.txt b/i18npool/source/breakiterator/data/dict_word_nodash.txt
index 4c5c80823041..279cc50e5b66 100644
--- a/i18npool/source/breakiterator/data/dict_word_nodash.txt
+++ b/i18npool/source/breakiterator/data/dict_word_nodash.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/dict_word_prepostdash.txt b/i18npool/source/breakiterator/data/dict_word_prepostdash.txt
index 0a60cc5d4924..fb29b478af21 100644
--- a/i18npool/source/breakiterator/data/dict_word_prepostdash.txt
+++ b/i18npool/source/breakiterator/data/dict_word_prepostdash.txt
@@ -30,7 +30,7 @@ $Hangul = [:Script = HANGUL:];
 $PrePostDashHyphen = [ [:name = HYPHEN-MINUS:] ];
 
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/edit_word.txt b/i18npool/source/breakiterator/data/edit_word.txt
index ff8321262269..92b344c19d41 100644
--- a/i18npool/source/breakiterator/data/edit_word.txt
+++ b/i18npool/source/breakiterator/data/edit_word.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
+$ALetter   = [[:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/edit_word_he.txt b/i18npool/source/breakiterator/data/edit_word_he.txt
index 0061f552e1bf..0b5908814e08 100644
--- a/i18npool/source/breakiterator/data/edit_word_he.txt
+++ b/i18npool/source/breakiterator/data/edit_word_he.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
+$ALetter   = [[:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/edit_word_hu.txt b/i18npool/source/breakiterator/data/edit_word_hu.txt
index 5262726cbce5..4a08acab0029 100644
--- a/i18npool/source/breakiterator/data/edit_word_hu.txt
+++ b/i18npool/source/breakiterator/data/edit_word_hu.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
+$ALetter   = [[:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
                 [:name = PERCENT SIGN:] [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
                 [:name = SECTION SIGN:] [:name = DEGREE SIGN:] [:name = EURO SIGN:]
                 [:name = HYPHEN-MINUS:] [:name = EN DASH:] [:name = EM DASH:]
diff --git a/sw/qa/complex/writer/TextPortionEnumerationTest.java b/sw/qa/complex/writer/TextPortionEnumerationTest.java
index 62c3624e4c8e..3170b930ff9c 100644
--- a/sw/qa/complex/writer/TextPortionEnumerationTest.java
+++ b/sw/qa/complex/writer/TextPortionEnumerationTest.java
@@ -3187,26 +3187,31 @@ public class TextPortionEnumerationTest
         xTextCursor.gotoStart(false);
         XWordCursor xWordCursor = UnoRuntime.queryInterface(XWordCursor.class, xTextCursor);
 
-        bSuccess = xWordCursor.gotoNextWord(true);
+        bSuccess = xWordCursor.gotoNextWord(true);              //at start of "words"
         assertTrue("gotoNextWord(): failed", bSuccess);
         {
             String string = xTextCursor.getString();
             assertEquals("gotoNextWord(): wrong string",
                          "Two ", string);
         }
-        bSuccess = xWordCursor.gotoNextWord(false);
+        bSuccess = xWordCursor.gotoNextWord(false);             //at end of "words", cannot leave metafield
         assertFalse("gotoNextWord(): succeeded", bSuccess);
         xTextCursor.collapseToEnd();
-        bSuccess = xWordCursor.gotoPreviousWord(true);
+        bSuccess = xWordCursor.gotoPreviousWord(true);          //at start of "words"
         assertTrue("gotoPreviousWord(): failed", bSuccess);
         {
             String string = xTextCursor.getString();
             assertEquals("gotoPreviousWord(): wrong string",
                          "words", string);
         }
-        bSuccess = xWordCursor.gotoPreviousWord(false);
+        bSuccess = xWordCursor.gotoPreviousWord(false);         //at start of "Two"
+        assertTrue("gotoPreviousWord(): failed", bSuccess);
+
+
+        bSuccess = xWordCursor.gotoPreviousWord(false);         //cannot leave metafield
         assertFalse("gotoPreviousWord(): succeeded", bSuccess);
-        bSuccess = xWordCursor.gotoEndOfWord(true);
+
+        bSuccess = xWordCursor.gotoEndOfWord(true);             //at end of "Two"
         assertTrue("gotoEndOfWord(): failed", bSuccess);
         {
             String string = xTextCursor.getString();
@@ -3288,6 +3293,23 @@ public class TextPortionEnumerationTest
         assertFalse("gotoEndOfParagraph(): succeeded", bSuccess);
     }
 
+    /** See https://bugs.freedesktop.org/show_bug.cgi?id=49629
+        ensure that gotoEndOfWord does not fail when footnote is at word end*/
+    @Test public void testXTextCursor() throws Exception
+    {
+        RangeInserter inserter = new RangeInserter(m_xDoc);
+        XText xDocText = m_xDoc.getText();
+        XTextCursor xDocTextCursor = xDocText.createTextCursor();
+        inserter.insertText(xDocTextCursor, "Text");
+        XWordCursor xWordCursor = UnoRuntime.queryInterface(XWordCursor.class, xDocTextCursor);
+        xWordCursor.gotoEndOfWord(false);
+        inserter.insertFootnote(xDocTextCursor, "footnote");
+        xDocTextCursor.gotoStart(false);
+        boolean bSuccess = xWordCursor.gotoEndOfWord(true);
+        assertTrue("gotoEndOfWord(): failed", bSuccess);
+        String string = xWordCursor.getString();
+        assertEquals("gotoEndOfWord(): wrong string", "Text", string);
+    }
 
     abstract class AttachHelper
     {
diff --git a/sw/qa/core/swdoc-test.cxx b/sw/qa/core/swdoc-test.cxx
index f8c1317fead0..49b0930189c2 100644
--- a/sw/qa/core/swdoc-test.cxx
+++ b/sw/qa/core/swdoc-test.cxx
@@ -58,6 +58,8 @@
 #include "swscanner.hxx"
 #include "swmodule.hxx"
 #include "swtypes.hxx"
+#include "fmtftn.hxx"
+#include "fmtrfmrk.hxx"
 
 SO2_DECL_REF(SwDocShell)
 SO2_IMPL_REF(SwDocShell)
@@ -222,7 +224,7 @@ void SwDocTest::testSwScanner()
     SwNodeIndex aIdx(m_pDoc->GetNodes().GetEndOfContent(), -1);
     SwPaM aPaM(aIdx);
 
-    const SwTxtNode* pTxtNode = aPaM.GetNode()->GetTxtNode();
+    SwTxtNode* pTxtNode = aPaM.GetNode()->GetTxtNode();
 
     CPPUNIT_ASSERT_MESSAGE("Has Text Node", pTxtNode);
 
@@ -336,6 +338,36 @@ void SwDocTest::testSwScanner()
         CPPUNIT_ASSERT_MESSAGE("Should be 5", aDocStat.nWord == 5);
     }
 
+    //See https://bugs.freedesktop.org/show_bug.cgi?id=49629
+    {
+        SwDocStat aDocStat;
+
+        m_pDoc->AppendTxtNode(*aPaM.GetPoint());
+        m_pDoc->InsertString(aPaM, rtl::OUString("Apple"));
+        pTxtNode = aPaM.GetNode()->GetTxtNode();
+        xub_StrLen nPos = aPaM.GetPoint()->nContent.GetIndex();
+        SwFmtFtn aFtn;
+        aFtn.SetNumStr(rtl::OUString("banana"));
+        SwTxtAttr* pTA = pTxtNode->InsertItem(aFtn, nPos, nPos);
+        CPPUNIT_ASSERT(pTA);
+        CPPUNIT_ASSERT(pTxtNode->Len() == 6); //Apple + 0x02
+        pTxtNode->CountWords(aDocStat, 0, pTxtNode->Len());
+        CPPUNIT_ASSERT(aDocStat.nWord == 1);
+        CPPUNIT_ASSERT_MESSAGE("footnote should be expanded", aDocStat.nChar == 11);
+
+        xub_StrLen nNextPos = aPaM.GetPoint()->nContent.GetIndex();
+        CPPUNIT_ASSERT(nNextPos == nPos+1);
+        SwFmtRefMark aRef(rtl::OUString("refmark"));
+        pTA = pTxtNode->InsertItem(aRef, nNextPos, nNextPos);
+        CPPUNIT_ASSERT(pTA);
+
+        aDocStat.Reset();
+        pTxtNode->SetWordCountDirty(true);
+        pTxtNode->CountWords(aDocStat, 0, pTxtNode->Len());
+        CPPUNIT_ASSERT(aDocStat.nWord == 1);
+        CPPUNIT_ASSERT_MESSAGE("refmark anchor should not be counted", aDocStat.nChar == 11);
+    }
+
     //See https://bugs.freedesktop.org/show_bug.cgi?id=46757
     {
         SwDocStat aDocStat;
diff --git a/sw/source/core/txtnode/ndtxt.cxx b/sw/source/core/txtnode/ndtxt.cxx
index dcacdb4f280e..1a710a814d47 100644
--- a/sw/source/core/txtnode/ndtxt.cxx
+++ b/sw/source/core/txtnode/ndtxt.cxx
@@ -3106,22 +3106,45 @@ const ModelToViewHelper::ConversionMap*
     for ( sal_uInt16 i = 0; pSwpHints2 && i < pSwpHints2->Count(); ++i )
     {
         const SwTxtAttr* pAttr = (*pSwpHints2)[i];
-        if ( RES_TXTATR_FIELD == pAttr->Which() )
+        bool bReplace = false;
+        xub_StrLen nFieldPos;
+        rtl::OUString aExpand;
+        switch (pAttr->Which())
+        {
+            case RES_TXTATR_FIELD:
+                bReplace = true;
+                aExpand =
+                    static_cast<SwTxtFld const*>(pAttr)->GetFld().GetFld()
+                        ->ExpandField(true);
+                nFieldPos = *pAttr->GetStart();
+                break;
+            case RES_TXTATR_FTN:
+                {
+                    bReplace = true;
+                    const SwFmtFtn& rFtn = static_cast<SwTxtFtn const*>(pAttr)->GetFtn();
+                    const SwDoc *pDoc = GetDoc();
+                    aExpand = rFtn.GetViewNumStr(*pDoc);
+                    nFieldPos = *pAttr->GetStart();
+                }
+                break;
+            default:
+                if (pAttr->HasDummyChar())
+                {
+                    bReplace = true;
+                    nFieldPos = *pAttr->GetStart();
+                }
+                break;
+        }
+
+        if (bReplace)
         {
-            const XubString aExpand(
-                static_cast<SwTxtFld const*>(pAttr)->GetFld().GetFld()
-                    ->ExpandField(true));
-            if ( aExpand.Len() > 0 )
-            {
-                const xub_StrLen nFieldPos = *pAttr->GetStart();
-                rRetText = rRetText.replaceAt( nPos + nFieldPos, 1, aExpand );
-                if ( !pConversionMap )
-                    pConversionMap = new ModelToViewHelper::ConversionMap;
-                pConversionMap->push_back(
-                        ModelToViewHelper::ConversionMapEntry(
-                            nFieldPos, nPos + nFieldPos ) );
-                nPos += ( aExpand.Len() - 1 );
-            }
+            rRetText = rRetText.replaceAt( nPos + nFieldPos, 1, aExpand );
+            if ( !pConversionMap )
+                pConversionMap = new ModelToViewHelper::ConversionMap;
+            pConversionMap->push_back(
+                    ModelToViewHelper::ConversionMapEntry(
+                        nFieldPos, nPos + nFieldPos ) );
+            nPos += ( aExpand.getLength() - 1 );
         }
     }