summaryrefslogtreecommitdiff
path: root/i18npool
diff options
context:
space:
mode:
authorKhaled Hosny <khaledhosny@eglug.org>2017-01-08 01:02:20 +0200
committerEike Rathke <erack@redhat.com>2017-01-17 10:52:31 +0000
commit278eabab2b5bdc95a51d501fcdb46c216ded3baa (patch)
tree9b018f5644113c1b3fe6bbc86d757966b4aa99b3 /i18npool
parent7fa20da88a4778bdf20d6ca0491fe97ae03ab554 (diff)
tdf#105170: Ignore diacritics on precomposed chars
When ignoring diacritics, the old code just ignore combining marks which works when using decomposed forms (NFD) but does not work for precomposed forms (NFC). Instead, we now decompose, strip marks, then recompose, and use a nice icu::Transliterator that does the hard work for us. As a bonus, we should now handle surrogate pairs fine (most of the time). The new code (in ignoreDiacritics_CTL::folding()) might not be as efficient as the old code that used transliteration_Ignore::folding(), but it is less ugly and more easier to resonate with, or so I hope. Change-Id: If48c8be30527580cdd68f20b40a6533c5f258d83 Reviewed-on: https://gerrit.libreoffice.org/32826 Tested-by: Jenkins <ci@libreoffice.org> Reviewed-by: Eike Rathke <erack@redhat.com>
Diffstat (limited to 'i18npool')
-rw-r--r--i18npool/inc/transliteration_Ignore.hxx19
-rw-r--r--i18npool/source/transliteration/ignoreDiacritics_CTL.cxx79
2 files changed, 87 insertions, 11 deletions
diff --git a/i18npool/inc/transliteration_Ignore.hxx b/i18npool/inc/transliteration_Ignore.hxx
index e6573b6adfaa..3b67d64120e7 100644
--- a/i18npool/inc/transliteration_Ignore.hxx
+++ b/i18npool/inc/transliteration_Ignore.hxx
@@ -21,6 +21,7 @@
#include <transliteration_commonclass.hxx>
#include <i18nutil/oneToOneMapping.hxx>
+#include <unicode/translit.h>
typedef sal_Unicode (*TransFunc)(const sal_Unicode);
@@ -91,9 +92,25 @@ TRANSLITERATION_IGNORE(Space_ja_JP)
TRANSLITERATION_IGNORE(TraditionalKana_ja_JP)
TRANSLITERATION_IGNORE(TraditionalKanji_ja_JP)
TRANSLITERATION_IGNORE(ZiZu_ja_JP)
-TRANSLITERATION_IGNORE(Diacritics_CTL)
TRANSLITERATION_IGNORE(Kashida_CTL)
+class ignoreDiacritics_CTL : public transliteration_Ignore
+{
+ icu::Transliterator* m_transliterator;
+
+public:
+ ignoreDiacritics_CTL();
+
+ OUString SAL_CALL
+ folding(const OUString& rInStr, sal_Int32 nStartPos, sal_Int32 nCount, css::uno::Sequence<sal_Int32>& rOffset)
+ throw(css::uno::RuntimeException, std::exception) override;
+
+ sal_Unicode SAL_CALL
+ transliterateChar2Char(sal_Unicode nInChar)
+ throw(css::uno::RuntimeException,
+ css::i18n::MultipleCharsOutputException, std::exception) override;
+};
+
#undef TRANSLITERATION_IGNORE
#define TRANSLITERATION_IGNORE( name ) \
diff --git a/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx b/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx
index 8d6292c38d0d..c910b94f8a96 100644
--- a/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx
+++ b/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx
@@ -9,25 +9,84 @@
#include <transliteration_Ignore.hxx>
#include <unicode/uchar.h>
+#include <unicode/translit.h>
namespace com { namespace sun { namespace star { namespace i18n {
-sal_Unicode
-ignoreDiacritics_CTL_translator (const sal_Unicode c)
-{
- if(u_getIntPropertyValue(c, UCHAR_GENERAL_CATEGORY) == U_NON_SPACING_MARK)
- return 0xffff; // Skip this character
-
- return c;
-}
-
ignoreDiacritics_CTL::ignoreDiacritics_CTL()
{
- func = ignoreDiacritics_CTL_translator;
+ func = nullptr;
table = nullptr;
map = nullptr;
transliterationName = "ignoreDiacritics_CTL";
implementationName = "com.sun.star.i18n.Transliteration.ignoreDiacritics_CTL";
+
+ UErrorCode nStatus = U_ZERO_ERROR;
+ m_transliterator = icu::Transliterator::createInstance("NFD; [:M:] Remove; NFC",
+ UTRANS_FORWARD, nStatus);
+ if (U_FAILURE(nStatus))
+ m_transliterator = nullptr;
+}
+
+sal_Unicode SAL_CALL
+ignoreDiacritics_CTL::transliterateChar2Char(sal_Unicode nInChar)
+ throw(css::uno::RuntimeException,
+ css::i18n::MultipleCharsOutputException, std::exception)
+{
+ if (!m_transliterator)
+ throw css::uno::RuntimeException();
+
+ UnicodeString aChar(nInChar);
+ m_transliterator->transliterate(aChar);
+
+ if (aChar.isEmpty())
+ return 0xffff; // Skip this character.
+
+ if (aChar.length() > 1)
+ return nInChar; // Don't know what to do here, return the otiginal.
+
+ return aChar[0];
+}
+
+OUString SAL_CALL
+ignoreDiacritics_CTL::folding(const OUString& rInStr, sal_Int32 nStartPos,
+ sal_Int32 nCount, css::uno::Sequence<sal_Int32>& rOffset)
+ throw(css::uno::RuntimeException, std::exception)
+{
+ if (!m_transliterator)
+ throw css::uno::RuntimeException();
+
+ OUString aOutStr;
+
+ sal_Int32 nPosition = nStartPos;
+ sal_Int32 nOffset = 0;
+ if (useOffset)
+ rOffset.realloc(nCount);
+
+ while (nPosition < nStartPos + nCount)
+ {
+ sal_Int32 nIndex = nPosition;
+ UChar32 nChar = rInStr.iterateCodePoints(&nIndex);
+ UnicodeString aChar(nChar);
+ m_transliterator->transliterate(aChar);
+
+ if (useOffset && nOffset + aChar.length() > rOffset.getLength())
+ rOffset.realloc(rOffset.getLength() + aChar.length());
+
+ for (int32_t i = 0; i < aChar.length(); i++)
+ {
+ aOutStr += OUStringLiteral1(aChar[i]);
+ if (useOffset)
+ rOffset[nOffset++] = nPosition;
+ }
+
+ nPosition = nIndex;
+ }
+
+ if (useOffset)
+ rOffset.realloc(aOutStr.getLength());
+
+ return aOutStr;
}
} } } }