fdo#80403: TokenHandler impl. FastTokenHandlerBase

getTokenFromUTF8: calls getTokenDirect
getTokenDirect: uses perfect hash

New makefile creates perfect hash table with perl-script gentoken.pl.
I found the script on https://wiki.openoffice.org/wiki/XFastTokenHandler and made some changes.
XMLTokens are defined in tokens.txt

Change-Id: Id04134a896ee082e2d0ba55a715fede19ff04928
Reviewed-on: https://gerrit.libreoffice.org/12335
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>
Tested-by: Michael Meeks <michael.meeks@collabora.com>

7 files changed, 56 insertions, 10 deletions

diff --git a/editeng/CppunitTest_editeng_core.mk b/editeng/CppunitTest_editeng_core.mk
index 488a5d039eba..f66caec8a243 100644
--- a/editeng/CppunitTest_editeng_core.mk
+++ b/editeng/CppunitTest_editeng_core.mk
@@ -27,6 +27,7 @@ $(eval $(call gb_CppunitTest_use_libraries,editeng_core, \
     lng \
     sal \
     salhelper \
+    sax \
     sot \
     svl \
     svt \
diff --git a/editeng/CustomTarget_generated.mk b/editeng/CustomTarget_generated.mk
new file mode 100644
index 000000000000..f3329df5b79b
--- /dev/null
+++ b/editeng/CustomTarget_generated.mk
@@ -0,0 +1,27 @@
+# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+$(eval $(call gb_CustomTarget_CustomTarget,editeng/generated))
+
+SRC := $(SRCDIR)/editeng/source/misc
+PL := $(SRCDIR)/solenv/bin/gentoken.pl
+INC := $(call gb_CustomTarget_get_workdir,editeng/generated)
+
+$(INC)/tokens.hxx $(INC)/tokens.gperf : $(SRC)/tokens.txt $(PL)
+	mkdir -p $(INC)
+	$(PERL) $(PL) $(SRC)/tokens.txt $(INC)/tokens.gperf
+
+$(INC)/tokens.cxx : $(INC)/tokens.gperf
+	$(GPERF) --compare-strncmp --readonly-tables --output-file=$(INC)/tokens.cxx $(INC)/tokens.gperf
+	sed -i -e "s/(char\*)0/(char\*)0, XML_TOKEN_INVALID/g" $(INC)/tokens.cxx
+	sed -i -e "/^#line/d" $(INC)/tokens.cxx
+
+$(call gb_CustomTarget_get_target,editeng/generated) : $(INC)/tokens.cxx
+
+# vim: set noet sw=4 ts=4:
diff --git a/editeng/Library_editeng.mk b/editeng/Library_editeng.mk
index c1cc0940920f..22f4c086085d 100644
--- a/editeng/Library_editeng.mk
+++ b/editeng/Library_editeng.mk
@@ -24,6 +24,8 @@ $(eval $(call gb_Library_set_include,editeng,\
     -I$(SRCDIR)/editeng/inc \
 ))
 
+$(eval $(call gb_Library_use_custom_headers,editeng,editeng/generated))
+
 $(eval $(call gb_Library_set_precompiled_header,editeng,$(SRCDIR)/editeng/inc/pch/precompiled_editeng))
 
 $(eval $(call gb_Library_add_defs,editeng,\
@@ -140,6 +142,7 @@ $(eval $(call gb_Library_use_libraries,editeng,\
     cppu \
     sal \
     salhelper \
+    sax \
     i18nlangtag \
     i18nutil \
 	$(gb_UWINAPI) \
diff --git a/editeng/Module_editeng.mk b/editeng/Module_editeng.mk
index de8840b9c34c..22f36027a565 100644
--- a/editeng/Module_editeng.mk
+++ b/editeng/Module_editeng.mk
@@ -20,6 +20,7 @@
 $(eval $(call gb_Module_Module,editeng))
 
 $(eval $(call gb_Module_add_targets,editeng,\
+        CustomTarget_generated \
 	Library_editeng \
 ))
 
diff --git a/editeng/source/misc/SvXMLAutoCorrectTokenHandler.cxx b/editeng/source/misc/SvXMLAutoCorrectTokenHandler.cxx
index 81982a63ee0c..f669291f53fb 100644
--- a/editeng/source/misc/SvXMLAutoCorrectTokenHandler.cxx
+++ b/editeng/source/misc/SvXMLAutoCorrectTokenHandler.cxx
@@ -10,6 +10,7 @@
 #include <SvXMLAutoCorrectTokenHandler.hxx>
 #include <xmloff/xmltoken.hxx>
 #include <com/sun/star/xml/sax/FastToken.hpp>
+#include <tokens.cxx>
 
 using namespace ::css::uno;
 using namespace ::xmloff::token;
@@ -25,14 +26,7 @@ SvXMLAutoCorrectTokenHandler::~SvXMLAutoCorrectTokenHandler()
 sal_Int32 SAL_CALL SvXMLAutoCorrectTokenHandler::getTokenFromUTF8( const Sequence< sal_Int8 >& Identifier )
      throw (::css::uno::RuntimeException, std::exception)
 {
-    switch( Identifier.getLength() )
-    {
-        case 4:     return XML_NAME;
-        case 5:     return XML_BLOCK;
-        case 10:    return XML_BLOCK_LIST;
-        case 16:    return XML_ABBREVIATED_NAME;
-        default:    return css::xml::sax::FastToken::DONTKNOW;
-    }
+    return getTokenDirect( reinterpret_cast< const char* >( Identifier.getConstArray() ), Identifier.getLength() );
 }
 
 Sequence< sal_Int8 > SAL_CALL SvXMLAutoCorrectTokenHandler::getUTF8Identifier( sal_Int32 )
@@ -41,4 +35,12 @@ Sequence< sal_Int8 > SAL_CALL SvXMLAutoCorrectTokenHandler::getUTF8Identifier( s
     return Sequence< sal_Int8 >();
 }
 
+sal_Int32 SvXMLAutoCorrectTokenHandler::getTokenDirect( const char *pTag, sal_Int32 nLength ) const
+{
+    if( !nLength )
+        nLength = strlen( pTag );
+    const struct xmltoken* pToken = Perfect_Hash::in_word_set( pTag, nLength );
+    return pToken ? pToken->nToken : XML_TOKEN_INVALID;
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/editeng/source/misc/SvXMLAutoCorrectTokenHandler.hxx b/editeng/source/misc/SvXMLAutoCorrectTokenHandler.hxx
index dfe876213835..eaf15d64339d 100644
--- a/editeng/source/misc/SvXMLAutoCorrectTokenHandler.hxx
+++ b/editeng/source/misc/SvXMLAutoCorrectTokenHandler.hxx
@@ -16,6 +16,7 @@
 #include <cppuhelper/implbase1.hxx>
 #include <com/sun/star/xml/sax/XFastTokenHandler.hpp>
 #include <com/sun/star/xml/sax/FastToken.hpp>
+#include <sax/fastattribs.hxx>
 
 using namespace ::css::xml::sax;
 using namespace ::xmloff::token;
@@ -30,10 +31,11 @@ enum SvXMLAutoCorrectToken : sal_Int32
 };
 
 class SvXMLAutoCorrectTokenHandler : public
-        cppu::WeakImplHelper1< css::xml::sax::XFastTokenHandler >
+        cppu::WeakImplHelper1< css::xml::sax::XFastTokenHandler >,
+        public sax_fastparser::FastTokenHandlerBase
 {
 public:
-    SvXMLAutoCorrectTokenHandler();
+    explicit SvXMLAutoCorrectTokenHandler();
     virtual ~SvXMLAutoCorrectTokenHandler();
 
     //XFastTokenHandler
@@ -41,6 +43,9 @@ public:
         throw (css::uno::RuntimeException, std::exception) SAL_OVERRIDE;
     virtual css::uno::Sequence< sal_Int8 > SAL_CALL getUTF8Identifier( sal_Int32 Token )
         throw (css::uno::RuntimeException, std::exception) SAL_OVERRIDE;
+
+    // Much faster direct C++ shortcut to the method that matters
+    virtual sal_Int32 getTokenDirect( const char *pToken, sal_Int32 nLength ) const SAL_OVERRIDE;
 };
 
 #endif // EDITENG_SOURCE_MISC_SVXMLAUTOCORRECTTOKENHANDLER_HXX
diff --git a/editeng/source/misc/tokens.txt b/editeng/source/misc/tokens.txt
new file mode 100644
index 000000000000..0b5a64607b3e
--- /dev/null
+++ b/editeng/source/misc/tokens.txt
@@ -0,0 +1,7 @@
+abbreviated-name
+block
+block-list
+list-name
+name
+package-name
+unformatted-text