Refactor building perfect hash tabels for tokens

moved the defines to be usable in more modules
moved perl-script to be usable in more modules

Change-Id: If893e4b7e9dc670646bdf097cb6fa5710e45f1ac

2 files changed, 115 insertions, 0 deletions

diff --git a/solenv/bin/generate-tokens.pl b/solenv/bin/generate-tokens.pl
new file mode 100644
index 000000000000..b6391a760358
--- /dev/null
+++ b/solenv/bin/generate-tokens.pl
@@ -0,0 +1,65 @@
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This file incorporates work covered by the following license notice:
+#
+#   Licensed to the Apache Software Foundation (ASF) under one or more
+#   contributor license agreements. See the NOTICE file distributed
+#   with this work for additional information regarding copyright
+#   ownership. The ASF licenses this file to you under the Apache
+#   License, Version 2.0 (the "License"); you may not use this file
+#   except in compliance with the License. You may obtain a copy of
+#   the License at http://www.apache.org/licenses/LICENSE-2.0 .
+#
+
+$ARGV0 = shift @ARGV;
+$ARGV1 = shift @ARGV;
+$ARGV2 = shift @ARGV;
+$ARGV3 = shift @ARGV;
+
+open ( IDFILE, ">$ARGV1" ) or die "Error: cannot open output file: $!";
+open ( NAMEFILE, ">$ARGV2" ) or die "Error: cannot open output file: $!";
+open ( GPERFFILE, ">$ARGV3" ) or die "Error: cannot open output file: $!";
+
+print( GPERFFILE "%language=C++\n" );
+print( GPERFFILE "%global-table\n" );
+print( GPERFFILE "%null-strings\n" );
+print( GPERFFILE "%struct-type\n" );
+print( GPERFFILE "struct xmltoken {\n" );
+print( GPERFFILE "    const sal_Char *name;\n" );
+print( GPERFFILE "    sal_Int32 nToken;\n" );
+print( GPERFFILE "};\n" );
+print( GPERFFILE "%%\n" );
+
+open( INFILE, $ARGV0 ) or die "Error: cannot open input file: $!";
+
+$i = 0;
+while ( <INFILE> )
+{
+    # trim newline
+    chomp( $_ );
+    # trim leading/trailing whitespace
+    $_ =~ s/^\s*//g;
+    $_ =~ s/\s*$//g;
+    # check for valid characters
+    $_ =~ /^[a-zA-Z0-9-_]+$/ or die "Error: invalid character in token '$_'";
+    $id = "XML_$_";
+    $id =~ s/-/_/g;
+    $tokens{$_} = $id;
+    print( IDFILE "const sal_Int32 $id = $i;\n" );
+    print( NAMEFILE "\"$_\",\n" );
+    print( GPERFFILE "$_,$id\n" );
+    ++$i;
+}
+close ( INFILE );
+
+print( IDFILE "const sal_Int32 XML_TOKEN_COUNT = $i;\n" );
+print( GPERFFILE "%%\n" );
+
+close( IDFILE );
+close( NAMEFILE );
+close( GPERFFILE );
diff --git a/solenv/gbuild/CustomTarget.mk b/solenv/gbuild/CustomTarget.mk
index f68d7da18ecf..8f5ef05d4089 100644
--- a/solenv/gbuild/CustomTarget.mk
+++ b/solenv/gbuild/CustomTarget.mk
@@ -76,4 +76,54 @@ $(1) : $(2) $(if $(WITH_LANG),$(call gb_Executable_get_runtime_dependencies,ulfe
 
 endef
 
+#$(call gb_CustomTarget_token_hash,oox/generated,tokenhash.inc,tokenhash.gperf)
+define gb_CustomTarget_token_hash
+$(call gb_CustomTarget_get_target,$(1)) : $(call gb_CustomTarget_get_workdir,$(1))/$(2)
+$(call gb_CustomTarget_get_workdir,$(1))/$(2) : $(call gb_CustomTarget_get_workdir,$(1))/misc/$(3)
+	$$(call gb_Output_announce,$$(subst $(WORKDIR)/,,$$@),build,GPF,1)
+	$(GPERF) --compare-strncmp --switch=2 --readonly-tables $$< \
+		| sed -e 's/char\*)0/(char\*)0, 0/g' | grep -v '^#line' > $$@
+
+endef
+
+#$(call gb_CustomTarget_generate_tokens,oox/generated,oox,oox/source/token,
+#namespaces,namespace,namespaces.txt,namespaces-strict,namespaces.pl)
+define gb_CustomTarget_generate_tokens
+$(call gb_CustomTarget_get_workdir,$(1))/misc/$(5)ids.inc \
+$(call gb_CustomTarget_get_workdir,$(1))/$(5)names.inc \
+$(if $(6),$(call gb_CustomTarget_get_workdir,$(1))/misc/$(6)) \
+$(if $(7),$(call gb_CustomTarget_get_workdir,$(1))/$(7)names.inc) : \
+	$(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx
+	touch $$@
+
+$(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx : \
+	$(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.pl) \
+	$(SRCDIR)/$(3)/$(4).txt \
+	$(SRCDIR)/$(3)/$(4).hxx.head \
+	$(SRCDIR)/$(3)/$(4).hxx.tail
+	$$(call gb_Output_announce,$$(subst $(WORKDIR)/,,$$@),build,PRL,1)
+	mkdir -p $(call gb_CustomTarget_get_workdir,$(1))/misc \
+	    	$(call gb_CustomTarget_get_workdir,$(1)) \
+		$(call gb_CustomTarget_get_workdir,$(1))/$(2)/token
+	perl $(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.pl) \
+	    	$(SRCDIR)/$(3)/$(4).txt \
+		$(call gb_CustomTarget_get_workdir,$(1))/misc/$(5)ids.inc \
+		$(call gb_CustomTarget_get_workdir,$(1))/$(5)names.inc \
+		$(if $(6), $(call gb_CustomTarget_get_workdir,$(1))/misc/$(6)) \
+		$(if $(7), $(SRCDIR)/$(3)/$(7).txt \
+			$(call gb_CustomTarget_get_workdir,$(1))/$(7)names.inc) \
+	&& cat $(SRCDIR)/$(3)/$(4).hxx.head \
+		$(call gb_CustomTarget_get_workdir,$(1))/misc/$(5)ids.inc \
+		$(SRCDIR)/$(3)/$(4).hxx.tail \
+		> $(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx \
+	&& touch $$@
+
+$(call gb_CustomTarget_get_target,$(1)) : \
+    $(call gb_CustomTarget_get_workdir,$(1))/$(5)names.inc \
+    $(if $(7),$(call gb_CustomTarget_get_workdir,$(1))/$(7)names.inc) \
+    $(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx \
+    $(if $(6),$(call gb_CustomTarget_get_workdir,$(1))/misc/$(6)) \
+
+endef
+
 # vim: set noet sw=4: