Convert filter token generator from Perl to Python.

See tdf#130911 for motivation.

Change-Id: Iad0960d5e6298236dea57e37930dfad1f13628c6
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/90127
Tested-by: Jenkins
Reviewed-by: Michael Stahl <michael.stahl@cib.de>

3 files changed, 66 insertions, 70 deletions

diff --git a/filter/CustomTarget_svg.mk b/filter/CustomTarget_svg.mk
index 2904da9c7e38..065e26e1bb1c 100644
--- a/filter/CustomTarget_svg.mk
+++ b/filter/CustomTarget_svg.mk
@@ -14,7 +14,7 @@ filter_SVGSRC := $(SRCDIR)/filter/source/svg
 filter_SVGWORK := $(call gb_CustomTarget_get_workdir,filter/source/svg)
 
 filter_SRC_svg_Tokens := $(filter_SVGSRC)/tokens.txt
-filter_SRC_svg_GenToken := $(filter_SVGSRC)/gentoken.pl
+filter_SRC_svg_GenToken := $(filter_SVGSRC)/gentoken.py
 filter_SRC_svg_PresentationEngine := $(filter_SVGSRC)/presentation_engine.js
 filter_SRC_svg_Js2Hxx := $(filter_SVGSRC)/js2hxx.py
 
@@ -24,12 +24,15 @@ filter_GEN_svg_Tokens_cxx := $(filter_SVGWORK)/tokens.cxx
 filter_GEN_svg_Script_hxx := $(filter_SVGWORK)/svgscript.hxx
 
 $(filter_GEN_svg_Tokens_gperf) : \
+			$(call gb_ExternalExecutable_get_dependencies,python) \
 			$(filter_SRC_svg_GenToken) $(filter_SRC_svg_Tokens) \
 			| $(filter_SVGWORK)/.dir
 	$(call gb_Output_announce,$(subst $(WORKDIR)/,,$@),build,GPF,3)
 	$(call gb_Helper_abbreviate_dirs, \
-		$(PERL) $(filter_SRC_svg_GenToken) $(filter_SRC_svg_Tokens) \
-			$(filter_GEN_svg_Tokens_hxx) $(filter_GEN_svg_Tokens_gperf))
+		$(call gb_ExternalExecutable_get_command,python) \
+			$(filter_SRC_svg_GenToken) \
+			$(filter_SRC_svg_Tokens) $(filter_GEN_svg_Tokens_hxx) \
+			$(filter_GEN_svg_Tokens_gperf))
 
 # dummy rule: both files generated by recipe above
 $(filter_GEN_svg_Tokens_hxx) : $(filter_GEN_svg_Tokens_gperf)
diff --git a/filter/source/svg/gentoken.pl b/filter/source/svg/gentoken.pl
deleted file mode 100644
index b8c138bbbb78..000000000000
--- a/filter/source/svg/gentoken.pl
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env perl
-#
-#
-# This file is part of the LibreOffice project.
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-
-
-$ARGV0 = shift @ARGV;
-$ARGV1 = shift @ARGV;
-$ARGV2 = shift @ARGV;
-
-open ( TOKENS, $ARGV0 ) || die "can't open token file: $!";
-my %tokens;
-
-while ( defined ($line = <TOKENS>) )
-{
-    if( !($line =~ /^#/) )
-    {
-        chomp($line);
-        @token = split(/\s+/,$line);
-        if ( not defined ($token[1]) )
-        {
-            $token[1] = "XML_".$token[0];
-            $token[1] =~ tr/\-\.\:/___/;
-            $token[1] =~ s/\+/PLUS/g;
-            $token[1] =~ s/\-/MINUS/g;
-        }
-
-        $tokens{$token[0]} = uc($token[1]);
-    }
-}
-close ( TOKENS );
-
-open ( HXX, ">$ARGV1" ) || die "can't open tokens.hxx file: $!";
-open ( GPERF, ">$ARGV2" ) || die "can't open tokens.gperf file: $!";
-
-print ( GPERF "%language=C++\n" );
-print ( GPERF "%global-table\n" );
-print ( GPERF "%null-strings\n" );
-print ( GPERF "%struct-type\n" );
-print ( GPERF "struct xmltoken\n" );
-print ( GPERF "{\n" );
-print ( GPERF "  const char *name; sal_Int32 nToken; \n" );
-print ( GPERF "};\n" );
-print ( GPERF "%%\n" );
-
-print ( HXX "#ifndef INCLUDED_AUTOGEN_TOKEN_HXX\n" );
-print ( HXX "#define INCLUDED_AUTOGEN_TOKEN_HXX\n\n" );
-print ( HXX "#include <sal/types.h>\n\n" );
-
-$i = 0;
-foreach( sort(keys(%tokens)) )
-{
-    $i = $i + 1;
-    print( HXX "const sal_Int32 $tokens{$_} = $i;\n" );
-    print( GPERF "$_,$tokens{$_}\n" );
-}
-print ( GPERF "%%\n" );
-print ( HXX "const sal_Int32 XML_TOKEN_COUNT = $i;\n" );
-print ( HXX "const sal_Int32 XML_TOKEN_INVALID = -1;\n\n" );
-print ( HXX "#endif\n" );
-close ( HXX );
-close ( GPERF );
diff --git a/filter/source/svg/gentoken.py b/filter/source/svg/gentoken.py
new file mode 100644
index 000000000000..c78d066d7c97
--- /dev/null
+++ b/filter/source/svg/gentoken.py
@@ -0,0 +1,60 @@
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+import sys
+
+tokenfile_name = sys.argv[1]
+hxx_name = sys.argv[2]
+gperf_name = sys.argv[3]
+
+gperf_header = r"""%language=C++
+%global-table
+%null-strings
+%struct-type
+struct xmltoken
+{
+  const char *name; sal_Int32 nToken;
+}
+%%
+"""
+
+tokens = {}
+
+with open(tokenfile_name) as tokenfile:
+   for line in tokenfile:
+       line = line.strip()
+       if line:
+           arr = line.split()
+           if len(arr) < 2:
+               t = "XML_" + arr[0]
+               t = t.replace('-', '_').replace('.', '_').replace(':', '_')
+               t = t.replace('+', 'PLUS')
+               arr.append(t)
+           tokens[arr[0]] = arr[1].upper()
+
+hxx = open(hxx_name, 'w')
+gperf = open(gperf_name, 'w')
+
+gperf.write(gperf_header)
+
+hxx.write("#ifndef INCLUDED_AUTOGEN_TOKEN_HXX\n")
+hxx.write("#define INCLUDED_AUTOGEN_TOKEN_HXX\n\n")
+hxx.write("#include <sal/types.h>\n\n" )
+
+i = 0;
+for token in sorted(tokens.keys()):
+    i += 1;
+    hxx.write("const sal_Int32 {} = {};\n".format(tokens[token], i))
+    gperf.write("{},{}\n".format(token, tokens[token]))
+
+gperf.write("%%\n")
+hxx.write("const sal_Int32 XML_TOKEN_COUNT = {};\n".format(i))
+hxx.write("const sal_Int32 XML_TOKEN_INVALID = -1;\n\n")
+hxx.write("#endif\n")
+
+hxx.close()
+gperf.close()