genlang, added genKey function

KID generation is an integrated part of the POT files.

Updated lex files for simplification

Change-Id: I8ba64e7119edc5267b2acd75c468ed2ff1cf16c2

16 files changed, 45 insertions, 32 deletions

diff --git a/l10ntools/Executable_genlang.mk b/l10ntools/Executable_genlang.mk
index 16d49dd776bb..c05f97939c57 100644
--- a/l10ntools/Executable_genlang.mk
+++ b/l10ntools/Executable_genlang.mk
@@ -14,6 +14,11 @@ $(eval $(call gb_Executable_set_include,genlang,\
     $$(INCLUDE) \
 ))
 
+$(eval $(call gb_Executable_use_externals,genlang,\
+    boost_headers \
+))
+
+
 $(eval $(call gb_Executable_add_scanners,genlang,\
     l10ntools/source/gLexPo   \
     l10ntools/source/gLexUi   \
diff --git a/l10ntools/inc/gConv.hxx b/l10ntools/inc/gConv.hxx
index d329a5ac0295..3b5c1d12a120 100644
--- a/l10ntools/inc/gConv.hxx
+++ b/l10ntools/inc/gConv.hxx
@@ -42,7 +42,7 @@ class convert_gen
         virtual void doExecute() = 0;
 
         // utility functions for converters
-        int lexRead(char *sBuf, int nMax_size);
+        void lexRead(char *sBuf, size_t *result, size_t nMax_size);
         static void lexStrncpy(char* s1, const char * s2, int n);
         string& copySource(char const *yyText, bool bDoClear = true);
 
diff --git a/l10ntools/inc/gConvPo.hxx b/l10ntools/inc/gConvPo.hxx
index 5be3657a70bc..53257c2393e0 100644
--- a/l10ntools/inc/gConvPo.hxx
+++ b/l10ntools/inc/gConvPo.hxx
@@ -61,8 +61,8 @@ class convert_po : public convert_gen
         string  msId;
         string  msStr;
         string  msKey;
-        bool         mbFuzzy;
-        filebuf outBuffer;
+        bool    mbFuzzy;
+        filebuf mfOutBuffer;
 
         void doExecute() override;
         string genKeyId(const string& text);
diff --git a/l10ntools/source/gConv.cxx b/l10ntools/source/gConv.cxx
index 1070e88fa565..39f08259ad87 100644
--- a/l10ntools/source/gConv.cxx
+++ b/l10ntools/source/gConv.cxx
@@ -191,29 +191,28 @@ bool convert_gen::prepareFile()
 
 
 
-int convert_gen::lexRead(char *sBuf, int nMax_size)
+void convert_gen::lexRead(char *sBuf, size_t *result, size_t nMax_size)
 {
-    int nResult = 0;
-
     // did we hit eof
-    if (miSourceReadIndex != -1) {
+    if (miSourceReadIndex == -1)
+        *result = 0;
+    else {
         // assume we can copy all that are left.
-        nResult = msSourceBuffer.size() - miSourceReadIndex;
+        *result = msSourceBuffer.size() - miSourceReadIndex;
 
         // space enough for the whole line ?
-        if (nResult <= nMax_size) {
-            msSourceBuffer.copy(sBuf, nResult, miSourceReadIndex);
+        if (*result <= nMax_size) {
+            msSourceBuffer.copy(sBuf, *result, miSourceReadIndex);
             l10nMem::showDebug(sBuf);
             miSourceReadIndex = -1;
         }
         else {
             msSourceBuffer.copy(sBuf, nMax_size, miSourceReadIndex);
             l10nMem::showDebug(sBuf);
-            nResult = nMax_size;
+            *result = nMax_size;
             miSourceReadIndex += nMax_size;
         }
     }
-    return nResult;
 }
 
 
diff --git a/l10ntools/source/gConvPo.cxx b/l10ntools/source/gConvPo.cxx
index 21b18cfa1d79..1483d099aae4 100644
--- a/l10ntools/source/gConvPo.cxx
+++ b/l10ntools/source/gConvPo.cxx
@@ -161,14 +161,14 @@ void convert_po::startSave(const string& sName,
 
     // create directories as needed
     createDir(string(""), sFilePath);
-    outBuffer.open(sFilePath.c_str(), ios::out | ios::binary);
+    mfOutBuffer.open(sFilePath.c_str(), ios::out | ios::binary);
 
-    if (!outBuffer.is_open())
+    if (!mfOutBuffer.is_open())
         throw l10nMem::showError("Cannot open " + sFilePath + " for writing");
 
     l10nMem::showDebug("writing file (" + sFilePath + ")");
 
-    ostream outFile(&outBuffer);
+    ostream outFile(&mfOutBuffer);
 
     // Set header
     auto t = std::time(nullptr);
@@ -204,14 +204,14 @@ void convert_po::save(const string& sFileName,
                       bool               bFuzzy)
 {
     string sName;
-    ostream outFile(&outBuffer);
+    ostream outFile(&mfOutBuffer);
     int newPos;
 
     // isolate filename
     newPos = sFileName.find_last_of("/\\", sFileName.length());
     sName = sFileName.substr(newPos + 1, sFileName.length());
 
-    outFile << endl << "#. " << genKeyId(sName + sText) << endl;
+    outFile << endl << "#. " << genKeyId(sName + sKey + sResource + sENUStext) << endl;
     if (sComment.length())
         outFile << "#. " << sComment << endl;
     outFile << "#: " << sName << endl
@@ -229,15 +229,24 @@ void convert_po::save(const string& sFileName,
 
 void convert_po::endSave()
 {
-    outBuffer.close();
+    mfOutBuffer.close();
 }
 
 
 
 string convert_po::genKeyId(const string& text)
 {
+    string newText(text);
     boost::crc_32_type aCRC32;
-    aCRC32.process_bytes(text.c_str(), text.length());
+    int i;
+
+    for (i = 0; (i = newText.find("\\\\", 0)) != (int)string::npos;) {
+        newText.erase(i, 1);
+    }
+    for (i = 0; (i = newText.find("\\\"", 0)) != (int)string::npos;) {
+        newText.erase(i, 1);
+    }
+    aCRC32.process_bytes(newText.c_str(), newText.length());
     unsigned int nCRC = aCRC32.checksum();
     string key;
 
diff --git a/l10ntools/source/gConvSrc.cxx b/l10ntools/source/gConvSrc.cxx
index e2784851bc01..cc53ea8ea046 100644
--- a/l10ntools/source/gConvSrc.cxx
+++ b/l10ntools/source/gConvSrc.cxx
@@ -152,13 +152,13 @@ void convert_src::setList(char *syyText)
 void convert_src::setNL(char *syyText, bool bMacro)
 {
     int         nL;
-    string sKey;
+    string sKey, x;
 
     copySource(syyText);
 
     if (msTextName.size() && mbValuePresent && mbEnUs) {
         // locate key and extract it
-        buildKey(sKey);
+        buildKey(x);
 
         for (nL = -1;;) {
             nL = msValue.find("\\\"", nL+1);
@@ -231,7 +231,7 @@ void convert_src::setListItem(char const *syyText, bool bIsStart)
             msName = "dummy";
             mcStack.push_back(msName);
         }
-        msTextName         = "item";
+        msTextName    = "item";
         mbExpectValue =
         mbExpectName  =
         mbInListItem  = true;
diff --git a/l10ntools/source/gLexPo.l b/l10ntools/source/gLexPo.l
index af0d2313c1ae..e0f23124a630 100644
--- a/l10ntools/source/gLexPo.l
+++ b/l10ntools/source/gLexPo.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_po *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr potext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexSrc.l b/l10ntools/source/gLexSrc.l
index ef05201f41e3..d55956260c5b 100644
--- a/l10ntools/source/gLexSrc.l
+++ b/l10ntools/source/gLexSrc.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_src *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr srctext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexTree.l b/l10ntools/source/gLexTree.l
index 768ed5b7acc5..ca3d4b63a7c8 100644
--- a/l10ntools/source/gLexTree.l
+++ b/l10ntools/source/gLexTree.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_tree *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr treetext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexUi.l b/l10ntools/source/gLexUi.l
index d7dd86b65222..2ad64b42b868 100644
--- a/l10ntools/source/gLexUi.l
+++ b/l10ntools/source/gLexUi.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_ui *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr uitext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexUlf.l b/l10ntools/source/gLexUlf.l
index 1492666cd8f0..8da3539b700c 100644
--- a/l10ntools/source/gLexUlf.l
+++ b/l10ntools/source/gLexUlf.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_ulf *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr ulftext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexXcs.l b/l10ntools/source/gLexXcs.l
index 98a0983ebf45..f0bb97bafa2b 100644
--- a/l10ntools/source/gLexXcs.l
+++ b/l10ntools/source/gLexXcs.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_xcs *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr xcstext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexXcu.l b/l10ntools/source/gLexXcu.l
index 4f88c9b3c3f7..fb3e1872f319 100644
--- a/l10ntools/source/gLexXcu.l
+++ b/l10ntools/source/gLexXcu.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_xcu *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr xcutext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexXhp.l b/l10ntools/source/gLexXhp.l
index 802b3060d35d..b8577a44dfa4 100644
--- a/l10ntools/source/gLexXhp.l
+++ b/l10ntools/source/gLexXhp.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_xhp *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr xhptext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexXml.l b/l10ntools/source/gLexXml.l
index 357f0bada40f..fd07001f19bb 100644
--- a/l10ntools/source/gLexXml.l
+++ b/l10ntools/source/gLexXml.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_xml *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr xmltext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy 
diff --git a/l10ntools/source/gLexXrm.l b/l10ntools/source/gLexXrm.l
index 8567016a2836..12c1ec9c2648 100644
--- a/l10ntools/source/gLexXrm.l
+++ b/l10ntools/source/gLexXrm.l
@@ -26,7 +26,7 @@ using namespace std;
 
 #define LOCptr ((convert_xrm *)convert_gen::mcImpl)
 #define YYLMAX 64000
-#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, max_size)
+#define YY_INPUT(buf,result,max_size) LOCptr->lexRead(buf, &result, max_size)
 #define YY_NO_UNISTD_H 1
 #define yytext_ptr xrmtext_ptr
 #define yy_flex_strncpy convert_gen::lexStrncpy