Fix problems with LibreOffice document generation (local and portal)

* Fixed problems with LibreOffice local document generation script
  (mkdocs.sh) by interpreting backslash escapes when generating module
  links using "echo -e" and using double quotes around Markdown
  generated text. The result is placed at docs/ folder.
* Fixed problems with LibreOffice portal document generation script
  (mkdocs_portal.sh and mkonedoc.sh) by doing above fixes and also
  adding code from the local document generation script. The result is
  placed at https://docs.libreoffice.org/
* Only the README.md files are processed, and not README* or readme.txt*
* Fixed redmine #3515 "Remove non-necessary subfolders from
  docs.libreoffice.org" by excluding these 10 folders:
  "autom4te.cache dictionaries docs helpcompiler helpcontent2 include
  instdir lo translations workdir"

Change-Id: Id230872b8583a575bfeb6fc77cc9f96d5982908f
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/113773
Tested-by: Jenkins
Reviewed-by: Thorsten Behrens <thorsten.behrens@allotropia.de>

2 files changed, 42 insertions, 21 deletions

diff --git a/solenv/bin/mkdocs.sh b/solenv/bin/mkdocs.sh
index 0f92ff87e532..4b70787787e8 100755
--- a/solenv/bin/mkdocs.sh
+++ b/solenv/bin/mkdocs.sh
@@ -85,8 +85,7 @@ function proc_text {
 }
 
 function proc_text_markdown {
-  sed -re ' s/\[\[([-_a-zA-Z0-9]+)\]\]/<a href="\1.html">\1<\/a>/g' - \
-  | sed -re ' s/\[git:([^]]+)\]/<a href="https:\/\/cgit.freedesktop.org\/libreoffice\/core\/tree\/\1">\1<\/a>/g'
+  sed -re ' s/\[git:([^]]+)\]/<a href="\.\.\/\1">\1<\/a>/g'
 }
 
 function check_cmd {
@@ -226,10 +225,10 @@ echo "generating index page"
 header "LibreOffice Modules" " " "$BASE_OUTPUT/index.html"
 for module_name in *; do
   if [ -d $module_name ]; then
-    cur_file=$(echo $module_name/README* $module_name/readme.txt*)
+    cur_file=$(echo $module_name/README.md)
     if [ -f "$cur_file" ]; then
       # write index.html entry
-      text="<h2><a href=\"${module_name}.html\">${module_name}</a></h2>\n"
+      text=$(echo -e "<h2><a href=\"${module_name}.html\">${module_name}</a></h2>\n")
 
       if [ ${cur_file: -3} == ".md" ]; then
         # This is a markdown file.
@@ -239,7 +238,7 @@ for module_name in *; do
       else
         text="${text}$(head -n1 $cur_file | proc_text)"
       fi
-      echo -e $text >> "$BASE_OUTPUT/index.html"
+      echo -e "$text" >> "$BASE_OUTPUT/index.html"
 
       # write detailed module content
       header "$module_name" "<a href=\"index.html\">LibreOffice</a> &raquo; ${module_name}" "$BASE_OUTPUT/${module_name}.html"
@@ -249,12 +248,12 @@ for module_name in *; do
         text="${text} &nbsp; <a href=\"${module_name}/html/classes.html\">Doxygen</a>"
       fi
       text="${text} </p><p>&nbsp;</p>"
-      echo -e $text >> "$BASE_OUTPUT/${module_name}.html"
+      echo -e "$text" >> "$BASE_OUTPUT/${module_name}.html"
 
       if [ ${cur_file: -3} == ".md" ]; then
         # This is a markdown file.
         text="$(${markdown} $cur_file | proc_text_markdown)"
-        echo $text >> "$BASE_OUTPUT/${module_name}.html"
+        echo -e "$text" >> "$BASE_OUTPUT/${module_name}.html"
       else
         proc_text < $cur_file >> "$BASE_OUTPUT/${module_name}.html"
       fi
@@ -265,9 +264,13 @@ for module_name in *; do
   fi
 done
 
-if [ ${#empty_modules[*]} -gt 0 ]; then
+if [ ${#empty_modules[*]} -gt 10 ]; then
   echo -e "<p>&nbsp;</p><p>READMEs were not available for these modules:</p><ul>\n" >> "$BASE_OUTPUT/index.html"
   for module_name in "${empty_modules[@]}"; do
+    # Do not process these directories
+    if [[ "$module_name" =~ ^(autom4te.cache|dictionaries|docs|helpcompiler|helpcontent2|include|instdir|lo|translations|workdir)$ ]]; then
+      continue
+    fi
     echo -e "<li><a href=\"https://cgit.freedesktop.org/libreoffice/core/tree/${module_name}\">${module_name}</a></li>\n" >> "$BASE_OUTPUT/index.html"
   done
   echo -e "</ul>\n" >> "$BASE_OUTPUT/index.html"
diff --git a/solenv/bin/mkdocs_portal.sh b/solenv/bin/mkdocs_portal.sh
index 852e3448179d..284af9e741f5 100755
--- a/solenv/bin/mkdocs_portal.sh
+++ b/solenv/bin/mkdocs_portal.sh
@@ -4,6 +4,7 @@ if [ -n "$debug" ] ; then
     set -x
 fi
 
+markdown="markdown"
 SRCDIR="$1"
 BASE_OUTPUT="$2"
 
@@ -88,6 +89,10 @@ function proc_text
         | awk 'BEGIN { print "<p>" } { print } END { print "</p>" }'
 }
 
+function proc_text_markdown {
+  sed -re ' s/\[git:([^]]+)\]/<a href="https:\/\/cgit.freedesktop.org\/libreoffice\/core\/tree\/\1">\1<\/a>/g'
+}
+
 # generate entry page
 
 echo "generating index page"
@@ -95,28 +100,37 @@ header "LibreOffice Modules" " " "$BASE_OUTPUT/index.html"
 
 for module_name in *; do
     if [ -d $module_name ]; then
-        cur_file=
-        if [ -f $module_name/readme.txt ] ; then
-            cur_file="$module_name/readme.txt"
-        elif [ -f $module_name/README ] ; then
-            cur_file="$module_name/README"
-        fi
-        if [ -n "$cur_file" ]; then
+        cur_file=$(echo $module_name/README.md)
+        if [ -f "$cur_file" ]; then
             # write index.html entry
-            text="<h2><a href=\"${module_name}.html\">${module_name}</a></h2>\n"
-            text="${text}$(head -n1 $cur_file | proc_text )"
-            echo -e $text >> "$BASE_OUTPUT/index.html"
+            text=$(echo -e "<h2><a href=\"${module_name}.html\">${module_name}</a></h2>\n")
+            if [ ${cur_file: -3} == ".md" ]; then
+                # This is a markdown file.
+                header_text="$(head -n1 $cur_file)"
+                header_text="$(echo ${header_text} | sed -e 's/^\#*//g')"
+                text="${text}${header_text}"
+            else
+                text="${text}$(head -n1 $cur_file | proc_text)"
+            fi
+            echo -e "$text" >> "$BASE_OUTPUT/index.html"
 
             # write detailed module content
             header "$module_name" "<a href=\"index.html\">LibreOffice</a> &raquo; ${module_name}" "$BASE_OUTPUT/${module_name}.html"
             text="<p><b>View module in:</b>"
             text="${text} &nbsp; <a href=\"https://cgit.freedesktop.org/libreoffice/core/tree/${module_name}\">cgit</a>"
-            if [ -d ./docs/${module_name} ] ; then
+            if $(echo $INPUT_PROJECTS | grep -q $module_name); then
                 text="${text} &nbsp; <a href=\"${module_name}/html/classes.html\">Doxygen</a>"
             fi
             text="${text} </p><p>&nbsp;</p>"
-            echo -e $text >> "$BASE_OUTPUT/${module_name}.html"
+            echo -e "$text" >> "$BASE_OUTPUT/${module_name}.html"
+
+        if [ ${cur_file: -3} == ".md" ]; then
+            # This is a markdown file.
+            text="$(${markdown} $cur_file | proc_text_markdown)"
+            echo -e "$text" >> "$BASE_OUTPUT/${module_name}.html"
+        else
             proc_text < $cur_file >> "$BASE_OUTPUT/${module_name}.html"
+        fi
             footer "$BASE_OUTPUT/${module_name}.html"
         else
             empty_modules[${#empty_modules[*]}]=$module_name
@@ -124,9 +138,13 @@ for module_name in *; do
     fi
 done
 
-if [ ${#empty_modules[*]} -gt 0 ]; then
+if [ ${#empty_modules[*]} -gt 10 ]; then
     echo -e "<p>&nbsp;</p><p>READMEs were not available for these modules:</p><ul>\n" >> "$BASE_OUTPUT/index.html"
     for module_name in "${empty_modules[@]}"; do
+        if [[ "$module_name" =~ ^(autom4te.cache|dictionaries|docs|helpcompiler|helpcontent2|include|instdir|lo|translations|workdir)$ ]]
+        then
+            continue
+        fi
         echo -e "<li><a href=\"https://cgit.freedesktop.org/libreoffice/core/tree/${module_name}\">${module_name}</a></li>\n" >> "$BASE_OUTPUT/index.html"
     done
     echo -e "</ul>\n" >> "$BASE_OUTPUT/index.html"