Properly initialize gpgme-w32spawn.exe path on Windows

On Windows, gpgme expects gpgme-w32spawn.exe to be in the same directory as the current
process executable. This assumption might be wrong, e.g., for bundled python, which is
in instdir/program/python-core-x.y.z/bin, while gpgme-w32spawn.exe is in instdir/program.
In this case, if an operation in a python script requires initializing gpgme, it will be
interrupted by a modal warning box telling that gpgme-w32spawn.exe was not found.

If we can't find gpgme-w32spawn.exe in the current executable location, then try to find
the spawn executable, and inform gpgme about actual location using gpgme_set_global_flag.

Change-Id: Ie30a0d4a6666767e8c54f1bdc67b67570d6ea47a
Reviewed-on: https://gerrit.libreoffice.org/71014
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
(cherry picked from commit cca6830288f8d4f541077ead5ee229e02304601d)
Reviewed-on: https://gerrit.libreoffice.org/71036
Tested-by: Mike Kaganski <mike.kaganski@collabora.com>

3 files changed, 66 insertions, 0 deletions

diff --git a/external/gpgmepp/UnpackedTarball_gpgmepp.mk b/external/gpgmepp/UnpackedTarball_gpgmepp.mk
index 1f7496b9b043..c4e52811af54 100644
--- a/external/gpgmepp/UnpackedTarball_gpgmepp.mk
+++ b/external/gpgmepp/UnpackedTarball_gpgmepp.mk
@@ -23,6 +23,7 @@ $(eval $(call gb_UnpackedTarball_add_patches,gpgmepp, \
     $(if $(filter MSC,$(COM)),external/gpgmepp/w32-fix-libtool.patch.1) \
     $(if $(filter MSC,$(COM)),external/gpgmepp/w32-add-initializer.patch.1) \
     external/gpgmepp/w32-build-fixes-2.patch \
+    external/gpgmepp/add-gpgme_set_global_flag-wrapper.patch \
     $(if $(filter LINUX,$(OS)),external/gpgmepp/asan.patch) \
     $(if $(filter LINUX,$(OS)),external/gpgmepp/rpath.patch) \
 ))
diff --git a/external/gpgmepp/add-gpgme_set_global_flag-wrapper.patch b/external/gpgmepp/add-gpgme_set_global_flag-wrapper.patch
new file mode 100644
index 000000000000..fade88a3e2b2
--- /dev/null
+++ b/external/gpgmepp/add-gpgme_set_global_flag-wrapper.patch
@@ -0,0 +1,27 @@
+diff -ur lang/cpp/src/context.cpp.old lang/cpp/src/context.cpp
+--- lang/cpp/src/context.cpp.old	2019-04-20 16:03:09.575605300 +0300
++++ lang/cpp/src/context.cpp	2019-04-20 16:06:43.078419700 +0300
+@@ -1656,6 +1656,11 @@
+     return Error(gpgme_engine_check_version(p));
+ }
+ 
++int GpgME::setGlobalFlag(const char *name, const char *value)
++{
++    return gpgme_set_global_flag(name, value);
++}
++
+ static const unsigned long supported_features = 0
+         | GpgME::ValidatingKeylistModeFeature
+         | GpgME::CancelOperationFeature
+diff -ur lang/cpp/src/global.h lang/cpp/src/global.h
+--- lang/cpp/src/global.h.old	2016-09-07 09:48:44.000000000 +0300
++++ lang/cpp/src/global.h	2019-04-20 16:06:41.918912600 +0300
+@@ -95,6 +95,8 @@
+ GPGMEPP_EXPORT Error checkEngine(Protocol proto);
+ GPGMEPP_EXPORT Error checkEngine(Engine engine);
+ 
++GPGMEPP_EXPORT int setGlobalFlag(const char *name, const char *value);
++
+ GPGMEPP_EXPORT GIOChannel *getGIOChannel(int fd);
+ GPGMEPP_EXPORT QIODevice   *getQIODevice(int fd);
+ 
diff --git a/xmlsecurity/source/gpg/SecurityEnvironment.cxx b/xmlsecurity/source/gpg/SecurityEnvironment.cxx
index 152e8c600019..6ba1bced5cfa 100644
--- a/xmlsecurity/source/gpg/SecurityEnvironment.cxx
+++ b/xmlsecurity/source/gpg/SecurityEnvironment.cxx
@@ -16,6 +16,14 @@
 #include <comphelper/servicehelper.hxx>
 #include <list>
 
+#ifdef _WIN32
+#include <config_folders.h>
+#include <osl/file.hxx>
+#include <osl/process.h>
+#include <rtl/bootstrap.hxx>
+#include <tools/urlobj.hxx>
+#endif
+
 #include <key.h>
 #include <keylistresult.h>
 #include <xmlsec-wrapper.h>
@@ -27,6 +35,36 @@ using namespace css::lang;
 
 SecurityEnvironmentGpg::SecurityEnvironmentGpg()
 {
+#ifdef _WIN32
+    // On Windows, gpgme expects gpgme-w32spawn.exe to be in the same directory as the current
+    // process executable. This assumption might be wrong, e.g., for bundled python, which is
+    // in instdir/program/python-core-x.y.z/bin, while gpgme-w32spawn.exe is in instdir/program.
+    // If we can't find gpgme-w32spawn.exe in the current executable location, then try to find
+    // the spawn executable, and inform gpgme about actual location using gpgme_set_global_flag.
+    static bool bSpawnPathInitialized = [] {
+        auto accessUrl = [](const INetURLObject& url) {
+            osl::File file(url.GetMainURL(INetURLObject::DecodeMechanism::NONE));
+            return file.open(osl_File_OpenFlag_Read) == osl::FileBase::E_None;
+        };
+        OUString sPath;
+        osl_getExecutableFile(&sPath.pData);
+        INetURLObject aPathUrl(sPath);
+        aPathUrl.setName("gpgme-w32spawn.exe");
+        if (!accessUrl(aPathUrl))
+        {
+            sPath = "$BRAND_BASE_DIR/" LIBO_LIBEXEC_FOLDER "/gpgme-w32spawn.exe";
+            rtl::Bootstrap::expandMacros(sPath);
+            aPathUrl.SetURL(sPath);
+            if (accessUrl(aPathUrl))
+            {
+                aPathUrl.removeSegment();
+                GpgME::setGlobalFlag("w32-inst-dir",
+                                     aPathUrl.getFSysPath(FSysStyle::Dos).toUtf8().getStr());
+            }
+        }
+        return true;
+    }();
+#endif
     GpgME::Error err = GpgME::checkEngine(GpgME::OpenPGP);
     if (err)
         throw RuntimeException("The GpgME library failed to initialize for the OpenPGP protocol.");