tdf#134754: Gracefully handle EINVAL from mmap MAP_JIT on old macOS

(cherry picked from commit 6cab5c9170dc167838f1aebafc47153cd84713b4, with
cca1240fe5884f184af489f5326e96892d1ae975 "Related tdf#134754: Detect failed mmap
on macOS" folded in)

Change-Id: Idfb148fad55c7c6b6e6f4f4b5316fd3b086f7d2e
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/107365
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/107395

1 files changed, 17 insertions, 1 deletions

diff --git a/bridges/source/cpp_uno/shared/vtablefactory.cxx b/bridges/source/cpp_uno/shared/vtablefactory.cxx
index 52309c6ec617..90c414290c1a 100644
--- a/bridges/source/cpp_uno/shared/vtablefactory.cxx
+++ b/bridges/source/cpp_uno/shared/vtablefactory.cxx
@@ -85,7 +85,21 @@ extern "C" void * allocExec(
     p = mmap(
         nullptr, n, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON | MAP_JIT, -1,
         0);
-#else
+    if (p != MAP_FAILED) {
+        goto done;
+    }
+    {
+        auto const e = errno;
+        SAL_INFO("bridges.osx", "mmap failed with " << e);
+        if (e != EINVAL) {
+            p = nullptr;
+            goto done;
+        }
+    }
+    // At least some macOS 10.13 machines are reported to fail the above mmap with EINVAL (see
+    // tdf#134754 "Crash on macOS 10.13 opening local HSQLDB-based odb file in Base on LibreOffice 7
+    // rc1", so in that case retry with the "traditional" approach:
+#endif
     p = mmap(
         nullptr, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1,
         0);
@@ -97,6 +111,8 @@ extern "C" void * allocExec(
         munmap (p, n);
         p = nullptr;
     }
+#if defined MACOSX
+done:
 #endif
 #elif defined _WIN32
     p = VirtualAlloc(nullptr, n, MEM_COMMIT, PAGE_EXECUTE_READWRITE);