fdo#38913: Prevent invalid parameter handler crashes

It appears that on Windows at least some jvm.dll versions can cause calls to
_fileno(NULL), which leads to a call of the invalid parameter handler (see
<http://msdn.microsoft.com/en-us/library/ksazx244%28v=vs.80%29.aspx> "Parameter
Validation: Visual Studio 2005").  The default handler causes the application to
crash, so install a "harmless" one instead.

Change-Id: Id6a3ffb63f70b0c65546bc933e994c8dbf35203c
(cherry picked from commit a82e532ce006c54b2740de74d1da5d11307da7c1)
Reviewed-on: https://gerrit.libreoffice.org/565
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

1 files changed, 29 insertions, 0 deletions

diff --git a/sal/osl/w32/salinit.cxx b/sal/osl/w32/salinit.cxx
index f7ae518dfef4..50fb3ba88789 100644
--- a/sal/osl/w32/salinit.cxx
+++ b/sal/osl/w32/salinit.cxx
@@ -26,6 +26,10 @@
  *
  ************************************************************************/
 
+#include "sal/config.h"
+
+#include <iostream>
+#include <stdlib.h>
 
 #include "system.h"
 #include <osl/process.h>
@@ -35,6 +39,23 @@
 extern "C" {
 #endif
 
+// _set_invalid_parameter_handler appears unavailable with MinGW:
+#if defined _MSC_VER
+namespace {
+
+extern "C" void invalidParameterHandler(
+    wchar_t const * expression, wchar_t const * function, wchar_t const * file,
+    unsigned int line, SAL_UNUSED_PARAMETER uintptr_t)
+{
+    std::wcerr
+        << L"Invalid parameter in \"" << (expression ? expression : L"???")
+        << L"\" (" << (function ? function : L"???") << ") at "
+        << (file ? file : L"???") << L':' << line << std::endl;
+}
+
+}
+#endif
+
 // Prototypes for initialization and deinitialization of SAL library
 
 void SAL_CALL sal_detail_initialize(int argc, char ** argv)
@@ -86,6 +107,14 @@ void SAL_CALL sal_detail_initialize(int argc, char ** argv)
         // How to handle a very unlikely error ???
     }
 
+#if defined _MSC_VER // appears unavailable with MinGW
+    // It appears that at least some jvm.dll versions can cause calls to
+    // _fileno(NULL), which leads to a call of the invalid parameter handler,
+    // and the default handler causes the application to crash, so install a
+    // "harmless" one (cf. fdo#38913):
+    _set_invalid_parameter_handler(&invalidParameterHandler);
+#endif
+
     osl_setCommandArgs(argc, argv);
 }