fix for fdo#62323 bad conversion of Hex strings for certain values

Basic hex literals are basic Integer ( e.g. 4 byte ) types with
-2,147,483,648 through 2,147,483,647 range. Interally the scanner
was using a long to form/scan the literal, this led to bad behaviour
on 64bit linux ( where normally long -> 8 bytes )

(cherry picked from commit 4c9a08e78b6e2c5d19628281bd4141c268299bea)

and squash of
fix for fdo#62323 fix duplicated and wrong tests

Change-Id: I73dc238f7de59367a0a9d00e5421ea6675b4f556
(cherry picked from commit fbf8ae82411d56189f844f9f00ccc6cf6b0827bf)
Reviewed-on: https://gerrit.libreoffice.org/5527
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

5 files changed, 75 insertions, 1 deletions

diff --git a/basic/qa/basic_coverage/test_hexliteral_negIntLimit-2.vb b/basic/qa/basic_coverage/test_hexliteral_negIntLimit-2.vb
new file mode 100644
index 000000000000..c660486a5618
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_negIntLimit-2.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&H80000000)
+  If lngDecimal = -2147483648 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/qa/basic_coverage/test_hexliteral_negIntLimit.vb b/basic/qa/basic_coverage/test_hexliteral_negIntLimit.vb
new file mode 100644
index 000000000000..f33d74cf8dd9
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_negIntLimit.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&HFFFFFFFF)
+  If lngDecimal = -1 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/qa/basic_coverage/test_hexliteral_posIntLimit.vb b/basic/qa/basic_coverage/test_hexliteral_posIntLimit.vb
new file mode 100644
index 000000000000..a03dadc42095
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_posIntLimit.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&H7FFFFFFF)
+  If lngDecimal = 2147483647 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/qa/basic_coverage/test_hexliteral_zeroIntLimit.vb b/basic/qa/basic_coverage/test_hexliteral_zeroIntLimit.vb
new file mode 100644
index 000000000000..1a02bdebfb6f
--- /dev/null
+++ b/basic/qa/basic_coverage/test_hexliteral_zeroIntLimit.vb
@@ -0,0 +1,18 @@
+'
+' This file is part of the LibreOffice project.
+'
+' This Source Code Form is subject to the terms of the Mozilla Public
+' License, v. 2.0. If a copy of the MPL was not distributed with this
+' file, You can obtain one at http://mozilla.org/MPL/2.0/.
+'
+
+
+Function doUnitTest as Integer
+  Dim lngDecimal as Long
+  lngDecimal = Clng(&H0)
+  If lngDecimal = 0 Then
+     doUnitTest = 1
+  Else
+     doUnitTest = 0
+  End If
+End Function
diff --git a/basic/source/comp/scanner.cxx b/basic/source/comp/scanner.cxx
index 87265e7839e2..d2728abc6fca 100644
--- a/basic/source/comp/scanner.cxx
+++ b/basic/source/comp/scanner.cxx
@@ -410,7 +410,9 @@ bool SbiScanner::NextSym()
                 return true;
         }
         bNumber = true;
-        long l = 0;
+        // Hex literals are signed Integers ( as defined by basic
+        // e.g. -2,147,483,648 through 2,147,483,647 (signed)
+        sal_Int32 l = 0;
         int i;
         bool bBufOverflow = false;
         while(nCol < aLine.getLength() &&  theBasicCharClass::get().isAlphaNumeric(aLine[nCol] & 0xFF, bCompatible))