xmlsecurity PDF NSS verify: handle SHA1_WITH_RSA

SHA1_WITH_RSA is a signing algorithm, not a digest one, but let's
accept it, so LO on Linux can verify a signature generated by LO on
Windows.

It's annoying that equivalent mapping in NSS is not part of their public
API.

Change-Id: I97186fcc1d118f922e5ee3cb472aa5b52bc4b5ca

1 files changed, 15 insertions, 2 deletions

diff --git a/xmlsecurity/source/pdfio/pdfdocument.cxx b/xmlsecurity/source/pdfio/pdfdocument.cxx
index 6822e149c47b..bfd66d240f29 100644
--- a/xmlsecurity/source/pdfio/pdfdocument.cxx
+++ b/xmlsecurity/source/pdfio/pdfdocument.cxx
@@ -1760,7 +1760,20 @@ bool PDFDocument::ValidateSignature(SvStream& rStream, PDFObjectElement* pSignat
     }
 
     SECItem aAlgorithm = NSS_CMSSignedData_GetDigestAlgs(pCMSSignedData)[0]->algorithm;
-    HASH_HashType eHashType = HASH_GetHashTypeByOidTag(SECOID_FindOIDTag(&aAlgorithm));
+    SECOidTag eOidTag = SECOID_FindOIDTag(&aAlgorithm);
+
+    // Map a sign algorithm to a digest algorithm.
+    // See NSS_CMSUtil_MapSignAlgs(), which is private to us.
+    switch (eOidTag)
+    {
+    case SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION:
+        eOidTag = SEC_OID_SHA1;
+        break;
+    default:
+        break;
+    }
+
+    HASH_HashType eHashType = HASH_GetHashTypeByOidTag(eOidTag);
     HASHContext* pHASHContext = HASH_Create(eHashType);
     if (!pHASHContext)
     {
@@ -1796,7 +1809,7 @@ bool PDFDocument::ValidateSignature(SvStream& rStream, PDFObjectElement* pSignat
 
     // Find out what is the expected length of the hash.
     unsigned int nMaxResultLen = 0;
-    switch (SECOID_FindOIDTag(&aAlgorithm))
+    switch (eOidTag)
     {
     case SEC_OID_SHA1:
         nMaxResultLen = msfilter::SHA1_HASH_LENGTH;