Keep nss types in signature class

If one wants the ability to use a different library for signing and
verification than nss, it is a good idea to not directly expose the NSS
types to the rest of poppler.

9 files changed, 157 insertions, 62 deletions

diff --git a/poppler/Form.cc b/poppler/Form.cc
index 5e885fa1..738bfe30 100644
--- a/poppler/Form.cc
+++ b/poppler/Form.cc
@@ -613,7 +613,7 @@ bool FormWidgetSignature::signDocument(const char *saveFilename, const char *cer
     // calculate a signature over tmp_buffer with the certificate to get its size
     unsigned char tmp_buffer[4];
     memcpy(tmp_buffer, "PDF", 4);
-    SignatureHandler sigHandler(certNickname, SEC_OID_SHA256);
+    SignatureHandler sigHandler(certNickname, HashAlgorithm::Sha256);
     sigHandler.updateHash(tmp_buffer, 4);
     const std::unique_ptr<GooString> tmpSignature = sigHandler.signDetached(password);
     if (!tmpSignature) {
diff --git a/poppler/HashAlgorithm.h b/poppler/HashAlgorithm.h
new file mode 100644
index 00000000..5253a76a
--- /dev/null
+++ b/poppler/HashAlgorithm.h
@@ -0,0 +1,25 @@
+//========================================================================
+//
+// HashAlgorithm.h
+//
+// This file is licensed under the GPLv2 or later
+//
+// Copyright 2023 g10 Code GmbH, Author: Sune Stolborg Vuorela <sune@vuorela.dk>
+//========================================================================
+
+#ifndef HASH_ALGORITHM_H
+#define HASH_ALGORITHM_H
+
+enum class HashAlgorithm
+{
+    Unknown,
+    Md2,
+    Md5,
+    Sha1,
+    Sha256,
+    Sha384,
+    Sha512,
+    Sha224,
+};
+
+#endif // HASH_ALGORITHM_H
diff --git a/poppler/SignatureHandler.cc b/poppler/SignatureHandler.cc
index 89bcc5dd..0ea44e46 100644
--- a/poppler/SignatureHandler.cc
+++ b/poppler/SignatureHandler.cc
@@ -491,16 +491,85 @@ static SECStatus my_NSS_CMSSignerInfo_AddAuthAttr(NSSCMSSignerInfo *signerinfo,
     return my_NSS_CMSAttributeArray_AddAttr(signerinfo->cmsg->poolp, &(signerinfo->authAttr), attr);
 }
 
-unsigned int SignatureHandler::digestLength(SECOidTag digestAlgId)
+static SECOidTag ConvertHashAlgorithmToNss(HashAlgorithm digestAlgId)
 {
     switch (digestAlgId) {
+    case HashAlgorithm::Md2:
+        return SEC_OID_MD2;
+    case HashAlgorithm::Md5:
+        return SEC_OID_MD5;
+    case HashAlgorithm::Sha1:
+        return SEC_OID_SHA1;
+    case HashAlgorithm::Sha256:
+        return SEC_OID_SHA256;
+    case HashAlgorithm::Sha384:
+        return SEC_OID_SHA384;
+    case HashAlgorithm::Sha512:
+        return SEC_OID_SHA512;
+    case HashAlgorithm::Sha224:
+        return SEC_OID_SHA224;
+    case HashAlgorithm::Unknown:
+        return SEC_OID_UNKNOWN;
+    }
+    return SEC_OID_UNKNOWN;
+}
+
+static HashAlgorithm ConvertHashAlgorithmFromNss(SECOidTag digestAlgId)
+{
+    switch (digestAlgId) {
+    case SEC_OID_MD2:
+        return HashAlgorithm::Md2;
+    case SEC_OID_MD5:
+        return HashAlgorithm::Md5;
     case SEC_OID_SHA1:
-        return 20;
+        return HashAlgorithm::Sha1;
     case SEC_OID_SHA256:
-        return 32;
+        return HashAlgorithm::Sha256;
     case SEC_OID_SHA384:
-        return 48;
+        return HashAlgorithm::Sha384;
     case SEC_OID_SHA512:
+        return HashAlgorithm::Sha512;
+    case SEC_OID_SHA224:
+        return HashAlgorithm::Sha224;
+    default:
+        return HashAlgorithm::Unknown;
+    }
+}
+
+static HashAlgorithm ConvertHashTypeFromNss(HASH_HashType type)
+{
+    switch (type) {
+    case HASH_AlgMD2:
+        return HashAlgorithm::Md2;
+    case HASH_AlgMD5:
+        return HashAlgorithm::Md5;
+    case HASH_AlgSHA1:
+        return HashAlgorithm::Sha1;
+    case HASH_AlgSHA256:
+        return HashAlgorithm::Sha256;
+    case HASH_AlgSHA384:
+        return HashAlgorithm::Sha384;
+    case HASH_AlgSHA512:
+        return HashAlgorithm::Sha512;
+    case HASH_AlgSHA224:
+        return HashAlgorithm::Sha224;
+    case HASH_AlgNULL:
+    case HASH_AlgTOTAL:
+        return HashAlgorithm::Unknown;
+    }
+    return HashAlgorithm::Unknown;
+}
+
+unsigned int SignatureHandler::digestLength(HashAlgorithm digestAlgId)
+{
+    switch (digestAlgId) {
+    case HashAlgorithm::Sha1:
+        return 20;
+    case HashAlgorithm::Sha256:
+        return 32;
+    case HashAlgorithm::Sha384:
+        return 48;
+    case HashAlgorithm::Sha512:
         return 64;
     default:
         printf("ERROR: Unrecognized Hash ID\n");
@@ -508,17 +577,17 @@ unsigned int SignatureHandler::digestLength(SECOidTag digestAlgId)
     }
 }
 
-SECOidTag SignatureHandler::getHashOidTag(const char *digestName)
+HashAlgorithm SignatureHandler::getHashOidTag(const char *digestName)
 {
-    SECOidTag tag = SEC_OID_UNKNOWN;
+    HashAlgorithm tag = HashAlgorithm::Unknown;
     if (strcmp(digestName, "SHA1") == 0) {
-        tag = SEC_OID_SHA1;
+        tag = HashAlgorithm::Sha1;
     } else if (strcmp(digestName, "SHA256") == 0) {
-        tag = SEC_OID_SHA256;
+        tag = HashAlgorithm::Sha256;
     } else if (strcmp(digestName, "SHA384") == 0) {
-        tag = SEC_OID_SHA384;
+        tag = HashAlgorithm::Sha384;
     } else if (strcmp(digestName, "SHA512") == 0) {
-        tag = SEC_OID_SHA512;
+        tag = HashAlgorithm::Sha512;
     }
     return tag;
 }
@@ -570,12 +639,12 @@ const char *SignatureHandler::getSignerSubjectDN()
     return signing_cert->subjectName;
 }
 
-HASH_HashType SignatureHandler::getHashAlgorithm()
+HashAlgorithm SignatureHandler::getHashAlgorithm()
 {
     if (hash_context && hash_context->hashobj) {
-        return hash_context->hashobj->type;
+        return ConvertHashTypeFromNss(hash_context->hashobj->type);
     }
-    return HASH_AlgNULL;
+    return HashAlgorithm::Unknown;
 }
 
 time_t SignatureHandler::getSigningTime()
@@ -807,13 +876,13 @@ SignatureHandler::SignatureHandler(unsigned char *p7, int p7_length) : hash_cont
     }
 }
 
-SignatureHandler::SignatureHandler(const char *certNickname, SECOidTag digestAlgTag)
+SignatureHandler::SignatureHandler(const char *certNickname, HashAlgorithm digestAlgTag)
     : hash_length(digestLength(digestAlgTag)), digest_alg_tag(digestAlgTag), CMSitem(), hash_context(nullptr), CMSMessage(nullptr), CMSSignedData(nullptr), CMSSignerInfo(nullptr), signing_cert(nullptr), temp_certs(nullptr)
 {
     setNSSDir({});
     CMSMessage = NSS_CMSMessage_Create(nullptr);
     signing_cert = CERT_FindCertByNickname(CERT_GetDefaultCertDB(), certNickname);
-    hash_context = HASH_Create(HASH_GetHashTypeByOidTag(digestAlgTag));
+    hash_context = HASH_Create(HASH_GetHashTypeByOidTag(ConvertHashAlgorithmToNss(digestAlgTag)));
 }
 
 SignatureHandler::SignatureHandler() : hash_length(), digest_alg_tag(), CMSitem(), hash_context(nullptr), CMSMessage(nullptr), CMSSignedData(nullptr), CMSSignerInfo(nullptr), signing_cert(nullptr), temp_certs(nullptr)
@@ -826,9 +895,10 @@ HASHContext *SignatureHandler::initHashContext()
 {
 
     SECItem usedAlgorithm = NSS_CMSSignedData_GetDigestAlgs(CMSSignedData)[0]->algorithm;
-    hash_length = digestLength(SECOID_FindOIDTag(&usedAlgorithm));
+    auto hashAlgorithm = SECOID_FindOIDTag(&usedAlgorithm);
+    hash_length = digestLength(ConvertHashAlgorithmFromNss(hashAlgorithm));
     HASH_HashType hashType;
-    hashType = HASH_GetHashTypeByOidTag(SECOID_FindOIDTag(&usedAlgorithm));
+    hashType = HASH_GetHashTypeByOidTag(hashAlgorithm);
     return HASH_Create(hashType);
 }
 
@@ -844,7 +914,7 @@ void SignatureHandler::restartHash()
     if (hash_context) {
         HASH_Destroy(hash_context);
     }
-    hash_context = HASH_Create(HASH_GetHashTypeByOidTag(digest_alg_tag));
+    hash_context = HASH_Create(HASH_GetHashTypeByOidTag(ConvertHashAlgorithmToNss(digest_alg_tag)));
 }
 
 SignatureHandler::~SignatureHandler()
diff --git a/poppler/SignatureHandler.h b/poppler/SignatureHandler.h
index 77d5cca8..a391b8a6 100644
--- a/poppler/SignatureHandler.h
+++ b/poppler/SignatureHandler.h
@@ -24,6 +24,7 @@
 #include "SignatureInfo.h"
 #include "CertificateInfo.h"
 #include "poppler_private_export.h"
+#include "HashAlgorithm.h"
 
 #include <vector>
 #include <functional>
@@ -46,12 +47,12 @@ class POPPLER_PRIVATE_EXPORT SignatureHandler
 public:
     explicit SignatureHandler();
     SignatureHandler(unsigned char *p7, int p7_length);
-    SignatureHandler(const char *certNickname, SECOidTag digestAlgTag);
+    SignatureHandler(const char *certNickname, HashAlgorithm digestAlgTag);
     ~SignatureHandler();
     time_t getSigningTime();
     std::string getSignerName();
     const char *getSignerSubjectDN();
-    HASH_HashType getHashAlgorithm();
+    HashAlgorithm getHashAlgorithm();
     void setSignature(unsigned char *, int);
     void updateHash(unsigned char *data_block, int data_len);
     void restartHash();
@@ -62,7 +63,7 @@ public:
     static std::vector<std::unique_ptr<X509CertificateInfo>> getAvailableSigningCertificates();
     std::unique_ptr<GooString> signDetached(const char *password) const;
 
-    static SECOidTag getHashOidTag(const char *digestName);
+    static HashAlgorithm getHashOidTag(const char *digestName);
 
     // Initializes the NSS dir with the custom given directory
     // calling it with an empty string means use the default firefox db, /etc/pki/nssdb, ~/.pki/nssdb
@@ -91,7 +92,7 @@ private:
     SignatureHandler(const SignatureHandler &);
     SignatureHandler &operator=(const SignatureHandler &);
 
-    unsigned int digestLength(SECOidTag digestAlgId);
+    unsigned int digestLength(HashAlgorithm digestAlgId);
     NSSCMSMessage *CMS_MessageCreate(SECItem *cms_item);
     NSSCMSSignedData *CMS_SignedDataCreate(NSSCMSMessage *cms_msg);
     NSSCMSSignerInfo *CMS_SignerInfoCreate(NSSCMSSignedData *cms_sig_data);
@@ -99,7 +100,7 @@ private:
     static void outputCallback(void *arg, const char *buf, unsigned long len);
 
     unsigned int hash_length;
-    SECOidTag digest_alg_tag;
+    HashAlgorithm digest_alg_tag;
     SECItem CMSitem;
     HASHContext *hash_context;
     NSSCMSMessage *CMSMessage;
diff --git a/poppler/SignatureInfo.cc b/poppler/SignatureInfo.cc
index 923860b0..7450c115 100644
--- a/poppler/SignatureInfo.cc
+++ b/poppler/SignatureInfo.cc
@@ -24,12 +24,6 @@
 #include <cstdlib>
 #include <cstring>
 
-#ifdef ENABLE_NSS3
-#    include <hasht.h>
-#else
-static const int HASH_AlgNULL = -1;
-#endif
-
 /* Constructor & Destructor */
 
 SignatureInfo::SignatureInfo()
@@ -39,7 +33,7 @@ SignatureInfo::SignatureInfo()
     cert_info = nullptr;
     signer_name = nullptr;
     subject_dn = nullptr;
-    hash_type = HASH_AlgNULL;
+    hash_type = HashAlgorithm::Unknown;
     signing_time = 0;
     sig_subfilter_supported = false;
 }
@@ -51,7 +45,7 @@ SignatureInfo::SignatureInfo(SignatureValidationStatus sig_val_status, Certifica
     cert_info = nullptr;
     signer_name = nullptr;
     subject_dn = nullptr;
-    hash_type = HASH_AlgNULL;
+    hash_type = HashAlgorithm::Unknown;
     signing_time = 0;
     sig_subfilter_supported = false;
 }
@@ -94,7 +88,7 @@ const GooString &SignatureInfo::getReason() const
     return reason;
 }
 
-int SignatureInfo::getHashAlgorithm() const
+HashAlgorithm SignatureInfo::getHashAlgorithm() const
 {
     return hash_type;
 }
@@ -143,7 +137,7 @@ void SignatureInfo::setReason(const GooString *signingReason)
     reason = GooString(signingReason->toStr());
 }
 
-void SignatureInfo::setHashAlgorithm(int type)
+void SignatureInfo::setHashAlgorithm(HashAlgorithm type)
 {
     hash_type = type;
 }
diff --git a/poppler/SignatureInfo.h b/poppler/SignatureInfo.h
index eb1f0077..0a3b5b4d 100644
--- a/poppler/SignatureInfo.h
+++ b/poppler/SignatureInfo.h
@@ -24,6 +24,7 @@
 
 #include "poppler_private_export.h"
 #include "goo/GooString.h"
+#include "HashAlgorithm.h"
 
 enum SignatureValidationStatus
 {
@@ -66,7 +67,7 @@ public:
     const char *getSubjectDN() const;
     const GooString &getLocation() const;
     const GooString &getReason() const;
-    int getHashAlgorithm() const; // Returns a NSS3 HASH_HashType or -1 if compiled without NSS3
+    HashAlgorithm getHashAlgorithm() const; // Returns the used HashAlgorithm, and unknown if compiled without signature support
     time_t getSigningTime() const;
     bool isSubfilterSupported() const { return sig_subfilter_supported; }
     const X509CertificateInfo *getCertificateInfo() const;
@@ -78,7 +79,7 @@ public:
     void setSubjectDN(const char *);
     void setLocation(const GooString *);
     void setReason(const GooString *);
-    void setHashAlgorithm(int);
+    void setHashAlgorithm(HashAlgorithm);
     void setSigningTime(time_t);
     void setSubFilterSupport(bool isSupported) { sig_subfilter_supported = isSupported; }
     void setCertificateInfo(std::unique_ptr<X509CertificateInfo>);
@@ -91,7 +92,7 @@ private:
     char *subject_dn;
     GooString location;
     GooString reason;
-    int hash_type;
+    HashAlgorithm hash_type;
     time_t signing_time;
     bool sig_subfilter_supported;
 };
diff --git a/qt5/src/poppler-form.cc b/qt5/src/poppler-form.cc
index a4111511..9e17bee3 100644
--- a/qt5/src/poppler-form.cc
+++ b/qt5/src/poppler-form.cc
@@ -780,7 +780,7 @@ bool CertificateInfo::checkPassword(const QString &password) const
 {
 #ifdef ENABLE_NSS3
     Q_D(const CertificateInfo);
-    SignatureHandler sigHandler(d->nick_name.toUtf8().constData(), SEC_OID_SHA256);
+    SignatureHandler sigHandler(d->nick_name.toUtf8().constData(), HashAlgorithm::Sha256);
     unsigned char buffer[5];
     memcpy(buffer, "test", 5);
     sigHandler.updateHash(buffer, 5);
@@ -805,7 +805,7 @@ public:
     QString signer_subject_dn;
     QString location;
     QString reason;
-    int hash_algorithm;
+    HashAlgorithm hash_algorithm;
     time_t signing_time;
     QList<qint64> range_bounds;
     qint64 docLength;
@@ -859,20 +859,22 @@ SignatureValidationInfo::HashAlgorithm SignatureValidationInfo::hashAlgorithm()
     Q_D(const SignatureValidationInfo);
 
     switch (d->hash_algorithm) {
-    case HASH_AlgMD2:
+    case ::HashAlgorithm::Md2:
         return HashAlgorithmMd2;
-    case HASH_AlgMD5:
+    case ::HashAlgorithm::Md5:
         return HashAlgorithmMd5;
-    case HASH_AlgSHA1:
+    case ::HashAlgorithm::Sha1:
         return HashAlgorithmSha1;
-    case HASH_AlgSHA256:
+    case ::HashAlgorithm::Sha256:
         return HashAlgorithmSha256;
-    case HASH_AlgSHA384:
+    case ::HashAlgorithm::Sha384:
         return HashAlgorithmSha384;
-    case HASH_AlgSHA512:
+    case ::HashAlgorithm::Sha512:
         return HashAlgorithmSha512;
-    case HASH_AlgSHA224:
+    case ::HashAlgorithm::Sha224:
         return HashAlgorithmSha224;
+    case ::HashAlgorithm::Unknown:
+        return HashAlgorithmUnknown;
     }
 #endif
     return HashAlgorithmUnknown;
diff --git a/qt6/src/poppler-form.cc b/qt6/src/poppler-form.cc
index 6f264bd0..a3762cc4 100644
--- a/qt6/src/poppler-form.cc
+++ b/qt6/src/poppler-form.cc
@@ -780,7 +780,7 @@ bool CertificateInfo::checkPassword(const QString &password) const
 {
 #ifdef ENABLE_NSS3
     Q_D(const CertificateInfo);
-    SignatureHandler sigHandler(d->nick_name.toUtf8().constData(), SEC_OID_SHA256);
+    SignatureHandler sigHandler(d->nick_name.toUtf8().constData(), HashAlgorithm::Sha256);
     unsigned char buffer[5];
     memcpy(buffer, "test", 5);
     sigHandler.updateHash(buffer, 5);
@@ -805,7 +805,7 @@ public:
     QString signer_subject_dn;
     QString location;
     QString reason;
-    int hash_algorithm;
+    HashAlgorithm hash_algorithm;
     time_t signing_time;
     QList<qint64> range_bounds;
     qint64 docLength;
@@ -859,20 +859,22 @@ SignatureValidationInfo::HashAlgorithm SignatureValidationInfo::hashAlgorithm()
     Q_D(const SignatureValidationInfo);
 
     switch (d->hash_algorithm) {
-    case HASH_AlgMD2:
+    case ::HashAlgorithm::Md2:
         return HashAlgorithmMd2;
-    case HASH_AlgMD5:
+    case ::HashAlgorithm::Md5:
         return HashAlgorithmMd5;
-    case HASH_AlgSHA1:
+    case ::HashAlgorithm::Sha1:
         return HashAlgorithmSha1;
-    case HASH_AlgSHA256:
+    case ::HashAlgorithm::Sha256:
         return HashAlgorithmSha256;
-    case HASH_AlgSHA384:
+    case ::HashAlgorithm::Sha384:
         return HashAlgorithmSha384;
-    case HASH_AlgSHA512:
+    case ::HashAlgorithm::Sha512:
         return HashAlgorithmSha512;
-    case HASH_AlgSHA224:
+    case ::HashAlgorithm::Sha224:
         return HashAlgorithmSha224;
+    case ::HashAlgorithm::Unknown:
+        return HashAlgorithmUnknown;
     }
 #endif
     return HashAlgorithmUnknown;
diff --git a/utils/pdfsig.cc b/utils/pdfsig.cc
index 1e4301fc..9326b74f 100644
--- a/utils/pdfsig.cc
+++ b/utils/pdfsig.cc
@@ -488,25 +488,25 @@ int main(int argc, char *argv[])
         printf("  - Signing Time: %s\n", time_str = getReadableTime(sig_info->getSigningTime()));
         printf("  - Signing Hash Algorithm: ");
         switch (sig_info->getHashAlgorithm()) {
-        case HASH_AlgMD2:
+        case HashAlgorithm::Md2:
             printf("MD2\n");
             break;
-        case HASH_AlgMD5:
+        case HashAlgorithm::Md5:
             printf("MD5\n");
             break;
-        case HASH_AlgSHA1:
+        case HashAlgorithm::Sha1:
             printf("SHA1\n");
             break;
-        case HASH_AlgSHA256:
+        case HashAlgorithm::Sha256:
             printf("SHA-256\n");
             break;
-        case HASH_AlgSHA384:
+        case HashAlgorithm::Sha384:
             printf("SHA-384\n");
             break;
-        case HASH_AlgSHA512:
+        case HashAlgorithm::Sha512:
             printf("SHA-512\n");
             break;
-        case HASH_AlgSHA224:
+        case HashAlgorithm::Sha224:
             printf("SHA-224\n");
             break;
         default: