oox: add Digest class which uses NSS or OpenSSL for digest calc.

Document encryption and decryption uses either NSS or OpenSSL to
calculate digest. Digest class hides the implementation details
between the two implementations. Previously, functions sha1 and
sha512 were used for this, but were less generic.

Change-Id: I60119e2ab9c5c1f4a2b02bc417c3c89c53a63fda

4 files changed, 169 insertions, 83 deletions

diff --git a/include/oox/crypto/CryptTools.hxx b/include/oox/crypto/CryptTools.hxx
index ae2c6f6f4209..dde778d4df8c 100644
--- a/include/oox/crypto/CryptTools.hxx
+++ b/include/oox/crypto/CryptTools.hxx
@@ -28,9 +28,11 @@
 #include <openssl/evp.h>
 #include <openssl/sha.h>
 #endif // USE_TLS_OPENSSL
+
 #if USE_TLS_NSS
 #include <nss.h>
 #include <pk11pub.h>
+#include <sechash.h>
 #endif // USE_TLS_NSS
 
 #include <rtl/digest.h>
@@ -115,12 +117,45 @@ public:
                     sal_uInt32 inputLength = 0);
 };
 
-const sal_uInt32 SHA1_LENGTH    = 20;
-const sal_uInt32 SHA512_LENGTH  = 64;
+class Digest
+{
+public:
+    enum DigestType
+    {
+        UNKNOWN,
+        SHA1,
+        SHA512
+    };
+
+    static const sal_uInt32 DIGEST_LENGTH_SHA1;
+    static const sal_uInt32 DIGEST_LENGTH_SHA512;
+
+private:
+    DigestType meType;
 
-bool sha1( std::vector<sal_uInt8>& output, std::vector<sal_uInt8>& input );
+#if USE_TLS_OPENSSL
+    EVP_MD_CTX* mpContext;
+#endif
+
+#if USE_TLS_NSS
+    HASHContext* mpContext;
+#endif
+
+public:
+    Digest(DigestType eType);
+    virtual ~Digest();
+
+    bool update(std::vector<sal_uInt8>& input);
+    bool finalize(std::vector<sal_uInt8>& digest);
+
+    sal_uInt32 getLength();
+
+    static bool sha1(  std::vector<sal_uInt8>& digest, std::vector<sal_uInt8>& input);
+    static bool sha512(std::vector<sal_uInt8>& digest, std::vector<sal_uInt8>& input);
+};
 
-bool sha512( std::vector<sal_uInt8>& output, std::vector<sal_uInt8>& input );
+bool sha1(  std::vector<sal_uInt8>& digest, std::vector<sal_uInt8>& input);
+bool sha512(std::vector<sal_uInt8>& digest, std::vector<sal_uInt8>& input);
 
 } // namespace core
 } // namespace oox
diff --git a/oox/source/crypto/AgileEngine.cxx b/oox/source/crypto/AgileEngine.cxx
index 86d7025b83d1..df034ca9d13c 100644
--- a/oox/source/crypto/AgileEngine.cxx
+++ b/oox/source/crypto/AgileEngine.cxx
@@ -26,9 +26,9 @@ bool hashCalc( std::vector<sal_uInt8>& output,
                const OUString& algorithm )
 {
     if (algorithm == "SHA1")
-        return sha1(output, input);
+        return Digest::sha1(output, input);
     else if (algorithm == "SHA512")
-        return sha512(output, input);
+        return Digest::sha512(output, input);
     return false;
 }
 
diff --git a/oox/source/crypto/CryptTools.cxx b/oox/source/crypto/CryptTools.cxx
index 60e544908dd6..414c6dbc09b2 100644
--- a/oox/source/crypto/CryptTools.cxx
+++ b/oox/source/crypto/CryptTools.cxx
@@ -186,93 +186,144 @@ sal_uInt32 Encrypt::update(vector<sal_uInt8>& output, vector<sal_uInt8>& input,
     return static_cast<sal_uInt32>(outputLength);
 }
 
-bool sha1(vector<sal_uInt8>& output, vector<sal_uInt8>& input)
-{
-    bool aResult = false;
+// Digest
 
 #if USE_TLS_OPENSSL
-    output.clear();
-    output.resize(SHA_DIGEST_LENGTH, 0);
+const sal_uInt32 Digest::DIGEST_LENGTH_SHA1 = SHA_DIGEST_LENGTH;
+const sal_uInt32 Digest::DIGEST_LENGTH_SHA512 = SHA512_DIGEST_LENGTH;
+#endif
+#if USE_TLS_NSS
+const sal_uInt32 Digest::DIGEST_LENGTH_SHA1 = SHA1_LENGTH;
+const sal_uInt32 Digest::DIGEST_LENGTH_SHA512 = SHA512_LENGTH;
+#endif
 
-    SHA_CTX context;
-    SHA1_Init(&context);
-    SHA1_Update(&context, &input[0], input.size());
-    SHA1_Final(&output[0], &context);
-    aResult = true;
+namespace
+{
+
+#if USE_TLS_OPENSSL
+const EVP_MD* lclOpenSSLgetEngine(Digest::DigestType eType)
+{
+    switch(eType)
+    {
+        case Digest::SHA1:
+            return EVP_sha1();
+        case Digest::SHA512:
+            return EVP_sha512();
+        default:
+            break;
+    }
+    return NULL;
+}
 #endif
 
 #if USE_TLS_NSS
-    output.clear();
-    output.resize(SHA1_LENGTH, 0);
+HASH_HashType lclNSSgetHashType(Digest::DigestType eType)
+{
+    switch(eType)
+    {
+        case Digest::SHA1:
+            return HASH_AlgSHA1;
+        case Digest::SHA512:
+            return HASH_AlgSHA512;
+        default:
+            break;
+    }
+    return HASH_AlgNULL;
+}
+#endif
 
-    // Initialize NSS, database functions are not needed
-    NSS_NoDB_Init(NULL);
-    SECStatus status;
+}
 
-    PK11Context* mContext = PK11_CreateDigestContext(SEC_OID_SHA1);
-    status = PK11_DigestBegin(mContext);
-    if (status != SECSuccess)
-        return false;
+Digest::Digest(DigestType eType) :
+    meType(eType)
+{
+    #if USE_TLS_OPENSSL
+    mpContext = EVP_MD_CTX_create();
+    EVP_DigestInit_ex(mpContext, lclOpenSSLgetEngine(eType), NULL);
+    #endif
 
-    status = PK11_DigestOp(mContext, &input[0], input.size());
-    if (status != SECSuccess)
-        return false;
+    #if USE_TLS_NSS
+    NSS_NoDB_Init(NULL);
+    mpContext = HASH_Create(lclNSSgetHashType(eType));
+    HASH_Begin(mpContext);
+    #endif
+}
 
-    unsigned int outputLength = 0;
+Digest::~Digest()
+{
+    #if USE_TLS_OPENSSL
+    if(mpContext)
+        EVP_MD_CTX_destroy(mpContext);
+    #endif
+
+    #if USE_TLS_NSS
+    if(mpContext)
+        HASH_Destroy(mpContext);
+    #endif
+}
 
-    status = PK11_DigestFinal(mContext, &output[0], &outputLength, SHA1_LENGTH);
-    if (status != SECSuccess || outputLength != SHA1_LENGTH)
-        return false;
+sal_uInt32 Digest::getLength()
+{
+    switch(meType)
+    {
+        case SHA1:
+            return DIGEST_LENGTH_SHA1;
+        case SHA512:
+            return DIGEST_LENGTH_SHA512;
+        default:
+            break;
+    }
+    return 0;
+}
 
-    PK11_DestroyContext(mContext, PR_TRUE);
+bool Digest::update(std::vector<sal_uInt8>& input)
+{
+    #if USE_TLS_OPENSSL
+    EVP_DigestUpdate(mpContext, &input[0], input.size());
+    #endif
+    #if USE_TLS_NSS
+    HASH_Update(mpContext, &input[0], input.size());
+    #endif
+    return true;
+}
 
-    aResult = true;
-#endif
-    return aResult;
+bool Digest::finalize(std::vector<sal_uInt8>& digest)
+{
+    digest.clear();
+    sal_uInt32 digestWrittenLength;
+
+    #if USE_TLS_OPENSSL
+    digest.resize(getLength(), 0);
+    EVP_DigestFinal_ex(mpContext, &digest[0], &digestWrittenLength);
+    #endif
+
+    #if USE_TLS_NSS
+    sal_uInt32 digestLength = getLength();
+    digest.resize(digestLength, 0);
+    HASH_End(mpContext, &digest[0], &digestWrittenLength, digestLength);
+    #endif
+    return true;
 }
 
-bool sha512(vector<sal_uInt8>& output, vector<sal_uInt8>& input)
+bool Digest::sha1(vector<sal_uInt8>& output, vector<sal_uInt8>& input)
 {
     bool aResult = false;
 
-#if USE_TLS_OPENSSL
-    output.clear();
-    output.resize(SHA512_DIGEST_LENGTH, 0);
-
-    SHA512_CTX context;
-    SHA512_Init(&context);
-    SHA512_Update(&context, &input[0], input.size());
-    SHA512_Final(&output[0], &context);
+    Digest aDigest(SHA1);
+    aDigest.update(input);
+    aDigest.finalize(output);
     aResult = true;
-#endif
-
-#if USE_TLS_NSS
-    output.clear();
-    output.resize(SHA512_LENGTH, 0);
-
-    // Initialize NSS, database functions are not needed
-    NSS_NoDB_Init(NULL);
-    SECStatus status;
-
-    PK11Context* mContext = PK11_CreateDigestContext(SEC_OID_SHA512);
-    status = PK11_DigestBegin(mContext);
-    if (status != SECSuccess)
-        return false;
-
-    status = PK11_DigestOp(mContext, &input[0], input.size());
-    if (status != SECSuccess)
-        return false;
-
-    unsigned int outputLength = 0;
-
-    status = PK11_DigestFinal(mContext, &output[0], &outputLength, SHA512_LENGTH);
-    if (status != SECSuccess || outputLength != SHA512_LENGTH)
-        return false;
+    return aResult;
+}
 
-    PK11_DestroyContext(mContext, PR_TRUE);
+bool Digest::sha512(vector<sal_uInt8>& output, vector<sal_uInt8>& input)
+{
+    bool aResult = false;
 
+    Digest aDigest(SHA512);
+    aDigest.update(input);
+    aDigest.finalize(output);
     aResult = true;
-#endif
     return aResult;
 }
 
diff --git a/oox/source/crypto/Standard2007Engine.cxx b/oox/source/crypto/Standard2007Engine.cxx
index 29f1e148615d..3d06cba20591 100644
--- a/oox/source/crypto/Standard2007Engine.cxx
+++ b/oox/source/crypto/Standard2007Engine.cxx
@@ -50,9 +50,9 @@ EncryptionStandardHeader::EncryptionStandardHeader()
     reserved2    = 0;
 }
 
-EncryptionVerifierAES::EncryptionVerifierAES()
-    : saltSize(SALT_LENGTH)
-    , encryptedVerifierHashSize(RTL_DIGEST_LENGTH_SHA1)
+EncryptionVerifierAES::EncryptionVerifierAES() :
+    saltSize(SALT_LENGTH),
+    encryptedVerifierHashSize(Digest::DIGEST_LENGTH_SHA1)
 {
     memset(salt, 0, sizeof(salt));
     memset(encryptedVerifier, 0, sizeof(encryptedVerifier));
@@ -92,7 +92,7 @@ bool Standard2007Engine::generateVerifier()
 
     vector<sal_uInt8> hash(RTL_DIGEST_LENGTH_SHA1, 0);
     mInfo.verifier.encryptedVerifierHashSize = RTL_DIGEST_LENGTH_SHA1;
-    sha1(hash, verifier);
+    Digest::sha1(hash, verifier);
     hash.resize(ENCRYPTED_VERIFIER_HASH_LENGTH, 0);
 
     vector<sal_uInt8> encryptedHash(ENCRYPTED_VERIFIER_HASH_LENGTH, 0);
@@ -122,31 +122,31 @@ bool Standard2007Engine::calculateEncryptionKey(const OUString& rPassword)
         initialData.begin() + saltSize);
 
     // use "hash" vector for result of sha1 hashing
-    vector<sal_uInt8> hash(RTL_DIGEST_LENGTH_SHA1, 0);
+    vector<sal_uInt8> hash(Digest::DIGEST_LENGTH_SHA1, 0);
 
     // calculate SHA1 hash of initialData
-    sha1(hash, initialData);
+    Digest::sha1(hash, initialData);
 
     // data = iterator (4bytes) + hash
-    vector<sal_uInt8> data(RTL_DIGEST_LENGTH_SHA1 + 4, 0);
+    vector<sal_uInt8> data(Digest::DIGEST_LENGTH_SHA1 + 4, 0);
 
     for (sal_Int32 i = 0; i < 50000; ++i)
     {
         ByteOrderConverter::writeLittleEndian( &data[0], i );
         std::copy(hash.begin(), hash.end(), data.begin() + 4);
-        sha1(hash, data);
+        Digest::sha1(hash, data);
     }
     std::copy(hash.begin(), hash.end(), data.begin() );
-    std::fill(data.begin() + RTL_DIGEST_LENGTH_SHA1, data.end(), 0 );
+    std::fill(data.begin() + Digest::DIGEST_LENGTH_SHA1, data.end(), 0 );
 
-    sha1(hash, data);
+    Digest::sha1(hash, data);
 
     // derive key
     vector<sal_uInt8> buffer(64, 0x36);
     for( sal_uInt32 i = 0; i < hash.size(); ++i )
         buffer[i] ^= hash[i];
 
-    sha1(hash, buffer);
+    Digest::sha1(hash, buffer);
     std::copy(hash.begin(), hash.begin() + mKey.size(), mKey.begin());
 
     return true;
@@ -178,7 +178,7 @@ bool Standard2007Engine::generateEncryptionKey(const OUString& password)
     Decrypt::aes128ecb(verifierHash, encryptedHash, mKey);
 
     vector<sal_uInt8> hash(RTL_DIGEST_LENGTH_SHA1, 0);
-    sha1(hash, verifier);
+    Digest::sha1(hash, verifier);
 
     return std::equal( hash.begin(), hash.end(), verifierHash.begin() );
 }