implement CryptoAPI RC4+SHA1 encryption scheme for xls import

there might be other variants out there in practice, but this
works for default encrypted xls of excel 2013

(cherry picked from commit 1473ce030314027c01c98f513407ed0897328585)

contains...

be more c++y

Change-Id: I673b00e111a734bc626ba3d3f6ecf9692f1ce7db
(cherry picked from commit fcf7f503b64b2cf7dbef019fb43dde033e3538e8)

hash len is 20 for xls cryptoapi configuration

Change-Id: I6e06b8578959b8147043179db57e875b1d98d57d
(cherry picked from commit 491884350ce74f36044b3348bd66356ad1c234af)

clarify hash lengths a bit more

Change-Id: I66cea4b9efb82d29e6c052ac11a6f2eaca823ce0
(cherry picked from commit 8db1b13e721683d0263925c8e0300dd86a022814)

hash len isn't going to change depending on who implements it

Change-Id: Iee585cba4acad74c11d083085153e2af96c8894f
(cherry picked from commit fcc846e8f29839eaace7e1d28746abea8f4b598a)

move some useful header information to mscodec for reuse

Change-Id: Ic7adf3ed3c8279cc93a06975d6fb337210d1af87
(cherry picked from commit fc514cbf30be1613fdf4d4b7c12cbd55ca08b9b0)

rework things in light of now available documentation

Change-Id: If5f75e27191017a8c6a3929e2a70d21840f157c6
(cherry picked from commit 3fabbd0a22219464545f933fc28c869a6fa89546)

split MSCodec_Std97 into a baseclass MSCodec97

(cherry picked from commit 06916c839b16866b47235306d2db50850df0ad7c)

Change-Id: Ia3c41a048169c78684800def94e53fc9f3201e30

Change-Id: I91c0e1d1d95fbd1c68966650e7ac7d23276bcbe3
Reviewed-on: https://gerrit.libreoffice.org/30128
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Michael Stahl <mstahl@redhat.com>

3 files changed, 37 insertions, 66 deletions

diff --git a/oox/source/crypto/CryptTools.cxx b/oox/source/crypto/CryptTools.cxx
index 120487789d89..4bb3ec2f2d28 100644
--- a/oox/source/crypto/CryptTools.cxx
+++ b/oox/source/crypto/CryptTools.cxx
@@ -9,6 +9,7 @@
  */
 
 #include "oox/crypto/CryptTools.hxx"
+#include <filter/msfilter/mscodec.hxx>
 #include <com/sun/star/uno/RuntimeException.hpp>
 
 namespace oox {
@@ -196,15 +197,6 @@ sal_uInt32 Encrypt::update(vector<sal_uInt8>& output, vector<sal_uInt8>& input,
 
 // Digest
 
-#if USE_TLS_OPENSSL
-const sal_uInt32 Digest::DIGEST_LENGTH_SHA1 = SHA_DIGEST_LENGTH;
-const sal_uInt32 Digest::DIGEST_LENGTH_SHA512 = SHA512_DIGEST_LENGTH;
-#endif
-#if USE_TLS_NSS
-const sal_uInt32 Digest::DIGEST_LENGTH_SHA1 = SHA1_LENGTH;
-const sal_uInt32 Digest::DIGEST_LENGTH_SHA512 = SHA512_LENGTH;
-#endif
-
 namespace
 {
 
@@ -275,9 +267,9 @@ sal_uInt32 Digest::getLength()
     switch(meType)
     {
         case SHA1:
-            return DIGEST_LENGTH_SHA1;
+            return msfilter::SHA1_HASH_LENGTH;
         case SHA512:
-            return DIGEST_LENGTH_SHA512;
+            return msfilter::SHA512_HASH_LENGTH;
         default:
             break;
     }
diff --git a/oox/source/crypto/DocumentDecryption.cxx b/oox/source/crypto/DocumentDecryption.cxx
index eea0c629d641..fa51235e7757 100644
--- a/oox/source/crypto/DocumentDecryption.cxx
+++ b/oox/source/crypto/DocumentDecryption.cxx
@@ -241,7 +241,7 @@ bool DocumentDecryption::readAgileEncryptionInfo(Reference< XInputStream >& xInp
         info.cipherAlgorithm == "AES" &&
         info.cipherChaining  == "ChainingModeCBC" &&
         info.hashAlgorithm   == "SHA1" &&
-        info.hashSize        == 20)
+        info.hashSize        == msfilter::SHA1_HASH_LENGTH)
     {
         return true;
     }
@@ -251,7 +251,7 @@ bool DocumentDecryption::readAgileEncryptionInfo(Reference< XInputStream >& xInp
         info.cipherAlgorithm == "AES" &&
         info.cipherChaining  == "ChainingModeCBC" &&
         info.hashAlgorithm   == "SHA512" &&
-        info.hashSize        == 64 )
+        info.hashSize        == msfilter::SHA512_HASH_LENGTH)
     {
         return true;
     }
@@ -263,10 +263,10 @@ bool DocumentDecryption::readStandard2007EncryptionInfo(BinaryInputStream& rStre
 {
     Standard2007Engine* engine = new Standard2007Engine();
     mEngine.reset(engine);
-    StandardEncryptionInfo& info = engine->getInfo();
+    msfilter::StandardEncryptionInfo& info = engine->getInfo();
 
     info.header.flags = rStream.readuInt32();
-    if( getFlag( info.header.flags, ENCRYPTINFO_EXTERNAL ) )
+    if( getFlag( info.header.flags, msfilter::ENCRYPTINFO_EXTERNAL ) )
         return false;
 
     sal_uInt32 nHeaderSize = rStream.readuInt32();
@@ -297,18 +297,18 @@ bool DocumentDecryption::readStandard2007EncryptionInfo(BinaryInputStream& rStre
         return false;
 
     // check flags and algorithm IDs, required are AES128 and SHA-1
-    if( !getFlag( info.header.flags , ENCRYPTINFO_CRYPTOAPI ) )
+    if( !getFlag( info.header.flags, msfilter::ENCRYPTINFO_CRYPTOAPI ) )
         return false;
 
-    if( !getFlag( info.header.flags, ENCRYPTINFO_AES ) )
+    if( !getFlag( info.header.flags, msfilter::ENCRYPTINFO_AES ) )
         return false;
 
     // algorithm ID 0 defaults to AES128 too, if ENCRYPTINFO_AES flag is set
-    if( info.header.algId != 0 && info.header.algId != ENCRYPT_ALGO_AES128 )
+    if( info.header.algId != 0 && info.header.algId != msfilter::ENCRYPT_ALGO_AES128 )
         return false;
 
     // hash algorithm ID 0 defaults to SHA-1 too
-    if( info.header.algIdHash != 0 && info.header.algIdHash != ENCRYPT_HASH_SHA1 )
+    if( info.header.algIdHash != 0 && info.header.algIdHash != msfilter::ENCRYPT_HASH_SHA1 )
         return false;
 
     if( info.verifier.encryptedVerifierHashSize != 20 )
@@ -332,12 +332,12 @@ bool DocumentDecryption::readEncryptionInfo()
 
     switch (aVersion)
     {
-        case VERSION_INFO_2007_FORMAT:
-        case VERSION_INFO_2007_FORMAT_SP2:
+        case msfilter::VERSION_INFO_2007_FORMAT:
+        case msfilter::VERSION_INFO_2007_FORMAT_SP2:
             mCryptoType = STANDARD_2007; // Set encryption info format
             bResult = readStandard2007EncryptionInfo( aBinaryInputStream );
             break;
-        case VERSION_INFO_AGILE:
+        case msfilter::VERSION_INFO_AGILE:
             mCryptoType = AGILE; // Set encryption info format
             aBinaryInputStream.skip(4);
             bResult = readAgileEncryptionInfo( xEncryptionInfo );
diff --git a/oox/source/crypto/Standard2007Engine.cxx b/oox/source/crypto/Standard2007Engine.cxx
index 34fa2265a719..ecf9d8c5dfe8 100644
--- a/oox/source/crypto/Standard2007Engine.cxx
+++ b/oox/source/crypto/Standard2007Engine.cxx
@@ -42,27 +42,6 @@ static const OUString lclCspName = "Microsoft Enhanced RSA and AES Cryptographic
 
 } // namespace
 
-EncryptionStandardHeader::EncryptionStandardHeader()
-{
-    flags        = 0;
-    sizeExtra    = 0;
-    algId        = 0;
-    algIdHash    = 0;
-    keyBits      = 0;
-    providedType = 0;
-    reserved1    = 0;
-    reserved2    = 0;
-}
-
-EncryptionVerifierAES::EncryptionVerifierAES() :
-    saltSize(SALT_LENGTH),
-    encryptedVerifierHashSize(Digest::DIGEST_LENGTH_SHA1)
-{
-    memset(salt, 0, sizeof(salt));
-    memset(encryptedVerifier, 0, sizeof(encryptedVerifier));
-    memset(encryptedVerifierHash, 0, sizeof(encryptedVerifierHash));
-}
-
 Standard2007Engine::Standard2007Engine() :
     CryptoEngine()
 {}
@@ -76,23 +55,23 @@ bool Standard2007Engine::generateVerifier()
     if (mKey.size() != 16)
         return false;
 
-    vector<sal_uInt8> verifier(ENCRYPTED_VERIFIER_LENGTH);
-    vector<sal_uInt8> encryptedVerifier(ENCRYPTED_VERIFIER_LENGTH);
+    vector<sal_uInt8> verifier(msfilter::ENCRYPTED_VERIFIER_LENGTH);
+    vector<sal_uInt8> encryptedVerifier(msfilter::ENCRYPTED_VERIFIER_LENGTH);
 
     lclRandomGenerateValues(&verifier[0], verifier.size());
 
     vector<sal_uInt8> iv;
     Encrypt aEncryptorVerifier(mKey, iv, Crypto::AES_128_ECB);
-    if (aEncryptorVerifier.update(encryptedVerifier, verifier) != ENCRYPTED_VERIFIER_LENGTH)
+    if (aEncryptorVerifier.update(encryptedVerifier, verifier) != msfilter::ENCRYPTED_VERIFIER_LENGTH)
         return false;
     std::copy(encryptedVerifier.begin(), encryptedVerifier.end(), mInfo.verifier.encryptedVerifier);
 
-    vector<sal_uInt8> hash(RTL_DIGEST_LENGTH_SHA1, 0);
-    mInfo.verifier.encryptedVerifierHashSize = RTL_DIGEST_LENGTH_SHA1;
+    vector<sal_uInt8> hash(msfilter::SHA1_HASH_LENGTH, 0);
+    mInfo.verifier.encryptedVerifierHashSize = msfilter::SHA1_HASH_LENGTH;
     Digest::sha1(hash, verifier);
-    hash.resize(ENCRYPTED_VERIFIER_HASH_LENGTH, 0);
+    hash.resize(msfilter::SHA256_HASH_LENGTH, 0);
 
-    vector<sal_uInt8> encryptedHash(ENCRYPTED_VERIFIER_HASH_LENGTH, 0);
+    vector<sal_uInt8> encryptedHash(msfilter::SHA256_HASH_LENGTH, 0);
 
     Encrypt aEncryptorHash(mKey, iv, Crypto::AES_128_ECB);
     aEncryptorHash.update(encryptedHash, hash, hash.size());
@@ -119,13 +98,13 @@ bool Standard2007Engine::calculateEncryptionKey(const OUString& rPassword)
         initialData.begin() + saltSize);
 
     // use "hash" vector for result of sha1 hashing
-    vector<sal_uInt8> hash(Digest::DIGEST_LENGTH_SHA1, 0);
+    vector<sal_uInt8> hash(msfilter::SHA1_HASH_LENGTH, 0);
 
     // calculate SHA1 hash of initialData
     Digest::sha1(hash, initialData);
 
     // data = iterator (4bytes) + hash
-    vector<sal_uInt8> data(Digest::DIGEST_LENGTH_SHA1 + 4, 0);
+    vector<sal_uInt8> data(msfilter::SHA1_HASH_LENGTH + 4, 0);
 
     for (sal_Int32 i = 0; i < 50000; ++i)
     {
@@ -134,7 +113,7 @@ bool Standard2007Engine::calculateEncryptionKey(const OUString& rPassword)
         Digest::sha1(hash, data);
     }
     std::copy(hash.begin(), hash.end(), data.begin() );
-    std::fill(data.begin() + Digest::DIGEST_LENGTH_SHA1, data.end(), 0 );
+    std::fill(data.begin() + msfilter::SHA1_HASH_LENGTH, data.end(), 0 );
 
     Digest::sha1(hash, data);
 
@@ -156,16 +135,16 @@ bool Standard2007Engine::generateEncryptionKey(const OUString& password)
 
     calculateEncryptionKey(password);
 
-    vector<sal_uInt8> encryptedVerifier(ENCRYPTED_VERIFIER_LENGTH);
+    vector<sal_uInt8> encryptedVerifier(msfilter::ENCRYPTED_VERIFIER_LENGTH);
     std::copy(
         mInfo.verifier.encryptedVerifier,
-        mInfo.verifier.encryptedVerifier + ENCRYPTED_VERIFIER_LENGTH,
+        mInfo.verifier.encryptedVerifier + msfilter::ENCRYPTED_VERIFIER_LENGTH,
         encryptedVerifier.begin());
 
-    vector<sal_uInt8> encryptedHash(ENCRYPTED_VERIFIER_HASH_LENGTH);
+    vector<sal_uInt8> encryptedHash(msfilter::SHA256_HASH_LENGTH);
     std::copy(
         mInfo.verifier.encryptedVerifierHash,
-        mInfo.verifier.encryptedVerifierHash + ENCRYPTED_VERIFIER_HASH_LENGTH,
+        mInfo.verifier.encryptedVerifierHash + msfilter::SHA256_HASH_LENGTH,
         encryptedHash.begin());
 
     vector<sal_uInt8> verifier(encryptedVerifier.size(), 0);
@@ -174,7 +153,7 @@ bool Standard2007Engine::generateEncryptionKey(const OUString& password)
     vector<sal_uInt8> verifierHash(encryptedHash.size(), 0);
     Decrypt::aes128ecb(verifierHash, encryptedHash, mKey);
 
-    vector<sal_uInt8> hash(RTL_DIGEST_LENGTH_SHA1, 0);
+    vector<sal_uInt8> hash(msfilter::SHA1_HASH_LENGTH, 0);
     Digest::sha1(hash, verifier);
 
     return std::equal( hash.begin(), hash.end(), verifierHash.begin() );
@@ -204,11 +183,11 @@ bool Standard2007Engine::decrypt(
 
 void Standard2007Engine::writeEncryptionInfo(const OUString& password, BinaryXOutputStream& rStream)
 {
-    mInfo.header.flags        = ENCRYPTINFO_AES | ENCRYPTINFO_CRYPTOAPI;
-    mInfo.header.algId        = ENCRYPT_ALGO_AES128;
-    mInfo.header.algIdHash    = ENCRYPT_HASH_SHA1;
-    mInfo.header.keyBits      = ENCRYPT_KEY_SIZE_AES_128;
-    mInfo.header.providedType = ENCRYPT_PROVIDER_TYPE_AES;
+    mInfo.header.flags        = msfilter::ENCRYPTINFO_AES | msfilter::ENCRYPTINFO_CRYPTOAPI;
+    mInfo.header.algId        = msfilter::ENCRYPT_ALGO_AES128;
+    mInfo.header.algIdHash    = msfilter::ENCRYPT_HASH_SHA1;
+    mInfo.header.keyBits      = msfilter::ENCRYPT_KEY_SIZE_AES_128;
+    mInfo.header.providedType = msfilter::ENCRYPT_PROVIDER_TYPE_AES;
 
     lclRandomGenerateValues(mInfo.verifier.salt, mInfo.verifier.saltSize);
     const sal_Int32 keyLength = mInfo.header.keyBits / 8;
@@ -222,11 +201,11 @@ void Standard2007Engine::writeEncryptionInfo(const OUString& password, BinaryXOu
     if (!generateVerifier())
         return;
 
-    rStream.WriteUInt32(VERSION_INFO_2007_FORMAT);
+    rStream.WriteUInt32(msfilter::VERSION_INFO_2007_FORMAT);
 
     sal_uInt32 cspNameSize = (lclCspName.getLength() * 2) + 2;
 
-    sal_uInt32 encryptionHeaderSize = static_cast<sal_uInt32>(sizeof(EncryptionStandardHeader));
+    sal_uInt32 encryptionHeaderSize = static_cast<sal_uInt32>(sizeof(msfilter::EncryptionStandardHeader));
 
     rStream.WriteUInt32( mInfo.header.flags );
     sal_uInt32 headerSize = encryptionHeaderSize + cspNameSize;
@@ -236,7 +215,7 @@ void Standard2007Engine::writeEncryptionInfo(const OUString& password, BinaryXOu
     rStream.writeUnicodeArray(lclCspName);
     rStream.WriteUInt16(0);
 
-    sal_uInt32 encryptionVerifierSize = static_cast<sal_uInt32>(sizeof(EncryptionVerifierAES));
+    sal_uInt32 encryptionVerifierSize = static_cast<sal_uInt32>(sizeof(msfilter::EncryptionVerifierAES));
     rStream.writeMemory(&mInfo.verifier, encryptionVerifierSize);
 }