diff options
author | Michael Stahl <mstahl@redhat.com> | 2018-01-16 17:47:52 +0100 |
---|---|---|
committer | Michael Stahl <mstahl@redhat.com> | 2018-01-16 22:35:14 +0100 |
commit | ea794efe656d3ab2dd4e414aa023fd2983088e20 (patch) | |
tree | ed3b13014678f42df10d297a8eae366b2636e151 | |
parent | 1929f23143e37cd50fc90425f5bd610827bff745 (diff) |
tdf#114939 sdext: don't use StarOffice MD5 in PDF import
Always use real MD5 here, to avoid interop issues.
Change-Id: Id6f43952bace1654ac761aecc77676d933737d1d
-rw-r--r-- | sdext/source/pdfimport/filterdet.cxx | 17 | ||||
-rw-r--r-- | sdext/source/pdfimport/pdfparse/pdfentries.cxx | 56 |
2 files changed, 35 insertions, 38 deletions
diff --git a/sdext/source/pdfimport/filterdet.cxx b/sdext/source/pdfimport/filterdet.cxx index 99c0f0359fa4..6a0d1b699cb8 100644 --- a/sdext/source/pdfimport/filterdet.cxx +++ b/sdext/source/pdfimport/filterdet.cxx @@ -33,6 +33,7 @@ #include <com/sun/star/io/XSeekable.hpp> #include <com/sun/star/io/TempFile.hpp> #include <comphelper/fileurl.hxx> +#include <comphelper/hash.hxx> #include <cppuhelper/supportsservice.hxx> #include <memory> #include <string.h> @@ -429,16 +430,15 @@ bool checkDocChecksum( const OUString& rInPDFFileURL, } // open file and calculate actual checksum up to index nBytes - sal_uInt8 nActualChecksum[ RTL_DIGEST_LENGTH_MD5 ]; - memset( nActualChecksum, 0, sizeof(nActualChecksum) ); - rtlDigest aActualDigest = rtl_digest_createMD5(); + ::std::vector<unsigned char> nChecksum; + ::comphelper::Hash aDigest(::comphelper::HashType::MD5); oslFileHandle aRead = nullptr; oslFileError aErr = osl_File_E_None; if( (aErr = osl_openFile(rInPDFFileURL.pData, &aRead, osl_File_OpenFlag_Read )) == osl_File_E_None ) { - sal_Int8 aBuf[4096]; + sal_uInt8 aBuf[4096]; sal_uInt32 nCur = 0; sal_uInt64 nBytesRead = 0; while( nCur < nBytes ) @@ -451,15 +451,16 @@ bool checkDocChecksum( const OUString& rInPDFFileURL, } nPass = static_cast<sal_uInt32>(nBytesRead); nCur += nPass; - rtl_digest_updateMD5( aActualDigest, aBuf, nPass ); + aDigest.update(aBuf, nPass); } - rtl_digest_getMD5( aActualDigest, nActualChecksum, sizeof(nActualChecksum) ); + + nChecksum = aDigest.finalize(); osl_closeFile( aRead ); } - rtl_digest_destroyMD5( aActualDigest ); // compare the contents - return (0 == memcmp( nActualChecksum, nTestChecksum, sizeof( nActualChecksum ) )); + return nChecksum.size() == RTL_DIGEST_LENGTH_MD5 + && (0 == memcmp(nChecksum.data(), nTestChecksum, nChecksum.size())); } uno::Reference< io::XStream > getAdditionalStream( const OUString& rInPDFFileURL, diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx index b973bd40f9eb..d9828d355afa 100644 --- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx +++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx @@ -20,6 +20,8 @@ #include <pdfparse.hxx> +#include <comphelper/hash.hxx> + #include <rtl/strbuf.hxx> #include <rtl/ustring.hxx> #include <rtl/ustrbuf.hxx> @@ -1013,7 +1015,6 @@ struct PDFFileImplData sal_uInt32 m_nPEntry; OString m_aDocID; rtlCipher m_aCipher; - rtlDigest m_aDigest; sal_uInt8 m_aDecryptionKey[ENCRYPTION_KEY_LEN+5]; // maximum handled key length @@ -1024,8 +1025,7 @@ struct PDFFileImplData m_nStandardRevision( 0 ), m_nKeyLength( 0 ), m_nPEntry( 0 ), - m_aCipher( nullptr ), - m_aDigest( nullptr ) + m_aCipher( nullptr ) { memset( m_aOEntry, 0, sizeof( m_aOEntry ) ); memset( m_aUEntry, 0, sizeof( m_aUEntry ) ); @@ -1036,8 +1036,6 @@ struct PDFFileImplData { if( m_aCipher ) rtl_cipher_destroyARCFOUR( m_aCipher ); - if( m_aDigest ) - rtl_digest_destroyMD5( m_aDigest ); } }; } @@ -1073,16 +1071,15 @@ bool PDFFile::decrypt( const sal_uInt8* pInBuffer, sal_uInt32 nLen, sal_uInt8* p m_pData->m_aDecryptionKey[i++] = sal_uInt8(nGeneration&0xff); m_pData->m_aDecryptionKey[i++] = sal_uInt8((nGeneration>>8)&0xff); - sal_uInt8 aSum[ENCRYPTION_KEY_LEN]; - rtl_digest_updateMD5( m_pData->m_aDigest, m_pData->m_aDecryptionKey, i ); - rtl_digest_getMD5( m_pData->m_aDigest, aSum, sizeof( aSum ) ); + ::std::vector<unsigned char> const aSum(::comphelper::Hash::calculateHash( + m_pData->m_aDecryptionKey, i, ::comphelper::HashType::MD5)); if( i > 16 ) i = 16; rtlCipherError aErr = rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode, - aSum, i, + aSum.data(), i, nullptr, 0 ); if( aErr == rtl_Cipher_E_None ) aErr = rtl_cipher_decodeARCFOUR( m_pData->m_aCipher, @@ -1116,32 +1113,32 @@ static sal_uInt32 password_to_key( const OString& rPwd, sal_uInt8* pOutKey, PDFF // encrypt pad string sal_Char aPadPwd[ENCRYPTION_BUF_LEN]; pad_or_truncate_to_32( rPwd, aPadPwd ); - rtl_digest_updateMD5( pData->m_aDigest, aPadPwd, sizeof( aPadPwd ) ); + ::comphelper::Hash aDigest(::comphelper::HashType::MD5); + aDigest.update(reinterpret_cast<unsigned char const*>(aPadPwd), sizeof(aPadPwd)); if( ! bComputeO ) { - rtl_digest_updateMD5( pData->m_aDigest, pData->m_aOEntry, 32 ); + aDigest.update(pData->m_aOEntry, 32); sal_uInt8 aPEntry[4]; aPEntry[0] = static_cast<sal_uInt8>(pData->m_nPEntry & 0xff); aPEntry[1] = static_cast<sal_uInt8>((pData->m_nPEntry >> 8 ) & 0xff); aPEntry[2] = static_cast<sal_uInt8>((pData->m_nPEntry >> 16) & 0xff); aPEntry[3] = static_cast<sal_uInt8>((pData->m_nPEntry >> 24) & 0xff); - rtl_digest_updateMD5( pData->m_aDigest, aPEntry, sizeof(aPEntry) ); - rtl_digest_updateMD5( pData->m_aDigest, pData->m_aDocID.getStr(), pData->m_aDocID.getLength() ); + aDigest.update(aPEntry, sizeof(aPEntry)); + aDigest.update(reinterpret_cast<unsigned char const*>(pData->m_aDocID.getStr()), pData->m_aDocID.getLength()); } - sal_uInt8 nSum[RTL_DIGEST_LENGTH_MD5]; - rtl_digest_getMD5( pData->m_aDigest, nSum, sizeof(nSum) ); + ::std::vector<unsigned char> nSum(aDigest.finalize()); if( pData->m_nStandardRevision == 3 ) { for( int i = 0; i < 50; i++ ) { - rtl_digest_updateMD5( pData->m_aDigest, nSum, sizeof(nSum) ); - rtl_digest_getMD5( pData->m_aDigest, nSum, sizeof(nSum) ); + nSum = ::comphelper::Hash::calculateHash(nSum.data(), nSum.size(), + ::comphelper::HashType::MD5); } } sal_uInt32 nLen = pData->m_nKeyLength; if( nLen > RTL_DIGEST_LENGTH_MD5 ) nLen = RTL_DIGEST_LENGTH_MD5; - memcpy( pOutKey, nSum, nLen ); + memcpy( pOutKey, nSum.data(), nLen ); return nLen; } @@ -1150,13 +1147,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData ) // see PDF reference 1.4 Algorithm 3.6 bool bValid = false; sal_uInt8 aKey[ENCRYPTION_KEY_LEN]; - sal_uInt8 nEncryptedEntry[ENCRYPTION_BUF_LEN]; - memset( nEncryptedEntry, 0, sizeof(nEncryptedEntry) ); sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, pData, false ); // save (at this time potential) decryption key for later use memcpy( pData->m_aDecryptionKey, aKey, nKeyLen ); if( pData->m_nStandardRevision == 2 ) { + sal_uInt8 nEncryptedEntry[ENCRYPTION_BUF_LEN]; + memset( nEncryptedEntry, 0, sizeof(nEncryptedEntry) ); // see PDF reference 1.4 Algorithm 3.4 // encrypt pad string rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, @@ -1169,14 +1166,15 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData ) else if( pData->m_nStandardRevision == 3 ) { // see PDF reference 1.4 Algorithm 3.5 - rtl_digest_updateMD5( pData->m_aDigest, nPadString, sizeof( nPadString ) ); - rtl_digest_updateMD5( pData->m_aDigest, pData->m_aDocID.getStr(), pData->m_aDocID.getLength() ); - rtl_digest_getMD5( pData->m_aDigest, nEncryptedEntry, sizeof(nEncryptedEntry) ); + ::comphelper::Hash aDigest(::comphelper::HashType::MD5); + aDigest.update(nPadString, sizeof(nPadString)); + aDigest.update(reinterpret_cast<unsigned char const*>(pData->m_aDocID.getStr()), pData->m_aDocID.getLength()); + ::std::vector<unsigned char> nEncryptedEntry(aDigest.finalize()); rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, aKey, sizeof(aKey), nullptr, 0 ); rtl_cipher_encodeARCFOUR( pData->m_aCipher, - nEncryptedEntry, 16, - nEncryptedEntry, 16 ); // encrypt in place + nEncryptedEntry.data(), 16, + nEncryptedEntry.data(), 16 ); // encrypt in place for( int i = 1; i <= 19; i++ ) // do it 19 times, start with 1 { sal_uInt8 aTempKey[ENCRYPTION_KEY_LEN]; @@ -1186,10 +1184,10 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData ) rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, aTempKey, sizeof(aTempKey), nullptr, 0 ); rtl_cipher_encodeARCFOUR( pData->m_aCipher, - nEncryptedEntry, 16, - nEncryptedEntry, 16 ); // encrypt in place + nEncryptedEntry.data(), 16, + nEncryptedEntry.data(), 16 ); // encrypt in place } - bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 16 ) == 0); + bValid = (memcmp( nEncryptedEntry.data(), pData->m_aUEntry, 16 ) == 0); } return bValid; } @@ -1214,8 +1212,6 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const if( ! m_pData->m_aCipher ) m_pData->m_aCipher = rtl_cipher_createARCFOUR(rtl_Cipher_ModeStream); - if( ! m_pData->m_aDigest ) - m_pData->m_aDigest = rtl_digest_createMD5(); // first try user password bool bValid = check_user_password( rPwd, m_pData.get() ); |