mav60: #164341# fix problems with the new implementation

3 files changed, 174 insertions, 24 deletions

diff --git a/xmlsecurity/source/xmlsec/nss/ciphercontext.cxx b/xmlsecurity/source/xmlsec/nss/ciphercontext.cxx
index 64c619d0dfae..9a7cb560ba8a 100644
--- a/xmlsecurity/source/xmlsec/nss/ciphercontext.cxx
+++ b/xmlsecurity/source/xmlsec/nss/ciphercontext.cxx
@@ -27,12 +27,15 @@
 
 #include <precompiled_xmlsecurity.hxx>
 
+#include <osl/time.h>
+#include <rtl/random.h>
 #include <rtl/ref.hxx>
+
 #include "ciphercontext.hxx"
 
 using namespace ::com::sun::star;
 
-uno::Reference< xml::crypto::XCipherContext > OCipherContext::Create( CK_MECHANISM_TYPE nNSSCipherID, const uno::Sequence< ::sal_Int8 >& aKey, const uno::Sequence< ::sal_Int8 >& aInitializationVector, bool bEncryption )
+uno::Reference< xml::crypto::XCipherContext > OCipherContext::Create( CK_MECHANISM_TYPE nNSSCipherID, const uno::Sequence< ::sal_Int8 >& aKey, const uno::Sequence< ::sal_Int8 >& aInitializationVector, bool bEncryption, bool bW3CPadding )
 {
     ::rtl::Reference< OCipherContext > xResult = new OCipherContext;
 
@@ -50,9 +53,12 @@ uno::Reference< xml::crypto::XCipherContext > OCipherContext::Create( CK_MECHANI
                 xResult->m_pContext = PK11_CreateContextBySymKey( nNSSCipherID, bEncryption ? CKA_ENCRYPT : CKA_DECRYPT, xResult->m_pSymKey, xResult->m_pSecParam);
                 if ( xResult->m_pContext )
                 {
-                    xResult->m_bPadding = ( PK11_GetPadMechanism( nNSSCipherID ) == nNSSCipherID );
+                    xResult->m_bEncryption = bEncryption;
+                    xResult->m_bW3CPadding = bW3CPadding;
+                    xResult->m_bPadding = bW3CPadding || ( PK11_GetPadMechanism( nNSSCipherID ) == nNSSCipherID );
                     xResult->m_nBlockSize = PK11_GetBlockSize( nNSSCipherID, xResult->m_pSecParam );
-                    return xResult.get();
+                    if ( xResult->m_nBlockSize <= SAL_MAX_INT8 )
+                        return xResult.get();
                 }
             }
         }
@@ -103,19 +109,73 @@ uno::Sequence< ::sal_Int8 > SAL_CALL OCipherContext::convertWithCipherContext( c
     if ( m_bDisposed )
         throw lang::DisposedException();
 
-    if ( !m_bPadding && aData.getLength() % m_nBlockSize )
-        throw lang::IllegalArgumentException( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "CBC without padding is used, the data should contain complete blocks only." ) ), uno::Reference< uno::XInterface >(), 1 );
+    uno::Sequence< sal_Int8 > aToConvert;
+    if ( aData.getLength() )
+    {
+        sal_Int32 nOldLastBlockLen = m_aLastBlock.getLength();
+        OSL_ENSURE( nOldLastBlockLen <= m_nBlockSize, "Unexpected last block size!" );
+
+        sal_Int32 nAvailableData = nOldLastBlockLen + aData.getLength();
+        sal_Int32 nToConvertLen = nAvailableData;
+        if ( m_bEncryption || !m_bW3CPadding )
+        {
+            if ( nAvailableData % m_nBlockSize == 0 )
+                nToConvertLen = nAvailableData;
+            else if ( nAvailableData < m_nBlockSize )
+                nToConvertLen = 0;
+            else
+                nToConvertLen = nAvailableData - nAvailableData % m_nBlockSize;
+        }
+        else
+        {
+            // decryption with W3C padding needs at least one block for finalizing
+            if ( nAvailableData < m_nBlockSize * 2 )
+                nToConvertLen = 0;
+            else
+                nToConvertLen = nAvailableData - nAvailableData % m_nBlockSize - m_nBlockSize;
+        }
+
+        aToConvert.realloc( nToConvertLen );
+        if ( nToConvertLen == 0 )
+        {
+            m_aLastBlock.realloc( nOldLastBlockLen + aData.getLength() );
+            rtl_copyMemory( m_aLastBlock.getArray() + nOldLastBlockLen, aData.getConstArray(), aData.getLength() );
+            // aToConvert stays empty
+        }
+        else if ( nToConvertLen < nOldLastBlockLen )
+        {
+            rtl_copyMemory( aToConvert.getArray(), m_aLastBlock.getConstArray(), nToConvertLen );
+            rtl_copyMemory( m_aLastBlock.getArray(), m_aLastBlock.getConstArray() + nToConvertLen, nOldLastBlockLen - nToConvertLen );
+            m_aLastBlock.realloc( nOldLastBlockLen - nToConvertLen + aData.getLength() );
+            rtl_copyMemory( m_aLastBlock.getArray() + nOldLastBlockLen - nToConvertLen, aData.getConstArray(), aData.getLength() );
+        }
+        else
+        {
+            rtl_copyMemory( aToConvert.getArray(), m_aLastBlock.getConstArray(), nOldLastBlockLen );
+            if ( nToConvertLen > nOldLastBlockLen )
+                rtl_copyMemory( aToConvert.getArray() + nOldLastBlockLen, aData.getConstArray(), nToConvertLen - nOldLastBlockLen );
+            m_aLastBlock.realloc( nAvailableData - nToConvertLen );
+            rtl_copyMemory( m_aLastBlock.getArray(), aData.getConstArray() + nToConvertLen - nOldLastBlockLen, nAvailableData - nToConvertLen );
+        }
+    }
 
-    int nResultLen = 0;
-    uno::Sequence< sal_Int8 > aResult( aData.getLength() );
-    if ( PK11_CipherOp( m_pContext, reinterpret_cast< unsigned char* >( aResult.getArray() ), &nResultLen, aResult.getLength(), const_cast< unsigned char* >( reinterpret_cast< const unsigned char* >( aData.getConstArray() ) ), aData.getLength() ) != SECSuccess )
+    uno::Sequence< sal_Int8 > aResult;
+    OSL_ENSURE( aToConvert.getLength() % m_nBlockSize == 0, "Unexpected size of the data to encrypt!" );
+    if ( aToConvert.getLength() )
     {
-        m_bBroken = true;
-        Dispose();
-        throw uno::RuntimeException();
+        int nResultLen = 0;
+        aResult.realloc( aToConvert.getLength() + m_nBlockSize );
+        if ( PK11_CipherOp( m_pContext, reinterpret_cast< unsigned char* >( aResult.getArray() ), &nResultLen, aResult.getLength(), const_cast< unsigned char* >( reinterpret_cast< const unsigned char* >( aToConvert.getConstArray() ) ), aToConvert.getLength() ) != SECSuccess )
+        {
+            m_bBroken = true;
+            Dispose();
+            throw uno::RuntimeException();
+        }
+
+        m_nConverted += aToConvert.getLength();
+        aResult.realloc( nResultLen );
     }
 
-    aResult.realloc( nResultLen );
     return aResult;
 }
 
@@ -130,17 +190,87 @@ uno::Sequence< ::sal_Int8 > SAL_CALL OCipherContext::finalizeCipherContextAndDis
     if ( m_bDisposed )
         throw lang::DisposedException();
 
-    unsigned nResultLen = 0;
-    uno::Sequence< sal_Int8 > aResult( m_nBlockSize );
-    if ( PK11_DigestFinal( m_pContext, reinterpret_cast< unsigned char* >( aResult.getArray() ), &nResultLen, aResult.getLength() ) != SECSuccess )
+    OSL_ENSURE( m_nBlockSize <= SAL_MAX_INT8, "Unexpected block size!" );
+    OSL_ENSURE( m_nConverted % m_nBlockSize == 0, "Unexpected amount of bytes is already converted!" );
+    sal_Int32 nSizeForPadding = ( m_nConverted + m_aLastBlock.getLength() ) % m_nBlockSize;
+
+    // if it is decryption, the amount of data should be rounded to the block size even in case of padding
+    if ( ( !m_bPadding || !m_bEncryption ) && nSizeForPadding )
+        throw uno::RuntimeException( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "The data should contain complete blocks only." ) ), uno::Reference< uno::XInterface >() );
+
+    if ( m_bW3CPadding && m_bEncryption )
+    {
+        // in this case the last block should be smaller than standtard block
+        // it will be increased with the padding
+        OSL_ENSURE( m_aLastBlock.getLength() < m_nBlockSize, "Unexpected size of cashed incomplete last block!" );
+
+        // W3CPadding handling for encryption
+        sal_Int32 nPaddingSize = m_nBlockSize - nSizeForPadding;
+        sal_Int32 nOldLastBlockLen = m_aLastBlock.getLength();
+        m_aLastBlock.realloc( nOldLastBlockLen + nPaddingSize );
+
+        if ( nPaddingSize > 1 )
+        {
+            TimeValue aTime;
+            osl_getSystemTime( &aTime );
+            rtlRandomPool aRandomPool = rtl_random_createPool();
+            rtl_random_addBytes( aRandomPool, &aTime, 8 );
+            rtl_random_getBytes( aRandomPool, m_aLastBlock.getArray() + nOldLastBlockLen, nPaddingSize - 1 );
+            rtl_random_destroyPool ( aRandomPool );
+        }
+        m_aLastBlock[m_aLastBlock.getLength() - 1] = nPaddingSize;
+    }
+
+    // finally should the last block be smaller than two standard blocks
+    OSL_ENSURE( m_aLastBlock.getLength() < m_nBlockSize * 2 , "Unexpected size of cashed incomplete last block!" );
+
+    uno::Sequence< sal_Int8 > aResult;
+    if ( m_aLastBlock.getLength() )
+    {
+        int nPrefResLen = 0;
+        aResult.realloc( m_aLastBlock.getLength() + m_nBlockSize );
+        if ( PK11_CipherOp( m_pContext, reinterpret_cast< unsigned char* >( aResult.getArray() ), &nPrefResLen, aResult.getLength(), const_cast< unsigned char* >( reinterpret_cast< const unsigned char* >( m_aLastBlock.getConstArray() ) ), m_aLastBlock.getLength() ) != SECSuccess )
+        {
+            m_bBroken = true;
+            Dispose();
+            throw uno::RuntimeException();
+        }
+
+        aResult.realloc( nPrefResLen );
+        m_aLastBlock.realloc( 0 );
+    }
+
+    sal_Int32 nPrefixLen = aResult.getLength();
+    aResult.realloc( nPrefixLen + m_nBlockSize * 2 );
+    unsigned nFinalLen = 0;
+    if ( PK11_DigestFinal( m_pContext, reinterpret_cast< unsigned char* >( aResult.getArray() + nPrefixLen ), &nFinalLen, aResult.getLength() - nPrefixLen ) != SECSuccess )
     {
         m_bBroken = true;
         Dispose();
         throw uno::RuntimeException();
     }
 
-    aResult.realloc( nResultLen );
-    return aResult;
+    aResult.realloc( nPrefixLen + nFinalLen );
+
+    if ( m_bW3CPadding && !m_bEncryption )
+    {
+        // W3CPadding handling for decryption
+        // aResult should have anough data, since we let m_aLastBlock be big enough in case of decryption
+        OSL_ENSURE( aResult.getLength() >= m_nBlockSize, "Not enough data to handle the padding!" );
 
+        sal_Int8 nBytesToRemove = aResult[aResult.getLength() - 1];
+        if ( nBytesToRemove <= 0 || nBytesToRemove > aResult.getLength() )
+        {
+            m_bBroken = true;
+            Dispose();
+            throw uno::RuntimeException();
+        }
+
+        aResult.realloc( aResult.getLength() - nBytesToRemove );
+    }
+
+    Dispose();
+
+    return aResult;
 }
 
diff --git a/xmlsecurity/source/xmlsec/nss/ciphercontext.hxx b/xmlsecurity/source/xmlsec/nss/ciphercontext.hxx
index 2ad8fc4ba62d..1574a6238bd4 100644
--- a/xmlsecurity/source/xmlsec/nss/ciphercontext.hxx
+++ b/xmlsecurity/source/xmlsec/nss/ciphercontext.hxx
@@ -43,8 +43,14 @@ private:
     PK11SymKey* m_pSymKey;
     SECItem* m_pSecParam;
     PK11Context* m_pContext;
+
     sal_Int32 m_nBlockSize;
+    ::com::sun::star::uno::Sequence< sal_Int8 > m_aLastBlock;
+
+    bool m_bEncryption;
     bool m_bPadding;
+    bool m_bW3CPadding;
+    sal_Int64 m_nConverted;
 
     bool m_bDisposed;
     bool m_bBroken;
@@ -56,8 +62,11 @@ private:
     , m_pSymKey( NULL )
     , m_pSecParam( NULL )
     , m_pContext( NULL )
-    , m_bPadding( false )
     , m_nBlockSize( 0 )
+    , m_bEncryption( false )
+    , m_bPadding( false )
+    , m_bW3CPadding( false )
+    , m_nConverted( 0 )
     , m_bDisposed( false )
     , m_bBroken( false )
     {}
@@ -69,7 +78,7 @@ public:
         Dispose();
     }
 
-    static ::com::sun::star::uno::Reference< ::com::sun::star::xml::crypto::XCipherContext > Create( CK_MECHANISM_TYPE nNSSCipherID, const ::com::sun::star::uno::Sequence< ::sal_Int8 >& aKey, const ::com::sun::star::uno::Sequence< ::sal_Int8 >& aInitializationVector, bool bEncryption );
+    static ::com::sun::star::uno::Reference< ::com::sun::star::xml::crypto::XCipherContext > Create( CK_MECHANISM_TYPE nNSSCipherID, const ::com::sun::star::uno::Sequence< ::sal_Int8 >& aKey, const ::com::sun::star::uno::Sequence< ::sal_Int8 >& aInitializationVector, bool bEncryption, bool bW3CPadding );
 
     // XCipherContext
     virtual ::com::sun::star::uno::Sequence< ::sal_Int8 > SAL_CALL convertWithCipherContext( const ::com::sun::star::uno::Sequence< ::sal_Int8 >& aData ) throw (::com::sun::star::lang::IllegalArgumentException, ::com::sun::star::lang::DisposedException, ::com::sun::star::uno::RuntimeException);
diff --git a/xmlsecurity/source/xmlsec/nss/seinitializer_nssimpl.cxx b/xmlsecurity/source/xmlsec/nss/seinitializer_nssimpl.cxx
index a614fac36a6e..81fe6857bf75 100644
--- a/xmlsecurity/source/xmlsec/nss/seinitializer_nssimpl.cxx
+++ b/xmlsecurity/source/xmlsec/nss/seinitializer_nssimpl.cxx
@@ -127,11 +127,19 @@ struct InitNSSInitialize
         }
 };
 
+struct GetNSSInitStaticMutex
+{
+    ::osl::Mutex* operator()()
+    {
+        static ::osl::Mutex aNSSInitMutex;
+        return &aNSSInitMutex;
+    }
+};
+
 bool * initNSS( const css::uno::Reference< css::lang::XMultiServiceFactory > &xMSF )
 {
-    return rtl_Instance< bool, InitNSSInitialize,
-        ::osl::MutexGuard, ::osl::GetGlobalMutex >::create(
-            InitNSSInitialize( xMSF ), ::osl::GetGlobalMutex());
+    return rtl_Instance< bool, InitNSSInitialize, ::osl::MutexGuard, GetNSSInitStaticMutex >
+                ::create( InitNSSInitialize( xMSF ), GetNSSInitStaticMutex() );
 }
 
 void deleteRootsModule()
@@ -478,9 +486,12 @@ css::uno::Reference< css::xml::crypto::XCipherContext > SAL_CALL SEInitializer_N
     throw (css::lang::IllegalArgumentException, css::uno::RuntimeException)
 {
     CK_MECHANISM_TYPE nNSSCipherID = -1;
-    if ( nCipherID == css::xml::crypto::CipherID::AES_CBC )
+    bool bW3CPadding = false;
+    if ( nCipherID == css::xml::crypto::CipherID::AES_CBC_W3C_PADDING )
     {
         nNSSCipherID = CKM_AES_CBC;
+        bW3CPadding = true;
+
         if ( aKey.getLength() != 16 && aKey.getLength() != 24 && aKey.getLength() != 32 )
             throw css::lang::IllegalArgumentException( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "Unexpected key length." ) ), css::uno::Reference< css::uno::XInterface >(), 2 );
 
@@ -496,7 +507,7 @@ css::uno::Reference< css::xml::crypto::XCipherContext > SAL_CALL SEInitializer_N
         if ( aInitializationVector.getLength() != PK11_GetIVLength( nNSSCipherID ) )
             throw css::lang::IllegalArgumentException( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "Unexpected length of initialization vector." ) ), css::uno::Reference< css::uno::XInterface >(), 3 );
 
-        xResult = OCipherContext::Create( nNSSCipherID, aKey, aInitializationVector, bEncryption );
+        xResult = OCipherContext::Create( nNSSCipherID, aKey, aInitializationVector, bEncryption, bW3CPadding );
     }
 
     return xResult;