Added possibility to explicitly access compressed streams; #361

Author: phax

phase4-lib/src/main/java/com/helger/phase4/attachment/IAS4Attachment.java

@@ -88,6 +88,27 @@ public interface IAS4Attachment
   @Nullable
   IHasInputStream getInputStreamProvider ();
 
+  /**
+   * Get the input stream provider for the attachment data in compressed form - that is the form
+   * over which the digital signature digests are calculated. For incoming messages this is the data
+   * as it was received (after eventual decryption), before the decompression took place. For
+   * outgoing messages this is the data as it is transmitted (before eventual encryption). This is
+   * primarily meant to preserve the compressed payload data for non-repudiation purposes, because
+   * e.g. GZIP compression is not deterministic, and the compressed data cannot be re-created from
+   * the decompressed data. The returned provider is backed by resources (usually temporary files)
+   * that are deleted when the message processing is finished - so the data must be consumed while
+   * the message is processed.
+   *
+   * @return The input stream provider for the compressed attachment data. May be <code>null</code>
+   *         if the attachment is not compressed.
+   * @since 4.5.4
+   */
+  @Nullable
+  default IHasInputStream getCompressedSourceStreamProvider ()
+  {
+    return null;
+  }
+
   /**
    * @return <code>true</code> if the input stream backing this attachment can be read multiple
    *         times, <code>false</code> if not.

phase4-lib/src/main/java/com/helger/phase4/attachment/WSS4JAttachment.java

@@ -80,6 +80,7 @@ public class WSS4JAttachment extends Attachment implements IAS4Attachment
 
   private final AS4ResourceHelper m_aResHelper;
   private IHasInputStream m_aISP;
+  private IHasInputStream m_aCompressedISP;
   private EContentTransferEncoding m_eCTE = EContentTransferEncoding.BINARY;
   private EAS4CompressionMode m_eCompressionMode;
   private Charset m_aCharset;
@@ -186,6 +187,26 @@ public void setSourceStreamProvider (@NonNull final IHasInputStream aISP)
     m_aISP = aISP;
   }
 
+  @Override
+  @Nullable
+  public IHasInputStream getCompressedSourceStreamProvider ()
+  {
+    return m_aCompressedISP;
+  }
+
+  /**
+   * Set the input stream provider for the attachment data in compressed form. See
+   * {@link #getCompressedSourceStreamProvider()} for the semantics.
+   *
+   * @param aISP
+   *        The input stream provider to use. May be <code>null</code>.
+   * @since 4.5.4
+   */
+  public void setCompressedSourceStreamProvider (@Nullable final IHasInputStream aISP)
+  {
+    m_aCompressedISP = aISP;
+  }
+
   @NonNull
   public final EContentTransferEncoding getContentTransferEncoding ()
   {
@@ -305,6 +326,7 @@ public String toString ()
                                        .append ("Headers", getHeaders ())
                                        .append ("ResourceManager", m_aResHelper)
                                        .append ("ISP", m_aISP)
+                                       .append ("CompressedISP", m_aCompressedISP)
                                        .append ("CTE", m_eCTE)
                                        .append ("CM", m_eCompressionMode)
                                        .append ("Charset", m_aCharset)
@@ -433,7 +455,14 @@ public static WSS4JAttachment createOutgoingFileAttachment (@NonNull final File
 
     // Set a stream provider that can be read multiple times (opens a new
     // FileInputStream internally)
-    ret.setSourceStreamProvider (HasInputStream.multiple ( () -> FileHelper.getBufferedInputStream (aRealFile)));
+    final IHasInputStream aISP = HasInputStream.multiple ( () -> FileHelper.getBufferedInputStream (aRealFile));
+    ret.setSourceStreamProvider (aISP);
+    if (eCompressionMode != null)
+    {
+      // Preserve the compressed data for non-repudiation purposes - the
+      // signature digests are calculated over the compressed data
+      ret.setCompressedSourceStreamProvider (aISP);
+    }
     return ret;
   }
 
@@ -494,7 +523,11 @@ public static WSS4JAttachment createOutgoingFileAttachment (final byte @NonNull
             aOS.write (aSrcData);
           }
       }
-      ret.setSourceStreamProvider (HasInputStream.multiple ( () -> FileHelper.getBufferedInputStream (aRealFile)));
+      final IHasInputStream aISP = HasInputStream.multiple ( () -> FileHelper.getBufferedInputStream (aRealFile));
+      ret.setSourceStreamProvider (aISP);
+      // Preserve the compressed data for non-repudiation purposes - the
+      // signature digests are calculated over the compressed data
+      ret.setCompressedSourceStreamProvider (aISP);
     }
     else
     {

phase4-lib/src/main/java/com/helger/phase4/client/AS4ClientUserMessage.java

@@ -716,6 +716,8 @@ public AS4ClientBuiltMessage buildMessage (@NonNull @Nonempty final String sMess
 
     // 1. compress
     // Is done when the attachments are added
+    if (aCallback != null && bAttachmentsPresent)
+      aCallback.onBuiltAttachments (m_aAttachments);
 
     // 2. sign and/or encrypt
     Document aResultSoapDoc = aPureSoapDoc;

phase4-lib/src/main/java/com/helger/phase4/client/IAS4ClientBuildMessageCallback.java

@@ -19,6 +19,9 @@
 import org.jspecify.annotations.NonNull;
 import org.w3c.dom.Document;
 
+import com.helger.annotation.Nonempty;
+import com.helger.collection.commons.ICommonsList;
+import com.helger.phase4.attachment.WSS4JAttachment;
 import com.helger.phase4.messaging.mime.AS4MimeMessage;
 import com.helger.phase4.model.message.AbstractAS4Message;
 
@@ -39,6 +42,24 @@ public interface IAS4ClientBuildMessageCallback
   default void onAS4Message (@NonNull final AbstractAS4Message <?> aMsg)
   {}
 
+  /**
+   * Called for all attachments of a User Message, after eventual compression was applied, but
+   * before signing and encryption took place. This is the way to access the compressed attachment
+   * data over which the digital signature digests are calculated - e.g. to preserve it for
+   * non-repudiation purposes, because GZIP compression is not deterministic, and the compressed
+   * data cannot be re-created from the uncompressed data. Use
+   * {@link com.helger.phase4.attachment.IAS4Attachment#getCompressedSourceStreamProvider()} to
+   * access the compressed data of compressed attachments. The attachment data is backed by
+   * temporary files that are deleted when the sending process is finished, so the data must be
+   * consumed inside this method. Only called for User Messages with at least one attachment.
+   *
+   * @param aAttachments
+   *        The outgoing attachments. Never <code>null</code> nor empty. Must not be modified.
+   * @since 4.5.4
+   */
+  default void onBuiltAttachments (@NonNull @Nonempty final ICommonsList <WSS4JAttachment> aAttachments)
+  {}
+
   /**
    * Called for the unsigned and unencrypted SOAP document. This method is called for all types.
    *

phase4-lib/src/main/java/com/helger/phase4/incoming/AS4IncomingHandler.java

@@ -16,6 +16,7 @@
  */
 package com.helger.phase4.incoming;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -43,6 +44,7 @@
 import com.helger.base.io.iface.IHasInputStream;
 import com.helger.base.io.nonblocking.NonBlockingByteArrayInputStream;
 import com.helger.base.io.stream.HasInputStream;
+import com.helger.base.io.stream.StreamHelper;
 import com.helger.base.spi.ServiceLoaderHelper;
 import com.helger.base.state.ESuccess;
 import com.helger.base.string.StringHelper;
@@ -56,6 +58,7 @@
 import com.helger.http.CHttp;
 import com.helger.http.CHttpHeader;
 import com.helger.http.header.HttpHeaderMap;
+import com.helger.io.file.FileHelper;
 import com.helger.mime.IMimeType;
 import com.helger.mime.parse.MimeTypeParser;
 import com.helger.phase4.attachment.AS4DecompressException;
@@ -577,6 +580,47 @@ private static void _processSoapHeaderElements (@NonNull final SoapHeaderElement
     }
   }
 
+  /**
+   * Create an input stream provider that can be read multiple times, by lazily copying the data of
+   * the provided single-read input stream provider to a temporary file on first access.
+   *
+   * @param aResHelper
+   *        The resource helper to create the temporary file with. May not be <code>null</code>.
+   * @param aSrcISP
+   *        The single-read source input stream provider. May not be <code>null</code>.
+   * @return A non-<code>null</code> input stream provider that can be read multiple times.
+   */
+  @NonNull
+  private static IHasInputStream _createReadMultipleISP (@NonNull final AS4ResourceHelper aResHelper,
+                                                         @NonNull final IHasInputStream aSrcISP)
+  {
+    final Wrapper <File> aTempFileWrapper = new Wrapper <> ();
+    return new HasInputStream ( () -> {
+      try
+      {
+        File aTempFile = aTempFileWrapper.get ();
+        if (aTempFile == null)
+        {
+          final InputStream aSrcIS = aSrcISP.getInputStream ();
+          if (aSrcIS == null)
+            throw new IllegalStateException ("Failed to create InputStream from " + aSrcISP);
+
+          aTempFile = aResHelper.createTempFile ();
+          try (final OutputStream aOS = FileHelper.getBufferedOutputStream (aTempFile))
+          {
+            StreamHelper.copyInputStreamToOutputStream (aSrcIS, aOS);
+          }
+          aTempFileWrapper.set (aTempFile);
+        }
+        return FileHelper.getBufferedInputStream (aTempFile);
+      }
+      catch (final IOException ex)
+      {
+        throw new AS4DecompressException (ex);
+      }
+    }, true);
+  }
+
   private static void _decompressAttachments (@NonNull final ICommonsList <WSS4JAttachment> aIncomingDecryptedAttachments,
                                               @NonNull final Ebms3UserMessage aUserMessage,
                                               @NonNull final IAS4IncomingMessageState aIncomingState)
@@ -587,7 +631,21 @@ private static void _decompressAttachments (@NonNull final ICommonsList <WSS4JAt
       final EAS4CompressionMode eCompressionMode = aIncomingState.getAttachmentCompressionMode (aIncomingAttachment.getId ());
       if (eCompressionMode != null)
       {
-        final IHasInputStream aOldISP = aIncomingAttachment.getInputStreamProvider ();
+        IHasInputStream aCompressedISP = aIncomingAttachment.getInputStreamProvider ();
+        if (!aCompressedISP.isReadMultiple ())
+        {
+          // E.g. decrypted attachments can be read only once - make sure the
+          // compressed data can be read multiple times: once for the
+          // decompression and e.g. once for non-repudiation evidence storage
+          aCompressedISP = _createReadMultipleISP (aIncomingAttachment.getResHelper (), aCompressedISP);
+        }
+
+        // Preserve the compressed data - the signature digests are calculated
+        // over the compressed data, and e.g. GZIP compression is not
+        // reproducible (#361)
+        aIncomingAttachment.setCompressedSourceStreamProvider (aCompressedISP);
+
+        final IHasInputStream aOldISP = aCompressedISP;
         aIncomingAttachment.setSourceStreamProvider (new HasInputStream ( () -> {
           try
           {

phase4-lib/src/test/java/com/helger/phase4/attachment/WSS4JAttachmentTest.java

@@ -16,6 +16,7 @@
  */
 package com.helger.phase4.attachment;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
@@ -31,6 +32,7 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import com.helger.base.io.iface.IHasInputStream;
 import com.helger.base.io.stream.StreamHelper;
 import com.helger.io.file.SimpleFileIO;
 import com.helger.mime.CMimeType;
@@ -83,7 +85,7 @@ public void testBasic ()
   @Test
   public void testOutgoingBytes () throws IOException
   {
-    final byte [] aBytes = "<?xml version='1.0'?>".getBytes (StandardCharsets.ISO_8859_1);
+    final byte [] aBytes = "<?xml version='1.0'?>".getBytes (StandardCharsets.UTF_8);
 
     try (final AS4ResourceHelper aResHelper = new AS4ResourceHelper ())
     {
@@ -96,9 +98,12 @@ public void testOutgoingBytes () throws IOException
       assertEquals (CMimeType.APPLICATION_XML.getAsString (), a.getMimeType ());
       assertEquals (CMimeType.APPLICATION_XML.getAsString (), a.getUncompressedMimeType ());
 
+      // Not compressed - no compressed data present
+      assertNull (a.getCompressedSourceStreamProvider ());
+
       // Read content
       final byte [] aRead = StreamHelper.getAllBytes (a.getSourceStream ());
-      assertEquals (new String (aBytes, StandardCharsets.ISO_8859_1), new String (aRead, StandardCharsets.ISO_8859_1));
+      assertEquals (new String (aBytes, StandardCharsets.UTF_8), new String (aRead, StandardCharsets.UTF_8));
     }
   }
 
@@ -107,7 +112,7 @@ public void testOutgoingFile () throws IOException
   {
     final File f = m_aRule.newFile ("test.xml");
     final String sXMLContent = "<?xml version='1.0'?>";
-    SimpleFileIO.writeFile (f, sXMLContent.getBytes (StandardCharsets.ISO_8859_1));
+    SimpleFileIO.writeFile (f, sXMLContent.getBytes (StandardCharsets.UTF_8));
 
     try (final AS4ResourceHelper aResHelper = new AS4ResourceHelper ())
     {
@@ -122,16 +127,19 @@ public void testOutgoingFile () throws IOException
       assertEquals (CMimeType.APPLICATION_XML.getAsString (), a.getMimeType ());
       assertEquals (CMimeType.APPLICATION_XML.getAsString (), a.getUncompressedMimeType ());
 
+      // Not compressed - no compressed data present
+      assertNull (a.getCompressedSourceStreamProvider ());
+
       // Read content
       final byte [] aRead = StreamHelper.getAllBytes (a.getSourceStream ());
-      assertEquals (sXMLContent, new String (aRead, StandardCharsets.ISO_8859_1));
+      assertEquals (sXMLContent, new String (aRead, StandardCharsets.UTF_8));
     }
   }
 
   @Test
   public void testOutgoingCompression () throws IOException
   {
-    final byte [] aXmlBytes = "<?xml version='1.0'?>".getBytes (StandardCharsets.ISO_8859_1);
+    final byte [] aXmlBytes = "<?xml version='1.0'?>".getBytes (StandardCharsets.UTF_8);
 
     try (final AS4ResourceHelper aResHelper = new AS4ResourceHelper ())
     {
@@ -153,8 +161,19 @@ public void testOutgoingCompression () throws IOException
       assertTrue (aRead.length > 0);
 
       // It is definitely not the XML
-      assertNotEquals (new String (aRead, StandardCharsets.ISO_8859_1),
-                       new String (aXmlBytes, StandardCharsets.ISO_8859_1));
+      assertNotEquals (new String (aRead, StandardCharsets.UTF_8), new String (aXmlBytes, StandardCharsets.UTF_8));
+
+      // The compressed data must be preserved (issue #361)
+      final IHasInputStream aCompressedISP = a.getCompressedSourceStreamProvider ();
+      assertNotNull (aCompressedISP);
+
+      // It must be identical to the source stream content
+      assertArrayEquals (aRead, StreamHelper.getAllBytes (aCompressedISP.getInputStream ()));
+
+      // Decompressing the preserved compressed data must yield the original
+      // payload
+      final byte [] aDecompressed = StreamHelper.getAllBytes (EAS4CompressionMode.GZIP.getDecompressStream (aCompressedISP.getInputStream ()));
+      assertArrayEquals (aXmlBytes, aDecompressed);
     }
   }
 }