keccak: Pass fuzz tests vs OpenSSL

Author: mratsim

constantine.nimble

@@ -362,7 +362,8 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[
 
   # Hashing vs OpenSSL
   # ----------------------------------------------------------
-  ("tests/t_hash_sha256_vs_openssl.nim", false), # skip OpenSSL tests on Windows
+  ("tests/t_hash_sha256_vs_openssl.nim", false),
+  ("tests/t_hash_sha3_vs_openssl.nim", false),
 
   # Ciphers
   # ----------------------------------------------------------

constantine/hashes.nim

@@ -63,7 +63,15 @@ func hash*(
 # Exports
 # -----------------------------------------------------------------------
 
-import ./hashes/h_sha256
-export h_sha256
+import ./hashes/[
+  h_keccak,
+  h_sha256
+]
+export
+  h_keccak,
+  h_sha256
 
-static: doAssert sha256 is CryptoHash
+static:
+  doAssert keccak256 is CryptoHash
+  doAssert sha256 is CryptoHash
+  doAssert sha3_256 is CryptoHash

constantine/hashes/h_keccak.nim

@@ -6,10 +6,8 @@
 #   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-import ../zoo_exports
-
 import
-  ../platforms/[abstractions, views],
+  constantine/platforms/[abstractions, views],
   ./keccak/keccak_generic
 
 # Keccak, the hash function underlying SHA3
@@ -93,9 +91,8 @@ type
     #   Similarly after a squeeze, absorb_offset is incremented by the sponge rate.
     #   The real offset can be recovered with a substraction
     #   to properly update the state.
-
     H {.align: 64.}: KeccakState
-    buf {.align: 64.}: array[bits div 8, byte]
+    buf {.align: 64.}: array[200 - 2*(bits div 8), byte]
     absorb_offset: int32
     squeeze_offset: int32
 
@@ -125,11 +122,13 @@ func absorbBuffer(ctx: var KeccakContext) {.inline.} =
 
 template digestSize*(H: type KeccakContext): int =
   ## Returns the output size in bytes
-  KeccakContext.bits shr 3
+  # hardcoded for now or concept match issue with CryptoHash
+  32
 
 template internalBlockSize*(H: type KeccakContext): int =
   ## Returns the byte size of the hash function ingested blocks
-  2 * (KeccakContext.bits shr 3)
+  # hardcoded for now or concept match issue with CryptoHash
+  200
 
 func init*(ctx: var KeccakContext) {.inline.} =
   ## Initialize or reinitialize a Keccak context
@@ -145,9 +144,6 @@ func absorb*(ctx: var KeccakContext, message: openArray[byte]) =
   ## Additionally ensure that the message(s) passed were stored
   ## in memory considered secure for your threat model.
 
-  if message.len == 0:
-    return
-
   var pos = int ctx.absorb_offset
   var cur = 0
   var bytesLeft = message.len
@@ -183,13 +179,11 @@ func absorb*(ctx: var KeccakContext, message: openArray[byte]) =
     ctx.buf.rawCopy(dStart = pos, message, sStart = cur, len = bytesLeft)
 
   # Epilogue
-  ctx.absorb_offset = int32 bytesLeft
+  ctx.absorb_offset = int32(pos+bytesLeft)
   # Signal that the next squeeze transition needs a permute
   ctx.squeeze_offset = int32 ctx.rate()
 
 func squeeze*(ctx: var KeccakContext, digest: var openArray[byte]) =
-  if digest.len == 0:
-    return
 
   var pos = ctx.squeeze_offset
   var cur = 0
@@ -246,9 +240,11 @@ func update*(ctx: var KeccakContext, message: openArray[byte]) =
   ## in memory considered secure for your threat model.
   ctx.absorb(message)
 
-func finish*[N: static int](ctx: var KeccakContext, digest: var array[N, byte]) =
+func finish*(ctx: var KeccakContext, digest: var array[32, byte]) =
   ## Finalize a Keccak computation and output the
-  ## message digest to the `digest` buffer
+  ## message digest to the `digest` buffer.
+  ##
+  ## An `update` MUST be called before finish even with empty message.
   ##
   ## Security note: this does not clear the internal buffer.
   ## if sensitive content is used, use "ctx.clear()"
@@ -260,16 +256,3 @@ func clear*(ctx: var KeccakContext) =
   ## Clear the context internal buffers
   # TODO: ensure compiler cannot optimize the code away
   ctx.reset()
-
-when isMainModule:
-  import constantine/serialization/codecs
-
-  var msg: array[32, byte]
-  var digest: array[32, byte]
-  var ctx: keccak256
-
-  ctx.init()
-  ctx.update(msg)
-  ctx.finish(digest)
-
-  echo digest.toHex()

constantine/hashes/h_sha256.nim

@@ -6,11 +6,11 @@
 #   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-import ../zoo_exports
+import constantine/zoo_exports
 
 import
-  ../platforms/[abstractions, views],
-  ../serialization/endians,
+  constantine/platforms/[abstractions, views],
+  constantine/serialization/endians,
   ./sha256/sha256_generic
 
 when UseASM_X86_32:

constantine/hashes/keccak/keccak_generic.nim

@@ -231,10 +231,10 @@ func xorInSingle(H: var KeccakState, val: byte, offset: int) {.inline.} =
   let lane = uint64(val) shl slot # All bits but the one set in `val` are 0, and 0 is neutral element of xor
   H.state[offset shr 3] ^= lane
 
-func xorInBlock_generic(H: var KeccakState, msg: array[64, byte]) {.inline.} =
+func xorInBlock_generic(H: var KeccakState, msg: array[200 - 2*32, byte]) {.inline.} =
   ## Add new data into the Keccak state
   # This can benefit from vectorized instructions
-  for i in 0 ..< 8:
+  for i in 0 ..< msg.len div 8:
     H.state[i] ^= uint64.fromBytes(msg, i*8, littleEndian)
 
 func xorInPartial*(H: var KeccakState, msg: openArray[byte]) =
@@ -254,7 +254,7 @@ func xorInPartial*(H: var KeccakState, msg: openArray[byte]) =
   #   Lastly, this is only called when transitioning
   #   between absorbing and squeezing, for hashing
   #   this means once, however long a message to hash is.
-  var blck: array[64, byte] # zero-init
+  var blck: array[200 - 2*32, byte] # zero-init
   rawCopy(blck, 0, msg, 0, msg.len)
   H.xorInBlock_generic(blck)
 
@@ -283,7 +283,7 @@ func copyOutPartial*(
   # Implementation details:
   #   we could avoid a temporary block
   #   see `xorInPartial` for rationale
-  var blck {.noInit.}: array[64, byte]
+  var blck {.noInit.}: array[200 - 2*32, byte]
   H.copyOutWords(blck)
   rawCopy(dst, 0, blck, hByteOffset, dst.len)
 
@@ -303,8 +303,8 @@ func hashMessageBlocks_generic*(
   ##      a permutation is needed in-between
 
   var message = message
-  const rate = 64      # TODO: make a generic Keccak state with auto-derived rate
-  const numRounds = 24 # TODO: auto derive number of rounds
+  const rate = 200 - 2*32 # TODO: make a generic Keccak state with auto-derived rate
+  const numRounds = 24    # TODO: auto derive number of rounds
   for _ in 0 ..< numBlocks:
     let msg = cast[ptr array[rate, byte]](message)
     H.xorInBlock_generic(msg[])
@@ -321,8 +321,8 @@ func squeezeDigestBlocks_generic*(
   ## i.e. previous operation cannot be an absorb
   ##      a permutation is needed in-between
   var digest = digest
-  const rate = 64      # TODO: make a generic Keccak state with auto-derived rate
-  const numRounds = 24 # TODO: auto derive number of rounds
+  const rate = 200 - 2*32 # TODO: make a generic Keccak state with auto-derived rate
+  const numRounds = 24    # TODO: auto derive number of rounds
   for _ in 0 ..< numBlocks:
     let msg = cast[ptr array[rate, byte]](digest)
     H.copyOutWords(msg[])

tests/t_hash_sha256_vs_openssl.nim

@@ -28,30 +28,19 @@ else:
 # OpenSSL wrapper
 # --------------------------------------------------------------------
 
-# OpenSSL removed direct use of their SHA256 function. https://github.com/openssl/openssl/commit/4d49b68504cc494e552bce8e0b82ec8b501d5abe
-# It isn't accessible anymore in Windows CI on Github Action.
-# But the new API isn't expose on Linux :/
-
-# TODO: fix Windows
-when not defined(windows):
-  proc SHA256[T: byte|char](
-        msg: openarray[T],
-        digest: ptr array[32, byte] = nil
-      ): ptr array[32, byte] {.noconv, dynlib: DLLSSLName, importc.}
-
-  # proc EVP_Q_digest[T: byte|char](
-  #                 ossl_libctx: pointer,
-  #                 algoName: cstring,
-  #                 propq: cstring,
-  #                 data: openArray[T],
-  #                 digest: var array[32, byte],
-  #                 size: ptr uint): int32 {.noconv, dynlib: DLLSSLName, importc.}
-
-  proc SHA256_OpenSSL[T: byte|char](
-        digest: var array[32, byte],
-        s: openArray[T]) =
-    discard SHA256(s, digest.addr)
-    # discard EVP_Q_digest(nil, "SHA256", nil, s, digest, nil)
+proc EVP_Q_digest[T: byte|char](
+                ossl_libctx: pointer,
+                algoName: cstring,
+                propq: cstring,
+                data: openArray[T],
+                digest: var array[32, byte],
+                size: ptr uint): int32 {.noconv, dynlib: DLLSSLName, importc.}
+
+proc SHA256_OpenSSL[T: byte|char](
+      digest: var array[32, byte],
+      s: openArray[T]) =
+  # discard SHA256(s, digest.addr)
+  discard EVP_Q_digest(nil, "SHA256", nil, s, digest, nil)
 
 # Test
 # --------------------------------------------------------------------
@@ -84,16 +73,15 @@ proc sanityABC2 =
 
   doAssert bufCt == hashed
 
-when not defined(windows):
-  proc innerTest(rng: var RngState, sizeRange: Slice[int]) =
-    let size = rng.random_unsafe(sizeRange)
-    let msg = rng.random_byte_seq(size)
+proc innerTest(rng: var RngState, sizeRange: Slice[int]) =
+  let size = rng.random_unsafe(sizeRange)
+  let msg = rng.random_byte_seq(size)
 
-    var bufCt, bufOssl: array[32, byte]
+  var bufCt, bufOssl: array[32, byte]
 
-    sha256.hash(bufCt, msg)
-    SHA256_OpenSSL(bufOssl, msg)
-    doAssert bufCt == bufOssl, "Test failed with message of length " & $size
+  sha256.hash(bufCt, msg)
+  SHA256_OpenSSL(bufOssl, msg)
+  doAssert bufCt == bufOssl, "Test failed with message of length " & $size
 
 proc chunkTest(rng: var RngState, sizeRange: Slice[int]) =
   let size = rng.random_unsafe(sizeRange)
@@ -131,12 +119,9 @@ proc main() =
   var rng: RngState
   rng.seed(0xFACADE)
 
-  when not defined(windows):
-    echo "SHA256 - 0 <= size < 64 - exhaustive"
-    for i in 0 ..< 64:
-      rng.innerTest(i .. i)
-  else:
-    echo "SHA256 - 0 <= size < 64 - exhaustive [SKIPPED]"
+  echo "SHA256 - 0 <= size < 64 - exhaustive"
+  for i in 0 ..< 64:
+    rng.innerTest(i .. i)
 
   echo "SHA256 - 0 <= size < 64 - exhaustive chunked"
   for i in 0 ..< 64:
@@ -146,18 +131,14 @@ proc main() =
   for _ in 0 ..< SmallSizeIters:
     rng.chunkTest(0 ..< 1024)
 
-  when not defined(windows):
-    echo "SHA256 - 64 <= size < 1024B"
-    for _ in 0 ..< SmallSizeIters:
-      rng.innerTest(0 ..< 1024)
-
-    echo "SHA256 - 1MB <= size < 50MB"
-    for _ in 0 ..< LargeSizeIters:
-      rng.innerTest(1_000_000 ..< 50_000_000)
+  echo "SHA256 - 64 <= size < 1024B"
+  for _ in 0 ..< SmallSizeIters:
+    rng.innerTest(0 ..< 1024)
 
-    echo "SHA256 - Differential testing vs OpenSSL - SUCCESS"
+  echo "SHA256 - 1MB <= size < 50MB"
+  for _ in 0 ..< LargeSizeIters:
+    rng.innerTest(1_000_000 ..< 50_000_000)
 
-  else:
-    echo "SHA256 - Differential testing vs OpenSSL - [SKIPPED]"
+  echo "SHA256 - Differential testing vs OpenSSL - SUCCESS"
 
 main()