updated to remove bignum dep

Author: kristofr

src/blob/handlers/BlockBlobHandler.ts

@@ -1,6 +1,7 @@
-import { convertRawHeadersToMetadata } from "../../common/utils/utils";
 import {
   computeTransactionalChecksums,
+  convertRawHeadersToMetadata,
+  getCRC64FromStream,
   getMD5FromStream,
   getMD5FromString,
   newEtag
@@ -210,18 +211,31 @@ export default class BlockBlobHandler
       );
     }
 
-    // Compute MD5 and CRC64 in a single pass over the stored extent
-    const stream = await this.extentStore.readExtent(
-      persistency,
-      context.contextId
-    );
-    const { md5: calculatedContentMD5, crc64: calculatedCRC64 } =
-      await computeTransactionalChecksums(stream);
+    // Only read the stored extent when at least one transactional checksum was provided.
+    // Compute only what is needed to avoid unnecessary CPU work.
+    let calculatedContentMD5: Uint8Array | undefined;
+    let calculatedCRC64: Uint8Array | undefined;
+
+    if (contentMD5 !== undefined || contentCRC64 !== undefined) {
+      const stream = await this.extentStore.readExtent(
+        persistency,
+        context.contextId
+      );
+      if (contentMD5 !== undefined && contentCRC64 !== undefined) {
+        const result = await computeTransactionalChecksums(stream);
+        calculatedContentMD5 = result.md5;
+        calculatedCRC64 = result.crc64;
+      } else if (contentMD5 !== undefined) {
+        calculatedContentMD5 = await getMD5FromStream(stream);
+      } else {
+        calculatedCRC64 = await getCRC64FromStream(stream);
+      }
+    }
 
     if (contentMD5 !== undefined) {
       if (typeof contentMD5 === "string") {
         const calculatedContentMD5String = Buffer.from(
-          calculatedContentMD5
+          calculatedContentMD5!
         ).toString("base64");
         if (contentMD5 !== calculatedContentMD5String) {
           throw StorageErrorFactory.getInvalidOperation(
@@ -230,7 +244,7 @@ export default class BlockBlobHandler
           );
         }
       } else {
-        if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) {
+        if (!Buffer.from(contentMD5).equals(calculatedContentMD5!)) {
           throw StorageErrorFactory.getInvalidOperation(
             context.contextId!,
             "Provided contentMD5 doesn't match."
@@ -240,7 +254,7 @@ export default class BlockBlobHandler
     }
 
     if (contentCRC64 !== undefined) {
-      if (!Buffer.from(contentCRC64).equals(Buffer.from(calculatedCRC64))) {
+      if (!Buffer.from(contentCRC64).equals(Buffer.from(calculatedCRC64!))) {
         throw StorageErrorFactory.getInvalidOperation(
           context.contextId!,
           "Provided transactional CRC64 doesn't match."

src/common/utils/utils.ts

@@ -174,55 +174,68 @@ export async function getMD5FromStream(
 // Algorithm and lookup-table approach adapted from the Azure Storage JavaScript SDK (MIT License):
 // https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/storage/storage-blob/src/utils/crc64.ts
 // Polynomial: 0x42F0E1EBA9EA3693 (ECMA-182 standard, unreflected, init=0, xorout=0)
-const CRC64_POLY = 0x42f0e1eba9ea3693n;
+// Represented as two 32-bit halves (hi, lo) to avoid BigInt.
+const CRC64_POLY_HI = 0x42f0e1eb;
+const CRC64_POLY_LO = 0xa9ea3693;
 
-const CRC64_TABLE: readonly bigint[] = (() => {
-  const table: bigint[] = new Array(256);
+// Flat table: entry i occupies [i*2] (hi) and [i*2+1] (lo).
+const CRC64_TABLE: readonly number[] = (() => {
+  const table: number[] = new Array(512);
   for (let i = 0; i < 256; i++) {
-    let crc = BigInt(i) << 56n;
+    let hi = (i << 24) >>> 0;
+    let lo = 0;
     for (let j = 0; j < 8; j++) {
-      if ((crc & 0x8000000000000000n) !== 0n) {
-        crc = ((crc << 1n) ^ CRC64_POLY) & 0xffffffffffffffffn;
+      if ((hi & 0x80000000) !== 0) {
+        hi = (((hi << 1) | (lo >>> 31)) ^ CRC64_POLY_HI) >>> 0;
+        lo = ((lo << 1) ^ CRC64_POLY_LO) >>> 0;
       } else {
-        crc = (crc << 1n) & 0xffffffffffffffffn;
+        hi = ((hi << 1) | (lo >>> 31)) >>> 0;
+        lo = (lo << 1) >>> 0;
       }
     }
-    table[i] = crc;
+    table[i * 2] = hi;
+    table[i * 2 + 1] = lo;
   }
   return table;
 })();
 
-function crc64Accumulate(crc: bigint, chunk: Uint8Array): bigint {
+function crc64Accumulate(
+  crcHi: number, crcLo: number, chunk: Uint8Array
+): [number, number] {
   for (let i = 0; i < chunk.length; i++) {
-    const index = Number((crc >> 56n) ^ BigInt(chunk[i])) & 0xff;
-    crc = ((crc << 8n) ^ CRC64_TABLE[index]) & 0xffffffffffffffffn;
+    const index = ((crcHi >>> 24) ^ chunk[i]) & 0xff;
+    const tHi = CRC64_TABLE[index * 2];
+    const tLo = CRC64_TABLE[index * 2 + 1];
+    crcHi = (((crcHi << 8) | (crcLo >>> 24)) ^ tHi) >>> 0;
+    crcLo = ((crcLo << 8) ^ tLo) >>> 0;
   }
-  return crc;
+  return [crcHi, crcLo];
 }
 
-function bigintToUint8Array(n: bigint): Uint8Array {
+function crc64ToUint8Array(hi: number, lo: number): Uint8Array {
   const buf = Buffer.allocUnsafe(8);
-  buf.writeUInt32BE(Number(n >> 32n) >>> 0, 0);
-  buf.writeUInt32BE(Number(n & 0xffffffffn) >>> 0, 4);
+  buf.writeUInt32BE(hi >>> 0, 0);
+  buf.writeUInt32BE(lo >>> 0, 4);
   return buf;
 }
 
 export function getCRC64FromString(text: string): Uint8Array {
-  return bigintToUint8Array(crc64Accumulate(0n, Buffer.from(text)));
+  const [hi, lo] = crc64Accumulate(0, 0, Buffer.from(text));
+  return crc64ToUint8Array(hi, lo);
 }
 
 export async function getCRC64FromStream(
   stream: NodeJS.ReadableStream
 ): Promise<Uint8Array> {
   return new Promise<Uint8Array>((resolve, reject) => {
-    let crc = 0n;
+    let hi = 0, lo = 0;
     stream
       .on("data", (chunk: Buffer | string) => {
         const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
-        crc = crc64Accumulate(crc, data);
+        [hi, lo] = crc64Accumulate(hi, lo, data);
       })
       .on("end", () => {
-        resolve(bigintToUint8Array(crc));
+        resolve(crc64ToUint8Array(hi, lo));
       })
       .on("error", reject);
   });
@@ -237,15 +250,15 @@ export async function computeTransactionalChecksums(
 ): Promise<{ md5: Uint8Array; crc64: Uint8Array }> {
   const hash = createHash("md5");
   return new Promise((resolve, reject) => {
-    let crc = 0n;
+    let hi = 0, lo = 0;
     stream
       .on("data", (chunk: Buffer | string) => {
         const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
         hash.update(data);
-        crc = crc64Accumulate(crc, data);
+        [hi, lo] = crc64Accumulate(hi, lo, data);
       })
       .on("end", () => {
-        resolve({ md5: hash.digest(), crc64: bigintToUint8Array(crc) });
+        resolve({ md5: hash.digest(), crc64: crc64ToUint8Array(hi, lo) });
       })
       .on("error", reject);
   });