Add popcnt_avx2_medium() for arrays < 1024 bytes

Author: kimwalisch

CHANGELOG.md

@@ -25,6 +25,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `__popcnt64` do). The runtime POPCNT check is unchanged.
 - The 1..=7 byte scalar tail no longer copies through a `memcpy` libcall; it is
   packed into a `u64` with an inlinable shift-or loop.
+- Medium arrays (~64 bytes to ~1 KB) now use a lightweight AVX2 `popcnt256` loop
+  rather than the scalar path or Harley-Seal — roughly 1.2–3x faster than scalar
+  across that range on a modern AVX2 CPU. The AVX2 dispatch switches to
+  Harley-Seal at 1 KB, matching the lookup-vs-Harley-Seal crossover measured
+  across Haswell..Cascadelake in the sse-popcount benchmarks, so older x86 CPUs
+  stay on their fastest kernel too.
 
 ## [0.2.0] - 2026-07-01
 

examples/benchmark.rs

@@ -95,7 +95,7 @@ fn algorithm(bytes: usize) -> &'static str {
         let popcnt_hw = cfg!(target_feature = "popcnt") || is_x86_feature_detected!("popcnt");
         if avx512 && bytes >= 40 {
             "AVX512"
-        } else if avx2 && bytes >= 512 {
+        } else if avx2 && bytes >= 64 {
             "AVX2"
         } else if popcnt_hw {
             "POPCNT"

src/lib.rs

@@ -220,10 +220,14 @@ fn popcnt_x86(bytes: &[u8]) -> u64 {
     {
         let mut cnt = 0u64;
         let mut rest = bytes;
-        // AVX2 only wins for arrays >= 512 bytes.
-        if bytes.len() >= 512 {
+        // A plain `popcnt256` loop for the medium range, Harley-Seal from ~1 KB.
+        if bytes.len() >= 64 {
             let n = bytes.len() / 32 * 32;
-            cnt += unsafe { popcnt_avx2(&bytes[..n]) };
+            cnt += if bytes.len() >= 1024 {
+                unsafe { popcnt_avx2(&bytes[..n]) }
+            } else {
+                unsafe { popcnt_avx2_medium(&bytes[..n]) }
+            };
             rest = &bytes[n..];
         }
         cnt + popcnt_scalar_static(rest)
@@ -310,10 +314,17 @@ fn popcnt_x86_runtime(bytes: &[u8]) -> u64 {
     let mut cnt = 0u64;
     let mut rest = bytes;
 
-    // AVX2 only wins for arrays >= 512 bytes.
-    if bytes.len() >= 512 && is_x86_feature_detected!("avx2") {
+    // AVX2: a plain `popcnt256` loop for the medium range, Harley-Seal once it
+    // pays off. The `popcnt256` loop beats scalar from ~64 bytes and beats
+    // Harley-Seal until ~1 KB — the lookup-vs-Harley-Seal crossover across
+    // Haswell..Cascadelake in the sse-popcount benchmarks is 1..2 KB.
+    if bytes.len() >= 64 && is_x86_feature_detected!("avx2") {
         let n = bytes.len() / 32 * 32;
-        cnt += unsafe { popcnt_avx2(&bytes[..n]) };
+        cnt += if bytes.len() >= 1024 {
+            unsafe { popcnt_avx2(&bytes[..n]) }
+        } else {
+            unsafe { popcnt_avx2_medium(&bytes[..n]) }
+        };
         rest = &bytes[n..];
     }
 
@@ -510,6 +521,44 @@ fn popcnt_avx2(bytes: &[u8]) -> u64 {
     lanes[0] + lanes[1] + lanes[2] + lanes[3]
 }
 
+/// Plain 2-accumulator `popcnt256` loop for medium arrays (~64 bytes to ~1 KB).
+/// Harley-Seal's fixed CSA-reduction epilogue makes it lose to just running
+/// `popcnt256` in a loop until ~1 KB, and this beats the scalar path from ~64
+/// bytes up. `bytes.len()` must be a multiple of 32.
+#[cfg(all(
+    any(target_arch = "x86", target_arch = "x86_64"),
+    not(target_feature = "avx512vpopcntdq"),
+    any(target_feature = "avx2", feature = "std")
+))]
+#[target_feature(enable = "avx2")]
+#[inline]
+fn popcnt_avx2_medium(bytes: &[u8]) -> u64 {
+    let mut acc0 = _mm256_setzero_si256();
+    let mut acc1 = _mm256_setzero_si256();
+
+    let (pairs, tail) = bytes.as_chunks::<64>();
+    for chunk in pairs {
+        let p = chunk.as_ptr().cast::<__m256i>();
+        // SAFETY: `chunk` is 64 bytes, so both 32-byte loads are in bounds.
+        unsafe {
+            acc0 = _mm256_add_epi64(acc0, popcnt256(_mm256_loadu_si256(p.add(0))));
+            acc1 = _mm256_add_epi64(acc1, popcnt256(_mm256_loadu_si256(p.add(1))));
+        }
+    }
+
+    // 0 or 1 leftover 32-byte vector (`bytes.len()` is a multiple of 32).
+    let (vecs, _) = tail.as_chunks::<32>();
+    for chunk in vecs {
+        let v = unsafe { _mm256_loadu_si256(chunk.as_ptr().cast::<__m256i>()) };
+        acc0 = _mm256_add_epi64(acc0, popcnt256(v));
+    }
+
+    let cnt = _mm256_add_epi64(acc0, acc1);
+    // SAFETY: `__m256i` and `[u64; 4]` are both 32 bytes with no invalid bit patterns.
+    let lanes: [u64; 4] = unsafe { core::mem::transmute(cnt) };
+    lanes[0] + lanes[1] + lanes[2] + lanes[3]
+}
+
 // ── AVX512 ──────────────────────────────────────────────────────────────────
 
 /// AVX512-VPOPCNTDQ population count, handling any length: a 4×-unrolled