Skip to content

Commit 4afab4d

Browse files
touch-of-greytouch-of-grey
committed
bench: rework fineweb FTS bench to mirror the ShardWriter backpressure bench
Rewrites mem_wal_fineweb_fts as a CLI-arg bench modeled on the upstream mem_wal_shard_writer_backpressure bench: same Mode matrix (async/sync × index/no-index), same ShardWriter wiring, JSON output. Payload is real HuggingFace FineWeb text and the maintained index is the in-memory FTS index. Splits the run into independent --phase write|read invocations so a process never holds two ShardWriter lifecycles in sequence — that was the deadlock in the first iteration. The driver runs every config under a timeout watchdog so a hang costs one window, not days.
1 parent e5e39d9 commit 4afab4d

2 files changed

Lines changed: 618 additions & 723 deletions

File tree

‎bench/run_fineweb_fts.sh‎

Lines changed: 84 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -1,121 +1,113 @@
11
#!/usr/bin/env bash
2-
# Driver for the fineweb FTS benchmark.
2+
# Driver for the FineWeb FTS benchmark panel.
33
#
4-
# Runs the 12 configs (3 memtable sizes × durable yes/no × FTS yes/no), saves
5-
# each result.json locally and to S3, and at the end prints a small summary.
4+
# Write panel : 12 configs = 4 modes (async/sync × idx/no-idx) × 3 memtable
5+
# sizes (100k / 500k / 1M). Each config ingests 1M rows.
6+
# Read panel : 6 configs = 2 indexed modes × 3 memtable sizes. Each
7+
# ingests `size` rows into an auto-flush-disabled MemTable,
8+
# times the FTS queries, flushes, and replays on disk.
69
#
7-
# Usage:
8-
# ./bench/run_fineweb_fts.sh [run_id]
10+
# Every config runs as its own process under a `timeout` watchdog, so a
11+
# hang costs one timeout window, not days. result.json is uploaded to S3.
912
#
10-
# Env vars (optional):
11-
# DATASET_PREFIX default: s3://jack-devland-build/bench/mem-fts-fineweb
12-
# BENCH_BASE_ROWS default: 1000000
13-
# BENCH_INGEST_ROWS default: 1000000
14-
# BENCH_BATCH_SIZE default: 1000
15-
# AWS_DEFAULT_REGION default: us-east-1
13+
# Usage: ./bench/run_fineweb_fts.sh [run_id]
1614

17-
set -euo pipefail
15+
set -uo pipefail
1816

1917
cd "$(dirname "${BASH_SOURCE[0]}")/.."
2018

2119
RUN_ID="${1:-$(date -u +%Y%m%dT%H%M%SZ)}"
2220
DATASET_PREFIX="${DATASET_PREFIX:-s3://jack-devland-build/bench/mem-fts-fineweb}"
23-
BENCH_BASE_ROWS="${BENCH_BASE_ROWS:-1000000}"
24-
BENCH_INGEST_ROWS="${BENCH_INGEST_ROWS:-1000000}"
25-
BENCH_BATCH_SIZE="${BENCH_BATCH_SIZE:-1000}"
21+
SEED_ROWS="${SEED_ROWS:-1000000}"
22+
BATCH_ROWS="${BATCH_ROWS:-1000}"
23+
CALLS="${CALLS:-1000}"
24+
CACHE_DIR="${CACHE_DIR:-/mnt/data/fineweb}"
25+
CONFIG_TIMEOUT="${CONFIG_TIMEOUT:-3600}"
2626
export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
2727

2828
LOCAL_DIR="bench/results/${RUN_ID}"
2929
mkdir -p "$LOCAL_DIR"
3030

31-
BIN="target/release/mem_wal_fineweb_fts"
32-
if [ ! -x "$BIN" ]; then
31+
BIN="$(find target/release/deps -maxdepth 1 -type f -perm -111 -name 'mem_wal_fineweb_fts-*' ! -name '*.d' 2>/dev/null | sort | tail -1)"
32+
if [ -z "$BIN" ]; then
3333
echo "building bench binary..."
34-
cargo build --release -p lance --bench mem_wal_fineweb_fts
35-
# criterion-style bench output goes to deps/; resolve it.
36-
BIN="$(ls -t target/release/deps/mem_wal_fineweb_fts-* | grep -v '\.d$' | head -1)"
34+
cargo bench -p lance --bench mem_wal_fineweb_fts --no-run
35+
BIN="$(find target/release/deps -maxdepth 1 -type f -perm -111 -name 'mem_wal_fineweb_fts-*' ! -name '*.d' 2>/dev/null | sort | tail -1)"
3736
fi
38-
echo "using bench binary: $BIN"
39-
40-
CONFIGS=(
41-
"100000 0 0"
42-
"100000 0 1"
43-
"100000 1 0"
44-
"100000 1 1"
45-
"500000 0 0"
46-
"500000 0 1"
47-
"500000 1 0"
48-
"500000 1 1"
49-
"1000000 0 0"
50-
"1000000 0 1"
51-
"1000000 1 0"
52-
"1000000 1 1"
53-
)
54-
55-
echo "=== Run $RUN_ID =="
56-
echo " prefix: $DATASET_PREFIX"
57-
echo " base_rows: $BENCH_BASE_ROWS ingest_rows: $BENCH_INGEST_ROWS batch_size: $BENCH_BATCH_SIZE"
37+
echo "bench binary: $BIN"
38+
echo "run id: $RUN_ID"
5839
echo ""
5940

60-
for cfg in "${CONFIGS[@]}"; do
61-
read -r MT D F <<< "$cfg"
62-
if [ "$MT" = "1000000" ]; then SZ="1M"; elif [ "$MT" = "500000" ]; then SZ="500k"; else SZ="100k"; fi
63-
NAME="mt${SZ}_durable${D}_fts${F}"
64-
OUT="$LOCAL_DIR/${NAME}.json"
65-
LOG="$LOCAL_DIR/${NAME}.log"
66-
echo ">>> $NAME"
67-
if [ -f "$OUT" ]; then
68-
echo " result already exists, skipping"
69-
continue
41+
run_one() {
42+
local name="$1"; shift
43+
local out="$LOCAL_DIR/${name}.json"
44+
local log="$LOCAL_DIR/${name}.log"
45+
echo ">>> $name"
46+
if [ -f "$out" ]; then
47+
echo " already done, skipping"
48+
return
7049
fi
71-
set +e
72-
BENCH_RUN_ID="$RUN_ID" \
73-
DATASET_PREFIX="$DATASET_PREFIX" \
74-
BENCH_MAX_MEMTABLE_ROWS="$MT" \
75-
DURABLE_WRITE="$D" \
76-
FTS_ENABLED="$F" \
77-
BENCH_BASE_ROWS="$BENCH_BASE_ROWS" \
78-
BENCH_INGEST_ROWS="$BENCH_INGEST_ROWS" \
79-
BENCH_BATCH_SIZE="$BENCH_BATCH_SIZE" \
80-
BENCH_CACHE_DIR="${BENCH_CACHE_DIR:-/mnt/data/fineweb}" \
81-
RESULT_FILE="$OUT" \
82-
"$BIN" --bench --nocapture 2>&1 | tee "$LOG"
83-
RC=${PIPESTATUS[0]}
84-
set -e
85-
if [ "$RC" -ne 0 ]; then
86-
echo " !!! config failed (rc=$RC); see $LOG"
87-
fi
88-
# Upload to S3 alongside the dataset.
89-
if [ -f "$OUT" ]; then
90-
aws s3 cp "$OUT" "$DATASET_PREFIX/$RUN_ID/results/${NAME}.json" || true
91-
aws s3 cp "$LOG" "$DATASET_PREFIX/$RUN_ID/results/${NAME}.log" || true
50+
timeout "$CONFIG_TIMEOUT" "$BIN" --bench "$@" --output "$out" > "$log" 2>&1
51+
local rc=$?
52+
if [ "$rc" -eq 124 ]; then
53+
echo " !!! TIMED OUT after ${CONFIG_TIMEOUT}s"
54+
elif [ "$rc" -ne 0 ]; then
55+
echo " !!! failed rc=$rc (see $log)"
56+
else
57+
echo " ok"
9258
fi
59+
[ -f "$out" ] && aws s3 cp "$out" "$DATASET_PREFIX/$RUN_ID/results/${name}.json" >/dev/null 2>&1
60+
aws s3 cp "$log" "$DATASET_PREFIX/$RUN_ID/results/${name}.log" >/dev/null 2>&1
61+
}
62+
63+
# ---- write panel: 4 modes × 3 sizes ----
64+
for mode in async_noidx async_idx sync_noidx sync_idx; do
65+
for sz in 100000 500000 1000000; do
66+
case "$sz" in
67+
1000000) tag=1M ;;
68+
500000) tag=500k ;;
69+
*) tag=100k ;;
70+
esac
71+
run_one "write_${mode}_mt${tag}" \
72+
--phase write --mode "$mode" \
73+
--uri "$DATASET_PREFIX/$RUN_ID/w_${mode}_mt${tag}" \
74+
--seed-rows "$SEED_ROWS" --batch-rows "$BATCH_ROWS" --calls "$CALLS" \
75+
--max-memtable-rows "$sz" --cache-dir "$CACHE_DIR"
76+
done
77+
done
78+
79+
# ---- read panel: 2 indexed modes × 3 sizes ----
80+
for mode in async_idx sync_idx; do
81+
for sz in 100000 500000 1000000; do
82+
case "$sz" in
83+
1000000) tag=1M ;;
84+
500000) tag=500k ;;
85+
*) tag=100k ;;
86+
esac
87+
run_one "read_${mode}_mt${tag}" \
88+
--phase read --mode "$mode" \
89+
--uri "$DATASET_PREFIX/$RUN_ID/r_${mode}_mt${tag}" \
90+
--seed-rows "$SEED_ROWS" --batch-rows "$BATCH_ROWS" \
91+
--read-rows "$sz" --cache-dir "$CACHE_DIR"
92+
done
9393
done
9494

9595
echo ""
9696
echo "=== summary ==="
97-
python3 - <<PY
98-
import glob, json, os
99-
results = []
100-
for p in sorted(glob.glob(os.path.join("$LOCAL_DIR", "*.json"))):
97+
python3 - "$LOCAL_DIR" <<'PY'
98+
import glob, json, os, sys
99+
d = sys.argv[1]
100+
print(f"{'config':28s} {'rows/s':>10} {'put_p99_ms':>11} {'mt_p95_ms':>10} {'cons_mean':>10}")
101+
for p in sorted(glob.glob(os.path.join(d, "*.json"))):
101102
try:
102-
with open(p) as f: r = json.load(f)
103-
results.append(r)
103+
r = json.load(open(p))
104104
except Exception as e:
105-
print(f" failed to read {p}: {e}")
106-
107-
print(f"{'config':30s} {'rows/s':>10} {'p95_ms':>7} {'mt_p95_ms':>10} {'cons_mean':>10}")
108-
for r in results:
109-
name = r["config_name"]
110-
tp = r["ingest"]["rows_per_sec"]
111-
p95 = r["ingest"]["put_p95_ms"]
112-
rd = r.get("read")
113-
mt = rd["mt_latency_p95_ms"] if rd else 0
114-
cm = rd["consistency_mean"] if rd else 0
115-
print(f"{name:30s} {tp:>10.0f} {p95:>7.2f} {mt:>10.2f} {cm:>10.3f}")
105+
print(f" bad {p}: {e}"); continue
106+
name = os.path.basename(p)[:-5]
107+
if r.get("phase") == "write":
108+
print(f"{name:28s} {r['throughput_rows_per_sec']:>10.0f} {r['put_p99_ms']:>11.2f} {'-':>10} {'-':>10}")
109+
else:
110+
print(f"{name:28s} {'-':>10} {'-':>11} {r['mt_latency_p95_ms']:>10.3f} {r['consistency_mean']:>10.3f}")
116111
PY
117-
118112
echo ""
119-
echo "Results:"
120-
echo " local: $LOCAL_DIR"
121-
echo " s3: $DATASET_PREFIX/$RUN_ID/results/"
113+
echo "results: $LOCAL_DIR + $DATASET_PREFIX/$RUN_ID/results/"

0 commit comments

Comments
 (0)