Merge pull request #118 from ClickHouse/revert-117-mongo-snappy

Revert "Remove MongoDB snappy config"

Author: rschu1ze

clickhouse/ddl.sql

@@ -1,7 +1,7 @@
 CREATE TABLE bluesky
 (
     `data` JSON(
-        max_dynamic_paths = 0, -- will become the default for large uses in future
+        max_dynamic_paths = 0,
         kind LowCardinality(String),
         commit.operation LowCardinality(String),
         commit.collection LowCardinality(String),

mongodb/create_and_load.sh

@@ -1,27 +1,29 @@
 #!/bin/bash
 
 # Check if the required arguments are provided
-if [[ $# -lt 6 ]]; then
-    echo "Usage: $0 <DB_NAME> <COLLECTION_NAME> <DATA_DIRECTORY> <NUM_FILES> <SUCCESS_LOG> <ERROR_LOG>"
+if [[ $# -lt 7 ]]; then
+    echo "Usage: $0 <DB_NAME> <COLLECTION_NAME> <DDL_FILE> <DATA_DIRECTORY> <NUM_FILES> <SUCCESS_LOG> <ERROR_LOG>"
     exit 1
 fi
 
 # Arguments
 DB_NAME="$1"
 COLLECTION_NAME="$2"
-DATA_DIRECTORY="$3"
-NUM_FILES="$4"
-SUCCESS_LOG="$5"
-ERROR_LOG="$6"
+DDL_FILE="$3"
+DATA_DIRECTORY="$4"
+NUM_FILES="$5"
+SUCCESS_LOG="$6"
+ERROR_LOG="$7"
 
 # Validate arguments
+[[ ! -f "$DDL_FILE" ]] && { echo "Error: DDL file '$DDL_FILE' does not exist."; exit 1; }
 [[ ! -d "$DATA_DIRECTORY" ]] && { echo "Error: Data directory '$DATA_DIRECTORY' does not exist."; exit 1; }
 [[ ! "$NUM_FILES" =~ ^[0-9]+$ ]] && { echo "Error: NUM_FILES must be a positive integer."; exit 1; }
 
 # Create database and execute DDL file
 mongosh --quiet --eval "
     db = db.getSiblingDB('$DB_NAME');
-    load('ddl.js');
+    load('$DDL_FILE');
 "
 
 echo "Loading data"

mongodb/ddl_snappy.js

@@ -0,0 +1,6 @@
+db.createCollection(
+   "bluesky",
+   { storageEngine: { wiredTiger: { configString: "block_compressor=snappy" } } }
+);
+
+db.bluesky.createIndex({"kind": 1, "commit.operation": 1, "commit.collection": 1, "did": 1, "time_us": 1});

mongodb/ddl.js mongodb/ddl_zstd.jsmongodb/ddl.js renamed to mongodb/ddl_zstd.js

@@ -36,42 +36,50 @@ fi
 
 benchmark() {
     local size=$1
-    local compression=x
+    local compression=$2
     # Check DATA_DIRECTORY contains the required number of files to run the benchmark
     file_count=$(find "$DATA_DIRECTORY" -type f | wc -l)
     if (( file_count < size )); then
         echo "Error: Not enough files in '$DATA_DIRECTORY'. Required: $size, Found: $file_count."
         exit 1
     fi
-    ./create_and_load.sh "bluesky_${size}m" bluesky "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
-    ./total_size.sh "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.total_size"
-    ./data_size.sh "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.data_size"
-    ./index_size.sh "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.index_size"
-    ./count.sh "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.count"
-    #./query_results.sh "bluesky_${size}m" | tee "${OUTPUT_PREFIX}_bluesky_${size}m.query_results"
-    ./index_usage.sh "bluesky_${size}m" | tee "${OUTPUT_PREFIX}_bluesky_${size}m.index_usage"
-    ./benchmark.sh "bluesky_${size}m" "${OUTPUT_PREFIX}_bluesky_${size}m.results_runtime"
-    ./drop_table.sh "bluesky_${size}m"
+    ./create_and_load.sh "bluesky_${size}m_${compression}" bluesky "ddl_${compression}.js" "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
+    ./total_size.sh "bluesky_${size}m_${compression}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${compression}.total_size"
+    ./data_size.sh "bluesky_${size}m_${compression}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${compression}.data_size"
+    ./index_size.sh "bluesky_${size}m_${compression}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${compression}.index_size"
+    ./count.sh "bluesky_${size}m_${compression}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${compression}.count"
+    #./query_results.sh "bluesky_${size}m_${compression}" | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${compression}.query_results"
+    ./index_usage.sh "bluesky_${size}m_${compression}" | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${compression}.index_usage"
+    ./benchmark.sh "bluesky_${size}m_${compression}" "${OUTPUT_PREFIX}_bluesky_${size}m_${compression}.results_runtime"
+    ./drop_table.sh "bluesky_${size}m_${compression}"
 }
 
 case $CHOICE in
     2)
-        benchmark 10
+        benchmark 10 snappy
+        benchmark 10 zstd
         ;;
     3)
-        benchmark 100
+        benchmark 100 snappy
+        benchmark 100 zstd
         ;;
     4)
-        benchmark 1000
+        benchmark 1000 snappy
+        benchmark 1000 zstd
         ;;
     5)
-        benchmark 1
-        benchmark 10
-        benchmark 100
-        benchmark 1000
+        benchmark 1 snappy
+        benchmark 1 zstd
+        benchmark 10 snappy
+        benchmark 10 zstd
+        benchmark 100 snappy
+        benchmark 100 zstd
+        benchmark 1000 snappy
+        benchmark 1000 zstd
         ;;
     *)
-        benchmark 1
+        benchmark 1 snappy
+        benchmark 1 zstd
         ;;
 esac