Support FoundationDB for consensus and timestamp oracle (#33819)

Implements a consensus and timestamp oracle backend to talk to
FoundationDB.
---------

Signed-off-by: Moritz Hoffmann <[email protected]>

Author: antiguru

.gitignore

@@ -25,6 +25,7 @@ node_modules
 /test/feature-benchmark/tmp/*.td
 /test/zippy/tmp/*.td
 /test/**/tmp_*.toml
+/test/**/fdb_*.cluster
 junit_*.xml
 parallel-workload-queries.log
 perf.data*

Cargo.lock

@@ -41,6 +41,7 @@ members = [
     "src/expr-parser",
     "src/expr-test-util",
     "src/fivetran-destination",
+    "src/foundationdb",
     "src/frontegg-auth",
     "src/frontegg-client",
     "src/frontegg-mock",
@@ -165,6 +166,7 @@ default-members = [
     "src/expr-derive-impl",
     "src/expr-parser",
     "src/expr-test-util",
+    "src/foundationdb",
     "src/frontegg-auth",
     "src/frontegg-client",
     "src/frontegg-mock",

Cargo.toml

@@ -290,6 +290,8 @@ case "$cmd" in
             --env BUILDKITE_PULL_REQUEST
             --env DOCKERHUB_USERNAME
             --env DOCKERHUB_ACCESS_TOKEN
+            # For configuring the metadata store
+            --env EXTERNAL_METADATA_STORE
         )
 
         if [[ $detach_container == "true" ]]; then

bin/ci-builder

@@ -355,6 +355,12 @@ RUN arch_zizmor=$(echo "$ARCH_GCC" | sed -e "s/amd64/x86_64/") \
     && tar xvf zizmor.tar.gz \
     && mv zizmor /usr/local/bin
 
+RUN arch_fdb=$(echo "$ARCH_GCC" | sed -e "s/x86_64/amd64/") \
+    && curl -fsSL https://github.com/apple/foundationdb/releases/download/7.3.71/foundationdb-clients_7.3.71-1_$arch_fdb.deb > foundationdb-clients.deb \
+    && if [ $ARCH_GCC = x64_64 ]; then echo '695193b8c6f8af9ec083221611b5f2925ef7a5e3c5e3c1d0af65d0dfbe99d13d foundationdb-clients.deb' | sha256sum --check; fi \
+    && if [ $ARCH_GCC = aarch64 ]; then echo 'db1bbb72d57685a1c212c6456e6f0dfd1266c5c10c4adbc98d23a5d91bdbaff7 foundationdb-clients.deb' | sha256sum --check; fi \
+    && dpkg -i foundationdb-clients.deb
+
 # Hardcode some known SSH hosts, or else SSH will ask whether the host is
 # trustworthy on the first connection.
 COPY ssh_known_hosts /etc/ssh/ssh_known_hosts

ci/builder/Dockerfile

@@ -409,6 +409,20 @@ steps:
               composition: testdrive
               args: [--default-size=8, --azurite, --slow]
 
+      - id: testdrive-fdb
+        label: "Testdrive (FoundationDB)"
+        depends_on: build-aarch64
+        timeout_in_minutes: 40
+        inputs: [test/testdrive]
+        parallelism: 20
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: testdrive
+              env:
+                - EXTERNAL_METADATA_STORE: foundationdb
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+
       - id: testdrive-in-cloudtest
         label: "Full Testdrive in Cloudtest (K8s)"
         depends_on: build-aarch64

ci/nightly/pipeline.template.yml

@@ -35,6 +35,17 @@ if read_list BUILDKITE_PLUGIN_MZCOMPOSE_ARGS; then
     done
 fi
 
+# Export environment variables passed via the env parameter
+# Buildkite creates vars like BUILDKITE_PLUGIN_MZCOMPOSE_ENV_0_VARNAME=value
+# We need to strip the prefix and the index number (0_, 1_, etc.)
+while IFS='=' read -r name value; do
+    # Strip the BUILDKITE_PLUGIN_MZCOMPOSE_ENV_ prefix
+    env_name="${name#BUILDKITE_PLUGIN_MZCOMPOSE_ENV_}"
+    # Strip the index number and underscore (e.g., "0_VARNAME" -> "VARNAME")
+    env_name="${env_name#*_}"
+    export "$env_name=$value"
+done < <(env | grep "^BUILDKITE_PLUGIN_MZCOMPOSE_ENV_" || true)
+
 STEP_START_TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
 
 # Clean up cores here so that just killed processes' core files are ignored

ci/plugins/mzcompose/hooks/command

@@ -15,6 +15,8 @@ configuration:
   properties:
     args:
       type: array
+    env:
+      type: object
     run:
       type: string
     composition: