Skip to content

Commit 5e4042f

Browse files
committed
testdrive against fdb
Signed-off-by: Moritz Hoffmann <mh@materialize.com>
1 parent 1602023 commit 5e4042f

File tree

15 files changed

+433
-266
lines changed

15 files changed

+433
-266
lines changed

ci/builder/Dockerfile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,12 @@ RUN curl -fsSL https://amazon-inspector-sbomgen.s3.amazonaws.com/1.8.1/linux/$AR
352352
&& mv inspector-sbomgen-1.8.1/linux/$ARCH_GO/inspector-sbomgen /usr/local/bin \
353353
&& chmod +x /usr/local/bin/inspector-sbomgen
354354

355+
RUN arch_fdb=$(echo "$ARCH_GCC" | sed -e "s/x86_64/amd64/") \
356+
&& curl -fsSL https://github.com/apple/foundationdb/releases/download/7.3.71/foundationdb-clients_7.3.71-1_$arch_fdb.deb > foundationdb-clients.deb \
357+
&& if [ $ARCH_GCC = X64_64 ]; then echo '695193b8c6f8af9ec083221611b5f2925ef7a5e3c5e3c1d0af65d0dfbe99d13d foundationdb-clients.deb' | sha256sum --check; fi \
358+
&& if [ $ARCH_GCC = aarch64 ]; then echo 'db1bbb72d57685a1c212c6456e6f0dfd1266c5c10c4adbc98d23a5d91bdbaff7 foundationdb-clients.deb' | sha256sum --check; fi \
359+
&& dpkg -i foundationdb-clients.deb
360+
355361
# Hardcode some known SSH hosts, or else SSH will ask whether the host is
356362
# trustworthy on the first connection.
357363

ci/test/pipeline.template.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,19 @@ steps:
297297
agents:
298298
queue: hetzner-aarch64-8cpu-16gb
299299

300+
- id: testdrive
301+
label: "Testdrive (FoundationDB)"
302+
depends_on: build-aarch64
303+
timeout_in_minutes: 40
304+
inputs: [test/testdrive]
305+
parallelism: 20
306+
plugins:
307+
- ./ci/plugins/mzcompose:
308+
composition: testdrive
309+
args: [--foundationdb]
310+
agents:
311+
queue: hetzner-aarch64-8cpu-16gb
312+
300313
- id: cluster-tests
301314
label: "Cluster tests"
302315
depends_on: build-aarch64

misc/foundationdb/fdb.cluster

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# FoundationDB in Docker
2+
docker:docker@foundationdb:4500
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
## foundationdb.conf
2+
##
3+
## Configuration file for FoundationDB server processes
4+
## Full documentation is available at
5+
## https://apple.github.io/foundationdb/configuration.html#the-configuration-file
6+
7+
[fdbmonitor]
8+
user = foundationdb
9+
group = foundationdb
10+
11+
[general]
12+
restart-delay = 60
13+
## by default, restart-backoff = restart-delay-reset-interval = restart-delay
14+
# initial-restart-delay = 0
15+
# restart-backoff = 60
16+
# restart-delay-reset-interval = 60
17+
cluster-file = /etc/foundationdb/fdb.cluster
18+
# delete-envvars =
19+
# kill-on-configuration-change = true
20+
21+
## Default parameters for individual fdbserver processes
22+
[fdbserver]
23+
command = /usr/sbin/fdbserver
24+
public-address = auto:$ID
25+
listen-address = public
26+
datadir = /var/lib/foundationdb/data/$ID
27+
logdir = /var/log/foundationdb
28+
# logsize = 10MiB
29+
# maxlogssize = 100MiB
30+
# machine-id =
31+
# datacenter-id =
32+
# class =
33+
# memory = 8GiB
34+
# storage-memory = 1GiB
35+
# cache-memory = 2GiB
36+
# metrics-cluster =
37+
# metrics-prefix =
38+
39+
## An individual fdbserver process with id 4500
40+
## Parameters set here override defaults from the [fdbserver] section
41+
[fdbserver.4500]
42+
43+
[backup_agent]
44+
command = /usr/lib/foundationdb/backup_agent/backup_agent
45+
logdir = /var/log/foundationdb
46+
47+
[backup_agent.1]

misc/images/materialized-base/Dockerfile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,11 @@
1515
# deployed to production, but the version needs to be bumped whenever features
1616
# that the console depends upon are removed (to a version of the console that
1717
# doesn't depend on those features).
18+
1819
FROM materialize/console:25.2.3 AS console
1920

21+
FROM foundationdb/foundationdb:7.3.71 as fdb
22+
2023
MZFROM ubuntu-base
2124

2225
ARG CI_SANITIZER=none
@@ -47,6 +50,9 @@ COPY postgresql.conf pg_hba.conf /etc/postgresql/
4750
COPY --from=console /usr/share/nginx/html /usr/share/nginx/html
4851
COPY console_nginx.template /etc/nginx/templates/default.conf.template
4952

53+
COPY --from=fdb /usr/lib/libfdb_c.so /usr/lib/
54+
COPY --from=fdb /usr/bin/fdbcli /usr/bin/
55+
5056
# Configure the console to listen on port 6874 and proxy API requests through to
5157
# the Materialize instance that will be started and listening for requests on
5258
# port 6876.

misc/images/prod-base/Dockerfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
# This is a separate mzimage so that we don't have to re-install the apt things
1111
# every time we get a CI builder with a cold cache.
1212

13+
FROM foundationdb/foundationdb:7.3.71 as fdb
14+
1315
MZFROM ubuntu-base
1416

1517
RUN groupadd --system --gid=999 materialize \
@@ -30,3 +32,6 @@ RUN apt-get update \
3032
&& chown materialize /scratch \
3133
&& mkdir /mzdata \
3234
&& chown materialize /mzdata
35+
36+
COPY --from=fdb /usr/lib/libfdb_c.so /usr/lib/
37+
COPY --from=fdb /usr/bin/fdbcli /usr/bin/

misc/images/ubuntu-base/Dockerfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99

1010
FROM ubuntu:plucky-20251001
1111

12+
FROM foundationdb/foundationdb:7.3.71 as fdb
13+
1214
# Ensure any Rust binaries that crash print a backtrace.
1315
ENV RUST_BACKTRACE=1
1416
# Disable backtrace collection in Rust libraries, such as `anyhow`. Collecting
@@ -24,3 +26,6 @@ RUN sed -i -e 's#http://archive\.ubuntu\.com#http://us-east-1.ec2.archive.ubuntu
2426
-e 's#http://ports\.ubuntu\.com#http://us-east-1.ec2.ports.ubuntu.com#' /etc/apt/sources.list.d/ubuntu.sources
2527

2628
RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends eatmydata
29+
30+
COPY --from=fdb /usr/lib/libfdb_c.so /usr/lib/
31+
COPY --from=fdb /usr/bin/fdbcli /usr/bin/

misc/python/materialize/cli/run.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -271,11 +271,11 @@ def main() -> int:
271271
if args.program == "environmentd":
272272
_handle_lingering_services(kill=args.reset)
273273
scratch = MZ_ROOT / "scratch"
274-
# dbconn = _connect_sql(args.postgres)
275-
# for schema in ["consensus", "tsoracle", "storage"]:
276-
# if args.reset:
277-
# _run_sql(dbconn, f"DROP SCHEMA IF EXISTS {schema} CASCADE")
278-
# _run_sql(dbconn, f"CREATE SCHEMA IF NOT EXISTS {schema}")
274+
dbconn = _connect_sql(args.postgres)
275+
for schema in ["consensus", "tsoracle", "storage"]:
276+
if args.reset:
277+
_run_sql(dbconn, f"DROP SCHEMA IF EXISTS {schema} CASCADE")
278+
_run_sql(dbconn, f"CREATE SCHEMA IF NOT EXISTS {schema}")
279279
# Keep this after clearing out Postgres. Otherwise there is a race
280280
# where a ctrl-c could leave persist with references in Postgres to
281281
# files that have been deleted. There's no race if we reset in the
@@ -320,7 +320,7 @@ def main() -> int:
320320
f"--orchestrator-process-prometheus-service-discovery-directory={MZDATA}/prometheus",
321321
f"--orchestrator-process-scratch-directory={scratch}",
322322
"--secrets-controller=local-file",
323-
f"--persist-consensus-url={args.consensus}?options=--search_path=consensus",
323+
f"--persist-consensus-url={consensus}?options=--search_path=consensus",
324324
f"--persist-blob-url={args.blob}",
325325
f"--timestamp-oracle-url={args.postgres}?options=--search_path=tsoracle",
326326
f"--environment-id={environment_id}",
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
import os
11+
from textwrap import dedent
12+
13+
from materialize import MZ_ROOT
14+
from materialize.mzcompose import loader
15+
from materialize.mzcompose.service import (
16+
Service,
17+
ServiceConfig,
18+
)
19+
from materialize.mzcompose.services.cockroach import Cockroach
20+
21+
22+
class FoundationDB(Service):
23+
def __init__(
24+
self,
25+
name: str = "foundationdb",
26+
image: str | None = None,
27+
ports: list[str] = ["4500"],
28+
environment: list[str] = [
29+
"FDB_NETWORKING_MODE=container",
30+
],
31+
volumes: list[str] = [],
32+
restart: str = "no",
33+
version: str = "7.3.71",
34+
) -> None:
35+
# command: list[str] = [
36+
# "postgres",
37+
# "-c",
38+
# "wal_level=logical",
39+
# "-c",
40+
# f"max_wal_senders={max_wal_senders}",
41+
# "-c",
42+
# f"max_replication_slots={max_replication_slots}",
43+
# "-c",
44+
# "max_connections=5000",
45+
# ] + extra_command
46+
47+
# if setup_materialize:
48+
# path = os.path.relpath(
49+
# MZ_ROOT / "misc" / "postgres" / "setup_materialize.sql",
50+
# loader.composition_path,
51+
# )
52+
# volumes = volumes + [
53+
# f"{path}:/docker-entrypoint-initdb.d/z_setup_materialize.sql"
54+
# ]
55+
#
56+
# environment = environment + ["PGPORT=26257"]
57+
58+
env_extra = [
59+
f"FDB_COORDINATOR_PORT={ports[0]}",
60+
f"FDB_PORT={ports[0]}",
61+
]
62+
63+
# command = dedent(
64+
# """
65+
# /usr/bin/tini -g -- /var/fdb/scripts/fdb.bash &
66+
# sleep 5
67+
# fdbcli -C /etc/foundationdb/fdb.cluster --exec "configure new single memory"
68+
# fdbcli -C /etc/foundationdb/fdb.cluster --exec "status"
69+
# wait
70+
# """
71+
# )
72+
73+
if image is None:
74+
image = f"foundationdb/foundationdb:{version}"
75+
76+
config: ServiceConfig = {"image": image}
77+
78+
volumes += [f"{MZ_ROOT}/misc/foundationdb/:/etc/foundationdb/"]
79+
80+
config.update(
81+
{
82+
"image": image,
83+
# "allow_host_ports": True,
84+
# "command": ["bash", "-c", command],
85+
"ports": ports,
86+
"environment": env_extra + environment,
87+
# "healthcheck": {
88+
# "test": [
89+
# "CMD",
90+
# "fdbcli",
91+
# "--exec",
92+
# "configure single memory ; status",
93+
# ],
94+
# "interval": "1s",
95+
# "start_period": "30s",
96+
# },
97+
"restart": restart,
98+
"volumes": volumes,
99+
}
100+
)
101+
super().__init__(name=name, config=config)
102+
103+
104+
# class PostgresMetadata(Postgres):
105+
# def __init__(self, restart: str = "no") -> None:
106+
# super().__init__(
107+
# name="postgres-metadata",
108+
# setup_materialize=True,
109+
# ports=["26257"],
110+
# restart=restart,
111+
# )
112+
113+
114+
# CockroachOrPostgresMetadata = (
115+
# Cockroach if os.getenv("BUILDKITE_TAG", "") != "" else PostgresMetadata
116+
# )
117+
#
118+
# METADATA_STORE: str = (
119+
# "cockroach" if CockroachOrPostgresMetadata == Cockroach else "postgres-metadata"
120+
# )

misc/python/materialize/mzcompose/services/materialized.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ def __init__(
100100
networks: (
101101
dict[str, dict[str, list[str]]] | dict[str, dict[str, str]] | None
102102
) = None,
103+
consensus_foundationdb: bool = False,
103104
) -> None:
104105
if name is None:
105106
name = "materialized"
@@ -259,6 +260,10 @@ def __init__(
259260
# v0.92.0).
260261
f"MZ_ADAPTER_STASH_URL=postgres://root@{address}:26257?options=--search_path=adapter",
261262
]
263+
if consensus_foundationdb:
264+
command += [
265+
f"--persist-consensus-url=foundationdb:?options=--search_path=consensus",
266+
]
262267

263268
command += [
264269
"--orchestrator-process-tcp-proxy-listen-addr=0.0.0.0",
@@ -336,6 +341,14 @@ def __init__(
336341

337342
volumes += [f"{os.getcwd()}/license_key:/license_key/license_key"]
338343

344+
if (
345+
image_version is None or image_version >= "v0.160.0-dev"
346+
) and consensus_foundationdb:
347+
print("Using foundationdb for consensus")
348+
volumes += [
349+
f"{MZ_ROOT}/misc/foundationdb/fdb.cluster:/etc/foundationdb/fdb.cluster"
350+
]
351+
339352
if use_default_volumes:
340353
volumes += DEFAULT_MZ_VOLUMES
341354
volumes += volumes_extra
@@ -387,3 +400,19 @@ class DeploymentStatus(Enum):
387400
"-f",
388401
"localhost:6878/api/leader/status",
389402
]
403+
404+
405+
class MetadataStoreType(Enum):
406+
INTERNAL = "internal"
407+
COCKROACHDB = "cockroachdb"
408+
FOUNDATIONDB = "foundationdb"
409+
410+
def consensus_url(self, mz_service: str) -> str:
411+
if self == MetadataStoreType.INTERNAL:
412+
return f"postgres://root@{mz_service}:26257?options=--search_path=consensus"
413+
elif self == MetadataStoreType.COCKROACHDB:
414+
return f"--persist-consensus-url=postgres://root@cockroach:26257?options=--search_path=consensus"
415+
elif self == MetadataStoreType.FOUNDATIONDB:
416+
return "foundationdb:?options=--search_path=consensus"
417+
else:
418+
raise RuntimeError(f"unknown metadata store type: {self.value}")

0 commit comments

Comments
 (0)