millpond/justfile at main · PostHog/millpond · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
# Millpond — Kafka to DuckLake

# Default recipe: list all available recipes
default:
    @just --list

# === Dev ===

# Install dependencies
[group('dev')]
sync:
    uv sync

# Install git hooks
[group('dev')]
install-hooks:
    git config core.hooksPath .githooks

# Run the application
[group('dev')]
run:
    uv run millpond

# Format code
[group('dev')]
fmt:
    uv run ruff format millpond/

# Check formatting
[group('dev')]
fmt-check:
    uv run ruff format --check millpond/

# Lint code
[group('dev')]
lint:
    uv run ruff check millpond/

# Lint and fix
[group('dev')]
lint-fix:
    uv run ruff check --fix millpond/

# === Test ===

# Run unit tests
[group('test')]
test:
    uv run python -m pytest tests/unit

# Run integration tests
[group('test')]
test-integration:
    uv run python -m pytest tests/integration

# Run E2E test (brings up docker-compose stack automatically)
[group('test')]
test-e2e:
    uv run python -m pytest tests/e2e -v -s

# Full CI check
[group('test')]
ci: fmt-check lint test

# === Docker ===

# Generate SSL certs for local Kafka (one-time setup)
[group('docker')]
ssl-certs:
    ./test/generate-ssl-certs.sh

# Start the docker-compose dev environment (Kafka, Postgres, MinIO, producer, 2 millpond pods)
[group('docker')]
up:
    docker compose build
    docker compose up -d

# Start the docker-compose dev environment with SSL Kafka
[group('docker')]
up-ssl: ssl-certs
    docker compose -f docker-compose.yaml -f docker-compose.ssl.yaml build
    docker compose -f docker-compose.yaml -f docker-compose.ssl.yaml up -d

# Stop the docker-compose dev environment
[group('docker')]
down:
    docker compose down -v

# Stop the SSL docker-compose dev environment
[group('docker')]
down-ssl:
    docker compose -f docker-compose.yaml -f docker-compose.ssl.yaml down -v

# Open a DuckDB shell attached to the local DuckLake (requires `just up` first)
[group('docker')]
duck:
    duckdb -init test/ducklake-init.sql

# Open the Grafana dashboard (requires `just up` first)
[group('docker')]
dashboard:
    open http://localhost:3000/d/millpond/millpond

# Open the MinIO console (requires `just up` first, login: minioadmin/minioadmin)
[group('docker')]
minio:
    open http://localhost:9001

# Open the sizing calculator
[group('docker')]
sizing:
    open tools/sizing-calculator.html

# === Build ===

# Build Docker image
[group('build')]
build:
    docker build -t millpond .

# Clean build artifacts
[group('build')]
clean:
    rm -rf .venv dist *.egg-info __pycache__ millpond/__pycache__

# === Metrics ===

# Compare LOC against ducklake-kafka-connect
[group('dev')]
loc:
    #!/usr/bin/env bash
    set -euo pipefail
    tmpdir=$(mktemp -d)
    trap "rm -rf $tmpdir" EXIT
    git clone --quiet --depth 1 https://github.com/PostHog/ducklake-kafka-connect.git "$tmpdir/dkc" 2>/dev/null
    echo ""
    echo "┌─────────────────────────────────────────────────────────────────┐"
    echo "│              LOC Comparison: Millpond vs Kafka Connect         │"
    echo "└─────────────────────────────────────────────────────────────────┘"
    echo ""
    echo "=== Millpond (Python) ==="
    cloc --quiet millpond/ tests/
    echo ""
    echo "=== ducklake-kafka-connect (Java) ==="
    cloc --quiet "$tmpdir/dkc/src/"
    echo ""
    # Summary table
    mp_prod=$(cloc --csv --quiet millpond/ | tail -1 | cut -d, -f5)
    mp_test=$(cloc --csv --quiet tests/ | tail -1 | cut -d, -f5)
    dkc_prod=$(cloc --csv --quiet "$tmpdir/dkc/src/main/" | tail -1 | cut -d, -f5)
    dkc_test=$(cloc --csv --quiet "$tmpdir/dkc/src/test/" "$tmpdir/dkc/src/integrationTest/" | tail -1 | cut -d, -f5)
    echo "┌──────────────────┬──────────────────┬──────────────┬───────┐"
    echo "│                  │ Kafka Connect    │ Millpond     │ Ratio │"
    echo "├──────────────────┼──────────────────┼──────────────┼───────┤"
    printf "│ Production       │ %'16d │ %'12d │ %4.1fx │\n" "$dkc_prod" "$mp_prod" "$(echo "$dkc_prod/$mp_prod" | bc -l)"
    printf "│ Tests            │ %'16d │ %'12d │ %4.1fx │\n" "$dkc_test" "$mp_test" "$(echo "$dkc_test/$mp_test" | bc -l)"
    printf "│ Total            │ %'16d │ %'12d │ %4.1fx │\n" "$((dkc_prod+dkc_test))" "$((mp_prod+mp_test))" "$(echo "($dkc_prod+$dkc_test)/($mp_prod+$mp_test)" | bc -l)"
    echo "└──────────────────┴──────────────────┴──────────────┴───────┘"