Skip to content

Commit b4fa570

Browse files
TheBuilderJRTheBuilderJR
committed
Build OTLP content encoding mismatch lab
0 parents  commit b4fa570

18 files changed

Lines changed: 2442 additions & 0 deletions

‎.dockerignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
.git
2+
.github
3+
build
4+
__pycache__
5+
*.pyc

‎.gitattributes‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
*.vala linguist-detectable=true
2+
public/** linguist-vendored
3+
service/** linguist-vendored
4+
tests/** linguist-vendored
5+
Dockerfile linguist-vendored
6+
*.yaml linguist-vendored
7+
Makefile linguist-vendored

‎.github/workflows/ci.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: ci
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
test:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
16+
- name: Install Vala toolchain
17+
run: |
18+
sudo apt-get update
19+
sudo apt-get install --yes --no-install-recommends valac
20+
- name: Check model, service, and frontend
21+
run: make check
22+
23+
container:
24+
runs-on: ubuntu-latest
25+
needs: test
26+
steps:
27+
- uses: actions/checkout@v4
28+
- name: Build image
29+
run: docker build --tag content-encoding-mismatch-lab .
30+
- name: Smoke test
31+
run: |
32+
docker run --detach --name lab --publish 8080:8080 content-encoding-mismatch-lab
33+
for attempt in $(seq 1 20); do
34+
if curl --fail --silent http://127.0.0.1:8080/healthz; then
35+
break
36+
fi
37+
sleep 1
38+
done
39+
curl --fail --silent http://127.0.0.1:8080/ > /tmp/index.html
40+
curl --fail --silent \
41+
'http://127.0.0.1:8080/api/simulate?mode=passthrough' \
42+
| python3 -c 'import json,sys; assert json.load(sys.stdin)["summary"]["effective_delivery_percent"] == 100'
43+
curl --fail --silent \
44+
'http://127.0.0.1:8080/api/telemetry?mode=stripped' \
45+
| python3 -c 'import json,sys; assert json.load(sys.stdin)["collector_log"]["attributes"]["error.type"] == "gzip_header_missing"'
46+
- name: Container logs
47+
if: failure()
48+
run: |
49+
if docker container inspect lab >/dev/null 2>&1; then
50+
docker logs lab
51+
fi

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
build/
2+
__pycache__/
3+
*.py[cod]
4+
.DS_Store
5+
.idea/
6+
.vscode/

‎Dockerfile‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
FROM debian:bookworm-slim AS builder
2+
3+
RUN apt-get update \
4+
&& apt-get install -y --no-install-recommends \
5+
build-essential nodejs python3 valac \
6+
&& rm -rf /var/lib/apt/lists/*
7+
8+
WORKDIR /src
9+
COPY . .
10+
RUN make check
11+
12+
FROM python:3.13-slim-bookworm
13+
14+
RUN apt-get update \
15+
&& apt-get install -y --no-install-recommends libglib2.0-0 \
16+
&& rm -rf /var/lib/apt/lists/* \
17+
&& useradd --create-home --uid 10001 --shell /usr/sbin/nologin lab
18+
19+
WORKDIR /app
20+
COPY --from=builder /src/build/encoding-contract /app/build/encoding-contract
21+
COPY service /app/service
22+
COPY public /app/public
23+
24+
RUN chown -R lab:lab /app
25+
USER 10001
26+
27+
ENV HOST=0.0.0.0 \
28+
PORT=8080 \
29+
MODEL_BIN=/app/build/encoding-contract \
30+
PYTHONDONTWRITEBYTECODE=1 \
31+
PYTHONUNBUFFERED=1
32+
EXPOSE 8080
33+
34+
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
35+
CMD ["python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=2)"]
36+
37+
CMD ["python3", "service/server.py"]

‎LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2026 telemetry.sh
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎Makefile‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
PYTHON ?= python3
2+
VALAC ?= valac
3+
MODEL := build/encoding-contract
4+
5+
.PHONY: all build check vala-check python-check js-check run clean
6+
7+
all: build
8+
9+
build: $(MODEL)
10+
11+
$(MODEL): model/encoding_contract.vala
12+
mkdir -p build
13+
$(VALAC) --target-glib=2.56 --enable-checking --fatal-warnings \
14+
-X -Wall -X -Wextra -o $(MODEL) $<
15+
16+
check: build vala-check python-check js-check
17+
18+
vala-check:
19+
$(MODEL) >/dev/null
20+
$(MODEL) --mode stale >/dev/null
21+
$(MODEL) --mode passthrough >/dev/null
22+
$(MODEL) --mode normalize >/dev/null
23+
24+
python-check:
25+
$(PYTHON) -m unittest discover -s tests -v
26+
$(PYTHON) -m py_compile service/server.py
27+
28+
js-check:
29+
node --check public/app.js
30+
31+
run: build
32+
$(PYTHON) service/server.py
33+
34+
clean:
35+
rm -f $(MODEL)

‎README.md‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Content-Encoding Mismatch Lab
2+
3+
The bytes say gzip. The header says nothing.
4+
5+
This interactive lab models an OTLP/HTTP proxy that mutates a compressed body
6+
and its `Content-Encoding` metadata independently. The edge forwards every
7+
request, but the collector chooses its decoder from a contract that no longer
8+
describes the bytes.
9+
10+
The default scenario sends 240 trace batches per minute with 200 spans in each.
11+
A header-stripping proxy is rolled out to 55% of traffic:
12+
13+
| Signal | Edge view | Collector view |
14+
| --- | ---: | ---: |
15+
| Batches forwarded / minute | 240 | — |
16+
| Batches decoded / minute | — | 108 |
17+
| Apparent delivery | 100% | — |
18+
| Effective delivery | — | 45% |
19+
| Spans rejected / minute | 0 | 26,400 |
20+
21+
Every result is computed by the compiled Vala model in
22+
[`model/encoding_contract.vala`](model/encoding_contract.vala). The browser
23+
contains no duplicate simulation math.
24+
25+
## Why this happens
26+
27+
OTLP/HTTP permits clients to gzip the request body. When they do, the stable
28+
OTLP specification requires the request to include
29+
`Content-Encoding: gzip`. A collector uses that metadata to select the
30+
decompression step before decoding the OTLP Protobuf payload.
31+
32+
Two proxy transformations break the contract:
33+
34+
```text
35+
header stripped
36+
Content-Encoding: (absent) bytes: 1f 8b 08 00…
37+
collector selects Protobuf result: invalid wire type
38+
39+
header stale
40+
Content-Encoding: gzip bytes: 0a 8f 9c 06…
41+
collector selects gzip result: gzip magic missing
42+
```
43+
44+
Neither case is a transient outage. The OTLP specification classifies
45+
permanently undecodable input as bad data, for which the server returns
46+
`HTTP 400 Bad Request` and the client must not retry.
47+
48+
Primary references:
49+
50+
- [OTLP specification](https://opentelemetry.io/docs/specs/otlp/)
51+
- [OTLP exporter compression configuration](https://opentelemetry.io/docs/specs/otel/protocol/exporter/)
52+
- [Vala compiler documentation](https://docs.vala.dev/tutorials/programming-language/main/07-00-tools/07-01-valac.html)
53+
54+
## The repairs
55+
56+
The controls demonstrate two sound proxy contracts:
57+
58+
1. **Pass through** — forward `Content-Encoding: gzip` and the gzipped bytes
59+
without changing either.
60+
2. **Normalize** — decompress the body and remove `Content-Encoding`, leaving
61+
ordinary Protobuf bytes for the collector.
62+
63+
The invariant is simple: the header and body must be transformed together.
64+
65+
## What telemetry.sh reveals
66+
67+
A proxy request counter reports perfect delivery in both broken modes. The lab
68+
emits the signals needed to reconcile transport hops:
69+
70+
- `lab.otlp.apparent_delivery{hop="proxy"}` counts successfully forwarded
71+
requests.
72+
- `lab.otlp.effective_delivery{hop="collector"}` counts decoded requests.
73+
- `lab.otlp.decode_failures{error.type=...}` separates missing-header and
74+
stale-header failures.
75+
- `lab.otlp.rejected_spans{error.type=...}` quantifies permanent loss.
76+
- `lab.otlp.wire_bytes{representation=...}` shows when proxy decompression
77+
silently expands traffic.
78+
- a shared trace ID connects the forwarding proxy span to the collector decode
79+
log, including `Content-Encoding`, first bytes, and selected decoder.
80+
81+
Try the investigation queries in the UI:
82+
83+
```text
84+
metrics | where name in ("lab.otlp.apparent_delivery", "lab.otlp.effective_delivery") | chart value by hop
85+
logs | where error.type in ("gzip_header_missing", "gzip_magic_missing") | summarize count() by proxy.version
86+
traces | where http.route == "/v1/traces" | project trace_id, proxy.content_encoding, collector.decoder, otel.status_code
87+
```
88+
89+
This is the power of correlated telemetry: two individually plausible views
90+
become an obvious contradiction when joined.
91+
92+
## Run locally
93+
94+
Install Vala 0.56, a C compiler, Python 3, and Node.js, then:
95+
96+
```bash
97+
make check
98+
make run
99+
```
100+
101+
Open <http://localhost:8080>.
102+
103+
The live endpoints are:
104+
105+
```text
106+
GET /healthz
107+
GET /api/simulate
108+
GET /api/telemetry
109+
```
110+
111+
Example repairs:
112+
113+
```bash
114+
curl 'http://localhost:8080/api/simulate?mode=passthrough'
115+
curl 'http://localhost:8080/api/simulate?mode=normalize'
116+
```
117+
118+
## Run with Docker
119+
120+
```bash
121+
docker compose up --build
122+
```
123+
124+
The multi-stage image compiles the Vala model and runs the full test suite,
125+
then copies only the model, Python service, and static UI into an unprivileged
126+
runtime image.
127+
128+
## Repository layout
129+
130+
```text
131+
.
132+
├── model/encoding_contract.vala # header/body contract model
133+
├── service/server.py # static site, API, correlated evidence
134+
├── tests/ # model, service, and UI contracts
135+
├── public/ # dependency-free packet workbench
136+
├── telemetry/signals.md # evidence catalog
137+
├── Dockerfile
138+
└── compose.yaml
139+
```
140+
141+
## License
142+
143+
MIT

‎compose.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
services:
2+
lab:
3+
build: .
4+
ports:
5+
- "8080:8080"
6+
init: true
7+
read_only: true
8+
tmpfs:
9+
- /tmp
10+
security_opt:
11+
- no-new-privileges:true

0 commit comments

Comments
 (0)