fix(private-server): run the MCP endpoint statelessly

Claude Code (and any client) connected, then got 'Not Found: Session not
found' on the request after initialize. rmcp's default stateful mode keeps
sessions in process memory; behind the multi-replica deployment a follow-up
request routed to a different pod than the one that ran initialize, so the
session was unknown and rmcp 404'd. Clients surface that as a dropped
connection / 'tools fetch failed'.

This is a read-only request/response API with no server-initiated push, so
disable sessions (stateful_mode=false) and return plain application/json
(json_response=true). Each POST is then self-contained and any replica can
serve it; there's also no long-lived SSE stream for the ingress to drop.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: passcod

crates/private-server/src/mcp.rs

@@ -1024,6 +1024,17 @@ fn mcp_err(e: impl std::fmt::Display) -> McpError {
 /// Build the tower service nested into the axum router at `/api/mcp`.
 pub fn service(state: AppState) -> StreamableHttpService<CanopyMcp, LocalSessionManager> {
 	let mut config = StreamableHttpServerConfig::default();
+	// Stateless: each request is self-contained, with no server-side session.
+	// The default stateful mode keeps sessions in process memory and 404s
+	// ("Session not found") any follow-up request that a load balancer routes to
+	// a different replica than the one that handled `initialize` — which is
+	// exactly what a multi-replica deployment behind the Tailscale ingress does.
+	// This is a read-only request/response API with no server-initiated push, so
+	// sessions buy us nothing.
+	config.stateful_mode = false;
+	// Return plain `application/json` per request instead of an SSE stream. With
+	// no streaming there's no long-lived response for a proxy to buffer or drop.
+	config.json_response = true;
 	// rmcp's `allowed_hosts` defaults to loopback only — a DNS-rebinding defense
 	// aimed at browser-facing localhost MCP servers. That threat doesn't apply
 	// here: the endpoint is reachable only through the Tailscale ingress (which

crates/private-server/tests/mcp.rs

@@ -1,18 +1,19 @@
 //! Tests for the read-only MCP query interface mounted at `/api/mcp`.
 //!
-//! The debug build bypasses Tailscale auth, so no headers are needed. These
-//! drive the real protocol path (initialize → tools/call over Streamable HTTP)
-//! against seeded data and assert the structured results, so they cover both
-//! the wiring and each tool's data shaping.
+//! The endpoint runs in stateless mode (no server-side session), so each POST
+//! is self-contained: a `tools/call` needs no prior `initialize`, just an
+//! `MCP-Protocol-Version` header. Responses are plain `application/json`. The
+//! debug build bypasses Tailscale auth, so no identity headers are needed.
 
 use commons_tests::diesel_async::SimpleAsyncConnection;
 
-/// The Streamable HTTP transport requires the client to accept both JSON and
-/// SSE on the POST leg.
+/// Clients must accept both JSON and SSE on the POST leg.
 const ACCEPT: &str = "application/json, text/event-stream";
+/// Required on every non-initialize request in stateless mode.
+const PROTO: &str = "2025-06-18";
 
-/// Extract the JSON-RPC envelope from a Streamable HTTP response body, which is
-/// either a bare JSON object or an SSE stream whose `data:` line carries it.
+/// Extract the JSON-RPC envelope from a response body (plain JSON in
+/// json-response mode, or an SSE `data:` line otherwise).
 fn parse_envelope(body: &str) -> serde_json::Value {
 	let trimmed = body.trim_start();
 	if trimmed.starts_with('{') {
@@ -26,50 +27,14 @@ fn parse_envelope(body: &str) -> serde_json::Value {
 	serde_json::from_str(data.trim()).expect("json in SSE data line")
 }
 
-/// Initialize a session and return its id. Expanded inline so the test never
-/// has to name `axum_test::TestServer`.
-macro_rules! init_session {
-	($private:expr) => {{
-		let init = $private
-			.post("/api/mcp")
-			.add_header("accept", ACCEPT)
-			.json(&serde_json::json!({
-				"jsonrpc": "2.0", "id": 1, "method": "initialize",
-				"params": {
-					"protocolVersion": "2025-06-18",
-					"capabilities": {},
-					"clientInfo": { "name": "canopy-tests", "version": "0" }
-				}
-			}))
-			.await;
-		assert_eq!(init.status_code().as_u16(), 200, "initialize should 200");
-		let session = init
-			.headers()
-			.get("mcp-session-id")
-			.expect("session id")
-			.to_str()
-			.unwrap()
-			.to_owned();
-		$private
-			.post("/api/mcp")
-			.add_header("accept", ACCEPT)
-			.add_header("mcp-session-id", &session)
-			.json(&serde_json::json!({
-				"jsonrpc": "2.0", "method": "notifications/initialized"
-			}))
-			.await;
-		session
-	}};
-}
-
 /// Call a tool and return its `structuredContent`. Asserts the call succeeded
 /// (no JSON-RPC error and `isError` is not true).
 macro_rules! call_tool {
-	($private:expr, $session:expr, $name:expr, $args:expr) => {{
+	($private:expr, $name:expr, $args:expr) => {{
 		let resp = $private
 			.post("/api/mcp")
 			.add_header("accept", ACCEPT)
-			.add_header("mcp-session-id", &$session)
+			.add_header("mcp-protocol-version", PROTO)
 			.json(&serde_json::json!({
 				"jsonrpc": "2.0", "id": 2, "method": "tools/call",
 				"params": { "name": $name, "arguments": $args }
@@ -93,7 +58,7 @@ macro_rules! call_tool {
 }
 
 /// Seed a group, two servers (one grouped + monitored with a fresh healthy
-/// status, one ungrouped), and a backup config + capability + schedule.
+/// status, one ungrouped), and a status carrying a version + platform.
 const GROUP: &str = "11111111-1111-1111-1111-111111111111";
 const SRV_GROUPED: &str = "22222222-2222-2222-2222-222222222222";
 const SRV_UNGROUPED: &str = "33333333-3333-3333-3333-333333333333";
@@ -116,11 +81,26 @@ async fn seed(conn: &mut impl SimpleAsyncConnection) {
 #[tokio::test(flavor = "multi_thread")]
 async fn initialize_and_list_tools() {
 	commons_tests::server::run(async |_conn, _public, private| {
-		let session = init_session!(private);
+		// initialize works (and returns tool capability), but is not required
+		// before other calls in stateless mode.
+		let init = private
+			.post("/api/mcp")
+			.add_header("accept", ACCEPT)
+			.json(&serde_json::json!({
+				"jsonrpc": "2.0", "id": 1, "method": "initialize",
+				"params": {
+					"protocolVersion": PROTO,
+					"capabilities": {},
+					"clientInfo": { "name": "canopy-tests", "version": "0" }
+				}
+			}))
+			.await;
+		assert_eq!(init.status_code().as_u16(), 200, "initialize should 200");
+
 		let list = private
 			.post("/api/mcp")
 			.add_header("accept", ACCEPT)
-			.add_header("mcp-session-id", &session)
+			.add_header("mcp-protocol-version", PROTO)
 			.json(&serde_json::json!({
 				"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}
 			}))
@@ -143,12 +123,26 @@ async fn initialize_and_list_tools() {
 	.await
 }
 
+#[tokio::test(flavor = "multi_thread")]
+async fn tools_call_needs_no_session() {
+	// Regression: the endpoint is stateless, so a `tools/call` must succeed on
+	// its own — no `initialize` handshake and no session id. (The default
+	// stateful mode 404s any request routed to a replica that didn't handle
+	// `initialize`, which is what broke real clients behind the load balancer.)
+	commons_tests::server::run(async |mut conn, _public, private| {
+		seed(&mut conn).await;
+		let summary = call_tool!(private, "fleet_summary", serde_json::json!({}));
+		assert!(summary["total_servers"].as_u64().unwrap() >= 2);
+	})
+	.await
+}
+
 #[tokio::test(flavor = "multi_thread")]
 async fn oauth_discovery_is_404_not_spa() {
-	// Regression: MCP clients probe `/.well-known/oauth-*` for OAuth metadata.
-	// The SPA fallback used to answer 200 text/html, which clients fail to parse
-	// as JSON and report as "needs authentication". A 404 tells them there's no
-	// OAuth, so they connect with the ambient (Tailscale) identity instead.
+	// MCP clients probe `/.well-known/oauth-*` for OAuth metadata. The SPA
+	// fallback used to answer 200 text/html, which clients fail to parse as JSON
+	// and report as "needs authentication". A 404 tells them there's no OAuth, so
+	// they connect with the ambient (Tailscale) identity instead.
 	commons_tests::server::run(async |_conn, _public, private| {
 		for path in [
 			"/.well-known/oauth-protected-resource",
@@ -180,7 +174,7 @@ async fn accepts_non_loopback_host() {
 			.json(&serde_json::json!({
 				"jsonrpc": "2.0", "id": 1, "method": "initialize",
 				"params": {
-					"protocolVersion": "2025-06-18",
+					"protocolVersion": PROTO,
 					"capabilities": {},
 					"clientInfo": { "name": "canopy-tests", "version": "0" }
 				}
@@ -200,31 +194,20 @@ async fn accepts_non_loopback_host() {
 async fn find_servers_filters_and_decorates() {
 	commons_tests::server::run(async |mut conn, _public, private| {
 		seed(&mut conn).await;
-		let session = init_session!(private);
 
 		// Unfiltered: both seeded servers.
-		let all = call_tool!(private, session, "find_servers", serde_json::json!({}));
+		let all = call_tool!(private, "find_servers", serde_json::json!({}));
 		assert!(all["total_matched"].as_u64().unwrap() >= 2);
 
 		// Filter by kind=facility → only the ungrouped one.
-		let facility = call_tool!(
-			private,
-			session,
-			"find_servers",
-			serde_json::json!({ "kind": "facility" })
-		);
+		let facility = call_tool!(private, "find_servers", serde_json::json!({ "kind": "facility" }));
 		let servers = facility["servers"].as_array().unwrap();
 		assert_eq!(servers.len(), 1);
 		assert_eq!(servers[0]["id"], SRV_UNGROUPED);
 		assert_eq!(servers[0]["health"], "healthy");
 
 		// Query matches by name.
-		let q = call_tool!(
-			private,
-			session,
-			"find_servers",
-			serde_json::json!({ "query": "central" })
-		);
+		let q = call_tool!(private, "find_servers", serde_json::json!({ "query": "central" }));
 		let names: Vec<&str> = q["servers"]
 			.as_array()
 			.unwrap()
@@ -233,11 +216,11 @@ async fn find_servers_filters_and_decorates() {
 			.collect();
 		assert_eq!(names, vec!["Prod Central"]);
 
-		// Bad enum → invalid params (protocol error).
+		// Bad enum → error (protocol or tool-level).
 		let bad = private
 			.post("/api/mcp")
 			.add_header("accept", ACCEPT)
-			.add_header("mcp-session-id", &session)
+			.add_header("mcp-protocol-version", PROTO)
 			.json(&serde_json::json!({
 				"jsonrpc": "2.0", "id": 9, "method": "tools/call",
 				"params": { "name": "find_servers", "arguments": { "kind": "nonsense" } }
@@ -256,14 +239,8 @@ async fn find_servers_filters_and_decorates() {
 async fn get_server_detail_and_not_found() {
 	commons_tests::server::run(async |mut conn, _public, private| {
 		seed(&mut conn).await;
-		let session = init_session!(private);
 
-		let detail = call_tool!(
-			private,
-			session,
-			"get_server",
-			serde_json::json!({ "server_id": SRV_GROUPED })
-		);
+		let detail = call_tool!(private, "get_server", serde_json::json!({ "server_id": SRV_GROUPED }));
 		assert_eq!(detail["name"], "Prod Central");
 		assert_eq!(detail["group_name"], "Prod Group");
 		assert_eq!(detail["latest_status"]["version"], "2.34.1");
@@ -273,7 +250,7 @@ async fn get_server_detail_and_not_found() {
 		let missing = private
 			.post("/api/mcp")
 			.add_header("accept", ACCEPT)
-			.add_header("mcp-session-id", &session)
+			.add_header("mcp-protocol-version", PROTO)
 			.json(&serde_json::json!({
 				"jsonrpc": "2.0", "id": 3, "method": "tools/call",
 				"params": {
@@ -292,9 +269,8 @@ async fn get_server_detail_and_not_found() {
 async fn group_listing_and_detail() {
 	commons_tests::server::run(async |mut conn, _public, private| {
 		seed(&mut conn).await;
-		let session = init_session!(private);
 
-		let groups = call_tool!(private, session, "find_groups", serde_json::json!({}));
+		let groups = call_tool!(private, "find_groups", serde_json::json!({}));
 		let g = groups["groups"]
 			.as_array()
 			.unwrap()
@@ -305,12 +281,7 @@ async fn group_listing_and_detail() {
 		assert_eq!(g["member_count"], 1);
 		assert_eq!(g["highest_rank"], "production");
 
-		let detail = call_tool!(
-			private,
-			session,
-			"get_group",
-			serde_json::json!({ "group_id": GROUP })
-		);
+		let detail = call_tool!(private, "get_group", serde_json::json!({ "group_id": GROUP }));
 		assert_eq!(detail["name"], "Prod Group");
 		let members = detail["members"].as_array().unwrap();
 		assert_eq!(members.len(), 1);
@@ -323,9 +294,8 @@ async fn group_listing_and_detail() {
 async fn fleet_summary_rolls_up() {
 	commons_tests::server::run(async |mut conn, _public, private| {
 		seed(&mut conn).await;
-		let session = init_session!(private);
 
-		let s = call_tool!(private, session, "fleet_summary", serde_json::json!({}));
+		let s = call_tool!(private, "fleet_summary", serde_json::json!({}));
 		assert!(s["total_servers"].as_u64().unwrap() >= 2);
 		assert_eq!(s["counts"]["by_kind"]["facility"], 1);
 		assert_eq!(s["version_distribution"]["2.34.1"], 1);
@@ -338,14 +308,8 @@ async fn fleet_summary_rolls_up() {
 async fn backup_problems_scan_runs() {
 	commons_tests::server::run(async |mut conn, _public, private| {
 		seed(&mut conn).await;
-		let session = init_session!(private);
 		// No ready backup config seeded, so the scan returns an empty, well-formed set.
-		let p = call_tool!(
-			private,
-			session,
-			"find_backup_problems",
-			serde_json::json!({})
-		);
+		let p = call_tool!(private, "find_backup_problems", serde_json::json!({}));
 		assert!(p["count"].is_number());
 		assert!(p["problems"].is_array());
 	})