feat: automatic tool-count trimming to stay within per-provider limits

Claude Code with many MCP servers easily exceeds Anthropic's 128-tool limit,
returning a hard 400. Bildy now trims the tool list to fit the resolved
provider's ceiling before forwarding, using a smart strategy: tools that
appear in recent conversation history (tool_use blocks) are kept first so
the most relevant tools survive the cut.

Per-provider limits (conservative where undocumented):
  anthropic/openai/groq: 128
  cloudflare: 100
  cerebras/nvidia: 64

Override globally via bildy.config.json: { "routing": { "maxTools": N } }

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: stackbilt-admin

packages/llm-gateway/src/config.ts

@@ -86,6 +86,7 @@ export function resolveConfig(options?: { port?: number; cwd?: string }): Gatewa
       shadowMode: fileConfig.routing?.shadowMode ?? defaultConfig.routing.shadowMode,
       shadowRoutes: { ...defaultConfig.routing.shadowRoutes, ...fileConfig.routing?.shadowRoutes },
       anthropicDirectProxy: fileConfig.routing?.anthropicDirectProxy ?? defaultConfig.routing.anthropicDirectProxy,
+      maxTools: fileConfig.routing?.maxTools ?? defaultConfig.routing.maxTools,
     },
     cache: {
       ...defaultConfig.cache,

packages/llm-gateway/src/policy/tool-trim.ts

@@ -0,0 +1,100 @@
+import type { LLMMessage, LLMRequest, LLMTool } from "../types.js";
+
+// Per-provider tool count ceilings.
+// Anthropic/OpenAI: 128 (documented). Others: conservative estimates from
+// empirical testing — lower limits fail with cryptic 400/422 errors.
+const PROVIDER_TOOL_LIMITS: Record<string, number> = {
+  anthropic: 128,
+  openai: 128,
+  groq: 128,
+  cerebras: 64,
+  cloudflare: 100,
+  nvidia: 64,
+};
+
+export const DEFAULT_TOOL_LIMIT = 128;
+
+export function getProviderToolLimit(provider: string): number {
+  return PROVIDER_TOOL_LIMITS[provider.toLowerCase()] ?? DEFAULT_TOOL_LIMIT;
+}
+
+export interface ToolTrimResult {
+  request: LLMRequest;
+  trimmed: number;
+  originalCount: number;
+}
+
+/**
+ * Trim tool list to fit the provider's maximum.
+ * Prioritizes tools that appear in recent conversation history (most recently
+ * called first), then fills remaining slots in declaration order.
+ * Returns the original request unchanged if no trimming is needed.
+ */
+export function trimToolsForProvider(
+  request: LLMRequest,
+  provider: string,
+  configLimit?: number,
+): ToolTrimResult {
+  const tools = request.tools;
+  const limit = configLimit ?? getProviderToolLimit(provider);
+
+  if (!tools || tools.length <= limit) {
+    return { request, trimmed: 0, originalCount: tools?.length ?? 0 };
+  }
+
+  const originalCount = tools.length;
+  const recentlyUsed = extractRecentlyUsedToolNames(request.messages);
+  const toolByName = new Map<string, LLMTool>(tools.map((t) => [t.function.name, t]));
+  const kept = new Set<string>();
+  const result: LLMTool[] = [];
+
+  // First pass: recently-used tools in recency order (most recent first)
+  for (const name of recentlyUsed) {
+    if (result.length >= limit) break;
+    const tool = toolByName.get(name);
+    if (tool && !kept.has(name)) {
+      result.push(tool);
+      kept.add(name);
+    }
+  }
+
+  // Second pass: fill remaining slots in declaration order
+  for (const tool of tools) {
+    if (result.length >= limit) break;
+    const name = tool.function.name;
+    if (!kept.has(name)) {
+      result.push(tool);
+      kept.add(name);
+    }
+  }
+
+  return {
+    request: { ...request, tools: result },
+    trimmed: originalCount - result.length,
+    originalCount,
+  };
+}
+
+/**
+ * Walk messages in reverse order collecting tool names from tool_use blocks.
+ * Returns names in reverse-chronological order (most recently called first).
+ */
+function extractRecentlyUsedToolNames(messages: LLMMessage[]): string[] {
+  const seen = new Set<string>();
+  const result: string[] = [];
+
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (!msg.content || typeof msg.content === "string") continue;
+
+    const blocks = msg.content as Array<{ type?: string; name?: string }>;
+    for (const block of blocks) {
+      if (block.type === "tool_use" && typeof block.name === "string" && !seen.has(block.name)) {
+        result.push(block.name);
+        seen.add(block.name);
+      }
+    }
+  }
+
+  return result;
+}

packages/llm-gateway/src/server.ts

@@ -11,6 +11,7 @@ import { openAIResponsesAdapter } from "./adapters/openai-responses.js";
 import { buildResponseCacheKey } from "./cache/keys.js";
 import { FileCache } from "./cache/sqlite-cache.js";
 import { ANTHROPIC_INPUT_COST_PER_TOKEN, ANTHROPIC_OUTPUT_COST_PER_TOKEN, classifyRequest, computeShadowDecision, type ShadowDecision } from "./policy/classify.js";
+import { trimToolsForProvider } from "./policy/tool-trim.js";
 import { defaultCompatibilityRegistry, selectCompatibleProvider } from "./policy/compatibility.js";
 import {
   buildGatewayModelAliases,
@@ -449,16 +450,25 @@ async function executeRequest(params: {
   const provider = shadowMode ? "anthropic" : resolvedProvider;
   const modelOverride = shadowMode ? undefined : modelResolution.model;
 
+  // Trim tool list to the resolved provider's maximum before forwarding.
+  // Prioritizes recently-used tools from conversation history so the most
+  // relevant tools survive the cut.
+  const toolTrim = trimToolsForProvider(requestToForward, provider, params.config.routing.maxTools);
+  const requestToSend = toolTrim.trimmed > 0 ? toolTrim.request : requestToForward;
+  if (toolTrim.trimmed > 0) {
+    console.log(`[gateway] ${params.context.requestId} tool-trim: ${toolTrim.originalCount} → ${toolTrim.originalCount - toolTrim.trimmed} tools (provider=${provider} limit=${toolTrim.originalCount - toolTrim.trimmed})`);
+  }
+
   const safeResponseCacheRoute = activeRoute === "summary";
   const shouldCheckResponseCache =
     params.config.cache.enabled
     && params.config.cache.responseCache
     && safeResponseCacheRoute
-    && !requestToForward.stream
-    && (requestToForward.sampling?.temperature ?? 0) <= 0.2;
+    && !requestToSend.stream
+    && (requestToSend.sampling?.temperature ?? 0) <= 0.2;
 
   if (shouldCheckResponseCache) {
-    const responseKey = buildResponseCacheKey(requestToForward);
+    const responseKey = buildResponseCacheKey(requestToSend);
     const cachedPayload = params.cache.getResponse(responseKey);
     if (cachedPayload) {
       const cachedResponse = JSON.parse(cachedPayload) as LLMResponse;
@@ -481,11 +491,11 @@ async function executeRequest(params: {
 
   let shadow: ShadowDecision | undefined;
   if (shadowMode) {
-    shadow = computeShadowDecision(requestToForward, activeRoute, resolvedProvider);
+    shadow = computeShadowDecision(requestToSend, activeRoute, resolvedProvider);
   }
 
   const result = await routeViaProviders(
-    requestToForward,
+    requestToSend,
     activeRoute,
     provider,
     params.context.requestId,
@@ -494,7 +504,7 @@ async function executeRequest(params: {
   );
 
   if (shouldCheckResponseCache && result.response.outputText) {
-    const responseKey = buildResponseCacheKey(requestToForward);
+    const responseKey = buildResponseCacheKey(requestToSend);
     params.cache.setResponse(responseKey, JSON.stringify(result.response));
   }
 

packages/llm-gateway/src/types.ts

@@ -116,6 +116,8 @@ export interface RoutingConfig {
   shadowRoutes?: Partial<Record<RouteClass, boolean>>;
   anthropicDirectProxy?: boolean;
   routes: Record<RouteClass, string[]>;
+  /** Override per-provider tool limits. Set to a number to cap all providers at that value. */
+  maxTools?: number;
 }
 
 export interface GatewayConfig {