add kvevent online example for uds path

Author: delavet

.github/workflows/ci-release-uds-tokenizer.yaml

@@ -34,8 +34,7 @@ jobs:
           image-name: llm-d-uds-tokenizer
           registry: ghcr.io/llm-d
           github-token: ${{ secrets.GHCR_TOKEN }}
-          context: services/uds_tokenizer
-          dockerfile: services/uds_tokenizer/Dockerfile
+          dockerfile: Dockerfile.tokenizer
 
       - name: Run Trivy scan
         uses: ./.github/actions/trivy-scan

Dockerfile.uds

@@ -0,0 +1,59 @@
+# Copyright 2025 The llm-d Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build Stage: using Go 1.24.1 image
+FROM quay.io/projectquay/golang:1.24 AS builder
+ARG TARGETOS
+ARG TARGETARCH
+
+WORKDIR /workspace
+
+# Install system-level dependencies first. This layer is very stable.
+USER root
+# Install EPEL repository directly and then ZeroMQ, as epel-release is not in default repos.
+# Install all necessary dependencies including Python 3.12 for chat-completions templating.
+# The builder is based on UBI8, so we need epel-release-8.
+RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' && \
+    dnf install -y gcc-c++ libstdc++ libstdc++-devel clang zeromq-devel pkgconfig && \
+    dnf clean all
+
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the source code.
+COPY . .
+
+RUN make build-uds
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM registry.access.redhat.com/ubi9/ubi:latest
+WORKDIR /
+# Install zeromq runtime library needed by the manager.
+# The final image is UBI9, so we need epel-release-9.
+USER root
+RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' && \
+    dnf install -y zeromq libxcrypt-compat && \
+    dnf clean all
+
+# Copy the compiled Go application
+COPY --from=builder /workspace/bin/llm-d-kv-cache /app/kv-cache-manager
+USER 65532:65532
+
+# Set the entrypoint to the kv-cache-manager binary
+ENTRYPOINT ["/app/kv-cache-manager"]

Makefile

@@ -240,8 +240,8 @@ build: build-uds build-embedded ## Build both UDS-only and embedded binaries
 
 .PHONY: build-uds
 build-uds: check-go download-zmq ## Build without embedded tokenizers (no Python required)
-	@printf "\033[33;1m==== Building (UDS-only, no embedded tokenizers) ====\033[0m\n"
-	@go build ./pkg/...
+	@printf "\033[33;1m==== Building application binary (with uds tokenizers) ====\033[0m\n"
+	@go build -o bin/$(PROJECT_NAME) examples/kv_events/online_uds/main.go
 	@echo "✅ UDS-only build succeeded"
 
 .PHONY: build-embedded

examples/kv_events/online/main.go

@@ -62,12 +62,6 @@ const (
 	envExternalTokenization = "EXTERNAL_TOKENIZATION"
 )
 
-// ChatCompletionsRequest holds the fields needed for chat-completions rendering.
-type ChatCompletionsRequest struct {
-	Model string `json:"model"`
-	*types.RenderChatRequest
-}
-
 func main() {
 	baseLogger := zap.New(zap.UseDevMode(true))
 	log.SetLogger(baseLogger)
@@ -319,13 +313,20 @@ func setupUnifiedHTTPEndpoints(
 			return
 		}
 
-		var req ChatCompletionsRequest
+		var req struct {
+			Model    string               `json:"model"`
+			Messages []types.Conversation `json:"messages"`
+		}
 		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 			http.Error(w, "Invalid request body", http.StatusBadRequest)
 			return
 		}
 
-		pods, err := kvCacheIndexer.GetPodScores(ctx, req.RenderChatRequest, "", req.Model, nil)
+		renderChatReq := &types.RenderChatRequest{
+			Conversation: req.Messages,
+		}
+
+		pods, err := kvCacheIndexer.GetPodScores(ctx, renderChatReq, "", req.Model, nil)
 		if err != nil {
 			http.Error(w, fmt.Sprintf("Failed to get score request: %v", err), http.StatusInternalServerError)
 			return