add dmrlet - lightweight node agent for Docker Model Runner

dmrlet is a "Kubelet for AI" that runs inference containers directly
with zero YAML overhead. It provides a simple CLI to serve models:

  dmrlet serve ai/smollm2
  # Pulls model, starts inference container, exposes OpenAI API

Key features:
- Reuses existing pkg/distribution for model management
- containerd integration for container lifecycle
- GPU detection and passthrough (NVIDIA/AMD)
- Auto port allocation (30000-30999 range)
- Health checking with configurable timeout
- Backend auto-detection (llama-server for GGUF, vLLM for safetensors)

Commands: serve, stop, list, pull, version

Signed-off-by: Eric Curtin <eric.curtin@docker.com>

Author: ericcurtin

Makefile

@@ -26,14 +26,24 @@ DOCKER_BUILD_ARGS := \
 BUILD_DMR ?= 1
 
 # Main targets
-.PHONY: build run clean test integration-tests test-docker-ce-installation docker-build docker-build-multiplatform docker-run docker-build-vllm docker-run-vllm docker-build-sglang docker-run-sglang docker-run-impl help validate lint docker-build-diffusers docker-run-diffusers vllm-metal-build vllm-metal-install vllm-metal-dev vllm-metal-clean
+.PHONY: build build-dmrlet run clean test integration-tests test-docker-ce-installation docker-build docker-build-multiplatform docker-run docker-build-vllm docker-run-vllm docker-build-sglang docker-run-sglang docker-run-impl help validate lint docker-build-diffusers docker-run-diffusers vllm-metal-build vllm-metal-install vllm-metal-dev vllm-metal-clean
 # Default target
 .DEFAULT_GOAL := build
 
 # Build the Go application
 build:
 	CGO_ENABLED=1 go build -ldflags="-s -w" -o $(APP_NAME) .
 
+# Build dmrlet binary
+build-dmrlet:
+	@echo "Building dmrlet..."
+	@VERSION=$$(git describe --tags --always --dirty 2>/dev/null || echo "dev"); \
+	GIT_COMMIT=$$(git rev-parse HEAD 2>/dev/null || echo "unknown"); \
+	BUILD_DATE=$$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "unknown"); \
+	cd cmd/dmrlet && CGO_ENABLED=0 go build -ldflags="-s -w -X 'main.Version=$${VERSION}' -X 'main.GitCommit=$${GIT_COMMIT}' -X 'main.BuildDate=$${BUILD_DATE}'" -o dmrlet .
+	mv cmd/dmrlet/dmrlet .
+	@echo "Built: dmrlet"
+
 # Run the application locally
 run: build
 	@LLAMACPP_BIN="llamacpp/install/bin"; \
@@ -46,6 +56,7 @@ run: build
 # Clean build artifacts
 clean:
 	rm -f $(APP_NAME)
+	rm -f dmrlet
 	rm -f model-runner.sock
 	rm -rf $(MODELS_PATH)
 
@@ -219,6 +230,7 @@ vllm-metal-clean:
 help:
 	@echo "Available targets:"
 	@echo "  build				- Build the Go application"
+	@echo "  build-dmrlet			- Build dmrlet binary (lightweight node agent)"
 	@echo "  run				- Run the application locally"
 	@echo "  clean				- Clean build artifacts"
 	@echo "  test				- Run tests"

cmd/dmrlet/commands/list.go

@@ -0,0 +1,88 @@
+package commands
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/olekukonko/tablewriter"
+	"github.com/olekukonko/tablewriter/renderer"
+	"github.com/olekukonko/tablewriter/tw"
+	"github.com/spf13/cobra"
+)
+
+func newListCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "list",
+		Aliases: []string{"ls"},
+		Short:   "List running models",
+		Long: `List all running inference models managed by dmrlet.
+
+Examples:
+  dmrlet list
+  dmrlet ls`,
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runList(cmd)
+		},
+	}
+
+	return cmd
+}
+
+func runList(cmd *cobra.Command) error {
+	ctx := cmd.Context()
+
+	if err := initManager(ctx); err != nil {
+		return fmt.Errorf("initializing manager: %w", err)
+	}
+
+	running, err := manager.List(ctx)
+	if err != nil {
+		return fmt.Errorf("listing models: %w", err)
+	}
+
+	if len(running) == 0 {
+		cmd.Println("No running models")
+		return nil
+	}
+
+	table := tablewriter.NewTable(os.Stdout,
+		tablewriter.WithRenderer(renderer.NewBlueprint(tw.Rendition{
+			Borders: tw.BorderNone,
+			Settings: tw.Settings{
+				Separators: tw.Separators{
+					BetweenColumns: tw.Off,
+				},
+				Lines: tw.Lines{
+					ShowHeaderLine: tw.Off,
+				},
+			},
+		})),
+		tablewriter.WithConfig(tablewriter.Config{
+			Header: tw.CellConfig{
+				Formatting: tw.CellFormatting{
+					AutoFormat: tw.Off,
+				},
+				Alignment: tw.CellAlignment{Global: tw.AlignLeft},
+				Padding:   tw.CellPadding{Global: tw.Padding{Left: "", Right: "  "}},
+			},
+			Row: tw.CellConfig{
+				Alignment: tw.CellAlignment{Global: tw.AlignLeft},
+				Padding:   tw.CellPadding{Global: tw.Padding{Left: "", Right: "  "}},
+			},
+		}),
+	)
+	table.Header([]string{"MODEL", "BACKEND", "PORT", "ENDPOINT"})
+
+	for _, m := range running {
+		table.Append([]string{
+			m.ModelRef,
+			string(m.Backend),
+			fmt.Sprintf("%d", m.Port),
+			m.Endpoint,
+		})
+	}
+
+	table.Render()
+	return nil
+}

cmd/dmrlet/commands/pull.go

@@ -0,0 +1,44 @@
+package commands
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func newPullCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "pull MODEL",
+		Short: "Pull a model without serving",
+		Long: `Pull a model from Docker Hub or HuggingFace without starting an inference container.
+This is useful for pre-downloading models.
+
+Examples:
+  dmrlet pull ai/smollm2
+  dmrlet pull huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runPull(cmd, args[0])
+		},
+	}
+
+	return cmd
+}
+
+func runPull(cmd *cobra.Command, modelRef string) error {
+	ctx := cmd.Context()
+
+	if err := initStore(); err != nil {
+		return fmt.Errorf("initializing store: %w", err)
+	}
+
+	cmd.Printf("Pulling model: %s\n", modelRef)
+
+	if err := store.EnsureModel(ctx, modelRef, os.Stdout); err != nil {
+		return fmt.Errorf("pulling model: %w", err)
+	}
+
+	cmd.Printf("\nModel pulled successfully: %s\n", modelRef)
+	return nil
+}

cmd/dmrlet/commands/root.go

@@ -0,0 +1,146 @@
+// Package commands implements the dmrlet CLI commands.
+package commands
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	goruntime "runtime"
+	"syscall"
+
+	"github.com/docker/model-runner/pkg/dmrlet/inference"
+	"github.com/docker/model-runner/pkg/dmrlet/models"
+	"github.com/docker/model-runner/pkg/dmrlet/runtime"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var (
+	// Global flags
+	verbose bool
+	logJSON bool
+
+	// Shared state
+	log     *logrus.Entry
+	store   *models.Store
+	rt      runtime.Runner
+	manager *inference.Manager
+)
+
+// rootCmd is the root command for dmrlet.
+var rootCmd = &cobra.Command{
+	Use:   "dmrlet",
+	Short: "Lightweight node agent for Docker Model Runner",
+	Long: `dmrlet is a lightweight node agent for Docker Model Runner - a "Kubelet for AI"
+that runs inference containers directly with zero YAML overhead.
+
+Example:
+  dmrlet serve ai/smollm2
+  # Pulls model, starts inference container, exposes OpenAI API at http://localhost:30000/v1`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		// Skip initialization for help and version commands
+		if cmd.Name() == "help" || cmd.Name() == "version" {
+			return nil
+		}
+
+		// Setup logging
+		logger := logrus.New()
+		if verbose {
+			logger.SetLevel(logrus.DebugLevel)
+		} else {
+			logger.SetLevel(logrus.InfoLevel)
+		}
+		if logJSON {
+			logger.SetFormatter(&logrus.JSONFormatter{})
+		}
+
+		// Check DMRLET_LOG_LEVEL environment variable
+		if level := os.Getenv("DMRLET_LOG_LEVEL"); level != "" {
+			if lvl, err := logrus.ParseLevel(level); err == nil {
+				logger.SetLevel(lvl)
+			}
+		}
+
+		log = logger.WithField("component", "dmrlet")
+
+		return nil
+	},
+	SilenceUsage:  true,
+	SilenceErrors: true,
+}
+
+// Execute runs the root command.
+func Execute() error {
+	// Setup context with signal handling
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
+
+	return rootCmd.ExecuteContext(ctx)
+}
+
+func init() {
+	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose output")
+	rootCmd.PersistentFlags().BoolVar(&logJSON, "log-json", false, "Output logs in JSON format")
+
+	rootCmd.AddCommand(
+		newServeCmd(),
+		newStopCmd(),
+		newListCmd(),
+		newPullCmd(),
+		newVersionCmd(),
+	)
+}
+
+// initStore initializes the model store.
+func initStore() error {
+	if store != nil {
+		return nil
+	}
+
+	var err error
+	store, err = models.NewStore(
+		models.WithLogger(log),
+	)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// initRuntime initializes the runtime (native process on macOS, containerd elsewhere).
+func initRuntime(ctx context.Context) error {
+	if rt != nil {
+		return nil
+	}
+
+	var err error
+	switch goruntime.GOOS {
+	case "darwin":
+		rt = runtime.NewProcessRuntime(log)
+	default:
+		rt, err = runtime.NewRuntime(ctx,
+			runtime.WithRuntimeLogger(log),
+		)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// initManager initializes the inference manager.
+func initManager(ctx context.Context) error {
+	if err := initStore(); err != nil {
+		return err
+	}
+	if err := initRuntime(ctx); err != nil {
+		return err
+	}
+
+	if manager == nil {
+		manager = inference.NewManager(store, rt,
+			inference.WithManagerLogger(log),
+		)
+	}
+	return nil
+}