Liquid4All
diff --git a/‎.gitignore‎
Lines changed: 3 additions & 0 deletions b/‎.gitignore‎
Lines changed: 3 additions & 0 deletions
diff --git a/‎README.md‎
Lines changed: 90 additions & 3 deletions b/‎README.md‎
Lines changed: 90 additions & 3 deletions
diff --git a/‎job_configs/eval_standalone_example.yaml‎
Lines changed: 19 additions & 0 deletions b/‎job_configs/eval_standalone_example.yaml‎
Lines changed: 19 additions & 0 deletions
diff --git a/‎job_configs/sft_with_async_eval_example.yaml‎
Lines changed: 79 additions & 0 deletions b/‎job_configs/sft_with_async_eval_example.yaml‎
Lines changed: 79 additions & 0 deletions
diff --git a/‎job_configs/vlm_grpo_grounding_example.yaml‎
Lines changed: 1 addition & 1 deletion b/‎job_configs/vlm_grpo_grounding_example.yaml‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎pyproject.toml‎
Lines changed: 2 additions & 0 deletions b/‎pyproject.toml‎
Lines changed: 2 additions & 0 deletions
diff --git a/‎src/leap_finetune/__init__.py‎
Lines changed: 1 addition & 0 deletions b/‎src/leap_finetune/__init__.py‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎src/leap_finetune/cli/main.py‎
Lines changed: 55 additions & 6 deletions b/‎src/leap_finetune/cli/main.py‎
Lines changed: 55 additions & 6 deletions
@@ -41,3 +41,6 @@ scratch/
 .pytest_cache/
 .ruff_cache/
 tests/slurm/generated/
+
+# wandb local run dirs (offline logs, large + transient)
+wandb/
@@ -134,6 +134,7 @@ Useful starter configs:
 | MoE SFT                 | [`job_configs/moe_sft_example.yaml`](./job_configs/moe_sft_example.yaml)                       |
 | MoE DPO                 | [`job_configs/moe_dpo_example.yaml`](./job_configs/moe_dpo_example.yaml)                       |
 | Expert-parallel MoE SFT | [`job_configs/moe_ep_sft_example.yaml`](./job_configs/moe_ep_sft_example.yaml)                 |
+| Standalone eval         | [`job_configs/eval_standalone_example.yaml`](./job_configs/eval_standalone_example.yaml)       |
 
 ## CLI and Python Usage
 
@@ -143,6 +144,8 @@ CUDA/vLLM stack:
 ```bash
 uv run leap-finetune job_configs/sft_example.yaml
 uv run leap-finetune run job_configs/sft_example.yaml
+uv run leap-finetune job_configs/eval_standalone_example.yaml
+uv run leap-finetune eval job_configs/eval_standalone_example.yaml --output results.json
 ```
 
 Install the command as a reusable tool from a checkout:
@@ -151,6 +154,8 @@ Install the command as a reusable tool from a checkout:
 uv tool install --editable . --force
 leap-finetune /absolute/path/to/config.yaml
 leap-finetune slurm /absolute/path/to/config.yaml --output-dir /absolute/path/to/slurms
+leap-finetune /absolute/path/to/eval_config.yaml
+leap-finetune eval /absolute/path/to/eval_config.yaml --output /absolute/path/to/results.json
 ```
 
 `uv tool install` creates an isolated tool environment. Use explicit config
@@ -174,6 +179,14 @@ from leap_finetune import run_config
 run_config("/absolute/path/to/config.yaml")
 ```
 
+Standalone evals use the same entry point:
+
+```python
+from leap_finetune import run_config
+
+metrics = run_config("/absolute/path/to/eval_config.yaml")
+```
+
 Run that file inside an environment where `leap-finetune` is installed:
 
 ```bash
@@ -672,11 +685,11 @@ is simpler and faster.
 
 ## Evaluation
 
-Run benchmarks during training at every `eval_steps` by adding a `benchmarks:`
-section:
+Run benchmarks during training at every `eval_steps` by adding an `evals:`
+section. The legacy `benchmarks:` alias still parses.
 
 ```yaml
-benchmarks:
+evals:
   max_new_tokens: 128
   benchmarks:
     - name: "mmmu_val"
@@ -696,6 +709,80 @@ metrics include `short_answer`, `grounding_iou`, `mcq_gen`, and
 See the [Evaluation Guide](./src/leap_finetune/evaluation/README.md) for data
 format examples, YAML reference, and custom metrics.
 
+Run the same eval suite without training:
+
+```bash
+uv run leap-finetune job_configs/eval_standalone_example.yaml
+```
+
+Standalone eval configs use `model_name` or `checkpoint`, `evals:`, and an
+optional `backend:` block. They do not include `dataset`, `training_type`,
+`training_config`, or `async_eval`. Text evals default to `modality: text`;
+set `modality: vlm` only for standalone VLM evals.
+
+Use the explicit `eval` subcommand when you want CLI-only eval options such as
+writing metrics to JSON:
+
+```bash
+uv run leap-finetune eval job_configs/eval_standalone_example.yaml --output results.json
+```
+
+The same path is available from Python:
+
+```python
+from leap_finetune import run_config
+
+metrics = run_config("job_configs/eval_standalone_example.yaml")
+```
+
+### Async Eval (vLLM)
+
+By default, every `eval_steps` blocks training until benchmarks finish. For
+large generation suites this dominates wall-clock time. Add an `async_eval`
+block to run benchmarks **without blocking training**, using vLLM for the
+actual generation. Results are logged to wandb with `benchmark/step` and
+`train/global_step` fields so dashboards can align benchmark metrics to the
+training step that triggered them.
+
+Three modes (default is `sync` = today's behavior):
+
+| Mode       | Engine          | Pauses training? | GPUs reserved                    | Latency                   | Multi-node training  | Best for                                              |
+| ---------- | --------------- | ---------------- | -------------------------------- | ------------------------- | -------------------- | ----------------------------------------------------- |
+| `sync`     | HF transformers | Yes              | None                             | Immediate                 | ✓                    | Small/fast eval suites; default                       |
+| `sidecar`  | vLLM            | **No**           | None (slurm-scheduled per cycle) | Slurm queue + eval time   | ✓                    | Tight clusters; eval should be free of training cost  |
+| `reserved` | vLLM            | **No**           | N throughout the run             | ~30–60s respawn per cycle | **Single-node only** | Spare GPUs on one node, want predictable eval latency |
+
+`reserved` mode carves its GPUs off the same SLURM allocation as
+training via the driver's `CUDA_VISIBLE_DEVICES`, which only affects
+the head node. Multi-node training will raise `NotImplementedError`
+at startup — use `sidecar` instead, which scales to any node count.
+
+Async sidecar mode serves generation through vLLM and falls back to an HF model
+inside the sidecar for benchmark types vLLM cannot serve, such as logprob
+scoring. Reserved mode keeps a persistent vLLM server and should be used for
+generation benchmarks; use `sync` or `sidecar` for logprob suites.
+
+```yaml
+# Opt in by adding this block. See job_configs/sft_with_async_eval_example.yaml
+async_eval:
+  mode: sidecar # sync (default) | sidecar | reserved
+  vllm_gpus: 1
+  tensor_parallel_size: 1
+  gpu_memory_utilization: 0.9
+
+  # mode=sidecar: short sbatch job per eval_steps
+  sbatch:
+    time: "00:30:00"
+    # partition / account default to inheriting from the parent job
+
+  # mode=reserved: long-running vllm-serve on dedicated GPUs (single-node only for v1)
+  reserved:
+    weight_reload: respawn
+    server_port: 8100
+```
+
+Failures are isolated: if eval crashes or sbatch is rejected, training continues. After `failure.max_consecutive` consecutive failures the callback disables itself for the rest of the run. See [`job_configs/sft_with_async_eval_example.yaml`](./job_configs/sft_with_async_eval_example.yaml) for a full example.
+
 ### Post-Training Evaluation with lmms-eval
 
 For standard VLM benchmarks such as MMMU, OCRBench, RefCOCO, and POPE, use an
 
@@ -0,0 +1,19 @@
+# Standalone benchmark evaluation. No dataset or training_config is required.
+
+project_name: "standalone_eval_example"
+model_name: "LFM2-1.2B"
+
+evals:
+  max_new_tokens: 128
+  benchmarks:
+    - name: "tiny_qa"
+      path: "../tests/e2e/fixtures/tiny_qa_bench.jsonl"
+      metric: "short_answer"
+      limit: 10
+
+backend:
+  type: "hf" # hf | vllm
+  # vLLM-only settings:
+  tensor_parallel_size: 1
+  gpu_memory_utilization: 0.9
+  dtype: "bfloat16"
@@ -0,0 +1,79 @@
+# SFT with async vLLM-backed benchmark evaluation.
+#
+# Three modes are available under `async_eval`:
+#
+#   sync     - default; current behavior. Eval blocks training. Use when
+#              eval is fast enough or you want zero new infrastructure.
+#
+#   sidecar  - at each `eval_steps`, the trainer fires `sbatch` for a short
+#              eval job that runs vLLM on the latest checkpoint. Training
+#              never pauses, never reserves eval GPUs. Eval results lag by
+#              `slurm queue + eval time` and log `benchmark/step` /
+#              `train/global_step` so dashboards can align them to the
+#              originating training step. Requires running under SLURM.
+#
+#   reserved - carve N dedicated GPUs at job start for an OpenAI-compatible
+#              vLLM server. Worker rank 0 runs a helper thread that respawns
+#              the server with each new checkpoint, runs benchmarks over HTTP,
+#              and drains results to wandb. Predictable latency at the cost
+#              of N reserved GPUs throughout the run.
+#              Single-node only for v1.
+#
+# All three modes log `benchmark/<bench>/<metric>` keys with `benchmark/step`
+# so dashboards can use a consistent training-step axis.
+
+project_name: "sft_async_eval_example"
+model_name: "LFM2-1.2B"
+training_type: "sft"
+
+dataset:
+  path: "HuggingFaceTB/smoltalk"
+  type: "sft"
+  limit: 1000
+
+training_config:
+  extends: "DEFAULT_SFT"
+  num_train_epochs: 3
+  per_device_train_batch_size: 2
+  learning_rate: 2e-5
+  eval_strategy: "steps"
+  eval_steps: 200
+
+# === Evals: same shape as today's sync eval suite ===
+evals:
+  max_new_tokens: 256
+  benchmarks:
+    - name: gsm8k
+      path: openai/gsm8k
+      metric: gsm8k
+
+# === Async eval (opt-in). Pick ONE of the modes below. ===
+
+# --- Mode: sidecar (default recommendation, no GPU reservation) ---
+async_eval:
+  mode: sidecar
+  vllm_gpus: 1
+  tensor_parallel_size: 1
+  gpu_memory_utilization: 0.9
+  sbatch:
+    # Defaults to inheriting $SLURM_JOB_PARTITION / $SLURM_JOB_ACCOUNT
+    # partition: defq
+    # account: my_account
+    time: "00:30:00"
+    extra_args: []          # e.g. ["--qos=high", "--constraint=h100"]
+  on_overlap: skip          # skip (default) | queue
+  failure:
+    max_consecutive: 3
+
+# --- Mode: reserved (uncomment to use; comment out the sidecar block above) ---
+# async_eval:
+#   mode: reserved
+#   vllm_gpus: 1
+#   tensor_parallel_size: 1
+#   gpu_memory_utilization: 0.9
+#   reserved:
+#     weight_reload: respawn   # only `respawn` is implemented in v1
+#     server_port: 8100
+#   on_overlap: skip
+#   failure:
+#     max_consecutive: 3
@@ -96,7 +96,7 @@ training_config:
 # logs `benchmark/<name>/<metric>` to stdout + wandb. Each entry points at a
 # jsonl/parquet of `{messages: [user, assistant]}` samples; the last message
 # is treated as ground truth.
-benchmarks:
+evals:
   max_new_tokens: 256
   benchmarks:
     - name: "refcoco_m"
 
@@ -118,6 +118,8 @@ requires-dist = ["torch", "einops", "ninja", "packaging", "setuptools", "wheel"]
 testpaths = ["tests"]
 markers = [
     "configs: config parsing, validation, override, filtering (no GPU)",
+    "distribution: launch, resource planning, and backend integration tests (no GPU unless explicitly marked)",
+    "evaluation: benchmark, metric, and async evaluation tests",
     "rl: RL data, rewards, rollout, and environment tests",
     "dense: dense model end-to-end training tests",
     "vlm: VLM end-to-end training tests",
 
@@ -18,6 +18,7 @@
 
 TOKENIZATION_CACHE_DIR = LEAP_FINETUNE_DIR / ".cache" / "tokenized"
 
+
 __all__ = [
     "BASE_OUTPUT_PATH",
     "DPO_OUTPUT_PATH",
 
@@ -33,6 +33,20 @@ def _parse_cli_args():
             args = parser.parse_args()
             config_path_arg = args.config_path
             output_dir_arg = args.output_dir
+        elif sys.argv[1] == "eval":
+            command = "eval"
+            parser = argparse.ArgumentParser(description="Run standalone evals")
+            parser.add_argument("command", choices=["eval"])
+            parser.add_argument("config_path", help="Path to YAML eval config file")
+            parser.add_argument(
+                "--output",
+                "-o",
+                help="Optional JSON metrics output path",
+                default=None,
+            )
+            args = parser.parse_args()
+            config_path_arg = args.config_path
+            output_dir_arg = args.output
         elif sys.argv[1] == "run":
             command = "run"
             parser = argparse.ArgumentParser(description="Run training job")
@@ -68,6 +82,18 @@ def _generate_slurm_script(config_path_arg: str | None, output_dir_arg: str | No
     generate_slurm_script(config_path, config_dict, output_dir, auto_submit=False)
 
 
+def _run_standalone_eval(config_path_arg: str | None, output_path_arg: str | None):
+    if not config_path_arg:
+        print("No eval config file provided.")
+        print("Usage: leap-finetune eval <path_to_eval_config.yaml>")
+        sys.exit(1)
+
+    from leap_finetune.evaluation.runner import run_eval_config
+
+    results = run_eval_config(config_path_arg, output_path=output_path_arg)
+    print(yaml.safe_dump(results, sort_keys=True))
+
+
 def _assert_local_cuda_available() -> None:
     try:
         import torch
@@ -94,24 +120,31 @@ def check_and_handle_slurm(
     return _impl(config_path_arg, config_dict=config_dict)
 
 
-def run_config(config_path) -> None:
-    """Launch a training job from a YAML config path or typed JobConfig.
+def run_config(config_path, *, output_path: str | pathlib.Path | None = None):
+    """Launch a training job or standalone eval from a config path/model.
 
-    This is the programmatic equivalent of `leap-finetune <config>`. It keeps
-    the same backend dispatch behavior: configs with `slurm`, `kuberay`, or
-    `modal` sections submit remotely; other configs launch local Ray training.
+    This is the programmatic equivalent of `leap-finetune <config>`. Training
+    configs keep the same backend dispatch behavior: configs with `slurm`,
+    `kuberay`, or `modal` sections submit remotely; other training configs
+    launch local Ray training. Eval-only configs run benchmarks without
+    starting training.
     """
-    from leap_finetune.config import JobConfig
+    from leap_finetune.config import EvalRunConfig, JobConfig
     from leap_finetune.config.parser import (
         materialize_job_config,
         normalized_job_config_dict,
+        parse_eval_config,
         parse_job_config,
         print_job_config_summary,
     )
 
     parsed_job = None
     config_dict = None
     config_path_arg = None
+    if isinstance(config_path, EvalRunConfig):
+        from leap_finetune.evaluation.runner import run_eval_config as _run_eval_config
+
+        return _run_eval_config(config_path, output_path=output_path)
     if isinstance(config_path, JobConfig):
         parsed_job = config_path
         config_dict = normalized_job_config_dict(parsed_job)
@@ -134,6 +167,18 @@ def run_config(config_path) -> None:
     if check_and_handle_modal(config_path_arg, config_dict=config_dict):
         return
 
+    if parsed_job is None:
+        try:
+            eval_config = parse_eval_config(config_path_arg)
+        except Exception:
+            eval_config = None
+        else:
+            from leap_finetune.evaluation.runner import (
+                run_eval_config as _run_eval_config,
+            )
+
+            return _run_eval_config(eval_config, output_path=output_path)
+
     _assert_local_cuda_available()
 
     # Heavy imports deferred to here to keep remote-submit codepaths fast.
@@ -169,11 +214,15 @@ def main() -> None:
     if command == "slurm":
         _generate_slurm_script(config_path_arg, output_dir_arg)
         return
+    if command == "eval":
+        _run_standalone_eval(config_path_arg, output_dir_arg)
+        return
 
     if not config_path_arg:
         print("No config file provided. Please provide a path to a YAML config file.")
         print("Usage: leap-finetune <path_to_config.yaml>")
         print("   or: leap-finetune slurm <path_to_config.yaml>")
+        print("   or: leap-finetune eval <path_to_eval_config.yaml>")
         sys.exit(1)
 
     run_config(config_path_arg)