vllm-optimizer 0.1.0a11__py3-none-any.whl

vllm_optimizer/__init__.py

@@ -0,0 +1,7 @@
+"""vLLM Optimizer package."""
+
+__version__ = "0.1.0a11"
+
+from .orchestrator import Orchestrator, RunOutcome
+
+__all__ = ["Orchestrator", "RunOutcome"]

vllm_optimizer/benchmarks/__init__.py

@@ -0,0 +1,13 @@
+"""Benchmark backend adapters."""
+
+from .configuration import (
+    configured_engine, configured_min_repeats, configured_repeats,
+    configured_runs, configured_warmup_repeats,
+)
+from .guidellm import GuideLLMPlan, build_plan, parse_result
+
+__all__ = [
+    "GuideLLMPlan", "build_plan", "configured_engine", "configured_min_repeats",
+    "configured_repeats", "configured_runs", "configured_warmup_repeats",
+    "parse_result",
+]

vllm_optimizer/benchmarks/configuration.py

@@ -0,0 +1,84 @@
+"""Shared benchmark configuration validation."""
+
+from collections.abc import Mapping
+import re
+
+from vllm_optimizer.config.models import VTuneConfig
+
+_RUN_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$")
+_ENGINES = {"guidellm", "vllm"}
+
+
+def configured_engine(config: VTuneConfig) -> str:
+    value = config.benchmark.get("engine", "guidellm")
+    if not isinstance(value, str) or value not in _ENGINES:
+        raise ValueError("benchmark.engine must be 'guidellm' or 'vllm'")
+    return value
+
+
+def configured_repeats(config: VTuneConfig) -> int:
+    value = config.benchmark.get("repeats", 1)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise ValueError("benchmark.repeats must be a positive integer")
+    return value
+
+
+def configured_warmup_repeats(config: VTuneConfig) -> int:
+    value = config.benchmark.get("warmup_repeats", 0)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        raise ValueError("benchmark.warmup_repeats must be a non-negative integer")
+    return value
+
+
+def configured_min_repeats(config: VTuneConfig) -> int:
+    value = config.benchmark.get("min_repeats", 1)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise ValueError("benchmark.min_repeats must be a positive integer")
+    if value > configured_repeats(config):
+        raise ValueError("benchmark.min_repeats cannot exceed benchmark.repeats")
+    return value
+
+
+def configured_runs(config: VTuneConfig) -> tuple[Mapping[str, object], ...]:
+    unknown = set(config.benchmark) - {
+        "engine", "runs", "repeats", "warmup_repeats", "min_repeats",
+    }
+    if unknown:
+        raise ValueError(f"Unsupported benchmark setting(s): {', '.join(sorted(unknown))}")
+    values = config.benchmark.get("runs")
+    if not isinstance(values, list) or not values:
+        raise ValueError("'benchmark.runs' must be a non-empty list")
+    engine, names = configured_engine(config), set()
+    runs: list[Mapping[str, object]] = []
+    for index, value in enumerate(values):
+        run = _mapping(value, f"run {index}")
+        name = run.get("name")
+        if not isinstance(name, str) or not _RUN_NAME.fullmatch(name):
+            raise ValueError("benchmark run names must use letters, numbers, '_' or '-'")
+        if name in names:
+            raise ValueError(f"duplicate benchmark run name: {name}")
+        names.add(name)
+        _validate_run(engine, run, index, name)
+        runs.append(run)
+    return tuple(runs)
+
+
+def _validate_run(engine: str, run: Mapping[str, object], index: int, name: str) -> None:
+    allowed = ({"name", "request_format", "profile", "constraints", "data"}
+               if engine == "guidellm" else {"name", "args"})
+    if unknown := set(run) - allowed:
+        raise ValueError(
+            f"Unsupported setting(s) in benchmark run {index}: {', '.join(sorted(unknown))}"
+        )
+    if engine == "guidellm":
+        data = run.get("data")
+        if not isinstance(data, list) or len(data) != 1:
+            raise ValueError(f"benchmark run '{name}' must configure exactly one dataset")
+    else:
+        _mapping(run.get("args", {}), f"vllm benchmark run '{name}' args")
+
+
+def _mapping(value: object, label: str) -> Mapping[str, object]:
+    if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
+        raise ValueError(f"{label} must be an object")
+    return value

vllm_optimizer/benchmarks/guidellm.py

@@ -0,0 +1,126 @@
+"""GuideLLM configuration, command construction, and result normalization."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+import json
+from pathlib import Path
+
+from vllm_optimizer.config.models import VTuneConfig
+from vllm_optimizer.config.runtime import model_path
+from vllm_optimizer.domain.benchmark import BenchmarkResult, WorkloadResult
+from vllm_optimizer.benchmarks.timing import normalize_durations
+from vllm_optimizer.benchmarks.metrics import normalize_guidellm_metrics
+
+@dataclass(frozen=True, slots=True)
+class GuideLLMPlan:
+    run_name: str
+    argv: tuple[str, ...]
+    directory: Path
+    json_path: Path
+    log_path: Path
+
+
+def configured_runs(config: VTuneConfig) -> tuple[Mapping[str, object], ...]:
+    from vllm_optimizer.benchmarks.configuration import configured_runs as validate
+    return validate(config)
+
+
+def configured_repeats(config: VTuneConfig) -> int:
+    from vllm_optimizer.benchmarks.configuration import configured_repeats as validate
+    return validate(config)
+
+
+def build_plan(
+    config: VTuneConfig, run: Mapping[str, object], endpoint: str, artifacts: Path,
+) -> GuideLLMPlan:
+    name = run["name"]
+    request_format = run.get("request_format", "/v1/completions")
+    if not isinstance(request_format, str) or not request_format.strip():
+        raise ValueError("benchmark request_format must be a non-empty string")
+    directory = Path(artifacts) / str(name)
+    json_path = directory / "results.json"
+    argv = [
+        "guidellm", "run", "--backend",
+        _serialize({"kind": "openai_http", "target": endpoint,
+                    "model": model_path(config), "request_format": request_format}),
+        "--profile", _serialize(_mapping(run.get("profile"), "profile")),
+    ]
+    for option, label in (("constraints", "constraint"), ("data", "data")):
+        values = run.get(option, [])
+        if not isinstance(values, list):
+            raise ValueError(f"benchmark run '{option}' must be a list")
+        for value in values:
+            if label == "constraint" and isinstance(value, dict) and value.get("kind") == "max_duration":
+                value = {**value, "seconds": normalize_durations(value.get("seconds"))}
+            argv.extend((f"--{label}", _serialize(_mapping(value, label))))
+    argv.extend(("--output", f"kind=json,path={json_path}"))
+    return GuideLLMPlan(str(name), tuple(argv), directory, json_path,
+                        directory / "benchmark.log")
+
+
+def parse_result(path: Path, run_name: str) -> BenchmarkResult:
+    source = Path(path)
+    try:
+        root = _mapping(json.loads(source.read_text(encoding="utf-8")), "result")
+    except (OSError, UnicodeError, json.JSONDecodeError) as error:
+        raise ValueError(f"Cannot read GuideLLM JSON result: {error}") from error
+    metadata = _mapping(root.get("metadata"), "metadata")
+    version = metadata.get("guidellm_version")
+    if not isinstance(version, str) or not version.strip():
+        raise ValueError("GuideLLM result is missing its version")
+    benchmarks = root.get("benchmarks")
+    if not isinstance(benchmarks, list) or not benchmarks:
+        raise ValueError("GuideLLM result must contain benchmarks")
+    workloads = tuple(
+        WorkloadResult(
+            index, _mapping(item.get("config"), f"benchmark {index} config"),
+            _normalized_metrics(_mapping(item.get("metrics"), f"benchmark {index} metrics")),
+        )
+        for index, value in enumerate(benchmarks)
+        for item in (_mapping(value, f"benchmark {index}"),)
+    )
+    return BenchmarkResult(run_name, "guidellm", version, workloads, source)
+
+
+def _normalized_metrics(raw: Mapping[str, object]) -> dict[str, object]:
+    metrics = normalize_guidellm_metrics(raw)
+    totals = raw.get("request_totals")
+    if (isinstance(totals, Mapping) and isinstance(totals.get("total"), int)
+            and not isinstance(totals.get("total"), bool)):
+        metrics["request_total"] = totals["total"]
+    return metrics
+
+
+def _serialize(values: Mapping[str, object]) -> str:
+    kind = values.get("kind")
+    if not isinstance(kind, str) or not kind.strip():
+        raise ValueError("GuideLLM options require a non-empty 'kind'")
+    return ",".join(pair for key, value in values.items() for pair in _pairs(key, value))
+
+
+def _pairs(path: str, value: object) -> tuple[str, ...]:
+    if isinstance(value, Mapping):
+        return tuple(pair for key, item in value.items()
+                     for pair in _pairs(f"{path}.{key}", item))
+    if isinstance(value, list | tuple):
+        return tuple(pair for index, item in enumerate(value)
+                     for pair in _pairs(f"{path}[{index}]", item))
+    if isinstance(value, bool):
+        rendered = str(value).lower()
+    if isinstance(value, str | int | float) and not isinstance(value, bool):
+        rendered = str(value)
+    elif value is None:
+        rendered = "null"
+    elif not isinstance(value, bool):
+        raise ValueError(f"Unsupported GuideLLM value at '{path}'")
+    if "," in rendered:
+        raise ValueError(f"GuideLLM string values cannot contain commas: '{path}'")
+    return (f"{path}={rendered}",)
+
+
+def _mapping(value: object, label: str) -> Mapping[str, object]:
+    if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
+        raise ValueError(f"GuideLLM {label} must be an object")
+    return value

vllm_optimizer/benchmarks/metrics.py

@@ -0,0 +1,122 @@
+"""Canonical metric normalization shared by benchmark adapters."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+
+_GUIDELLM_METRICS = {
+    "requests_per_second": "requests_per_second",
+    "output_tokens_per_second": "output_tokens_per_second",
+    "total_tokens_per_second": "tokens_per_second",
+    "time_to_first_token_ms": "time_to_first_token_ms",
+    "time_per_output_token_ms": "time_per_output_token_ms",
+    "inter_token_latency_ms": "inter_token_latency_ms",
+}
+_VLLM_THROUGHPUT = {
+    "requests_per_second": ("requests_per_second", "request_throughput"),
+    "output_tokens_per_second": ("output_tokens_per_second", "output_throughput"),
+    "total_tokens_per_second": ("total_tokens_per_second", "total_token_throughput"),
+}
+_VLLM_LATENCY = {
+    "time_to_first_token_ms": "ttft",
+    "time_per_output_token_ms": "tpot",
+    "inter_token_latency_ms": "itl",
+    "end_to_end_latency_ms": "e2el",
+}
+
+
+def normalize_guidellm_metrics(raw: Mapping[str, object]) -> dict[str, object]:
+    """Return GuideLLM metrics with backend-neutral canonical fields."""
+    result = dict(raw)
+    for canonical, source in _GUIDELLM_METRICS.items():
+        if summary := metric_summary(raw.get(source)):
+            result[canonical] = summary
+    if summary := metric_summary(raw.get("request_latency"), scale=1000.0):
+        result["end_to_end_latency_ms"] = summary
+    result["request_totals"] = request_totals(raw.get("request_totals"))
+    return result
+
+
+def normalize_vllm_metrics(raw: Mapping[str, object]) -> dict[str, object]:
+    """Return vLLM Bench Serve metrics with the same canonical fields."""
+    result = dict(raw)
+    for canonical, aliases in _VLLM_THROUGHPUT.items():
+        value = next((raw[name] for name in aliases if name in raw), None)
+        if summary := metric_summary(value):
+            result[canonical] = summary
+    for canonical, stem in _VLLM_LATENCY.items():
+        if summary := _vllm_latency(raw, stem):
+            result[canonical] = summary
+    result["request_totals"] = request_totals(
+        raw.get("request_totals"), completed=raw.get("completed"),
+        failed=raw.get("failed"), requested=raw.get("num_prompts"),
+    )
+    return result
+
+
+def metric_summary(value: object, scale: float = 1.0) -> dict[str, float]:
+    """Extract average, median, and P99 without fabricating absent values."""
+    if (number := _number(value)) is not None:
+        return {"average": number * scale}
+    if not isinstance(value, Mapping):
+        return {}
+    successful = value.get("successful")
+    source = successful if isinstance(successful, Mapping) else value
+    result: dict[str, float] = {}
+    for target, names in {
+        "average": ("average", "mean"),
+        "median": ("median", "p50"),
+        "p99": ("p99",),
+    }.items():
+        number = next((_number(source.get(name)) for name in names
+                       if _number(source.get(name)) is not None), None)
+        if number is not None:
+            result[target] = number * scale
+    percentiles = source.get("percentiles")
+    if isinstance(percentiles, Mapping):
+        for target, name in (("median", "p50"), ("p99", "p99")):
+            if target not in result and (number := _number(percentiles.get(name))) is not None:
+                result[target] = number * scale
+    return result
+
+
+def request_totals(
+    value: object, *, completed: object = None, failed: object = None,
+    requested: object = None,
+) -> dict[str, int]:
+    source = value if isinstance(value, Mapping) else {}
+    successful = _count(source.get("successful", completed))
+    errored = _count(source.get("errored", source.get("failed", failed)))
+    incomplete = _count(source.get("incomplete"))
+    result = {
+        "successful": successful,
+        "errored": errored,
+        "incomplete": incomplete,
+    }
+    total = _optional_count(source.get("total", requested))
+    if total is not None:
+        result["incomplete"] = max(incomplete, total - successful - errored)
+    return result
+
+
+def _vllm_latency(raw: Mapping[str, object], stem: str) -> dict[str, float]:
+    values = {
+        "average": raw.get(f"mean_{stem}_ms"),
+        "median": raw.get(f"median_{stem}_ms", raw.get(f"p50_{stem}_ms")),
+        "p99": raw.get(f"p99_{stem}_ms"),
+    }
+    return {name: number for name, value in values.items()
+            if (number := _number(value)) is not None}
+
+
+def _number(value: object) -> float | None:
+    return float(value) if isinstance(value, int | float) and not isinstance(value, bool) else None
+
+
+def _count(value: object) -> int:
+    return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0
+
+
+def _optional_count(value: object) -> int | None:
+    return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None

vllm_optimizer/benchmarks/timing.py

@@ -0,0 +1,77 @@
+"""Duration parsing and GuideLLM timeout estimation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+import re
+
+_DURATION = re.compile(r"^(\d+(?:\.\d+)?)\s*([smh]?)$")
+_UNITS = {"": 1, "s": 1, "m": 60, "h": 3600}
+_REQUEST_COUNT_TIMEOUT = 3600.0
+
+
+def parse_duration(value: object, label: str = "duration") -> float:
+    if isinstance(value, bool):
+        raise ValueError(f"'{label}' must be a positive duration")
+    if isinstance(value, int | float):
+        seconds = float(value)
+    elif isinstance(value, str) and (match := _DURATION.fullmatch(value.strip().lower())):
+        seconds = float(match.group(1)) * _UNITS[match.group(2)]
+    else:
+        raise ValueError(f"'{label}' must use seconds, '30s', '2m', or '1h'")
+    if seconds <= 0:
+        raise ValueError(f"'{label}' must be positive")
+    return seconds
+
+
+def timeout_for_run(run: Mapping[str, object], configured: object = None) -> float:
+    """Return an explicit timeout or estimate a configured duration safely."""
+    if configured is not None:
+        return parse_duration(configured, "timeouts.benchmark")
+    if _has_request_constraint(run) and _duration_constraint(run) is None:
+        return _REQUEST_COUNT_TIMEOUT
+    duration = _duration_constraint(run)
+    if duration is None:
+        return 180.0
+    expected = duration * _strategy_count(run.get("profile"))
+    return expected + max(30.0, expected * 0.25)
+
+
+def normalize_durations(value: object) -> object:
+    """Convert duration strings recursively before passing values to GuideLLM."""
+    if isinstance(value, list):
+        return [normalize_durations(item) for item in value]
+    if isinstance(value, str):
+        return parse_duration(value)
+    return value
+
+
+def _duration_constraint(run: Mapping[str, object]) -> float | None:
+    constraints = run.get("constraints", [])
+    if not isinstance(constraints, list):
+        return None
+    for constraint in constraints:
+        if isinstance(constraint, Mapping) and constraint.get("kind") == "max_duration":
+            seconds = constraint.get("seconds")
+            if isinstance(seconds, list):
+                return sum(parse_duration(item, "max_duration.seconds") for item in seconds)
+            return parse_duration(seconds, "max_duration.seconds")
+    return None
+
+
+def _has_request_constraint(run: Mapping[str, object]) -> bool:
+    constraints = run.get("constraints", [])
+    return (isinstance(constraints, list)
+            and any(isinstance(item, Mapping) and item.get("kind") == "max_requests"
+                    for item in constraints))
+
+
+def _strategy_count(profile: object) -> int:
+    if not isinstance(profile, Mapping):
+        return 1
+    for key in ("streams", "rates"):
+        value = profile.get(key)
+        if isinstance(value, list):
+            return max(1, len(value))
+    size = profile.get("sweep_size")
+    return size if isinstance(size, int) and size > 0 else 1

vllm_optimizer/benchmarks/vllm.py

@@ -0,0 +1,92 @@
+"""vLLM Bench Serve command construction and result normalization."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from importlib.metadata import PackageNotFoundError, version
+import json
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from vllm_optimizer.config.models import VTuneConfig
+from vllm_optimizer.config.runtime import model_path
+from vllm_optimizer.domain.benchmark import BenchmarkResult, WorkloadResult
+from vllm_optimizer.benchmarks.metrics import normalize_vllm_metrics
+
+_RESERVED = {"backend", "model", "host", "port", "base-url", "save-result", "append-result",
+             "result-dir", "result-filename"}
+
+
+@dataclass(frozen=True, slots=True)
+class VLLMBenchPlan:
+    run_name: str
+    argv: tuple[str, ...]
+    directory: Path
+    json_path: Path
+    log_path: Path
+
+
+def build_plan(config: VTuneConfig, run: Mapping[str, object], endpoint: str,
+               artifacts: Path) -> VLLMBenchPlan:
+    name = str(run["name"])
+    args = _mapping(run.get("args", {}), f"vllm benchmark run '{name}' args")
+    normalized = {key.replace("_", "-"): value for key, value in args.items()}
+    if protected := _RESERVED.intersection(normalized):
+        raise ValueError(
+            "vLLM Optimizer controls vLLM benchmark argument(s): "
+            f"{', '.join(sorted(protected))}"
+        )
+    parsed = urlsplit(endpoint)
+    directory = Path(artifacts) / name
+    result = directory / "results.json"
+    argv = ["vllm", "bench", "serve", "--backend", "vllm",
+            "--model", model_path(config),
+            "--host", parsed.hostname or "127.0.0.1", "--port", str(parsed.port or 8000)]
+    for key, value in normalized.items():
+        argv.extend(_argument(key, value))
+    argv.extend(("--save-result", "--result-dir", str(directory),
+                 "--result-filename", result.name, "--disable-tqdm"))
+    return VLLMBenchPlan(name, tuple(argv), directory, result,
+                         directory / "benchmark.log")
+
+
+def parse_result(path: Path, run_name: str) -> BenchmarkResult:
+    source = Path(path)
+    try:
+        document = _mapping(json.loads(source.read_text(encoding="utf-8")), "result")
+    except (OSError, UnicodeError, json.JSONDecodeError) as error:
+        raise ValueError(f"Cannot read vLLM benchmark JSON result: {error}") from error
+    metrics = normalize_vllm_metrics(document)
+    if isinstance(document.get("num_prompts"), int) and not isinstance(document.get("num_prompts"), bool):
+        metrics["request_total"] = document["num_prompts"]
+    configuration = {key: document[key] for key in (
+        "backend", "model_id", "num_prompts", "request_rate", "burstiness",
+        "max_concurrency", "dataset_name",
+    ) if key in document}
+    return BenchmarkResult(run_name, "vllm", _version(),
+                           (WorkloadResult(0, configuration, metrics),), source)
+
+
+def _argument(name: str, value: object) -> tuple[str, ...]:
+    flag = f"--{name}"
+    if isinstance(value, bool):
+        return (flag,) if value else ()
+    if isinstance(value, list):
+        return tuple(item for entry in value for item in (flag, str(entry)))
+    if value is None or not isinstance(value, str | int | float):
+        raise ValueError(f"Unsupported vLLM benchmark value for '{name}'")
+    return flag, str(value)
+
+
+def _version() -> str:
+    try:
+        return version("vllm")
+    except PackageNotFoundError:
+        return "unknown"
+
+
+def _mapping(value: object, label: str) -> Mapping[str, object]:
+    if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
+        raise ValueError(f"{label} must be an object")
+    return value

vllm_optimizer/cli.py

@@ -0,0 +1,117 @@
+"""Command-line entry points for vLLM Optimizer."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+from collections.abc import Sequence
+from pathlib import Path
+import sys
+
+from vllm_optimizer.config.errors import ConfigError
+from vllm_optimizer.config.loader import load_config
+from vllm_optimizer.config.runtime import model_path
+from vllm_optimizer.cli_options import CLIUsageError, validate_cli_options
+from vllm_optimizer.lifecycle import load_retry_plan
+from vllm_optimizer.orchestrator import Orchestrator
+from vllm_optimizer.reporting.offline import regenerate_report
+from vllm_optimizer.reproduction.display import reproduce_trial
+from vllm_optimizer.reproduction.export import export_vllm_command
+from vllm_optimizer.terminal import with_debug_logging
+from vllm_optimizer.terminal_style import styled
+
+
+def build_parser(prog: str = "vllm-opt") -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=prog, description="Benchmark and optimize vLLM serving configurations.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=f"""Commands:
+  {prog} --config experiment.yaml                 Start an experiment
+  {prog} validate --config experiment.yaml        Validate configuration only
+  {prog} retry --run RUN --trial ID [--trial ID]  Retry selected trial(s)
+  {prog} reproduce --run RUN --trial ID           Print a saved trial command
+  {prog} export --run RUN --trial ID              Export a vLLM serve command
+  {prog} report --run RUN [--output DIRECTORY]    Regenerate an HTML report""",
+    )
+    parser.add_argument(
+        "action", nargs="?", choices=("validate", "export", "reproduce", "retry", "report"),
+        help="Post-run action; omit to start a new experiment.")
+    parser.add_argument("--config", "-c", metavar="YAML",
+                        help="Experiment YAML for a new run or validation.")
+    parser.add_argument("--run", type=Path, metavar="DIRECTORY",
+                        help="Existing immutable run directory.")
+    parser.add_argument("--trial", action="append", metavar="ID",
+                        help="Trial ID; repeat the option when retrying several trials.")
+    parser.add_argument("--output", type=Path, metavar="DIRECTORY",
+                        help="Destination for an offline regenerated report.")
+    parser.add_argument("--verbose", action="store_true",
+                        help="Override logging.level with DEBUG and stream child logs.")
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    return _run(argv, "vllm-opt")
+
+
+def legacy_main(argv: Sequence[str] | None = None) -> int:
+    print(
+        "The 'vtune' command is deprecated; use 'vllm-opt' instead.",
+        file=sys.stderr,
+    )
+    return _run(argv, "vtune")
+
+
+def _run(argv: Sequence[str] | None, prog: str) -> int:
+    args = build_parser(prog).parse_args(argv)
+    try:
+        validate_cli_options(args)
+        if args.action == "export":
+            print(export_vllm_command(args.run, args.trial[0]))
+            return 0
+        if args.action == "reproduce":
+            print(reproduce_trial(args.run, args.trial[0]))
+            return 0
+        if args.action == "retry":
+            plan = load_retry_plan(args.run, args.trial)
+            for warning in plan.warnings:
+                print(f"Integrity warning: {warning}", file=sys.stderr)
+            retry_config = with_debug_logging(plan.config) if args.verbose else plan.config
+            outcome = asyncio.run(Orchestrator(
+                retry_config, plan.trials, plan.source_run_id, plan.sources,
+            ).run())
+        elif args.action == "report":
+            generated = regenerate_report(args.run, args.output)
+            print(f"Offline report regenerated: {generated.directory}")
+            for warning in generated.warnings:
+                print(f"Integrity warning: {warning}", file=sys.stderr)
+            return 0
+        else:
+            config = load_config(args.config)
+            config = with_debug_logging(config) if args.verbose else config
+            if args.action == "validate":
+                Orchestrator(config).validate()
+                print(f"Configuration valid: {config.experiment.name}")
+                print(f"Model: {model_path(config)}")
+                return 0
+            outcome = asyncio.run(Orchestrator(config).run())
+    except CLIUsageError as error:
+        print(styled(f"Command error: {error}", "red", sys.stderr), file=sys.stderr)
+        return 2
+    except ConfigError as error:
+        print(styled(f"Configuration error: {error}", "red", sys.stderr), file=sys.stderr)
+        return 2
+    except (OSError, TypeError, ValueError) as error:
+        print(styled(f"Experiment error: {error}", "red", sys.stderr), file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        print("Experiment interrupted", file=sys.stderr)
+        return 130
+    print(outcome.summary)
+    if outcome.status == "interrupted":
+        return 130
+    completed = any(report.status.value == "completed" for report in outcome.trials)
+    return 0 if completed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())