servebench 1.0.0__py3-none-any.whl

servebench/__init__.py

@@ -0,0 +1,21 @@
+"""servebench: load-test any OpenAI-compatible LLM endpoint."""
+
+__version__ = "1.0.0"
+
+from .api import benchmark, check_regression, compare
+from .check import CheckReport, CheckThresholds
+from .results import LevelResult, RequestResult, RunResult, Stats, Summary
+
+__all__ = [
+    "benchmark",
+    "check_regression",
+    "compare",
+    "CheckThresholds",
+    "CheckReport",
+    "RunResult",
+    "LevelResult",
+    "RequestResult",
+    "Summary",
+    "Stats",
+    "__version__",
+]

servebench/api.py

@@ -0,0 +1,189 @@
+"""Stable Python API for programmatic LLM serving benchmarks.
+
+Allows executing benchmarks, running quick capacity discovery, evaluating regressions,
+and comparing runs directly in Python scripts, test harnesses, or Jupyter notebooks.
+
+Example:
+    >>> import servebench
+    >>> res = servebench.benchmark(
+    ...     url="http://127.0.0.1:8000/v1",
+    ...     model="llama-3-8b",
+    ...     mode="concurrency",
+    ...     levels=[1, 4, 8],
+    ...     duration=10.0,
+    ... )
+    >>> peak = max(lv.summary.output_throughput for lv in res.levels)
+    >>> print(f"Peak throughput: {peak:.1f} tok/s")
+"""
+
+from __future__ import annotations
+
+import asyncio
+import platform
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+from urllib.parse import urlparse
+
+from . import __version__, detect, system
+from .check import CheckReport, CheckThresholds, evaluate_regression
+from .client import RequestSpec
+from .load import run_sweep
+from .prompts import Prompt, load_prompts
+from .report import load_run, save_run
+from .results import Environment, RunConfig, RunResult
+from .tokens import make_counter
+from .workloads import PRESETS, generate_workload_requests, load_trace_file
+
+
+def _build_env(url: str, notes: str | None = None) -> Environment:
+    server = detect.identify(url)
+    host = urlparse(url).hostname or ""
+    return Environment(
+        servebench_version=__version__,
+        python=platform.python_version(),
+        platform=f"{platform.system()} {platform.release()}",
+        timestamp=datetime.now(UTC).isoformat(timespec="seconds"),
+        local_server=host in ("127.0.0.1", "localhost", "::1", "0.0.0.0"),
+        engine=server.engine if server else None,
+        engine_version=server.version if server else None,
+        notes=notes,
+        **system.describe_for_env(),
+    )
+
+
+def benchmark(
+    url: str,
+    model: str,
+    *,
+    mode: Literal["concurrency", "rate"] = "concurrency",
+    levels: list[float] | None = None,
+    num_requests: int | None = 20,
+    duration: float | None = None,
+    input_len: int = 256,
+    output_len: int = 128,
+    preset: str | None = None,
+    burstiness: float = 1.0,
+    trace: str | Path | None = None,
+    shared_prefix_tokens: int | None = None,
+    gpu_telemetry: bool = True,
+    scrape_metrics: str | None = None,
+    endpoint: Literal["chat", "completions"] = "chat",
+    slo_ttft_ms: float | None = 500.0,
+    slo_tpot_ms: float | None = 50.0,
+    warmup: int = 2,
+    seed: int = 0,
+    timeout: float = 300.0,
+    api_key: str | None = None,
+    label: str | None = None,
+    save_path: str | Path | None = None,
+) -> RunResult:
+    """Run a serving benchmark synchronously and return a RunResult."""
+    if preset and preset in PRESETS:
+        input_len = PRESETS[preset]["input_len"]
+        output_len = PRESETS[preset]["output_len"]
+
+    if levels is None:
+        levels = [1.0, 2.0, 4.0] if mode == "concurrency" else [5.0, 10.0]
+
+    env = _build_env(url)
+    label_val = label or f"{model} ({mode})"
+
+    spec = RequestSpec(
+        url=url,
+        model=model,
+        endpoint=endpoint,
+        output_len=output_len,
+        ignore_eos=True,
+        temperature=0.0,
+        api_key=api_key,
+    )
+
+    if trace:
+        def prompts_for(n: int, _s: int) -> list[Prompt]:
+            return load_trace_file(trace, n)
+    elif shared_prefix_tokens:
+        def prompts_for(n: int, s: int) -> list[Prompt]:
+            return generate_workload_requests(
+                n,
+                input_len,
+                output_len,
+                shared_prefix_tokens=shared_prefix_tokens,
+                seed=s,
+            )
+    else:
+        def prompts_for(n: int, s: int) -> list[Prompt]:
+            return load_prompts("synthetic", n, input_len, s)
+
+    counter = make_counter(None)
+    config = RunConfig(
+        url=url,
+        model=model,
+        endpoint=endpoint,
+        input_len=input_len,
+        output_len=output_len,
+        mode=mode,
+        levels=levels,
+        num_requests=num_requests,
+        duration=duration,
+        warmup=warmup,
+        burstiness=burstiness,
+        gpu_telemetry=gpu_telemetry,
+        scrape_metrics=scrape_metrics,
+    )
+
+    levels_out = asyncio.run(
+        run_sweep(
+            spec,
+            prompts_for,
+            mode,
+            levels,
+            num_requests,
+            duration=duration,
+            warmup_requests=warmup,
+            timeout=timeout,
+            seed=seed,
+            slo_ttft_ms=slo_ttft_ms,
+            slo_tpot_ms=slo_tpot_ms,
+            counter=counter,
+            burstiness=burstiness,
+            gpu_telemetry=gpu_telemetry,
+            metrics_url=scrape_metrics,
+        )
+    )
+
+    run_res = RunResult(
+        label=label_val,
+        config=config,
+        environment=env,
+        levels=levels_out,
+    )
+
+    if save_path:
+        save_run(run_res, Path(save_path))
+
+    return run_res
+
+
+def check_regression(
+    candidate: RunResult | Path | str,
+    baseline: RunResult | Path | str,
+    thresholds: CheckThresholds | None = None,
+) -> CheckReport:
+    """Evaluate performance regressions between candidate and baseline runs."""
+    cand = load_run(Path(candidate)) if isinstance(candidate, (str, Path)) else candidate
+    base = load_run(Path(baseline)) if isinstance(baseline, (str, Path)) else baseline
+    return evaluate_regression(cand, base, thresholds)
+
+
+def compare(runs: list[RunResult | Path | str]) -> list[RunResult]:
+    """Load and normalize multiple benchmark runs for comparison."""
+    loaded: list[RunResult] = []
+    for r in runs:
+        if isinstance(r, (str, Path)):
+            loaded.append(load_run(Path(r)))
+        elif isinstance(r, RunResult):
+            loaded.append(r)
+        else:
+            raise TypeError(f"Expected RunResult, Path, or str, got {type(r)}")
+    return loaded

servebench/capacity.py

@@ -0,0 +1,104 @@
+"""Find the highest load that still meets the SLOs.
+
+Search: double the load from `start` until a level fails (or `max_value` passes), then
+binary-search between the last pass and the first fail until the gap is within `tolerance`
+of the passing value (and at least 1 for whole-number concurrency).
+
+A level passes when no request failed and at least `target` of the requests met the SLOs.
+Each level is measured once, so a noisy server near its limit can pass a level above one
+that failed; the search assumes latency grows with load.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+
+from .results import CapacityResult, LevelResult
+
+
+@dataclass
+class SearchResult:
+    capacity: float | None
+    first_fail: float | None
+    reason: str  # "converged", "max_reached" or "start_failed"
+    tried: list[tuple[float, bool]] = field(default_factory=list)
+
+
+async def search_capacity(
+    probe: Callable[[float], Awaitable[bool]],
+    *,
+    start: float,
+    max_value: float,
+    integer: bool,
+    tolerance: float = 0.1,
+) -> SearchResult:
+    """Generic search over `probe(value) -> passed`; knows nothing about HTTP."""
+    tried: list[tuple[float, bool]] = []
+
+    async def check(v: float) -> bool:
+        ok = await probe(v)
+        tried.append((v, ok))
+        return ok
+
+    lo: float | None = None
+    v = min(start, max_value)
+    while True:
+        if not await check(v):
+            hi = v
+            break
+        lo = v
+        if v >= max_value:
+            return SearchResult(lo, None, "max_reached", tried)
+        v = min(v * 2, max_value)
+    if lo is None:
+        return SearchResult(None, hi, "start_failed", tried)
+
+    min_gap = 1 if integer else 0
+    while hi - lo > max(min_gap, tolerance * lo):
+        mid = (lo + hi) // 2 if integer else (lo + hi) / 2
+        if await check(mid):
+            lo = mid
+        else:
+            hi = mid
+    return SearchResult(lo, hi, "converged", tried)
+
+
+def passes(level: LevelResult, target: float) -> bool:
+    s = level.summary
+    return s.failed == 0 and s.slo_attainment is not None and s.slo_attainment >= target
+
+
+async def find_capacity(
+    measure: Callable[[float], Awaitable[LevelResult]],
+    mode: str,
+    *,
+    start: float = 1,
+    max_value: float = 256,
+    target: float = 0.9,
+    tolerance: float = 0.1,
+    slo_ttft_ms: float | None = None,
+    slo_tpot_ms: float | None = None,
+) -> tuple[list[LevelResult], CapacityResult]:
+    """Run the search with real levels. Returns every measured level, sorted by load."""
+    if slo_ttft_ms is None and slo_tpot_ms is None:
+        raise ValueError("find-capacity needs at least one SLO (TTFT or TPOT)")
+    if not 0 < target <= 1:
+        raise ValueError("target must be in (0, 1]")
+    integer = mode == "concurrency"
+    if integer:
+        start, max_value = max(1, int(start)), max(1, int(max_value))
+    levels: list[LevelResult] = []
+
+    async def probe(v: float) -> bool:
+        level = await measure(v)
+        levels.append(level)
+        return passes(level, target)
+
+    res = await search_capacity(probe, start=start, max_value=max_value, integer=integer,
+                                tolerance=tolerance)
+    result = CapacityResult(
+        mode=mode, target=target, slo_ttft_ms=slo_ttft_ms, slo_tpot_ms=slo_tpot_ms,
+        capacity=res.capacity, first_fail=res.first_fail, reason=res.reason,
+    )
+    return sorted(levels, key=lambda lv: lv.value), result

servebench/check.py

@@ -0,0 +1,246 @@
+"""CI Performance Regression Gate (`servebench check`).
+
+Compares candidate benchmark run against a baseline run. Computes relative degradation
+percentages for TTFT p50, TPOT p50, output throughput, goodput, and error rates across
+matched load levels. Exits with code 1 if regressions exceed configured tolerance thresholds.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+from rich.console import Console
+from rich.table import Table
+
+from .results import RunResult
+
+
+class CheckThresholds(BaseModel):
+    max_ttft_pct: float = Field(
+        15.0, description="Max allowed TTFT p50 latency increase percentage"
+    )
+    max_tpot_pct: float = Field(
+        10.0, description="Max allowed TPOT p50 latency increase percentage"
+    )
+    max_throughput_pct: float = Field(
+        10.0, description="Max allowed output throughput (tok/s) drop percentage"
+    )
+    max_goodput_pct: float = Field(10.0, description="Max allowed goodput drop percentage")
+    max_error_rate_pct: float = Field(1.0, description="Max allowed request error rate percentage")
+
+
+class MetricComparison(BaseModel):
+    metric: str
+    baseline: float
+    candidate: float
+    unit: str
+    change_pct: float
+    passed: bool
+    threshold_pct: float
+
+
+class LevelComparison(BaseModel):
+    label: str
+    passed: bool
+    metrics: list[MetricComparison] = Field(default_factory=list)
+    failures: list[str] = Field(default_factory=list)
+
+
+class CheckReport(BaseModel):
+    passed: bool
+    candidate_label: str
+    baseline_label: str
+    levels: list[LevelComparison] = Field(default_factory=list)
+    failures: list[str] = Field(default_factory=list)
+
+
+def evaluate_regression(
+    candidate: RunResult,
+    baseline: RunResult,
+    thresholds: CheckThresholds | None = None,
+) -> CheckReport:
+    """Evaluate performance regressions of candidate run against baseline run."""
+    thresh = thresholds or CheckThresholds()
+
+    # Index baseline levels by label (e.g. c=1, c=4, rate=10)
+    baseline_by_label = {lv.label: lv for lv in baseline.levels}
+    level_comparisons: list[LevelComparison] = []
+    overall_failures: list[str] = []
+
+    for c_lv in candidate.levels:
+        b_lv = baseline_by_label.get(c_lv.label)
+        if not b_lv:
+            # If no direct matching level, skip
+            continue
+
+        c_sum = c_lv.summary
+        b_sum = b_lv.summary
+
+        metrics: list[MetricComparison] = []
+        level_failures: list[str] = []
+
+        # 1. TTFT p50 (lower is better; positive diff = latency degradation)
+        b_ttft = (b_sum.ttft.p50 or 0.0) * 1000.0
+        c_ttft = (c_sum.ttft.p50 or 0.0) * 1000.0
+        if b_ttft > 0:
+            ttft_diff_pct = ((c_ttft - b_ttft) / b_ttft) * 100.0
+            ttft_pass = ttft_diff_pct <= thresh.max_ttft_pct
+            metrics.append(
+                MetricComparison(
+                    metric="TTFT p50",
+                    baseline=round(b_ttft, 1),
+                    candidate=round(c_ttft, 1),
+                    unit="ms",
+                    change_pct=round(ttft_diff_pct, 1),
+                    passed=ttft_pass,
+                    threshold_pct=thresh.max_ttft_pct,
+                )
+            )
+            if not ttft_pass:
+                fail_msg = (
+                    f"[{c_lv.label}] TTFT p50 degraded by +{ttft_diff_pct:.1f}% "
+                    f"({b_ttft:.1f}ms -> {c_ttft:.1f}ms, limit +{thresh.max_ttft_pct:.1f}%)"
+                )
+                level_failures.append(fail_msg)
+                overall_failures.append(fail_msg)
+
+        # 2. TPOT p50 (lower is better; positive diff = latency degradation)
+        b_tpot = (b_sum.tpot.p50 or 0.0) * 1000.0
+        c_tpot = (c_sum.tpot.p50 or 0.0) * 1000.0
+        if b_tpot > 0:
+            tpot_diff_pct = ((c_tpot - b_tpot) / b_tpot) * 100.0
+            tpot_pass = tpot_diff_pct <= thresh.max_tpot_pct
+            metrics.append(
+                MetricComparison(
+                    metric="TPOT p50",
+                    baseline=round(b_tpot, 2),
+                    candidate=round(c_tpot, 2),
+                    unit="ms",
+                    change_pct=round(tpot_diff_pct, 1),
+                    passed=tpot_pass,
+                    threshold_pct=thresh.max_tpot_pct,
+                )
+            )
+            if not tpot_pass:
+                fail_msg = (
+                    f"[{c_lv.label}] TPOT p50 degraded by +{tpot_diff_pct:.1f}% "
+                    f"({b_tpot:.2f}ms -> {c_tpot:.2f}ms, limit +{thresh.max_tpot_pct:.1f}%)"
+                )
+                level_failures.append(fail_msg)
+                overall_failures.append(fail_msg)
+
+        # 3. Output Throughput (higher is better; positive diff = throughput drop)
+        b_tok_s = b_sum.output_throughput
+        c_tok_s = c_sum.output_throughput
+        if b_tok_s > 0:
+            tok_drop_pct = ((b_tok_s - c_tok_s) / b_tok_s) * 100.0
+            tok_pass = tok_drop_pct <= thresh.max_throughput_pct
+            metrics.append(
+                MetricComparison(
+                    metric="Throughput",
+                    baseline=round(b_tok_s, 1),
+                    candidate=round(c_tok_s, 1),
+                    unit="tok/s",
+                    change_pct=round(-tok_drop_pct, 1),  # negative indicates drop
+                    passed=tok_pass,
+                    threshold_pct=thresh.max_throughput_pct,
+                )
+            )
+            if not tok_pass:
+                limit_s = f"-{thresh.max_throughput_pct:.1f}%"
+                fail_msg = (
+                    f"[{c_lv.label}] Output throughput dropped by {tok_drop_pct:.1f}% "
+                    f"({b_tok_s:.1f} -> {c_tok_s:.1f} tok/s, limit {limit_s})"
+                )
+                level_failures.append(fail_msg)
+                overall_failures.append(fail_msg)
+
+        # 4. Error Rate
+        total_reqs = len(c_lv.requests)
+        failed_reqs = sum(1 for r in c_lv.requests if not r.ok)
+        error_rate_pct = (failed_reqs / total_reqs * 100.0) if total_reqs > 0 else 0.0
+        err_pass = error_rate_pct <= thresh.max_error_rate_pct
+        metrics.append(
+            MetricComparison(
+                metric="Error Rate",
+                baseline=0.0,
+                candidate=round(error_rate_pct, 2),
+                unit="%",
+                change_pct=round(error_rate_pct, 2),
+                passed=err_pass,
+                threshold_pct=thresh.max_error_rate_pct,
+            )
+        )
+        if not err_pass:
+            fail_msg = (
+                f"[{c_lv.label}] Request error rate {error_rate_pct:.1f}% exceeded "
+                f"maximum threshold of {thresh.max_error_rate_pct:.1f}%"
+            )
+            level_failures.append(fail_msg)
+            overall_failures.append(fail_msg)
+
+        level_comparisons.append(
+            LevelComparison(
+                label=c_lv.label,
+                passed=len(level_failures) == 0,
+                metrics=metrics,
+                failures=level_failures,
+            )
+        )
+
+    all_passed = len(overall_failures) == 0 and len(level_comparisons) > 0
+    return CheckReport(
+        passed=all_passed,
+        candidate_label=candidate.label,
+        baseline_label=baseline.label,
+        levels=level_comparisons,
+        failures=overall_failures,
+    )
+
+
+def print_check_report(report: CheckReport, console: Console | None = None) -> None:
+    """Render check regression table to terminal."""
+    con = console or Console()
+
+    status_str = (
+        "[bold green]PASSED[/bold green]" if report.passed else "[bold red]FAILED[/bold red]"
+    )
+    tbl = Table(
+        title=f"CI Performance Regression Gate: {status_str}",
+        box=None,
+        pad_edge=False,
+        header_style="bold underline",
+    )
+    tbl.add_column("Level", justify="left")
+    tbl.add_column("Metric", justify="left")
+    tbl.add_column("Baseline", justify="right")
+    tbl.add_column("Candidate", justify="right")
+    tbl.add_column("Change", justify="right")
+    tbl.add_column("Status", justify="center")
+
+    for lv in report.levels:
+        for m in lv.metrics:
+            if m.metric in ("TTFT p50", "TPOT p50"):
+                sign = "+" if m.change_pct > 0 else ""
+                diff_str = f"{sign}{m.change_pct:.1f}%"
+            elif m.metric == "Throughput":
+                diff_str = f"{m.change_pct:+.1f}%"
+            else:
+                diff_str = f"{m.candidate:.1f}%"
+
+            stat = "[green]PASS[/green]" if m.passed else "[red]FAIL[/red]"
+            base_str = f"{m.baseline} {m.unit}"
+            cand_str = f"{m.candidate} {m.unit}"
+            tbl.add_row(lv.label, m.metric, base_str, cand_str, diff_str, stat)
+
+    con.print(tbl)
+    con.print()
+
+    if report.failures:
+        con.print("[bold red]Performance Regressions Detected:[/bold red]")
+        for f in report.failures:
+            con.print(f"  [red]✗ {f}[/red]")
+    else:
+        con.print(
+            "[bold green]✓ All performance metrics are within specified "
+            "regression tolerances.[/bold green]"
+        )