gbench 0.5.0__py3-none-any.whl

gbench/__init__.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""gbench - Open Model Performance Benchmark Suite.
+
+A production-grade, modular benchmarking tool for comprehensive evaluation
+of open models served by vLLM or any OpenAI-compatible endpoint (text and multimodal).
+"""
+
+__version__ = "0.5.0"
+__author__ = "Luciano Martins"
+
+import os
+import re
+import sys
+
+# Process-wide Gemini API key multi-key support & auto-sanitization.
+# GEMINI_API_KEYS (a comma/semicolon list) wins when both it and GEMINI_API_KEY are set, as in
+# base.gemini_api_keys(); otherwise GEMINI_API_KEY (one key or a list) is used. The chosen list is
+# written back to GEMINI_API_KEYS, and GEMINI_API_KEY is set to its first key for third-party
+# libraries (LiteLLM, OpenAI SDK) that read a single key.
+def _split_gemini_keys(raw: str) -> list:
+    return [k.strip() for k in re.split(r"[;,]", raw or "") if k.strip()]
+
+
+_GEMINI_KEYS = (_split_gemini_keys(os.environ.get("GEMINI_API_KEYS", ""))
+                or _split_gemini_keys(os.environ.get("GEMINI_API_KEY", "")))
+if _GEMINI_KEYS:
+    os.environ["GEMINI_API_KEYS"] = ",".join(_GEMINI_KEYS)
+    os.environ["GEMINI_API_KEY"] = _GEMINI_KEYS[0]
+
+# Auto-patch google.genai.Client globally so any eval or plugin can pass raw or multi-keys safely
+try:
+    from google import genai
+    _orig_genai_client_init = genai.Client.__init__
+
+    def _patched_genai_client_init(self, *args, **kwargs):
+        if "api_key" in kwargs and kwargs["api_key"]:
+            keys = _split_gemini_keys(str(kwargs["api_key"]))
+            if len(keys) > 1:
+                kwargs["api_key"] = keys[0]
+        elif not kwargs.get("api_key"):
+            keys = (_split_gemini_keys(os.environ.get("GEMINI_API_KEYS", ""))
+                    or _split_gemini_keys(os.environ.get("GEMINI_API_KEY", "")))
+            if keys:
+                kwargs["api_key"] = keys[0]
+        _orig_genai_client_init(self, *args, **kwargs)
+
+    genai.Client.__init__ = _patched_genai_client_init
+except Exception:
+    pass
+

gbench/__main__.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Main entry point for gbench package."""
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

gbench/analysis/__init__.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Analysis package for statistical processing."""
+
+from .statistics import (
+    aggregate_benchmark_results,
+    compute_statistics,
+    format_statistics_summary,
+    validate_repeatability,
+)
+
+__all__ = [
+    "aggregate_benchmark_results",
+    "compute_statistics",
+    "format_statistics_summary",
+    "validate_repeatability",
+]

gbench/analysis/perf_load.py

@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pure functions behind the stress pillar's open-loop load and its saturation gate.
+
+No network, no clock: the runner owns the I/O and calls these with planned send times
+and the records the client workers return. Every threshold here is relative (to the
+point's own latency, rate or SLO), never an absolute number tuned to one GPU, so the
+same gate is meaningful at 0.1 req/s on a laptop GPU and at 100 req/s on a large host.
+"""
+
+import math
+from typing import Dict, List, Optional, Sequence
+
+import numpy as np
+
+# ── Schedule ───────────────────────────────────────────────────────────────────
+
+def merged_schedule(rate: float, n: int, seed: int) -> List[float]:
+    """n Poisson arrival offsets (seconds after T0) at ``rate`` req/s from ONE seeded
+    generator: the whole point's arrival process, independent of how many client
+    processes later send it. The first arrival is an exponential gap after T0, never
+    at T0 itself."""
+    rng = np.random.default_rng(int(seed) & 0xFFFFFFFF)
+    return [float(x) for x in np.cumsum(rng.exponential(1.0 / float(rate), int(n)))]
+
+
+def assign_workers(n: int, k: int) -> List[List[int]]:
+    """Request indices per worker: arrival i goes to worker i mod k."""
+    k = max(1, int(k))
+    return [list(range(w, int(n), k)) for w in range(k)]
+
+
+def request_count(qps: float, min_prompts: int, min_steady: int,
+                  latency_est_s: Optional[float], c: float = 3.0,
+                  window_s: float = 60.0, cap: Optional[int] = None) -> int:
+    """Requests for one point; never fewer than the old floor max(min_prompts, qps·60).
+
+    Also enough to (a) cover c x the loaded latency plus 10 s, so the queue-growth test
+    sees a window several latencies long, and (b) leave min_steady requests in the
+    cohort after the first L of sends is excluded (planned send in [T0 + L, T_last]).
+    ``cap`` (an explicit --stress-max-prompts) is honoured as a recorded cap."""
+    n = max(int(min_prompts), int(round(qps * window_s)))
+    if latency_est_s and latency_est_s > 0:
+        n = max(n, int(math.ceil(qps * (c * latency_est_s + 10.0))))
+        n = max(n, int(math.ceil(qps * latency_est_s)) + int(min_steady) + 1)
+    else:
+        n = max(n, int(min_steady) + 1)
+    if cap:
+        n = min(n, int(cap))
+    return n
+
+
+def offered_rate_ci(n: int, span_s: float, alpha: float = 0.05) -> Dict[str, float]:
+    """Realized offered rate of a Poisson schedule whose n-th arrival is at span_s:
+    the unbiased estimate (n - 1) / span and the exact chi-square interval
+    [chi2(a/2, 2n), chi2(1 - a/2, 2n)] / (2 span)."""
+    if n <= 1 or span_s <= 0:
+        return {"rate": 0.0, "low": 0.0, "high": 0.0}
+    from scipy.stats import chi2
+    return {"rate": (n - 1) / span_s,
+            "low": float(chi2.ppf(alpha / 2, 2 * n)) / (2 * span_s),
+            "high": float(chi2.ppf(1 - alpha / 2, 2 * n)) / (2 * span_s)}
+
+
+def percentile_inf(values: Sequence[float], q: float = 99.0) -> float:
+    """Linear-interpolated percentile that treats +inf (a request that never finished)
+    as larger than everything: inf when the rank lands on or next to one. numpy's
+    percentile returns nan there (inf - inf)."""
+    v = np.sort(np.asarray(values, dtype=float))
+    if len(v) == 0:
+        return float("inf")
+    rank = q / 100.0 * (len(v) - 1)
+    lo, hi = int(math.floor(rank)), int(math.ceil(rank))
+    if math.isinf(v[hi]):
+        return float("inf")
+    return float(v[lo] + (v[hi] - v[lo]) * (rank - lo))
+
+
+# ── Cohort ─────────────────────────────────────────────────────────────────────
+
+def cohort(records: Sequence[dict], t_lo: float, t_hi: float) -> List[dict]:
+    """Requests whose PLANNED send time is in [t_lo, t_hi], in planned order. Selection
+    never looks at completion order, so slow requests can't be dropped from it."""
+    sel = [r for r in records if r.get("planned") is not None and t_lo <= r["planned"] <= t_hi]
+    return sorted(sel, key=lambda r: r["planned"])
+
+
+# ── Saturation: queue growth ───────────────────────────────────────────────────
+
+SATURATED = "SATURATED"
+NOT_DETECTED = "NOT_DETECTED"
+INDETERMINATE = "INDETERMINATE"
+
+# Calibration (docs/perf_evidence/step5b/final_calibration_2026-10-15.txt; 2000 seeds per
+# healthy point, Clopper-Pearson 95% CIs; 1/4-slot capacity and batching servers, sigma
+# 1.0, a fast batching server, 256 slots, unlimited servers at 0.13-100 req/s):
+#   certification size (~1300-request cohorts, which decide the headline): false
+#   SATURATED 0-0.1% at 0.8x/0.9x (upper CI <= 0.92%), detection 98.75-100% at 1.25x;
+#   sweep size (the bracket): false SATURATED up to 1.2% at 0.8x and 1.5-3.6% at 0.9x on
+#   1-4-slot servers (0 on large/fast/unlimited ones), detection 10-77% at 1.25x on small
+#   servers; a false alarm there only moves the bracket conservatively.
+# Live gemma-4-E2B (256 / 4 / 1 slots and a KV-limited server): 0/60 false at <= 0.9x; 32/40
+# overloaded points flagged (all 40 also failed the P99 TTFT SLO).
+# Undetected overload is NOT_DETECTED, never "keeping up"; the SLO gates then decide.
+QG_BATCHES = 10
+QG_ALPHA = 1e-4
+# Material growth: TTFT rising faster than this many seconds per second of wall time.
+# A queue fed at rho x capacity grows its wait by (rho - 1) s/s, so 0.05 means rho >~
+# 1.05; a continuous-batching server settling into a larger batch raises TTFT by
+# milliseconds over tens of seconds (~0.001 s/s), far below it. Dimensionless, so the
+# same on any hardware.
+QG_MATERIAL_SLOPE = 0.05
+
+
+def _batch_means(x: np.ndarray, y: np.ndarray, batches: int, stat: str = "median"):
+    idx = np.array_split(np.arange(len(x)), batches)
+    f = np.median if stat == "median" else np.mean
+    return (np.array([x[i].mean() for i in idx]), np.array([f(y[i]) for i in idx]))
+
+
+def _t_crit(df: int, one_sided_alpha: float) -> float:
+    """Student t critical value (scipy is a core gbench dependency)."""
+    from scipy.stats import t as _t
+    return float(_t.ppf(1.0 - one_sided_alpha, df))
+
+
+def queue_growth(times: Sequence[float], values: Sequence[float],
+                 batches: int = QG_BATCHES, alpha: float = QG_ALPHA,
+                 material_slope: float = QG_MATERIAL_SLOPE, stat: str = "median") -> dict:
+    """Is the queue growing over the cohort?
+
+    ``values`` is a queueing-sensitive latency (TTFT, ms) at each request's planned send
+    ``times`` (s). The cohort is cut into ``batches`` equal consecutive groups and the
+    slope of the batch medians over time is estimated by least squares on those medians
+    (batching makes the variance robust to the strong autocorrelation of queueing
+    delays, which an i.i.d. test on single requests ignores; medians stop one very slow
+    request from faking a trend):
+
+      SATURATED      slope > 0 with one-sided significance ``alpha`` AND TTFT growing
+                     faster than ``material_slope`` seconds per second of wall time (a
+                     queue fed at rho x capacity grows its wait by rho - 1 s/s)
+      NOT_DETECTED   otherwise (not proof of keeping up: the SLO gates decide)
+      INDETERMINATE  too few requests for the test
+    """
+    x = np.asarray(times, dtype=float)
+    y = np.asarray(values, dtype=float)
+    ok = np.isfinite(x) & np.isfinite(y)
+    x, y = x[ok], y[ok]
+    out = {"outcome": INDETERMINATE, "n": int(len(x)), "batches": int(batches),
+           "alpha": alpha, "material_slope_s_per_s": material_slope}
+    if len(x) < 2 * batches or batches < 4:
+        out["reason"] = "too few requests for the batch test"
+        return out
+    order = np.argsort(x)
+    x, y = x[order], y[order]
+    bx, by = _batch_means(x, y, batches, stat)
+    if float(x[-1] - x[0]) <= 0:
+        out["reason"] = "degenerate cohort"
+        return out
+    xc = bx - bx.mean()
+    sxx = float((xc ** 2).sum())
+    slope = float((xc * (by - by.mean())).sum() / sxx)
+    resid = by - (by.mean() + slope * xc)
+    df = batches - 2
+    se = float(math.sqrt(max(1e-18, (resid ** 2).sum() / df / sxx)))
+    tc = _t_crit(df, alpha)
+    out.update({"slope_s_per_s": slope / 1000.0, "slope_se_s_per_s": se / 1000.0,
+                "t_stat": slope / se if se > 0 else float("inf")})
+    out["outcome"] = (SATURATED if (slope / se > tc and slope / 1000.0 > material_slope)
+                      else NOT_DETECTED)
+    return out
+
+
+def in_flight_at(records: Sequence[dict], times: Sequence[float]) -> List[int]:
+    """Client-side in-flight count (sent and not yet finished, successfully or not) at
+    each instant; a request with no end at all counts as in flight forever."""
+    sends = np.sort([r["send"] for r in records if r.get("send") is not None])
+    dones = np.sort([r["done"] if r.get("done") is not None else math.inf
+                     for r in records if r.get("send") is not None])
+    return [int(np.searchsorted(sends, t, "right") - np.searchsorted(dones, t, "right"))
+            for t in times]
+
+
+# ── Secondary: completion fraction ─────────────────────────────────────────────
+
+def completion_bound_s(ttft_slo_ms: float, itl_slo_ms: float, max_tokens: int) -> float:
+    """The end-to-end time the SLOs imply for one request: TTFT SLO + (max_tokens - 1)
+    ITL SLOs. Independent of the measured latency, so the gate is not circular."""
+    return (float(ttft_slo_ms) + max(0, int(max_tokens) - 1) * float(itl_slo_ms)) / 1000.0
+
+
+def completion_fraction(cohort_recs: Sequence[dict], t_last_arrival: float,
+                        bound_s: float) -> Optional[float]:
+    """Share of the cohort that finished successfully by t_last_arrival + bound_s."""
+    if not cohort_recs:
+        return None
+    deadline = t_last_arrival + bound_s
+    done = sum(1 for r in cohort_recs
+               if r.get("ok") and r.get("done") is not None and r["done"] <= deadline)
+    return done / len(cohort_recs)

gbench/analysis/perf_method.py

@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pure functions behind the performance pillars' measurement method.
+
+Everything here is deterministic and side-effect free (no network, no clock), so
+the statistics the stress and serving pillars gate on can be unit-tested over many
+seeds without a server. The runners own the I/O; this module owns the maths.
+"""
+
+import math
+from typing import Dict, Optional, Tuple
+
+# Bumped whenever a change alters a reported number or a pass/fail decision, so old
+# and new results are never compared silently. Each component versions one part of
+# the method. The serving and stress scaffolds pin methodology_version (as
+# dataset_snapshot.workload_version), so a bump changes their scaffold_id and breaks
+# the series instead of silently extending it. The offline throughput pillar uses
+# none of this and is not pinned.
+METHODOLOGY_COMPONENTS: Dict[str, int] = {
+    # 2: every request unique by construction (salt + nonce, coded images), one chat
+    # layout, the served context window for multimodal text (perf_payload).
+    # 3: vLLM requests send skip_special_tokens False, so a reply of special tokens is
+    # timed like any other (it streamed as empty chunks and read as never finished).
+    "payload": 3,
+    # 2: one merged Poisson schedule from the parent, sent at absolute times by any
+    # number of workers; request count never below max(min_prompts, qps x 60), sized up
+    # for the loaded latency; no prompt cap by default.
+    "schedule": 2,
+    # 2: the server ITL gate is PASS / FAIL / INDETERMINATE on the bucket counts.
+    # 3: every check on a planned-send cohort; saturation = queue growth of TTFT (batch
+    # medians, alpha 1e-4, material growth 0.05 s/s) + completion fraction >= 0.90 within
+    # the SLO-implied e2e; schedule-based client-bound checks; INCONCLUSIVE points.
+    # 4: the headline is the median repeat's bracket pass, certified with
+    # percentile-sized probes (bisection to +-9%, budgeted) or labelled UNCERTIFIED; no
+    # interpolated limit; drain before every point; contaminated points re-measured.
+    "gate": 4,
+    # 2: the server ITL P99 reads the CUMULATIVE histogram delta (v1 summed the
+    # running totals again and reached 99% one or more buckets too early).
+    # 3: client ITL split into inter-chunk (what users see) and per-token (chunk gaps
+    # divided by the tokens each chunk carried); the per-token value is the fallback gate.
+    "itl_metric": 3,
+    # 2: per-token TPOT = (e2e - ttft) / (completion_tokens - 1) from the server's usage,
+    # not the mean gap between streamed chunks (a chunk can carry several tokens).
+    "tpot_metric": 2,
+    # 2: per point / iteration prefix- and multimodal-cache deltas, prompt-length check.
+    "cache_telemetry": 2,
+    # 2: remote text serving on aiohttp (was a 32-thread pool that capped batch sizes
+    # above 32 and opened a connection per request); peak in-flight recorded.
+    "serving_transport": 2,
+    # 2: serving input_length_actual is the median over iterations (+ template tokens),
+    # ignore_eos_honored is observed (n/N requests reached max_tokens), per-request
+    # samples saved, honest multimodal banner.
+    "reporting": 2,
+}
+
+
+# The components each pillar's numbers depend on. Serving never reads the server ITL
+# histogram or runs the sweep, so a schedule/gate/itl_metric bump must not break the
+# serving series.
+PILLAR_COMPONENTS: Dict[str, Tuple[str, ...]] = {
+    "serving": ("payload", "tpot_metric", "cache_telemetry", "serving_transport", "reporting"),
+    "stress": tuple(METHODOLOGY_COMPONENTS),
+}
+
+
+def methodology_record(pillar: str) -> dict:
+    """The methodology stamp a serving or stress result carries: the components that
+    pillar's numbers depend on, and a version string derived from them."""
+    comps = {k: METHODOLOGY_COMPONENTS[k] for k in PILLAR_COMPONENTS[pillar]}
+    return {
+        "methodology_version": ".".join(f"{k}{v}" for k, v in sorted(comps.items())),
+        "methodology_components": comps,
+    }
+
+
+# ── Prometheus histogram (vllm:inter_token_latency_seconds) ────────────────────
+
+def hist_delta(before: Optional[tuple], after: Optional[tuple]) -> Optional[Tuple[Dict[float, float], float]]:
+    """Per-bucket delta of two cumulative histogram snapshots ({le: count}, total).
+
+    The buckets stay CUMULATIVE: delta[le] = number of observations <= le made
+    between the snapshots. None when a snapshot is missing or nothing was observed.
+    """
+    if not before or not after:
+        return None
+    b_buckets, b_cnt = before
+    a_buckets, a_cnt = after
+    d_cnt = float(a_cnt) - float(b_cnt)
+    if d_cnt <= 0:
+        return None
+    # A counter reset, a server restart or a series first exposed mid-point gives an
+    # impossible delta; refuse it (the caller falls back to the client value) rather
+    # than report a 'provable' verdict from it.
+    if set(a_buckets) != set(b_buckets):
+        return None
+    delta = {}
+    prev = 0.0
+    for le in sorted(a_buckets):
+        v = float(a_buckets[le]) - float(b_buckets[le])
+        if v < -1e-9 or v + 1e-9 < prev or v > d_cnt + 1e-9:
+            return None
+        delta[le] = v
+        prev = v
+    if math.inf in delta and abs(delta[math.inf] - d_cnt) > 1e-6:
+        return None
+    return delta, d_cnt
+
+
+def hist_quantile_bucket(delta: Dict[float, float], total: float,
+                         pct: float = 99.0) -> Tuple[float, float]:
+    """(lower_edge, upper_edge) in SECONDS of the bucket holding the pct quantile.
+
+    Reads the cumulative counts directly: the first edge whose cumulative count
+    reaches pct% of the total. No in-bucket interpolation (a multi-modal ITL
+    distribution makes that false precision). lower_edge is 0.0 for the first
+    bucket; upper_edge is inf for the overflow bucket.
+    """
+    les = sorted(delta)
+    target = pct / 100.0 * total
+    lower = 0.0
+    for le in les:
+        if delta[le] >= target:
+            return lower, le
+        lower = le
+    return lower, math.inf
+
+
+def _fmt_edge_ms(sec: float) -> str:
+    if math.isinf(sec):
+        return "inf"
+    ms = sec * 1000.0
+    return f"{ms:.0f}" if abs(ms - round(ms)) < 1e-6 else f"{ms:g}"
+
+
+def bucket_label(lo_s: float, hi_s: float) -> str:
+    """'(50,75]ms', or '(1,2.5]s' when the upper edge is 1 s or more (slow hardware)."""
+    if not math.isinf(hi_s) and hi_s < 1.0:
+        return f"({_fmt_edge_ms(lo_s)},{_fmt_edge_ms(hi_s)}]ms"
+    if math.isinf(hi_s) and lo_s < 1.0:
+        return f"({_fmt_edge_ms(lo_s)}ms,inf]"
+    hi = "inf" if math.isinf(hi_s) else f"{hi_s:g}"
+    return f"({lo_s:g},{hi}]s"
+
+
+def smallest_finite_edge_ms(buckets: Dict[float, float]) -> Optional[float]:
+    finite = [le for le in buckets if not math.isinf(le)]
+    return min(finite) * 1000.0 if finite else None
+
+
+def hist_itl_gate(delta: Dict[float, float], total: float, limit_ms: float,
+                  pct: float = 99.0) -> dict:
+    """Gate a P99 inter-token-latency SLO on a histogram delta, without interpolation.
+
+    Let lo = the largest finite edge <= limit and hi = the smallest edge >= limit.
+    The share of gaps <= limit is bounded by [cum(lo), cum(hi)] / total, so:
+      PASS           cum(lo) >= pct%  (provably at least pct% within the limit)
+      FAIL           cum(hi) <  pct%  (provably fewer than pct% within the limit)
+      INDETERMINATE  otherwise (the limit sits inside a bucket that straddles pct%),
+                     or the limit is below the smallest finite edge (no lo exists).
+    When the limit is itself an edge, lo == hi and the outcome is never
+    INDETERMINATE. The caller falls back to the client per-token value on
+    INDETERMINATE and records that it did.
+
+    Returns outcome, p99_itl_server_ms (the upper edge of the P99 bucket, numeric),
+    p99_itl_server_bucket ("(50,75]ms"), itl_gate_edge_ms (lo, None if none),
+    itl_frac_over_limit (1 - cum(lo)/total, an upper bound on the true share over
+    the limit; None if no lo), and the bucket deltas in ms.
+    """
+    limit_s = float(limit_ms) / 1000.0
+    les = sorted(delta)
+    finite = [le for le in les if not math.isinf(le)]
+    lo = max((le for le in finite if le <= limit_s + 1e-12), default=None)
+    hi = min((le for le in les if le >= limit_s - 1e-12), default=math.inf)
+    target = pct / 100.0 * total
+    cum_lo = delta.get(lo, 0.0) if lo is not None else None
+    cum_hi = delta.get(hi, total) if not math.isinf(hi) else total
+
+    if lo is not None and cum_lo >= target:
+        outcome = "PASS"
+    elif cum_hi < target:
+        outcome = "FAIL"
+    else:
+        outcome = "INDETERMINATE"
+
+    b_lo, b_hi = hist_quantile_bucket(delta, total, pct)
+    return {
+        "outcome": outcome,
+        "p99_itl_server_ms": (math.inf if math.isinf(b_hi) else b_hi * 1000.0),
+        "p99_itl_server_bucket": bucket_label(b_lo, b_hi),
+        "itl_gate_edge_ms": (lo * 1000.0 if lo is not None else None),
+        "itl_frac_over_limit": ((1.0 - cum_lo / total) if lo is not None else None),
+        "itl_hist_delta_ms": {_fmt_edge_ms(le): delta[le] for le in les},
+        "itl_hist_count": total,
+    }
+
+
+# ── /metrics snapshot (vLLM Prometheus text) ────────────────────────────────────
+
+# Counters summed over every engine / API-server label. The "external_" prefix-cache
+# series (KV connector) is a different cache and is not summed into these.
+_COUNTERS = {
+    "vllm:prefix_cache_hits_total": "prefix_cache_hits",
+    "vllm:prefix_cache_queries_total": "prefix_cache_queries",
+    "vllm:mm_cache_hits_total": "mm_cache_hits",
+    "vllm:mm_cache_queries_total": "mm_cache_queries",
+    "vllm:prompt_tokens_total": "prompt_tokens",
+    "vllm:generation_tokens_total": "generation_tokens",
+    "vllm:request_success_total": "request_success",   # all finished_reason labels
+    "vllm:spec_decode_num_draft_tokens_total": "spec_draft_tokens",
+}
+_GAUGES = {
+    "vllm:num_requests_running": "running",
+    "vllm:num_requests_waiting": "waiting",
+}
+_ITL_BUCKET = "vllm:inter_token_latency_seconds_bucket"
+_ITL_COUNT = "vllm:inter_token_latency_seconds_count"
+
+
+def parse_metrics(text: str) -> dict:
+    """One /metrics scrape as {counter: total, gauge: total, "itl": (buckets, count),
+    "cache_config": {label: value}}. Missing series are absent (never 0), so a caller
+    can tell 'not exposed' from 'zero'."""
+    import re
+
+    out: dict = {}
+    buckets: dict = {}
+    itl_count = None
+    for ln in text.splitlines():
+        if not ln or ln.startswith("#"):
+            continue
+        name, _, rest = ln.partition("{") if "{" in ln.split(" ", 1)[0] else (ln.split(" ", 1)[0], "", "")
+        try:
+            value = float(ln.rsplit(" ", 1)[-1])
+        except ValueError:
+            continue
+        if name in _COUNTERS:
+            k = _COUNTERS[name]
+            out[k] = out.get(k, 0.0) + value
+        elif name in _GAUGES:
+            k = _GAUGES[name]
+            out[k] = out.get(k, 0.0) + value
+        elif name == _ITL_BUCKET:
+            m = re.search(r'le="([^"]+)"', rest)
+            if m:
+                le = math.inf if m.group(1) == "+Inf" else float(m.group(1))
+                buckets[le] = buckets.get(le, 0.0) + value
+        elif name == _ITL_COUNT:
+            itl_count = (itl_count or 0.0) + value
+        elif name == "vllm:cache_config_info" and "cache_config" not in out:
+            out["cache_config"] = dict(re.findall(r'(\w+)="([^"]*)"', rest))
+    if buckets and itl_count is not None:       # an empty histogram is 0, not absent
+        out["itl"] = (buckets, itl_count)
+    return out
+
+
+def counter_delta(before: Optional[dict], after: Optional[dict], key: str) -> Optional[float]:
+    """after[key] - before[key], or None when either scrape lacks the series."""
+    if not before or not after or key not in before or key not in after:
+        return None
+    return float(after[key]) - float(before[key])
+
+
+# ── Client-side per-token timing ───────────────────────────────────────────────
+
+def per_token_tpot_ms(e2e_ms: Optional[float], ttft_ms: Optional[float],
+                      completion_tokens: Optional[int]) -> Optional[float]:
+    """Per-token TPOT of one request: (e2e - ttft) / (completion_tokens - 1).
+
+    Uses the server's token count (final usage chunk), so a streamed chunk carrying
+    several tokens can't inflate it. None without a count or with fewer than 2 tokens."""
+    if e2e_ms is None or ttft_ms is None or not completion_tokens or completion_tokens < 2:
+        return None
+    return max(0.0, (float(e2e_ms) - float(ttft_ms)) / (int(completion_tokens) - 1))
+
+
+def per_token_gaps(gaps_ms: list, tokens_per_chunk: list) -> list:
+    """Per-token gaps: each inter-chunk gap divided by the tokens the later chunk carried
+    (one value per token, so a 3-token chunk after 30 ms gives three 10 ms gaps)."""
+    out = []
+    for g, n in zip(gaps_ms, tokens_per_chunk):
+        n = max(1, int(n or 1))
+        out.extend([float(g) / n] * n)
+    return out
+
+
+def bucket_index(value_ms: float, edges_s: list) -> int:
+    """Index of the histogram bucket (sorted upper edges in seconds, +Inf last) that
+    holds value_ms."""
+    v = float(value_ms) / 1000.0
+    for k, le in enumerate(sorted(edges_s)):
+        if v <= le + 1e-12:
+            return k
+    return len(edges_s) - 1