inferencekit-cli 0.1.0__py3-none-any.whl

inferencekit/serve/metrics.py

@@ -0,0 +1,203 @@
+"""Numbers from timed requests: per request, per bucket (single-user mode) and per load level.
+
+Per request (all from client timestamps, see client.py):
+  ttft_ms        t_first - t_send
+  e2e_ms         t_done - t_send
+  decode_tok_s   (n_tokens - k_first) / (t_last - t_first), k_first = tokens in the first token event
+                 (1 when the server sent no ids). "Tokens after the first, over time after the first",
+                 counted from ids so a server that packs several tokens into one event is not undercounted.
+  tpot_ms        (t_last - t_first) / (n_tokens - k_first) = 1000 / decode_tok_s
+  itl_ms         gaps between consecutive token events (per event, as most serving benchmarks report)
+
+Percentiles are nearest-rank (always an observed value), the same convention the bootstrap uses, so a
+point estimate and its CI agree.
+
+Single-user summaries: value = median (and p95) over every request of a bucket; 95% CI by cluster
+bootstrap with the *round* as the unit (stats.bootstrap_ci), because requests inside one round share
+the server's state and are not independent.
+
+Load-level summaries: p50/p95/p99 over the level's requests; the p95 CI is a plain bootstrap over
+requests (under load, requests interact, so treat it as optimistic; the record says so).
+"""
+
+from __future__ import annotations
+
+from inferencekit import evidence as ev
+from inferencekit import stats
+from inferencekit.serve.client import RequestResult
+
+SPEED_METRICS = (("ttft_ms", "ms"), ("tpot_ms", "ms"), ("decode_tok_s", "tok/s"), ("e2e_ms", "ms"))
+
+
+def pct(values: list[float], p: float) -> float | None:
+    """Nearest-rank percentile (an observed value), as inside stats.bootstrap_ci."""
+    return stats.nearest_rank(sorted(values), p)
+
+
+def _ms(a: int | None, b: int | None) -> float | None:
+    return (a - b) / 1e6 if a is not None and b is not None else None
+
+
+def request_row(res: RequestResult, **extra) -> dict:
+    """One request's numbers (no text): a row of requests.jsonl."""
+    ttft = _ms(res.t_first_ns, res.t_send_ns)
+    e2e = _ms(res.t_done_ns if res.t_done_ns is not None else res.t_end_ns, res.t_send_ns)
+    n = res.n_tokens
+    k_first = res.events[0].n_ids if res.events and res.events[0].n_ids else 1
+    span = _ms(res.t_last_ns, res.t_first_ns)
+    after = n - k_first if n is not None else None
+    decode = tpot = None
+    if res.ok and after and after > 0 and span and span > 0:
+        decode = after / (span / 1000.0)
+        tpot = span / after
+    per_event = [e.n_ids for e in res.events if e.n_ids]
+    row = {"id": res.id, "bucket": res.bucket, "salt": res.salt, "ok": res.ok, "error": res.error, "status": res.status,
+           "ttft_ms": ttft if res.ok else None, "tpot_ms": tpot, "decode_tok_s": decode,
+           "e2e_ms": e2e if res.ok else None, "connect_ms": _ms(res.t_connected_ns, res.t_send_ns),
+           "n_tokens": n, "token_count_method": res.token_count_method, "count_mismatch": res.count_mismatch,
+           "n_ids": res.n_ids, "n_usage": res.n_usage,
+           "finish_reason": res.finish_reason, "n_token_events": len(res.events),
+           "tokens_per_event_max": max(per_event) if per_event else None,
+           "send_lag_ms": _ms(res.t_send_ns, res.t_scheduled_ns), "t_send_ns": res.t_send_ns,
+           "t_end_ns": res.t_end_ns}
+    row.update(extra)
+    return row
+
+
+def itl_ms(res: RequestResult) -> list[float]:
+    ev_ = res.events
+    return [(b.t_ns - a.t_ns) / 1e6 for a, b in zip(ev_, ev_[1:])]
+
+
+def timings(res: RequestResult) -> dict:
+    """The per-event timeline stored with each output (ms after t_send, and ids per event)."""
+    return {"t_send_ns": res.t_send_ns,
+            "event_ms": [round((e.t_ns - res.t_send_ns) / 1e6, 4) for e in res.events],
+            "event_token_ids": [e.n_ids for e in res.events]}
+
+
+# --------------------------------------------------------------------------- single-user mode
+
+def bucket_metrics(rows: list[dict], buckets: list[str], resamples: int, seed: int = 0) -> tuple[list[dict], dict]:
+    """Evidence metrics per bucket (and 'all'): median and p95, with CIs over rounds. Also a plain dict."""
+    out, table = [], {}
+    for subject in list(buckets) + ["all"]:
+        sel = [r for r in rows if r["ok"] and (subject == "all" or r["bucket"] == subject)]
+        table[subject] = {"n_requests": len(sel)}
+        for name, unit in SPEED_METRICS:
+            vals = [(r[name], r["round"]) for r in sel if r.get(name) is not None]
+            values, units = [v for v, _ in vals], [u for _, u in vals]
+            if not values:
+                out.append(ev.metric(name, None, unit, "median", n=0, subject=subject))
+                table[subject][name] = None
+                continue
+            ci = stats.bootstrap_ci(values, units, ps=(50, 95), resamples=resamples, seed=seed)
+            med, p95 = pct(values, 50), pct(values, 95)
+            out.append(ev.metric(name, med, unit, "median", n=len(values), ci95=ci[50], subject=subject))
+            out.append(ev.metric(name, p95, unit, "p95", n=len(values), ci95=ci[95], subject=subject))
+            table[subject][name] = {"median": med, "p95": p95, "ci95_median": list(ci[50]),
+                                    "ci95_p95": list(ci[95]), "n": len(values), "rounds": len(set(units))}
+    return out, table
+
+
+def round_rows(rows: list[dict], round_walltimes: dict[int, tuple[float, float]]) -> list[dict]:
+    """One row per round (medians over the round's requests), in the row shape of bench decode records
+    (``compare`` reads that shape; it accepts serve_benchmark records only once that lands at merge)."""
+    out = []
+    for r, (t0, t1) in sorted(round_walltimes.items()):
+        sel = [x for x in rows if x["round"] == r and x["ok"]]
+        row = {"index": r, "t_start": t0, "t_end": t1, "n_requests": len(sel),
+               "n_failed": sum(1 for x in rows if x["round"] == r and not x["ok"])}
+        for name, key in (("decode_tok_s", "decode_tok_s"), ("ttft_ms", "ttft_ms"), ("tpot_ms", "tpot_ms"),
+                          ("total_ms", "e2e_ms")):
+            vals = [x[key] for x in sel if x.get(key) is not None]
+            row[name] = pct(vals, 50) if vals else None
+        out.append(row)
+    return out
+
+
+# --------------------------------------------------------------------------- load mode
+
+def _pcts(values: list[float]) -> dict:
+    if not values:
+        return {"p50": None, "p95": None, "p99": None, "n": 0}
+    return {"p50": pct(values, 50), "p95": pct(values, 95), "p99": pct(values, 99), "n": len(values)}
+
+
+def meets_slo(row: dict, slo: dict) -> bool:
+    """Per-request SLO attainment (for goodput): succeeded, TTFT and TPOT inside their targets."""
+    if not row["ok"]:
+        return False
+    if slo.get("ttft_p95_ms") is not None and (row["ttft_ms"] is None or row["ttft_ms"] > slo["ttft_p95_ms"]):
+        return False
+    if slo.get("tpot_p95_ms") is not None and row["tpot_ms"] is not None and row["tpot_ms"] > slo["tpot_p95_ms"]:
+        return False
+    return True
+
+
+def level_summary(rows: list[dict], itls: list[float], slo: dict, gpu_count: int, resamples: int = 2000,
+                  seed: int = 0) -> dict:
+    """One load level: latency percentiles, failures, goodput, throughput and whether the SLO held."""
+    ok = [r for r in rows if r["ok"]]
+    t0 = min((r["t_send_ns"] for r in rows), default=0)
+    t1 = max((r["t_end_ns"] for r in rows), default=0)
+    window_s = (t1 - t0) / 1e9 if rows else 0.0
+    out_tokens = sum(r["n_tokens"] or 0 for r in ok)
+    s = {"n_requests": len(rows), "n_ok": len(ok), "n_failed": len(rows) - len(ok), "window_s": window_s,
+         "ttft_ms": _pcts([r["ttft_ms"] for r in ok if r["ttft_ms"] is not None]),
+         "tpot_ms": _pcts([r["tpot_ms"] for r in ok if r["tpot_ms"] is not None]),
+         "e2e_ms": _pcts([r["e2e_ms"] for r in ok if r["e2e_ms"] is not None]),
+         "itl_ms": _pcts(itls),
+         "output_tokens": out_tokens,
+         "output_tok_s": out_tokens / window_s if window_s > 0 else None,
+         "output_tok_s_per_gpu": out_tokens / window_s / max(1, gpu_count) if window_s > 0 else None,
+         "goodput_rps": sum(1 for r in rows if meets_slo(r, slo)) / window_s if window_s > 0 else None,
+         "achieved_rps": len(ok) / window_s if window_s > 0 else None,
+         # Little's law: time-averaged requests in flight over the window
+         "mean_inflight": sum((r["t_end_ns"] - r["t_send_ns"]) for r in rows) / 1e9 / window_s if window_s else None,
+         "send_lag_ms": _pcts([r["send_lag_ms"] for r in rows if r.get("send_lag_ms") is not None])}
+    for name in ("ttft_ms", "tpot_ms"):
+        vals = [r[name] for r in ok if r[name] is not None]
+        s[name]["ci95_p95"] = list(stats.bootstrap_ci(vals, ps=(95,), resamples=resamples, seed=seed)[95])
+    s["slo_checks"] = slo_checks(s, slo)
+    s["slo_ok"] = all(c["ok"] for c in s["slo_checks"].values()) if s["slo_checks"] else None
+    return s
+
+
+def slo_checks(s: dict, slo: dict) -> dict:
+    checks = {}
+    if slo.get("ttft_p95_ms") is not None:
+        v = s["ttft_ms"]["p95"]
+        checks["ttft_p95_ms"] = {"value": v, "target": slo["ttft_p95_ms"], "ok": v is not None and v <= slo["ttft_p95_ms"]}
+    if slo.get("tpot_p95_ms") is not None:
+        v = s["tpot_ms"]["p95"]
+        checks["tpot_p95_ms"] = {"value": v, "target": slo["tpot_p95_ms"], "ok": v is not None and v <= slo["tpot_p95_ms"]}
+    if checks:  # failures only count when an SLO was given
+        mf = slo.get("max_fail") or 0
+        checks["failed"] = {"value": s["n_failed"], "target": mf, "ok": s["n_failed"] <= mf and s["n_ok"] > 0}
+    return checks
+
+
+def n_star(levels: list[dict]) -> tuple[float | None, str]:
+    """Highest level at which the SLO holds at that level and at every lower one.
+
+    Returns (level or None, reason). A level that passes above a failing one does not count.
+    """
+    if not levels:
+        return None, "no load levels ran"
+    best = None
+    for lv in sorted(levels, key=lambda x: x["level"]):
+        if lv["summary"].get("slo_ok") is None:
+            return None, "no SLO given"
+        if not lv["summary"]["slo_ok"]:
+            failed = [k for k, c in lv["summary"]["slo_checks"].items() if not c["ok"]]
+            return best, f"SLO first fails at level {lv['level']:g} ({', '.join(failed)})"
+        best = lv["level"]
+    return best, "SLO holds at every level tried: the true N* may be above the grid"
+
+
+def usd_per_1m_tokens(tok_s: float | None, price_usd_h: float | None, gpu_count: int = 1) -> float | None:
+    """$ per 1M output tokens = hourly price of the GPUs / tokens produced per hour x 1e6."""
+    if not tok_s or not price_usd_h or tok_s <= 0:
+        return None
+    return price_usd_h * max(1, gpu_count) / (tok_s * 3600.0) * 1e6

inferencekit/serve/profiles.py

@@ -0,0 +1,105 @@
+"""Per-engine request profiles: how to ask each OpenAI-compatible server for an exact output length,
+streamed token ids and a usage block, using only fields that server documents.
+
+Why it matters: a fair speed comparison needs every engine to generate the same number of tokens
+(otherwise an engine that stops early at EOS looks faster), and an honest decode speed needs a token
+count that does not depend on how many tokens a server packs into one SSE event.
+
+Sources (checked 13 Oct 2026; fields can move between releases, so the client parses every known
+location and the record says which one was used):
+  vLLM     vllm/entrypoints/openai/chat_completion/protocol.py (ChatCompletionRequest: ignore_eos,
+           min_tokens, return_token_ids, return_tokens_as_token_ids, stream_options;
+           ChatCompletionResponseStreamChoice.token_ids "for tracing the tokens") and
+           docs.vllm.ai/en/v0.12.0/serving/openai_compatible_server/ ("Extra parameters": extra
+           fields are merged into the JSON body; return_token_ids needs vLLM >= 0.10.2).
+  SGLang   python/sglang/srt/entrypoints/openai/protocol.py (ignore_eos, min_tokens, stream_options,
+           SglExt.output_ids) and serving_chat.py (return_token_ids is rejected when stream=true, so it
+           is not requested; usage arrives in a final chunk when stream_options.include_usage is set).
+  llama.cpp tools/server/README.md (ignore_eos and other /completion fields are accepted on the
+           OpenAI-compatible endpoints) and tools/server/server-task.cpp (logprob entries carry "id";
+           usage arrives in a trailing chunk when include_usage is set). llama-server has no
+           min_tokens. Asking for logprobs makes it compute token probabilities every step, which
+           costs decode time, so ids are requested only with --token-ids logprobs.
+  TRT-LLM  tensorrt_llm/serve/openai_protocol.py (ChatCompletionRequest / CompletionRequest:
+           ignore_eos, min_tokens, stream_options{include_usage}; CompletionResponseStreamChoice.token_ids;
+           no request field to ask for chat token ids).
+  generic  plain OpenAI fields only (max_tokens, stream, stream_options.include_usage).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+ENGINE_KINDS = ("vllm", "sglang", "llamacpp", "trtllm", "generic")
+APIS = ("chat", "completions")
+TOKEN_ID_MODES = ("auto", "logprobs", "off")
+
+
+@dataclass(frozen=True)
+class Profile:
+    name: str
+    # Body fields that force exactly max_tokens; "{n}" is replaced by max_tokens.
+    exact_length: dict = field(default_factory=dict)
+    # Body fields that make the server stream generated token ids at (near) zero cost, per API.
+    ids_free: dict = field(default_factory=dict)
+    # Body fields that stream token ids through logprobs (may slow decode), per API.
+    ids_logprobs: dict = field(default_factory=dict)
+    note: str = ""
+
+
+PROFILES: dict[str, Profile] = {
+    "vllm": Profile(
+        "vllm",
+        exact_length={"ignore_eos": True, "min_tokens": "{n}"},
+        ids_free={"chat": {"return_token_ids": True}, "completions": {"return_token_ids": True}},
+        ids_logprobs={"chat": {"logprobs": True, "top_logprobs": 0, "return_tokens_as_token_ids": True},
+                      "completions": {"logprobs": 0, "return_tokens_as_token_ids": True}},
+        note="token ids from choices[].token_ids (return_token_ids, vLLM >= 0.10.2)"),
+    "sglang": Profile(
+        "sglang",
+        exact_length={"ignore_eos": True, "min_tokens": "{n}"},
+        note="return_token_ids is not supported with streaming; tokens counted from usage unless the "
+             "server adds sglext.output_ids"),
+    "llamacpp": Profile(
+        "llamacpp",
+        exact_length={"ignore_eos": True},
+        ids_logprobs={"chat": {"logprobs": True, "top_logprobs": 1}, "completions": {"logprobs": 1}},
+        note="no min_tokens; ids only through logprobs (costs decode time), else usage"),
+    "trtllm": Profile(
+        "trtllm",
+        exact_length={"ignore_eos": True, "min_tokens": "{n}"},
+        note="completions stream choices may carry token_ids; chat has none, so usage is used"),
+    "generic": Profile("generic", note="standard OpenAI fields only: no exact-length control"),
+}
+
+
+def build_body(api: str, engine_kind: str, model: str, messages: list[dict], max_tokens: int, *,
+               token_ids: str = "auto", exact_length: bool = True, extra: dict | None = None) -> tuple[dict, dict]:
+    """The JSON body for one streamed request, and a description of what was asked for.
+
+    Greedy decoding (temperature 0) by default so outputs are comparable across engines; ``extra``
+    (the user's --extra-body) is merged last and can override anything.
+    """
+    prof = PROFILES[engine_kind]
+    body: dict = {"model": model}
+    if api == "chat":
+        body["messages"] = list(messages)
+    else:
+        body["prompt"] = "\n\n".join(str(m.get("content", "")) for m in messages)
+    body.update({"max_tokens": int(max_tokens), "temperature": 0.0, "stream": True,
+                 "stream_options": {"include_usage": True}})
+    asked = {"exact_length": None, "token_ids": None}
+    if exact_length and prof.exact_length:
+        body.update({k: (int(max_tokens) if v == "{n}" else v) for k, v in prof.exact_length.items()})
+        asked["exact_length"] = sorted(prof.exact_length)
+    if token_ids != "off":
+        fields = prof.ids_free.get(api)
+        how = "free"
+        if not fields and token_ids == "logprobs":
+            fields, how = prof.ids_logprobs.get(api), "logprobs"
+        if fields:
+            body.update(fields)
+            asked["token_ids"] = {"via": how, "fields": sorted(fields)}
+    if extra:
+        body.update(extra)
+    return body, asked

inferencekit/serve/prompts_io.py

@@ -0,0 +1,125 @@
+"""Read a prompt set: JSONL, one request per line (the format ``prompts build`` writes).
+
+    {"id": "short-0007", "bucket": "short", "messages": [{"role": "user", "content": "..."}],
+     "max_tokens": 256, "source": "dataset/license", "kind": "chat"}
+
+This module only reads. Building and sealing sets is ``inferencekit prompts ...``. The file may
+be gzipped (``.jsonl.gz``, as the shipped dev sets are). The set's identity is the sha256 of the
+uncompressed JSONL bytes, the same hash its manifest carries, so a benchmark record can be
+matched to the exact prompts it ran.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from inferencekit import prompts as _sets
+from inferencekit.registry import OpError
+
+BUCKETS = ("very_short", "short", "medium", "long")
+
+
+@dataclass(frozen=True)
+class Prompt:
+    id: str
+    bucket: str
+    messages: tuple  # OpenAI chat messages (dicts); a tuple so the dataclass stays hashable
+    max_tokens: int | None
+    kind: str = "chat"
+    extra: dict = field(default_factory=dict, compare=False, hash=False)
+
+    def as_completion_prompt(self) -> str:
+        """Plain text for /v1/completions: the message contents joined (no chat template)."""
+        return "\n\n".join(str(m.get("content", "")) for m in self.messages)
+
+
+@dataclass(frozen=True)
+class PromptSet:
+    name: str
+    path: Path
+    sha256: str
+    prompts: tuple[Prompt, ...]
+    manifest: dict | None = None
+
+    def buckets(self) -> list[str]:
+        seen = []
+        for p in self.prompts:
+            if p.bucket not in seen:
+                seen.append(p.bucket)
+        return sorted(seen, key=lambda b: (BUCKETS.index(b) if b in BUCKETS else len(BUCKETS), b))
+
+    def identity(self) -> dict:
+        return {"name": self.name, "sha256": self.sha256}
+
+
+def load(path: str | Path, buckets: list[str] | None = None, limit_per_bucket: int | None = None) -> PromptSet:
+    """Load and check a prompt set. ``buckets`` / ``limit_per_bucket`` select a subset (file order)."""
+    p = _sets.existing(path)
+    if not p.is_file():
+        raise OpError(f"prompt set {path} not found")
+    try:
+        raw = _sets.read_bytes(p)
+        text = raw.decode("utf-8")
+    except (OSError, ValueError) as e:  # a truncated or corrupt .jsonl.gz, or not UTF-8
+        raise OpError(f"prompt set {p.name}: cannot read ({e})") from None
+    sha = hashlib.sha256(raw).hexdigest()
+    manifest = _manifest(p)
+    if manifest and manifest.get("sha256") and manifest["sha256"] != sha:
+        raise OpError(f"prompt set {p.name}: sha256 {sha[:12]}... does not match its manifest "
+                      f"({str(manifest['sha256'])[:12]}...): the file changed after it was built")
+    prompts, ids, per_bucket = [], set(), {}
+    for n, line in enumerate(text.splitlines(), 1):
+        if not line.strip():
+            continue
+        try:
+            obj = json.loads(line)
+        except ValueError as e:
+            raise OpError(f"{p.name}:{n}: not JSON ({e})") from None
+        prompt = _parse(obj, f"{p.name}:{n}")
+        if prompt.id in ids:
+            raise OpError(f"{p.name}:{n}: duplicate id {prompt.id!r}")
+        ids.add(prompt.id)
+        if buckets and prompt.bucket not in buckets:
+            continue
+        k = per_bucket.get(prompt.bucket, 0)
+        if limit_per_bucket and k >= limit_per_bucket:
+            continue
+        per_bucket[prompt.bucket] = k + 1
+        prompts.append(prompt)
+    if not prompts:
+        raise OpError(f"prompt set {p.name}: no prompts" + (f" in buckets {buckets}" if buckets else ""))
+    name = (manifest or {}).get("name") or _sets.set_stem(p)
+    return PromptSet(name=name, path=p, sha256=sha, prompts=tuple(prompts), manifest=manifest)
+
+
+def _manifest(p: Path) -> dict | None:
+    m = _sets.manifest_path(p)
+    if not m.is_file():
+        return None
+    try:
+        value = json.loads(m.read_text())
+    except ValueError:
+        raise OpError(f"{m.name}: not JSON") from None
+    return value if isinstance(value, dict) else None
+
+
+def _parse(obj, where: str) -> Prompt:
+    if not isinstance(obj, dict):
+        raise OpError(f"{where}: each line must be a JSON object")
+    pid, bucket, messages = obj.get("id"), obj.get("bucket"), obj.get("messages")
+    if not isinstance(pid, str) or not pid or "/" in pid or pid.startswith("."):
+        raise OpError(f"{where}: 'id' must be a non-empty string without '/'")
+    if not isinstance(bucket, str) or not bucket or "/" in bucket or bucket.startswith("."):
+        raise OpError(f"{where}: 'bucket' must be a non-empty string without '/'")
+    if not isinstance(messages, list) or not messages or not all(
+            isinstance(m, dict) and isinstance(m.get("role"), str) for m in messages):
+        raise OpError(f"{where}: 'messages' must be a non-empty list of {{role, content}} objects")
+    max_tokens = obj.get("max_tokens")
+    if max_tokens is not None and (not isinstance(max_tokens, int) or isinstance(max_tokens, bool) or max_tokens < 1):
+        raise OpError(f"{where}: 'max_tokens' must be a positive integer")
+    extra = {k: v for k, v in obj.items() if k not in ("id", "bucket", "messages", "max_tokens", "kind")}
+    return Prompt(id=pid, bucket=bucket, messages=tuple(messages), max_tokens=max_tokens,
+                  kind=str(obj.get("kind") or "chat"), extra=extra)