benchkit-cli 0.1.0__py3-none-any.whl

benchkit/mc_viewer/dist/worker.js.LICENSE.txt

@@ -0,0 +1,19 @@
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <feross@feross.org> <http://feross.org>
+ * @license  MIT
+ */
+
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */
+
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/** @license URI.js v4.4.1 (c) 2011 Gary Court. License: http://github.com/garycourt/uri-js */

benchkit/metrics.py

@@ -0,0 +1,100 @@
+"""Throughput metrics shared by execution, front-ends and reports."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+
+def throughput_metrics(
+    *,
+    output_tokens: int,
+    generation_time_s: float,
+    request_time_s: float,
+    wall_time_s: float,
+) -> dict[str, float]:
+    """Calculate stream speed, total throughput and occupied request slots.
+
+    ``generation_time_s`` is the sum of server-reported decode times, while
+    ``request_time_s`` is the sum of complete per-task request durations.
+    Their denominators intentionally differ: stream speed describes decoder
+    performance and effective concurrency describes client/server saturation.
+    """
+    return {
+        "tok_s_per_stream": (
+            output_tokens / generation_time_s if generation_time_s > 0 else 0.0
+        ),
+        "tok_s_aggregate": (output_tokens / wall_time_s if wall_time_s > 0 else 0.0),
+        "concurrency_eff": (request_time_s / wall_time_s if wall_time_s > 0 else 0.0),
+    }
+
+
+def _number(value: object) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def stream_tok_s(result: Mapping[str, object]) -> float:
+    """Read stream throughput from new or legacy result data."""
+    if "tok_s_per_stream" in result:
+        return _number(result["tok_s_per_stream"])
+    return _number(result.get("tok_s"))
+
+
+def effective_concurrency(result: Mapping[str, object]) -> float:
+    """Read or derive occupied request slots from a result."""
+    if "concurrency_eff" in result:
+        return _number(result["concurrency_eff"])
+    wall_time = _number(result.get("total_time"))
+    if wall_time <= 0:
+        return 0.0
+    derived = (
+        _number(result.get("avg_response_time"))
+        * _number(result.get("total"))
+        / wall_time
+    )
+    configured = _number(result.get("concurrency"))
+    return min(configured, derived) if configured > 0 else derived
+
+
+def aggregate_tok_s(result: Mapping[str, object]) -> float:
+    """Read or derive wall-clock output throughput from a result."""
+    if "tok_s_aggregate" in result:
+        return _number(result["tok_s_aggregate"])
+
+    wall_time = _number(result.get("total_time"))
+    tasks = result.get("tasks")
+    if wall_time > 0 and isinstance(tasks, list):
+        token_values = [
+            _number(task.get("output_tokens"))
+            for task in tasks
+            if isinstance(task, Mapping) and "output_tokens" in task
+        ]
+        if token_values:
+            return sum(token_values) / wall_time
+
+    # Legacy CSV summaries do not contain tasks. Stream throughput multiplied
+    # by measured slot occupancy is the closest derivation available.
+    return stream_tok_s(result) * effective_concurrency(result)
+
+
+def is_decision(result: Mapping[str, object]) -> bool:
+    """Whether a result came from a native decision model."""
+    return result.get("harness") == "decision"
+
+
+def decision_speed(result: Mapping[str, object]) -> tuple[str, str]:
+    """Decision models emit no tokens, so speed is decisions/s plus latency."""
+    return (
+        f"{_number(result.get('decisions_per_s')):.1f} dec/s",
+        f"p50 {_number(result.get('decision_latency_p50_ms')):.0f} ms · "
+        f"p95 {_number(result.get('decision_latency_p95_ms')):.0f} ms",
+    )
+
+
+def latency_text(result: Mapping[str, object]) -> str:
+    """Typical request time: median ms for decisions, mean seconds otherwise."""
+    if is_decision(result):
+        return f"{_number(result.get('decision_latency_p50_ms')):.0f}ms"
+    return f"{result.get('avg_response_time', 0)}s"

benchkit/ollama.py

@@ -0,0 +1,60 @@
+"""Ollama API client."""
+
+import os
+
+import httpx
+
+DEFAULT_HOST = "http://localhost:11434"
+
+
+def get_host() -> str:
+    return os.environ.get("OLLAMA_HOST", DEFAULT_HOST).rstrip("/")
+
+
+def list_models(host: str) -> list[dict]:
+    r = httpx.get(f"{host}/api/tags", timeout=10)
+    r.raise_for_status()
+    models = r.json().get("models", [])
+    return sorted(models, key=lambda m: m.get("size", 0))
+
+
+def unload_model(host: str, model: str) -> None:
+    """Evict model from Ollama VRAM by setting keep_alive to 0."""
+    httpx.post(
+        f"{host}/api/generate",
+        json={"model": model, "keep_alive": 0},
+        timeout=30,
+    )
+
+
+def generate(host: str, model: str, prompt: str) -> dict:
+    r = httpx.post(
+        f"{host}/api/generate",
+        json={
+            "model": model,
+            "prompt": prompt,
+            "stream": False,
+            "options": {"temperature": 0.0},
+        },
+        timeout=300,
+    )
+    r.raise_for_status()
+    data = r.json()
+
+    response = data.get("response", "")
+    done_reason = data.get("done_reason", "")
+
+    eval_count = data.get("eval_count", 0)
+    eval_duration_ns = data.get("eval_duration", 0)
+    total_duration_ns = data.get("total_duration", 0)
+    tok_s = eval_count / (eval_duration_ns / 1e9) if eval_duration_ns > 0 else 0.0
+    response_time_s = total_duration_ns / 1e9
+
+    return {
+        "response": response,
+        "tok_s": tok_s,
+        "eval_count": eval_count,
+        "eval_duration_ns": eval_duration_ns,
+        "response_time_s": response_time_s,
+        "done_reason": done_reason,
+    }

benchkit/perf.py

@@ -0,0 +1,353 @@
+"""Controlled model-speed profiling for llama.cpp-compatible servers."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import platform
+import statistics
+from collections.abc import Callable
+from dataclasses import asdict, dataclass
+from datetime import datetime
+
+from benchkit.client import InferenceClient
+
+DEFAULT_DEPTHS = (0, 4096, 16384, 32768, 65536)
+DEPTH_TOLERANCE = 0.01
+DRIFT_TOLERANCE = 0.10
+MIN_RELIABLE_PP_TOKENS = 256
+FILLER = (
+    "A local inference server processes this deterministic BenchKit workload. "
+    "The passage is intentionally ordinary so tokenization remains stable "
+    "across repetitions while context depth changes. "
+)
+
+
+@dataclass(frozen=True)
+class PerfConfig:
+    """Inputs that make a performance profile reproducible."""
+
+    model: str
+    depths: tuple[int, ...] = DEFAULT_DEPTHS
+    gen_tokens: int = 128
+    reps: int = 5
+    warmups: int = 1
+    config_note: str = ""
+
+
+def parse_depths(value: str) -> tuple[int, ...]:
+    """Parse comma-separated token depths such as ``minimal,4k,16k``."""
+    depths: list[int] = []
+    for raw in value.split(","):
+        item = raw.strip().lower()
+        if item in {"minimal", "min", "0"}:
+            depth = 0
+        else:
+            multiplier = 1
+            if item.endswith("k"):
+                multiplier, item = 1024, item[:-1]
+            elif item.endswith("m"):
+                multiplier, item = 1024 * 1024, item[:-1]
+            try:
+                depth = int(float(item) * multiplier)
+            except ValueError as exc:
+                raise ValueError(f"Invalid depth: {raw.strip()!r}") from exc
+            if depth < 1:
+                raise ValueError(f"Invalid depth: {raw.strip()!r}")
+        if depth not in depths:
+            depths.append(depth)
+    if not depths:
+        raise ValueError("At least one context depth is required")
+    return tuple(depths)
+
+
+def depth_label(depth: int) -> str:
+    if depth == 0:
+        return "minimal"
+    if depth % 1024 == 0:
+        return f"{depth // 1024}k"
+    return f"{depth:,}"
+
+
+def _filler_text(length: int) -> str:
+    repetitions = max(1, length // len(FILLER) + 1)
+    return (FILLER * repetitions)[:length]
+
+
+def _prompt_for(
+    client: InferenceClient,
+    model: str,
+    target_depth: int,
+    announce: Callable[[str], None],
+) -> tuple[str | list[int], str, list[dict]]:
+    if target_depth == 0:
+        return (
+            "BenchKit speed profile. Continue with concise factual prose:",
+            "minimal",
+            [],
+        )
+
+    content = _filler_text(target_depth * 8)
+    tokens = client.tokenize(model, content)
+    if tokens is not None:
+        while len(tokens) < target_depth:
+            content += _filler_text(target_depth * 2)
+            tokens = client.tokenize(model, content)
+            if tokens is None:
+                break
+        if tokens is not None and len(tokens) >= target_depth:
+            return tokens[:target_depth], "tokenizer", []
+
+    # llama-swap commonly proxies the OpenAI routes but not llama.cpp's native
+    # tokenizer. Calibrate text length against the completion response instead
+    # of silently trusting a characters-per-token estimate.
+    prompt = _filler_text(target_depth * 5)
+    calibration: list[dict] = []
+    for attempt in range(3):
+        announce(f"{depth_label(target_depth)} · calibrating input {attempt + 1}/3")
+        try:
+            measured = client.profile_completion(model, prompt, 1)
+        except Exception as exc:
+            calibration.append(
+                {
+                    "attempt": attempt + 1,
+                    "characters": len(prompt),
+                    "error": f"{type(exc).__name__}: {exc}",
+                }
+            )
+            break
+
+        actual = int(measured.get("input_tokens") or 0)
+        calibration.append(
+            {
+                "attempt": attempt + 1,
+                "characters": len(prompt),
+                "actual_input_tokens": actual,
+            }
+        )
+        if not actual:
+            break
+        error_ratio = abs(actual - target_depth) / target_depth
+        if error_ratio <= DEPTH_TOLERANCE:
+            return prompt, "calibrated", calibration
+        prompt = _filler_text(max(1, round(len(prompt) * target_depth / actual)))
+
+    calibrated = any(item.get("actual_input_tokens") for item in calibration)
+    return prompt, "calibrated" if calibrated else "estimated", calibration
+
+
+def _summary(values: list[float]) -> dict[str, float]:
+    if not values:
+        return {"median": 0.0, "min": 0.0, "max": 0.0}
+    return {
+        "median": round(statistics.median(values), 3),
+        "min": round(min(values), 3),
+        "max": round(max(values), 3),
+    }
+
+
+def _summarize_case(
+    requested_depth: int,
+    depth_method: str,
+    calibration: list[dict],
+    runs: list[dict],
+    errors: list[str],
+) -> dict:
+    successful = [run for run in runs if not run.get("error")]
+    actual_depths = [run["input_tokens"] for run in successful if run["input_tokens"]]
+    actual_depth = int(statistics.median(actual_depths)) if actual_depths else 0
+    depth_deviation = (
+        (actual_depth - requested_depth) / requested_depth
+        if requested_depth and actual_depth
+        else 0.0
+    )
+    return {
+        "depth": requested_depth,
+        "depth_label": depth_label(requested_depth),
+        "actual_input_tokens": actual_depth,
+        "depth_method": depth_method,
+        "exact_depth": depth_method == "tokenizer",
+        "depth_deviation": round(depth_deviation, 5),
+        "depth_within_tolerance": (
+            requested_depth == 0
+            or (actual_depth > 0 and abs(depth_deviation) <= DEPTH_TOLERANCE)
+        ),
+        "calibration": calibration,
+        "pp_reliable": actual_depth >= MIN_RELIABLE_PP_TOKENS,
+        "successful_reps": len(successful),
+        "errors": errors,
+        "pp_tps": _summary([run["pp_tps"] for run in successful]),
+        "tg_tps": _summary([run["tg_tps"] for run in successful]),
+        "ttft_s": _summary([run["ttft_s"] for run in successful]),
+        "wall_time_s": _summary([run["wall_time_s"] for run in successful]),
+        "wall_tg_tps": _summary([run["wall_tg_tps"] for run in successful]),
+        "overhead_available": any(
+            run.get("server_time_s", 0) > 0 for run in successful
+        ),
+        "overhead_s": _summary([run["overhead_s"] for run in successful]),
+        "output_tokens": _summary([float(run["output_tokens"]) for run in successful]),
+        "runs": runs,
+    }
+
+
+def run_profile(
+    client: InferenceClient,
+    config: PerfConfig,
+    *,
+    model_meta: dict | None = None,
+    progress: Callable[[str], None] | None = None,
+) -> dict:
+    """Run a serial context-depth sweep and return a complete raw profile."""
+
+    def announce(message: str) -> None:
+        if progress is not None:
+            progress(message)
+
+    cases: list[dict] = []
+    startup: dict | None = None
+    first_prompt: str | list[int] | None = None
+
+    # Run this before tokenization because llama-swap may need to load the model
+    # even for its native tokenizer endpoint.
+    if config.warmups:
+        announce("First request · startup probe")
+        try:
+            measured = client.profile_completion(
+                config.model,
+                "BenchKit startup probe. Continue with concise factual prose:",
+                min(config.gen_tokens, 32),
+            )
+            startup = {
+                "ttft_s": round(measured["ttft_s"], 3),
+                "wall_time_s": round(measured["wall_time_s"], 3),
+                "label": "may include llama-swap model loading",
+            }
+        except Exception as exc:
+            startup = {
+                "error": f"{type(exc).__name__}: {exc}",
+                "label": "first request",
+            }
+
+    for depth in config.depths:
+        label = depth_label(depth)
+        announce(f"Preparing {label} context")
+        prompt, depth_method, calibration = _prompt_for(
+            client, config.model, depth, announce
+        )
+        if first_prompt is None:
+            first_prompt = prompt
+
+        for warmup in range(config.warmups):
+            announce(f"{label} · warmup {warmup + 1}/{config.warmups}")
+            with contextlib.suppress(Exception):
+                client.profile_completion(
+                    config.model, prompt, min(config.gen_tokens, 32)
+                )
+
+        runs: list[dict] = []
+        errors: list[str] = []
+        for repetition in range(config.reps):
+            announce(f"{label} · run {repetition + 1}/{config.reps}")
+            try:
+                measured = client.profile_completion(
+                    config.model, prompt, config.gen_tokens
+                )
+                runs.append(
+                    {
+                        "repetition": repetition + 1,
+                        **{
+                            key: round(value, 4) if isinstance(value, float) else value
+                            for key, value in measured.items()
+                            if key != "response"
+                        },
+                    }
+                )
+            except Exception as exc:
+                error = f"{type(exc).__name__}: {exc}"
+                errors.append(error)
+                runs.append({"repetition": repetition + 1, "error": error})
+
+        cases.append(
+            _summarize_case(
+                depth,
+                depth_method,
+                calibration,
+                runs,
+                errors,
+            )
+        )
+
+    drift_check: dict | None = None
+    if first_prompt is not None and cases:
+        announce(f"{cases[0]['depth_label']} · end-of-sweep drift check")
+        try:
+            measured = client.profile_completion(
+                config.model, first_prompt, config.gen_tokens
+            )
+            initial_tg = float(cases[0]["tg_tps"]["median"])
+            final_tg = float(measured["tg_tps"])
+            change = (final_tg - initial_tg) / initial_tg if initial_tg else 0.0
+            drift_check = {
+                "depth": cases[0]["depth"],
+                "depth_label": cases[0]["depth_label"],
+                "initial_tg_tps": round(initial_tg, 3),
+                "final_tg_tps": round(final_tg, 3),
+                "change": round(change, 5),
+                "stable": bool(initial_tg) and abs(change) <= DRIFT_TOLERANCE,
+                "tolerance": DRIFT_TOLERANCE,
+            }
+        except Exception as exc:
+            drift_check = {
+                "depth": cases[0]["depth"],
+                "depth_label": cases[0]["depth_label"],
+                "error": f"{type(exc).__name__}: {exc}",
+                "stable": False,
+                "tolerance": DRIFT_TOLERANCE,
+            }
+
+    tg_depth_medians = [
+        float(case["tg_tps"]["median"])
+        for case in cases
+        if case["successful_reps"] and case["tg_tps"]["median"] > 0
+    ]
+    pp_depth_medians = [
+        float(case["pp_tps"]["median"])
+        for case in cases
+        if (
+            case["successful_reps"]
+            and case["pp_reliable"]
+            and case["pp_tps"]["median"] > 0
+        )
+    ]
+    sweep_summary = {
+        "tg_tps_median": round(statistics.median(tg_depth_medians), 3)
+        if tg_depth_medians
+        else 0.0,
+        "pp_tps_median": round(statistics.median(pp_depth_medians), 3)
+        if pp_depth_medians
+        else 0.0,
+        "tg_depths": len(tg_depth_medians),
+        "pp_depths": len(pp_depth_medians),
+    }
+
+    return {
+        "kind": "benchkit-performance-profile",
+        "version": 1,
+        "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
+        "model": config.model,
+        "provider": client.label,
+        "host": client.host,
+        "environment": {
+            "hardware": os.environ.get("BENCHKIT_HARDWARE", ""),
+            "system": platform.system(),
+            "machine": platform.machine(),
+            "config_note": config.config_note
+            or os.environ.get("BENCHKIT_PERF_CONFIG", ""),
+        },
+        "model_metadata": model_meta or {},
+        "settings": asdict(config),
+        "startup": startup,
+        "drift_check": drift_check,
+        "sweep_summary": sweep_summary,
+        "cases": cases,
+    }