inferbench-cli 0.1.0__py3-none-any.whl

inferbench/hardware/detect.py

@@ -0,0 +1,109 @@
+"""
+Hardware detection, ported from src/hardware/detect.ts. Node's `os` module
+gives platform/arch/cpu-model/total-memory for free; Python's standard
+library has no single equivalent, so this module composes `sys.platform`,
+`platform.machine()`, POSIX `os.sysconf()`, and a couple of small,
+fixed-argv subprocess calls (never a shell string, never user input) to
+reach the same shape of data on macOS and Linux. Every value here degrades
+to a safe fallback ("unknown" / 0.0) rather than raising, matching the
+original's contract of always returning a usable HardwareProfile.
+"""
+from __future__ import annotations
+
+import os
+import platform
+import subprocess
+import sys
+
+from ..types import HardwareProfile
+
+
+def _detect_platform() -> str:
+    # Node's os.platform() and Python's sys.platform use the same tokens
+    # for the platforms this tool targets: "darwin", "linux", "win32".
+    return sys.platform
+
+
+def _detect_arch() -> str:
+    machine = platform.machine().lower()
+    if machine in ("x86_64", "amd64"):
+        return "x64"
+    if machine == "aarch64":
+        return "arm64"
+    return machine or "unknown"
+
+
+def _total_memory_gb() -> float:
+    try:
+        page_size = os.sysconf("SC_PAGE_SIZE")
+        phys_pages = os.sysconf("SC_PHYS_PAGES")
+        total_bytes = page_size * phys_pages
+        return round(total_bytes / 1024**3, 1)
+    except (ValueError, OSError, AttributeError):
+        pass
+    if sys.platform == "win32":
+        try:
+            import ctypes
+
+            class _MemoryStatusEx(ctypes.Structure):
+                _fields_ = [
+                    ("dwLength", ctypes.c_ulong),
+                    ("dwMemoryLoad", ctypes.c_ulong),
+                    ("ullTotalPhys", ctypes.c_ulonglong),
+                    ("ullAvailPhys", ctypes.c_ulonglong),
+                    ("ullTotalPageFile", ctypes.c_ulonglong),
+                    ("ullAvailPageFile", ctypes.c_ulonglong),
+                    ("ullTotalVirtual", ctypes.c_ulonglong),
+                    ("ullAvailVirtual", ctypes.c_ulonglong),
+                    ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
+                ]
+
+            stat = _MemoryStatusEx()
+            stat.dwLength = ctypes.sizeof(_MemoryStatusEx)
+            ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))  # type: ignore[attr-defined]
+            return round(stat.ullTotalPhys / 1024**3, 1)
+        except Exception:  # noqa: BLE001 -- best-effort fallback, never fatal
+            pass
+    return 0.0
+
+
+def _cpu_model() -> str:
+    if sys.platform == "darwin":
+        try:
+            result = subprocess.run(
+                ["sysctl", "-n", "machdep.cpu.brand_string"],
+                capture_output=True,
+                text=True,
+                timeout=2,
+                check=True,
+            )
+            model = result.stdout.strip()
+            if model:
+                return model
+        except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            pass
+    elif sys.platform.startswith("linux"):
+        try:
+            with open("/proc/cpuinfo", encoding="utf-8") as handle:
+                for line in handle:
+                    if line.lower().startswith("model name"):
+                        _, _, value = line.partition(":")
+                        model = value.strip()
+                        if model:
+                            return model
+        except OSError:
+            pass
+    processor = platform.processor()
+    return processor if processor else "unknown"
+
+
+def detect_hardware() -> HardwareProfile:
+    plat = _detect_platform()
+    arch = _detect_arch()
+    return HardwareProfile(
+        platform=plat,
+        arch=arch,
+        total_memory_gb=_total_memory_gb(),
+        cpu_model=_cpu_model(),
+        is_apple_silicon=(plat == "darwin" and arch == "arm64"),
+    )

inferbench/harness/measure.py

@@ -0,0 +1,103 @@
+"""
+Ported from src/harness/measure.ts. Sends one timed chat-completion request
+to an engine's OpenAI-compatible server and measures wall-clock time plus
+reported token counts -- the ONE shared measurement code path for every
+engine (the architecture decision that replaced per-engine benchmark-CLI
+parsing; omlx has no CLI benchmark tool at all, verified against its real
+README before the original tool's adapter code was written).
+
+Elapsed time is measured across the FULL response body, not just the
+headers: urlopen() returns once headers arrive, before the response body
+(and therefore generation) has finished streaming. The original TypeScript
+harness had a real bug here once -- measuring right after `await fetch(...)`
+resolved, which produced a physically impossible 64,646 tok/s during a live
+end-to-end run before it was caught and fixed. This port measures after
+`response.read()` completes, deliberately not reintroducing that bug.
+"""
+from __future__ import annotations
+
+import json
+import socket
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+
+from ..errors import BenchmarkParseError, EngineRequestTimeoutError
+from ..types import CompletionResult
+
+
+@dataclass
+class TimedCompletionOptions:
+    engine: str
+    base_url: str
+    model_id: str
+    prompt: str
+    max_tokens: int
+    timeout_ms: int
+
+
+def timed_completion(opts: TimedCompletionOptions) -> CompletionResult:
+    body = json.dumps(
+        {
+            "model": opts.model_id,
+            "messages": [{"role": "user", "content": opts.prompt}],
+            "max_tokens": opts.max_tokens,
+            "stream": False,
+        }
+    ).encode("utf-8")
+
+    request = urllib.request.Request(
+        f"{opts.base_url}/v1/chat/completions",
+        data=body,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+
+    start = time.perf_counter()
+    try:
+        response = urllib.request.urlopen(  # noqa: S310 -- fixed, internally-constructed URL only
+            request, timeout=opts.timeout_ms / 1000
+        )
+        raw_bytes = response.read()
+    except urllib.error.HTTPError as err:
+        error_body = err.read().decode("utf-8", errors="replace") if err.fp else ""
+        raise BenchmarkParseError(opts.engine, f"HTTP {err.code}: {error_body}") from err
+    except (socket.timeout, TimeoutError) as err:
+        raise EngineRequestTimeoutError(opts.engine, opts.timeout_ms) from err
+    except urllib.error.URLError as err:
+        if isinstance(err.reason, (socket.timeout, TimeoutError)):
+            raise EngineRequestTimeoutError(opts.engine, opts.timeout_ms) from err
+        raise
+
+    elapsed_ms = (time.perf_counter() - start) * 1000
+    raw_text = raw_bytes.decode("utf-8", errors="replace")
+
+    try:
+        payload = json.loads(raw_text)
+    except json.JSONDecodeError as err:
+        raise BenchmarkParseError(opts.engine, raw_text) from err
+
+    usage = payload.get("usage") if isinstance(payload, dict) else None
+    completion_tokens = usage.get("completion_tokens") if isinstance(usage, dict) else None
+    if not isinstance(completion_tokens, (int, float)) or isinstance(completion_tokens, bool):
+        completion_tokens = None
+
+    prompt_tokens_details = usage.get("prompt_tokens_details") if isinstance(usage, dict) else None
+    cached_raw = (
+        prompt_tokens_details.get("cached_tokens") if isinstance(prompt_tokens_details, dict) else None
+    )
+    cached_prompt_tokens = (
+        int(cached_raw) if isinstance(cached_raw, (int, float)) and not isinstance(cached_raw, bool) else 0
+    )
+
+    tokens_per_second = (
+        completion_tokens / (elapsed_ms / 1000) if completion_tokens and elapsed_ms > 0 else None
+    )
+
+    return CompletionResult(
+        elapsed_ms=round(elapsed_ms),
+        completion_tokens=int(completion_tokens) if completion_tokens is not None else None,
+        cached_prompt_tokens=cached_prompt_tokens,
+        tokens_per_second=round(tokens_per_second, 2) if tokens_per_second is not None else None,
+    )

inferbench/harness/spawn_server.py

@@ -0,0 +1,69 @@
+"""
+Ported from src/harness/spawn-server.ts. Spawns a long-running engine
+server process and polls a URL until it responds, so callers get a server
+that is actually ready rather than racing against startup.
+
+Security note: `command` and `args` are always a fixed argv list passed to
+subprocess.Popen with no shell involved (no shell=True anywhere in this
+module or its callers) -- the model spec a user passes on --model becomes
+a single argv element, never string-concatenated into a shell command, so
+there is no command-injection path through it. This mirrors the same
+argv-array discipline the original TypeScript harness documents in its own
+comments.
+"""
+from __future__ import annotations
+
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Callable, List, Optional
+
+from ..errors import EngineStartTimeoutError
+
+
+@dataclass
+class SpawnedServer:
+    process: subprocess.Popen
+    stop: Callable[[], None]
+
+
+def spawn_server_and_wait_ready(
+    *,
+    engine: str,
+    command: str,
+    args: List[str],
+    ready_check_url: str,
+    timeout_ms: int,
+    poll_interval_ms: int = 500,
+    verbose: bool = False,
+) -> SpawnedServer:
+    stdio = None if verbose else subprocess.DEVNULL
+    process = subprocess.Popen([command, *args], stdout=stdio, stderr=stdio)  # noqa: S603 -- fixed argv, no shell
+
+    def stop() -> None:
+        if process.poll() is None:
+            process.kill()
+
+    deadline = time.monotonic() + timeout_ms / 1000
+    last_error: Optional[BaseException] = None
+
+    while time.monotonic() < deadline:
+        if process.poll() is not None:
+            raise RuntimeError(
+                f"{engine}: process exited early (code {process.returncode}) before becoming ready"
+            )
+        try:
+            with urllib.request.urlopen(ready_check_url, timeout=2) as response:  # noqa: S310
+                if 200 <= response.status < 300:
+                    return SpawnedServer(process=process, stop=stop)
+        except Exception as err:  # noqa: BLE001 -- polling loop; any failure just means "not ready yet"
+            last_error = err
+        time.sleep(poll_interval_ms / 1000)
+
+    stop()
+    timeout_err = EngineStartTimeoutError(engine, timeout_ms)
+    if last_error is not None:
+        timeout_err.__cause__ = last_error
+    raise timeout_err

inferbench/prompts.py

@@ -0,0 +1,22 @@
+"""
+Default varied prompt set, ported verbatim from src/prompts.ts: 8 distinct
+prompts across different topics and lengths, deliberately never repeating
+the exact same prompt, to avoid the prefix-cache skew observed during the
+original TypeScript tool's real validation (a repeated identical prompt let
+llama.cpp reuse 39/40 cached prompt tokens on the second call, which is not
+a fair steady-state comparison).
+"""
+from __future__ import annotations
+
+DEFAULT_PROMPTS: list[str] = [
+    "Explain in one paragraph why the sky is blue.",
+    "Write a short haiku about autumn leaves falling in a quiet forest.",
+    "What are three practical tips for someone learning to cook rice perfectly every time?",
+    "Summarize the plot of a story about a lighthouse keeper who discovers a message in a bottle.",
+    "List five differences between a cat and a dog as household pets.",
+    "Describe how a bicycle chain transfers power from the pedals to the rear wheel.",
+    "Give a brief explanation of why leaves change color in autumn, covering chlorophyll and other pigments.",
+    "What is the difference between weather and climate? Explain with a simple example.",
+]
+
+WARMUP_PROMPT = "Say hello."

inferbench/recommend/config.py

@@ -0,0 +1,29 @@
+"""
+Ported from src/recommend/config.ts.
+
+v0.1 recommendation rule: highest measured average tok/s among engines that
+were actually installed and tested. Deliberately simple -- richer
+multi-factor scoring (memory, cost) is intentionally deferred until real
+usage shows this simple rule picks wrong recommendations.
+"""
+from __future__ import annotations
+
+from typing import List, Optional
+
+from ..types import EngineBenchmarkResult, Recommendation
+
+
+def recommend(results: List[EngineBenchmarkResult]) -> Optional[Recommendation]:
+    candidates = [r for r in results if r.installed and r.avg_tokens_per_second is not None]
+    if not candidates:
+        return None
+
+    best = max(candidates, key=lambda r: r.avg_tokens_per_second or 0)
+
+    return Recommendation(
+        engine=best.engine,
+        reason=(
+            f"highest measured throughput on this run ({best.avg_tokens_per_second} tok/s avg) "
+            "-- specific to this hardware and model, not a universal ranking"
+        ),
+    )

inferbench/report/json_report.py

@@ -0,0 +1,66 @@
+"""
+Ported from src/report/json.ts, extended with a snake_case -> camelCase
+key conversion so a report written by this CLI's --out is field-for-field
+compatible with the npm CLI's own JSON report (both usable by the same
+downstream tooling) -- see docs/concepts.md.
+"""
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import asdict
+from typing import Any, Dict
+
+from ..types import BenchmarkReport
+
+
+class UnsafeOutputPathError(ValueError):
+    pass
+
+
+def _to_camel_case(key: str) -> str:
+    head, *tail = key.split("_")
+    return head + "".join(part.title() for part in tail)
+
+
+def _camelize(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {_to_camel_case(k): _camelize(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_camelize(v) for v in value]
+    return value
+
+
+def report_to_dict(report: BenchmarkReport) -> Dict[str, Any]:
+    """BenchmarkReport never holds a StartedServer (that only lives on the
+    adapter's return value during a run, not in the final report), so a
+    plain dataclasses.asdict() is safe here -- nothing unconvertible in the
+    tree."""
+    return _camelize(asdict(report))
+
+
+# --out is a plain CLI flag today, but this CLI is also meant to be invoked
+# programmatically by agents that may derive the value from less-trusted
+# input (a fetched benchmark config, an LLM-generated argument list, etc).
+# A relative path containing ".." segments can escape the intended output
+# location entirely (--out ../../../etc/cron.d/x) -- reject any --out value
+# that resolves outside the current working directory. An explicit absolute
+# path is still allowed: that's a value the caller typed/passed directly,
+# not one that silently escaped via traversal.
+def _assert_safe_output_path(file_path: str) -> None:
+    if os.path.isabs(file_path):
+        return
+    cwd = os.path.realpath(os.getcwd())
+    resolved = os.path.realpath(os.path.join(cwd, file_path))
+    if resolved != cwd and not resolved.startswith(cwd + os.sep):
+        raise UnsafeOutputPathError(
+            f'--out "{file_path}" resolves outside the current working directory '
+            f"({resolved}). Pass an absolute path if you intend to write outside "
+            "the working directory."
+        )
+
+
+def write_json_report(report: BenchmarkReport, file_path: str) -> None:
+    _assert_safe_output_path(file_path)
+    with open(file_path, "w", encoding="utf-8") as handle:
+        json.dump(report_to_dict(report), handle, indent=2)

inferbench/types.py

@@ -0,0 +1,95 @@
+"""
+Shared data types, ported from src/types.ts. Field names follow Python
+convention (snake_case) rather than the TypeScript originals' camelCase --
+the JSON report writer (report/json_report.py) converts back to camelCase
+so a --json/--out report is field-for-field compatible with the npm CLI's
+own JSON output, even though the two libraries' native call shapes differ.
+See docs/concepts.md for the full data model and the snake_case/camelCase
+note.
+"""
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass, field
+from typing import Callable, List, Optional, Protocol, runtime_checkable
+
+
+@dataclass(frozen=True)
+class HardwareProfile:
+    platform: str
+    arch: str
+    total_memory_gb: float
+    cpu_model: str
+    is_apple_silicon: bool
+
+
+@dataclass
+class StartedServer:
+    process: subprocess.Popen
+    base_url: str
+    model_id: str
+    stop: Callable[[], None]
+
+
+@dataclass
+class CompletionResult:
+    elapsed_ms: float
+    completion_tokens: Optional[int]
+    cached_prompt_tokens: int
+    tokens_per_second: Optional[float]
+
+
+@dataclass
+class PromptRunResult:
+    prompt: str
+    ok: bool
+    error: Optional[str] = None
+    result: Optional[CompletionResult] = None
+
+
+@dataclass
+class EngineBenchmarkResult:
+    engine: str
+    installed: bool
+    error: Optional[str] = None
+    runs: List[PromptRunResult] = field(default_factory=list)
+    avg_tokens_per_second: Optional[float] = None
+    min_tokens_per_second: Optional[float] = None
+    max_tokens_per_second: Optional[float] = None
+
+
+@dataclass
+class Recommendation:
+    engine: str
+    reason: str
+
+
+@dataclass
+class BenchmarkReport:
+    timestamp: str
+    hardware: HardwareProfile
+    model: str
+    engines: List[EngineBenchmarkResult]
+    recommendation: Optional[Recommendation]
+
+
+@runtime_checkable
+class EngineAdapter(Protocol):
+    """Structural type -- any object with this shape (name, binary,
+    is_installed(), start_server(...)) works as an adapter, mirroring the
+    TS interface's duck-typed usage. LlamaCppAdapter and OmlxAdapter both
+    satisfy this without inheriting from it explicitly."""
+
+    name: str
+    binary: str
+
+    def is_installed(self) -> bool: ...
+
+    def start_server(
+        self,
+        *,
+        model: str,
+        port: int,
+        timeout_ms: int,
+        verbose: bool = False,
+    ) -> StartedServer: ...