inferbench-cli 0.1.0__py3-none-any.whl

inferbench/__init__.py

@@ -0,0 +1,59 @@
+"""
+Programmatic / agent-native entry point.
+
+    from inferbench import benchmark_engine, detect_hardware, all_engines
+
+    hardware = detect_hardware()
+    adapters = all_engines()
+    results = [benchmark_engine(a, model="bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M") for a in adapters]
+
+Same core orchestration the CLI calls; inferbench.cli is a thin
+argument-parsing wrapper over these functions.
+
+This is the Python port of the inferbench-cli npm package
+(https://www.npmjs.com/package/inferbench-cli). Both distributions
+benchmark the same two supported local-inference engines (omlx,
+llama.cpp) against a fixed, varied prompt set over each engine's own
+OpenAI-compatible HTTP server; see
+https://github.com/RudrenduPaul/InferBench for the canonical
+documentation and the original TypeScript source.
+"""
+from .benchmark import benchmark_engine
+from .cost.cloud_comparison import CLOUD_PRICING_PER_1K_OUTPUT_TOKENS, CostComparison, compare_to_cloud
+from .engines.registry import SUPPORTED_ENGINES, all_engines, resolve_engines
+from .hardware.detect import detect_hardware
+from .recommend.config import recommend
+from .report.json_report import report_to_dict, write_json_report
+from .types import (
+    BenchmarkReport,
+    CompletionResult,
+    EngineBenchmarkResult,
+    HardwareProfile,
+    PromptRunResult,
+    Recommendation,
+    StartedServer,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "benchmark_engine",
+    "detect_hardware",
+    "all_engines",
+    "resolve_engines",
+    "recommend",
+    "compare_to_cloud",
+    "report_to_dict",
+    "write_json_report",
+    "SUPPORTED_ENGINES",
+    "CLOUD_PRICING_PER_1K_OUTPUT_TOKENS",
+    "BenchmarkReport",
+    "CompletionResult",
+    "CostComparison",
+    "EngineBenchmarkResult",
+    "HardwareProfile",
+    "PromptRunResult",
+    "Recommendation",
+    "StartedServer",
+    "__version__",
+]

inferbench/benchmark.py

@@ -0,0 +1,121 @@
+"""
+Ported from src/benchmark.ts. Benchmarks a single engine. Never raises for
+an engine-level failure (not installed, server start timeout, request
+failure) -- engines are isolated from each other by design, so one bad
+engine never blocks results from the others. Callers get a result object
+with installed=False or per-run errors instead.
+"""
+from __future__ import annotations
+
+from typing import Callable, List, Optional
+
+from .harness.measure import TimedCompletionOptions, timed_completion
+from .prompts import DEFAULT_PROMPTS, WARMUP_PROMPT
+from .types import EngineAdapter, EngineBenchmarkResult, PromptRunResult
+
+DEFAULT_MAX_TOKENS = 200
+DEFAULT_SERVER_START_TIMEOUT_MS = 5 * 60 * 1000
+DEFAULT_REQUEST_TIMEOUT_MS = 2 * 60 * 1000
+
+_next_port = 41000
+
+
+def _claim_port() -> int:
+    global _next_port
+    port = _next_port
+    _next_port += 1
+    return port
+
+
+def benchmark_engine(
+    adapter: EngineAdapter,
+    *,
+    model: str,
+    max_tokens: Optional[int] = None,
+    server_start_timeout_ms: Optional[int] = None,
+    request_timeout_ms: Optional[int] = None,
+    prompts: Optional[List[str]] = None,
+    verbose: bool = False,
+    on_progress: Optional[Callable[[str], None]] = None,
+) -> EngineBenchmarkResult:
+    progress = on_progress or (lambda _line: None)
+    prompt_list = prompts if prompts is not None else DEFAULT_PROMPTS
+    tokens_cap = max_tokens if max_tokens is not None else DEFAULT_MAX_TOKENS
+
+    if not adapter.is_installed():
+        progress(f"{adapter.name}: not installed, skipped")
+        return EngineBenchmarkResult(
+            engine=adapter.name,
+            installed=False,
+            error=f'binary "{adapter.binary}" not found',
+        )
+
+    progress(f"{adapter.name}: starting server...")
+    try:
+        server = adapter.start_server(
+            model=model,
+            port=_claim_port(),
+            timeout_ms=server_start_timeout_ms or DEFAULT_SERVER_START_TIMEOUT_MS,
+            verbose=verbose,
+        )
+    except Exception as err:  # noqa: BLE001 -- any start-up failure reports as a failed result, never raises
+        message = str(err)
+        progress(f"{adapter.name}: failed to start ({message})")
+        return EngineBenchmarkResult(engine=adapter.name, installed=True, error=message)
+
+    try:
+        progress(f"{adapter.name}: warming up...")
+        try:
+            timed_completion(
+                TimedCompletionOptions(
+                    engine=adapter.name,
+                    base_url=server.base_url,
+                    model_id=server.model_id,
+                    prompt=WARMUP_PROMPT,
+                    max_tokens=16,
+                    timeout_ms=request_timeout_ms or DEFAULT_REQUEST_TIMEOUT_MS,
+                )
+            )
+        except Exception as err:  # noqa: BLE001 -- any warm-up failure reports as a failed result
+            message = str(err)
+            progress(f"{adapter.name}: warm-up failed ({message})")
+            return EngineBenchmarkResult(
+                engine=adapter.name,
+                installed=True,
+                error=f"warm-up failed: {message}",
+            )
+
+        runs: List[PromptRunResult] = []
+        for i, prompt in enumerate(prompt_list):
+            progress(f"{adapter.name}: [{i + 1}/{len(prompt_list)}] benchmarking...")
+            try:
+                result = timed_completion(
+                    TimedCompletionOptions(
+                        engine=adapter.name,
+                        base_url=server.base_url,
+                        model_id=server.model_id,
+                        prompt=prompt,
+                        max_tokens=tokens_cap,
+                        timeout_ms=request_timeout_ms or DEFAULT_REQUEST_TIMEOUT_MS,
+                    )
+                )
+                runs.append(PromptRunResult(prompt=prompt, ok=True, result=result))
+            except Exception as err:  # noqa: BLE001 -- one failed prompt must not abort the sweep
+                runs.append(PromptRunResult(prompt=prompt, ok=False, error=str(err)))
+
+        valid = [
+            r.result.tokens_per_second
+            for r in runs
+            if r.ok and r.result is not None and r.result.tokens_per_second
+        ]
+
+        return EngineBenchmarkResult(
+            engine=adapter.name,
+            installed=True,
+            runs=runs,
+            avg_tokens_per_second=round(sum(valid) / len(valid), 2) if valid else None,
+            min_tokens_per_second=min(valid) if valid else None,
+            max_tokens_per_second=max(valid) if valid else None,
+        )
+    finally:
+        server.stop()

inferbench/cli.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""
+Ported from src/cli.ts (which uses `commander`); this port uses the stdlib
+`argparse` to avoid a CLI-framework dependency. Flags and defaults are kept
+identical in name and meaning to the npm CLI. Console entry point:
+`inferbench run [options]`, installed via the `inferbench` console-script
+defined in python/pyproject.toml.
+
+One deliberate, documented divergence from the TypeScript CLI: argparse's
+own parse errors (a missing required --model, an unrecognized flag) exit
+with code 2, the standard argparse/Unix convention, rather than commander's
+exit code 1 for the same case. The "ran, but no supported engine was
+installed" case still exits 1 on both CLIs, matching NoEnginesFoundError's
+documented contract. See docs/concepts.md for the full exit-code table.
+"""
+from __future__ import annotations
+
+import argparse
+import json as json_module
+import sys
+from datetime import datetime, timezone
+from typing import List
+
+from .benchmark import benchmark_engine
+from .engines.registry import SUPPORTED_ENGINES, all_engines, resolve_engines
+from .errors import NoEnginesFoundError, UsageError
+from .hardware.detect import detect_hardware
+from .recommend.config import recommend
+from .report.json_report import report_to_dict, write_json_report
+from .types import BenchmarkReport
+
+_VERSION = "0.1.0"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="inferbench",
+        description=(
+            "Benchmarks local-LLM-inference engines (omlx, llama.cpp) on your own "
+            "hardware, live."
+        ),
+    )
+    parser.add_argument("--version", action="version", version=f"inferbench {_VERSION}")
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    run_parser = subparsers.add_parser("run", help="Benchmark installed engines against a model")
+    run_parser.add_argument(
+        "--model", required=True, help="Model spec (engine-specific, see README)"
+    )
+    run_parser.add_argument(
+        "--engines",
+        default=None,
+        help=(
+            "Comma-separated engines to test (default: all supported -- "
+            f"{', '.join(SUPPORTED_ENGINES)})"
+        ),
+    )
+    run_parser.add_argument(
+        "--max-tokens", default="200", help="Max completion tokens per prompt"
+    )
+    run_parser.add_argument(
+        "--json", action="store_true", help="Output machine-readable JSON instead of a human table"
+    )
+    run_parser.add_argument("--out", default=None, help="Also write the full JSON report to this file")
+    run_parser.add_argument(
+        "--verbose", action="store_true", help="Show raw engine server stdout/stderr"
+    )
+
+    return parser
+
+
+def _parse_max_tokens(value: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError:
+        raise UsageError(
+            f'Invalid --max-tokens value "{value}": must be a whole number, e.g. --max-tokens 200'
+        ) from None
+    if parsed <= 0:
+        raise UsageError(f'Invalid --max-tokens value "{value}": must be a positive number')
+    return parsed
+
+
+def _run_command(args: argparse.Namespace) -> None:
+    adapters = resolve_engines(args.engines.split(",")) if args.engines else all_engines()
+    max_tokens = _parse_max_tokens(args.max_tokens)
+
+    hardware = detect_hardware()
+    if not args.json:
+        print(
+            f"Hardware: {hardware.cpu_model} ({hardware.platform}/{hardware.arch}), "
+            f"{hardware.total_memory_gb}GB\n"
+        )
+
+    results = []
+    for adapter in adapters:
+        result = benchmark_engine(
+            adapter,
+            model=args.model,
+            max_tokens=max_tokens,
+            verbose=args.verbose,
+            on_progress=None if args.json else print,
+        )
+        results.append(result)
+
+    if not any(r.installed for r in results):
+        raise NoEnginesFoundError()
+
+    report = BenchmarkReport(
+        timestamp=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+        hardware=hardware,
+        model=args.model,
+        engines=results,
+        recommendation=recommend(results),
+    )
+
+    if args.out:
+        write_json_report(report, args.out)
+
+    if args.json:
+        print(json_module.dumps(report_to_dict(report), indent=2))
+        return
+
+    print("\nResults:")
+    for r in results:
+        if not r.installed:
+            print(f"  {r.engine}: not installed, skipped")
+            continue
+        if r.avg_tokens_per_second is None:
+            print(f"  {r.engine}: FAILED ({r.error or 'no successful runs'})")
+            continue
+        n = sum(1 for run in r.runs if run.ok)
+        print(
+            f"  {r.engine}: avg {r.avg_tokens_per_second} tok/s "
+            f"(range {r.min_tokens_per_second}-{r.max_tokens_per_second}, n={n})"
+        )
+
+    if report.recommendation:
+        print(f"\nRecommendation: {report.recommendation.engine} -- {report.recommendation.reason}")
+
+    if args.out:
+        print(f"\nFull report: {args.out}")
+
+
+def run_cli(argv: List[str]) -> int:
+    """`argv` follows the sys.argv convention: argv[0] is the program name,
+    the real arguments start at argv[1]. Returns the process exit code (0
+    on a successful run with at least one engine tested; 1 on a usage error
+    or when no supported engine is installed). Invalid CLI input calls
+    sys.exit(2) directly via argparse, same as build_parser()'s own error
+    handling."""
+    parser = build_parser()
+    args = parser.parse_args(argv[1:])
+
+    if args.command != "run":
+        parser.print_help()
+        return 0
+
+    try:
+        _run_command(args)
+        return 0
+    except (UsageError, NoEnginesFoundError) as err:
+        print(str(err), file=sys.stderr)
+        return 1
+
+
+def main() -> None:
+    sys.exit(run_cli(sys.argv))
+
+
+if __name__ == "__main__":
+    main()

inferbench/cost/cloud_comparison.py

@@ -0,0 +1,42 @@
+"""
+Ported from src/cost/cloud_comparison.ts.
+
+Static, versioned pricing snapshot -- not a live API call. Prices drift;
+this table is a directional reference, not a real-time quote. Update
+PRICING_SNAPSHOT_DATE whenever the numbers below are refreshed.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+PRICING_SNAPSHOT_DATE = "2026-07-15"
+
+CLOUD_PRICING_PER_1K_OUTPUT_TOKENS: Dict[str, float] = {
+    "claude-5-haiku": 0.0008,
+    "claude-5-sonnet": 0.006,
+}
+
+
+@dataclass
+class CostComparison:
+    cloud_model: str
+    cloud_cost_per_1k_tokens_usd: float
+    pricing_snapshot_date: str
+    note: str
+
+
+def compare_to_cloud(cloud_model: str) -> Optional[CostComparison]:
+    price = CLOUD_PRICING_PER_1K_OUTPUT_TOKENS.get(cloud_model)
+    if price is None:
+        return None
+    return CostComparison(
+        cloud_model=cloud_model,
+        cloud_cost_per_1k_tokens_usd=price,
+        pricing_snapshot_date=PRICING_SNAPSHOT_DATE,
+        note=(
+            "Local hardware's own amortized cost is not included -- this compares raw "
+            "generation cost only. Local inference has $0 marginal per-token cost once "
+            "hardware is already owned."
+        ),
+    )

inferbench/engines/llamacpp.py

@@ -0,0 +1,61 @@
+"""
+Ported from src/engines/llamacpp.ts. `model` is treated as a Hugging Face
+repo spec (e.g. "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M") -- llama.cpp's
+own `-hf` flag downloads and caches it automatically, no manual step needed.
+"""
+from __future__ import annotations
+
+import subprocess
+
+from ..errors import EngineNotFoundError, UsageError
+from ..harness.spawn_server import spawn_server_and_wait_ready
+from ..types import StartedServer
+
+_BINARY = "llama-server"
+
+
+class LlamaCppAdapter:
+    name = "llama.cpp"
+    binary = _BINARY
+
+    def is_installed(self) -> bool:
+        try:
+            subprocess.run(  # noqa: S603, S607 -- fixed argv, binary name only, no shell
+                [_BINARY, "--version"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=True,
+            )
+            return True
+        except (OSError, subprocess.CalledProcessError):
+            return False
+
+    def start_server(
+        self,
+        *,
+        model: str,
+        port: int,
+        timeout_ms: int,
+        verbose: bool = False,
+    ) -> StartedServer:
+        if not self.is_installed():
+            raise EngineNotFoundError(self.name, _BINARY)
+        if model.startswith("-"):
+            raise UsageError(
+                f'Invalid --model value "{model}": cannot start with "-" '
+                "(would be parsed as a flag by llama-server, not a model spec)"
+            )
+        spawned = spawn_server_and_wait_ready(
+            engine=self.name,
+            command=_BINARY,
+            args=["-hf", model, "--port", str(port), "--host", "127.0.0.1"],
+            ready_check_url=f"http://127.0.0.1:{port}/v1/models",
+            timeout_ms=timeout_ms,
+            verbose=verbose,
+        )
+        return StartedServer(
+            process=spawned.process,
+            base_url=f"http://127.0.0.1:{port}",
+            model_id="default",
+            stop=spawned.stop,
+        )

inferbench/engines/omlx.py

@@ -0,0 +1,97 @@
+"""
+Ported from src/engines/omlx.ts.
+
+omlx has no CLI benchmark tool of its own (verified against its real
+README -- its "Performance Benchmark" feature is a GUI-only, one-click
+action in its admin dashboard). This adapter only starts the server; all
+measurement goes through the shared HTTP harness in harness/measure.py,
+same as every other engine.
+
+`model` is treated as a model-directory subdirectory name under
+~/.omlx/models/<model> -- omlx's `serve` command has no positional model
+argument and discovers models from --model-dir subdirectories. Unlike
+llama.cpp, omlx does not auto-download an arbitrary Hugging Face repo from
+a CLI flag, so the model must already be present locally -- an honest v0.1
+limitation, not hidden from the user.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any, Optional
+
+from ..errors import EngineNotFoundError, EngineStartTimeoutError
+from ..harness.spawn_server import spawn_server_and_wait_ready
+from ..types import StartedServer
+
+_BINARY = "omlx"
+
+
+class OmlxAdapter:
+    name = "omlx"
+    binary = _BINARY
+
+    def is_installed(self) -> bool:
+        try:
+            subprocess.run(  # noqa: S603, S607 -- fixed argv, binary name only, no shell
+                [_BINARY, "--version"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=True,
+            )
+            return True
+        except (OSError, subprocess.CalledProcessError):
+            return False
+
+    def start_server(
+        self,
+        *,
+        model: str,
+        port: int,
+        timeout_ms: int,
+        verbose: bool = False,
+    ) -> StartedServer:
+        if not self.is_installed():
+            raise EngineNotFoundError(self.name, _BINARY)
+
+        model_dir = str(Path.home() / ".omlx" / "models")
+        spawned = spawn_server_and_wait_ready(
+            engine=self.name,
+            command=_BINARY,
+            args=["serve", "--model-dir", model_dir, "--port", str(port)],
+            ready_check_url=f"http://127.0.0.1:{port}/v1/models",
+            timeout_ms=timeout_ms,
+            verbose=verbose,
+        )
+
+        model_id = self._resolve_model_id(port, model)
+        if not model_id:
+            spawned.stop()
+            raise EngineStartTimeoutError(self.name, timeout_ms)
+
+        return StartedServer(
+            process=spawned.process,
+            base_url=f"http://127.0.0.1:{port}",
+            model_id=model_id,
+            stop=spawned.stop,
+        )
+
+    def _resolve_model_id(self, port: int, requested_model: str) -> Optional[str]:
+        try:
+            with urllib.request.urlopen(  # noqa: S310 -- fixed, internally-constructed loopback URL
+                f"http://127.0.0.1:{port}/v1/models", timeout=5
+            ) as response:
+                if response.status != 200:
+                    return None
+                payload: Any = json.loads(response.read().decode("utf-8"))
+        except (urllib.error.URLError, TimeoutError, ValueError):
+            return None
+
+        models = payload.get("data") if isinstance(payload, dict) else None
+        if not models:
+            return None
+        exact = next((m for m in models if m.get("id") == requested_model), None)
+        return exact["id"] if exact else models[0].get("id")

inferbench/engines/registry.py

@@ -0,0 +1,31 @@
+"""Ported from src/engines/registry.ts."""
+from __future__ import annotations
+
+from typing import Callable, Dict, List
+
+from ..errors import UsageError
+from ..types import EngineAdapter
+from .llamacpp import LlamaCppAdapter
+from .omlx import OmlxAdapter
+
+_ADAPTERS: Dict[str, Callable[[], EngineAdapter]] = {
+    "omlx": OmlxAdapter,
+    "llama.cpp": LlamaCppAdapter,
+}
+
+SUPPORTED_ENGINES: List[str] = list(_ADAPTERS.keys())
+
+
+def resolve_engines(names: List[str]) -> List[EngineAdapter]:
+    """Dedupes and validates a user-supplied --engines list."""
+    deduped = list(dict.fromkeys(names))
+    unknown = [n for n in deduped if n not in _ADAPTERS]
+    if unknown:
+        raise UsageError(
+            f"Unknown engine(s): {', '.join(unknown)}. Supported: {', '.join(SUPPORTED_ENGINES)}"
+        )
+    return [_ADAPTERS[n]() for n in deduped]
+
+
+def all_engines() -> List[EngineAdapter]:
+    return [_ADAPTERS[n]() for n in SUPPORTED_ENGINES]

inferbench/errors.py

@@ -0,0 +1,48 @@
+"""
+Exception hierarchy, ported 1:1 from src/errors.ts. Every error message is
+kept identical in wording to the TypeScript original so the two CLIs report
+the same failure text for the same underlying condition.
+"""
+from __future__ import annotations
+
+
+class EngineNotFoundError(Exception):
+    def __init__(self, engine: str, binary: str) -> None:
+        self.engine = engine
+        self.binary = binary
+        super().__init__(f'{engine}: binary "{binary}" not found on PATH -- skipped')
+
+
+class EngineStartTimeoutError(Exception):
+    def __init__(self, engine: str, timeout_ms: int) -> None:
+        self.engine = engine
+        self.timeout_ms = timeout_ms
+        super().__init__(f"{engine}: server did not become ready within {timeout_ms}ms")
+
+
+class EngineRequestTimeoutError(Exception):
+    def __init__(self, engine: str, timeout_ms: int) -> None:
+        self.engine = engine
+        self.timeout_ms = timeout_ms
+        super().__init__(f"{engine}: request timed out after {timeout_ms}ms")
+
+
+class BenchmarkParseError(Exception):
+    def __init__(self, engine: str, raw_output: str) -> None:
+        self.engine = engine
+        self.raw_output = raw_output
+        super().__init__(f"{engine}: could not parse benchmark response")
+
+
+class NoEnginesFoundError(Exception):
+    def __init__(self) -> None:
+        super().__init__(
+            "No supported engines found on this machine. Install omlx (brew install omlx) "
+            "or llama.cpp (brew install llama.cpp) and try again."
+        )
+
+
+class UsageError(Exception):
+    """Raised for invalid CLI input recognized at the library layer (e.g. an
+    unknown --engines name). Distinct from argparse's own parse errors,
+    which exit 2 directly -- see cli.py."""