chimeraforge 0.2.0__py3-none-any.whl

chimeraforge/__init__.py

@@ -0,0 +1,8 @@
+"""ChimeraForge — LLM deployment optimizer.
+
+Backed by 70,000+ real measurements across 26 technical reports
+on consumer GPUs. Plan, benchmark, evaluate, and optimize LLM
+inference deployments.
+"""
+
+__version__ = "0.2.0"

chimeraforge/bench/__init__.py

@@ -0,0 +1,43 @@
+"""ChimeraForge Benchmarking Engine — run real LLM inference benchmarks.
+
+Public API:
+    run_benchmark       Run a single benchmark configuration
+    run_quant_sweep     Sweep across quantization levels
+    run_context_sweep   Sweep across context lengths
+    save_results        Persist results as JSON
+    BenchmarkResult     Top-level result container
+    RunMetrics          Per-run timing metrics
+    AggregateMetrics    Statistical summary
+    get_backend         Instantiate a backend adapter
+"""
+
+from chimeraforge.bench.backends import get_backend
+from chimeraforge.bench.metrics import (
+    AggregateMetrics,
+    BenchmarkResult,
+    EnvironmentInfo,
+    RunMetrics,
+    StatSummary,
+)
+from chimeraforge.bench.profiles import WorkloadProfile, get_profile
+from chimeraforge.bench.runner import (
+    run_benchmark,
+    run_context_sweep,
+    run_quant_sweep,
+    save_results,
+)
+
+__all__ = [
+    "AggregateMetrics",
+    "BenchmarkResult",
+    "EnvironmentInfo",
+    "RunMetrics",
+    "StatSummary",
+    "WorkloadProfile",
+    "get_backend",
+    "get_profile",
+    "run_benchmark",
+    "run_context_sweep",
+    "run_quant_sweep",
+    "save_results",
+]

chimeraforge/bench/backends/__init__.py

@@ -0,0 +1,33 @@
+"""Backend registry — maps backend names to adapter classes."""
+
+from __future__ import annotations
+
+from chimeraforge.bench.backends.base import Backend
+from chimeraforge.bench.backends.ollama import OllamaBackend
+from chimeraforge.bench.backends.tgi import TGIBackend
+from chimeraforge.bench.backends.vllm import VLLMBackend
+
+BACKEND_REGISTRY: dict[str, type[Backend]] = {
+    "ollama": OllamaBackend,
+    "vllm": VLLMBackend,
+    "tgi": TGIBackend,
+}
+
+
+def get_backend(name: str, **kwargs: object) -> Backend:
+    """Instantiate a backend by name.
+
+    Args:
+        name: Backend identifier ("ollama", "vllm", or "tgi").
+        **kwargs: Passed to the backend constructor (e.g. base_url).
+
+    Returns:
+        Configured Backend instance.
+
+    Raises:
+        ValueError: If backend name is unknown.
+    """
+    cls = BACKEND_REGISTRY.get(name)
+    if cls is None:
+        raise ValueError(f"Unknown backend: {name}. Available: {list(BACKEND_REGISTRY)}")
+    return cls(**kwargs)

chimeraforge/bench/backends/base.py

@@ -0,0 +1,56 @@
+"""Abstract backend interface for LLM serving backends."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from chimeraforge.bench.metrics import RunMetrics
+
+
+class Backend(ABC):
+    """Abstract interface for LLM serving backends.
+
+    Each backend adapter translates the common generate() call into
+    the backend-specific HTTP API, extracts timing metrics from the
+    response, and returns a standardized RunMetrics object.
+    """
+
+    name: str
+
+    @abstractmethod
+    async def health_check(self) -> tuple[bool, str]:
+        """Check if the backend is reachable.
+
+        Returns:
+            Tuple of (ok, message). If not ok, message describes the error.
+        """
+
+    @abstractmethod
+    async def check_model(self, model: str) -> tuple[bool, str]:
+        """Check if a model is available on the backend.
+
+        Returns:
+            Tuple of (ok, message). If not ok, message includes remediation.
+        """
+
+    @abstractmethod
+    async def generate(
+        self,
+        model: str,
+        prompt: str,
+        options: dict | None = None,
+    ) -> RunMetrics:
+        """Run a single generation and return metrics.
+
+        Args:
+            model: Model name or tag.
+            prompt: Input prompt text.
+            options: Backend-specific generation options.
+
+        Returns:
+            RunMetrics with timing and token counts.
+        """
+
+    @abstractmethod
+    async def get_version(self) -> str | None:
+        """Return the backend version string, or None if unavailable."""

chimeraforge/bench/backends/ollama.py

@@ -0,0 +1,117 @@
+"""Ollama backend adapter.
+
+Implements the Backend interface against the Ollama REST API.
+Uses stream=False to extract eval_count / eval_duration / prompt_eval_duration
+from the final JSON response, matching the banterhearts measurement pattern.
+"""
+
+from __future__ import annotations
+
+import httpx
+
+from chimeraforge.bench.backends.base import Backend
+from chimeraforge.bench.metrics import RunMetrics
+
+
+class OllamaBackend(Backend):
+    """Ollama serving backend (http://localhost:11434 by default)."""
+
+    name = "ollama"
+
+    def __init__(self, base_url: str = "http://localhost:11434") -> None:
+        self.base_url = base_url.rstrip("/")
+        self._client: httpx.AsyncClient | None = None
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None or self._client.is_closed:
+            self._client = httpx.AsyncClient(timeout=300)
+        return self._client
+
+    async def close(self) -> None:
+        """Close the underlying HTTP client."""
+        if self._client and not self._client.is_closed:
+            await self._client.aclose()
+
+    async def health_check(self) -> tuple[bool, str]:
+        """GET / -- Ollama returns 'Ollama is running'."""
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/", timeout=10)
+            if resp.status_code == 200:
+                return True, "Ollama is running"
+            return False, f"Ollama returned status {resp.status_code}"
+        except httpx.ConnectError:
+            return False, f"Ollama not running at {self.base_url}"
+        except httpx.TimeoutException:
+            return False, f"Ollama timed out at {self.base_url}"
+
+    async def check_model(self, model: str) -> tuple[bool, str]:
+        """POST /api/show to verify model availability."""
+        try:
+            client = await self._get_client()
+            resp = await client.post(
+                f"{self.base_url}/api/show",
+                json={"name": model},
+                timeout=30,
+            )
+            if resp.status_code == 200:
+                return True, ""
+            return False, f"Model not found. Run: ollama pull {model}"
+        except httpx.ConnectError:
+            return False, f"Ollama not running at {self.base_url}"
+
+    async def generate(
+        self,
+        model: str,
+        prompt: str,
+        options: dict | None = None,
+    ) -> RunMetrics:
+        """POST /api/generate with stream=False, extract timing metrics."""
+        payload: dict = {
+            "model": model,
+            "prompt": prompt,
+            "stream": False,
+        }
+        if options:
+            payload["options"] = options
+
+        client = await self._get_client()
+        resp = await client.post(
+            f"{self.base_url}/api/generate",
+            json=payload,
+            timeout=300,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+
+        eval_count = data.get("eval_count", 0)
+        eval_duration_ns = data.get("eval_duration", 0)
+        prompt_eval_duration_ns = data.get("prompt_eval_duration", 0)
+        total_duration_ns = data.get("total_duration", 0)
+
+        eval_duration_ms = eval_duration_ns / 1e6
+        prompt_eval_duration_ms = prompt_eval_duration_ns / 1e6
+        total_duration_ms = total_duration_ns / 1e6
+
+        throughput = eval_count / (eval_duration_ns / 1e9) if eval_duration_ns > 0 else 0.0
+        ttft = prompt_eval_duration_ms
+
+        return RunMetrics(
+            tokens_generated=eval_count,
+            throughput_tps=throughput,
+            ttft_ms=ttft,
+            total_duration_ms=total_duration_ms,
+            prompt_eval_duration_ms=prompt_eval_duration_ms,
+            eval_duration_ms=eval_duration_ms,
+        )
+
+    async def get_version(self) -> str | None:
+        """GET /api/version."""
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/api/version", timeout=10)
+            if resp.status_code == 200:
+                return resp.json().get("version")
+        except Exception:
+            pass
+        return None

chimeraforge/bench/backends/tgi.py

@@ -0,0 +1,149 @@
+"""TGI (Text Generation Inference) backend adapter.
+
+Implements the Backend interface against the HuggingFace TGI HTTP API.
+TGI exposes /generate with detailed timing in the response.
+"""
+
+from __future__ import annotations
+
+import time
+
+import httpx
+
+from chimeraforge.bench.backends.base import Backend
+from chimeraforge.bench.metrics import RunMetrics
+
+
+class TGIBackend(Backend):
+    """HuggingFace TGI serving backend (http://localhost:8080 by default)."""
+
+    name = "tgi"
+
+    def __init__(self, base_url: str = "http://localhost:8080") -> None:
+        self.base_url = base_url.rstrip("/")
+        self._client: httpx.AsyncClient | None = None
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None or self._client.is_closed:
+            self._client = httpx.AsyncClient(timeout=300)
+        return self._client
+
+    async def close(self) -> None:
+        """Close the underlying HTTP client."""
+        if self._client and not self._client.is_closed:
+            await self._client.aclose()
+
+    async def health_check(self) -> tuple[bool, str]:
+        """GET /health to check TGI availability."""
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/health", timeout=10)
+            if resp.status_code == 200:
+                return True, "TGI is running"
+            return False, f"TGI returned status {resp.status_code}"
+        except httpx.ConnectError:
+            return False, f"TGI not running at {self.base_url}"
+        except httpx.TimeoutException:
+            return False, f"TGI timed out at {self.base_url}"
+
+    async def check_model(self, model: str) -> tuple[bool, str]:
+        """GET /info to verify model is loaded.
+
+        TGI loads a single model at startup, so we verify the loaded
+        model_id matches exactly or that the model name appears as a
+        path component of the loaded model_id (e.g. "llama-3b" matches
+        "meta-llama/Llama-3.2-3B-Instruct").
+        """
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/info", timeout=30)
+            if resp.status_code != 200:
+                return False, f"Cannot get model info (status {resp.status_code})"
+            data = resp.json()
+            loaded = data.get("model_id", "")
+            # Exact match or model is a path component of loaded model_id
+            if model == loaded:
+                return True, ""
+            # Check if model name appears after a "/" in the loaded ID
+            loaded_parts = loaded.lower().split("/")
+            if model.lower() in loaded_parts:
+                return True, ""
+            return False, (
+                f"TGI has '{loaded}' loaded, not '{model}'. Restart TGI with the desired model."
+            )
+        except httpx.ConnectError:
+            return False, f"TGI not running at {self.base_url}"
+
+    async def generate(
+        self,
+        model: str,
+        prompt: str,
+        options: dict | None = None,
+    ) -> RunMetrics:
+        """POST /generate, extract timing from details."""
+        opts = options or {}
+        payload = {
+            "inputs": prompt,
+            "parameters": {
+                "max_new_tokens": opts.get("max_new_tokens", 256),
+                "temperature": opts.get("temperature", 0.7),
+            },
+        }
+
+        client = await self._get_client()
+        t0 = time.perf_counter()
+        resp = await client.post(
+            f"{self.base_url}/generate",
+            json=payload,
+            timeout=300,
+        )
+        total_s = time.perf_counter() - t0
+        resp.raise_for_status()
+        data = resp.json()
+
+        # TGI response structure
+        details = data.get("details", {})
+        generated_tokens = details.get("generated_tokens", 0)
+        total_duration_ms = total_s * 1000
+
+        # TGI timing fields vary by version; try multiple known field names
+        prefill_time = (
+            details.get("prefill_time")  # TGI 2.x (seconds)
+            or details.get("prefill_duration_ns", 0) / 1e9  # hypothetical ns
+        )
+        decode_time = (
+            details.get("decode_time")  # TGI 2.x (seconds)
+            or details.get("decode_duration_ns", 0) / 1e9
+        )
+
+        if prefill_time and prefill_time > 0:
+            prompt_eval_duration_ms = prefill_time * 1000
+            eval_duration_ms = decode_time * 1000 if decode_time else total_duration_ms
+        else:
+            # Timing not available from TGI response
+            prompt_eval_duration_ms = 0.0
+            eval_duration_ms = total_duration_ms
+
+        throughput = generated_tokens / total_s if total_s > 0 else 0.0
+        ttft = prompt_eval_duration_ms if prompt_eval_duration_ms > 0 else -1.0
+
+        return RunMetrics(
+            tokens_generated=generated_tokens,
+            throughput_tps=throughput,
+            ttft_ms=ttft,
+            total_duration_ms=total_duration_ms,
+            prompt_eval_duration_ms=prompt_eval_duration_ms,
+            eval_duration_ms=eval_duration_ms,
+        )
+
+    async def get_version(self) -> str | None:
+        """GET /info and extract version."""
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/info", timeout=10)
+            if resp.status_code == 200:
+                data = resp.json()
+                return data.get("version")
+        except Exception:
+            pass
+        return None

chimeraforge/bench/backends/vllm.py

@@ -0,0 +1,123 @@
+"""vLLM backend adapter.
+
+Implements the Backend interface against the vLLM OpenAI-compatible API.
+vLLM exposes /v1/completions with usage stats in the response.
+"""
+
+from __future__ import annotations
+
+import time
+
+import httpx
+
+from chimeraforge.bench.backends.base import Backend
+from chimeraforge.bench.metrics import RunMetrics
+
+
+class VLLMBackend(Backend):
+    """vLLM serving backend (OpenAI-compatible, http://localhost:8000 by default)."""
+
+    name = "vllm"
+
+    def __init__(self, base_url: str = "http://localhost:8000") -> None:
+        self.base_url = base_url.rstrip("/")
+        self._client: httpx.AsyncClient | None = None
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None or self._client.is_closed:
+            self._client = httpx.AsyncClient(timeout=300)
+        return self._client
+
+    async def close(self) -> None:
+        """Close the underlying HTTP client."""
+        if self._client and not self._client.is_closed:
+            await self._client.aclose()
+
+    async def health_check(self) -> tuple[bool, str]:
+        """GET /health or /v1/models to check availability."""
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/health", timeout=10)
+            if resp.status_code == 200:
+                return True, "vLLM is running"
+            # Fallback: try /v1/models
+            resp = await client.get(f"{self.base_url}/v1/models", timeout=10)
+            if resp.status_code == 200:
+                return True, "vLLM is running"
+            return False, f"vLLM returned status {resp.status_code}"
+        except httpx.ConnectError:
+            return False, f"vLLM not running at {self.base_url}"
+        except httpx.TimeoutException:
+            return False, f"vLLM timed out at {self.base_url}"
+
+    async def check_model(self, model: str) -> tuple[bool, str]:
+        """GET /v1/models and check if model is listed."""
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/v1/models", timeout=30)
+            if resp.status_code != 200:
+                return False, f"Cannot list models (status {resp.status_code})"
+            data = resp.json()
+            model_ids = [m["id"] for m in data.get("data", [])]
+            if model in model_ids:
+                return True, ""
+            return False, (
+                f"Model '{model}' not found. Available: {', '.join(model_ids) or 'none'}"
+            )
+        except httpx.ConnectError:
+            return False, f"vLLM not running at {self.base_url}"
+
+    async def generate(
+        self,
+        model: str,
+        prompt: str,
+        options: dict | None = None,
+    ) -> RunMetrics:
+        """POST /v1/completions, extract usage and timing."""
+        opts = options or {}
+        payload = {
+            "model": model,
+            "prompt": prompt,
+            "max_tokens": opts.get("max_tokens", 256),
+            "temperature": opts.get("temperature", 0.7),
+        }
+
+        client = await self._get_client()
+        t0 = time.perf_counter()
+        resp = await client.post(
+            f"{self.base_url}/v1/completions",
+            json=payload,
+            timeout=300,
+        )
+        total_s = time.perf_counter() - t0
+        resp.raise_for_status()
+        data = resp.json()
+
+        usage = data.get("usage", {})
+        completion_tokens = usage.get("completion_tokens", 0)
+        total_duration_ms = total_s * 1000
+
+        # vLLM non-streaming API doesn't expose TTFT; throughput from wall clock
+        eval_duration_ms = total_duration_ms
+        throughput = completion_tokens / total_s if total_s > 0 else 0.0
+
+        return RunMetrics(
+            tokens_generated=completion_tokens,
+            throughput_tps=throughput,
+            ttft_ms=-1.0,  # Not measurable without streaming
+            total_duration_ms=total_duration_ms,
+            prompt_eval_duration_ms=0.0,
+            eval_duration_ms=eval_duration_ms,
+        )
+
+    async def get_version(self) -> str | None:
+        """GET /version or extract from /v1/models metadata."""
+        try:
+            client = await self._get_client()
+            resp = await client.get(f"{self.base_url}/version", timeout=10)
+            if resp.status_code == 200:
+                data = resp.json()
+                return data.get("version", str(data))
+        except Exception:
+            pass
+        return None