innards 0.1.0__py3-none-any.whl

innards/__init__.py

@@ -0,0 +1,19 @@
+"""Innards: see inside open-weight models, one message at a time.
+
+>>> from innards import Session, AsyncSession, compare
+>>> from innards.kv import predict
+>>> from innards.lineage import explain
+>>> from innards.schema import TurnRecord
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("innards")
+except PackageNotFoundError:  # pragma: no cover - running from a source tree
+    __version__ = "0.0.0"
+
+from innards.schema import Turn, TurnRecord
+from innards.session import AsyncSession, CompareResult, Session, compare
+
+__all__ = ["AsyncSession", "CompareResult", "Session", "Turn", "TurnRecord", "__version__", "compare"]

innards/__main__.py

@@ -0,0 +1,7 @@
+"""``python -m innards`` runs the CLI."""
+
+import sys
+
+from innards.cli import main
+
+sys.exit(main())

innards/_text.py

@@ -0,0 +1,76 @@
+"""Incremental detokenization for streaming, with stop-string hold-back."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+
+__all__ = ["IncrementalDecoder", "strip_stop"]
+
+
+def strip_stop(text: str, stop_strings: Sequence[str]) -> tuple[str, bool]:
+    """Cut ``text`` at the first stop string. Returns ``(text, stopped)``."""
+    cut = min((i for i in (text.find(s) for s in stop_strings if s) if i >= 0), default=-1)
+    return (text[:cut], True) if cut >= 0 else (text, False)
+
+
+class IncrementalDecoder:
+    """Turn a growing list of token ids into text deltas.
+
+    Decodes a short window (prefix/read offsets) instead of the whole sequence,
+    holds back incomplete UTF-8 sequences, and holds back any tail that could be
+    the start of a stop string.
+    """
+
+    def __init__(self, decode: Callable[[list[int]], str], stop_strings: Sequence[str] = ()) -> None:
+        self._decode = decode
+        self._stop = [s for s in stop_strings if s]
+        self._ids: list[int] = []
+        self._prefix = 0
+        self._read = 0
+        self._decoded = ""  # all text decoded so far
+        self._released = 0  # characters of _decoded handed to the caller
+        self.stopped = False
+
+    @property
+    def text(self) -> str:
+        """Full text so far, cut at the first stop string."""
+        return strip_stop(self._decoded, self._stop)[0]
+
+    def _hold(self, text: str) -> int:
+        """Length of the longest suffix of ``text`` that is a proper prefix of a stop string."""
+        best = 0
+        for stop in self._stop:
+            for n in range(min(len(stop) - 1, len(text)), 0, -1):
+                if text.endswith(stop[:n]):
+                    best = max(best, n)
+                    break
+        return best
+
+    def push(self, ids: Sequence[int]) -> str:
+        if self.stopped:
+            return ""
+        self._ids.extend(ids)
+        prefix_text = self._decode(self._ids[self._prefix : self._read])
+        new_text = self._decode(self._ids[self._prefix :])
+        if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
+            self._decoded += new_text[len(prefix_text) :]
+            self._prefix, self._read = self._read, len(self._ids)
+        return self._release(final=False)
+
+    def flush(self) -> str:
+        if not self.stopped and self._read < len(self._ids):
+            tail = self._decode(self._ids[self._prefix :])[len(self._decode(self._ids[self._prefix : self._read])) :]
+            self._decoded += tail
+            self._read = len(self._ids)
+        return self._release(final=True)
+
+    def _release(self, final: bool) -> str:
+        text, stopped = strip_stop(self._decoded, self._stop)
+        if stopped:
+            self.stopped = True
+            end = len(text)
+        else:
+            end = len(text) if final else len(text) - self._hold(text)
+        out = text[self._released : end] if end > self._released else ""
+        self._released = max(self._released, end)
+        return out

innards/backends/__init__.py

@@ -0,0 +1,39 @@
+"""Engine adapters. ``hf`` (in-process Hugging Face Transformers) ships in this release."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from innards.backends.base import (
+    Backend,
+    ContextFullError,
+    KVReading,
+    PreparedTurn,
+    available_backends,
+    get_backend,
+    register_backend,
+)
+
+__all__ = [
+    "Backend",
+    "ContextFullError",
+    "KVReading",
+    "PreparedTurn",
+    "available_backends",
+    "get_backend",
+    "register_backend",
+]
+
+
+def _hf_factory(model: str, **kwargs: Any) -> Backend:
+    try:
+        from innards.backends.hf import HFBackend
+    except ImportError as exc:  # torch / transformers missing
+        raise ImportError(
+            "The 'hf' backend needs torch and transformers. Install them with: "
+            "pip install 'innards[hf]'  (or: uv add 'innards[hf]')"
+        ) from exc
+    return HFBackend(model, **kwargs)
+
+
+register_backend("hf", _hf_factory)

innards/backends/base.py

@@ -0,0 +1,156 @@
+"""Backend interface. HF Transformers ships in this release; vLLM, Ollama and API backends plug in here."""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+from innards.kv.config import ModelSpec
+from innards.observe.events import TurnEnd, TurnKey, TurnStats
+from innards.schema import MemoryInfo
+
+__all__ = [
+    "Backend",
+    "ContextFullError",
+    "Deliver",
+    "Emit",
+    "KVReading",
+    "PreparedTurn",
+    "available_backends",
+    "get_backend",
+    "register_backend",
+]
+
+Deliver = Callable[[Any], None]
+"""Receives each chunk of generated token ids for the caller's stream (output path)."""
+Emit = Callable[[int, TurnKey | None, Any], bool]
+"""Observer hot-path hook: ``emit(kind, key, data)``; must never block."""
+
+
+class ContextFullError(RuntimeError):
+    """The prompt plus ``max_new_tokens`` would exceed the model's context window."""
+
+
+@dataclass(slots=True)
+class PreparedTurn:
+    """A rendered prompt and the cache-reuse decision for it."""
+
+    input_ids: list[int]
+    cached_tokens: int
+    """Prompt tokens already resident in the kept cache (prefix reuse)."""
+    prefix_reused: bool
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def prompt_tokens(self) -> int:
+        return len(self.input_ids)
+
+    @property
+    def recomputed_tokens(self) -> int:
+        return self.prompt_tokens - self.cached_tokens
+
+
+@dataclass(frozen=True, slots=True)
+class KVReading:
+    """Measured KV, produced in an observer worker from a backend snapshot."""
+
+    total_bytes: int
+    per_layer_bytes: tuple[int, ...]
+    by_layer_type: dict[str, int]
+    source: str
+    layer_tokens: tuple[int | None, ...] = ()
+
+
+@runtime_checkable
+class Backend(Protocol):
+    """What a Session needs from an engine adapter."""
+
+    name: str
+    model_id: str
+
+    def load(self) -> None: ...
+
+    @property
+    def spec(self) -> ModelSpec: ...
+
+    @property
+    def dtype(self) -> str | None: ...
+
+    @property
+    def kv_dtype(self) -> str: ...
+
+    @property
+    def max_new_tokens(self) -> int: ...
+
+    def runtime_facts(self) -> dict[str, Any]:
+        """``host``, ``device``, ``engine_version`` for RuntimeInfo."""
+        ...
+
+    def prepare(self, messages: list[dict[str, str]]) -> PreparedTurn:
+        """Render the conversation and decide how much of the kept cache to reuse."""
+        ...
+
+    def generate(
+        self,
+        prepared: PreparedTurn,
+        *,
+        deliver: Deliver | None = None,
+        emit: Emit | None = None,
+        key: TurnKey | None = None,
+        cancel: threading.Event | None = None,
+    ) -> tuple[TurnEnd, list[int]]:
+        """Run one turn in the calling thread. Returns raw end-of-turn facts and generated ids."""
+        ...
+
+    def decode(self, token_ids: list[int]) -> str: ...
+
+    def final_text(self, token_ids: list[int]) -> str:
+        """Text stored in history and the record (stop strings removed, stripped)."""
+        ...
+
+    @property
+    def stop_strings(self) -> tuple[str, ...]: ...
+
+    def reset(self) -> None:
+        """Drop the kept cache (next turn prefills from scratch)."""
+        ...
+
+    def measure_kv(self, snapshot: Any) -> KVReading | None:
+        """Turn a cache snapshot into bytes. Runs in an observer worker."""
+        ...
+
+    def memory_sampler(self) -> Callable[[], int | None] | None: ...
+
+    def memory_info(self, end: TurnEnd, stats: TurnStats, kv_bytes: int | None) -> MemoryInfo:
+        """Memory split for a finished turn. Runs in an observer worker."""
+        ...
+
+    def kv_memory_budget(self) -> int | None:
+        """Bytes available for KV, for capacity projections."""
+        ...
+
+    def composition(self, messages: list[dict[str, str]], prompt_tokens: int) -> dict[str, int | None]:
+        """Token counts for system / history / new message. Runs in an observer worker."""
+        ...
+
+    def close(self) -> None: ...
+
+
+_BACKENDS: dict[str, Callable[..., Backend]] = {}
+
+
+def register_backend(name: str, factory: Callable[..., Backend]) -> None:
+    _BACKENDS[name] = factory
+
+
+def available_backends() -> list[str]:
+    return sorted(_BACKENDS)
+
+
+def get_backend(name: str, model: str, **kwargs: Any) -> Backend:
+    """Create a backend by name (``"hf"``)."""
+    if name not in _BACKENDS:
+        raise ValueError(f"Unknown backend {name!r}. Available: {', '.join(available_backends())}")
+    return _BACKENDS[name](model, **kwargs)