innards 0.0.1__py3-none-any.whl

innards/__init__.py

@@ -0,0 +1,79 @@
+"""Innards: see inside open-weight models while they run.
+
+Your code loads the model; Innards watches it::
+
+    import innards
+
+    watch = innards.observe(model, tokenizer, model_id="Qwen/Qwen3.5-0.8B", run="deltanet-study")
+    out = model.generate(**inputs, max_new_tokens=256)    # one turn
+    watch.view()                                          # the web view on this machine
+
+Feature analyzers run automatically for the features the model has, and can be called explicitly::
+
+    from innards import GatedDeltaNet
+    GatedDeltaNet(model="Qwen/Qwen3.5-0.8B", args=watch.last_turn())
+
+For MoE models, :mod:`innards.experts` records which experts each token and topic activates, during
+inference and training, and which experts a topic needs::
+
+    trace = innards.experts.trace(model, tokenizer, run="topic-experts")
+    with trace.topic("math"):
+        model.generate(**inputs, max_new_tokens=64)
+    trace.map().select("math", coverage=0.9)
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+from typing import Any
+
+try:
+    __version__ = version("innards")
+except PackageNotFoundError:  # pragma: no cover - running from a source tree
+    __version__ = "0.0.0"
+
+from innards import experts
+from innards.features import (
+    CSAHCA,
+    DSA,
+    MLA,
+    GatedAttention,
+    GatedDeltaNet,
+    KVContext,
+    MoE,
+    mHC,
+)
+from innards.observer.watch import CapturedTurn, Watch, observe
+from innards.schema import ConversationRecord, TurnRecord
+
+__all__ = [
+    "CSAHCA",
+    "DSA",
+    "MLA",
+    "CapturedTurn",
+    "ConversationRecord",
+    "GatedAttention",
+    "GatedDeltaNet",
+    "KVContext",
+    "MoE",
+    "TurnRecord",
+    "Watch",
+    "__version__",
+    "experts",
+    "mHC",
+    "observe",
+    "view",
+]
+
+
+def view(
+    root: str | None = None,
+    *,
+    host: str = "127.0.0.1",
+    port: int = 8000,
+    runner: Any = None,
+    open_browser: bool = False,
+    block: bool = True,
+) -> object:
+    """Serve the web view over a folder of runs (default ``./innards_runs``). Needs ``innards[view]``."""
+    from innards.view import serve
+
+    return serve(root=root, runner=runner, host=host, port=port, open_browser=open_browser, block=block)

innards/__main__.py

@@ -0,0 +1,7 @@
+"""``python -m innards`` runs the CLI."""
+
+import sys
+
+from innards.cli import main
+
+sys.exit(main())

innards/_env.py

@@ -0,0 +1,78 @@
+"""Versions and host facts recorded with every run, and the device helpers ``observe`` uses."""
+
+from __future__ import annotations
+
+import os
+import platform
+import socket
+from typing import Any
+
+__all__ = ["DEFAULT_BUDGET_GB", "budget_bytes", "device_name", "environment", "host_kind"]
+
+DEFAULT_BUDGET_GB = 16.0
+"""The memory budget Innards checks against (GPU memory, or unified memory on Apple silicon), unless
+``budget_gb`` or ``$INNARDS_GPU_BUDGET_GB`` says otherwise. 16 GB fits a local machine running models at 16 bit."""
+
+
+def budget_bytes(budget_gb: float | None = None) -> int:
+    value = budget_gb if budget_gb is not None else float(os.environ.get("INNARDS_GPU_BUDGET_GB") or DEFAULT_BUDGET_GB)
+    if value <= 0:
+        raise ValueError("The memory budget must be positive")
+    return int(value * 1e9)
+
+
+def host_kind() -> str:
+    if os.environ.get("INNARDS_HOST"):
+        return os.environ["INNARDS_HOST"]
+    if os.environ.get("KAGGLE_KERNEL_RUN_TYPE"):
+        return "kaggle"
+    if os.environ.get("COLAB_RELEASE_TAG"):
+        return "colab"
+    if os.environ.get("SM_CURRENT_HOST") or os.environ.get("SAGEMAKER_INTERNAL_IMAGE_URI"):
+        return "sagemaker"
+    if os.environ.get("AWS_EXECUTION_ENV") or os.environ.get("ECS_CONTAINER_METADATA_URI"):
+        return "aws"
+    return "local"
+
+
+def device_name(device: Any) -> str:
+    """``NVIDIA H200``, ``mps (arm64)`` or ``cpu (x86_64)``."""
+    kind = getattr(device, "type", str(device))
+    if kind == "cuda":
+        try:
+            import torch
+
+            index = device.index if getattr(device, "index", None) is not None else torch.cuda.current_device()
+            return str(torch.cuda.get_device_name(index))
+        except Exception:
+            return "cuda"
+    return f"{kind} ({platform.machine()})"
+
+
+def environment() -> dict[str, Any]:
+    """Versions and host facts, recorded once per run and per conversation."""
+    from innards import __version__
+
+    env: dict[str, Any] = {
+        "innards": __version__,
+        "python": platform.python_version(),
+        "platform": platform.platform(),
+        "machine": platform.machine(),
+        "host": socket.gethostname(),
+        "host_kind": host_kind(),
+    }
+    try:
+        import torch
+
+        env["torch"] = torch.__version__
+        env["cuda"] = bool(torch.cuda.is_available())
+        env["mps"] = bool(getattr(torch.backends, "mps", None) and torch.backends.mps.is_available())
+    except ImportError:
+        pass
+    try:
+        import transformers
+
+        env["transformers"] = transformers.__version__
+    except ImportError:
+        pass
+    return env

innards/_text.py

@@ -0,0 +1,76 @@
+"""Incremental detokenization for streaming, with stop-string hold-back."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+
+__all__ = ["IncrementalDecoder", "strip_stop"]
+
+
+def strip_stop(text: str, stop_strings: Sequence[str]) -> tuple[str, bool]:
+    """Cut ``text`` at the first stop string. Returns ``(text, stopped)``."""
+    cut = min((i for i in (text.find(s) for s in stop_strings if s) if i >= 0), default=-1)
+    return (text[:cut], True) if cut >= 0 else (text, False)
+
+
+class IncrementalDecoder:
+    """Turn a growing list of token ids into text deltas.
+
+    Decodes a short window (prefix/read offsets) instead of the whole sequence,
+    holds back incomplete UTF-8 sequences, and holds back any tail that could be
+    the start of a stop string.
+    """
+
+    def __init__(self, decode: Callable[[list[int]], str], stop_strings: Sequence[str] = ()) -> None:
+        self._decode = decode
+        self._stop = [s for s in stop_strings if s]
+        self._ids: list[int] = []
+        self._prefix = 0
+        self._read = 0
+        self._decoded = ""  # all text decoded so far
+        self._released = 0  # characters of _decoded handed to the caller
+        self.stopped = False
+
+    @property
+    def text(self) -> str:
+        """Full text so far, cut at the first stop string."""
+        return strip_stop(self._decoded, self._stop)[0]
+
+    def _hold(self, text: str) -> int:
+        """Length of the longest suffix of ``text`` that is a proper prefix of a stop string."""
+        best = 0
+        for stop in self._stop:
+            for n in range(min(len(stop) - 1, len(text)), 0, -1):
+                if text.endswith(stop[:n]):
+                    best = max(best, n)
+                    break
+        return best
+
+    def push(self, ids: Sequence[int]) -> str:
+        if self.stopped:
+            return ""
+        self._ids.extend(ids)
+        prefix_text = self._decode(self._ids[self._prefix : self._read])
+        new_text = self._decode(self._ids[self._prefix :])
+        if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
+            self._decoded += new_text[len(prefix_text) :]
+            self._prefix, self._read = self._read, len(self._ids)
+        return self._release(final=False)
+
+    def flush(self) -> str:
+        if not self.stopped and self._read < len(self._ids):
+            tail = self._decode(self._ids[self._prefix :])[len(self._decode(self._ids[self._prefix : self._read])) :]
+            self._decoded += tail
+            self._read = len(self._ids)
+        return self._release(final=True)
+
+    def _release(self, final: bool) -> str:
+        text, stopped = strip_stop(self._decoded, self._stop)
+        if stopped:
+            self.stopped = True
+            end = len(text)
+        else:
+            end = len(text) if final else len(text) - self._hold(text)
+        out = text[self._released : end] if end > self._released else ""
+        self._released = max(self._released, end)
+        return out

innards/cli.py

@@ -0,0 +1,308 @@
+"""Command line.
+
+innards view [folder]                         the web view over a folder of runs (or one run folder)
+innards validate <model> [--device --set]     the before-download checks for this machine
+innards predict --model <id> --context <n>    KV size from the config, no weights
+innards explain <family | model>              what an architecture changed from its predecessor
+innards lineup                                the pairs and what each newer model changed
+innards sync up <run> s3://bucket/prefix      copy a run folder to S3 (and ``sync down`` back)
+innards experts [folder] [--topic T]          MoE routing traces: top experts and the set a topic needs
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections.abc import Sequence
+from typing import Any
+
+from innards import __version__
+
+__all__ = ["main"]
+
+_UNITS = {"": 1, "b": 1, "k": 1000, "kb": 1000, "m": 10**6, "mb": 10**6, "g": 10**9, "gb": 10**9, "t": 10**12}
+_UNITS |= {"tb": 10**12, "kib": 2**10, "mib": 2**20, "gib": 2**30, "tib": 2**40}
+
+
+def parse_bytes(text: str) -> int:
+    """``"16GiB"``, ``"8 GB"``, ``"512m"`` or a plain integer."""
+    match = re.fullmatch(r"\s*([\d.]+)\s*([a-zA-Z]*)\s*", text)
+    if not match or match.group(2).lower() not in _UNITS:
+        raise argparse.ArgumentTypeError(f"not a size: {text!r} (try 16GiB, 8GB or 1073741824)")
+    return int(float(match.group(1)) * _UNITS[match.group(2).lower()])
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="innards", description="See inside open-weight models: KV cache, context growth and memory per message."
+    )
+    parser.add_argument("--version", action="version", version=f"innards {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    view = sub.add_parser("view", help="Serve the web view over a folder of runs")
+    view.add_argument(
+        "root", nargs="?", default=None, help="Folder of runs, or one run folder (default ./innards_runs)"
+    )
+    view.add_argument("--host", default="127.0.0.1", help="Address to bind (default 127.0.0.1; use an SSH tunnel)")
+    view.add_argument("--port", type=int, default=8000)
+    view.add_argument("--open", action="store_true", help="Open the page in a browser")
+
+    validate = sub.add_parser("validate", help="Check a model against this machine before downloading it")
+    validate.add_argument("model", help="Hub repo id or local folder")
+    validate.add_argument("--device", default=None, help="cuda, mps or cpu (default: the best available)")
+    validate.add_argument("--precision", default=None, help="auto (published), bf16, fp16, fp32 ...")
+    validate.add_argument("--set", default=None, help="Conversation set that sizes the KV check")
+    validate.add_argument(
+        "--budget-gb", type=float, default=None, help="Memory budget (default 16 or $INNARDS_GPU_BUDGET_GB)"
+    )
+    validate.add_argument("--json", action="store_true")
+
+    lineup = sub.add_parser("lineup", help="The lineup: pairs and what each newer model changed")
+    lineup.add_argument("--json", action="store_true")
+
+    sync = sub.add_parser("sync", help="Copy run folders to or from S3 (needs innards[aws])")
+    sync_sub = sync.add_subparsers(dest="direction", required=True)
+    up = sync_sub.add_parser("up", help="Upload a run folder: innards sync up <run> s3://bucket/prefix")
+    up.add_argument("run")
+    up.add_argument("url")
+    up.add_argument("--root", default=None, help="Folder of runs (default ./innards_runs)")
+    down = sync_sub.add_parser("down", help="Download runs: innards sync down s3://bucket/prefix[/run]")
+    down.add_argument("url")
+    down.add_argument("--root", default=None, help="Folder of runs (default ./innards_runs)")
+
+    predict = sub.add_parser("predict", help="Predict KV-cache size from a model config (no weights)")
+    predict.add_argument("--model", required=True, help="Hub repo id, local path or config.json")
+    predict.add_argument("--context", type=int, required=True, help="Context length in tokens")
+    predict.add_argument(
+        "--kv-dtype", default="float16", help="float32, float16, bfloat16 or float8 (default: float16)"
+    )
+    predict.add_argument("--memory", type=parse_bytes, default=None, help="KV memory budget for capacity, e.g. 16GiB")
+    predict.add_argument("--batch-size", type=int, default=1)
+    predict.add_argument("--revision", default=None, help="Hub revision")
+    predict.add_argument("--json", action="store_true", help="Print the prediction as JSON")
+
+    experts = sub.add_parser("experts", help="MoE routing traces: which experts a topic or token activates")
+    experts.add_argument(
+        "path", nargs="?", default=None, help="Folder of runs, a run folder or a trace folder (default ./innards_runs)"
+    )
+    experts.add_argument("--topic", default=None, help="Topic to show (default: list the traces and topics)")
+    experts.add_argument("--by", default="selection", choices=["selection", "gate", "lift"], help="Ranking measure")
+    experts.add_argument("--layer", type=int, action="append", default=None, help="Only these MoE layers")
+    experts.add_argument("--top", type=int, default=8, help="Experts to show per layer (default 8)")
+    experts.add_argument(
+        "--select", type=float, default=None, metavar="COVERAGE", help="The experts that cover this share, e.g. 0.9"
+    )
+    experts.add_argument("--out", default=None, help="Write the --select expert set to this JSON file")
+    experts.add_argument("--token", default=None, help="Where one token goes, per layer (reads the token shards)")
+    experts.add_argument("--json", action="store_true")
+
+    explain = sub.add_parser("explain", help="What an architecture changed from its predecessor")
+    explain.add_argument("target", help="Family (mha, gqa, mla, sliding_window, hybrid ...) or a model id/path")
+    explain.add_argument("--json", action="store_true")
+    return parser
+
+
+# --------------------------------------------------------------------------- predict / explain
+
+
+def _cmd_predict(args: argparse.Namespace) -> int:
+    from innards.kv import predict
+    from innards.lineage import explain
+    from innards.sinks.console import format_bytes
+
+    p = predict(
+        args.model,
+        args.context,
+        args.kv_dtype,
+        memory_bytes=args.memory,
+        batch_size=args.batch_size,
+        revision=args.revision,
+    )
+    if args.json:
+        print(json.dumps(p.to_dict(), indent=2))
+        return 0
+    groups = p.by_layer_type
+    attention = f" ({p.attention.upper()} attention)" if p.attention and p.attention != p.architecture else ""
+    print(f"{p.model_id or args.model} · {p.architecture}{attention} · KV dtype {p.kv_dtype}")
+    print(f"  per token: {p.formula}")
+    print(
+        f"  at {p.context_tokens:,} tokens: KV {format_bytes(p.kv_bytes)} ({p.kv_bytes:,} B) · "
+        f"global {format_bytes(groups['global'])} · sliding {format_bytes(groups['sliding'])} · "
+        f"recurrent {format_bytes(p.recurrent_state_bytes)}"
+    )
+    if p.max_context_tokens:
+        line = f"  max context: {p.max_context_tokens:,} tokens"
+        if p.memory_bytes is not None:
+            line += f" · fits in {format_bytes(p.memory_bytes)}: {p.capacity_tokens:,} tokens"
+        print(line)
+    entry = explain(p.architecture)
+    print(f"  lineage: {entry.family} ← {entry.predecessor or '-'}: {entry.change}")
+    for note in p.notes:
+        print(f"  note: {note}")
+    return 0
+
+
+def _cmd_explain(args: argparse.Namespace) -> int:
+    from innards.lineage import explain
+
+    entry = explain(args.target)
+    print(json.dumps(entry.to_dict(), indent=2) if args.json else str(entry))
+    return 0
+
+
+# --------------------------------------------------------------------------- view / validate / lineup / sync
+
+
+def _cmd_view(args: argparse.Namespace) -> int:
+    from innards.view import serve
+
+    serve(root=args.root, host=args.host, port=args.port, open_browser=args.open, block=True)
+    return 0
+
+
+def _cmd_validate(args: argparse.Namespace) -> int:
+    from innards.validate import validate
+
+    report = validate(
+        args.model, precision=args.precision, device=args.device, conversation_set=args.set, budget_gb=args.budget_gb
+    )
+    print(json.dumps(report.to_dict(), indent=2) if args.json else str(report))
+    return 0 if report.ok else 1
+
+
+def _cmd_lineup(args: argparse.Namespace) -> int:
+    from innards.lineage.lineup import lineup, what_changed
+
+    data = lineup()
+    if args.json:
+        print(json.dumps(data.to_dict(), indent=2))
+        return 0
+    for pair in data.pairs:
+        changed = what_changed(pair.newer, pair.previous)
+        added = ", ".join(a["name"] for a in changed["added"]) or "-"
+        print(f"{pair.feature}  [{pair.where}]")
+        print(f"  {pair.previous}  →  {pair.newer}")
+        print(f"  adds: {added}")
+        for newer, previous in pair.alternates:
+            print(f"  or:   {previous}  →  {newer}")
+    return 0
+
+
+def _cmd_sync(args: argparse.Namespace) -> int:
+    from innards.store import RunStore
+    from innards.sync import sync_down, sync_up
+
+    if args.direction == "up":
+        result = sync_up(RunStore(args.root), args.run, args.url)
+    else:
+        result = sync_down(args.url, RunStore(args.root).root)
+    print(result)
+    return 0
+
+
+def _pct(value: float | None) -> str:
+    return f"{value * 100:5.1f}%" if value is not None else "  n/a "
+
+
+def _cmd_experts(args: argparse.Namespace) -> int:
+    from pathlib import Path
+
+    from innards.experts import TokenRouting, load, traces
+    from innards.store import RunStore
+
+    path = Path(args.path) if args.path else RunStore().root
+    if args.topic is None and args.token is None:
+        found: list[dict[str, Any]]
+        if (path / "summary.json").is_file():
+            found = [{"run": None, "name": str(path), **_brief(load(path))}]
+        elif (path / "experts").is_dir():
+            found = [t for t in traces(path.parent) if t["run"] == path.name]
+        else:
+            found = traces(path)
+        if args.json:
+            print(json.dumps(found, indent=2))
+            return 0
+        if not found:
+            print(f"No expert traces under {path}")
+        for t in found:
+            where = f"{t['run']}/experts/{t['name']}" if t.get("run") else t["name"]
+            print(f"{where}  ·  {t['model']}  ·  {t['layers']} MoE layers")
+            for topic, count in t["tokens"].items():
+                print(f"  {topic}: {count:,} tokens")
+        return 0
+    experts = load(path)
+    layers = args.layer or experts.layers
+    if args.token is not None:
+        folder = experts.folder
+        if folder is None:
+            raise ValueError("--token reads one trace folder's token shards; pass that folder")
+        where = TokenRouting(folder).token(args.token, topic=args.topic)
+        if args.json:
+            print(json.dumps(where, indent=2))
+            return 0
+        print(f"{args.token!r}: {where['occurrences']:,} occurrences (measured; ids {where['ids']})")
+        for layer, row in zip(where["layers"], where["counts"], strict=True):
+            if layer in layers:
+                total = where["occurrences"] or 1
+                ranked = sorted(range(len(row)), key=lambda e: -row[e])[: args.top]
+                print(f"  layer {layer:>3}: " + "  ".join(f"e{e} {row[e] / total:.0%}" for e in ranked if row[e]))
+        return 0
+    topic = args.topic
+    if args.select is not None:
+        chosen = experts.select(topic, coverage=args.select, by="gate" if args.by == "gate" else "selection")
+        if args.out:
+            chosen.save(args.out)
+        print(json.dumps(chosen.to_dict(), indent=2) if args.json else chosen)
+        if args.out and not args.json:
+            print(f"(written to {args.out})")
+        return 0
+    rows = {layer: experts.top(topic, layer, n=args.top, by=args.by) for layer in layers}
+    if args.json:
+        print(json.dumps({str(k): v for k, v in rows.items()}, indent=2))
+        return 0
+    print(f"{experts.model} · {topic}: {experts.tokens(topic):,} tokens measured · ranked by {args.by} (derived)")
+    print("  selection = share of the topic's tokens whose top-k included the expert; lift = vs the other topics")
+    for layer, top in rows.items():
+        cells = "  ".join(
+            f"e{r['expert']} {_pct(r['selection']).strip()}"
+            + (f" ×{r['lift']:.1f}" if r["lift"] is not None and args.by == "lift" else "")
+            for r in top
+        )
+        print(f"  layer {layer:>3}: {cells}")
+    return 0
+
+
+def _brief(experts: object) -> dict[str, Any]:
+    from innards.experts import ExpertMap
+
+    assert isinstance(experts, ExpertMap)
+    return {
+        "model": experts.model,
+        "layers": len(experts.layers),
+        "tokens": {t: experts.tokens(t) for t in experts.topics},
+    }
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    commands = {
+        "predict": _cmd_predict,
+        "explain": _cmd_explain,
+        "view": _cmd_view,
+        "validate": _cmd_validate,
+        "lineup": _cmd_lineup,
+        "sync": _cmd_sync,
+        "experts": _cmd_experts,
+    }
+    try:
+        return commands[args.command](args)
+    except (ValueError, KeyError, FileNotFoundError, ImportError) as exc:
+        message = exc.args[0] if isinstance(exc, KeyError) and exc.args else exc
+        print(f"innards: error: {message}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

innards/collectors/__init__.py

@@ -0,0 +1,6 @@
+"""Collectors turn raw engine facts into TurnRecord fields. They run in observer workers."""
+
+from innards.collectors.memory import rss_sampler, split_memory, tensor_bytes, weights_bytes
+from innards.collectors.timing import timing_from
+
+__all__ = ["rss_sampler", "split_memory", "tensor_bytes", "timing_from", "weights_bytes"]

innards/collectors/memory.py

@@ -0,0 +1,85 @@
+"""Memory collectors: process RSS (psutil) and the weights / KV / activations split."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import Any
+
+from innards.schema import MemoryInfo
+
+__all__ = ["rss_sampler", "split_memory", "tensor_bytes", "weights_bytes"]
+
+log = logging.getLogger("innards.collectors")
+
+
+def rss_sampler() -> Callable[[], int | None] | None:
+    """Return a callable reading this process's RSS, or ``None`` if psutil is not installed."""
+    try:
+        import psutil
+    except ImportError:
+        log.debug("psutil not installed; CPU memory will be reported as null")
+        return None
+    process = psutil.Process()
+
+    def sample() -> int | None:
+        return int(process.memory_info().rss)
+
+    return sample
+
+
+def tensor_bytes(tensor: Any) -> int:
+    """Bytes a tensor really holds on its device (metadata only: no values are read).
+
+    Quantized weights can be tensor subclasses that report their logical dtype, so
+    ``numel * element_size`` would count them at the unpacked size. Their inner tensors (packed data
+    and scales) are counted instead.
+    """
+    if hasattr(type(tensor), "__tensor_flatten__"):  # plain tensors and Parameters do not have it
+        try:
+            names, _ctx = tensor.__tensor_flatten__()
+            return sum(tensor_bytes(getattr(tensor, name)) for name in names)
+        except Exception:
+            log.debug("could not unwrap %s; counting its logical size", type(tensor).__name__, exc_info=True)
+    return int(tensor.numel() * tensor.element_size())
+
+
+def weights_bytes(model: Any) -> int:
+    """Bytes of a model's parameters and buffers, each tensor counted once."""
+    seen: set[int] = set()
+    total = 0
+    for tensor in [*model.parameters(), *model.buffers()]:
+        if id(tensor) in seen:
+            continue
+        seen.add(id(tensor))
+        total += tensor_bytes(tensor)
+    return total
+
+
+def split_memory(
+    *,
+    weights_bytes: int | None,
+    kv_bytes: int | None,
+    peak_bytes: int | None,
+    source: str | None,
+    allocator_only: bool,
+    allocated_bytes: int | None = None,
+    budget_bytes: int | None = None,
+) -> MemoryInfo:
+    """Build MemoryInfo. ``activations = peak - weights - kv`` only when the peak counts tensors alone.
+
+    Process RSS also counts the Python runtime, libraries and allocator caches, so on
+    CPU the activation share is left ``None`` rather than guessed.
+    """
+    activations: int | None = None
+    if allocator_only and peak_bytes is not None and weights_bytes is not None and kv_bytes is not None:
+        activations = max(peak_bytes - weights_bytes - kv_bytes, 0)
+    return MemoryInfo(
+        weights_bytes=weights_bytes,
+        kv_bytes=kv_bytes,
+        activations_bytes=activations,
+        peak_bytes=peak_bytes,
+        allocated_bytes=allocated_bytes,
+        budget_bytes=budget_bytes,
+        source=source,
+    )

innards/collectors/timing.py

@@ -0,0 +1,32 @@
+"""Timing from streamer timestamps (``time.perf_counter_ns``)."""
+
+from __future__ import annotations
+
+from innards.observer.events import TurnEnd
+from innards.schema import TimingInfo
+
+__all__ = ["timing_from"]
+
+
+def _ms(start: int | None, end: int | None) -> float | None:
+    if start is None or end is None:
+        return None
+    return round((end - start) / 1e6, 3)
+
+
+def timing_from(t_request_ns: int | None, end: TurnEnd) -> TimingInfo:
+    """TTFT and total time are measured from the request; prefill from the start of ``generate``.
+
+    Decode speed counts tokens after the first one, over the time between the first
+    and last token, so it excludes prefill.
+    """
+    decode: float | None = None
+    first, last = end.t_first_token_ns, end.t_last_token_ns
+    if first is not None and last is not None and last > first and end.generated_tokens > 1:
+        decode = round((end.generated_tokens - 1) / ((last - first) / 1e9), 3)
+    return TimingInfo(
+        ttft_ms=_ms(t_request_ns, first),
+        prefill_ms=_ms(end.t_gen_start_ns, first),
+        decode_tokens_per_s=decode,
+        total_ms=_ms(t_request_ns, end.t_end_ns),
+    )