every-other-token 0.1.0__py3-none-any.whl

every_other_token/__init__.py

@@ -0,0 +1,32 @@
+"""See how sure an LLM was about every token of its answer.
+
+>>> from every_other_token import render
+>>> resp = client.chat.completions.create(model="gpt-4o-mini", messages=msgs,
+...                                       logprobs=True, top_logprobs=5)  # doctest: +SKIP
+>>> render(resp)  # in Jupyter: a confidence heatmap  # doctest: +SKIP
+
+The Python companion to the ``every-other-token`` Rust CLI. Pure Python, no
+dependencies. Accepts OpenAI Chat Completions and legacy Completions (SDK
+objects or dicts), lists of streamed chunks, any OpenAI-compatible server
+(vLLM, llama.cpp, LM Studio) and vLLM's offline ``RequestOutput``.
+"""
+
+from ._parse import Alternative, ResponseShapeError, Token, parse
+from ._render import Heatmap, Summary, band, render, render_text, render_tokens, summarize
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Alternative",
+    "Heatmap",
+    "ResponseShapeError",
+    "Summary",
+    "Token",
+    "band",
+    "parse",
+    "render",
+    "render_text",
+    "render_tokens",
+    "summarize",
+    "__version__",
+]

every_other_token/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

every_other_token/_parse.py

@@ -0,0 +1,376 @@
+"""Turn LLM responses that carry log-probabilities into a flat list of tokens.
+
+Supported shapes (objects from the ``openai`` SDK, plain dicts, or JSON):
+
+* OpenAI **Chat Completions** with ``logprobs=True``:
+  ``choices[i].logprobs.content[j] = {token, logprob, bytes, top_logprobs}``.
+  vLLM, llama.cpp, LM Studio and other OpenAI-compatible servers return the
+  same shape.
+* A list of **streamed chat chunks** (``stream=True``), each carrying its own
+  ``choices[i].logprobs.content`` slice.
+* OpenAI legacy **Completions** with ``logprobs=N``:
+  ``choices[i].logprobs = {tokens, token_logprobs, top_logprobs}``.
+* vLLM **offline** ``RequestOutput`` (``LLM.generate`` with
+  ``SamplingParams(logprobs=N)``): ``outputs[i].token_ids`` plus
+  ``outputs[i].logprobs[j] = {token_id: Logprob(logprob, rank, decoded_token)}``.
+
+The arithmetic matches the Rust crate: ``confidence = clamp(exp(logprob), 0, 1)``
+and per-token ``perplexity = exp(-logprob)``.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Any, Iterable, List, Mapping, Optional, Sequence, Tuple
+
+__all__ = ["Alternative", "Token", "parse", "ResponseShapeError"]
+
+
+class ResponseShapeError(ValueError):
+    """The response has no per-token log-probabilities in a shape we know."""
+
+
+@dataclass(frozen=True)
+class Alternative:
+    """One of the model's top candidates at a position."""
+
+    text: str
+    logprob: float
+
+    @property
+    def probability(self) -> float:
+        return _confidence(self.logprob)
+
+
+@dataclass(frozen=True)
+class Token:
+    """A generated token and how sure the model was about it.
+
+    ``logprob`` is ``None`` when the provider sent no number for this token
+    (for example the first prompt token of an ``echo=True`` completion).
+    """
+
+    text: str
+    logprob: Optional[float]
+    alternatives: Tuple[Alternative, ...] = field(default_factory=tuple)
+
+    @property
+    def confidence(self) -> Optional[float]:
+        """``exp(logprob)`` clamped to ``[0, 1]``, or ``None``."""
+        return None if self.logprob is None else _confidence(self.logprob)
+
+    @property
+    def perplexity(self) -> Optional[float]:
+        """Per-token perplexity ``exp(-logprob)``, or ``None``."""
+        if self.logprob is None:
+            return None
+        try:
+            return math.exp(-self.logprob)
+        except OverflowError:
+            return math.inf
+
+
+def _confidence(logprob: float) -> float:
+    if math.isnan(logprob):
+        return 0.0
+    if logprob >= 0:
+        return 1.0
+    return min(1.0, max(0.0, math.exp(logprob)))
+
+
+# ---------------------------------------------------------------------------
+# Generic accessors: work on dicts, pydantic models and plain objects alike.
+# ---------------------------------------------------------------------------
+
+_MISSING = object()
+
+
+def _get(obj: Any, key: str, default: Any = None) -> Any:
+    if obj is None:
+        return default
+    if isinstance(obj, Mapping):
+        return obj.get(key, default)
+    value = getattr(obj, key, _MISSING)
+    return default if value is _MISSING else value
+
+
+def _to_plain(resp: Any) -> Any:
+    """Convert SDK objects to dicts when they offer a lossless dump."""
+    if isinstance(resp, (Mapping, list, tuple)):
+        return resp
+    for method in ("model_dump", "to_dict", "dict"):
+        fn = getattr(resp, method, None)
+        if callable(fn):
+            try:
+                out = fn()
+            except TypeError:
+                continue
+            if isinstance(out, Mapping):
+                return out
+    return resp
+
+
+def _num(value: Any) -> Optional[float]:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Shape handlers
+# ---------------------------------------------------------------------------
+
+
+# (text, logprob, raw UTF-8 bytes or None, alternatives) before byte merging.
+_Entry = Tuple[str, Optional[float], Optional[bytes], Tuple[Alternative, ...]]
+
+
+def _merge_byte_fragments(entries: List[_Entry]) -> List[Token]:
+    """Join tokens that each hold part of one UTF-8 character.
+
+    OpenAI splits some characters (emoji, CJK) across tokens and sends each
+    fragment as ``"bytes:\\xe2\\x80"`` with the raw ``bytes``. We join the
+    fragments until they decode; the joined token's logprob is the sum (the
+    probability of the whole run), and its alternatives are the first
+    fragment's.
+    """
+    out: List[Token] = []
+    pending: List[_Entry] = []
+
+    def flush_raw() -> None:
+        for text, lp, _b, alts in pending:
+            out.append(Token(text, lp, alts))
+        pending.clear()
+
+    for entry in entries:
+        text, lp, raw, alts = entry
+        if raw is None and not pending:
+            out.append(Token(text, lp, alts))
+            continue
+        if raw is None:
+            flush_raw()
+            out.append(Token(text, lp, alts))
+            continue
+        pending.append(entry)
+        buf = b"".join(p[2] or b"" for p in pending)
+        try:
+            decoded = buf.decode("utf-8")
+        except UnicodeDecodeError:
+            if len(pending) >= 4:
+                flush_raw()
+            continue
+        if len(pending) == 1:
+            out.append(Token(text if not text.startswith("bytes:") else decoded, lp, alts))
+        else:
+            lps = [p[1] for p in pending]
+            total = None if any(v is None for v in lps) else sum(lps)  # type: ignore[arg-type]
+            out.append(Token(decoded, total, pending[0][3]))
+        pending.clear()
+    flush_raw()
+    return out
+
+
+def _chat_entries(content: Iterable[Any]) -> List[_Entry]:
+    entries = []
+    for item in content or []:
+        text = _get(item, "token", "") or ""
+        lp = _num(_get(item, "logprob"))
+        raw_bytes = _get(item, "bytes")
+        raw = bytes(raw_bytes) if isinstance(raw_bytes, (list, tuple, bytes, bytearray)) else None
+        alts = tuple(
+            Alternative(_get(a, "token", "") or "", float(_get(a, "logprob")))
+            for a in (_get(item, "top_logprobs") or [])
+            if _num(_get(a, "logprob")) is not None
+        )
+        entries.append((text, lp, raw, alts))
+    return entries
+
+
+def _from_completion_logprobs(logprobs: Any) -> Optional[List[Token]]:
+    tokens = _get(logprobs, "tokens")
+    if tokens is None:
+        return None
+    lps = _get(logprobs, "token_logprobs") or []
+    tops = _get(logprobs, "top_logprobs") or []
+    out = []
+    for i, text in enumerate(tokens):
+        lp = _num(lps[i]) if i < len(lps) else None
+        top = tops[i] if i < len(tops) else None
+        alts: Tuple[Alternative, ...] = ()
+        if isinstance(top, Mapping):
+            alts = tuple(
+                Alternative(str(k), float(v))
+                for k, v in sorted(top.items(), key=lambda kv: _desc(_num(kv[1])))
+                if _num(v) is not None
+            )
+        out.append(Token(str(text), lp, alts))
+    return out
+
+
+def _desc(value: Optional[float]) -> float:
+    """Sort key: highest value first, missing values last. 0.0 is a real value."""
+    return math.inf if value is None else -value
+
+
+def _has(obj: Any, key: str) -> bool:
+    if isinstance(obj, Mapping):
+        return key in obj
+    return hasattr(obj, key)
+
+
+def _logprobs_entries(logprobs: Any) -> Optional[List[_Entry]]:
+    """Raw entries of one ``choices[i].logprobs`` object, before byte merging.
+
+    Returns ``[]`` for a chat logprobs object with nothing in it (an empty
+    answer, ``max_tokens=0``, or ``content: null``), and ``None`` only when the
+    object is not a logprobs shape we know.
+    """
+    content = _get(logprobs, "content")
+    if isinstance(content, (list, tuple)):
+        return _chat_entries(content)
+    tokens = _get(logprobs, "tokens")
+    if isinstance(tokens, (list, tuple)):
+        return [(t.text, t.logprob, None, t.alternatives) for t in _from_completion_logprobs(logprobs) or []]
+    # A refusal carries its own token logprobs; those are the generated tokens.
+    refusal = _get(logprobs, "refusal")
+    if isinstance(refusal, (list, tuple)):
+        return _chat_entries(refusal)
+    if content is None and (_has(logprobs, "content") or _has(logprobs, "refusal")):
+        return []
+    return None
+
+
+def _from_vllm_output(output: Any) -> Optional[List[Token]]:
+    logprobs = _get(output, "logprobs")
+    token_ids = _get(output, "token_ids")
+    if logprobs is None or token_ids is None:
+        return None
+    out = []
+    for i, tid in enumerate(token_ids):
+        pos = logprobs[i] if i < len(logprobs) else None
+        if not pos:
+            out.append(Token(str(tid), None))
+            continue
+        # Keys may be ints (Python objects) or strings (after a JSON round trip).
+        chosen = pos.get(tid, pos.get(str(tid))) if isinstance(pos, Mapping) else None
+        by_rank = sorted(pos.items(), key=lambda kv: _rank(kv[1]))
+        alts = tuple(
+            Alternative(_decoded(lp, key), float(_get(lp, "logprob")))
+            for key, lp in by_rank
+            if _num(_get(lp, "logprob")) is not None
+        )
+        if chosen is None:
+            out.append(Token(str(tid), None, alts))
+        else:
+            out.append(Token(_decoded(chosen, tid), _num(_get(chosen, "logprob")), alts))
+    return out
+
+
+def _rank(lp: Any) -> Tuple[int, float]:
+    """Sort key: vLLM's rank when present, then most probable first."""
+    rank = _num(_get(lp, "rank"))
+    if rank is not None:
+        return (0, rank)
+    lpv = _num(_get(lp, "logprob"))
+    return (1, math.inf if lpv is None else -lpv)
+
+
+def _decoded(lp: Any, key: Any) -> str:
+    text = _get(lp, "decoded_token")
+    return str(key) if text is None else str(text)
+
+
+def _choice_entries(choice: Any) -> Optional[List[_Entry]]:
+    logprobs = _get(choice, "logprobs")
+    if logprobs is None:
+        return None
+    return _logprobs_entries(logprobs)
+
+
+def _pick(seq: Sequence[Any], index: int, what: str) -> Any:
+    for item in seq:
+        if _get(item, "index", None) == index:
+            return item
+    if 0 <= index < len(seq):
+        return seq[index]
+    raise ResponseShapeError(f"response has no {what} with index {index} (it has {len(seq)})")
+
+
+def parse(resp: Any, choice: int = 0) -> List[Token]:
+    """Extract the tokens of one choice, with their log-probabilities.
+
+    ``resp`` may be an ``openai`` SDK object, a dict, a JSON string, a list of
+    streamed chunks, or a vLLM ``RequestOutput`` (object or dict). Raises
+    :class:`ResponseShapeError` when no logprobs are present, with a hint on
+    how to ask for them.
+    """
+    if isinstance(resp, (str, bytes)):
+        import json
+
+        resp = json.loads(resp)
+    resp = _to_plain(resp)
+
+    # A list of streamed chunks (or the list of RequestOutputs LLM.generate returns).
+    if isinstance(resp, (list, tuple)):
+        n_outputs = sum(1 for item in resp if _get(item, "outputs") is not None)
+        if n_outputs == 1 and len(resp) == 1:
+            return parse(resp[0], choice)
+        if n_outputs:
+            raise ResponseShapeError(
+                f"got a list of {len(resp)} vLLM RequestOutputs (one per prompt); "
+                "pass one of them, for example render(outputs[0])"
+            )
+        # Collect every chunk's raw entries first and merge once: a character
+        # split into byte fragments usually arrives across several chunks.
+        entries: List[_Entry] = []
+        saw_logprobs = False
+        for chunk in resp:
+            chunk = _to_plain(chunk)
+            for c in _get(chunk, "choices") or []:
+                if (_get(c, "index", 0) or 0) != choice:
+                    continue
+                part = _choice_entries(c)
+                if part is not None:
+                    saw_logprobs = True
+                    entries.extend(part)
+        if not saw_logprobs:
+            raise ResponseShapeError(_NO_LOGPROBS)
+        return _merge_byte_fragments(entries)
+
+    choices = _get(resp, "choices")
+    if choices is not None:
+        picked = _pick(choices, choice, "choice")
+        picked_entries = _choice_entries(picked)
+        if picked_entries is None:
+            raise ResponseShapeError(_NO_LOGPROBS)
+        return _merge_byte_fragments(picked_entries)
+
+    outputs = _get(resp, "outputs")
+    if outputs is not None:
+        out = _from_vllm_output(_pick(outputs, choice, "output"))
+        if out is None:
+            raise ResponseShapeError(
+                "vLLM output has no logprobs: pass SamplingParams(logprobs=5) to LLM.generate"
+            )
+        return out
+
+    # A bare logprobs object (choices[i].logprobs) is accepted too.
+    # Only list-valued content/tokens/refusal count, so a chat message dict
+    # ({"role": ..., "content": "text"}) is not mistaken for one.
+    if any(isinstance(_get(resp, k), (list, tuple)) for k in ("content", "tokens", "refusal")):
+        bare = _logprobs_entries(resp)
+        if bare is not None:
+            return _merge_byte_fragments(bare)
+    raise ResponseShapeError(
+        "not a Chat Completion, Completion, list of stream chunks or vLLM RequestOutput"
+    )
+
+
+_NO_LOGPROBS = (
+    "response has no logprobs: call the API with logprobs=True (and top_logprobs=5 "
+    "for alternatives) for chat completions, or logprobs=5 for legacy completions"
+)

every_other_token/_render.py

@@ -0,0 +1,236 @@
+"""Render parsed tokens as an HTML confidence heatmap or ANSI terminal text."""
+
+from __future__ import annotations
+
+import html
+import math
+from dataclasses import dataclass
+from typing import Any, List, Optional, Sequence
+
+from ._parse import Token, parse
+
+__all__ = ["Heatmap", "Summary", "band", "render", "render_text", "summarize"]
+
+# Same thresholds and colours as the Rust CLI (render::ConfidenceThresholds)
+# and the web UI (.conf-high / .conf-mid / .conf-low).
+DEFAULT_HIGH = 0.7
+DEFAULT_MID = 0.4
+_RGB = {"high": (63, 185, 80), "mid": (227, 179, 65), "low": (248, 81, 73)}
+
+
+def band(confidence: Optional[float], high: float = DEFAULT_HIGH, mid: float = DEFAULT_MID) -> str:
+    """Classify a confidence as ``"high"``, ``"mid"``, ``"low"`` or ``"none"``."""
+    if confidence is None:
+        return "none"
+    if confidence >= high:
+        return "high"
+    if confidence >= mid:
+        return "mid"
+    return "low"
+
+
+@dataclass(frozen=True)
+class Summary:
+    """Whole-answer numbers.
+
+    ``perplexity`` is the sequence perplexity ``exp(-mean(logprob))`` over the
+    tokens that have a logprob; ``mean_confidence`` is the mean of
+    ``exp(logprob)``. ``low``/``mid``/``high`` count tokens per band.
+    """
+
+    tokens: int
+    scored: int
+    mean_confidence: Optional[float]
+    perplexity: Optional[float]
+    high: int
+    mid: int
+    low: int
+    least_confident: List[Token]
+
+
+def summarize(tokens: Sequence[Token], high: float = DEFAULT_HIGH, mid: float = DEFAULT_MID, worst: int = 5) -> Summary:
+    scored = [t for t in tokens if t.logprob is not None]
+    counts = {"high": 0, "mid": 0, "low": 0}
+    for t in scored:
+        counts[band(t.confidence, high, mid)] += 1
+    if scored:
+        mean_lp = sum(t.logprob for t in scored) / len(scored)  # type: ignore[misc]
+        mean_conf = sum(t.confidence for t in scored) / len(scored)  # type: ignore[misc]
+        try:
+            ppl: Optional[float] = math.exp(-mean_lp)
+        except OverflowError:
+            ppl = math.inf
+    else:
+        mean_conf = None
+        ppl = None
+    least = sorted(scored, key=lambda t: t.logprob)[:worst]  # type: ignore[arg-type,return-value]
+    return Summary(len(tokens), len(scored), mean_conf, ppl, counts["high"], counts["mid"], counts["low"], least)
+
+
+class Heatmap(str):
+    """The rendered HTML (a ``str``), shown as a heatmap by Jupyter.
+
+    Also carries ``tokens`` and ``summary`` for programmatic use. ``str(h)``
+    is the HTML fragment; :meth:`page` wraps it in a full HTML document.
+    """
+
+    tokens: List[Token]
+    summary: Summary
+
+    def __new__(cls, markup: str, tokens: List[Token], summary: Summary) -> "Heatmap":
+        obj = super().__new__(cls, markup)
+        obj.tokens = tokens
+        obj.summary = summary
+        return obj
+
+    def _repr_html_(self) -> str:
+        return str(self)
+
+    def __repr__(self) -> str:
+        s = self.summary
+        if not s.scored:
+            return f"<Heatmap {s.tokens} tokens, no logprobs>"
+        return (
+            f"<Heatmap {s.tokens} tokens, mean confidence {s.mean_confidence * 100:.1f}%, "  # type: ignore[operator]
+            f"perplexity {s.perplexity:.2f}>"
+        )
+
+    def page(self, title: str = "Token confidence") -> str:
+        """A standalone HTML document, ready to write to a ``.html`` file."""
+        return (
+            "<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\">"
+            "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">"
+            f"<title>{html.escape(title)}</title></head>"
+            "<body style=\"margin:0;padding:24px 16px;font-family:system-ui,sans-serif;"
+            "background:#fff;color:#1f2328\">"
+            f"{self}</body></html>\n"
+        )
+
+
+def _fmt_pct(c: Optional[float]) -> str:
+    return "n/a" if c is None else f"{c * 100:.1f}%"
+
+
+def _tooltip(t: Token, max_alts: int) -> str:
+    if t.logprob is None:
+        return "no logprob for this token"
+    lines = [
+        f"confidence {_fmt_pct(t.confidence)}",
+        f"logprob {t.logprob:.4f}",
+        f"perplexity {t.perplexity:.2f}",
+    ]
+    if t.alternatives:
+        lines.append("top alternatives:")
+        for a in t.alternatives[:max_alts]:
+            lines.append(f"  {a.text!r} {_fmt_pct(a.probability)}")
+    return "\n".join(lines)
+
+
+def _style(t: Token, high: float, mid: float) -> str:
+    b = band(t.confidence, high, mid)
+    if b == "none":
+        return "border-bottom:1px dashed #8b949e"
+    r, g, bl = _RGB[b]
+    c = t.confidence or 0.0
+    # Stronger tint the less sure the model was, so trouble spots stand out.
+    alpha = 0.12 + 0.5 * (1.0 - c)
+    return f"background:rgba({r},{g},{bl},{alpha:.2f});border-radius:2px"
+
+
+def render_tokens(
+    tokens: Sequence[Token],
+    *,
+    high: float = DEFAULT_HIGH,
+    mid: float = DEFAULT_MID,
+    max_alternatives: int = 5,
+    show_summary: bool = True,
+) -> Heatmap:
+    """Render already-parsed tokens. See :func:`render`."""
+    tokens = list(tokens)
+    s = summarize(tokens, high, mid)
+    spans = []
+    for t in tokens:
+        title = html.escape(_tooltip(t, max_alternatives), quote=True)
+        conf = "" if t.confidence is None else f" data-confidence=\"{t.confidence:.4f}\""
+        spans.append(
+            f"<span class=\"eot-tok eot-{band(t.confidence, high, mid)}\"{conf} "
+            f"title=\"{title}\" style=\"{_style(t, high, mid)}\">{html.escape(t.text)}</span>"
+        )
+    parts = [
+        "<div class=\"eot-heatmap\" style=\"font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;"
+        "font-size:14px;line-height:1.7\">"
+    ]
+    if show_summary:
+        parts.append(_summary_html(s, high, mid))
+    parts.append("<div class=\"eot-text\" style=\"white-space:pre-wrap;word-wrap:break-word\">")
+    parts.append("".join(spans))
+    parts.append("</div></div>")
+    return Heatmap("".join(parts), tokens, s)
+
+
+def _swatch(name: str, label: str) -> str:
+    r, g, b = _RGB[name]
+    return (
+        f"<span style=\"background:rgba({r},{g},{b},0.45);border-radius:2px;padding:0 4px;"
+        f"margin-right:6px\">{html.escape(label)}</span>"
+    )
+
+
+def _summary_html(s: Summary, high: float, mid: float) -> str:
+    if s.scored:
+        stats = (
+            f"{s.tokens} tokens &middot; mean confidence {_fmt_pct(s.mean_confidence)} &middot; "
+            f"perplexity {s.perplexity:.2f} &middot; {s.low} low / {s.mid} mid / {s.high} high"
+        )
+    else:
+        stats = f"{s.tokens} tokens &middot; no logprobs"
+    legend = (
+        _swatch("high", f">= {high:.0%}")
+        + _swatch("mid", f"{mid:.0%}-{high:.0%}")
+        + _swatch("low", f"< {mid:.0%}")
+    )
+    return (
+        "<div class=\"eot-summary\" style=\"font-family:system-ui,sans-serif;font-size:12px;"
+        f"opacity:.8;margin-bottom:6px\">{stats}<br>{legend}"
+        "<span>hover a token for its alternatives</span></div>"
+    )
+
+
+def render(
+    resp: Any,
+    *,
+    choice: int = 0,
+    high: float = DEFAULT_HIGH,
+    mid: float = DEFAULT_MID,
+    max_alternatives: int = 5,
+    show_summary: bool = True,
+) -> Heatmap:
+    """Render a response that has logprobs as an HTML confidence heatmap.
+
+    ``resp`` is anything :func:`every_other_token.parse` accepts: an OpenAI
+    ``ChatCompletion`` or ``Completion`` (SDK object or dict) created with
+    logprobs, a list of streamed chunks, a JSON string, or a vLLM
+    ``RequestOutput``. In Jupyter the result displays as the heatmap; it is
+    also a plain ``str`` of HTML.
+
+    Each token is tinted green (confidence >= ``high``), amber (>= ``mid``)
+    or red (below), more strongly the less confident the model was. Hovering
+    shows confidence, logprob, perplexity and the top alternatives.
+    """
+    return render_tokens(
+        parse(resp, choice=choice),
+        high=high,
+        mid=mid,
+        max_alternatives=max_alternatives,
+        show_summary=show_summary,
+    )
+
+
+_ANSI = {"high": "\x1b[32m", "mid": "\x1b[33m", "low": "\x1b[31m", "none": "\x1b[2m"}
+
+
+def render_text(tokens: Sequence[Token], *, color: bool = True, high: float = DEFAULT_HIGH, mid: float = DEFAULT_MID) -> str:
+    """The answer with each token coloured by band (ANSI), like the Rust CLI."""
+    if not color:
+        return "".join(t.text for t in tokens)
+    return "".join(f"{_ANSI[band(t.confidence, high, mid)]}{t.text}\x1b[0m" for t in tokens)

every_other_token/cli.py

@@ -0,0 +1,130 @@
+"""``eot-py``: render a saved LLM response with logprobs.
+
+Examples::
+
+    eot-py response.json -o heatmap.html      # standalone HTML page
+    eot-py response.json --format text        # coloured terminal output
+    cat response.json | eot-py - --format json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import List, Optional
+
+from . import __version__
+from ._parse import ResponseShapeError, parse
+from ._render import DEFAULT_HIGH, DEFAULT_MID, band, render_text, render_tokens, summarize
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="eot-py",
+        description=(
+            "Show how confident the model was about each token of a saved response "
+            "(OpenAI Chat Completion or Completion with logprobs, a JSON list of stream "
+            "chunks, or a vLLM RequestOutput dumped to JSON)."
+        ),
+    )
+    p.add_argument("input", help="path to the JSON response, or - for stdin")
+    p.add_argument("-o", "--output", help="write here instead of stdout")
+    p.add_argument(
+        "-f",
+        "--format",
+        choices=["html", "text", "json"],
+        default=None,
+        help="html (default with -o, a full page), text (default on a terminal) or json",
+    )
+    p.add_argument("--choice", type=int, default=0, help="which choice to render (default 0)")
+    p.add_argument("--high", type=float, default=DEFAULT_HIGH, help="confidence for green (default 0.7)")
+    p.add_argument("--mid", type=float, default=DEFAULT_MID, help="confidence for amber (default 0.4)")
+    p.add_argument("--no-color", action="store_true", help="plain text output")
+    p.add_argument("--title", default="Token confidence", help="HTML page title")
+    p.add_argument("--version", action="version", version=f"eot-py {__version__}")
+    return p
+
+
+def _read(path: str) -> str:
+    if path == "-":
+        return sys.stdin.read()
+    with open(path, encoding="utf-8") as fh:
+        return fh.read()
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = _build_parser().parse_args(argv)
+    if not 0.0 <= args.mid <= args.high <= 1.0:
+        print("eot-py: need 0 <= --mid <= --high <= 1", file=sys.stderr)
+        return 2
+    try:
+        data = json.loads(_read(args.input))
+    except (OSError, json.JSONDecodeError) as exc:
+        print(f"eot-py: cannot read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    try:
+        tokens = parse(data, choice=args.choice)
+    except ResponseShapeError as exc:
+        print(f"eot-py: {exc}", file=sys.stderr)
+        return 1
+
+    fmt = args.format or ("html" if args.output or not sys.stdout.isatty() else "text")
+    if fmt == "html":
+        out = render_tokens(tokens, high=args.high, mid=args.mid).page(args.title)
+    elif fmt == "json":
+        s = summarize(tokens, args.high, args.mid)
+        out = json.dumps(
+            {
+                "summary": {
+                    "tokens": s.tokens,
+                    "scored": s.scored,
+                    "mean_confidence": s.mean_confidence,
+                    "perplexity": s.perplexity,
+                    "high": s.high,
+                    "mid": s.mid,
+                    "low": s.low,
+                },
+                "tokens": [
+                    {
+                        "text": t.text,
+                        "logprob": t.logprob,
+                        "confidence": t.confidence,
+                        "perplexity": t.perplexity,
+                        "band": band(t.confidence, args.high, args.mid),
+                        "alternatives": [{"text": a.text, "logprob": a.logprob} for a in t.alternatives],
+                    }
+                    for t in tokens
+                ],
+            },
+            indent=2,
+            ensure_ascii=False,
+        ) + "\n"
+    else:
+        color = not args.no_color and not args.output and sys.stdout.isatty()
+        s = summarize(tokens, args.high, args.mid)
+        text = render_text(tokens, color=color, high=args.high, mid=args.mid)
+        if s.scored:
+            footer = (
+                f"\n\n{s.tokens} tokens, mean confidence {s.mean_confidence * 100:.1f}%, "  # type: ignore[operator]
+                f"perplexity {s.perplexity:.2f}, {s.low} low / {s.mid} mid / {s.high} high\n"
+            )
+        else:
+            footer = f"\n\n{s.tokens} tokens, no logprobs\n"
+        out = text + footer
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8", newline="\n") as fh:
+            fh.write(out)
+        print(f"wrote {args.output} ({len(tokens)} tokens)", file=sys.stderr)
+    else:
+        stream = sys.stdout
+        try:
+            stream.write(out)
+        except UnicodeEncodeError:
+            stream.buffer.write(out.encode("utf-8"))  # type: ignore[attr-defined]
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

every_other_token-0.1.0.dist-info/METADATA

@@ -0,0 +1,64 @@
+Metadata-Version: 2.5
+Name: every-other-token
+Version: 0.1.0
+Summary: Render per-token LLM confidence from logprobs as an HTML heatmap (Jupyter, CLI). OpenAI, vLLM and OpenAI-compatible servers.
+Project-URL: Homepage, https://every-other-token.vercel.app/
+Project-URL: Documentation, https://github.com/Mattbusel/Every-Other-Token/blob/main/docs/python.md
+Project-URL: Source, https://github.com/Mattbusel/Every-Other-Token/tree/main/python
+Project-URL: Changelog, https://github.com/Mattbusel/Every-Other-Token/blob/main/CHANGELOG.md
+Author: Matthew Busel
+License-Expression: MIT
+License-File: LICENSE
+Keywords: confidence,heatmap,interpretability,jupyter,llm,logprobs,openai,vllm
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Jupyter
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == 'test'
+Description-Content-Type: text/markdown
+
+# every-other-token (Python)
+
+See how sure an LLM was about every token of its answer. Ask for logprobs,
+pass the response to `render()`, and get an HTML heatmap: green where the
+model was confident, amber where it hesitated, red where it was guessing.
+Hover a token for its probability, logprob, perplexity and the model's top
+alternatives.
+
+```python
+from openai import OpenAI
+from every_other_token import render
+
+resp = OpenAI().chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": "Name three moons of Jupiter."}],
+    logprobs=True,
+    top_logprobs=5,
+)
+render(resp)  # in Jupyter this displays the heatmap
+```
+
+Works with OpenAI Chat Completions and legacy Completions (SDK objects or
+dicts), lists of streamed chunks, vLLM / llama.cpp / LM Studio
+(OpenAI-compatible servers) and vLLM's offline `RequestOutput`. Pure Python,
+no dependencies.
+
+From the shell:
+
+```
+eot-py response.json -o heatmap.html
+eot-py response.json --format text
+```
+
+Full guide: https://github.com/Mattbusel/Every-Other-Token/blob/main/docs/python.md
+
+This is the Python companion to the
+[every-other-token](https://crates.io/crates/every-other-token) Rust CLI,
+which streams live from OpenAI, Anthropic, Gemini, OpenRouter and local
+models. MIT licensed.

every_other_token-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+every_other_token/__init__.py,sha256=1BZQtEb4jQ1_LeDiIyyLhN_WA9x1Z9Dtyx4p9YnUMys,1037
+every_other_token/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
+every_other_token/_parse.py,sha256=-jvfGneC10pyhGVcRHAHSRkoGS4Z8JSb2uIYWVj7Iu4,13382
+every_other_token/_render.py,sha256=I-bOICdgiztnFRk4XEWPc9p1trHs85nkZoAOe8bYLvg,8356
+every_other_token/cli.py,sha256=2CzNrfWIIx5d_DPsaHqzP0T5TVtzB21uq1xgiFeba2c,4898
+every_other_token/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+every_other_token-0.1.0.dist-info/METADATA,sha256=JabmBYfhk4jTNB_xXQItyO-xnm8RBR--hMxxUDrRi4k,2450
+every_other_token-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+every_other_token-0.1.0.dist-info/entry_points.txt,sha256=XWGR8fo60Cpj2xhAlOn4UMvEAWf0bTV91oJbSBrPmSw,54
+every_other_token-0.1.0.dist-info/licenses/LICENSE,sha256=gHKCBtTMrskKy3zjTmhZG4z13TKr0b-f-XPY9AIooJo,1066
+every_other_token-0.1.0.dist-info/RECORD,,

every_other_token-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

every_other_token-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+eot-py = every_other_token.cli:main

every_other_token-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Mattbusel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.