backspin 0.5.1__py3-none-any.whl

backspin/replay.py

@@ -0,0 +1,327 @@
+"""Deterministic replay: run your agent again with recorded LLM responses.
+
+A :class:`Cassette` indexes the LLM calls of a recorded run. A stub client
+answers new calls from the cassette, matching by request fingerprint
+(model + messages) and falling back to call order — so you can re-run the
+agent offline, in tests, or against a fix, with the LLM held constant.
+
+What-if branching: :meth:`Cassette.mutate` alters one recorded answer and
+:func:`branch` records the mutated replay as a new run, so you can diff
+"what would have happened if the model had said X".
+"""
+from __future__ import annotations
+
+import copy
+import json
+import warnings
+from contextlib import contextmanager
+from types import SimpleNamespace
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from .fakes import FakeResponse, stream_chunks
+from .recorder import Recorder
+from .runfile import Run, fingerprint_request, load_run
+
+
+class ReplayMismatchWarning(UserWarning):
+    """A replayed request did not exactly match its recording."""
+
+
+class ReplayMismatch(Exception):
+    """The cassette ran out of recordings before the replay finished."""
+
+
+class Cassette:
+    """Indexed recordings of LLM calls from one run."""
+
+    def __init__(self, entries: List[Dict[str, Any]]):
+        self.entries = entries
+        self._cursor = 0
+
+    @classmethod
+    def from_run(cls, run: Run) -> "Cassette":
+        return cls([e for e in run.llm_calls() if e.get("response") is not None])
+
+    def __len__(self) -> int:
+        return len(self.entries)
+
+    def match(self, fingerprint: str) -> Tuple[Optional[Dict[str, Any]], bool]:
+        """Best-effort match against the remaining tape: exact fingerprint
+        first, then the next recording in call order.
+
+        Returns ``(entry, exact)``. ``entry`` is ``None`` when the cassette
+        is exhausted. Matching is sequential — each recording is consumed
+        at most once, so looping agents surface as exhaustion instead of
+        silently looping the same response forever.
+        """
+        for i in range(self._cursor, len(self.entries)):
+            if self.entries[i].get("fingerprint") == fingerprint:
+                self._cursor = i + 1
+                return self.entries[i], True
+        if self._cursor < len(self.entries):
+            entry = self.entries[self._cursor]
+            self._cursor += 1
+            return entry, False
+        return None, False
+
+    def take(self, fingerprint: str) -> Dict[str, Any]:
+        """Like :meth:`match` but raises on exhaustion and warns on fallback."""
+        entry, exact = self.match(fingerprint)
+        if entry is None:
+            raise ReplayMismatch(
+                f"no recording left in cassette (wanted fingerprint {fingerprint})"
+            )
+        if not exact:
+            warnings.warn(
+                f"request fingerprint {fingerprint} not in cassette; "
+                f"falling back to recording #{self._cursor} by call order",
+                ReplayMismatchWarning,
+                stacklevel=3,
+            )
+        return entry
+
+    def mutate(
+        self,
+        index: int,
+        *,
+        content: Optional[str] = None,
+        tool_arguments: Optional[Dict[str, Any]] = None,
+    ) -> "Cassette":
+        """What-if: return a copy with recording ``#index``'s answer altered.
+
+        ``content`` replaces the assistant message text; ``tool_arguments``
+        (a dict) replaces the first tool call's arguments, stored as JSON.
+        Requests are untouched, so replay matching still works — only the
+        answer the agent sees changes. Combine with :func:`branch` or
+        ``stub_client`` to measure the downstream effect.
+        """
+        if not 0 <= index < len(self.entries):
+            raise IndexError(
+                f"recording #{index} out of range (cassette has {len(self.entries)})"
+            )
+        entries = copy.deepcopy(self.entries)
+        resp = entries[index]["response"]
+        if "choices" in resp:
+            message = (resp["choices"] or [{}])[0].setdefault(
+                "message", {"role": "assistant"}
+            )
+            if content is not None:
+                message["content"] = content
+            if tool_arguments is not None:
+                calls = message.get("tool_calls")
+                if not calls:
+                    raise ValueError("that recording has no tool calls to mutate")
+                calls[0].setdefault("function", {})["arguments"] = json.dumps(tool_arguments)
+        elif "content" in resp:  # anthropic message: mutate first text block
+            if tool_arguments is not None:
+                raise ValueError(
+                    "tool_arguments mutation is not supported for anthropic tool_use blocks yet"
+                )
+            for block in resp["content"] or []:
+                if isinstance(block, dict) and block.get("type") == "text":
+                    block["text"] = content
+                    break
+            else:
+                if content is not None:
+                    resp.setdefault("content", []).append({"type": "text", "text": content})
+        else:
+            raise ValueError("unrecognized response shape; cannot mutate")
+        return Cassette(entries)
+
+
+class _StubCompletions:
+    """Duck-typed ``chat.completions`` answered from a cassette."""
+
+    def __init__(self, cassette: Cassette, *, async_mode: bool = False):
+        self.cassette = cassette
+        self.state = SimpleNamespace(calls=0, mismatches=[])
+        self._async = async_mode
+
+    def _respond(self, **kwargs: Any):
+        self.state.calls += 1
+        fp = fingerprint_request(kwargs.get("model"), kwargs.get("messages"))
+        entry, exact = self.cassette.match(fp)
+        if entry is None:
+            raise ReplayMismatch(
+                f"LLM call #{self.state.calls}: cassette exhausted "
+                f"(fingerprint={fp})"
+            )
+        if not exact:
+            self.state.mismatches.append(
+                {"call": self.state.calls, "requested": fp,
+                 "replayed": entry.get("fingerprint")}
+            )
+            warnings.warn(
+                f"LLM call #{self.state.calls}: fingerprint mismatch, "
+                f"replaying recording #{self.cassette._cursor} by call order",
+                ReplayMismatchWarning,
+                stacklevel=2,
+            )
+        data = dict(entry["response"])
+        if kwargs.get("stream"):
+            return iter(stream_chunks(data))
+        return FakeResponse(data)
+
+    def create(self, **kwargs: Any):
+        if self._async:
+            return self._respond_async(**kwargs)
+        return self._respond(**kwargs)
+
+    async def _respond_async(self, **kwargs: Any):
+        return self._respond(**kwargs)
+
+
+def stub_client(cassette: Cassette, *, async_mode: bool = False) -> SimpleNamespace:
+    """A client shaped like ``openai.OpenAI`` (the slice agents touch)."""
+    completions = _StubCompletions(cassette, async_mode=async_mode)
+    return SimpleNamespace(
+        chat=SimpleNamespace(completions=completions), state=completions.state
+    )
+
+
+@contextmanager
+def patch_openai(cassette: Cassette):
+    """Patch ``openai.OpenAI`` / ``openai.AsyncOpenAI`` so constructors
+    yield stub clients backed by ``cassette``. Restores on exit.
+
+    Lets unmodified agent code (``client = OpenAI()``) replay offline.
+    """
+    try:
+        import openai
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError(
+            "patch_openai() requires the openai package: pip install openai"
+        ) from exc
+    saved = (openai.OpenAI, openai.AsyncOpenAI)
+    sync_client = stub_client(cassette)
+    async_client = stub_client(cassette, async_mode=True)
+    # monkeypatching the constructors is the whole point of this helper
+    openai.OpenAI = lambda *a, **k: sync_client  # type: ignore[assignment,misc]
+    openai.AsyncOpenAI = lambda *a, **k: async_client  # type: ignore[assignment,misc]
+    try:
+        yield SimpleNamespace(sync=sync_client, async_=async_client)
+    finally:
+        openai.OpenAI, openai.AsyncOpenAI = saved  # type: ignore[misc]
+
+
+def branch(
+    run: Union[Run, str],
+    mutations: Dict[int, Dict[str, Any]],
+    *,
+    dir: str = "runs",
+    agent: Optional[str] = None,
+) -> str:
+    """What-if: record a replayed run with selected answers mutated.
+
+    ``mutations`` maps a 0-based LLM-call index to
+    :meth:`Cassette.mutate` kwargs, e.g. ``{1: {"content": "No."}}``. Every
+    recorded LLM request is re-issued in order against the mutated cassette
+    and captured into a new run file marked ``branch_of`` in its metadata.
+
+    Returns the new run's path. Diff it against the original with
+    ``diff_runs(original, branch, llm_only=True)`` — requests match until
+    the mutated answer flows back into a later request, which is exactly
+    where the two timelines split.
+    """
+    run_obj = load_run(run) if isinstance(run, str) else run
+    if not mutations:
+        raise ValueError("branch(): pass at least one mutation, e.g. {0: {'content': 'No.'}}")
+    cassette = Cassette.from_run(run_obj)
+    calls = run_obj.llm_calls()
+
+    # Text substitutions that carry mutations forward: when a mutated answer
+    # would have been fed back into a later request, rewrite that request's
+    # assistant messages to the mutated text. (Content-level rewriting; tool
+    # call arguments in follow-up requests are left as recorded.)
+    substitutions: Dict[str, str] = {}
+    for index in sorted(mutations):
+        if not 0 <= index < len(calls):
+            raise IndexError(
+                f"mutation step {index} out of range (run has {len(calls)} LLM calls)"
+            )
+        before = (
+            (calls[index].get("response") or {})
+            .get("choices", [{}])[0].get("message", {}).get("content")
+        )
+        cassette = cassette.mutate(index, **mutations[index])
+        after = (cassette.entries[index]["response"]["choices"][0]["message"].get("content"))
+        if before is not None and after != before:
+            substitutions[before] = after
+
+    rec = Recorder(
+        dir=dir,
+        agent=agent or run_obj.agent,
+        metadata={
+            "branch_of": run_obj.run_id,
+            "mutations": {str(k): v for k, v in sorted(mutations.items())},
+        },
+    )
+    with rec:
+        stub = rec.capture_openai(stub_client(cassette))
+        with warnings.catch_warnings():
+            # fingerprint fallbacks are the point of a branch: mutated
+            # answers change later requests by design
+            warnings.simplefilter("ignore", ReplayMismatchWarning)
+            for entry in calls:
+                if entry.get("response") is None:
+                    continue
+                request = dict(entry.get("request") or {})
+                messages = [
+                    dict(m, content=substitutions[m["content"]])
+                    if m.get("role") == "assistant" and m.get("content") in substitutions
+                    else m
+                    for m in (request.get("messages") or [])
+                ]
+                stub.chat.completions.create(
+                    model=request.get("model"), messages=messages
+                )
+    return rec.path
+
+
+def branch_agent(
+    fn: Callable,
+    run: Union[Run, str],
+    mutations: Dict[int, Dict[str, Any]],
+    *,
+    dir: str = "runs",
+    agent: Optional[str] = None,
+    extra_args: Optional[Tuple] = None,
+) -> str:
+    """Agent-level what-if: re-run the *actual agent function* against a
+    mutated cassette and record it as a branch run.
+
+    ``fn(client, rec, *extra_args)`` is your real agent function: ``client``
+    is a stub answering from the (mutated) cassette, and ``rec`` is the
+    branch run's recorder, so logs, spans, tool calls and downstream
+    requests all happen for real and preserve the run's full shape (unlike
+    :func:`branch`, which replays the request sequence only).
+
+    Returns the new run's path; metadata carries ``branch_of`` +
+    ``mutations``. Diff against the original with plain ``diff_runs`` —
+    shapes align, and the first divergence is where the mutated answer
+    changed the conversation.
+    """
+    run_obj = load_run(run) if isinstance(run, str) else run
+    if not callable(fn):
+        raise TypeError("branch_agent(): fn must be callable, taking (client, *extra_args)")
+    if not mutations:
+        raise ValueError("branch_agent(): pass at least one mutation, e.g. {0: {'content': 'No.'}}")
+    cassette = Cassette.from_run(run_obj)
+    for index in sorted(mutations):
+        cassette = cassette.mutate(index, **mutations[index])
+
+    rec = Recorder(
+        dir=dir,
+        agent=agent or run_obj.agent,
+        metadata={
+            "branch_of": run_obj.run_id,
+            "mutations": {str(k): v for k, v in sorted(mutations.items())},
+            "branch_level": "agent",
+        },
+    )
+    with rec:
+        stub = rec.capture_openai(stub_client(cassette))
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ReplayMismatchWarning)
+            fn(stub, rec, *(extra_args or ()))
+    return rec.path

backspin/runfile.py

@@ -0,0 +1,182 @@
+"""On-disk run format.
+
+A run is a single self-contained ``*.backspin.jsonl`` file. The first line
+is a header event; every following line is one recorded step — an LLM call,
+a tool call, a log line, an error, or any custom event. Because a run is
+one file, "please attach the failing run" becomes a practical bug-report
+ritual instead of a database dump.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+SCHEMA_VERSION = 1
+FILE_SUFFIX = ".backspin.jsonl"
+
+HEADER = "header"
+LLM = "llm"
+TOOL = "tool"
+LOG = "log"
+ERROR = "error"
+SPAN = "span"
+
+
+def new_run_id() -> str:
+    return uuid.uuid4().hex[:12]
+
+
+def backspin_version() -> str:
+    try:
+        from importlib.metadata import version
+
+        return version("backspin")
+    except Exception:
+        return "0.0.0+unknown"
+
+
+def canonical_json(obj: Any) -> str:
+    """Stable serialization, used for fingerprints."""
+    return json.dumps(
+        obj, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str
+    )
+
+
+def jsonable(obj: Any, max_len: int = 800) -> Any:
+    """Return obj if JSON-serializable, else a truncated repr.
+
+    A recorder must never crash the agent it is watching, so anything
+    exotic degrades to a repr instead of raising.
+    """
+    try:
+        json.dumps(obj)
+        return obj
+    except (TypeError, ValueError):
+        return repr(obj)[:max_len]
+
+
+def fingerprint_request(model: Optional[str], messages: Any) -> str:
+    """Short stable fingerprint of an LLM request (model + messages).
+
+    Sampling params (temperature, seed, ...) are deliberately excluded:
+    replay matching should survive them changing.
+    """
+    payload = canonical_json({"model": model, "messages": messages})
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def make_header(
+    run_id: str, agent: str, metadata: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    return {
+        "kind": HEADER,
+        "schema": SCHEMA_VERSION,
+        "run_id": run_id,
+        "agent": agent,
+        "created_at": time.time(),
+        "backspin_version": backspin_version(),
+        "metadata": dict(metadata or {}),
+    }
+
+
+def _token_sums(events: List[Dict[str, Any]]) -> Dict[str, int]:
+    prompt = completion = 0
+    for e in events:
+        usage = e.get("usage") or {}
+        prompt += usage.get("prompt_tokens") or 0
+        completion += usage.get("completion_tokens") or 0
+    return {"prompt_tokens": prompt, "completion_tokens": completion}
+
+
+@dataclass
+class Run:
+    """A loaded run file: header + ordered step events."""
+
+    header: Dict[str, Any]
+    events: List[Dict[str, Any]] = field(default_factory=list)
+    path: Optional[str] = None
+
+    @property
+    def run_id(self) -> str:
+        return str(self.header.get("run_id", ""))
+
+    @property
+    def agent(self) -> str:
+        return str(self.header.get("agent", ""))
+
+    @property
+    def metadata(self) -> Dict[str, Any]:
+        return dict(self.header.get("metadata") or {})
+
+    def by_kind(self, kind: str) -> List[Dict[str, Any]]:
+        return [e for e in self.events if e.get("kind") == kind]
+
+    def llm_calls(self) -> List[Dict[str, Any]]:
+        return self.by_kind(LLM)
+
+    def tool_calls(self) -> List[Dict[str, Any]]:
+        return self.by_kind(TOOL)
+
+    def totals(self) -> Dict[str, Any]:
+        tokens = _token_sums(self.events)
+        duration = sum(
+            e.get("duration_ms") or 0.0
+            for e in self.events
+            if e.get("kind") != "span"  # nested spans would double-count
+        )
+        from .cost import cost_report  # lazy: keeps this module dependency-light
+
+        cost = cost_report(self)
+        return {
+            "steps": len(self.events),
+            "llm_calls": len(self.llm_calls()),
+            "tool_calls": len(self.tool_calls()),
+            **tokens,
+            "total_tokens": tokens["prompt_tokens"] + tokens["completion_tokens"],
+            "duration_ms": round(duration, 1),
+            "cost_usd": cost["total_usd"],
+            "cost_complete": cost["complete"],
+        }
+
+    def summary(self) -> Dict[str, Any]:
+        """Everything list views (CLI / viewer) need about a run."""
+        return {
+            "name": os.path.basename(self.path) if self.path else self.run_id,
+            "run_id": self.run_id,
+            "agent": self.agent,
+            "created_at": self.header.get("created_at"),
+            "backspin_version": self.header.get("backspin_version"),
+            "metadata": self.metadata,
+            "totals": self.totals(),
+        }
+
+
+def load_run(path: str) -> Run:
+    """Load and validate a ``*.backspin.jsonl`` run file."""
+    header: Optional[Dict[str, Any]] = None
+    events: List[Dict[str, Any]] = []
+    with open(path, encoding="utf-8") as f:
+        for lineno, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                ev = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{lineno}: invalid JSON: {exc}") from exc
+            if not isinstance(ev, dict) or "kind" not in ev:
+                raise ValueError(f"{path}:{lineno}: event must be an object with a 'kind'")
+            if header is None:
+                if ev["kind"] != HEADER:
+                    raise ValueError(f"{path}: first event must be a header")
+                header = ev
+            else:
+                events.append(ev)
+    if header is None:
+        raise ValueError(f"{path}: empty run file")
+    return Run(header=header, events=events, path=os.path.abspath(path))

backspin/server.py

@@ -0,0 +1,91 @@
+"""Local viewer: a small FastAPI app that serves the timeline UI.
+
+Read-only over the runs directory: the API only ever exposes files that
+are plain names directly inside the resolved runs dir, and never serves
+anything outside it.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Dict, List
+
+from fastapi import FastAPI, HTTPException
+from fastapi.staticfiles import StaticFiles
+from starlette.responses import JSONResponse
+
+from .diff import diff_runs
+from .runfile import FILE_SUFFIX, load_run
+
+UI_DIR = Path(__file__).resolve().parent / "ui"
+
+
+def _resolve_runs_dir(runs_dir: str) -> Path:
+    root = Path(os.path.realpath(os.path.expanduser(runs_dir)))
+    if not root.is_dir():
+        raise NotADirectoryError(f"runs directory not found: {runs_dir}")
+    return root
+
+
+def _safe_run_path(root: Path, name: str) -> Path:
+    """Only bare file names directly inside ``root`` are ever readable."""
+    if (
+        not name.endswith(FILE_SUFFIX)
+        or "/" in name
+        or "\\" in name
+        or name in (".", "..")
+    ):
+        raise HTTPException(status_code=404, detail="no such run")
+    path = Path(os.path.realpath(root / name))
+    if os.path.dirname(str(path)) != str(root):
+        raise HTTPException(status_code=404, detail="no such run")
+    if not path.is_file():
+        raise HTTPException(status_code=404, detail="no such run")
+    return path
+
+
+def create_app(runs_dir: str = "runs") -> FastAPI:
+    app = FastAPI(title="backspin viewer", docs_url=None, redoc_url=None)
+    root = _resolve_runs_dir(runs_dir)
+
+    @app.get("/api/runs")
+    def list_runs() -> List[Dict[str, Any]]:
+        runs = []
+        for name in sorted(os.listdir(root), reverse=True):
+            if not name.endswith(FILE_SUFFIX):
+                continue
+            try:
+                runs.append(load_run(str(root / name)).summary())
+            except ValueError:
+                runs.append({"name": name, "agent": "?", "invalid": True})
+        return runs
+
+    @app.get("/api/run/{name}")
+    def get_run(name: str) -> Dict[str, Any]:
+        path = _safe_run_path(root, name)
+        try:
+            run = load_run(str(path))
+        except ValueError as exc:
+            raise HTTPException(status_code=422, detail=str(exc)) from exc
+        data = run.summary()
+        data["events"] = run.events
+        return data
+
+    @app.get("/api/diff")
+    def diff(a: str, b: str) -> Dict[str, Any]:
+        run_a = load_run(str(_safe_run_path(root, a)))
+        run_b = load_run(str(_safe_run_path(root, b)))
+        return diff_runs(run_a, run_b).to_dict()
+
+    @app.exception_handler(NotADirectoryError)
+    def bad_dir(_req, exc: NotADirectoryError):
+        return JSONResponse(status_code=500, content={"detail": str(exc)})
+
+    app.mount("/", StaticFiles(directory=str(UI_DIR), html=True), name="ui")
+    return app
+
+
+def serve(runs_dir: str = "runs", host: str = "127.0.0.1", port: int = 8787) -> None:
+    import uvicorn
+
+    uvicorn.run(create_app(runs_dir), host=host, port=port, log_level="warning")

backspin/share.py

@@ -0,0 +1,58 @@
+"""Single-file HTML sharing: one run + the whole viewer in one .html file.
+
+``build_share_html`` inlines the viewer's CSS/JS and the run's data, so a
+teammate can open the file in any browser and step through the run — no
+server, no install, no data leaves the file.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Optional
+
+from .runfile import load_run
+
+_UI_DIR = Path(__file__).resolve().parent / "ui"
+
+
+def _safe_json(obj: Any) -> str:
+    """JSON that is safe to embed inside a <script> block."""
+    return json.dumps(obj, ensure_ascii=False).replace("</", "<\\/")
+
+
+def build_share_html(run_path: str, title: Optional[str] = None) -> str:
+    run = load_run(run_path)
+    data = run.summary()
+    data["events"] = run.events
+
+    html = (_UI_DIR / "index.html").read_text(encoding="utf-8")
+    css = (_UI_DIR / "style.css").read_text(encoding="utf-8")
+    i18n = (_UI_DIR / "i18n.js").read_text(encoding="utf-8")
+    js = (_UI_DIR / "app.js").read_text(encoding="utf-8")
+
+    html = html.replace(
+        '<link rel="stylesheet" href="style.css">',
+        "<style>\n" + css + "\n</style>",
+    )
+    # i18n.js must be inlined before app.js (app.js calls its applyI18n/t).
+    embed = (
+        "<script>\n" + i18n + "\n</script>\n"
+        "<script>window.__BACKSPIN_EMBED__ = " + _safe_json(data) + ";</script>\n"
+        "<script>\n" + js + "\n</script>"
+    )
+    html = html.replace('<script src="i18n.js"></script>', "")
+    html = html.replace('<script src="app.js"></script>', embed)
+    if title:
+        html = html.replace("<title>backspin — agent 运行查看器</title>",
+                            "<title>" + title + "</title>")
+    return html
+
+
+def write_share_html(run_path: str, out_path: Optional[str] = None) -> str:
+    """Build the share file; returns the written path."""
+    run = load_run(run_path)
+    if out_path is None:
+        out_path = str(Path(run_path).with_suffix(".share.html"))
+    html = build_share_html(run_path, title=f"backspin — {run.agent}")
+    Path(out_path).write_text(html, encoding="utf-8")
+    return out_path