scrollback 0.1.0__py3-none-any.whl

scrollback/sources/codex.py

@@ -0,0 +1,281 @@
+"""Codex CLI source adapter (read-only JSONL rollouts).
+
+Codex (OpenAI's terminal coding agent) records each session as a "rollout"
+JSONL file under::
+
+    ~/.codex/sessions/<YYYY>/<MM>/<DD>/rollout-<timestamp>-<uuid>.jsonl
+
+Each line is a JSON event. The first line is typically a session-meta
+record (``{"type": "session_meta", ...}`` or a payload carrying ``id`` /
+``cwd`` / ``timestamp`` / ``instructions``); subsequent lines are response
+items, of which the conversational ones look like::
+
+    {"type": "response_item", "payload": {"type": "message",
+        "role": "user"|"assistant", "content": [{"type": "input_text"|
+        "output_text", "text": "..."}]}}
+
+and tool/function calls appear as ``function_call`` / ``local_shell_call``
+payloads. The exact shape has evolved across Codex versions, so this parser
+is intentionally tolerant: it pulls role + text from whatever message-like
+records it recognizes and skips the rest.
+
+NOTE: this adapter is written to the documented/observed Codex format but
+has not been verified against a live ~/.codex/sessions store on the
+development machine; field handling is deliberately defensive.
+
+All reads are read-only file reads; the rollout files are never modified.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from ..models import Message, Part, Session, _to_dt
+from .base import Source
+
+_DEFAULT_ROOT = Path.home() / ".codex" / "sessions"
+
+# rollout-2025-01-31T12-34-56-<uuid>.jsonl  ->  capture the uuid tail.
+_ROLLOUT_RE = re.compile(r"rollout-(?P<ts>[\dT:-]+)-(?P<uuid>[0-9a-fA-F-]{8,})\.jsonl$")
+
+
+def _env_root() -> Path:
+    override = os.environ.get("SCROLLBACK_CODEX_DIR")
+    if override:
+        p = Path(override).expanduser()
+        return p / "sessions" if (p / "sessions").is_dir() else p
+    return _DEFAULT_ROOT
+
+
+class CodexSource(Source):
+    name = "codex"
+    label = "Codex"
+
+    def __init__(self, root: Path | None = None) -> None:
+        self._root = root or _env_root()
+
+    def resume_command(self, session) -> str | None:
+        # Codex resumes a recorded session with `codex resume <id>` (documented;
+        # not verified on this machine). Run from the session directory.
+        import shlex
+
+        cmd = f"codex resume {session.id}"
+        if session.directory:
+            return f"cd {shlex.quote(session.directory)} && {cmd}"
+        return cmd
+
+    # -- availability / location -------------------------------------------
+
+    def is_available(self) -> bool:
+        return self._root.is_dir()
+
+    def location(self) -> Path | None:
+        return self._root if self.is_available() else None
+
+    # -- discovery ----------------------------------------------------------
+
+    def _rollout_files(self) -> Iterator[Path]:
+        # Rollouts are nested under YYYY/MM/DD; rglob keeps us version-proof
+        # against minor layout changes.
+        yield from sorted(self._root.rglob("rollout-*.jsonl"))
+
+    def _session_id_for(self, path: Path) -> str:
+        m = _ROLLOUT_RE.search(path.name)
+        return m.group("uuid") if m else path.stem
+
+    def _find_path(self, session_id: str) -> Path | None:
+        for f in self._rollout_files():
+            if self._session_id_for(f) == session_id or f.stem == session_id:
+                return f
+        cands = [f for f in self._rollout_files() if self._session_id_for(f).startswith(session_id)]
+        return cands[0] if len(cands) == 1 else None
+
+    # -- listing ------------------------------------------------------------
+
+    def list_sessions(self) -> Iterator[Session]:
+        if not self.is_available():
+            return iter(())
+        return self._list_sessions()
+
+    def _list_sessions(self) -> Iterator[Session]:
+        for f in self._rollout_files():
+            meta = _scan_meta(f)
+            if meta is None:
+                continue
+            yield Session(
+                id=self._session_id_for(f),
+                source=self.name,
+                title=meta["title"],
+                directory=meta["cwd"],
+                created=_to_dt(meta["first_ts"]),
+                updated=_to_dt(meta["last_ts"]),
+                model=meta["model"],
+                message_count=meta["msg_count"],
+                raw={"path": str(f)},
+            )
+
+    # -- single session -----------------------------------------------------
+
+    def load_session(self, session_id: str) -> Session | None:
+        if not self.is_available():
+            return None
+        path = self._find_path(session_id)
+        if path is None:
+            return None
+        meta = _scan_meta(path) or _empty_meta(path)
+        messages = list(_iter_messages(path))
+        return Session(
+            id=self._session_id_for(path),
+            source=self.name,
+            title=meta["title"],
+            directory=meta["cwd"],
+            created=_to_dt(meta["first_ts"]),
+            updated=_to_dt(meta["last_ts"]),
+            model=meta["model"],
+            message_count=len(messages),
+            messages=tuple(messages),
+            raw={"path": str(path)},
+        )
+
+
+# -- parsing helpers -------------------------------------------------------
+
+
+def _iter_lines(path: Path) -> Iterator[dict[str, Any]]:
+    try:
+        with path.open("r", encoding="utf-8", errors="replace") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    obj = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if isinstance(obj, dict):
+                    yield obj
+    except OSError:
+        return
+
+
+def _payload(obj: dict[str, Any]) -> dict[str, Any]:
+    """Codex wraps records in {'type':..., 'payload': {...}} in newer
+    versions; older ones are flat. Return the inner dict either way."""
+    p = obj.get("payload")
+    return p if isinstance(p, dict) else obj
+
+
+def _record_text(p: dict[str, Any]) -> str:
+    """Extract human-readable text from a message-like payload."""
+    content = p.get("content")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        out: list[str] = []
+        for block in content:
+            if isinstance(block, dict):
+                t = block.get("text") or block.get("input_text") or block.get("output_text")
+                if t:
+                    out.append(t)
+            elif isinstance(block, str):
+                out.append(block)
+        return "\n".join(out)
+    # some versions use a flat "text"
+    return p.get("text") or ""
+
+
+def _scan_meta(path: Path) -> dict[str, Any] | None:
+    cwd: str | None = None
+    model: str | None = None
+    title: str | None = None
+    first_ts: str | int | None = None
+    last_ts: str | int | None = None
+    msg_count = 0
+    seen = False
+
+    for obj in _iter_lines(path):
+        seen = True
+        ts = obj.get("timestamp") or obj.get("ts")
+        if ts is not None:
+            if first_ts is None:
+                first_ts = ts
+            last_ts = ts
+        p = _payload(obj)
+        if cwd is None:
+            cwd = obj.get("cwd") or p.get("cwd")
+        if model is None:
+            model = obj.get("model") or p.get("model")
+        role = p.get("role")
+        ptype = p.get("type") or obj.get("type")
+        if role in ("user", "assistant") or ptype == "message":
+            msg_count += 1
+            if title is None and role == "user":
+                txt = _record_text(p).strip()
+                if txt and not txt.startswith("<"):
+                    title = " ".join(txt.split())[:60]
+
+    if not seen:
+        return None
+    if cwd and not title:
+        title = Path(cwd).name
+    return {
+        "cwd": cwd,
+        "model": model,
+        "title": title or path.stem[:16],
+        "first_ts": first_ts,
+        "last_ts": last_ts,
+        "msg_count": msg_count,
+    }
+
+
+def _empty_meta(path: Path) -> dict[str, Any]:
+    return {"cwd": None, "model": None, "title": path.stem[:16],
+            "first_ts": None, "last_ts": None, "msg_count": 0}
+
+
+def _iter_messages(path: Path) -> Iterator[Message]:
+    idx = 0
+    for obj in _iter_lines(path):
+        p = _payload(obj)
+        role = p.get("role")
+        ptype = p.get("type") or obj.get("type")
+        created = _to_dt(obj.get("timestamp") or obj.get("ts"))
+
+        if role in ("user", "assistant"):
+            text = _record_text(p)
+            if not text.strip():
+                continue
+            yield Message(
+                id=f"{path.stem}:{idx}",
+                role=role,
+                created=created,
+                parts=(Part(id=f"{path.stem}:{idx}:0", type="text", text=text),),
+                model=p.get("model"),
+                raw=obj,
+            )
+            idx += 1
+        elif ptype in ("function_call", "local_shell_call", "tool_call"):
+            name = p.get("name") or p.get("tool") or "tool"
+            args = p.get("arguments") or p.get("input") or p.get("command")
+            text = f"$ {name} {json.dumps(args, ensure_ascii=False)}" if args is not None else f"$ {name}"
+            yield Message(
+                id=f"{path.stem}:{idx}",
+                role="assistant",
+                created=created,
+                parts=(Part(id=f"{path.stem}:{idx}:0", type="tool", text=text,
+                            tool_name=name, tool_status="call"),),
+                raw=obj,
+            )
+            idx += 1
+
+
+def _now() -> datetime:
+    from datetime import timezone
+
+    return datetime.now(timezone.utc)

scrollback/sources/opencode.py

@@ -0,0 +1,357 @@
+"""opencode source adapter (read-only SQLite).
+
+opencode stores sessions in a SQLite database (default
+~/.local/share/opencode/opencode.db) with three relevant tables:
+
+  session(id, title, directory, time_created, time_updated, model, agent,
+          parent_id, ...)
+  message(id, session_id, time_created, data)   -- data is JSON
+  part(id, message_id, session_id, time_created, data)  -- data is JSON
+
+We open the database strictly read-only (URI `mode=ro`) so we never lock
+it for writing or interfere with a running opencode. The DB may be large
+and live (WAL active); read-only queries are safe and see a consistent
+snapshot.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from ..models import Message, Part, Session, _to_dt
+from .base import Source
+
+_DEFAULT_DB = Path.home() / ".local" / "share" / "opencode" / "opencode.db"
+
+# Map opencode part `type` values to our PartType, with a renderer each.
+_TEXT_PART_TYPES = {"text", "reasoning"}
+
+
+def _env_db() -> Path:
+    override = os.environ.get("SCROLLBACK_OPENCODE_DB")
+    return Path(override).expanduser() if override else _DEFAULT_DB
+
+
+class OpenCodeSource(Source):
+    name = "opencode"
+    label = "opencode"
+
+    def __init__(self, db_path: Path | None = None) -> None:
+        self._db_path = db_path or _env_db()
+
+    def resume_command(self, session) -> str | None:
+        # `opencode --session <id>` resumes a session (verified via --help).
+        # Run it from the session's directory so the project context matches.
+        import shlex
+
+        cmd = f"opencode --session {session.id}"
+        if session.directory:
+            return f"cd {shlex.quote(session.directory)} && {cmd}"
+        return cmd
+
+    # -- availability / location -------------------------------------------
+
+    def is_available(self) -> bool:
+        return self._db_path.is_file()
+
+    def location(self) -> Path | None:
+        return self._db_path if self.is_available() else None
+
+    # -- read-only connection ----------------------------------------------
+
+    def _connect(self) -> sqlite3.Connection:
+        # `mode=ro` => read-only; never creates or writes. `immutable=0`
+        # so SQLite still consults the WAL for a consistent live snapshot.
+        uri = f"file:{self._db_path}?mode=ro"
+        conn = sqlite3.connect(uri, uri=True, timeout=5.0)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    # -- listing ------------------------------------------------------------
+
+    def list_sessions(self) -> Iterator[Session]:
+        if not self.is_available():
+            return iter(())
+        return self._list_sessions()
+
+    def _list_sessions(self) -> Iterator[Session]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                """
+                SELECT s.id, s.title, s.directory, s.time_created,
+                       s.time_updated, s.model, s.agent, s.parent_id,
+                       s.cost, s.tokens_input, s.tokens_output,
+                       (SELECT COUNT(*) FROM message m WHERE m.session_id = s.id)
+                           AS msg_count
+                FROM session s
+                ORDER BY s.time_updated DESC
+                """
+            ).fetchall()
+        for r in rows:
+            yield Session(
+                id=r["id"],
+                source=self.name,
+                title=r["title"] or "(untitled)",
+                directory=r["directory"],
+                created=_to_dt(r["time_created"]),
+                updated=_to_dt(r["time_updated"]),
+                model=_parse_model(r["model"]),
+                agent=r["agent"],
+                parent_id=r["parent_id"],
+                message_count=r["msg_count"],
+                cost=r["cost"],
+                tokens_input=r["tokens_input"],
+                tokens_output=r["tokens_output"],
+            )
+
+    # -- single session -----------------------------------------------------
+
+    def load_session(self, session_id: str) -> Session | None:
+        if not self.is_available():
+            return None
+        with self._connect() as conn:
+            srow = conn.execute(
+                "SELECT * FROM session WHERE id = ?", (session_id,)
+            ).fetchone()
+            if srow is None:
+                return None
+            mrows = conn.execute(
+                """
+                SELECT id, time_created, data FROM message
+                WHERE session_id = ?
+                ORDER BY time_created, id
+                """,
+                (session_id,),
+            ).fetchall()
+            prows = conn.execute(
+                """
+                SELECT id, message_id, time_created, data FROM part
+                WHERE session_id = ?
+                ORDER BY time_created, id
+                """,
+                (session_id,),
+            ).fetchall()
+
+        parts_by_message: dict[str, list[Part]] = {}
+        for pr in prows:
+            data = _loads(pr["data"])
+            part = _to_part(pr["id"], data)
+            if part is None:
+                continue
+            parts_by_message.setdefault(pr["message_id"], []).append(part)
+
+        messages: list[Message] = []
+        for mr in mrows:
+            data = _loads(mr["data"])
+            messages.append(
+                Message(
+                    id=mr["id"],
+                    role=data.get("role", "assistant"),
+                    created=_to_dt(mr["time_created"]),
+                    parts=tuple(parts_by_message.get(mr["id"], ())),
+                    model=_model_from_message(data),
+                    raw=data,
+                )
+            )
+
+        return self._session_from_row(srow, tuple(messages), message_count=len(messages))
+
+    def _session_from_row(
+        self, srow: sqlite3.Row, messages: tuple[Message, ...], *, message_count: int
+    ) -> Session:
+        return Session(
+            id=srow["id"],
+            source=self.name,
+            title=srow["title"] or "(untitled)",
+            directory=srow["directory"],
+            created=_to_dt(srow["time_created"]),
+            updated=_to_dt(srow["time_updated"]),
+            model=_parse_model(srow["model"]),
+            agent=srow["agent"],
+            parent_id=srow["parent_id"],
+            message_count=message_count,
+            cost=_col(srow, "cost"),
+            tokens_input=_col(srow, "tokens_input"),
+            tokens_output=_col(srow, "tokens_output"),
+            messages=messages,
+        )
+
+    # -- windowed loading ---------------------------------------------------
+
+    def load_session_meta(self, session_id: str) -> Session | None:
+        if not self.is_available():
+            return None
+        with self._connect() as conn:
+            srow = conn.execute(
+                "SELECT * FROM session WHERE id = ?", (session_id,)
+            ).fetchone()
+            if srow is None:
+                return None
+            count = conn.execute(
+                "SELECT COUNT(*) AS c FROM message WHERE session_id = ?", (session_id,)
+            ).fetchone()["c"]
+        return self._session_from_row(srow, (), message_count=count)
+
+    def load_messages(
+        self, session_id: str, *, offset: int = 0, limit: int | None = None
+    ) -> list[Message]:
+        if not self.is_available():
+            return []
+        with self._connect() as conn:
+            lim = -1 if limit is None else limit
+            mrows = conn.execute(
+                """
+                SELECT id, time_created, data FROM message
+                WHERE session_id = ?
+                ORDER BY time_created, id
+                LIMIT ? OFFSET ?
+                """,
+                (session_id, lim, offset),
+            ).fetchall()
+            if not mrows:
+                return []
+            ids = [mr["id"] for mr in mrows]
+            # Fetch parts in chunks: a single IN (...) can exceed SQLite's
+            # SQLITE_MAX_VARIABLE_NUMBER (999 on older builds) for big sessions.
+            prows = []
+            for chunk in _chunks(ids, 900):
+                placeholders = ",".join("?" * len(chunk))
+                prows.extend(
+                    conn.execute(
+                        f"""
+                        SELECT id, message_id, time_created, data FROM part
+                        WHERE message_id IN ({placeholders})
+                        ORDER BY time_created, id
+                        """,
+                        chunk,
+                    ).fetchall()
+                )
+
+        parts_by_message: dict[str, list[Part]] = {}
+        for pr in prows:
+            part = _to_part(pr["id"], _loads(pr["data"]))
+            if part is not None:
+                parts_by_message.setdefault(pr["message_id"], []).append(part)
+
+        messages: list[Message] = []
+        for mr in mrows:
+            data = _loads(mr["data"])
+            messages.append(
+                Message(
+                    id=mr["id"],
+                    role=data.get("role", "assistant"),
+                    created=_to_dt(mr["time_created"]),
+                    parts=tuple(parts_by_message.get(mr["id"], ())),
+                    model=_model_from_message(data),
+                    raw=data,
+                )
+            )
+        return messages
+
+
+# -- helpers ---------------------------------------------------------------
+
+
+def _chunks(seq: list, size: int):
+    """Yield successive `size`-length chunks of `seq`."""
+    for i in range(0, len(seq), size):
+        yield seq[i : i + size]
+
+
+def _col(row: sqlite3.Row, key: str) -> Any:
+    """Read a column from a Row, returning None if absent."""
+    try:
+        return row[key]
+    except (IndexError, KeyError):
+        return None
+
+
+def _loads(s: str | None) -> dict[str, Any]:
+    if not s:
+        return {}
+    try:
+        obj = json.loads(s)
+        return obj if isinstance(obj, dict) else {}
+    except (json.JSONDecodeError, TypeError):
+        return {}
+
+
+def _parse_model(model_field: str | None) -> str | None:
+    """session.model is JSON like {"id": "...", "providerID": "..."}."""
+    if not model_field:
+        return None
+    try:
+        obj = json.loads(model_field)
+        if isinstance(obj, dict):
+            return obj.get("id") or obj.get("modelID")
+    except (json.JSONDecodeError, TypeError):
+        pass
+    return model_field
+
+
+def _model_from_message(data: dict[str, Any]) -> str | None:
+    m = data.get("model")
+    if isinstance(m, dict):
+        return m.get("modelID") or m.get("id")
+    return data.get("modelID")
+
+
+def _to_part(part_id: str, data: dict[str, Any]) -> Part | None:
+    ptype = data.get("type", "unknown")
+    if ptype in _TEXT_PART_TYPES:
+        return Part(
+            id=part_id,
+            type=ptype,
+            text=data.get("text", "") or "",
+            raw=data,
+        )
+    if ptype == "tool":
+        state = data.get("state", {}) or {}
+        tool_name = data.get("tool")
+        status = state.get("status")
+        text = _render_tool(tool_name, state)
+        return Part(
+            id=part_id,
+            type="tool",
+            text=text,
+            tool_name=tool_name,
+            tool_status=status,
+            raw=data,
+        )
+    # Other part types (step-start/step-finish/patch/file/compaction) are
+    # kept with empty text; export/search can opt in via raw.
+    return Part(id=part_id, type=ptype if ptype in _KNOWN else "unknown", raw=data)
+
+
+_KNOWN = {
+    "text",
+    "reasoning",
+    "tool",
+    "file",
+    "patch",
+    "step-start",
+    "step-finish",
+    "compaction",
+}
+
+
+def _render_tool(tool_name: str | None, state: dict[str, Any]) -> str:
+    """A compact, searchable rendering of a tool call and its result."""
+    parts: list[str] = []
+    inp = state.get("input")
+    if inp is not None:
+        parts.append(f"$ {tool_name} {json.dumps(inp, ensure_ascii=False)}")
+    out = state.get("output")
+    if isinstance(out, str) and out:
+        parts.append(out)
+    elif out is not None:
+        parts.append(json.dumps(out, ensure_ascii=False))
+    err = state.get("error")
+    if err:
+        parts.append(f"[error] {err}")
+    return "\n".join(parts)

scrollback/sources/registry.py

@@ -0,0 +1,39 @@
+"""Registry of available source adapters.
+
+To add a new agent: implement a `Source` subclass and append it to
+`ALL_SOURCES`. Everything else (CLI, search, export, web) picks it up
+automatically.
+"""
+
+from __future__ import annotations
+
+from .aider import AiderSource
+from .base import Source
+from .claudecode import ClaudeCodeSource
+from .codex import CodexSource
+from .opencode import OpenCodeSource
+
+#: Every adapter the program knows about, in display order.
+ALL_SOURCES: tuple[type[Source], ...] = (
+    OpenCodeSource,
+    ClaudeCodeSource,
+    CodexSource,
+    AiderSource,
+)
+
+
+def all_sources() -> list[Source]:
+    """Instantiate every registered adapter (cheap; no I/O yet)."""
+    return [cls() for cls in ALL_SOURCES]
+
+
+def available_sources() -> list[Source]:
+    """Only adapters whose data store exists on this machine."""
+    return [s for s in all_sources() if s.is_available()]
+
+
+def get_source(name: str) -> Source | None:
+    for s in all_sources():
+        if s.name == name:
+            return s
+    return None