zeno-cli 0.3.4__py3-none-any.whl

zeno_session_intel/parsers/claude_code.py

@@ -0,0 +1,169 @@
+"""Claude Code transcript parser (JSONL, the format zeno already hooks - verified).
+
+Layout: ``~/.claude/projects/<project-slug>/<sessionId>.jsonl``, one JSON object per
+line. Lines are ``user`` / ``assistant`` / ``system`` / ``summary`` records carrying a
+``message`` object, plus ``uuid`` / ``parentUuid`` / ``timestamp`` / ``sessionId`` /
+``cwd`` / ``gitBranch`` / ``isSidechain``. Assistant ``message.usage`` is the cache-aware
+token block; assistant ``message.content`` is a list of text/thinking/tool_use blocks;
+user content is a string or a list including ``tool_result`` blocks.
+
+Read-only: the file is opened ``"r"`` and never modified. Never raises - a malformed line
+is skipped.
+
+CC gotcha: ``cache_read_input_tokens`` is large and recurs every turn (prompt caching),
+so each assistant message carries its own usage as ONE ledger event (deduped by message
+id); the rollup uses the ledger, never a naive per-line sum.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from ..model import ParsedMessage, ParsedSession, ToolResult, ToolUse, to_int
+
+
+def default_root() -> Path:
+    return Path(os.path.expanduser("~")) / ".claude" / "projects"
+
+
+def discover(root: Path) -> list[Path]:
+    if not root.exists():
+        return []
+    return sorted(root.glob("**/*.jsonl"))
+
+
+def _flatten_content(content: object) -> tuple[str, str, list[ToolUse], list[ToolResult]]:
+    """Return (text, thinking, tool_uses, tool_results) from a CC message.content."""
+    if isinstance(content, str):
+        return content, "", [], []
+    text_parts: list[str] = []
+    thinking_parts: list[str] = []
+    tool_uses: list[ToolUse] = []
+    tool_results: list[ToolResult] = []
+    if isinstance(content, list):
+        for block in content:
+            if not isinstance(block, dict):
+                if isinstance(block, str):
+                    text_parts.append(block)
+                continue
+            btype = block.get("type")
+            if btype == "text":
+                text_parts.append(str(block.get("text", "")))
+            elif btype == "thinking":
+                thinking_parts.append(str(block.get("thinking", "")))
+            elif btype == "tool_use":
+                tool_uses.append(
+                    ToolUse(
+                        name=str(block.get("name", "")),
+                        input_json=json.dumps(block.get("input", {}), sort_keys=True),
+                        id=str(block.get("id", "")),
+                    )
+                )
+            elif btype == "tool_result":
+                rc = block.get("content", "")
+                if isinstance(rc, list):
+                    rc = " ".join(
+                        str(b.get("text", "")) if isinstance(b, dict) else str(b) for b in rc
+                    )
+                tool_results.append(
+                    ToolResult(
+                        tool_use_id=str(block.get("tool_use_id", "")),
+                        content=str(rc),
+                        is_error=bool(block.get("is_error", False)),
+                    )
+                )
+    return "\n".join(p for p in text_parts if p), "\n".join(thinking_parts), tool_uses, tool_results
+
+
+def parse_file(path: Path) -> ParsedSession | None:
+    try:
+        raw = path.read_text(errors="replace")
+    except OSError:
+        return None
+    session_id = path.stem
+    messages: list[ParsedMessage] = []
+    project = ""
+    cwd = ""
+    git_branch = ""
+    started_at: str | None = None
+    ended_at: str | None = None
+    display_name: str | None = None
+    ordinal = 0
+
+    for line in raw.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except Exception:
+            continue
+        if not isinstance(obj, dict):
+            continue
+        rtype = obj.get("type")
+        if rtype == "summary":
+            display_name = display_name or str(obj.get("summary", "")) or None
+            continue
+        msg = obj.get("message")
+        if not isinstance(msg, dict):
+            continue
+        role = str(msg.get("role") or rtype or "")
+        if role not in ("user", "assistant", "system"):
+            continue
+        ts = obj.get("timestamp")
+        if isinstance(ts, str):
+            started_at = started_at or ts
+            ended_at = ts
+        cwd = cwd or str(obj.get("cwd", "") or "")
+        git_branch = git_branch or str(obj.get("gitBranch", "") or "")
+        sid = obj.get("sessionId")
+        if isinstance(sid, str) and sid:
+            session_id = sid
+        text, thinking, tool_uses, tool_results = _flatten_content(msg.get("content"))
+        usage = msg.get("usage") if isinstance(msg.get("usage"), dict) else {}
+        pm = ParsedMessage(
+            ordinal=ordinal,
+            role=role,
+            content=text,
+            thinking_text=thinking,
+            timestamp=ts if isinstance(ts, str) else None,
+            model=str(msg.get("model", "") or ""),
+            input_tokens=to_int(usage.get("input_tokens")),
+            output_tokens=to_int(usage.get("output_tokens")),
+            cache_creation_input_tokens=to_int(usage.get("cache_creation_input_tokens")),
+            cache_read_input_tokens=to_int(usage.get("cache_read_input_tokens")),
+            raw_usage=json.dumps(usage, sort_keys=True) if usage else "",
+            source_uuid=str(obj.get("uuid", "") or ""),
+            source_parent_uuid=str(obj.get("parentUuid", "") or ""),
+            is_sidechain=bool(obj.get("isSidechain", False)),
+            tool_uses=tool_uses,
+            tool_results=tool_results,
+            usage_dedup_key=str(msg.get("id") or obj.get("uuid") or ""),
+        )
+        messages.append(pm)
+        ordinal += 1
+
+    if not messages:
+        return None
+    project = path.parent.name
+    first_user = next((m.content for m in messages if m.role == "user" and m.content), None)
+    try:
+        mtime = int(path.stat().st_mtime)
+    except OSError:
+        mtime = None
+    return ParsedSession(
+        id=session_id,
+        agent="claude",
+        messages=messages,
+        project=project,
+        cwd=cwd,
+        git_branch=git_branch,
+        display_name=display_name or (first_user[:80] if first_user else None),
+        first_message=first_user,
+        started_at=started_at,
+        ended_at=ended_at,
+        file_path=str(path),
+        file_mtime=mtime,
+    )

zeno_session_intel/parsers/codex.py

@@ -0,0 +1,265 @@
+"""Codex (OpenAI) rollout parser - JSONL. EXPERIMENTAL: fixture-verified only.
+
+STATUS: experimental. zeno is Claude-Code-first; Codex is NOT a first-class capture path
+and is NOT installed on this box. This parser is validated against a synthesized fixture
+(``tests/fixtures/session_intel/codex_rollout.jsonl``) built from the public rollout format
+only - it is **fixture-verified, needs real-data confirmation**. The ingester gates Codex
+behind an explicit ``--tools`` opt-in so a wrong guess never corrupts real Claude Code data.
+See docs/TOOLS_STATUS.md and docs/adapters/codex.md.
+
+Layout: ``~/.codex/sessions/<YYYY>/<MM>/<DD>/rollout-<ts>-<uuid>.jsonl``. Record types:
+  - ``session_meta``        -> id / timestamp / cwd (payload-nested or flat)
+  - ``response_item`` ``message``          -> a turn (role + content[].text)
+  - ``response_item`` ``function_call``     -> a tool call (name/arguments/call_id)
+  - ``response_item`` ``function_call_output`` -> a tool result (call_id/output, exit code)
+  - ``event_msg`` ``token_count`` / ``usage``  -> token usage, attributed to the last turn
+
+Read-only; never raises (malformed lines skipped).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from ..model import ParsedMessage, ParsedSession, ToolResult, ToolUse, to_int
+
+_DEFAULT_MODEL = "gpt-5.3-codex"
+
+
+def default_root() -> Path:
+    return Path(os.path.expanduser("~")) / ".codex" / "sessions"
+
+
+def discover(root: Path) -> list[Path]:
+    if not root.exists():
+        return []
+    return sorted(root.glob("**/rollout-*.jsonl"))
+
+
+def _payload(obj: dict) -> dict:
+    p = obj.get("payload")
+    return p if isinstance(p, dict) else obj
+
+
+def _text_from_content(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts = []
+        for b in content:
+            if isinstance(b, dict):
+                parts.append(str(b.get("text", "")))
+            elif isinstance(b, str):
+                parts.append(b)
+        return "\n".join(p for p in parts if p)
+    return ""
+
+
+def _block(src: dict) -> dict:
+    return {
+        "input_tokens": to_int(src.get("input_tokens")),
+        "output_tokens": to_int(src.get("output_tokens")),
+        "cache_read_input_tokens": to_int(
+            src.get("cached_input_tokens", src.get("cache_read_input_tokens"))
+        ),
+        "reasoning_tokens": to_int(src.get("reasoning_output_tokens", src.get("reasoning_tokens"))),
+    }
+
+
+def _usage_block(info: dict) -> tuple[dict, bool]:
+    """Return (usage dict, is_cumulative) from a Codex token_count record.
+
+    Codex ``token_count`` totals under ``total_token_usage`` are CUMULATIVE across the
+    session; only ``last_token_usage`` (when present) is already a per-turn delta. The
+    caller subtracts the running cumulative so the ledger sums to the true session total
+    instead of overcounting. (Verified against the public Codex rollout format,
+    2026-06-17; still verified-by-fixture pending a real install.)"""
+    if isinstance(info.get("last_token_usage"), dict):
+        return _block(info["last_token_usage"]), False
+    if isinstance(info.get("total_token_usage"), dict):
+        return _block(info["total_token_usage"]), True
+    if isinstance(info.get("usage"), dict):
+        return _block(info["usage"]), False
+    return _block(info), False
+
+
+def parse_file(path: Path) -> ParsedSession | None:
+    try:
+        raw = path.read_text(errors="replace")
+    except OSError:
+        return None
+
+    session_id = path.stem
+    cwd = ""
+    model = ""
+    started_at: str | None = None
+    ended_at: str | None = None
+    messages: list[ParsedMessage] = []
+    ordinal = 0
+    # running cumulative, to turn Codex's cumulative token_count totals into per-turn deltas
+    prev_cum = {
+        "input_tokens": 0,
+        "output_tokens": 0,
+        "cache_read_input_tokens": 0,
+        "reasoning_tokens": 0,
+    }
+
+    def last_assistant() -> ParsedMessage | None:
+        for m in reversed(messages):
+            if m.role == "assistant":
+                return m
+        return None
+
+    for line in raw.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except Exception:
+            continue
+        if not isinstance(obj, dict):
+            continue
+        rtype = obj.get("type")
+        ts = obj.get("timestamp")
+        if isinstance(ts, str):
+            started_at = started_at or ts
+            ended_at = ts
+
+        if rtype == "session_meta":
+            p = _payload(obj)
+            sid = p.get("id") or obj.get("id")
+            if isinstance(sid, str) and sid:
+                session_id = sid
+            cwd = cwd or str(p.get("cwd", "") or "")
+            model = model or str(p.get("model", "") or "")
+            continue
+
+        if rtype == "response_item":
+            p = _payload(obj)
+            ptype = p.get("type")
+            if ptype == "message":
+                role = str(p.get("role", "") or "")
+                if role not in ("user", "assistant", "system"):
+                    continue
+                messages.append(
+                    ParsedMessage(
+                        ordinal=ordinal,
+                        role=role,
+                        content=_text_from_content(p.get("content")),
+                        timestamp=ts if isinstance(ts, str) else None,
+                        model=model or _DEFAULT_MODEL if role == "assistant" else "",
+                    )
+                )
+                ordinal += 1
+            elif ptype == "function_call":
+                tu = ToolUse(
+                    name=str(p.get("name", "") or ""),
+                    input_json=_canon_args(p.get("arguments")),
+                    id=str(p.get("call_id", "") or ""),
+                )
+                target = last_assistant()
+                if target is None:
+                    target = ParsedMessage(ordinal=ordinal, role="assistant", model=model)
+                    messages.append(target)
+                    ordinal += 1
+                target.tool_uses.append(tu)
+            elif ptype == "function_call_output":
+                out = p.get("output")
+                content, is_err = _decode_output(out)
+                tr = ToolResult(
+                    tool_use_id=str(p.get("call_id", "") or ""),
+                    content=content,
+                    is_error=is_err,
+                )
+                target = last_assistant()
+                if target is not None:
+                    target.tool_results.append(tr)
+            continue
+
+        if rtype in ("event_msg", "token_count"):
+            p = _payload(obj)
+            info = p.get("info") if isinstance(p.get("info"), dict) else p
+            if p.get("type") in ("token_count", None) or rtype == "token_count":
+                usage, is_cumulative = _usage_block(info)
+                if is_cumulative:
+                    # cumulative -> per-turn delta, then advance the running total. A
+                    # context-window reset / compaction makes the cumulative drop BELOW the
+                    # prior total (non-monotonic); a plain ``max(0, cur-prev)`` would clamp
+                    # that turn to 0 and silently drop its real tokens. On a reset the new
+                    # lower cumulative IS that turn's usage, so take the current value itself.
+                    delta = {
+                        k: (
+                            usage[k]
+                            if usage[k] < prev_cum.get(k, 0)
+                            else usage[k] - prev_cum.get(k, 0)
+                        )
+                        for k in usage
+                    }
+                    prev_cum = dict(usage)
+                    usage = delta
+                target = last_assistant()
+                if target is not None and any(usage.values()):
+                    target.input_tokens = usage["input_tokens"] or target.input_tokens
+                    target.output_tokens = usage["output_tokens"] or target.output_tokens
+                    target.cache_read_input_tokens = (
+                        usage["cache_read_input_tokens"] or target.cache_read_input_tokens
+                    )
+                    target.reasoning_tokens = usage["reasoning_tokens"] or target.reasoning_tokens
+                    target.raw_usage = json.dumps(usage, sort_keys=True)
+                    target.model = target.model or model or _DEFAULT_MODEL
+                    target.usage_dedup_key = f"{session_id}:tok:{target.ordinal}"
+
+    if not messages:
+        return None
+    first_user = next((m.content for m in messages if m.role == "user" and m.content), None)
+    try:
+        mtime = int(path.stat().st_mtime)
+    except OSError:
+        mtime = None
+    return ParsedSession(
+        id=session_id,
+        agent="codex",
+        messages=messages,
+        project=Path(cwd).name if cwd else path.parent.name,
+        cwd=cwd,
+        display_name=(first_user[:80] if first_user else None),
+        first_message=first_user,
+        started_at=started_at,
+        ended_at=ended_at,
+        file_path=str(path),
+        file_mtime=mtime,
+    )
+
+
+def _canon_args(arguments: object) -> str:
+    if isinstance(arguments, str):
+        try:
+            return json.dumps(json.loads(arguments), sort_keys=True)
+        except Exception:
+            return arguments
+    if isinstance(arguments, dict):
+        return json.dumps(arguments, sort_keys=True)
+    return ""
+
+
+def _decode_output(out: object) -> tuple[str, bool]:
+    """Codex function_call_output is often a JSON string with {output, metadata:{exit_code}}."""
+    if isinstance(out, str):
+        try:
+            obj = json.loads(out)
+        except Exception:
+            return out, False
+    elif isinstance(out, dict):
+        obj = out
+    else:
+        return "", False
+    if isinstance(obj, dict):
+        text = str(obj.get("output", obj.get("content", "")))
+        meta = obj.get("metadata")
+        exit_code = meta.get("exit_code") if isinstance(meta, dict) else obj.get("exit_code")
+        is_err = bool(exit_code) and str(exit_code) != "0"
+        return text, is_err
+    return str(obj), False

zeno_session_intel/parsers/cursor.py

@@ -0,0 +1,198 @@
+"""Cursor chat parser - a SQLite ``state.vscdb`` (verified against real local data).
+
+Unlike Claude Code / Codex (one JSONL file == one session), one Cursor ``state.vscdb``
+holds MANY composers (each composer == one session), so this module exposes
+``parse_sessions(path) -> list[ParsedSession]`` and the ingester iterates it.
+
+Layout (verified on dctrl's Mac, 2026-06-17):
+``~/Library/Application Support/Cursor/User/globalStorage/state.vscdb`` (+ per-workspace
+``workspaceStorage/<hash>/state.vscdb``), a ``cursorDiskKV(key, value)`` table where:
+  - ``composerData:<composerId>`` -> JSON header: ``composerId``, ``name``, ``createdAt``
+    (epoch ms), ``fullConversationHeadersOnly`` = ordered ``[{bubbleId, type}]`` (type
+    1 = user, 2 = assistant).
+  - ``bubbleId:<composerId>:<bubbleId>`` -> JSON message: ``text``, ``type``,
+    ``toolFormerData`` (``.name`` e.g. ``read_file``), ``tokenCount`` (often ``0/0``).
+
+Quirks handled: only the most-recently-active composer has its bubbles materialized (the
+rest are header-only -> tolerate composers with zero retrievable bubbles); Cursor may hold
+the DB open -> **copy-then-read** the file (+ WAL sidecars) to a temp dir to dodge
+``database is locked``. Token counts are best-effort (often 0), so Cursor cost is usually
+unpriced - never alert on it.
+
+Read-only on the source; never raises (a bad row is skipped). py3.9-safe.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import sqlite3
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ..model import ParsedMessage, ParsedSession, ToolResult, ToolUse, to_int
+
+
+def default_root() -> Path:
+    return Path(os.path.expanduser("~")) / "Library" / "Application Support" / "Cursor" / "User"
+
+
+def discover(root: Path) -> list[Path]:
+    if not root.exists():
+        return []
+    found = []
+    for sub in ("globalStorage/state.vscdb",):
+        p = root / sub
+        if p.exists():
+            found.append(p)
+    found.extend(sorted((root / "workspaceStorage").glob("*/state.vscdb")))
+    return found
+
+
+def _epoch_ms_to_iso(ms: object) -> str | None:
+    try:
+        return datetime.fromtimestamp(float(ms) / 1000.0, tz=timezone.utc).isoformat()
+    except (ValueError, TypeError, OverflowError, OSError):
+        return None
+
+
+def _open_copy(path: Path):
+    """Copy the vscdb (+ -wal/-shm) to a temp dir and open it, dodging Cursor's lock.
+    Returns (connection, tempdir) or (None, tempdir) on failure; caller cleans up."""
+    d = tempfile.mkdtemp(prefix="zeno-cursor-")
+    dst = os.path.join(d, "state.vscdb")
+    try:
+        for ext in ("", "-wal", "-shm"):
+            src = str(path) + ext
+            if os.path.exists(src):
+                shutil.copy(src, dst + ext)
+        con = sqlite3.connect(dst)
+        con.execute("PRAGMA query_only=1")
+        return con, d
+    except Exception:
+        return None, d
+
+
+def parse_sessions(path: Path) -> list[ParsedSession]:
+    con, tmpdir = _open_copy(path)
+    if con is None:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+        return []
+    sessions: list[ParsedSession] = []
+    try:
+        composers = con.execute(
+            "SELECT key, value FROM cursorDiskKV WHERE key LIKE 'composerData:%'"
+        ).fetchall()
+        for _key, raw in composers:
+            s = _parse_composer(con, raw, path)
+            if s is not None:
+                sessions.append(s)
+    except Exception:
+        pass
+    finally:
+        con.close()
+        shutil.rmtree(tmpdir, ignore_errors=True)
+    return sessions
+
+
+def _parse_composer(con: sqlite3.Connection, raw: str, path: Path) -> ParsedSession | None:
+    try:
+        cd = json.loads(raw)
+    except Exception:
+        return None
+    if not isinstance(cd, dict):
+        return None
+    composer_id = str(cd.get("composerId") or "")
+    headers = cd.get("fullConversationHeadersOnly")
+    if not composer_id or not isinstance(headers, list) or not headers:
+        return None  # header-only / empty composer -> tolerate (skip)
+
+    messages: list[ParsedMessage] = []
+    ordinal = 0
+    last_ms = None
+    for h in headers:
+        if not isinstance(h, dict):
+            continue
+        bid = h.get("bubbleId")
+        btype = h.get("type")
+        if not bid:
+            continue
+        row = con.execute(
+            "SELECT value FROM cursorDiskKV WHERE key = ?",
+            (f"bubbleId:{composer_id}:{bid}",),
+        ).fetchone()
+        if not row:
+            continue  # bubble not materialized (common for older composers)
+        try:
+            b = json.loads(row[0])
+        except Exception:
+            continue
+        if not isinstance(b, dict):
+            continue
+        role = "user" if btype == 1 else "assistant" if btype == 2 else "system"
+        tool_uses: list[ToolUse] = []
+        tool_results: list[ToolResult] = []
+        tfd = b.get("toolFormerData")
+        if isinstance(tfd, dict) and tfd.get("name"):
+            params = tfd.get("params")
+            tool_uses.append(
+                ToolUse(
+                    name=str(tfd.get("name")),
+                    input_json=json.dumps(params, sort_keys=True) if params else "",
+                    id=str(b.get("bubbleId") or bid),
+                )
+            )
+            status = str(tfd.get("status", "")).lower()
+            tool_results.append(
+                ToolResult(
+                    tool_use_id=str(b.get("bubbleId") or bid),
+                    content=str(tfd.get("result", "")),
+                    is_error=status in ("error", "failed", "cancelled"),
+                )
+            )
+        tc = b.get("tokenCount") if isinstance(b.get("tokenCount"), dict) else {}
+        created = b.get("createdAt")
+        if created is not None:
+            last_ms = created
+        messages.append(
+            ParsedMessage(
+                ordinal=ordinal,
+                role=role,
+                content=str(b.get("text") or ""),
+                thinking_text="",
+                timestamp=_epoch_ms_to_iso(created),
+                model="",  # Cursor does not expose a per-bubble model id -> unpriced
+                input_tokens=to_int(tc.get("inputTokens")),
+                output_tokens=to_int(tc.get("outputTokens")),
+                raw_usage=json.dumps(tc, sort_keys=True) if tc else "",
+                source_uuid=str(bid),
+                tool_uses=tool_uses,
+                tool_results=tool_results,
+                usage_dedup_key=str(bid),
+            )
+        )
+        ordinal += 1
+
+    if not messages:
+        return None
+    first_user = next((m.content for m in messages if m.role == "user" and m.content), None)
+    return ParsedSession(
+        id=composer_id,
+        agent="cursor",
+        messages=messages,
+        project=str(cd.get("name") or "") or "cursor",
+        display_name=(str(cd.get("name")) if cd.get("name") else None),
+        first_message=first_user,
+        started_at=_epoch_ms_to_iso(cd.get("createdAt")),
+        ended_at=_epoch_ms_to_iso(last_ms if last_ms is not None else cd.get("lastUpdatedAt")),
+        file_path=str(path),
+    )
+
+
+def parse_file(path: Path) -> ParsedSession | None:
+    """Single-session convenience (returns the first composer). The ingester uses
+    ``parse_sessions`` because one vscdb holds many."""
+    out = parse_sessions(path)
+    return out[0] if out else None