agentabacus 0.1.0__py3-none-any.whl

agentabacus/__init__.py

@@ -0,0 +1,3 @@
+"""agentabacus — local-first analytics for AI coding agents."""
+
+__version__ = "0.1.0"

agentabacus/adapters/__init__.py

@@ -0,0 +1,17 @@
+"""Adapter registry.
+
+Adding support for a new agent CLI is: write a module exposing
+`parse(path, kind, start_offset) -> Batch`, add a walker in discovery.py, and
+register it here. Nothing else in the codebase needs to change.
+"""
+
+from __future__ import annotations
+
+from . import claude_code, codex
+
+REGISTRY = {
+    claude_code.SOURCE: claude_code.parse,
+    codex.SOURCE: codex.parse,
+}
+
+__all__ = ["REGISTRY", "claude_code", "codex"]

agentabacus/adapters/base.py

@@ -0,0 +1,101 @@
+"""Adapter helpers.
+
+Design rule for every adapter: **be a tolerant parser**. These formats are
+undocumented, version-dependent, and change without notice. An adapter that
+raises on an unknown `type` turns a vendor's routine release into a crash for
+every user. Route on known shapes, count what you skipped, never throw.
+
+The strictness lives in schema.py instead -- messy at the edges, contract-
+enforced in the core.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from typing import Any, Iterator
+
+
+def parse_ts(value: Any) -> datetime | None:
+    """ISO-8601 (with or without 'Z') or epoch ms/seconds. Never raises."""
+    if value is None:
+        return None
+    if isinstance(value, (int, float)):
+        # Heuristic: anything past ~2001 in seconds is milliseconds here.
+        seconds = value / 1000 if value > 1e11 else value
+        try:
+            return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(tzinfo=None)
+        except (ValueError, OSError, OverflowError):
+            return None
+    if isinstance(value, str):
+        try:
+            dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return None
+        return dt.astimezone(timezone.utc).replace(tzinfo=None) if dt.tzinfo else dt
+    return None
+
+
+def sha256(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()
+
+
+def as_int(value: Any) -> int:
+    return value if isinstance(value, int) else 0
+
+
+def iter_lines(path, start_offset: int = 0) -> Iterator[tuple[dict, int]]:
+    """Yield (record, offset_after_this_line) for every COMPLETE line.
+
+    A trailing partial line -- the file was mid-write when we read it -- is not
+    yielded and does not advance the offset, so the next run re-reads it whole.
+    Malformed complete lines are skipped with a sentinel rather than raising.
+    """
+    with open(path, "rb") as fh:
+        fh.seek(start_offset)
+        offset = start_offset
+        for raw in fh:
+            if not raw.endswith(b"\n"):
+                break  # torn write; leave the offset where it was
+            offset += len(raw)
+            text = raw.decode("utf-8", "replace").strip()
+            if not text:
+                continue
+            try:
+                record = json.loads(text)
+            except (ValueError, TypeError):
+                yield ({"__unparsed__": True}, offset)
+                continue
+            if isinstance(record, dict):
+                yield (record, offset)
+            else:
+                yield ({"__unparsed__": True}, offset)
+
+
+# Tool inputs vary per tool; these are the keys that carry "what did it act on".
+_TARGET_KEYS = ("file_path", "path", "notebook_path", "command", "url", "pattern", "query")
+
+
+def tool_target(tool_input: Any, limit: int = 400) -> str | None:
+    if not isinstance(tool_input, dict):
+        return None
+    for key in _TARGET_KEYS:
+        value = tool_input.get(key)
+        if isinstance(value, str) and value:
+            return value[:limit]
+    return None
+
+
+def content_chars(content: Any) -> int:
+    if isinstance(content, str):
+        return len(content)
+    if isinstance(content, list):
+        total = 0
+        for block in content:
+            if isinstance(block, dict):
+                total += len(block.get("text") or "")
+            elif isinstance(block, str):
+                total += len(block)
+        return total
+    return 0

agentabacus/adapters/claude_code.py

@@ -0,0 +1,230 @@
+"""Claude Code adapter.
+
+Reads three shapes, all JSONL:
+
+  <slug>/<uuid>.jsonl                     main session transcript
+  <slug>/<uuid>/subagents/agent-*.jsonl   one per subagent thread
+  history.jsonl                           every prompt typed, survives GC
+
+Two things this adapter exists to get right:
+
+1. **Dedupe by requestId.** Consecutive assistant lines are content blocks of
+   ONE API response and each repeats the full usage. Aggregating with MAX per
+   requestId is the difference between correct numbers and numbers that are
+   2-3x too high. Verified against real transcripts: 16 assistant lines
+   collapsed to 6 requests, a 2.4-3.0x overcount if summed naively.
+
+2. **The cache TTL split.** `cache_creation.ephemeral_1h_input_tokens` and
+   `ephemeral_5m_input_tokens` bill at different multipliers (2x vs 1.25x of
+   base input). We keep them apart all the way to the cost view.
+
+Subagent files carry the PARENT's sessionId plus their own `agentId`, so the
+thread_id is what separates them -- not the session id.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..schema import Batch, Prompt, Session, ToolCall, Turn
+from .base import as_int, content_chars, iter_lines, parse_ts, sha256, tool_target
+
+SOURCE = "claude_code"
+
+
+def _project_slug(path: Path) -> str | None:
+    """The slug is the directory directly under `projects/`.
+
+    Derived by walking up rather than by a fixed parent index, because subagent
+    transcripts sit at two different depths (plain vs workflow subagents).
+    Note this is only a label: the slug encoding ('/' -> '-') is lossy and
+    irreversible, so `cwd` from inside the file is the authoritative path.
+    """
+    for parent in path.parents:
+        if parent.parent.name == "projects":
+            return parent.name
+    return None
+
+
+def _usage_fields(usage: dict) -> dict:
+    """Pull the token counts, keeping the two cache-write TTLs separate."""
+    creation = usage.get("cache_creation")
+    if isinstance(creation, dict):
+        w1h = as_int(creation.get("ephemeral_1h_input_tokens"))
+        w5m = as_int(creation.get("ephemeral_5m_input_tokens"))
+    else:
+        # Older/other shapes give only the total. Attribute it to the 5m rate,
+        # which is the cheaper of the two -- under-report rather than inflate.
+        w1h, w5m = 0, as_int(usage.get("cache_creation_input_tokens"))
+
+    details = usage.get("output_tokens_details")
+    thinking = as_int(details.get("thinking_tokens")) if isinstance(details, dict) else 0
+
+    server = usage.get("server_tool_use")
+    server = server if isinstance(server, dict) else {}
+
+    return {
+        "input_tokens": as_int(usage.get("input_tokens")),
+        "output_tokens": as_int(usage.get("output_tokens")),
+        "thinking_tokens": thinking,
+        "cache_read_tokens": as_int(usage.get("cache_read_input_tokens")),
+        "cache_write_5m_tokens": w5m,
+        "cache_write_1h_tokens": w1h,
+        "web_search_requests": as_int(server.get("web_search_requests")),
+        "web_fetch_requests": as_int(server.get("web_fetch_requests")),
+    }
+
+
+def parse_history(path: Path, start_offset: int = 0) -> Batch:
+    """history.jsonl: prompt text, project, sessionId, epoch-ms timestamp.
+
+    Only a hash and a length are kept. The prompt body never enters the
+    pipeline, so "does this leak my code?" is answerable from the schema
+    instead of from trust in a downstream filter.
+    """
+    batch = Batch(byte_offset=start_offset)
+    for record, offset in iter_lines(path, start_offset):
+        batch.byte_offset = offset
+        if record.get("__unparsed__"):
+            batch.skipped_lines += 1
+            continue
+        display = record.get("display")
+        if not isinstance(display, str):
+            continue
+        batch.prompts.append(
+            Prompt(
+                prompt_sha256=sha256(display),
+                session_id=record.get("sessionId"),
+                source=SOURCE,
+                ts=parse_ts(record.get("timestamp")),
+                project=record.get("project"),
+                char_len=len(display),
+            )
+        )
+    return batch
+
+
+def parse_transcript(path: Path, start_offset: int = 0, is_subagent: bool = False) -> Batch:
+    batch = Batch(byte_offset=start_offset)
+
+    turns: dict[str, Turn] = {}
+    tools: dict[str, ToolCall] = {}
+    meta: dict = {}
+    first_ts = last_ts = None
+    session_id = thread_id = None
+
+    for record, offset in iter_lines(path, start_offset):
+        batch.byte_offset = offset
+        if record.get("__unparsed__"):
+            batch.skipped_lines += 1
+            continue
+
+        rec_type = record.get("type")
+        ts = parse_ts(record.get("timestamp"))
+        if ts:
+            first_ts = ts if first_ts is None or ts < first_ts else first_ts
+            last_ts = ts if last_ts is None or ts > last_ts else last_ts
+
+        if record.get("sessionId"):
+            session_id = record["sessionId"]
+        # Subagent threads are identified by agentId, NOT by a distinct session id.
+        if record.get("agentId"):
+            thread_id = record["agentId"]
+        for key in ("cwd", "version", "gitBranch", "entrypoint", "attributionAgent"):
+            if record.get(key):
+                meta.setdefault(key, record[key])
+        if record.get("isSidechain"):
+            is_subagent = True
+
+        message = record.get("message")
+        message = message if isinstance(message, dict) else {}
+        blocks = message.get("content")
+        blocks = blocks if isinstance(blocks, list) else []
+
+        if rec_type == "assistant":
+            request_id = record.get("requestId") or message.get("id")
+            usage = message.get("usage")
+            if request_id and isinstance(usage, dict):
+                fields = _usage_fields(usage)
+                existing = turns.get(request_id)
+                if existing is None:
+                    turns[request_id] = Turn(
+                        request_id=request_id,
+                        session_id=session_id or "",
+                        thread_id=thread_id or "main",
+                        source=SOURCE,
+                        model_id=message.get("model"),
+                        ts=ts,
+                        effort=record.get("effort"),
+                        service_tier=usage.get("service_tier"),
+                        speed=usage.get("speed") or "standard",
+                        block_lines=1,
+                        **fields,
+                    )
+                else:
+                    # THE DEDUPE. Same request, another content block, same usage
+                    # repeated. MAX (not +=) so re-reads and partial lines are safe.
+                    for key, value in fields.items():
+                        setattr(existing, key, max(getattr(existing, key), value))
+                    existing.block_lines += 1
+                    if existing.ts is None or (ts and ts < existing.ts):
+                        existing.ts = ts
+
+            for block in blocks:
+                if isinstance(block, dict) and block.get("type") == "tool_use":
+                    tool_id = block.get("id")
+                    if not tool_id:
+                        continue
+                    tools[tool_id] = ToolCall(
+                        tool_use_id=tool_id,
+                        session_id=session_id or "",
+                        thread_id=thread_id or "main",
+                        source=SOURCE,
+                        request_id=request_id,
+                        ts=ts,
+                        tool_name=block.get("name"),
+                        target=tool_target(block.get("input")),
+                    )
+
+        elif rec_type == "user":
+            for block in blocks:
+                if not isinstance(block, dict) or block.get("type") != "tool_result":
+                    continue
+                tool_id = block.get("tool_use_id")
+                call = tools.get(tool_id) if tool_id else None
+                if call is None:
+                    continue
+                call.is_error = bool(block.get("is_error"))
+                call.result_chars = content_chars(block.get("content"))
+
+    if session_id is None:
+        return batch
+
+    thread = thread_id or "main"
+    batch.sessions.append(
+        Session(
+            session_id=session_id,
+            thread_id=thread,
+            source=SOURCE,
+            cwd=meta.get("cwd"),
+            project_slug=_project_slug(path),
+            git_branch=meta.get("gitBranch"),
+            cli_version=meta.get("version"),
+            entrypoint=meta.get("entrypoint"),
+            agent_type=meta.get("attributionAgent"),
+            parent_session_id=session_id if is_subagent else None,
+            is_subagent=is_subagent,
+            started_at=first_ts,
+            ended_at=last_ts,
+            transcript_path=str(path),
+        )
+    )
+    batch.turns = list(turns.values())
+    batch.tool_calls = list(tools.values())
+    return batch
+
+
+def parse(path: Path, kind: str, start_offset: int = 0) -> Batch:
+    if kind == "history":
+        return parse_history(path, start_offset)
+    return parse_transcript(path, start_offset, is_subagent=(kind == "subagent"))

agentabacus/adapters/codex.py

@@ -0,0 +1,122 @@
+"""Codex CLI adapter -- STRUCTURE UNVERIFIED.
+
+No Codex rollout files were available on the machine this was written against,
+so this adapter is deliberately shape-agnostic: it walks each record looking for
+anything that resembles a usage object and a request identifier, rather than
+asserting a layout it cannot confirm.
+
+That makes it useful as a starting point and as the template for a third
+adapter, but treat its output as provisional until someone verifies it against
+real files. Rows land with `source='codex'`, so they are trivial to exclude.
+
+Contributing a verified version is the single highest-value PR here: paste two
+or three real records into a test fixture and tighten the parsing.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ..schema import Batch, Session, Turn
+from .base import as_int, iter_lines, parse_ts
+
+SOURCE = "codex"
+
+# Both Anthropic-style and OpenAI-style names, since Codex may use either.
+_INPUT_KEYS = ("input_tokens", "prompt_tokens")
+_OUTPUT_KEYS = ("output_tokens", "completion_tokens")
+
+
+def _find_usage(node: Any, depth: int = 0):
+    """Depth-limited hunt for a dict that looks like a usage object."""
+    if depth > 6 or not isinstance(node, dict):
+        return None
+    looks_like_usage = any(k in node for k in _INPUT_KEYS) and any(
+        k in node for k in _OUTPUT_KEYS
+    )
+    if looks_like_usage:
+        return node
+    for value in node.values():
+        if isinstance(value, dict):
+            found = _find_usage(value, depth + 1)
+            if found is not None:
+                return found
+    return None
+
+
+def _first(node: dict, keys) -> Any:
+    for key in keys:
+        value = node.get(key)
+        if value:
+            return value
+    return None
+
+
+def parse(path: Path, kind: str = "transcript", start_offset: int = 0) -> Batch:
+    batch = Batch(byte_offset=start_offset)
+    session_id = path.stem
+    turns: dict[str, Turn] = {}
+    model_id = None
+    first_ts = last_ts = None
+
+    for record, offset in iter_lines(path, start_offset):
+        batch.byte_offset = offset
+        if record.get("__unparsed__"):
+            batch.skipped_lines += 1
+            continue
+
+        ts = parse_ts(_first(record, ("timestamp", "created_at", "time")))
+        if ts:
+            first_ts = ts if first_ts is None or ts < first_ts else first_ts
+            last_ts = ts if last_ts is None or ts > last_ts else last_ts
+        model_id = model_id or _first(record, ("model", "model_id"))
+
+        usage = _find_usage(record)
+        if not usage:
+            continue
+
+        request_id = (
+            _first(record, ("request_id", "response_id", "id"))
+            or f"{session_id}:{offset}"
+        )
+        fields = {
+            "input_tokens": as_int(_first(usage, _INPUT_KEYS)),
+            "output_tokens": as_int(_first(usage, _OUTPUT_KEYS)),
+            "cache_read_tokens": as_int(
+                _first(usage, ("cache_read_input_tokens", "cached_tokens"))
+            ),
+        }
+        existing = turns.get(request_id)
+        if existing is None:
+            turns[request_id] = Turn(
+                request_id=f"codex:{request_id}",
+                session_id=session_id,
+                thread_id="main",
+                source=SOURCE,
+                model_id=_first(record, ("model", "model_id")) or model_id,
+                ts=ts,
+                speed="standard",
+                block_lines=1,
+                **fields,
+            )
+        else:
+            # Same dedupe discipline as the Claude Code adapter: if a runtime
+            # repeats usage across records, MAX keeps the total honest.
+            for key, value in fields.items():
+                setattr(existing, key, max(getattr(existing, key), value))
+            existing.block_lines += 1
+
+    if turns:
+        batch.sessions.append(
+            Session(
+                session_id=session_id,
+                thread_id="main",
+                source=SOURCE,
+                started_at=first_ts,
+                ended_at=last_ts,
+                transcript_path=str(path),
+            )
+        )
+        batch.turns = list(turns.values())
+    return batch