tokenol 0.1.0__py3-none-any.whl

tokenol/__init__.py

@@ -0,0 +1,3 @@
+"""tokenol — Claude Code usage & efficiency audit tool."""
+
+__version__ = "0.1.0.dev0"

tokenol/assumptions.py

@@ -0,0 +1,51 @@
+"""AssumptionTag recorder — collects fired heuristics for report footer."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from tokenol.enums import AssumptionTag
+
+
+class AssumptionRecorder:
+    """Thread-unsafe (single-threaded CLI use only) assumption counter."""
+
+    def __init__(self) -> None:
+        self._counts: Counter[AssumptionTag] = Counter()
+
+    def record(self, tags: list[AssumptionTag]) -> None:
+        for t in tags:
+            self._counts[t] += 1
+
+    def fired(self) -> dict[AssumptionTag, int]:
+        return dict(self._counts)
+
+    def summary_lines(self) -> list[str]:
+        if not self._counts:
+            return []
+        lines = ["Assumptions fired:"]
+        for tag, count in sorted(self._counts.items(), key=lambda x: x[0].value):
+            lines.append(f"  {tag.value}: {count:,} event(s)")
+        return lines
+
+    def reset(self) -> None:
+        self._counts.clear()
+
+
+_recorder = AssumptionRecorder()
+
+
+def record(tags: list[AssumptionTag]) -> None:
+    _recorder.record(tags)
+
+
+def fired() -> dict[AssumptionTag, int]:
+    return _recorder.fired()
+
+
+def summary_lines() -> list[str]:
+    return _recorder.summary_lines()
+
+
+def reset() -> None:
+    _recorder.reset()

tokenol/cli.py

@@ -0,0 +1,408 @@
+"""tokenol — Claude Code usage & efficiency audit CLI."""
+
+from __future__ import annotations
+
+import logging
+import re
+import subprocess
+from datetime import date, datetime, timedelta, timezone
+from enum import Enum
+
+import typer
+from rich.console import Console
+
+from tokenol import assumptions as assumption_recorder
+from tokenol.ingest.builder import build_sessions, build_turns
+from tokenol.ingest.discovery import find_jsonl_files, get_config_dirs
+from tokenol.metrics.cost import rollup_by_date, rollup_by_hour
+from tokenol.metrics.rollups import (
+    build_model_rollups,
+    build_project_rollups,
+    build_session_rollup,
+)
+from tokenol.metrics.verdicts import compute_verdict
+from tokenol.metrics.windows import align_windows, project_window
+from tokenol.report.text import (
+    print_daily,
+    print_hourly,
+    print_live_full,
+    print_models,
+    print_projects,
+    print_sessions,
+)
+
+app = typer.Typer(
+    name="tokenol",
+    help="Audit Claude Code JSONL logs: cost, cache health, blow-up detection.",
+    add_completion=False,
+)
+
+# Single process-wide flag — set by the --all-projects option on each command.
+# Simpler than threading it through every helper signature.
+_SCAN_ALL: bool = False
+
+console = Console(stderr=False)
+err = Console(stderr=True)
+
+
+class LogLevel(str, Enum):
+    debug = "debug"
+    info = "info"
+    warning = "warning"
+
+
+class SortKey(str, Enum):
+    cost = "cost"
+    input = "input"
+    output = "output"
+    cache_read = "cache_read"
+    turns = "turns"
+    max_input = "max_input"
+    duration = "duration"
+
+
+def _parse_since(since: str) -> date:
+    """Parse e.g. '14d', '30d', '2026-04-01' into a date."""
+    since = since.strip()
+    if since.endswith("d"):
+        days = int(since[:-1])
+        return (datetime.now(tz=timezone.utc) - timedelta(days=days)).date()
+    return date.fromisoformat(since)
+
+
+def _parse_last(last: str) -> timedelta:
+    """Parse lookback duration like '20m', '2h', '30s'. Rejects bare numbers and zero."""
+    m = re.fullmatch(r"([1-9]\d*)([mhs])", last.strip())
+    if not m:
+        raise typer.BadParameter(
+            f"Invalid duration '{last}'. Use a positive number with unit: '20m', '2h', '30s'."
+        )
+    value = int(m.group(1))
+    unit = m.group(2)
+    if unit == "m":
+        return timedelta(minutes=value)
+    if unit == "h":
+        return timedelta(hours=value)
+    # unit == "s"
+    return timedelta(seconds=value)
+
+
+def _configure_logging(log_level: LogLevel) -> None:
+    level = getattr(logging, log_level.value.upper(), logging.INFO)
+    logging.basicConfig(level=level)
+
+
+def _set_scan_scope(all_projects: bool) -> None:
+    global _SCAN_ALL
+    _SCAN_ALL = all_projects
+
+
+_ALL_PROJECTS_OPT = typer.Option(
+    False,
+    "--all-projects",
+    "-A",
+    help="Scan every ~/.claude* directory, ignoring CLAUDE_CONFIG_DIR.",
+)
+
+
+def _timedelta_label(td: timedelta) -> str:
+    """Convert a timedelta to a short human label: '20min', '2hr', '30sec'."""
+    total = int(td.total_seconds())
+    if total % 3600 == 0:
+        return f"{total // 3600}hr"
+    if total % 60 == 0:
+        return f"{total // 60}min"
+    return f"{total}sec"
+
+
+def _load_turns(since: date | None = None):
+    assumption_recorder.reset()
+    dirs = get_config_dirs(all_projects=_SCAN_ALL)
+    paths = find_jsonl_files(dirs)
+    turns = build_turns(paths)
+    if since:
+        turns = [t for t in turns if t.timestamp.date() >= since]
+    return turns, paths
+
+
+def _load_turns_and_sessions(since: date | None = None):
+    assumption_recorder.reset()
+    dirs = get_config_dirs(all_projects=_SCAN_ALL)
+    paths = find_jsonl_files(dirs)
+    turns = build_turns(paths)
+    sessions = build_sessions(turns, paths=paths)
+    if since:
+        turns = [t for t in turns if t.timestamp.date() >= since]
+        sessions = [
+            s for s in sessions
+            if s.turns and s.turns[-1].timestamp.date() >= since
+        ]
+    return turns, sessions, paths
+
+
+@app.command()
+def daily(
+    since: str = typer.Option("14d", help="Start date, e.g. '14d' or '2026-04-01'"),
+    strict: bool = typer.Option(False, "--strict", help="Error on any assumption fallback"),
+    show_assumptions: bool = typer.Option(False, "--show-assumptions", help="Always print assumption footer"),
+    log_level: LogLevel = typer.Option(LogLevel.info, "--log-level"),  # noqa: B008
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+) -> None:
+    """Daily token and cost aggregates."""
+    _set_scan_scope(all_projects)
+    _configure_logging(log_level)
+    since_date = _parse_since(since)
+    turns, paths = _load_turns(since=since_date)
+    if strict and assumption_recorder.fired():
+        raise typer.BadParameter("Assumptions fired and --strict is set.")
+    rollups = rollup_by_date(turns, since=since_date)
+    print_daily(rollups, console=console, show_assumptions=show_assumptions)
+
+
+@app.command()
+def hourly(
+    day: str | None = typer.Argument(None, help="Date as YYYY-MM-DD (default: today)"),
+    strict: bool = typer.Option(False, "--strict"),
+    show_assumptions: bool = typer.Option(False, "--show-assumptions"),
+    log_level: LogLevel = typer.Option(LogLevel.info, "--log-level"),  # noqa: B008
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+) -> None:
+    """Hourly token and cost breakdown for one day."""
+    _set_scan_scope(all_projects)
+    _configure_logging(log_level)
+    target = date.fromisoformat(day) if day else date.today()
+    turns, paths = _load_turns()
+    if strict and assumption_recorder.fired():
+        raise typer.BadParameter("Assumptions fired and --strict is set.")
+    rollups = rollup_by_hour(turns, target_date=target)
+    print_hourly(rollups, console=console, show_assumptions=show_assumptions)
+
+
+@app.command()
+def live(
+    last: str = typer.Option(..., "--last", help="Lookback duration, e.g. '20m', '2h', '30s'"),
+    strict: bool = typer.Option(False, "--strict"),
+    show_assumptions: bool = typer.Option(False, "--show-assumptions"),
+    log_level: LogLevel = typer.Option(LogLevel.info, "--log-level"),  # noqa: B008
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+) -> None:
+    """Live burn-rate view for the active 5h window."""
+    _set_scan_scope(all_projects)
+    _configure_logging(log_level)
+    lookback = _parse_last(last)
+
+    turns, paths = _load_turns()
+    if strict and assumption_recorder.fired():
+        raise typer.BadParameter("Assumptions fired and --strict is set.")
+
+    now = datetime.now(tz=timezone.utc)
+
+    # Only consider turns within a reasonable past window (last 10h) to keep it fast
+    recent_all = [t for t in turns if t.timestamp >= now - timedelta(hours=10)]
+    windows = align_windows(recent_all)
+
+    if not windows:
+        console.print("[yellow]No active window found.[/yellow]")
+        raise typer.Exit(code=0)
+
+    active = windows[-1]
+
+    # Check if active window is actually current (within 5h of now)
+    if active.end < now:
+        console.print("[yellow]No active window found (all windows have ended).[/yellow]")
+        raise typer.Exit(code=0)
+
+    proj = project_window(active, now=now, lookback=lookback)
+
+    cutoff = now - lookback
+    recent_turns_count = sum(1 for t in active.turns if t.timestamp >= cutoff)
+
+    print_live_full(
+        active_window=active,
+        projection=proj,
+        recent_turns_count=recent_turns_count,
+        last_label=_timedelta_label(lookback),
+        console=console,
+    )
+
+    if proj["over_reference"]:
+        raise typer.Exit(code=1)
+
+
+@app.command()
+def sessions(
+    top: int = typer.Option(10, "--top", help="Number of sessions to show"),
+    sort: SortKey = typer.Option(SortKey.cost, "--sort", help="Sort key"),  # noqa: B008
+    since: str = typer.Option("14d", help="Start date, e.g. '14d' or '2026-04-01'"),
+    strict: bool = typer.Option(False, "--strict"),
+    show_assumptions: bool = typer.Option(False, "--show-assumptions"),
+    log_level: LogLevel = typer.Option(LogLevel.info, "--log-level"),  # noqa: B008
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+) -> None:
+    """Per-session detail table sorted by a chosen metric."""
+    _set_scan_scope(all_projects)
+    _configure_logging(log_level)
+    since_date = _parse_since(since)
+    turns, session_list, paths = _load_turns_and_sessions(since=since_date)
+
+    if strict and assumption_recorder.fired():
+        raise typer.BadParameter("Assumptions fired and --strict is set.")
+
+    # Build rollups and compute verdicts
+    rollups = []
+    for session in session_list:
+        sr = build_session_rollup(session)
+        sr.verdict = compute_verdict(sr)
+        rollups.append(sr)
+
+    _sort_attrs = {
+        SortKey.cost: lambda sr: sr.cost_usd,
+        SortKey.input: lambda sr: float(sr.input_tokens),
+        SortKey.output: lambda sr: float(sr.output_tokens),
+        SortKey.cache_read: lambda sr: float(sr.cache_read_tokens),
+        SortKey.turns: lambda sr: float(sr.turns),
+        SortKey.max_input: lambda sr: float(sr.max_turn_input),
+        SortKey.duration: lambda sr: (sr.last_ts - sr.first_ts).total_seconds(),
+    }
+    rollups.sort(key=_sort_attrs[sort], reverse=True)
+    rollups = rollups[:top]
+
+    print_sessions(rollups, console=console, show_assumptions=show_assumptions)
+
+
+@app.command()
+def projects(
+    since: str = typer.Option("14d", help="Start date, e.g. '14d' or '2026-04-01'"),
+    strict: bool = typer.Option(False, "--strict"),
+    show_assumptions: bool = typer.Option(False, "--show-assumptions"),
+    log_level: LogLevel = typer.Option(LogLevel.info, "--log-level"),  # noqa: B008
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+) -> None:
+    """Per-project rollup (grouped by cwd)."""
+    _set_scan_scope(all_projects)
+    _configure_logging(log_level)
+    since_date = _parse_since(since)
+    turns, session_list, paths = _load_turns_and_sessions(since=since_date)
+
+    if strict and assumption_recorder.fired():
+        raise typer.BadParameter("Assumptions fired and --strict is set.")
+
+    session_rollups = [build_session_rollup(s) for s in session_list]
+    project_rollups = build_project_rollups(session_rollups)
+    total_cost = sum(pr.cost_usd for pr in project_rollups)
+
+    print_projects(project_rollups, total_cost=total_cost, console=console, show_assumptions=show_assumptions)
+
+
+@app.command()
+def models(
+    since: str = typer.Option("14d", help="Start date, e.g. '14d' or '2026-04-01'"),
+    strict: bool = typer.Option(False, "--strict"),
+    show_assumptions: bool = typer.Option(False, "--show-assumptions"),
+    log_level: LogLevel = typer.Option(LogLevel.info, "--log-level"),  # noqa: B008
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+) -> None:
+    """Per-model rollup."""
+    _set_scan_scope(all_projects)
+    _configure_logging(log_level)
+    since_date = _parse_since(since)
+    turns, paths = _load_turns(since=since_date)
+
+    if strict and assumption_recorder.fired():
+        raise typer.BadParameter("Assumptions fired and --strict is set.")
+
+    model_rollups = build_model_rollups(turns)
+    print_models(model_rollups, console=console, show_assumptions=show_assumptions)
+
+
+@app.command()
+def verify(
+    since: str = typer.Option("14d"),
+    tolerance: float = typer.Option(0.02, help="Max acceptable fractional diff vs ccusage"),
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+) -> None:
+    """Cross-check tokenol totals against ccusage (if installed)."""
+    _set_scan_scope(all_projects)
+    since_date = _parse_since(since)
+    turns, paths = _load_turns(since=since_date)
+
+    our_cost = sum(t.cost_usd for t in turns)
+
+    # Try ccusage
+    try:
+        result = subprocess.run(
+            ["ccusage", "--json"],
+            capture_output=True, text=True, timeout=30,
+        )
+        if result.returncode != 0:
+            err.print("[yellow]ccusage returned non-zero exit — skipping comparison[/yellow]")
+            console.print(f"tokenol total: ${our_cost:.4f}")
+            return
+        import json
+        data = json.loads(result.stdout)
+        # ccusage JSON structure: list of day objects or a totals key
+        ccusage_cost = 0.0
+        if isinstance(data, list):
+            for row in data:
+                ccusage_cost += row.get("cost", 0) or row.get("totalCost", 0)
+        elif isinstance(data, dict):
+            ccusage_cost = data.get("totalCost", data.get("cost", 0))
+
+        diff = abs(our_cost - ccusage_cost) / max(ccusage_cost, 1e-9)
+        status = "OK" if diff <= tolerance else "FAIL"
+        console.print(f"tokenol: ${our_cost:.4f}  ccusage: ${ccusage_cost:.4f}  diff: {diff:.1%}  [{status}]")
+        if status == "FAIL":
+            raise typer.Exit(code=1)
+
+    except FileNotFoundError:
+        err.print("[dim]ccusage not on PATH — showing tokenol total only[/dim]")
+        console.print(f"tokenol total: ${our_cost:.4f}")
+
+
+@app.command()
+def serve(
+    port: int = typer.Option(8787, "--port", help="TCP port to bind."),
+    tick: str = typer.Option("5s", "--tick", help="SSE tick interval, e.g. '2s', '5s'."),
+    reference: float = typer.Option(50.0, "--reference", help="$/window alarm threshold."),
+    open_browser: bool = typer.Option(False, "--open", help="Open dashboard in default browser."),
+    all_projects: bool = _ALL_PROJECTS_OPT,  # noqa: B008
+    log_level: LogLevel = typer.Option(LogLevel.info, "--log-level"),  # noqa: B008
+) -> None:
+    """Start the live dashboard server."""
+    _configure_logging(log_level)
+    tick_td = _parse_last(tick)
+    tick_seconds = max(1, int(tick_td.total_seconds()))
+
+    try:
+        import uvicorn
+
+        from tokenol.serve.app import ServerConfig, create_app
+    except ImportError:
+        err.print(
+            "[red]tokenol[serve] extras not installed.[/red] "
+            "Run: pip install 'tokenol[serve]'"
+        )
+        raise typer.Exit(code=1) from None
+
+    config = ServerConfig(
+        all_projects=all_projects,
+        reference_usd=reference,
+        tick_seconds=tick_seconds,
+    )
+    application = create_app(config)
+    url = f"http://127.0.0.1:{port}"
+    err.print(f"tokenol dashboard → {url}")
+
+    if open_browser:
+        import webbrowser
+        webbrowser.open(url)
+
+    uvicorn.run(application, host="127.0.0.1", port=port, log_level=log_level.value.lower())
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

tokenol/enums.py

@@ -0,0 +1,26 @@
+from enum import Enum
+
+
+class BlockType(str, Enum):
+    TEXT = "text"
+    THINKING = "thinking"
+    TOOL_USE = "tool_use"
+    TOOL_RESULT = "tool_result"
+    REDACTED_THINKING = "redacted_thinking"
+    OTHER = "other"
+
+
+class BlowUpVerdict(str, Enum):
+    OK = "OK"
+    CONTEXT_CREEP = "CONTEXT_CREEP"
+    SIDECHAIN_HEAVY = "SIDECHAIN_HEAVY"
+    TOOL_ERROR_STORM = "TOOL_ERROR_STORM"
+    RUNAWAY_WINDOW = "RUNAWAY_WINDOW"
+
+
+class AssumptionTag(str, Enum):
+    WINDOW_BOUNDARY_HEURISTIC = "WINDOW_BOUNDARY_HEURISTIC"
+    UNKNOWN_MODEL_FALLBACK = "UNKNOWN_MODEL_FALLBACK"
+    DEDUP_PASSTHROUGH = "DEDUP_PASSTHROUGH"
+    INTERRUPTED_TURN_SKIPPED = "INTERRUPTED_TURN_SKIPPED"
+    GEMINI_UNPRICED = "GEMINI_UNPRICED"

tokenol/ingest/builder.py

@@ -0,0 +1,88 @@
+"""Build Turn objects from deduplicated RawEvents, applying cost computation."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from pathlib import Path
+
+from tokenol import assumptions as assumption_recorder
+from tokenol.ingest.parser import dedup_key, iter_assistant_events, parse_file
+from tokenol.metrics.cost import cost_for_turn
+from tokenol.model.events import Session, Turn, Usage
+
+
+def build_turns(paths: list[Path]) -> list[Turn]:
+    """Parse *paths*, deduplicate, compute cost, record assumptions."""
+    turns: list[Turn] = []
+
+    for ev, tags in iter_assistant_events(paths):
+        is_interrupted = ev.usage is None
+        usage = ev.usage if ev.usage is not None else Usage()
+
+        tc = cost_for_turn(ev.model, usage)
+        tags.extend(t for t in tc.assumptions if t not in tags)
+        assumption_recorder.record(tags)
+
+        key = dedup_key(ev) or ev.uuid or str(id(ev))
+
+        turns.append(
+            Turn(
+                dedup_key=key,
+                timestamp=ev.timestamp,
+                session_id=ev.session_id,
+                model=ev.model,
+                usage=usage,
+                is_sidechain=ev.is_sidechain,
+                stop_reason=ev.stop_reason,
+                assumptions=tags,
+                cost_usd=tc.total_usd,
+                is_interrupted=is_interrupted,
+                tool_use_count=ev.tool_use_count,
+                tool_error_count=ev.tool_error_count,
+            )
+        )
+
+    return turns
+
+
+def build_sessions(turns: list[Turn], paths: list[Path] | None = None) -> list[Session]:
+    """Group turns by session_id. One Session per JSONL file.
+
+    If *paths* is provided, scans each file once to extract cwd and source path.
+    """
+    cwd_by_session: dict[str, str] = {}
+    session_source: dict[str, str] = {}
+
+    if paths:
+        for path in paths:
+            sid = path.stem
+            session_source[sid] = str(path)
+            for raw_ev in parse_file(path):
+                if raw_ev.cwd and sid not in cwd_by_session:
+                    cwd_by_session[sid] = raw_ev.cwd
+                    break
+
+    session_turns: dict[str, list[Turn]] = defaultdict(list)
+    session_sidechain: dict[str, bool] = {}
+
+    for turn in turns:
+        sid = turn.session_id
+        session_turns[sid].append(turn)
+        if sid not in session_sidechain:
+            session_sidechain[sid] = turn.is_sidechain
+
+    sessions: list[Session] = []
+    for sid, t_list in session_turns.items():
+        t_list.sort(key=lambda t: t.timestamp)
+        sessions.append(
+            Session(
+                session_id=sid,
+                source_file=session_source.get(sid, ""),
+                is_sidechain=session_sidechain.get(sid, False),
+                cwd=cwd_by_session.get(sid),
+                turns=t_list,
+            )
+        )
+
+    sessions.sort(key=lambda s: s.turns[0].timestamp if s.turns else s.session_id)
+    return sessions

tokenol/ingest/discovery.py

@@ -0,0 +1,43 @@
+"""Discover JSONL files from ~/.claude* config directories."""
+
+from __future__ import annotations
+
+import glob
+import os
+from pathlib import Path
+
+
+def get_config_dirs(all_projects: bool = False) -> list[Path]:
+    """Return config directories to scan.
+
+    - If *all_projects* is True, scan every ~/.claude* directory (ignoring
+      CLAUDE_CONFIG_DIR). Useful when workspace isolation points the env var
+      at a single project but you want a cross-project view.
+    - Otherwise, if CLAUDE_CONFIG_DIR is set, honour it (single path, or
+      colon- or comma-separated list of paths). Matches ccusage behaviour.
+    - Otherwise, scan all ~/.claude* directories.
+    """
+    home = Path.home()
+    all_dirs = sorted(p for p in home.glob(".claude*") if p.is_dir())
+
+    if all_projects:
+        return all_dirs
+
+    env = os.environ.get("CLAUDE_CONFIG_DIR")
+    if env:
+        parts = [p for chunk in env.split(",") for p in chunk.split(":") if p]
+        return [Path(p) for p in parts]
+
+    return all_dirs
+
+
+def find_jsonl_files(dirs: list[Path] | None = None) -> list[Path]:
+    """Return all JSONL files under the given (or auto-discovered) dirs."""
+    if dirs is None:
+        dirs = get_config_dirs()
+
+    files: list[Path] = []
+    for d in dirs:
+        pattern = str(d / "projects" / "**" / "*.jsonl")
+        files.extend(Path(p) for p in glob.glob(pattern, recursive=True))
+    return sorted(files)