devin-metrics 0.2.0__py3-none-any.whl

devin_metrics/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

devin_metrics/aggregate.py

@@ -0,0 +1,199 @@
+"""Rollups over a :class:`MetricsSnapshot` — pure functions, JSON-ready dicts.
+
+Attribution rules:
+
+- Sessions group by ``working_directory`` (the project) — free with Devin's
+  session store.
+- Day buckets use the session's ``created_at`` in UTC.
+- ``cost_usd_total`` counts **all** acp usage rows (including orphan dbs with
+  no session row); ``cost_usd_by_sessions`` counts only matched sessions.
+- ``cost_usd is None`` means *unknown* (no acp data), never zero.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Iterable
+
+from devin_metrics.collect import MetricsSnapshot, SessionMetrics, UsageRecord
+
+
+def _day(ms: int) -> str:
+    return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).date().isoformat()
+
+
+def session_to_dict(s: SessionMetrics) -> dict[str, Any]:
+    return {
+        "id": s.id,
+        "title": s.title,
+        "project": s.working_directory,
+        "model": s.model,
+        "created": _day(s.created_at),
+        "created_at": s.created_at,
+        "duration_ms": s.duration_ms,
+        "messages": s.n_messages,
+        "tool_calls": s.n_tool_calls,
+        "cost_usd": s.cost_usd,
+        "input_tokens": s.input_tokens,
+        "output_tokens": s.output_tokens,
+        "context_tokens": s.context_tokens,
+    }
+
+
+def _sum_or_none(values: Iterable[float | int | None]) -> float | int | None:
+    known = [v for v in values if v is not None]
+    return sum(known) if known else None
+
+
+def _sum(values: Iterable[float | int]) -> float | int:
+    return sum(values)
+
+
+# -- summaries ---------------------------------------------------------------
+
+
+def summarize(snap: MetricsSnapshot) -> dict[str, Any]:
+    sessions = list(snap.sessions)
+    n = len(sessions)
+    with_cost = [s for s in sessions if s.cost_usd is not None]
+    return {
+        "sessions": n,
+        "sessions_with_cost": len(with_cost),
+        "hidden_sessions": sum(1 for s in sessions if s.hidden),
+        "first_seen": _day(min(s.created_at for s in sessions)) if n else None,
+        "last_seen": _day(max(s.last_activity_at for s in sessions)) if n else None,
+        "messages": _sum(s.n_messages for s in sessions),
+        "tool_calls": _sum(s.n_tool_calls for s in sessions),
+        "duration_ms_total": _sum(s.duration_ms for s in sessions),
+        "cost_usd_total": _sum_or_none(u.cost_usd for u in snap.usage),
+        "cost_usd_by_sessions": _sum_or_none(s.cost_usd for s in sessions),
+        "input_tokens_total": _sum_or_none(u.input_tokens for u in snap.usage),
+        "output_tokens_total": _sum_or_none(u.output_tokens for u in snap.usage),
+        "context_tokens_peak": max(
+            (s.context_tokens for s in sessions
+             if s.context_tokens is not None),
+            default=None,
+        ),
+        "sessions_with_context": sum(
+            1 for s in sessions if s.context_tokens is not None
+        ),
+        "avg_duration_ms": (_sum(s.duration_ms for s in sessions) / n) if n else None,
+        "avg_messages": (_sum(s.n_messages for s in sessions) / n) if n else None,
+        "avg_cost_usd": (
+            _sum(s.cost_usd for s in with_cost) / len(with_cost)  # type: ignore[arg-type]
+            if with_cost
+            else None
+        ),
+        "acp_available": snap.acp_available,
+        "acp_db_files": snap.acp_db_files,
+        "acp_db_errors": snap.acp_db_errors,
+        "matched_sessions": snap.matched_sessions,
+        "orphan_dbs": snap.orphan_dbs,
+        "top_longest": top_sessions(snap, n=5, key="duration_ms"),
+        "top_costliest": top_sessions(snap, n=5, key="cost_usd"),
+        "models": by_model(snap),
+    }
+
+
+def by_project(snap: MetricsSnapshot) -> list[dict[str, Any]]:
+    groups: dict[str, list[SessionMetrics]] = {}
+    for s in snap.sessions:
+        groups.setdefault(s.working_directory, []).append(s)
+    rows = [
+        {
+            "project": wd,
+            "sessions": len(members),
+            "messages": _sum(s.n_messages for s in members),
+            "tool_calls": _sum(s.n_tool_calls for s in members),
+            "duration_ms": _sum(s.duration_ms for s in members),
+            "cost_usd": _sum_or_none(s.cost_usd for s in members),
+            "input_tokens": _sum_or_none(s.input_tokens for s in members),
+            "output_tokens": _sum_or_none(s.output_tokens for s in members),
+        }
+        for wd, members in groups.items()
+    ]
+    rows.sort(
+        key=lambda r: (r["cost_usd"] is not None, r["cost_usd"] or 0, r["sessions"]),
+        reverse=True,
+    )
+    return rows
+
+
+def by_model(snap: MetricsSnapshot) -> list[dict[str, Any]]:
+    """Per-model stats merging session-declared models and acp usage rows."""
+    rows: dict[str, dict[str, Any]] = {}
+
+    def _row(model: str | None) -> dict[str, Any]:
+        name = model or "(unknown)"
+        return rows.setdefault(
+            name,
+            {
+                "model": name,
+                "sessions": 0,
+                "usage_records": 0,
+                "cost_usd": None,
+                "input_tokens": None,
+                "output_tokens": None,
+            },
+        )
+
+    for s in snap.sessions:
+        _row(s.model)["sessions"] += 1
+    for u in snap.usage:
+        r = _row(u.model)
+        r["usage_records"] += 1
+        if u.cost_usd is not None:
+            r["cost_usd"] = (r["cost_usd"] or 0.0) + u.cost_usd
+        if u.input_tokens is not None:
+            r["input_tokens"] = (r["input_tokens"] or 0) + u.input_tokens
+        if u.output_tokens is not None:
+            r["output_tokens"] = (r["output_tokens"] or 0) + u.output_tokens
+    out = list(rows.values())
+    out.sort(
+        key=lambda r: (r["cost_usd"] is not None, r["cost_usd"] or 0, r["sessions"]),
+        reverse=True,
+    )
+    return out
+
+
+def by_day(snap: MetricsSnapshot, days: int | None = None) -> list[dict[str, Any]]:
+    """Sessions bucketed by UTC creation date.
+
+    ``days=N`` keeps the N most recent *activity* days (anchored on the latest
+    day in the data, not on wall-clock now — deterministic for fixtures and
+    for inspecting old installs alike).
+    """
+    groups: dict[str, list[SessionMetrics]] = {}
+    for s in snap.sessions:
+        groups.setdefault(_day(s.created_at), []).append(s)
+    dates = sorted(groups)
+    if days is not None:
+        dates = dates[-days:] if days > 0 else []
+    return [
+        {
+            "date": d,
+            "sessions": len(groups[d]),
+            "messages": _sum(s.n_messages for s in groups[d]),
+            "tool_calls": _sum(s.n_tool_calls for s in groups[d]),
+            "duration_ms": _sum(s.duration_ms for s in groups[d]),
+            "cost_usd": _sum_or_none(s.cost_usd for s in groups[d]),
+        }
+        for d in dates
+    ]
+
+
+def top_sessions(
+    snap: MetricsSnapshot, n: int = 5, key: str = "duration_ms"
+) -> list[dict[str, Any]]:
+    """Top-N sessions. ``key``: ``duration_ms`` or ``cost_usd``.
+
+    For ``cost_usd`` sessions with unknown cost are excluded entirely.
+    """
+    if key == "cost_usd":
+        pool = [s for s in snap.sessions if s.cost_usd is not None]
+        ordered = sorted(pool, key=lambda s: s.cost_usd or 0, reverse=True)
+    elif key == "duration_ms":
+        ordered = sorted(snap.sessions, key=lambda s: s.duration_ms, reverse=True)
+    else:
+        raise ValueError(f"top_sessions: unknown key {key!r}")
+    return [session_to_dict(s) for s in ordered[:n]]

devin_metrics/churn.py

@@ -0,0 +1,141 @@
+"""Rework (churn) metrics over a built ``graph.db`` (ME-4).
+
+Churn = rework: distinct tool calls in the same session that touched the
+same file again. Read straight from the knowledge graph's
+``file_touched`` edges (each edge is one tool_call → file, owned by a
+session), so this needs a ``devin-graph build`` first — the graph is the
+source of truth, this module never touches sessions.db.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+
+def _connect_ro(path: Path) -> sqlite3.Connection:
+    con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+    con.execute("PRAGMA query_only = ON")
+    return con
+
+
+def churn_report(graph_db: str | Path) -> dict[str, Any]:
+    """Per-session and per-model rework stats from a graph.db.
+
+    Returns dict with ``sessions`` (one row per session that reworked at
+    least one file), ``models`` (aggregated by the session node's model
+    attr) and ``top_files`` (most-reworked paths overall).
+    """
+    path = Path(graph_db).expanduser()
+    if not path.exists():
+        raise FileNotFoundError(
+            f"{path}: no graph — run 'devin-graph build --graph {path}'")
+    con = _connect_ro(path)
+    try:
+        _require_schema(con)
+        reworked = con.execute(
+            "SELECT e.owner, e.dst, COUNT(*) AS calls"
+            " FROM edges e WHERE e.kind = 'file_touched'"
+            " GROUP BY e.owner, e.dst HAVING calls > 1"
+        ).fetchall()
+        touched = con.execute(
+            "SELECT owner, COUNT(DISTINCT dst) FROM edges"
+            " WHERE kind = 'file_touched' GROUP BY owner"
+        ).fetchall()
+        models = {
+            r[0]: _model_of(r[1])
+            for r in con.execute(
+                "SELECT key, attrs FROM nodes WHERE kind = 'session'"
+            ).fetchall()
+        }
+        paths = {
+            r[0]: r[1]
+            for r in con.execute(
+                "SELECT id, key FROM nodes WHERE kind = 'file'"
+            ).fetchall()
+        }
+    finally:
+        con.close()
+
+    distinct_files = {owner: n for owner, n in touched}
+    per_session: dict[str, dict[str, Any]] = {}
+    file_calls: dict[str, int] = {}
+    for session_id, file_id, calls in reworked:
+        row = per_session.setdefault(session_id, {
+            "session_id": session_id,
+            "model": models.get(session_id),
+            "files_touched": distinct_files.get(session_id, 0),
+            "reworked_files": 0,
+            "repeat_calls": 0,
+        })
+        row["reworked_files"] += 1
+        row["repeat_calls"] += calls - 1
+        file_calls[file_id] = file_calls.get(file_id, 0) + calls - 1
+
+    sessions = sorted(
+        per_session.values(), key=lambda r: -r["repeat_calls"])
+
+    per_model: dict[str, dict[str, Any]] = {}
+    for row in sessions:
+        m = row["model"] or "(unknown)"
+        agg = per_model.setdefault(m, {
+            "model": m, "sessions_with_churn": 0,
+            "reworked_files": 0, "repeat_calls": 0})
+        agg["sessions_with_churn"] += 1
+        agg["reworked_files"] += row["reworked_files"]
+        agg["repeat_calls"] += row["repeat_calls"]
+
+    return {
+        "graph": str(path),
+        "sessions_with_churn": len(sessions),
+        "sessions": sessions,
+        "models": sorted(
+            per_model.values(), key=lambda r: -r["repeat_calls"]),
+        "top_files": [
+            {"file": paths.get(fid, fid), "repeat_calls": n}
+            for fid, n in sorted(
+                file_calls.items(), key=lambda kv: -kv[1])[:20]
+        ],
+    }
+
+
+def _require_schema(con: sqlite3.Connection) -> None:
+    tables = {r[0] for r in con.execute(
+        "SELECT name FROM sqlite_master WHERE type = 'table'")}
+    cols = {r[1] for r in con.execute("PRAGMA table_info(edges)")}
+    if not ({"nodes", "edges"} <= tables and {"kind", "src", "dst", "owner"}
+            <= cols):
+        raise ValueError(
+            f"{con}: not a devin-graph database — "
+            "expected nodes/edges(kind, src, dst, owner) from "
+            "'devin-graph build'")
+
+
+def _model_of(attrs_json: str) -> str | None:
+    try:
+        attrs = json.loads(attrs_json)
+    except (TypeError, ValueError):
+        return None
+    m = attrs.get("model")
+    return m if isinstance(m, str) and m else None
+
+
+def render_churn(report: dict[str, Any]) -> str:
+    """Human-readable churn table."""
+    lines = [
+        f"churn report — {report['graph']}",
+        f"sessions with rework: {report['sessions_with_churn']}",
+        "",
+        f"{'MODEL':<24} {'SESS':>5} {'REWORKED FILES':>15} {'REPEAT CALLS':>13}",
+    ]
+    for m in report["models"]:
+        lines.append(
+            f"{m['model']:<24} {m['sessions_with_churn']:>5} "
+            f"{m['reworked_files']:>15} {m['repeat_calls']:>13}")
+    if report["top_files"]:
+        lines += ["", "most reworked files:"]
+        for f in report["top_files"][:10]:
+            lines.append(f"  {f['repeat_calls']:>4}×  {f['file']}")
+    return "\n".join(lines)

devin_metrics/cli.py

@@ -0,0 +1,246 @@
+"""``devin-metrics`` — thin CLI wrapper; all logic lives in the library.
+
+Subcommands (read-only, no network, all accept ``--json``):
+
+- ``summary``   headline numbers + per-model table + top-5 longest sessions
+- ``projects``  per-project (``working_directory``) session/activity table
+- ``daily``     per-day activity; ``--days N`` keeps the N most recent days
+
+Store locations default to the platform Devin data dir (``paths.py``);
+``--data-dir`` overrides the root, ``--sessions-db``/``--acp-dir`` override
+individual stores. A missing ``acp-messages`` dir degrades gracefully — a
+warning on stderr and ``-`` in the cost columns. A missing ``sessions.db``
+is fatal (exit 1): it is the only source of session rows.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+import sys
+from pathlib import Path
+from typing import Sequence
+
+from devin_internals.schema import SchemaError
+
+from devin_metrics.aggregate import by_day, by_project, summarize
+from devin_metrics.churn import churn_report, render_churn
+from devin_metrics.collect import MetricsSnapshot, collect
+from devin_metrics.dashboard.collect import collect_stats
+from devin_metrics.dashboard.render import render_html
+from devin_metrics.paths import (
+    acp_messages_dir,
+    default_acp_messages_dir,
+    default_data_dir,
+    sessions_db_path,
+)
+from devin_metrics.render import (
+    dumps_json,
+    render_daily_md,
+    render_projects_md,
+    render_summary_md,
+)
+
+
+def _resolve(args: argparse.Namespace) -> tuple[Path, Path]:
+    root = Path(args.data_dir).expanduser() if args.data_dir else default_data_dir()
+    sessions_db = (
+        Path(args.sessions_db).expanduser()
+        if args.sessions_db
+        else sessions_db_path(root)
+    )
+    acp_dir = (
+        Path(args.acp_dir).expanduser()
+        if args.acp_dir is not None
+        else acp_messages_dir(root) if args.data_dir
+        else default_acp_messages_dir()
+    )
+    return sessions_db, acp_dir
+
+
+def _snapshot(args: argparse.Namespace) -> MetricsSnapshot:
+    sessions_db, acp_dir = _resolve(args)
+    if not acp_dir.is_dir():
+        print(
+            f"warning: {acp_dir}: no such directory — "
+            "cost/token columns will show '-'",
+            file=sys.stderr,
+        )
+    return collect(sessions_db, acp_dir)
+
+
+def cmd_summary(args: argparse.Namespace) -> int:
+    summary = summarize(_snapshot(args))
+    print(dumps_json(summary) if args.json else render_summary_md(summary))
+    return 0
+
+
+def cmd_projects(args: argparse.Namespace) -> int:
+    rows = by_project(_snapshot(args))
+    print(dumps_json(rows) if args.json else render_projects_md(rows))
+    return 0
+
+
+def cmd_daily(args: argparse.Namespace) -> int:
+    rows = by_day(_snapshot(args), days=args.days)
+    print(dumps_json(rows) if args.json else render_daily_md(rows))
+    return 0
+
+
+def cmd_watch(args: argparse.Namespace) -> int:
+    """ME-2: advisory context guard — warns, never blocks.
+
+    Cost is unverifiable in local stores (ME-3), so the guard watches
+    ``context_tokens`` (peak prompt size per session) — the only real
+    resource signal that persists. Thresholds are advisory: sessions that
+    crossed them are listed, exit stays 0 unless --fail.
+    """
+    snap = _snapshot(args)
+    findings: list[dict] = []
+    by_ctx = sorted(
+        (s for s in snap.sessions if s.context_tokens is not None),
+        key=lambda s: s.context_tokens or 0, reverse=True)
+    over_session = [s for s in by_ctx if s.context_tokens > args.session_warn]
+    days: dict[str, int] = {}
+    for s in snap.sessions:
+        if s.context_tokens:
+            from devin_metrics.aggregate import _day
+            d = _day(s.created_at)
+            days[d] = days.get(d, 0) + s.context_tokens
+    over_days = {d: v for d, v in days.items() if v > args.daily_warn}
+    for s in over_session:
+        findings.append({"kind": "session", "session_id": s.id,
+                         "context_tokens": s.context_tokens,
+                         "threshold": args.session_warn})
+    for d, v in sorted(over_days.items()):
+        findings.append({"kind": "day", "date": d, "context_tokens": v,
+                         "threshold": args.daily_warn})
+    report = {
+        "advisory": True,
+        "thresholds": {"session_warn": args.session_warn,
+                       "daily_warn": args.daily_warn},
+        "sessions_observed": len(by_ctx),
+        "findings": findings,
+        "note": "advisory only — context_tokens is peak prompt size, "
+                "not cost (local stores have no cost data, verified ME-3)",
+    }
+    if args.json:
+        print(json.dumps(report, indent=2))
+    else:
+        print(f"watch (advisory): {len(by_ctx)} sessions observed · "
+              f"{len(findings)} finding(s)")
+        for f in findings:
+            if f["kind"] == "session":
+                print(f"  session {f['session_id'][:16]}… "
+                      f"{f['context_tokens']:,} tok > {f['threshold']:,}")
+            else:
+                print(f"  day {f['date']}  {f['context_tokens']:,} tok "
+                      f"> {f['threshold']:,}")
+        if not findings:
+            print("  all clear")
+    return 1 if (findings and args.fail) else 0
+
+
+def cmd_churn(args: argparse.Namespace) -> int:
+    report = churn_report(args.graph)
+    print(json.dumps(report, indent=2) if args.json
+          else render_churn(report))
+    return 0
+
+
+def cmd_dashboard(args: argparse.Namespace) -> int:
+    sessions_db, acp_dir = _resolve(args)
+    if not acp_dir.is_dir():
+        print(
+            f"warning: {acp_dir}: no such directory — "
+            "cost/token fields will be empty",
+            file=sys.stderr,
+        )
+    stats = collect_stats(sessions_db, acp_dir)
+    if args.json:
+        import json
+
+        print(json.dumps(stats, indent=2))
+        return 0
+    out = Path(args.out).expanduser()
+    out.write_text(render_html(stats), encoding="utf-8")
+    print(f"wrote {out} ({out.stat().st_size:,} bytes) — open it in a browser")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="devin-metrics",
+        description="Local-only metrics from Devin's session stores.",
+    )
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument(
+        "--data-dir", metavar="DIR",
+        help="Devin data dir (default: platform-specific, see docs/SPEC.md)",
+    )
+    common.add_argument(
+        "--sessions-db", metavar="DB",
+        help="path to sessions.db (default: <data-dir>/cli/sessions.db)",
+    )
+    common.add_argument(
+        "--acp-dir", metavar="DIR",
+        help="path to acp-messages dir (default: <data-dir>/User/acp-messages)",
+    )
+    common.add_argument("--json", action="store_true", help="JSON output")
+
+    sub = parser.add_subparsers(dest="command", required=True)
+    sub.add_parser("summary", parents=[common],
+                   help="headline numbers").set_defaults(func=cmd_summary)
+    sub.add_parser("projects", parents=[common],
+                   help="per-project session/activity table").set_defaults(func=cmd_projects)
+    p_daily = sub.add_parser("daily", parents=[common],
+                             help="activity over time")
+    p_daily.add_argument(
+        "--days", type=int, default=None, metavar="N",
+        help="keep the N most recent activity days (default: all)",
+    )
+    p_daily.set_defaults(func=cmd_daily)
+    p_dash = sub.add_parser("dashboard", parents=[common],
+                            help="render the static HTML dashboard")
+    p_dash.add_argument(
+        "--out", metavar="FILE", default="index.html",
+        help="output file (default: index.html)",
+    )
+    p_dash.set_defaults(func=cmd_dashboard)
+    p_churn = sub.add_parser(
+        "churn", help="rework stats from a graph.db (devin-graph build first)")
+    p_churn.add_argument("--graph", metavar="DB", default="graph.db",
+                         help="path to a built graph.db (default ./graph.db)")
+    p_churn.add_argument("--json", action="store_true")
+    p_churn.set_defaults(func=cmd_churn)
+    p_watch = sub.add_parser(
+        "watch", parents=[common],
+        help="advisory context guard — lists sessions/days over a token "
+             "threshold; never blocks (ME-2)")
+    p_watch.add_argument("--session-warn", type=int, default=400_000,
+                         metavar="N", help="flag sessions with peak context "
+                         "> N tokens (default: %(default)s)")
+    p_watch.add_argument("--daily-warn", type=int, default=2_000_000,
+                         metavar="N", help="flag days whose summed context "
+                         "> N tokens (default: %(default)s)")
+    p_watch.add_argument("--fail", action="store_true",
+                         help="exit 1 when findings exist (CI mode)")
+    p_watch.set_defaults(func=cmd_watch)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except SchemaError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    except (sqlite3.Error, OSError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())