devin-graph 0.1.0__py3-none-any.whl

devin_graph/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

devin_graph/cli.py

@@ -0,0 +1,297 @@
+"""Thin CLI wrapper — all logic lives in the library modules."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from devin_internals import SchemaError
+from devin_internals.parsers import SessionsStore
+
+from devin_graph.paths import default_state_vscdb
+from devin_graph.vscdb import extract_vscdb
+
+from devin_graph import __version__
+from devin_graph.paths import default_sessions_db
+from devin_graph.query import (
+    project_detail,
+    projects_graph,
+    sessions_for_file,
+    sessions_for_tool,
+    shared_files,
+    tools_for_project,
+)
+from devin_graph.store import GraphStore
+
+DEFAULT_GRAPH = "graph.db"
+
+
+def _open_sessions(db_arg: str | None) -> SessionsStore:
+    path = Path(db_arg).expanduser() if db_arg else default_sessions_db()
+    if path is None:
+        print(
+            "devin-graph: no sessions.db found in the default locations; "
+            "pass --sessions-db",
+            file=sys.stderr,
+        )
+        raise SystemExit(2)
+    if not path.exists():
+        print(f"devin-graph: {path}: no such file", file=sys.stderr)
+        raise SystemExit(2)
+    try:
+        return SessionsStore(path)
+    except SchemaError as exc:
+        print(f"devin-graph: {exc}", file=sys.stderr)
+        raise SystemExit(2)
+
+
+def _open_graph(graph_arg: str | None) -> GraphStore:
+    path = Path(graph_arg or DEFAULT_GRAPH).expanduser()
+    if not path.exists():
+        print(
+            f"devin-graph: {path}: no such graph file — build one with "
+            "'devin-graph build --graph graph.db'",
+            file=sys.stderr,
+        )
+        raise SystemExit(2)
+    return GraphStore(path)
+
+
+def _add_graph(sub: argparse.ArgumentParser) -> None:
+    sub.add_argument(
+        "--graph", default=DEFAULT_GRAPH,
+        help="path to the graph database (default: ./graph.db)")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="devin-graph",
+        description="Knowledge graph over Devin sessions: projects, files "
+        "and tools as queryable nodes (read-only on Devin stores).",
+    )
+    p.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    b = sub.add_parser("build", help="extract sessions.db → graph.db "
+                       "(incremental)")
+    b.add_argument(
+        "--sessions-db",
+        help="path to Devin's sessions.db (default: auto-detect "
+        "%%APPDATA%%/devin/cli/sessions.db or the Linux/macOS equivalent)")
+    _add_graph(b)
+    b.add_argument(
+        "--vscdb",
+        help="path to the GUI state.vscdb for gui_session coverage "
+        "(default: auto-detect; pass 'none' to disable)")
+    b.add_argument("--json", action="store_true",
+                   help="machine-readable JSON output")
+    b.set_defaults(func=_cmd_build)
+
+    q = sub.add_parser("query", help="canned queries over the graph")
+    qsub = q.add_subparsers(dest="what", required=True)
+    for name, helptext, argname in (
+        ("file", "which sessions touched a file", "path"),
+        ("tool", "which sessions/projects used a tool", "name"),
+        ("project", "sessions/tools/files of a project", "name"),
+    ):
+        sq = qsub.add_parser(name, help=helptext)
+        sq.add_argument(argname)
+        _add_graph(sq)
+        sq.add_argument("--json", action="store_true")
+        sq.set_defaults(func=_cmd_query)
+    sq = qsub.add_parser("projects-graph",
+                         help="project adjacency JSON for visualization")
+    _add_graph(sq)
+    sq.add_argument("--json", action="store_true")
+    sq.set_defaults(func=_cmd_query)
+    sq = qsub.add_parser("shared-files",
+                         help="files touched by two or more projects")
+    _add_graph(sq)
+    sq.add_argument("--json", action="store_true")
+    sq.set_defaults(func=_cmd_query)
+
+    e = sub.add_parser("export", help="dump nodes+edges (D3-friendly)")
+    e.add_argument("--format", choices=["json"], default="json")
+    _add_graph(e)
+    e.add_argument("--out", help="write to file instead of stdout")
+    e.set_defaults(func=_cmd_export)
+
+    sq2 = sub.add_parser(
+        "sql", help="read-only SQL over graph.db (SELECT/WITH only)")
+    sq2.add_argument("statement", help="a SELECT or WITH ... SELECT statement")
+    _add_graph(sq2)
+    sq2.add_argument("--json", action="store_true")
+    sq2.set_defaults(func=_cmd_sql)
+    v = sub.add_parser(
+        "view", help="self-contained HTML viewer (offline, no CDN)")
+    v.add_argument("--out", default="graph-view.html",
+                   help="output file (default: graph-view.html)")
+    v.add_argument("--limit", type=int, default=500,
+                   help="max nodes rendered (default 500; highest-degree "
+                   "nodes are kept when truncating)")
+    _add_graph(v)
+    v.set_defaults(func=_cmd_view)
+    return p
+
+
+def _cmd_build(args: argparse.Namespace) -> int:
+    graph_path = Path(args.graph).expanduser()
+    extras = []
+    vscdb = args.vscdb
+    if vscdb is None:
+        found = default_state_vscdb()
+        vscdb = str(found) if found else "none"
+    if vscdb != "none":
+        vpath = Path(vscdb).expanduser()
+        if vpath.is_file():
+            try:
+                extras.append(("vscdb", extract_vscdb(vpath)))
+            except ValueError as exc:
+                print(f"warning: {exc}", file=sys.stderr)
+        else:
+            print(f"warning: {vpath}: no such file — skipping GUI coverage",
+                  file=sys.stderr)
+    with _open_sessions(args.sessions_db) as ss, \
+            GraphStore(graph_path) as gs:
+        res = gs.build(ss, extras=extras)
+        nodes = len(gs.nodes())
+        edges = len(gs.edges())
+    if args.json:
+        print(json.dumps({
+            "graph": str(graph_path),
+            "extracted": len(res.extracted),
+            "skipped": res.skipped,
+            "removed": len(res.removed),
+            "nodes": nodes,
+            "edges": edges,
+        }, indent=2))
+    else:
+        print(
+            f"extracted: {len(res.extracted)} · skipped: {res.skipped} · "
+            f"removed: {len(res.removed)} · nodes: {nodes} · edges: {edges} "
+            f"→ {graph_path}")
+    return 0
+
+
+def _cmd_query(args: argparse.Namespace) -> int:
+    with _open_graph(args.graph) as gs:
+        if args.what == "file":
+            data = sessions_for_file(gs, args.path)
+        elif args.what == "tool":
+            data = sessions_for_tool(gs, args.name)
+        elif args.what == "project":
+            data = project_detail(gs, args.name)
+        elif args.what == "shared-files":
+            data = shared_files(gs)
+        else:
+            data = projects_graph(gs)
+    if args.json:
+        print(json.dumps(data, ensure_ascii=False, indent=2))
+    else:
+        _print_query(args.what, data)
+    return 0
+
+
+def _print_query(what: str, data: dict) -> None:
+    if what == "shared-files":
+        if not data["files"]:
+            print("no files shared between projects")
+        for f in data["files"]:
+            print(f"{f['file']}  ({len(f['projects'])} projects)")
+            print(f"  projects: {', '.join(f['projects'])}")
+            print(f"  sessions: {', '.join(f['sessions'])}")
+        return
+    if what == "projects-graph":
+        for n in data["nodes"]:
+            print(f"{n['id']}  ({len(n['sessions'])} sessions)")
+        for link in data["links"]:
+            print(f"{link['source']} ↔ {link['target']}  "
+                  f"weight={link['weight']}")
+        return
+    if "files" in data and what == "file":
+        for f in data["files"]:
+            print(f"file: {f}")
+    for s in data.get("sessions", []):
+        print(f"{s['id']}  {s.get('title') or 'Untitled'}  "
+              f"({s.get('project') or '-'})")
+    if what == "tool" and data.get("projects"):
+        print("projects: " + ", ".join(data["projects"]))
+    if what == "project":
+        if data.get("tools"):
+            print("tools: " + ", ".join(data["tools"]))
+        for f in data.get("files", []):
+            print(f"file: {f}")
+    if what == "file":
+        for t in data.get("tool_calls", []):
+            print(f"call: {t['id']} → {t['file']}")
+
+
+def _cmd_export(args: argparse.Namespace) -> int:
+    with _open_graph(args.graph) as gs:
+        data = gs.export()
+    text = json.dumps(data, ensure_ascii=False, indent=2)
+    if args.out:
+        Path(args.out).expanduser().write_text(text + "\n", encoding="utf-8")
+        print(f"exported {len(data['nodes'])} nodes, {len(data['edges'])} "
+              f"edges → {args.out}")
+    else:
+        print(text)
+    return 0
+
+
+def _cmd_view(args: argparse.Namespace) -> int:
+    from devin_graph.view import write_view
+    with _open_graph(args.graph) as gs:
+        out = write_view(gs.export(), args.out, limit=args.limit)
+    print(f"wrote {out} ({out.stat().st_size:,} bytes) — open it in a browser")
+    return 0
+
+
+def _cmd_sql(args: argparse.Namespace) -> int:
+    stmt = args.statement.strip().rstrip(";")
+    head = stmt.split(None, 1)[0].lower() if stmt else ""
+    if head not in ("select", "with"):
+        print("devin-graph: sql accepts SELECT/WITH only", file=sys.stderr)
+        return 2
+    import sqlite3
+
+    path = Path(args.graph).expanduser()
+    if not path.exists():
+        print(f"devin-graph: {path}: no such graph file", file=sys.stderr)
+        return 2
+    con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+    try:
+        con.execute("PRAGMA query_only = ON")
+        cur = con.execute(stmt)
+        cols = [d[0] for d in cur.description or []]
+        rows = cur.fetchall()
+    except sqlite3.Error as exc:
+        print(f"devin-graph: {exc}", file=sys.stderr)
+        return 2
+    finally:
+        con.close()
+    if args.json:
+        print(json.dumps(
+            [dict(zip(cols, r)) for r in rows], ensure_ascii=False, indent=2))
+    else:
+        print("  ".join(cols))
+        for r in rows:
+            print("  ".join("" if v is None else str(v) for v in r))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except (AttributeError, ValueError):
+            pass
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

devin_graph/extract.py

@@ -0,0 +1,300 @@
+"""Extract graph nodes/edges from a ``sessions.db`` (via ``SessionsStore``).
+
+Node kinds: ``session``, ``project`` (the session's ``working_directory``),
+``file`` (a path touched by a tool call), ``tool`` (tool name),
+``tool_call`` (one row of ``tool_call_state`` — the ground truth of what the
+agent actually invoked) and ``commit`` (a full git SHA referenced by a
+tool-call payload, via ``devin_internals.commits``).
+
+Edge kinds:
+
+- ``runs_in``       session → project
+- ``made_call``     session → tool_call
+- ``call_used``     tool_call → tool
+- ``tool_used``     session → tool
+- ``file_touched``  tool_call → file
+- ``produced``      session → commit (SHA seen in a ``git commit``/``git
+  push`` tool call — ``method="exact"``)
+- ``referenced``    session → commit (SHA seen elsewhere — ``method="seen"``)
+- ``gui_workspace`` gui_session → project (from ``state.vscdb`` — see
+  ``vscdb.py``; GUI sessions are keyed by their generated slug)
+
+``tool_call_json`` is marked *unstable* in SCHEMA.md, so payload decoding is
+deliberately defensive: file paths are collected from any recognised path-ish
+key anywhere in the payload, plus path-looking tokens inside command strings.
+Relative paths resolve against the session's ``working_directory``.
+"""
+
+from __future__ import annotations
+
+import json
+import posixpath
+import re
+from dataclasses import dataclass, field
+
+from devin_internals.commits import commit_references
+from typing import Any, Iterable, Iterator
+
+NODE_KINDS = ("session", "project", "file", "tool", "tool_call", "commit",
+              "gui_session")
+EDGE_KINDS = ("runs_in", "made_call", "call_used", "tool_used",
+              "file_touched", "produced", "referenced", "gui_workspace")
+
+# Payload keys whose string value(s) are file/dir paths.
+_PATH_KEYS = {
+    "path", "paths", "file", "files", "filename", "file_path", "filepath",
+    "fileName".lower(), "abs_path", "absolute_path", "target_file",
+}
+
+# Payload keys whose string value is a shell-like command to mine for paths.
+_COMMAND_KEYS = {"command", "cmd", "script", "shell_command", "input"}
+
+# Tool-name candidate keys, in priority order, checked at top level and one
+# level deep (rawInput/raw_input/input carry the real tool input in ACP).
+_TOOL_NAME_KEYS = ("name", "tool_name", "tool", "kind", "type", "title")
+_NESTED_KEYS = ("rawInput", "raw_input", "input", "args")
+
+# Path-looking token inside a command string: either contains a separator or
+# is a bare filename with a recognisable code/docs extension.
+_PATH_TOKEN_RE = re.compile(
+    r"(?:[A-Za-z]:[\\/]|\.{1,2}/|/)?[\w@+~.-]+(?:[\\/][\w@+~.-]+)+"
+    r"|[\w@+~.-]+\.(?:py|pyi|js|jsx|ts|tsx|mjs|cjs|json|toml|ya?ml|md|txt|rst"
+    r"|rs|go|c|h|cc|cpp|hpp|java|kt|rb|sh|bash|bat|ps1|sql|html?|css|scss|xml"
+    r"|csv|cfg|ini|env|lock|ipynb)\b"
+)
+_TOKEN_STRIP = "\"'`.,;:()[]{}"
+
+
+@dataclass(frozen=True)
+class Node:
+    kind: str
+    key: str
+    attrs: dict = field(default_factory=dict, compare=False)
+
+
+@dataclass(frozen=True)
+class Edge:
+    kind: str
+    src: tuple[str, str]  # (node kind, node key)
+    dst: tuple[str, str]
+    attrs: dict = field(default_factory=dict, compare=False)
+
+
+@dataclass
+class Extraction:
+    """Nodes/edges produced from one or more sessions (deduplicated)."""
+
+    nodes: list[Node] = field(default_factory=list)
+    edges: list[Edge] = field(default_factory=list)
+
+    def merged(self) -> "Extraction":
+        seen_n: dict[tuple[str, str], Node] = {}
+        seen_e: set[tuple] = set()
+        out = Extraction()
+        for n in self.nodes:
+            seen_n.setdefault((n.kind, n.key), n)
+        out.nodes = list(seen_n.values())
+        for e in self.edges:
+            k = (e.kind, e.src, e.dst)
+            if k not in seen_e:
+                seen_e.add(k)
+                out.edges.append(e)
+        return out
+
+
+def normalize_path(raw: Any) -> str | None:
+    """Canonical path string: ``/`` separators, no ``.`` segments, no quotes.
+
+    Returns ``None`` for empty/non-string input.
+    """
+    if not isinstance(raw, str):
+        return None
+    p = raw.strip().strip("\"'").strip()
+    if not p:
+        return None
+    p = p.replace("\\", "/")
+    is_abs = p.startswith("/") or re.match(r"^[A-Za-z]:/", p) is not None
+    p = posixpath.normpath(p)
+    if p == ".":
+        return None
+    if not is_abs and p.startswith("/"):
+        p = p.lstrip("/")
+    return p or None
+
+
+def _is_absolute(p: str) -> bool:
+    return p.startswith("/") or re.match(r"^[A-Za-z]:/", p) is not None
+
+
+def _resolve(path: str, cwd: str | None) -> str:
+    """Anchor relative paths to the session's working directory."""
+    if _is_absolute(path) or not cwd:
+        return path
+    base = normalize_path(cwd) or ""
+    return posixpath.normpath(f"{base}/{path}") if base else path
+
+
+def parse_payload(text: str | None) -> dict | None:
+    """Decode a ``tool_call_*_json`` payload; ``None`` on anything unusable."""
+    if not text:
+        return None
+    try:
+        data = json.loads(text)
+    except (TypeError, ValueError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def _walk_strings(node: Any) -> Iterator[str]:
+    if isinstance(node, dict):
+        for v in node.values():
+            yield from _walk_strings(v)
+    elif isinstance(node, list):
+        for v in node:
+            yield from _walk_strings(v)
+    elif isinstance(node, str):
+        yield node
+
+
+def _iter_keyed(node: Any) -> Iterator[tuple[str, Any]]:
+    """Yield ``(lowercased_key, value)`` for every dict entry, recursively."""
+    if isinstance(node, dict):
+        for k, v in node.items():
+            yield str(k).lower(), v
+            yield from _iter_keyed(v)
+    elif isinstance(node, list):
+        for v in node:
+            yield from _iter_keyed(v)
+
+
+def extract_tool_name(payload: dict) -> str | None:
+    """Best-effort tool name from an unstable payload shape."""
+    for key in _TOOL_NAME_KEYS:
+        v = payload.get(key)
+        if isinstance(v, str) and v.strip():
+            return v.strip()
+    for nest in _NESTED_KEYS:
+        sub = payload.get(nest)
+        if isinstance(sub, dict):
+            for key in _TOOL_NAME_KEYS:
+                v = sub.get(key)
+                if isinstance(v, str) and v.strip():
+                    return v.strip()
+    return None
+
+
+def _paths_from_command(text: str) -> set[str]:
+    found = set()
+    for m in _PATH_TOKEN_RE.finditer(text):
+        tok = m.group(0).strip(_TOKEN_STRIP)
+        preceded_by_scheme = re.search(
+            r"[A-Za-z][\w+.-]*:/+$", text[: m.start()]) is not None
+        if (not tok or "://" in tok or tok.startswith("//")
+                or preceded_by_scheme):
+            continue
+        if tok.lstrip("./").startswith("-"):  # a flag, not a path
+            continue
+        norm = normalize_path(tok)
+        if norm:
+            found.add(norm)
+    return found
+
+
+def extract_paths(payload: dict) -> set[str]:
+    """All file/dir paths referenced by a tool-call payload (normalized)."""
+    found: set[str] = set()
+    for key, value in _iter_keyed(payload):
+        if key in _PATH_KEYS:
+            if isinstance(value, str):
+                norm = normalize_path(value)
+                if norm and "://" not in norm:
+                    found.add(norm)
+            elif isinstance(value, (list, dict)):
+                for s in _walk_strings(value):
+                    norm = normalize_path(s)
+                    if norm and "://" not in norm:
+                        found.add(norm)
+        elif key in _COMMAND_KEYS and isinstance(value, str):
+            found |= _paths_from_command(value)
+    return found
+
+
+def extract_session(session, tool_calls: Iterable) -> Extraction:
+    """One session → nodes + edges.
+
+    ``session`` is a ``devin_internals.parsers.Session``; ``tool_calls`` are
+    its ``ToolCallState`` rows.
+    """
+    tool_calls = list(tool_calls)
+    ex = Extraction()
+    sid = session.id
+    cwd = normalize_path(session.working_directory)
+
+    ex.nodes.append(Node("session", sid, {
+        "title": session.title,
+        "model": session.model,
+        "created_at": session.created_at,
+        "last_activity_at": session.last_activity_at,
+        "hidden": session.hidden,
+    }))
+    if cwd:
+        ex.nodes.append(Node("project", cwd, {"name": posixpath.basename(cwd)}))
+        ex.edges.append(Edge("runs_in", ("session", sid), ("project", cwd)))
+
+    for tc in tool_calls:
+        call_key = f"{sid}:{tc.tool_call_id}"
+        ex.nodes.append(Node("tool_call", call_key, {
+            "tool_call_id": tc.tool_call_id,
+            "session_id": sid,
+        }))
+        ex.edges.append(
+            Edge("made_call", ("session", sid), ("tool_call", call_key)))
+
+        # Merge update then call payloads (call wins); tolerate NULL/junk.
+        payload = {}
+        for raw in (tc.tool_call_update_json, tc.tool_call_json):
+            data = parse_payload(raw)
+            if data:
+                payload = {**payload, **data}
+
+        tool = extract_tool_name(payload) if payload else None
+        if tool:
+            ex.nodes.append(Node("tool", tool, {}))
+            ex.edges.append(
+                Edge("call_used", ("tool_call", call_key), ("tool", tool)))
+            ex.edges.append(
+                Edge("tool_used", ("session", sid), ("tool", tool)))
+
+        for path in sorted(extract_paths(payload)):
+            resolved = _resolve(path, cwd)
+            ex.nodes.append(
+                Node("file", resolved, {"name": posixpath.basename(resolved)}))
+            ex.edges.append(Edge(
+                "file_touched", ("tool_call", call_key), ("file", resolved)))
+
+    # commit attribution: "produced" only when the SHA shows in a git
+    # commit/push tool call (exact); any other mention is "referenced".
+    for ref in commit_references(tool_calls):
+        text = " ".join(
+            p or "" for r in tool_calls if r.tool_call_id == ref.tool_call_id
+            for p in (r.tool_call_json, r.tool_call_update_json))
+        produced = "git commit" in text or "git push" in text
+        ex.nodes.append(Node("commit", ref.sha, {
+            "repo_url": ref.repo_url, "short": ref.sha[:7]}))
+        ex.edges.append(Edge(
+            "produced" if produced else "referenced",
+            ("session", sid), ("commit", ref.sha),
+            {"method": "exact" if produced else "seen",
+             "tool_call_id": ref.tool_call_id}))
+
+    return ex
+
+
+def extract_all(store) -> Extraction:
+    """Every session in ``store`` → merged, deduplicated graph."""
+    combined = Extraction()
+    for session in store.sessions():
+        part = extract_session(session, store.tool_call_state(session.id))
+        combined.nodes.extend(part.nodes)
+        combined.edges.extend(part.edges)
+    return combined.merged()