devin-graph 0.1.0__py3-none-any.whl

devin_graph/store.py

@@ -0,0 +1,273 @@
+"""Persistent knowledge-graph store: a local SQLite ``graph.db``.
+
+Layout: ``nodes`` / ``edges`` keyed on composite ids (``"kind:key"``), an
+``extracted_sessions`` ledger for incremental re-extract, and a ``meta`` table
+recording which source DB the graph was built from.
+
+Incremental rule: a session is re-extracted only when its
+``last_activity_at`` marker changed (or it is new); sessions deleted from the
+source are removed; shared nodes (project/file/tool) left with no incident
+edges are pruned. Nothing is ever written to the source store — the caller
+hands in a read-only ``SessionsStore``.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from devin_graph import identity
+from devin_graph.extract import Extraction, extract_session
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS nodes (
+  id    TEXT PRIMARY KEY,
+  kind  TEXT NOT NULL,
+  key   TEXT NOT NULL,
+  owner TEXT,
+  attrs TEXT NOT NULL DEFAULT '{}'
+);
+CREATE INDEX IF NOT EXISTS idx_nodes_kind ON nodes(kind);
+CREATE TABLE IF NOT EXISTS edges (
+  kind  TEXT NOT NULL,
+  src   TEXT NOT NULL,
+  dst   TEXT NOT NULL,
+  owner TEXT,
+  attrs TEXT NOT NULL DEFAULT '{}',
+  PRIMARY KEY (kind, src, dst)
+);
+CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src);
+CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst);
+CREATE TABLE IF NOT EXISTS extracted_sessions (
+  session_id       TEXT PRIMARY KEY,
+  last_activity_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS meta (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+"""
+
+# Node kinds owned by a single session (safe to delete on re-extract).
+_OWNED_KINDS = ("session", "tool_call")
+
+
+@dataclass(frozen=True)
+class StoredNode:
+    id: str  # "kind:key"
+    kind: str
+    key: str
+    attrs: dict
+
+
+@dataclass(frozen=True)
+class StoredEdge:
+    kind: str
+    src: str  # "kind:key"
+    dst: str
+    attrs: dict
+    owner: str | None = None  # session_id that produced the edge
+
+
+@dataclass
+class BuildResult:
+    extracted: list[str] = field(default_factory=list)
+    removed: list[str] = field(default_factory=list)
+    skipped: int = 0
+
+
+def _node_id(kind: str, key: str) -> str:
+    return f"{kind}:{key}"
+
+
+class GraphStore:
+    """Read/write handle on a ``graph.db`` file (created on demand)."""
+
+    def __init__(self, path: str | Path) -> None:
+        self.path = Path(path).expanduser()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._con = sqlite3.connect(self.path)
+        self._con.row_factory = sqlite3.Row
+        self._con.executescript(_SCHEMA)
+
+    # -- lifecycle ---------------------------------------------------------
+
+    def close(self) -> None:
+        self._con.close()
+
+    def __enter__(self) -> "GraphStore":
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    # -- read API ------------------------------------------------------------
+
+    def nodes(self, kind: str | None = None) -> list[StoredNode]:
+        sql = "SELECT id, kind, key, attrs FROM nodes"
+        args: tuple = ()
+        if kind is not None:
+            sql += " WHERE kind = ?"
+            args = (kind,)
+        return [
+            StoredNode(r["id"], r["kind"], r["key"], json.loads(r["attrs"]))
+            for r in self._con.execute(sql + " ORDER BY id", args)
+        ]
+
+    def edges(self, kind: str | None = None) -> list[StoredEdge]:
+        sql = "SELECT kind, src, dst, owner, attrs FROM edges"
+        args: tuple = ()
+        if kind is not None:
+            sql += " WHERE kind = ?"
+            args = (kind,)
+        return [
+            StoredEdge(r["kind"], r["src"], r["dst"],
+                       json.loads(r["attrs"]), r["owner"])
+            for r in self._con.execute(sql + " ORDER BY kind, src, dst", args)
+        ]
+
+    def node(self, node_id: str) -> StoredNode | None:
+        row = self._con.execute(
+            "SELECT id, kind, key, attrs FROM nodes WHERE id = ?",
+            (node_id,),
+        ).fetchone()
+        if row is None:
+            return None
+        return StoredNode(row["id"], row["kind"], row["key"],
+                          json.loads(row["attrs"]))
+
+    def out_edges(self, node_id: str) -> list[StoredEdge]:
+        return [
+            StoredEdge(r["kind"], r["src"], r["dst"],
+                       json.loads(r["attrs"]), r["owner"])
+            for r in self._con.execute(
+                "SELECT kind, src, dst, owner, attrs FROM edges WHERE src = ?"
+                " ORDER BY kind, dst",
+                (node_id,),
+            )
+        ]
+
+    def in_edges(self, node_id: str) -> list[StoredEdge]:
+        return [
+            StoredEdge(r["kind"], r["src"], r["dst"],
+                       json.loads(r["attrs"]), r["owner"])
+            for r in self._con.execute(
+                "SELECT kind, src, dst, owner, attrs FROM edges WHERE dst = ?"
+                " ORDER BY kind, src",
+                (node_id,),
+            )
+        ]
+
+    # -- build ---------------------------------------------------------------
+
+    def build(self, sessions_store,
+              extras: list[tuple[str, "Extraction"]] | None = None
+              ) -> BuildResult:
+        """Incrementally (re)extract every session in ``sessions_store``.
+
+        ``extras`` are ``(owner, Extraction)`` pairs merged under their own
+        owner key (e.g. ``"vscdb"`` for GUI coverage) so rebuilds replace
+        them atomically without touching session-owned rows.
+        """
+        result = BuildResult()
+        seen = {
+            r["session_id"]: r["last_activity_at"]
+            for r in self._con.execute(
+                "SELECT session_id, last_activity_at FROM extracted_sessions")
+        }
+        current_ids = set()
+        sessions = sessions_store.sessions()
+        with self._con:
+            for s in sessions:
+                current_ids.add(s.id)
+                if seen.get(s.id) == s.last_activity_at:
+                    result.skipped += 1
+                    continue
+                self._delete_owned(s.id)
+                self._insert(extract_session(
+                    s, sessions_store.tool_call_state(s.id)), owner=s.id)
+                self._con.execute(
+                    "INSERT OR REPLACE INTO extracted_sessions"
+                    " (session_id, last_activity_at) VALUES (?, ?)",
+                    (s.id, s.last_activity_at),
+                )
+                result.extracted.append(s.id)
+
+            for sid in sorted(set(seen) - current_ids):
+                self._delete_owned(sid)
+                self._con.execute(
+                    "DELETE FROM extracted_sessions WHERE session_id = ?",
+                    (sid,))
+                result.removed.append(sid)
+
+            for owner, ex in extras or []:
+                self._delete_owned(owner)
+                self._insert(ex, owner=owner)
+            self._prune_orphans()
+            prov = identity.provenance()
+            for key, value in (
+                ("source_db", str(sessions_store.path)),
+                ("schema_version",
+                 str(sessions_store.schema_info["schema_version"])),
+                ("machine_id", prov["machine_id"]),
+                ("profile", prov["profile"]),
+            ):
+                self._con.execute(
+                    "INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
+                    (key, value))
+        result.extracted.sort()
+        return result
+
+    def _delete_owned(self, session_id: str) -> None:
+        self._con.execute("DELETE FROM edges WHERE owner = ?", (session_id,))
+        self._con.execute("DELETE FROM nodes WHERE owner = ?", (session_id,))
+
+    def _insert(self, ex: Extraction, owner: str) -> None:
+        for n in ex.merged().nodes:
+            node_owner = owner if n.kind in _OWNED_KINDS else None
+            self._con.execute(
+                "INSERT OR REPLACE INTO nodes(id, kind, key, owner, attrs)"
+                " VALUES (?, ?, ?, ?, ?)",
+                (_node_id(n.kind, n.key), n.kind, n.key, node_owner,
+                 json.dumps(n.attrs, sort_keys=True)),
+            )
+        for e in ex.edges:
+            self._con.execute(
+                "INSERT OR REPLACE INTO edges(kind, src, dst, owner, attrs)"
+                " VALUES (?, ?, ?, ?, ?)",
+                (e.kind, _node_id(*e.src), _node_id(*e.dst), owner,
+                 json.dumps(e.attrs, sort_keys=True)),
+            )
+
+    def _prune_orphans(self) -> None:
+        """Drop shared nodes (no owner) left with no incident edges."""
+        self._con.execute(
+            "DELETE FROM nodes WHERE owner IS NULL"
+            " AND id NOT IN (SELECT src FROM edges)"
+            " AND id NOT IN (SELECT dst FROM edges)"
+        )
+
+    # -- export --------------------------------------------------------------
+
+    def export(self) -> dict:
+        """Deterministic D3-friendly dump: ``{meta, nodes, edges}``."""
+        meta = {
+            r["key"]: r["value"]
+            for r in self._con.execute("SELECT key, value FROM meta")
+        }
+        schema_version = meta.get("schema_version")
+        if schema_version is not None:
+            meta["schema_version"] = int(schema_version)
+        return {
+            "meta": meta,
+            "nodes": [
+                {"id": n.id, "kind": n.kind, "key": n.key, **n.attrs}
+                for n in self.nodes()
+            ],
+            "edges": [
+                {"kind": e.kind, "source": e.src, "target": e.dst, **e.attrs}
+                for e in self.edges()
+            ],
+        }

devin_graph/view.py

@@ -0,0 +1,144 @@
+"""GR-2: self-contained HTML viewer for graph.db.
+
+``devin-graph view`` renders the graph (or a filtered slice) into a single
+HTML file: embedded JSON + a small vanilla-JS force layout on SVG. No CDN,
+no external assets — it works offline by design, which is the whole point
+of the ecosystem. Large graphs are downsampled: ``--limit`` caps nodes and
+the viewer warns when it truncated.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import random
+from pathlib import Path
+from typing import Any
+
+_KIND_COLORS = {
+    "session": "#4f8ef7",
+    "project": "#f7a94f",
+    "file": "#5cb870",
+    "tool": "#b06ef7",
+    "tool_call": "#7a8599",
+    "commit": "#f75f5f",
+    "gui_session": "#4fc3c3",
+}
+
+_PAGE = """<!DOCTYPE html>
+<html lang="en"><head><meta charset="utf-8">
+<title>devin-graph view</title>
+<style>
+ body{{font-family:system-ui,sans-serif;margin:0;background:#0e1116;color:#dde3ee}}
+ header{{padding:10px 16px;border-bottom:1px solid #2a3242;font-size:13px}}
+ #stats{{color:#8b96ab}}
+ svg{{display:block;width:100vw;height:calc(100vh - 46px)}}
+ circle{{stroke:#0e1116;stroke-width:1}}
+ line{{stroke:#2a3242;stroke-width:.6;opacity:.7}}
+ #tip{{position:fixed;pointer-events:none;background:#1c2330;border:1px solid #3a4458;
+      padding:6px 9px;border-radius:4px;font-size:12px;display:none;max-width:420px}}
+ #legend span{{display:inline-block;margin-right:12px}}
+ .dot{{display:inline-block;width:9px;height:9px;border-radius:50%;margin-right:4px}}
+ #filter{{margin-left:14px;background:#1c2330;color:#dde3ee;border:1px solid #3a4458;
+          border-radius:4px;padding:3px 6px}}
+</style></head><body>
+<header><b>devin-graph</b> — {title}
+ <select id="filter"><option value="">all kinds</option>{options}</select>
+ <span id="stats"></span></header>
+<div id="legend" style="padding:6px 16px;font-size:12px">{legend}</div>
+<svg id="g"></svg><div id="tip"></div>
+<script>
+const DATA = {data};
+const nodes = DATA.nodes, edges = DATA.edges;
+const idx = new Map(nodes.map((n,i)=>[n.id,i]));
+const links = edges.filter(e=>idx.has(e.source)&&idx.has(e.target))
+  .map(e=>({{s:idx.get(e.source),t:idx.get(e.target)}}));
+const svg = document.getElementById('g'), tip = document.getElementById('tip');
+const W=innerWidth,H=innerHeight-46;
+nodes.forEach((n,i)=>{{n.x=W/2+Math.cos(i)*80+Math.random()*40;
+  n.y=H/2+Math.sin(i)*80+Math.random()*40;n.vx=0;n.vy=0;}});
+const NS='http://www.w3.org/2000/svg';
+const gLines=document.createElementNS(NS,'g'),gNodes=document.createElementNS(NS,'g');
+svg.appendChild(gLines);svg.appendChild(gNodes);
+const lEls=links.map(l=>{{const e=document.createElementNS(NS,'line');gLines.appendChild(e);return e;}});
+const nEls=nodes.map((n,i)=>{{const c=document.createElementNS(NS,'circle');
+  c.setAttribute('r',n.kind==='session'?6:(n.kind==='project'?7:4));
+  c.setAttribute('fill',DATA.colors[n.kind]||'#8b96ab');
+  c.onmousemove=ev=>{{tip.style.display='block';tip.style.left=ev.clientX+12+'px';
+    tip.style.top=ev.clientY+12+'px';tip.innerHTML=n.tip;}};
+  c.onmouseleave=()=>tip.style.display='none';
+  gNodes.appendChild(c);return c;}});
+function tick(){{
+ for(let i=0;i<nodes.length;i++)for(let j=i+1;j<nodes.length;j++){{
+  const a=nodes[i],b=nodes[j];let dx=a.x-b.x,dy=a.y-b.y,d2=dx*dx+dy*dy+0.01;
+  if(d2<40000){{const f=30/d2;a.vx+=dx*f;b.vx-=dx*f;a.vy+=dy*f;b.vy-=dy*f;}}}}
+ for(const l of links){{const a=nodes[l.s],b=nodes[l.t];
+  let dx=b.x-a.x,dy=b.y-a.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-60)/d*0.04;
+  a.vx+=dx*f;b.vx-=dx*f;a.vy+=dy*f;b.vy-=dy*f;}}
+ for(const n of nodes){{n.vx+=(W/2-n.x)*0.0005;n.vy+=(H/2-n.y)*0.0005;
+  n.vx*=0.85;n.vy*=0.85;n.x+=n.vx;n.y+=n.vy;}}
+ lEls.forEach((e,i)=>{{const l=links[i];
+  e.setAttribute('x1',nodes[l.s].x);e.setAttribute('y1',nodes[l.s].y);
+  e.setAttribute('x2',nodes[l.t].x);e.setAttribute('y2',nodes[l.t].y);}});
+ nEls.forEach((e,i)=>{{e.setAttribute('cx',nodes[i].x);e.setAttribute('cy',nodes[i].y);}});
+}}
+let frames=0;
+(function loop(){{tick();if(++frames<400)requestAnimationFrame(loop);}})();
+document.getElementById('stats').textContent=
+ ` · ${{nodes.length}} nodes · ${{links.length}} edges${{DATA.truncated?' · TRUNCATED (--limit)':''}}`;
+document.getElementById('filter').onchange=ev=>{{
+ const k=ev.target.value;
+ nEls.forEach((e,i)=>{{e.style.opacity=(!k||nodes[i].kind===k)?1:0.08;}});
+ lEls.forEach((e,i)=>{{const l=links[i];
+  e.style.opacity=(!k||nodes[l.s].kind===k||nodes[l.t].kind===k)?0.7:0.04;}});
+}};
+</script></body></html>"""
+
+
+def render_view(export: dict[str, Any], *, limit: int = 500,
+                seed: int = 42) -> str:
+    """graph.db export dict → self-contained HTML page."""
+    nodes = export.get("nodes", [])
+    edges = export.get("edges", [])
+    truncated = len(nodes) > limit
+    if truncated:
+        # keep the highest-degree nodes so the slice stays meaningful
+        degree: dict[str, int] = {}
+        for e in edges:
+            degree[e["source"]] = degree.get(e["source"], 0) + 1
+            degree[e["target"]] = degree.get(e["target"], 0) + 1
+        rng = random.Random(seed)
+        keep = {n["id"] for n in sorted(
+            nodes, key=lambda n: -degree.get(n["id"], 0))[:limit]}
+        nodes = [n for n in nodes if n["id"] in keep]
+        edges = [e for e in edges
+                 if e["source"] in keep and e["target"] in keep]
+
+    view_nodes = [{
+        "id": n["id"], "kind": n["kind"],
+        "tip": ("<b>" + html.escape(n["kind"]) + "</b> "
+                + html.escape(str(n.get("key", n["id"]))[:200])),
+    } for n in nodes]
+    data = {
+        "nodes": view_nodes,
+        "edges": edges,
+        "colors": _KIND_COLORS,
+        "truncated": truncated,
+    }
+    kinds = sorted({n["kind"] for n in view_nodes})
+    legend = "".join(
+        f'<span><span class="dot" style="background:{_KIND_COLORS.get(k, "#8b96ab")}"></span>{k}</span>'
+        for k in kinds)
+    options = "".join(f'<option value="{k}">{k}</option>' for k in kinds)
+    src = export.get("meta", {}).get("source_db", "graph.db")
+    return _PAGE.format(
+        title=html.escape(src),
+        data=json.dumps(data, separators=(",", ":")),
+        legend=legend, options=options)
+
+
+def write_view(export: dict[str, Any], out: str | Path,
+               limit: int = 500) -> Path:
+    p = Path(out).expanduser()
+    p.write_text(render_view(export, limit=limit), encoding="utf-8")
+    return p

devin_graph/vscdb.py

@@ -0,0 +1,117 @@
+"""Extract GUI-side nodes/edges from Devin's ``state.vscdb`` (GR-1).
+
+``state.vscdb`` is the Electron store (``ItemTable`` key/value) under
+``User/globalStorage/`` of the Devin config dir. The only keys this reads
+are the observed ``windsurfSpace.*`` family:
+
+- ``windsurfSpace.sessionWorkspace/<backend>/<slug>`` — JSON
+  ``{workspaceId, label, folders[], lastUpdated}`` binding a GUI session
+  (identified by a generated slug like ``canyon-newspaper``) to a workspace.
+- ``windsurfSpace.resourceToSpace`` — ``{space_id: [editor URIs]}``; URIs
+  look like ``vscode-cascade-editor:///cascade-acp/<backend>/<slug>``.
+- ``windsurfSpace.metadata`` — ``{space_id: {lastAccessed: ms}}``.
+
+Coverage is deliberately best-effort: unknown or malformed keys are skipped,
+never fatal, and the store is opened read-only (``mode=ro`` + ``query_only``).
+
+Produces ``gui_session`` nodes (key = slug) and ``gui_workspace`` edges
+``gui_session → project``. When a space points at a slug, its ``space_id``
+and ``lastAccessed`` enrich the session node's attrs.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from devin_graph.extract import EDGE_KINDS, NODE_KINDS, Edge, Extraction, Node
+
+SESSION_WS_PREFIX = "windsurfSpace.sessionWorkspace/"
+RESOURCE_TO_SPACE = "windsurfSpace.resourceToSpace"
+SPACE_METADATA = "windsurfSpace.metadata"
+_EDITOR_URI_PREFIX = "vscode-cascade-editor:///cascade-acp/"
+
+
+def extract_vscdb(path: str | Path) -> Extraction:
+    """``state.vscdb`` → gui_session/gui_workspace nodes and edges."""
+    p = Path(path).expanduser()
+    con = sqlite3.connect(f"file:{p}?mode=ro", uri=True)
+    try:
+        con.execute("PRAGMA query_only = ON")
+        rows = dict(
+            con.execute("SELECT key, value FROM ItemTable").fetchall())
+    except sqlite3.Error as exc:
+        raise ValueError(f"{p}: not a readable state.vscdb ({exc})") from exc
+    finally:
+        con.close()
+
+    ex = Extraction()
+    last_accessed = _space_last_accessed(rows)
+    space_of = _space_of_session(rows)
+
+    for key, raw in rows.items():
+        if not key.startswith(SESSION_WS_PREFIX):
+            continue
+        backend, sep, slug = key[len(SESSION_WS_PREFIX):].rpartition("/")
+        if not sep or not slug:
+            continue
+        data = _json_dict(raw)
+        workspace = data.get("workspaceId")
+        folders = data.get("folders") or []
+        attrs: dict[str, Any] = {
+            "backend": backend,
+            "label": data.get("label"),
+            "workspace": workspace,
+            "folders": folders,
+            "lastUpdated": data.get("lastUpdated"),
+        }
+        space = space_of.get(slug)
+        if space:
+            attrs["space_id"] = space
+            if space in last_accessed:
+                attrs["lastAccessed"] = last_accessed[space]
+        ex.nodes.append(Node("gui_session", slug, attrs))
+        if workspace:
+            ex.nodes.append(
+                Node("project", workspace, {"name": Path(workspace).name}))
+            ex.edges.append(
+                Edge("gui_workspace", ("gui_session", slug),
+                     ("project", workspace)))
+    return ex
+
+
+def _json_dict(raw: Any) -> dict:
+    if isinstance(raw, bytes):
+        raw = raw.decode("utf-8", "replace")
+    try:
+        data = json.loads(raw)
+    except (TypeError, ValueError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _space_last_accessed(rows: dict) -> dict[str, int]:
+    meta = _json_dict(rows.get(SPACE_METADATA))
+    out = {}
+    for space_id, m in meta.items():
+        if isinstance(m, dict) and isinstance(m.get("lastAccessed"), int):
+            out[space_id] = m["lastAccessed"]
+    return out
+
+
+def _space_of_session(rows: dict) -> dict[str, str]:
+    """session slug → space_id, via resourceToSpace editor URIs."""
+    out: dict[str, str] = {}
+    rts = _json_dict(rows.get(RESOURCE_TO_SPACE))
+    for space_id, uris in rts.items():
+        if not isinstance(uris, list):
+            continue
+        for uri in uris:
+            if isinstance(uri, str) and uri.startswith(_EDITOR_URI_PREFIX):
+                slug = uri[len(_EDITOR_URI_PREFIX):].rstrip("/").rsplit(
+                    "/", 1)[-1]
+                if slug:
+                    out.setdefault(slug, space_id)
+    return out