klew 0.1.0__py3-none-any.whl

klew/__init__.py

@@ -0,0 +1 @@
+"""Klew: a self-improving code-context layer for coding agents, on HydraDB."""

klew/adapters/git_miner.py

@@ -0,0 +1,113 @@
+"""Read commit history with the git CLI: replay tasks, ground truth, and co-change counts."""
+from __future__ import annotations
+
+import math
+import subprocess
+from collections import Counter
+from dataclasses import dataclass, field
+from itertools import combinations
+from pathlib import Path
+
+
+@dataclass
+class Commit:
+    sha: str
+    ts: int
+    subject: str
+    body: str
+    changed: list[str] = field(default_factory=list)   # modified/renamed-to .py paths
+    added: list[str] = field(default_factory=list)
+    deleted: list[str] = field(default_factory=list)
+
+    @property
+    def task(self) -> str:
+        body = "\n".join(ln for ln in self.body.splitlines()
+                         if not ln.lower().startswith(("co-authored-by", "signed-off-by")))
+        return (self.subject + "\n\n" + body).strip()
+
+
+def _git(root: Path, *args: str) -> str:
+    return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True,
+                          text=True, errors="replace").stdout
+
+
+def commits(root: Path, rev_range: str = "HEAD", max_count: int | None = None,
+            paths: tuple[str, ...] = ("*.py",)) -> list[Commit]:
+    """Non-merge commits touching `paths`, oldest first."""
+    args = ["log", "--no-merges", "--reverse", "-M", "--name-status",
+            "--format=%x1e%H%x1f%at%x1f%s%x1f%b%x1f"]
+    if max_count:
+        args.append(f"--max-count={max_count}")
+    out = _git(root, *args, rev_range, "--", *paths)
+    result: list[Commit] = []
+    for rec in out.split("\x1e"):
+        if not rec.strip():
+            continue
+        head, _, files = rec.rpartition("\x1f")
+        sha, ts, subject, body = (head.split("\x1f") + ["", "", "", ""])[:4]
+        c = Commit(sha.strip(), int(ts or 0), subject.strip(), body.strip())
+        for line in files.strip().splitlines():
+            parts = line.split("\t")
+            if len(parts) < 2:
+                continue
+            status, path = parts[0], parts[-1]
+            if not path.endswith(".py"):
+                continue
+            if status.startswith("A"):
+                c.added.append(path)
+            elif status.startswith("D"):
+                c.deleted.append(path)
+            else:
+                c.changed.append(path)
+        result.append(c)
+    if max_count and len(result) > max_count:
+        result = result[-max_count:]
+    return result
+
+
+def head_sha(root: Path, rev: str = "HEAD") -> str:
+    return _git(root, "rev-parse", rev).strip()
+
+
+def worktree(root: Path, sha: str, dest: Path) -> Path:
+    """Materialize `sha` at `dest` (a detached worktree) for snapshot indexing."""
+    dest = dest.resolve()
+    if dest.exists():
+        _git(dest, "checkout", "-q", "--detach", sha)
+        return dest
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    _git(root, "worktree", "prune")  # forget worktrees whose directories were deleted
+    _git(root, "worktree", "add", "-q", "--detach", str(dest), sha)
+    return dest
+
+
+def remove_worktree(root: Path, dest: Path) -> None:
+    try:
+        _git(root, "worktree", "remove", "--force", str(dest.resolve()))
+    except subprocess.CalledProcessError:
+        _git(root, "worktree", "prune")
+
+
+@dataclass
+class CoChange:
+    pair_counts: Counter
+    file_counts: Counter
+
+    def weight(self, a: str, b: str) -> float:
+        c = self.pair_counts.get((a, b)) or self.pair_counts.get((b, a)) or 0
+        if not c:
+            return 0.0
+        return c / math.sqrt(self.file_counts[a] * self.file_counts[b])
+
+
+def co_change(history: list[Commit], max_files: int = 12) -> CoChange:
+    """Count how often files change together. Large sweeping commits are ignored as noise."""
+    pairs: Counter = Counter()
+    files: Counter = Counter()
+    for c in history:
+        touched = sorted(set(c.changed + c.added))
+        if not touched or len(touched) > max_files:
+            continue
+        files.update(touched)
+        pairs.update(combinations(touched, 2))
+    return CoChange(pairs, files)

klew/adapters/hydra_store.py

@@ -0,0 +1,150 @@
+"""GraphStore backed by a HydraDB node over Bolt (Neo4j driver)."""
+from __future__ import annotations
+
+import threading
+from collections.abc import Iterable, Sequence
+
+from neo4j import GraphDatabase
+
+from .. import ids as idmod
+from ..config import HydraConfig
+from ..ports import Neighbor, Row
+from ..schema import EDGE_TYPES, NODE_FIELDS, EdgeType
+from . import queries as Q
+
+BATCH = 1000  # HydraDB caps UNWIND batches at 1024 items
+ID_CHUNK = 1000
+
+
+def _chunks(seq: Sequence, n: int):
+    for i in range(0, len(seq), n):
+        yield seq[i : i + n]
+
+
+class HydraGraphStore:
+    def __init__(self, cfg: HydraConfig | None = None):
+        cfg = cfg or HydraConfig()
+        self._driver = GraphDatabase.driver(cfg.uri, auth=("neo4j", cfg.token))
+        self._db = cfg.database
+        # Sessions are not thread-safe, the driver is: one session per thread lets the walker
+        # run a hop's expansions concurrently.
+        self._local = threading.local()
+        self._sessions: list = []
+        self._count_lock = threading.Lock()
+        self.queries = 0
+
+    # -- plumbing -------------------------------------------------------------------------
+    def _run(self, cypher: str, **params) -> list[Row]:
+        # HydraDB rejects explicit transactions, so every statement is an auto-commit RUN.
+        sess = getattr(self._local, "session", None)
+        if sess is None:
+            sess = self._local.session = self._driver.session(database=self._db)
+            with self._count_lock:
+                self._sessions.append(sess)
+        with self._count_lock:
+            self.queries += 1
+        return [r.data() for r in sess.run(cypher, params)]
+
+    def ping(self) -> None:
+        self._driver.verify_connectivity()
+
+    # -- writes ---------------------------------------------------------------------------
+    def upsert_nodes(self, label: str, rows: Sequence[Row]) -> None:
+        if not rows:
+            return
+        fields = tuple(f for f in NODE_FIELDS[label] if f in rows[0])
+        cypher = Q.upsert_nodes(label, fields)
+        for chunk in _chunks(list(rows), BATCH):
+            self._run(cypher, rows=[{"id": r["id"], **{f: r.get(f) for f in fields}} for r in chunk])
+
+    def upsert_edges(self, etype: EdgeType, rows: Sequence[Row]) -> None:
+        if not rows:
+            return
+        extra = tuple(k for k in rows[0] if k not in ("src", "dst", "eid"))
+        cypher = Q.upsert_edges(etype, extra)
+        for chunk in _chunks(list(rows), BATCH):
+            self._run(cypher, rows=[
+                {"src": r["src"], "dst": r["dst"],
+                 "eid": r.get("eid") or idmod.edge_id(etype.name, r["src"], r["dst"]),
+                 **{k: r.get(k) for k in extra}}
+                for r in chunk
+            ])
+
+    def delete_nodes(self, label: str, ids: Iterable[int]) -> None:
+        ids = list(dict.fromkeys(ids))
+        for chunk in _chunks(ids, ID_CHUNK):
+            self._run(Q.delete_nodes(label), ids=chunk)
+
+    def delete_repo(self, repo: str) -> None:
+        for label in NODE_FIELDS:
+            self._run(Q.delete_repo(label), repo=repo)
+
+    # -- reads ----------------------------------------------------------------------------
+    def expand(self, etype: EdgeType, direction: str, ids: Iterable[int]) -> list[Neighbor]:
+        ids = list(dict.fromkeys(ids))
+        out: list[Neighbor] = []
+        cypher = Q.expand(etype, direction)
+        for chunk in _chunks(ids, ID_CHUNK):
+            for r in self._run(cypher, ids=chunk):
+                a, b = r["src"], r["dst"]
+                # Edge ids are identity, not readable properties; recompute from the endpoints.
+                eid = idmod.edge_id(etype.name, a, b) if direction == "out" else idmod.edge_id(etype.name, b, a)
+                out.append(Neighbor(a, b, eid, 1.0 if r.get("w") is None else float(r["w"])))
+        return sorted(out)   # HydraDB row order varies between runs; callers get a stable order
+
+    def get_nodes(self, label: str, ids: Iterable[int], fields: Sequence[str]) -> list[Row]:
+        ids = list(dict.fromkeys(ids))
+        out: list[Row] = []
+        cypher = Q.get_nodes(label, tuple(fields))
+        for chunk in _chunks(ids, ID_CHUNK):
+            out.extend(self._run(cypher, ids=chunk))
+        return out
+
+    def scan_nodes(self, label: str, repo: str, fields: Sequence[str]) -> list[Row]:
+        return self._run(Q.scan_nodes(label, tuple(fields)), repo=repo)
+
+    def list_nodes(self, label: str, fields: Sequence[str]) -> list[Row]:
+        return self._run(Q.list_nodes(label, tuple(fields)))
+
+    def edge_weights(self, etype: EdgeType, pairs: Iterable[tuple[int, int]]) -> dict[tuple[int, int], Row]:
+        pairs = set(pairs)
+        srcs = list({s for s, _ in pairs})
+        found: dict[tuple[int, int], Row] = {}
+        cypher = Q.edge_stats(etype)
+        for chunk in _chunks(srcs, ID_CHUNK):
+            for r in self._run(cypher, ids=chunk):
+                key = (r["src"], r["dst"])
+                if key in pairs:
+                    found[key] = {"w": r.get("w"), "hits": r.get("hits") or 0,
+                                  "misses": r.get("misses") or 0}
+        return found
+
+    def count_edges(self, etype: EdgeType, repo: str) -> int:
+        rows = self._run(Q.count_edges(etype), repo=repo)
+        return int(rows[0]["c"]) if rows else 0
+
+    def key_paths(self, repo: str, src_label: str, src_keys, dst_label: str, dst_keys, rel_types,
+                  max_len: int = 3, limit: int = 5) -> list[list]:
+        """Native algo.MSpaths between nodes named by `key`. Keys repeat across repos, so only paths whose
+        nodes all belong to `repo` are kept. Each path alternates node dicts and relationship-type strings."""
+        rows = self._run(Q.ms_paths(src_label, list(src_keys), dst_label, list(dst_keys), list(rel_types), max_len))
+        out = []
+        for r in rows:
+            path = r["path"]
+            nodes = [x for x in path if isinstance(x, dict)]
+            if nodes and all(n.get("repo") == repo for n in nodes):
+                out.append(path)
+        out.sort(key=len)
+        return out[:limit]
+
+    def shortest_paths(self, etype: EdgeType, src: int, dst: int, max_len: int, k: int) -> list[list[Row]]:
+        rows = self._run(Q.shortest_paths(etype), src=src, dst=dst, max_len=max_len, k=k)
+        return [[x for x in r["path"] if isinstance(x, dict)] for r in rows]
+
+    def close(self) -> None:
+        for sess in self._sessions:
+            sess.close()
+        self._driver.close()
+
+
+__all__ = ["HydraGraphStore", "EDGE_TYPES"]

klew/adapters/memory_store.py

@@ -0,0 +1,158 @@
+"""In-memory GraphStore with the same semantics as HydraGraphStore.
+
+Used for unit tests, for frontend work before HydraDB is up, and as the reference
+implementation in the store contract test.
+"""
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+
+from .. import ids as idmod
+from ..ports import Neighbor, Row
+from ..schema import NODE_FIELDS, EdgeType
+
+
+class MemoryGraphStore:
+    def __init__(self) -> None:
+        self.nodes: dict[str, dict[int, Row]] = defaultdict(dict)
+        # etype -> src -> dst -> props
+        self.out: dict[str, dict[int, dict[int, Row]]] = defaultdict(lambda: defaultdict(dict))
+        self.inn: dict[str, dict[int, dict[int, Row]]] = defaultdict(lambda: defaultdict(dict))
+        self.queries = 0
+
+    def upsert_nodes(self, label: str, rows: Sequence[Row]) -> None:
+        self.queries += 1
+        for r in rows:
+            cur = self.nodes[label].setdefault(r["id"], {"id": r["id"]})
+            cur.update({k: v for k, v in r.items() if k in NODE_FIELDS[label]})
+
+    def upsert_edges(self, etype: EdgeType, rows: Sequence[Row]) -> None:
+        self.queries += 1
+        for r in rows:
+            s, d = r["src"], r["dst"]
+            if s not in self.nodes[etype.src] or d not in self.nodes[etype.dst]:
+                continue  # MATCH would not bind; HydraDB silently writes nothing
+            props = self.out[etype.name][s].setdefault(d, {})
+            props.update({k: v for k, v in r.items() if k not in ("src", "dst")})
+            props.setdefault("eid", idmod.edge_id(etype.name, s, d))
+            self.inn[etype.name][d][s] = props
+
+    def delete_nodes(self, label: str, ids: Iterable[int]) -> None:
+        self._drop({i for i in ids if i in self.nodes[label]})
+
+    def delete_repo(self, repo: str) -> None:
+        self._drop({i for lbl in self.nodes.values() for i, r in lbl.items() if r.get("repo") == repo})
+
+    def _drop(self, doomed: set[int]) -> None:
+        self.queries += 1
+        for lbl in self.nodes.values():
+            for i in list(lbl):
+                if i in doomed:
+                    del lbl[i]
+        for idx in (self.out, self.inn):
+            for adj in idx.values():
+                for a in list(adj):
+                    if a in doomed:
+                        del adj[a]
+                        continue
+                    for b in list(adj[a]):
+                        if b in doomed:
+                            del adj[a][b]
+
+    def expand(self, etype: EdgeType, direction: str, ids: Iterable[int]) -> list[Neighbor]:
+        self.queries += 1
+        adj = self.out[etype.name] if direction == "out" else self.inn[etype.name]
+        out = []
+        for a in dict.fromkeys(ids):
+            for b, p in adj.get(a, {}).items():
+                w = p.get("w")
+                eid = idmod.edge_id(etype.name, a, b) if direction == "out" else idmod.edge_id(etype.name, b, a)
+                out.append(Neighbor(a, b, eid, 1.0 if w is None else float(w)))
+        return sorted(out)
+
+    def get_nodes(self, label: str, ids: Iterable[int], fields: Sequence[str]) -> list[Row]:
+        self.queries += 1
+        tbl = self.nodes[label]
+        return [{"id": i, **{f: tbl[i].get(f) for f in fields}} for i in dict.fromkeys(ids) if i in tbl]
+
+    def scan_nodes(self, label: str, repo: str, fields: Sequence[str]) -> list[Row]:
+        self.queries += 1
+        return [{"id": i, **{f: r.get(f) for f in fields}}
+                for i, r in self.nodes[label].items() if r.get("repo") == repo]
+
+    def list_nodes(self, label: str, fields: Sequence[str]) -> list[Row]:
+        self.queries += 1
+        return [{"id": i, **{f: r.get(f) for f in fields}} for i, r in self.nodes[label].items()]
+
+    def edge_weights(self, etype: EdgeType, pairs: Iterable[tuple[int, int]]) -> dict[tuple[int, int], Row]:
+        self.queries += 1
+        found = {}
+        for s, d in set(pairs):
+            p = self.out[etype.name].get(s, {}).get(d)
+            if p is not None:
+                found[(s, d)] = {"w": p.get("w"), "hits": p.get("hits") or 0,
+                                 "misses": p.get("misses") or 0}
+        return found
+
+    def count_edges(self, etype: EdgeType, repo: str) -> int:
+        self.queries += 1
+        return sum(len(v) for s, v in self.out[etype.name].items()
+                   if self.nodes[etype.src].get(s, {}).get("repo") == repo)
+
+    def key_paths(self, repo: str, src_label: str, src_keys, dst_label: str, dst_keys, rel_types,
+                  max_len: int = 3, limit: int = 5) -> list[list]:
+        """Reference BFS with the same output shape as HydraDB's algo.MSpaths (both directions)."""
+        from collections import deque
+        self.queries += 1
+        by_key = {}
+        for lbl, tbl in self.nodes.items():
+            for i, r in tbl.items():
+                if r.get("repo") == repo:
+                    by_key[(lbl, r.get("key"))] = i
+        lookup = {i: r for tbl in self.nodes.values() for i, r in tbl.items()}
+        targets = {by_key[(dst_label, k)] for k in dst_keys if (dst_label, k) in by_key}
+        out = []
+        for k in src_keys:
+            s = by_key.get((src_label, k))
+            if s is None:
+                continue
+            q = deque([[s]])
+            while q and len(out) < limit * 4:
+                p = q.popleft()
+                if p[-1] in targets and len(p) > 1:
+                    out.append(p)
+                    continue
+                if (len(p) - 1) // 2 >= max_len:
+                    continue
+                for t in rel_types:
+                    for nb in list(self.out[t].get(p[-1], {})) + list(self.inn[t].get(p[-1], {})):
+                        if nb not in p[::2]:
+                            q.append(p + [t, nb])
+        res = [[x if isinstance(x, str) else {k2: v for k2, v in lookup[x].items() if k2 != "id"} for x in p]
+               for p in out]
+        res.sort(key=len)
+        return res[:limit]
+
+    def shortest_paths(self, etype: EdgeType, src: int, dst: int, max_len: int, k: int) -> list[list[Row]]:
+        self.queries += 1
+        # BFS over both directions, mirroring relDirection: 'both'
+        from collections import deque
+        q = deque([[src]])
+        paths = []
+        while q and len(paths) < k:
+            p = q.popleft()
+            if p[-1] == dst:
+                paths.append(p)
+                continue
+            if len(p) > max_len:
+                continue
+            nbrs = set(self.out[etype.name].get(p[-1], {})) | set(self.inn[etype.name].get(p[-1], {}))
+            for n in nbrs:
+                if n not in p:
+                    q.append(p + [n])
+        lookup = {i: r for lbl in self.nodes.values() for i, r in lbl.items()}
+        return [[{k2: v for k2, v in lookup[i].items() if k2 != "id"} for i in p] for p in paths]
+
+    def close(self) -> None:
+        pass

klew/adapters/queries.py

@@ -0,0 +1,91 @@
+"""Every Cypher statement Klew sends to HydraDB lives here.
+
+HydraDB dialect rules this file respects (verified against v0.4.0, legacy engine):
+- Vertex batch upsert must be `UNWIND $rows AS row MERGE (n {id: row.id}) SET n:Label, ...`.
+- Relationship batch upsert must MATCH both endpoints and MERGE with an edge id: `{id: row.eid}`.
+- Composite (list-of-maps) params are only legal as UNWIND input, so batched *reads* use
+  `WHERE a.id IN $ids` with a scalar list instead of UNWIND.
+- Exactly one relationship type per pattern; no type(r)/labels(n); RETURN only binding.prop or
+  aggregates; property values are scalars only (lists/maps are stored as JSON strings).
+- Reading r.<prop> works when the far endpoint is unlabeled.
+"""
+from __future__ import annotations
+
+from ..schema import EdgeType
+
+
+def upsert_nodes(label: str, fields: tuple[str, ...]) -> str:
+    sets = ", ".join(f"n.{f} = row.{f}" for f in fields)
+    return f"UNWIND $rows AS row MERGE (n {{id: row.id}}) SET n:{label}" + (f", {sets}" if sets else "")
+
+
+def upsert_edges(e: EdgeType, fields: tuple[str, ...]) -> str:
+    sets = ", ".join(f"r.{f} = row.{f}" for f in fields)
+    return (
+        f"UNWIND $rows AS row MATCH (s:{e.src} {{id: row.src}}), (d:{e.dst} {{id: row.dst}}) "
+        f"MERGE (s)-[r:{e.name} {{id: row.eid}}]->(d)" + (f" SET {sets}" if sets else "")
+    )
+
+
+def expand(e: EdgeType, direction: str) -> str:
+    if direction == "out":
+        return (f"MATCH (a:{e.src})-[r:{e.name}]->(b) WHERE a.id IN $ids "
+                "RETURN a.id AS src, b.id AS dst, r.w AS w")
+    return (f"MATCH (a:{e.dst})<-[r:{e.name}]-(b) WHERE a.id IN $ids "
+            "RETURN a.id AS src, b.id AS dst, r.w AS w")
+
+
+def edge_stats(e: EdgeType) -> str:
+    return (f"MATCH (a:{e.src})-[r:{e.name}]->(b) WHERE a.id IN $ids "
+            "RETURN a.id AS src, b.id AS dst, r.w AS w, r.hits AS hits, r.misses AS misses")
+
+
+def get_nodes(label: str, fields: tuple[str, ...]) -> str:
+    cols = ", ".join(f"n.{f} AS {f}" for f in fields)
+    return f"MATCH (n:{label}) WHERE n.id IN $ids RETURN n.id AS id, {cols}"
+
+
+def scan_nodes(label: str, fields: tuple[str, ...]) -> str:
+    cols = ", ".join(f"n.{f} AS {f}" for f in fields)
+    return f"MATCH (n:{label}) WHERE n.repo = $repo RETURN n.id AS id, {cols}"
+
+
+def list_nodes(label: str, fields: tuple[str, ...]) -> str:
+    cols = ", ".join(f"n.{f} AS {f}" for f in fields)
+    return f"MATCH (n:{label}) RETURN n.id AS id, {cols}"
+
+
+def count_edges(e: EdgeType) -> str:
+    return f"MATCH (a:{e.src})-[r:{e.name}]->(b) WHERE a.repo = $repo RETURN count(*) AS c"
+
+
+def delete_nodes(label: str) -> str:
+    return f"MATCH (n:{label}) WHERE n.id IN $ids DETACH DELETE n"
+
+
+def delete_repo(label: str) -> str:
+    return f"MATCH (n:{label}) WHERE n.repo = $repo DETACH DELETE n"
+
+
+def _lit(values) -> str:
+    """Procedure maps take literals only, so lists are inlined and escaped."""
+    return "[" + ", ".join("'" + str(v).replace("\\", "\\\\").replace("'", "\\'") + "'" for v in values) + "]"
+
+
+def ms_paths(src_label: str, src_keys, dst_label: str, dst_keys, rel_types, max_len: int) -> str:
+    """HydraDB's native multi-source path search (GraphBLAS), anchored on the string `key` property."""
+    return (
+        f"CALL algo.MSpaths({{sourceLabel: '{src_label}', sourceProperty: 'key', sourceValues: {_lit(src_keys)}, "
+        f"targetLabel: '{dst_label}', targetProperty: 'key', targetValues: {_lit(dst_keys)}, "
+        f"relTypes: {_lit(rel_types)}, relDirection: 'both', maxLen: {int(max_len)}}}) "
+        "YIELD path RETURN path"
+    )
+
+
+def shortest_paths(e: EdgeType) -> str:
+    # Procedure maps take scalars only; lists are inlined literals.
+    return (
+        "CALL algo.SPpaths({sourceNode: $src, targetNode: $dst, "
+        f"relTypes: ['{e.name}'], relDirection: 'both', maxLen: $max_len, pathCount: $k}}) "
+        "YIELD path, pathWeight RETURN path, pathWeight"
+    )