agentforge-graph 0.3.2__py3-none-any.whl

agentforge_graph/ingest/extractor.py

@@ -0,0 +1,353 @@
+"""``TreeSitterExtractor`` — pass 1 of ingestion (feat-002).
+
+File-isolated: parses one file and emits its ``FileSubgraph`` — definition
+nodes (File/Class/Function/Method) with ``CONTAINS`` edges, plus imports and
+call sites recorded as node *attrs* (not edges — their targets may live in
+other files, which pass 1 may not read). The graph-only resolver (pass 2)
+turns those attrs into ``IMPORTS``/``CALLS`` edges.
+
+Parsing uses the standalone ``tree_sitter`` package driven by a grammar from
+``tree-sitter-language-pack`` (``Parser(get_language(...))``, never
+``get_parser()`` — see the framework note on the ABI split).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import textwrap
+from collections import defaultdict
+from dataclasses import dataclass, field
+from functools import cache
+from typing import Any
+
+from tree_sitter import Language, Parser, Query, QueryCursor
+from tree_sitter import Node as TSNode
+from tree_sitter_language_pack import get_language
+
+from agentforge_graph.core import (
+    Descriptor,
+    Edge,
+    EdgeKind,
+    Extractor,
+    FileSubgraph,
+    NodeKind,
+    Provenance,
+    SourceFile,
+    SymbolID,
+)
+from agentforge_graph.core import (
+    Node as GraphNode,
+)
+
+from .pack import LanguagePack
+
+_CALLABLE = {NodeKind.FUNCTION, NodeKind.METHOD}
+_METHOD_OWNERS = {NodeKind.CLASS, NodeKind.INTERFACE}
+
+
+@cache
+def _language(grammar: str) -> Language:
+    return get_language(grammar)
+
+
+@dataclass
+class _Def:
+    """A captured definition, pre-symbol-id."""
+
+    ts_id: int
+    node: TSNode
+    kind: NodeKind
+    name: str
+    enclosing: int | None = None  # ts id of the nearest enclosing def
+    symbol_id: str = ""
+    bases: list[str] = field(default_factory=list)  # superclass names (INHERITS)
+    recv_var: str = ""  # Go: a method's receiver variable name (`s` in `func (s *T)`)
+    recv_type: str = ""  # Go: a method's receiver type name (`T`)
+    docstring: str = ""  # the symbol's docstring, cleaned (DESCRIBES, feat-010)
+
+
+def _text(node: TSNode, src: bytes) -> str:
+    return src[node.start_byte : node.end_byte].decode("utf-8", errors="replace")
+
+
+def _span(node: TSNode) -> tuple[int, int]:
+    return (node.start_point[0] + 1, node.end_point[0] + 1)
+
+
+def _signature(node: TSNode, src: bytes) -> str:
+    """The symbol's first source line (the def/class header), trimmed."""
+    text = _text(node, src)
+    return text.splitlines()[0].strip() if text else ""
+
+
+_STR_PREFIX = re.compile(r"^[rbfuRBFU]{0,2}")
+_JSDOC_LINE = re.compile(r"^\s*\*?\s?")  # a JSDoc/Javadoc line's leading ` * `
+
+
+def _clean_docstring(raw: str) -> str:
+    """The docstring body: strip a Python string literal's prefix + quotes, OR a
+    ``/** … */`` JSDoc/Javadoc comment's markers + per-line ``*``; then dedent."""
+    s = raw.strip()
+    if s.startswith("/*"):  # JSDoc / Javadoc block comment
+        s = s[2:]
+        if s.startswith("*"):  # the second `*` of `/**`
+            s = s[1:]
+        if s.endswith("*/"):
+            s = s[:-2]
+        return "\n".join(_JSDOC_LINE.sub("", ln) for ln in s.splitlines()).strip()
+    s = _STR_PREFIX.sub("", s, count=1)
+    for q in ('"""', "'''", '"', "'"):
+        if s.startswith(q) and s.endswith(q) and len(s) >= 2 * len(q):
+            s = s[len(q) : -len(q)]
+            break
+    return textwrap.dedent(s).strip()
+
+
+class TreeSitterExtractor(Extractor):
+    """Extracts a ``FileSubgraph`` from one source file, in isolation."""
+
+    def __init__(self, pack: LanguagePack, repo: str, commit: str = "") -> None:
+        self.pack = pack
+        self.repo = repo
+        self.commit = commit
+        self.name = f"tree-sitter-{pack.language}"
+        self._lang = _language(pack.grammar)
+        self._parser = Parser(self._lang)
+        self._structure_q = Query(self._lang, pack.structure_queries)
+        self._reference_q = Query(self._lang, pack.reference_queries)
+
+    def extract(self, file: SourceFile) -> FileSubgraph:
+        src = file.text.encode("utf-8")
+        root = self._parser.parse(src).root_node
+        prov = Provenance.parsed(self.name, self.commit)
+        file_id = SymbolID.for_symbol(self.pack.lang_slug, self.repo, file.path, "")
+
+        defs, imports, default_export, namespace = self._structure(root, src)
+        self._assign_symbol_ids(defs, file.path)
+        by_tsid = {d.ts_id: d for d in defs}
+        refs = self._references(root, src, by_tsid, file_id)
+
+        nodes: list[GraphNode] = []
+        file_attrs: dict[str, Any] = {}
+        if imports:
+            file_attrs["imports"] = imports
+        if default_export:
+            file_attrs["default_export"] = default_export
+        if namespace:
+            file_attrs["namespace"] = namespace  # PHP/Java/C# package (FQN resolution)
+        if file_id in refs:
+            file_attrs["refs"] = refs[file_id]
+        nodes.append(
+            GraphNode(
+                id=file_id,
+                kind=NodeKind.FILE,
+                name=file.path.rsplit("/", 1)[-1],
+                provenance=prov,
+                attrs=file_attrs,
+            )
+        )
+
+        edges: list[Edge] = []
+        for d in defs:
+            attrs: dict[str, Any] = {"signature": _signature(d.node, src)}
+            if d.symbol_id in refs:
+                attrs["refs"] = refs[d.symbol_id]
+            if d.bases:  # INHERITS: superclass names, resolved in pass 2
+                attrs["bases"] = d.bases
+            if d.recv_var:  # Go: receiver var/type, for receiver self-calls (pass 2)
+                attrs["recv_var"] = d.recv_var
+                attrs["recv_type"] = d.recv_type
+            nodes.append(
+                GraphNode(
+                    id=d.symbol_id,
+                    kind=d.kind,
+                    name=d.name,
+                    span=_span(d.node),
+                    provenance=prov,
+                    attrs=attrs,
+                )
+            )
+            parent_id = by_tsid[d.enclosing].symbol_id if d.enclosing in by_tsid else file_id
+            edges.append(
+                Edge(src=parent_id, dst=d.symbol_id, kind=EdgeKind.CONTAINS, provenance=prov)
+            )
+            # docstring -> a DocChunk that DESCRIBES the symbol (feat-010), so the
+            # docstring prose is embeddable + searchable, attached to its symbol.
+            if d.docstring:
+                desc = SymbolID.parse(d.symbol_id).descriptor + "docstring."
+                doc_id = SymbolID.for_symbol(self.pack.lang_slug, self.repo, file.path, desc)
+                nodes.append(
+                    GraphNode(
+                        id=doc_id,
+                        kind=NodeKind.DOC_CHUNK,
+                        name=d.name,
+                        provenance=prov,
+                        attrs={
+                            "path": file.path,
+                            "heading": d.name,
+                            "text": d.docstring,
+                            "describes": d.symbol_id,
+                            "content_hash": hashlib.sha256(d.docstring.encode()).hexdigest(),
+                        },
+                    )
+                )
+                edges.append(
+                    Edge(src=doc_id, dst=d.symbol_id, kind=EdgeKind.DESCRIBES, provenance=prov)
+                )
+
+        nodes.sort(key=lambda n: (n.span or (0, 0), n.id))
+        edges.sort(key=lambda e: (e.src, e.dst, e.kind.value))
+        return FileSubgraph(
+            path=file.path, content_hash=file.content_hash, nodes=nodes, edges=edges
+        )
+
+    # --- structure pass -------------------------------------------------
+
+    def _structure(
+        self, root: TSNode, src: bytes
+    ) -> tuple[list[_Def], list[dict[str, Any]], str, str]:
+        defs: list[_Def] = []
+        imports: list[dict[str, Any]] = []
+        default_export = ""  # CommonJS `module.exports = <name>` (BUG-006)
+        namespace = ""  # PHP/Java/C# package declaration (FQN import resolution)
+        class_bases: dict[int, list[str]] = defaultdict(list)  # class node id -> base names
+        method_recv: dict[int, tuple[str, str]] = {}  # method node id -> (recv var, recv type)
+        docstrings: dict[int, str] = {}  # def node id -> cleaned docstring (DESCRIBES)
+        rules = self.pack.descriptor_rules
+        for _pattern, caps in QueryCursor(self._structure_q).matches(root):
+            def_cap = next((c for c in caps if c.startswith("def.")), None)
+            if def_cap is not None:
+                kind = rules.kind_for(def_cap)
+                names = caps.get("name")
+                if kind is None or not names:
+                    continue
+                node = caps[def_cap][0]
+                defs.append(_Def(ts_id=node.id, node=node, kind=kind, name=_text(names[0], src)))
+            elif "base.name" in caps:
+                # a base class of a class definition (INHERITS); one match per base
+                cls = caps.get("base.def")
+                if cls:
+                    class_bases[cls[0].id].extend(_text(b, src) for b in caps["base.name"])
+            elif "recv.var" in caps:
+                # Go: a method's receiver `(s *T)` — bind the var name + type
+                meth, rvar, rtype = caps.get("recv.method"), caps["recv.var"], caps.get("recv.type")
+                if meth and rtype:
+                    method_recv[meth[0].id] = (_text(rvar[0], src), _text(rtype[0], src))
+            elif "docstring" in caps:
+                # a def/class docstring or JSDoc comment — DESCRIBES the symbol
+                owner = caps.get("doc.owner")
+                if owner:
+                    docstrings[owner[0].id] = _clean_docstring(_text(caps["docstring"][0], src))
+            elif "import" in caps:
+                mods = caps.get("import.module", [])
+                dflt = caps.get("import.default")
+                imports.append(
+                    {
+                        "module": _text(mods[0], src) if mods else "",
+                        "names": [_text(n, src) for n in caps.get("import.name", [])],
+                        # CommonJS default require binding: `const x = require(...)`
+                        "default": _text(dflt[0], src) if dflt else "",
+                        "line": caps["import"][0].start_point[0] + 1,
+                    }
+                )
+            elif "namespace" in caps:
+                ns = caps.get("namespace")
+                if ns:
+                    namespace = _text(ns[0], src)
+            elif "export" in caps:
+                ed = caps.get("export.default")
+                if ed:
+                    default_export = _text(ed[0], src)
+        for d in defs:
+            if d.ts_id in class_bases:
+                d.bases = class_bases[d.ts_id]
+            if d.ts_id in method_recv:
+                d.recv_var, d.recv_type = method_recv[d.ts_id]
+            if d.ts_id in docstrings:
+                d.docstring = docstrings[d.ts_id]
+        self._link_scopes(defs)
+        return defs, imports, default_export, namespace
+
+    def _link_scopes(self, defs: list[_Def]) -> None:
+        idset = {d.ts_id for d in defs}
+        by_tsid = {d.ts_id: d for d in defs}
+        for d in defs:
+            anc = d.node.parent
+            while anc is not None and anc.id not in idset:
+                anc = anc.parent
+            d.enclosing = anc.id if anc is not None else None
+            # a function whose nearest enclosing def is a class is a method
+            if (
+                d.kind is NodeKind.FUNCTION
+                and d.enclosing is not None
+                and by_tsid[d.enclosing].kind in _METHOD_OWNERS
+            ):
+                d.kind = NodeKind.METHOD
+
+    def _assign_symbol_ids(self, defs: list[_Def], path: str) -> None:
+        by_tsid = {d.ts_id: d for d in defs}
+        # overload disambiguator: nth same-named callable in the same scope (source order)
+        counter: dict[tuple[int | None, str], int] = defaultdict(int)
+        disamb: dict[int, int] = {}
+        for d in sorted(defs, key=lambda d: d.node.start_byte):
+            if d.kind in _CALLABLE:
+                key = (d.enclosing, d.name)
+                disamb[d.ts_id] = counter[key]
+                counter[key] += 1
+        for d in defs:
+            chain: list[_Def] = []
+            cur: _Def | None = d
+            while cur is not None:
+                chain.append(cur)
+                cur = by_tsid.get(cur.enclosing) if cur.enclosing is not None else None
+            chain.reverse()
+            descriptor = "".join(self._suffix(x, disamb.get(x.ts_id, 0)) for x in chain)
+            d.symbol_id = SymbolID.for_symbol(self.pack.lang_slug, self.repo, path, descriptor)
+
+    @staticmethod
+    def _suffix(d: _Def, disambiguator: int) -> str:
+        if d.kind in (NodeKind.CLASS, NodeKind.INTERFACE):
+            return Descriptor.type(d.name)
+        if d.kind in _CALLABLE:
+            return Descriptor.method(d.name, disambiguator)
+        return Descriptor.term(d.name)
+
+    # --- reference pass -------------------------------------------------
+
+    def _references(
+        self, root: TSNode, src: bytes, by_tsid: dict[int, _Def], file_id: str
+    ) -> dict[str, list[dict[str, Any]]]:
+        idset = set(by_tsid)
+        # Keyed by the call node so a bare + receiver-capturing pattern that both
+        # match the same call (Java/Ruby, where one node type serves `f()` and
+        # `recv.f()`) yield ONE ref — the receiver merged in. Distinct-node-type
+        # grammars (Py/TS/JS/C#/Rust/PHP/C++) never collide, so this is a no-op
+        # for them; insertion order preserves source order.
+        owner_of: dict[int, str] = {}
+        ref_of: dict[int, dict[str, Any]] = {}
+        for _pattern, caps in QueryCursor(self._reference_q).matches(root):
+            if "call" not in caps:
+                continue
+            callees = caps.get("call.callee")
+            if not callees:
+                continue
+            call_node = caps["call"][0]
+            ref = ref_of.get(call_node.id)
+            if ref is None:
+                anc = call_node.parent
+                while anc is not None and anc.id not in idset:
+                    anc = anc.parent
+                owner_of[call_node.id] = by_tsid[anc.id].symbol_id if anc is not None else file_id
+                ref = {"name": _text(callees[0], src), "line": call_node.start_point[0] + 1}
+                ref_of[call_node.id] = ref
+            # BUG-006: the receiver of an attribute call (`recv.f()`), when the pack
+            # captures it — lets the resolver bind `self.f()`/`this.f()` to the
+            # enclosing class's method and refuse to guess for other receivers.
+            recv = caps.get("call.recv")
+            if recv and "recv" not in ref:
+                ref["recv"] = _text(recv[0], src)
+        refs: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        for cid, ref in ref_of.items():
+            refs[owner_of[cid]].append(ref)
+        return dict(refs)

agentforge_graph/ingest/incremental/__init__.py

@@ -0,0 +1,25 @@
+"""Incremental indexing (feat-004): re-index only the diff.
+
+A thin coordination layer over the feat-002/003 primitives —
+``ChangeDetector`` diffs the working tree against the ``IndexMeta`` manifest,
+``IncrementalIndexer`` applies the resulting ``ChangeSet`` (delete → re-extract
+→ scoped re-resolve), and ``DirtySet`` records what each enricher must redo.
+Zero ``agentforge`` imports (ADR-0001).
+"""
+
+from __future__ import annotations
+
+from .detect import ChangeDetector, ChangeSet, DetectResult
+from .dirty import DirtySet
+from .indexer import IncrementalIndexer
+from .meta import IndexMeta, pack_fingerprint
+
+__all__ = [
+    "ChangeDetector",
+    "ChangeSet",
+    "DetectResult",
+    "DirtySet",
+    "IncrementalIndexer",
+    "IndexMeta",
+    "pack_fingerprint",
+]

agentforge_graph/ingest/incremental/detect.py

@@ -0,0 +1,118 @@
+"""``ChangeDetector`` — diff the working tree against the indexed manifest.
+
+The **content hash is the source of truth**: we walk the working tree once,
+hash every indexable file, and diff that against ``IndexMeta.files``. This is
+correct regardless of git state (dirty working tree, shallow clone, detached
+HEAD, rebase) and naturally catches uncommitted edits — the common case for an
+agent mid-flight. Git is then consulted *best-effort* only to promote a
+matching delete+add pair into a rename (nicer reporting); if git disagrees or
+is absent, the hash diff stands and a move simply reads as delete + add
+(accepted at 0.2, spec §3).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import subprocess
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from agentforge_graph.ingest.pack import PackRegistry
+from agentforge_graph.ingest.source import RepoSource
+
+from .meta import IndexMeta
+
+
+class ChangeSet(BaseModel):
+    """Files that changed since the last index, classified."""
+
+    added: list[str] = Field(default_factory=list)
+    modified: list[str] = Field(default_factory=list)
+    deleted: list[str] = Field(default_factory=list)
+    renamed: list[tuple[str, str]] = Field(default_factory=list)  # (old, new)
+
+    def is_empty(self) -> bool:
+        return not (self.added or self.modified or self.deleted or self.renamed)
+
+    def touched_paths(self) -> list[str]:
+        """Files to (re)extract: added, modified, and the new side of renames."""
+        return sorted({*self.added, *self.modified, *(new for _, new in self.renamed)})
+
+    def removed_paths(self) -> list[str]:
+        """Files to delete from the store: deleted, and the old side of renames."""
+        return sorted({*self.deleted, *(old for old, _ in self.renamed)})
+
+    def changed_paths(self) -> list[str]:
+        """Every path the diff touches on either side — the re-resolve seed."""
+        return sorted({*self.touched_paths(), *self.removed_paths()})
+
+
+class DetectResult(BaseModel):
+    changes: ChangeSet
+    file_hashes: dict[str, str]  # the fresh, full path -> content_hash manifest
+
+
+class ChangeDetector:
+    def __init__(self, repo_path: str | Path = ".") -> None:
+        self.repo_path = repo_path
+
+    async def detect(
+        self, source: RepoSource, meta: IndexMeta, registry: PackRegistry
+    ) -> DetectResult:
+        current = await asyncio.to_thread(self._current_hashes, source, registry)
+        prior = meta.files
+        added = [p for p in current if p not in prior]
+        modified = [p for p in current if p in prior and current[p] != prior[p]]
+        deleted = [p for p in prior if p not in current]
+        changes = ChangeSet(added=sorted(added), modified=sorted(modified), deleted=sorted(deleted))
+        self._refine_renames(changes, meta.indexed_commit)
+        return DetectResult(changes=changes, file_hashes=current)
+
+    @staticmethod
+    def _current_hashes(source: RepoSource, registry: PackRegistry) -> dict[str, str]:
+        return {sf.path: sf.content_hash for sf in source.iter_files(registry)}
+
+    def _refine_renames(self, changes: ChangeSet, base_commit: str) -> None:
+        """Best-effort: if git reports a committed rename old->new and our hash
+        diff independently saw `old` deleted and `new` added, collapse the pair
+        into a rename. Purely cosmetic — the indexer treats a rename as
+        delete(old)+add(new) anyway (§3), so a miss here changes nothing."""
+        if not base_commit:
+            return
+        added = set(changes.added)
+        deleted = set(changes.deleted)
+        for old, new in self._git_renames(base_commit):
+            if old in deleted and new in added:
+                changes.renamed.append((old, new))
+                changes.deleted.remove(old)
+                changes.added.remove(new)
+                deleted.discard(old)
+                added.discard(new)
+
+    def _git_renames(self, base_commit: str) -> list[tuple[str, str]]:
+        try:
+            out = subprocess.run(
+                [
+                    "git",
+                    "-C",
+                    str(self.repo_path),
+                    "diff",
+                    "--name-status",
+                    "-M",
+                    "--diff-filter=R",
+                    base_commit,
+                    "HEAD",
+                ],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+        except (subprocess.SubprocessError, OSError):
+            return []
+        pairs: list[tuple[str, str]] = []
+        for line in out.stdout.splitlines():
+            parts = line.split("\t")
+            if len(parts) == 3 and parts[0].startswith("R"):
+                pairs.append((parts[1], parts[2]))
+        return pairs

agentforge_graph/ingest/incremental/dirty.py

@@ -0,0 +1,61 @@
+"""``DirtySet`` — the one staleness API every enricher reads (feat-004).
+
+When an incremental refresh changes a file, the symbols it touched (plus their
+1-hop neighbours) are *dirtied* for every registered consumer — ``embeddings``
+now, ``summaries`` / ``pattern-tags`` / ``routes`` as feat-010/011/012 land.
+Each consumer drains its own cursor at its own cadence and marks the ids clean,
+so no enricher reinvents "what changed since I last ran". Persisted to
+``.ckg/dirty.json`` as ``{consumer: [symbol_id, ...]}`` — a side file, so a
+consumer cursor update never rewrites the index manifest (``meta.json``).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+_DIRTY = "dirty.json"
+
+
+class DirtySet:
+    # Known enrichment consumers: embeddings (feat-005), patterns + summaries (feat-012).
+    DEFAULT_CONSUMERS = ["embeddings", "patterns", "summaries"]
+
+    def __init__(self, root: str | Path, consumers: list[str] | None = None) -> None:
+        self._path = Path(root) / _DIRTY
+        self._consumers = list(consumers or self.DEFAULT_CONSUMERS)
+        self._state: dict[str, list[str]] = self._load()
+
+    def _load(self) -> dict[str, list[str]]:
+        if not self._path.exists():
+            return {}
+        data = json.loads(self._path.read_text())
+        return {k: list(v) for k, v in data.items()}
+
+    def _save(self) -> None:
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = self._path.with_name(_DIRTY + ".tmp")
+        tmp.write_text(json.dumps(self._state, indent=2, sort_keys=True))
+        tmp.replace(self._path)
+
+    async def add(self, ids: list[str]) -> None:
+        """Append ``ids`` to every registered consumer's dirty set (deduped,
+        order-stable)."""
+        if not ids:
+            return
+        for consumer in self._consumers:
+            have = self._state.setdefault(consumer, [])
+            seen = set(have)
+            for i in ids:
+                if i not in seen:
+                    seen.add(i)
+                    have.append(i)
+        self._save()
+
+    async def dirty_for(self, consumer: str) -> list[str]:
+        return list(self._state.get(consumer, []))
+
+    async def mark_clean(self, consumer: str, ids: list[str]) -> None:
+        drop = set(ids)
+        self._state[consumer] = [i for i in self._state.get(consumer, []) if i not in drop]
+        self._save()