agentforge-graph 0.3.2__py3-none-any.whl

agentforge_graph/enrich/summary_enricher.py

@@ -0,0 +1,211 @@
+"""``SummaryEnricher`` (feat-012) — bottom-up module summaries.
+
+Leaf-first over ``CONTAINS``: each file is summarised from its symbols
+(signatures) + imports, then one repo summary is synthesised from the file
+summaries. Summaries are ``Summary`` nodes (``SUMMARIZES`` → file / a synthesised
+``Repository`` node) with ``llm`` provenance, and are embedded
+(``source_type="summary"``) so a concept query can land on one and expand to the
+code. Budgeted (``BudgetPolicy``), resumable (``DirtySet("summaries")``), and
+idempotent: the ``Summary`` node is MERGE-updated and its (stable) ``SUMMARIZES``
+edge is created only when missing, and the vector is replaced by ref.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from agentforge_core.production.budget import BudgetPolicy
+from agentforge_core.production.exceptions import BudgetExceeded
+
+from agentforge_graph.core import (
+    Edge,
+    EdgeKind,
+    Embedded,
+    Node,
+    NodeKind,
+    Provenance,
+    SymbolID,
+)
+from agentforge_graph.store import Store
+
+from .report import SummaryReport
+from .summarizer import FileContext, Summarizer
+
+_SUMMARY_LANG = "summary"
+_REPO_PLACEHOLDER = "<repo>"
+
+
+def summary_id(repo: str, path: str) -> str:
+    return SymbolID.for_symbol(_SUMMARY_LANG, repo, path, "summary.")
+
+
+def repo_node_id(repo: str) -> str:
+    return SymbolID.for_symbol("repo", repo, _REPO_PLACEHOLDER, "repository.")
+
+
+class SummaryEnricher:
+    version = "summaries@1"
+
+    def __init__(
+        self,
+        repo: str,
+        summarizer: Summarizer,
+        *,
+        embedder: object | None = None,
+        max_words: int = 120,
+        levels: list[str] | None = None,
+        budget_usd: float = 2.0,
+        concurrency: int = 6,
+        commit: str = "",
+    ) -> None:
+        self.repo = repo
+        self.summarizer = summarizer
+        self.embedder = embedder
+        self.max_words = max_words
+        self.levels = levels or ["file", "repo"]
+        self.budget_usd = budget_usd
+        self.concurrency = max(1, concurrency)
+        self.commit = commit
+        self.last_done_ids: list[str] = []
+
+    async def enrich(self, store: Store, file_ids: list[str]) -> SummaryReport:
+        report = SummaryReport()
+        self.last_done_ids = []
+        if "file" not in self.levels:
+            return report
+
+        budget = BudgetPolicy(usd=self.budget_usd, max_tokens=10**12, max_iterations=10**12)
+        prov = Provenance.llm(self.version, 1.0, self.commit)
+        nodes: list[Node] = []
+        edges: list[Edge] = []
+        to_embed: list[tuple[str, str, str]] = []  # (summary_id, path, text)
+        file_summaries: list[tuple[str, str]] = []  # (path, text)
+
+        # contexts first (graph reads), then summarize in concurrent batches —
+        # cost accounted per batch outside the gather (ENH-002), bottom-up order
+        # preserved (the repo tier runs after all file summaries).
+        targets: list[tuple[str, FileContext]] = []
+        for fid in file_ids:
+            file_node = await store.graph.get(fid)
+            if file_node is not None and file_node.kind is NodeKind.FILE:
+                targets.append((fid, await self._file_context(store, file_node)))
+
+        for start in range(0, len(targets), self.concurrency):
+            batch = targets[start : start + self.concurrency]
+            try:
+                budget.check()
+            except BudgetExceeded:
+                report.budget_tripped = True
+                break
+            before = self.summarizer.cost_usd
+            summaries = await asyncio.gather(
+                *(self.summarizer.summarize_file(ctx, self.max_words) for _fid, ctx in batch)
+            )
+            budget.commit(self.summarizer.cost_usd - before)
+            for (fid, _ctx), summary in zip(batch, summaries, strict=True):
+                path = SymbolID.parse(fid).path
+                sid = summary_id(self.repo, path)
+                nodes.append(
+                    self._summary_node(sid, summary.text, "file", summary.model, path, prov)
+                )
+                edges.append(Edge(src=sid, dst=fid, kind=EdgeKind.SUMMARIZES, provenance=prov))
+                to_embed.append((sid, path, summary.text))
+                file_summaries.append((path, summary.text))
+                self.last_done_ids.append(fid)
+                report.files_summarized += 1
+
+        # repo tier (bottom-up from the file summaries) — also budget-gated
+        repo_ok = "repo" in self.levels and bool(file_summaries) and not report.budget_tripped
+        if repo_ok:
+            try:
+                budget.check()
+            except BudgetExceeded:
+                report.budget_tripped = True
+                repo_ok = False
+        if repo_ok:
+            before = self.summarizer.cost_usd
+            repo_summary = await self.summarizer.summarize_repo(
+                self.repo, file_summaries, self.max_words
+            )
+            budget.commit(self.summarizer.cost_usd - before)
+            rnode = repo_node_id(self.repo)
+            nodes.append(Node(id=rnode, kind=NodeKind.REPOSITORY, name=self.repo, provenance=prov))
+            rsid = summary_id(self.repo, _REPO_PLACEHOLDER)
+            nodes.append(
+                self._summary_node(rsid, repo_summary.text, "repo", repo_summary.model, "", prov)
+            )
+            edges.append(Edge(src=rsid, dst=rnode, kind=EdgeKind.SUMMARIZES, provenance=prov))
+            to_embed.append((rsid, "", repo_summary.text))
+            report.repo_summarized = True
+
+        report.cost_usd = round(self.summarizer.cost_usd, 6)
+
+        # Idempotent without edge churn: MERGE the summary nodes (this refreshes
+        # their text), then create each SUMMARIZES edge only if it's missing. The
+        # edge target is stable (a summary always summarizes the same file), so
+        # we never delete+recreate it — avoiding a Kuzu forward-rel-scan
+        # staleness bug (see docs/framework note).
+        if nodes:
+            await store.graph.add(list(nodes))
+        for edge in edges:
+            existing = await store.graph.adjacent(edge.src, [edge.kind], "out")
+            if not any(e.dst == edge.dst for e in existing):
+                await store.graph.add([edge])
+        await self._embed(store, to_embed)
+        return report
+
+    # --- helpers ----------------------------------------------------------
+
+    def _summary_node(
+        self, sid: str, text: str, level: str, model: str, path: str, prov: Provenance
+    ) -> Node:
+        return Node(
+            id=sid,
+            kind=NodeKind.SUMMARY,
+            name=f"summary:{path or self.repo}",
+            attrs={
+                "text": text,
+                "level": level,
+                "model": model,
+                "prompt_version": self.version,
+                "path": path,
+            },
+            provenance=prov,
+        )
+
+    async def _file_context(self, store: Store, file_node: Node) -> FileContext:
+        symbols: list[tuple[str, str]] = []
+        for e in await store.graph.adjacent(file_node.id, [EdgeKind.CONTAINS], "out"):
+            child = await store.graph.get(e.dst)
+            if child is not None and child.kind in (
+                NodeKind.CLASS,
+                NodeKind.FUNCTION,
+                NodeKind.METHOD,
+            ):
+                symbols.append((child.name, str(child.attrs.get("signature", ""))))
+        imports = [
+            str(imp.get("module", ""))
+            for imp in file_node.attrs.get("imports", [])
+            if imp.get("module")
+        ]
+        return FileContext(path=SymbolID.parse(file_node.id).path, symbols=symbols, imports=imports)
+
+    async def _embed(self, store: Store, items: list[tuple[str, str, str]]) -> None:
+        from agentforge_graph.embed import Embedder
+
+        if not items or not isinstance(self.embedder, Embedder):
+            return
+        vectors = await self.embedder.embed([text for _, _, text in items], "document")
+        embedded = [
+            Embedded(
+                ref=sid,
+                vector=vec,
+                kind=NodeKind.SUMMARY,
+                attrs={"path": path, "source_type": "summary", "model": self.embedder.name},
+            )
+            for (sid, path, _text), vec in zip(items, vectors, strict=True)
+        ]
+        # replace any prior vectors for these refs, then add
+        for sid, _path, _text in items:
+            await store.vectors.delete_where({"ref": sid})
+        await store.vectors.upsert(embedded)

agentforge_graph/enrich/taxonomy.py

@@ -0,0 +1,38 @@
+"""The fixed v1 design-pattern taxonomy (feat-012).
+
+A ``PatternTag`` is a shared taxonomy node (one per pattern name); a ``TAGGED``
+edge goes code-symbol → ``PatternTag`` with confidence + rationale in attrs.
+The list is locked at v1 (GoF core + architectural roles); extensible by config
+later. See spec §4.2.
+"""
+
+from __future__ import annotations
+
+from agentforge_graph.core import SymbolID
+
+TAXONOMY_V1: tuple[str, ...] = (
+    "Singleton",
+    "Factory",
+    "Builder",
+    "Adapter",
+    "Facade",
+    "Observer",
+    "Strategy",
+    "Decorator",
+    "Repository",
+    "Service",
+    "Controller",
+    "DTO",
+    "ValueObject",
+)
+
+_PATTERN_PATH = "<taxonomy>"
+
+
+def is_pattern(name: str) -> bool:
+    return name in TAXONOMY_V1
+
+
+def pattern_tag_id(repo: str, pattern: str) -> str:
+    """Stable id for a taxonomy node — a shared singleton per pattern name."""
+    return SymbolID.for_symbol("pattern", repo, _PATTERN_PATH, f"{pattern}.")

agentforge_graph/frameworks/__init__.py

@@ -0,0 +1,29 @@
+"""Framework-aware extractors (feat-011): export framework semantics —
+routes, ORM models, DI — as graph edges agents can traverse.
+
+A ``FrameworkPack`` rides feat-002's per-file extraction and emits framework
+nodes/edges into the file's ``FileSubgraph`` (so feat-004 incrementality
+applies for free). v0.4 MVP ships the **FastAPI** routes pack; ORM/DI and more
+frameworks follow over the same harness. Zero ``agentforge`` imports (ADR-0001).
+"""
+
+from __future__ import annotations
+
+from .base import FrameworkFacts, FrameworkPack
+from .detect import active_frameworks
+from .extractor import FrameworkExtractor
+from .registry import (
+    BUILTIN_FRAMEWORK_PACKS,
+    FrameworkRegistry,
+    builtin_framework_registry,
+)
+
+__all__ = [
+    "FrameworkFacts",
+    "FrameworkPack",
+    "FrameworkExtractor",
+    "FrameworkRegistry",
+    "BUILTIN_FRAMEWORK_PACKS",
+    "builtin_framework_registry",
+    "active_frameworks",
+]

agentforge_graph/frameworks/base.py

@@ -0,0 +1,75 @@
+"""The ``FrameworkPack`` ABC + ``FrameworkFacts`` (feat-011).
+
+A framework pack rides feat-002's extraction: given a parsed file it emits
+framework nodes/edges (``Route``/``DataModel``/``Service`` + ``HANDLED_BY``/…)
+attached to the symbols the *language* pack already produced (same SymbolID
+scheme). The facts are merged into the file's ``FileSubgraph`` (pipeline), so
+they inherit feat-004 incrementality for free — file-owned, ``parsed``
+provenance, never touched by the resolver's ``clear_resolved``.
+
+Detection is declarative: a pack lists the dependency names and import markers
+that mean "this repo uses me"; ``frameworks.detect`` does the scanning. Zero
+``agentforge`` imports (ADR-0001).
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel, Field
+
+from agentforge_graph.core import Edge, GraphStore, Node, SourceFile
+
+
+class FrameworkFacts(BaseModel):
+    """What a pack derived from one file. ``unresolved`` counts registrations
+    the pack recognised but could not extract statically (dynamic paths,
+    class-based handlers at MVP) — surfaced in the IndexReport, never dropped
+    silently."""
+
+    nodes: list[Node] = Field(default_factory=list)
+    edges: list[Edge] = Field(default_factory=list)
+    unresolved: int = 0
+
+
+class FrameworkPack(ABC):
+    """A framework's extraction rules over one language's parse trees."""
+
+    name: str = ""  # "fastapi"
+    language: str = ""  # the language pack this rides ("python")
+    language_slug: str = ""  # SymbolID slug of that language ("py")
+
+    @property
+    def slugs(self) -> tuple[str, ...]:
+        """The SymbolID slugs this pack extracts over. Defaults to the single
+        ``language_slug``; a pack spanning sibling languages (e.g. Express over
+        JS *and* TS) overrides this and uses ``file.language`` per file."""
+        return (self.language_slug,)
+
+    version: str = "1"  # bump on pattern changes (provenance + future --full)
+    dep_names: tuple[str, ...] = ()  # manifest dependency names that imply this framework
+    import_markers: tuple[str, ...] = ()  # source substrings that confirm use
+
+    def detect(self, dep_names: set[str], source_sample: str) -> bool:
+        """Active for this repo? A declared dependency, or an import marker in
+        the sampled source. Override for bespoke detection."""
+        if dep_names.intersection(self.dep_names):
+            return True
+        return any(marker in source_sample for marker in self.import_markers)
+
+    @abstractmethod
+    def extract(self, file: SourceFile, repo: str, commit: str) -> FrameworkFacts:
+        """Pass-1, file-isolated: emit framework nodes/edges for ``file``."""
+
+    async def resolve(self, store: GraphStore, commit: str = "") -> list[Edge]:
+        """Optional pass-2 cross-file stitching (ORM relationship/FK string
+        targets, router prefixes, string view refs). Reads the graph built in
+        pass-1 and returns resolved-provenance edges; the orchestrator clears the
+        previous generation and upserts the result (idempotent). Packs with no
+        cross-file step return ``[]``."""
+        return []
+
+    def coupled_files(self, path: str) -> bool:
+        """True for files whose change forces a framework re-resolve (e.g.
+        ``urls.py``). MVP: no pass-2, so always False."""
+        return False

agentforge_graph/frameworks/detect.py

@@ -0,0 +1,124 @@
+"""Framework detection (feat-011): which packs are active for a repo.
+
+Honours ``frameworks.enabled`` (``auto`` → detect, ``off`` → none, or an
+explicit list) plus ``frameworks.packs`` force-enable. Auto-detection reads
+dependency manifests (``pyproject.toml`` / ``requirements*.txt``) and, as a
+fallback, samples source text for each pack's import markers — so a repo that
+vendors a framework without a manifest still activates.
+"""
+
+from __future__ import annotations
+
+import re
+import tomllib
+from pathlib import Path
+
+from agentforge_graph.config import FrameworksConfig
+
+from .base import FrameworkPack
+from .registry import FrameworkRegistry
+
+_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+")
+_SAMPLE_CAP = 256 * 1024  # bytes of source to scan for import markers
+
+
+def _norm(name: str) -> str:
+    return name.strip().lower().replace("_", "-")
+
+
+def _dep_name(requirement: str) -> str:
+    """The distribution name from a PEP 508 requirement string
+    (``fastapi>=0.110`` → ``fastapi``)."""
+    m = _NAME_RE.match(requirement.strip())
+    return _norm(m.group(0)) if m else ""
+
+
+def _pyproject_deps(path: Path) -> set[str]:
+    try:
+        data = tomllib.loads(path.read_text(encoding="utf-8", errors="replace"))
+    except (OSError, tomllib.TOMLDecodeError):
+        return set()
+    deps: set[str] = set()
+    project = data.get("project", {})
+    for req in project.get("dependencies", []):
+        deps.add(_dep_name(str(req)))
+    for group in project.get("optional-dependencies", {}).values():
+        deps.update(_dep_name(str(req)) for req in group)
+    # poetry-style
+    poetry = data.get("tool", {}).get("poetry", {})
+    deps.update(_norm(k) for k in poetry.get("dependencies", {}))
+    return {d for d in deps if d and d != "python"}
+
+
+def _requirements_deps(path: Path) -> set[str]:
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return set()
+    deps: set[str] = set()
+    for line in text.splitlines():
+        line = line.strip()
+        if line and not line.startswith(("#", "-")):
+            deps.add(_dep_name(line))
+    return {d for d in deps if d}
+
+
+def _manifest_deps(repo_path: Path) -> set[str]:
+    """Best-effort dependency-name set from common manifests (names only),
+    lowercased with ``_``→``-`` so ``Flask_SQLAlchemy`` == ``flask-sqlalchemy``."""
+    deps: set[str] = set()
+    pyproject = repo_path / "pyproject.toml"
+    if pyproject.is_file():
+        deps |= _pyproject_deps(pyproject)
+    for req in repo_path.glob("requirements*.txt"):
+        deps |= _requirements_deps(req)
+    return deps
+
+
+def _source_sample(repo_path: Path, exts: set[str]) -> str:
+    """Concatenate up to ``_SAMPLE_CAP`` bytes of source (files matching the
+    active languages' extensions) for import-marker confirmation."""
+    chunks: list[str] = []
+    total = 0
+    for path in sorted(repo_path.rglob("*")):
+        if total >= _SAMPLE_CAP:
+            break
+        if not path.is_file() or path.suffix not in exts or ".ckg" in path.parts:
+            continue
+        try:
+            text = path.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        chunks.append(text)
+        total += len(text)
+    return "\n".join(chunks)
+
+
+def active_frameworks(
+    repo_path: str | Path,
+    config: str | Path | None,
+    registry: FrameworkRegistry,
+    language_extensions: set[str],
+) -> list[FrameworkPack]:
+    cfg = FrameworksConfig.load(config)
+    enabled = cfg.enabled
+    if enabled == "off":
+        return []
+
+    root = Path(repo_path)
+    forced = set(cfg.packs)
+
+    # Explicit list short-circuits detection (still honour force-enable).
+    if isinstance(enabled, list):
+        wanted = set(enabled) | forced
+        return [p for p in registry.packs if p.name in wanted]
+
+    # "auto": dependency manifest + import-marker fallback.
+    deps = _manifest_deps(root)
+    needs_sample = any(p.import_markers for p in registry.packs)
+    sample = _source_sample(root, language_extensions) if needs_sample else ""
+    active: list[FrameworkPack] = []
+    for pack in registry.packs:
+        if pack.name in forced or pack.detect(deps, sample):
+            active.append(pack)
+    return active

agentforge_graph/frameworks/extractor.py

@@ -0,0 +1,63 @@
+"""``FrameworkExtractor`` — run the active framework packs over one file
+(feat-011). Selects packs by the file's language and merges their
+``FrameworkFacts``. File-isolated and stateless, so it runs inside the same
+worker thread as the language extractor (pipeline)."""
+
+from __future__ import annotations
+
+from agentforge_graph.core import (
+    Edge,
+    EdgeKind,
+    GraphQuery,
+    GraphStore,
+    Node,
+    NodeKind,
+    SourceFile,
+)
+
+from .base import FrameworkFacts, FrameworkPack
+
+_ALL = 10_000_000  # effectively unbounded query for v0.1 graph sizes
+
+
+class FrameworkExtractor:
+    def __init__(self, packs: list[FrameworkPack]) -> None:
+        self._packs = list(packs)
+        self._by_slug: dict[str, list[FrameworkPack]] = {}
+        for pack in packs:
+            for slug in pack.slugs:
+                self._by_slug.setdefault(slug, []).append(pack)
+
+    @property
+    def active(self) -> bool:
+        return bool(self._packs)
+
+    def extract(self, file: SourceFile, repo: str, commit: str) -> FrameworkFacts:
+        merged = FrameworkFacts()
+        for pack in self._by_slug.get(file.language, []):
+            facts = pack.extract(file, repo, commit)
+            merged.nodes.extend(facts.nodes)
+            merged.edges.extend(facts.edges)
+            merged.unresolved += facts.unresolved
+        return merged
+
+    async def resolve(self, store: GraphStore, commit: str = "") -> tuple[int, int]:
+        """Run every active pack's cross-file pass-2 (ORM relationship/FK string
+        targets, …) and replace the previous generation of framework-resolved
+        edges. Globally idempotent: clears all ``RELATES_TO`` out of the current
+        framework nodes, then rebuilds from the whole-repo node set — so an
+        incremental resolve converges to the same graph as a full re-index
+        (feat-004). Returns ``(edges_resolved, targets_unresolved)``."""
+        if not self._packs:
+            return 0, 0
+        models = (await store.query(GraphQuery(kinds=[NodeKind.DATA_MODEL], limit=_ALL))).nodes
+        if models:
+            await store.clear_outgoing([m.id for m in models], EdgeKind.RELATES_TO)
+        pending = sum(len(m.attrs.get("relations") or []) for m in models)
+
+        edges: list[Node | Edge] = []
+        for pack in self._packs:
+            edges.extend(await pack.resolve(store, commit))
+        if edges:
+            await store.add(edges)
+        return len(edges), max(0, pending - len(edges))

agentforge_graph/frameworks/orm.py

@@ -0,0 +1,93 @@
+"""Shared ORM pass-2 helpers (feat-011).
+
+The cross-file ``RELATES_TO`` stitch is the same shape for every ORM pack: load
+the whole-repo model set, index it by class name and table, then turn each
+model's pending ``relations`` (recorded in pass-1) into edges via a unique-match
+lookup (ADR-0004 — never guess an ambiguous target). Only the per-relation
+target resolution differs per framework, so that is injected as a callback.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+
+from agentforge_graph.core import (
+    Edge,
+    EdgeKind,
+    GraphQuery,
+    GraphStore,
+    Node,
+    NodeKind,
+    Provenance,
+    SymbolID,
+)
+
+_ALL = 10_000_000  # effectively unbounded query for v0.1 graph sizes
+
+
+class ModelIndex:
+    """Whole-repo model lookup by class name and table, each mapping to the set
+    of model ids carrying it (so duplicates are detectable and never guessed)."""
+
+    def __init__(self, models: Iterable[Node]) -> None:
+        self.by_class: dict[str, set[str]] = {}
+        self.by_table: dict[str, set[str]] = {}
+        for m in models:
+            cls = str(m.attrs.get("model_class", ""))
+            if cls:
+                self.by_class.setdefault(cls, set()).add(m.id)
+            tbl = str(m.attrs.get("table", ""))
+            if tbl:
+                self.by_table.setdefault(tbl, set()).add(m.id)
+
+    def unique_class(self, name: str) -> str | None:
+        return _unique(self.by_class.get(name))
+
+    def unique_table(self, name: str) -> str | None:
+        return _unique(self.by_table.get(name))
+
+
+def _unique(ids: set[str] | None) -> str | None:
+    return next(iter(ids)) if ids and len(ids) == 1 else None
+
+
+async def framework_models(store: GraphStore, framework: str) -> list[Node]:
+    """Every ``DataModel`` node emitted by ``framework``."""
+    nodes = (await store.query(GraphQuery(kinds=[NodeKind.DATA_MODEL], limit=_ALL))).nodes
+    return [m for m in nodes if m.attrs.get("framework") == framework]
+
+
+def relations_to_edges(
+    models: list[Node],
+    index: ModelIndex,
+    resolve_target: Callable[[dict[str, str], ModelIndex], str | None],
+    prov: Provenance,
+) -> list[Edge]:
+    """Build deduped ``RELATES_TO`` edges from each model's pending relations.
+    ``resolve_target`` maps one relation dict to a target model id (or None when
+    external/ambiguous). Edges carry ``attrs.kind`` (relationship/fk/m2m/…) and
+    ``attrs.via`` (the field), owned by the source model's file for incremental
+    invalidation."""
+    edges: list[Edge] = []
+    seen: set[tuple[str, str, str]] = set()
+    for m in models:
+        for rel in m.attrs.get("relations") or []:
+            target_id = resolve_target(rel, index)
+            if target_id is None:
+                continue
+            kind = str(rel.get("kind", ""))
+            key = (m.id, target_id, kind)
+            if key in seen:
+                continue
+            seen.add(key)
+            edges.append(
+                Edge(
+                    src=m.id,
+                    dst=target_id,
+                    kind=EdgeKind.RELATES_TO,
+                    attrs={"kind": kind, "via": str(rel.get("field", ""))},
+                    provenance=prov,
+                    origin_path=SymbolID.parse(m.id).path,
+                )
+            )
+    return edges