agentforge-graph 0.3.2__py3-none-any.whl

agentforge_graph/ingest/packs/ruby/__init__.py

@@ -0,0 +1,37 @@
+"""The Ruby language pack (Tier A — structure + require_relative resolution).
+
+Ruby modules/classes nest methods; `require_relative "./x"` is path-based and
+resolves in-repo (``module_style="relative"``), while `require "gem"` stays
+external. Ruby also autoloads (Rails) and uses heavy metaprogramming, so the
+import graph is sparser than in static languages — symbol extraction is the
+primary value; receiver-qualified calls stay unresolved (ADR-0004).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from agentforge_graph.core import NodeKind
+from agentforge_graph.ingest.pack import DescriptorRules, LanguagePack
+
+_HERE = Path(__file__).parent
+
+RUBY_PACK = LanguagePack(
+    language="ruby",
+    lang_slug="rb",
+    grammar="ruby",
+    extensions=(".rb",),
+    structure_queries=(_HERE / "structure.scm").read_text(encoding="utf-8"),
+    reference_queries=(_HERE / "references.scm").read_text(encoding="utf-8"),
+    descriptor_rules=DescriptorRules(
+        kinds={
+            "def.class": NodeKind.CLASS,  # class + module
+            "def.function": NodeKind.FUNCTION,  # def (promoted to Method in a class)
+            "def.method": NodeKind.METHOD,  # def self.x
+            "def.variable": NodeKind.VARIABLE,  # constant assignment
+        }
+    ),
+    module_style="relative",  # require_relative paths are relative to the file
+    relative_bare=True,  # `require_relative "thor/x"` (bare) is still file-relative
+    wildcard_import=True,  # require_relative brings in all the file's top-level defs
+)

agentforge_graph/ingest/packs/ruby/references.scm

@@ -0,0 +1,12 @@
+; Ruby reference queries (feat-002, pack-ruby).
+; A call's `method:` field is the called name, whether or not it has a receiver
+; (`foo(...)`, `obj.foo(...)`, `Mod::Klass.foo(...)`). The second pattern captures
+; the receiver (@call.recv) so `self.foo()` binds to the enclosing class's method
+; (BUG-006); other receivers stay unresolved (member access, ADR-0004).
+
+(call
+  method: (identifier) @call.callee) @call
+
+(call
+  receiver: (_) @call.recv
+  method: (identifier) @call.callee) @call

agentforge_graph/ingest/packs/ruby/structure.scm

@@ -0,0 +1,37 @@
+; Ruby structure queries (feat-002, pack-ruby).
+; Shares the capture vocabulary so edge kinds mean the same as other packs.
+
+; --- definitions ---
+; module + class are named containers -> Class; nested defs become methods.
+(module
+  name: (constant) @name) @def.class
+
+(class
+  name: (constant) @name) @def.class
+
+; --- inheritance (INHERITS): `class B < A` ---
+(class
+  superclass: (superclass (constant) @base.name)) @base.def
+
+; `def foo` -> Function (promoted to Method when nested in a class/module body).
+(method
+  name: (identifier) @name) @def.function
+
+; `def self.foo` (class method) -> Method.
+(singleton_method
+  name: (identifier) @name) @def.method
+
+; constant assignment (`PI = 3.14`) -> Variable. Only `constant` (Uppercase) lefts
+; match, so local variables (lowercase identifiers) are not captured.
+(assignment
+  left: (constant) @name) @def.variable
+
+; --- imports ---
+; `require_relative "thor/command"` is always file-relative (bare or `./`),
+; resolved in-repo via relative_bare. Plain `require "gem"` is load-path based
+; (lib-root relative) and is left to a follow-up — capturing it here would
+; mis-resolve against the importer's dir, so only require_relative is taken.
+(call
+  method: (identifier) @_req
+  arguments: (argument_list (string (string_content) @import.module))
+  (#eq? @_req "require_relative")) @import

agentforge_graph/ingest/packs/rust/__init__.py

@@ -0,0 +1,39 @@
+"""The Rust language pack (Tier A — structure + path-derived module resolution).
+
+Rust's module path is implicit in the file layout (`src/a/b.rs` is module `a::b`),
+so the pack derives each file's module from its path (`namespace_from_path`) and
+resolves `use crate::a::b::Item` to the file declaring `Item` (FQN-style, sep
+`::`, with `crate::` stripped). Extracts struct/enum/union/impl (→Class), trait
+(→Interface), functions + methods, const/static (→Variable), type aliases. `impl`
+blocks attach their methods to the type. Method/path calls stay unresolved
+(ADR-0004); grouped/glob `use` is a follow-up.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from agentforge_graph.core import NodeKind
+from agentforge_graph.ingest.pack import DescriptorRules, LanguagePack
+
+_HERE = Path(__file__).parent
+
+RUST_PACK = LanguagePack(
+    language="rust",
+    lang_slug="rs",
+    grammar="rust",
+    extensions=(".rs",),
+    structure_queries=(_HERE / "structure.scm").read_text(encoding="utf-8"),
+    reference_queries=(_HERE / "references.scm").read_text(encoding="utf-8"),
+    descriptor_rules=DescriptorRules(
+        kinds={
+            "def.class": NodeKind.CLASS,  # struct + enum + union + impl
+            "def.interface": NodeKind.INTERFACE,  # trait
+            "def.function": NodeKind.FUNCTION,  # fn + trait method sig (promoted)
+            "def.variable": NodeKind.VARIABLE,  # const + static
+            "def.type": NodeKind.TYPE_ALIAS,
+        }
+    ),
+    namespace_sep="::",
+    namespace_from_path=True,  # module path is the file path, not a declaration
+)

agentforge_graph/ingest/packs/rust/references.scm

@@ -0,0 +1,12 @@
+; Rust reference queries (feat-002, pack-rust).
+; Plain call `f(...)` and method call `x.f(...)`. @call.recv captures the
+; receiver so `self.f()` binds to the enclosing impl's method (BUG-006); other
+; receivers stay unresolved (member access, ADR-0004).
+
+(call_expression
+  function: (identifier) @call.callee) @call
+
+(call_expression
+  function: (field_expression
+    value: (_) @call.recv
+    field: (field_identifier) @call.callee)) @call

agentforge_graph/ingest/packs/rust/structure.scm

@@ -0,0 +1,46 @@
+; Rust structure queries (feat-002, pack-rust).
+; Module path is derived from the file path (namespace_from_path), so a
+; `use crate::a::b::Item` resolves to the file declaring Item. `mod` blocks are
+; scopes, not def nodes (so items in them keep their correct kind).
+
+; --- definitions ---
+(struct_item
+  name: (type_identifier) @name) @def.class
+
+(enum_item
+  name: (type_identifier) @name) @def.class
+
+(union_item
+  name: (type_identifier) @name) @def.class
+
+; a trait is an interface (named method/contract container).
+(trait_item
+  name: (type_identifier) @name) @def.interface
+
+; `impl Type { … }` / `impl Trait for Type { … }` -> attach methods to the Type
+; (merges with the struct/enum node of the same name; methods nest -> Method).
+(impl_item
+  type: (type_identifier) @name) @def.class
+
+; free functions; in an impl/trait body they promote to Method by nesting.
+(function_item
+  name: (identifier) @name) @def.function
+
+; trait method signatures (`fn draw(&self);`).
+(function_signature_item
+  name: (identifier) @name) @def.function
+
+(const_item
+  name: (identifier) @name) @def.variable
+
+(static_item
+  name: (identifier) @name) @def.variable
+
+(type_item
+  name: (type_identifier) @name) @def.type
+
+; --- imports ---
+; `use crate::shapes::Shape;` -> path naming an item (FQN-style resolution).
+; Grouped/glob uses (`use a::{B, C}`, `use a::*`) are a follow-up.
+(use_declaration
+  (scoped_identifier) @import.module) @import

agentforge_graph/ingest/packs/typescript/__init__.py

@@ -0,0 +1,31 @@
+"""The TypeScript language pack (Tier A — structure + import resolution)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from agentforge_graph.core import NodeKind
+from agentforge_graph.ingest.pack import DescriptorRules, LanguagePack
+
+_HERE = Path(__file__).parent
+
+TYPESCRIPT_PACK = LanguagePack(
+    language="typescript",
+    lang_slug="ts",
+    grammar="typescript",
+    extensions=(".ts",),
+    structure_queries=(_HERE / "structure.scm").read_text(encoding="utf-8"),
+    reference_queries=(_HERE / "references.scm").read_text(encoding="utf-8"),
+    descriptor_rules=DescriptorRules(
+        kinds={
+            "def.class": NodeKind.CLASS,
+            "def.function": NodeKind.FUNCTION,  # promoted to METHOD inside a class
+            # ENH-008: broaden the TS surface beyond class/function/method.
+            "def.interface": NodeKind.INTERFACE,
+            "def.enum": NodeKind.CLASS,  # no dedicated Enum kind; a nominal type
+            "def.type": NodeKind.TYPE_ALIAS,
+            "def.variable": NodeKind.VARIABLE,  # module-level const tables/enums
+        }
+    ),
+    module_style="relative",  # TS imports are path specifiers (./util), not dotted
+)

agentforge_graph/ingest/packs/typescript/references.scm

@@ -0,0 +1,11 @@
+; TypeScript reference queries (feat-002, pack-ts).
+; Plain call `f(...)` and method/attribute call `recv.f(...)`. @call.recv is the
+; receiver (BUG-006), so `this.f()` binds to the enclosing class's method.
+
+(call_expression
+  function: (identifier) @call.callee) @call
+
+(call_expression
+  function: (member_expression
+    object: (_) @call.recv
+    property: (property_identifier) @call.callee)) @call

agentforge_graph/ingest/packs/typescript/structure.scm

@@ -0,0 +1,99 @@
+; TypeScript structure queries (feat-002, pack-ts).
+; Mirrors the Python pack's capture vocabulary so edge kinds mean the same.
+; Definitions may be wrapped in `export_statement`; queries match nested.
+
+; --- definitions ---
+(class_declaration
+  name: (type_identifier) @name) @def.class
+
+; `abstract class Foo {}` is a distinct node from class_declaration; capture it
+; as the same Class kind so abstract base classes + their methods are extracted
+; (BUG-005). JS has no `abstract`, so this is TS-only.
+(abstract_class_declaration
+  name: (type_identifier) @name) @def.class
+
+; --- inheritance (INHERITS): `class B extends A` (interfaces via `implements`
+; are a separate relation, not captured here). ---
+(class_declaration
+  (class_heritage (extends_clause value: (identifier) @base.name))) @base.def
+(abstract_class_declaration
+  (class_heritage (extends_clause value: (identifier) @base.name))) @base.def
+
+; qualified base `class B extends mod.Base` -> base `mod.Base`; the resolver splits
+; the receiver and binds it via the importing module alias.
+(class_declaration
+  (class_heritage (extends_clause value: (member_expression) @base.name))) @base.def
+(abstract_class_declaration
+  (class_heritage (extends_clause value: (member_expression) @base.name))) @base.def
+
+(function_declaration
+  name: (identifier) @name) @def.function
+
+; methods live in a class_body -> promoted to METHOD by the extractor
+(method_definition
+  name: (property_identifier) @name) @def.function
+
+; --- JSDoc/TSDoc docstrings (DESCRIBES) ---
+; a `/** … */` block comment immediately before a function/class/method becomes a
+; DocChunk that DESCRIBES the symbol (feat-010); `#match?` keeps only `/**` blocks.
+((comment) @docstring . (function_declaration) @doc.owner
+  (#match? @docstring "^/[*][*]"))
+((comment) @docstring . (class_declaration) @doc.owner
+  (#match? @docstring "^/[*][*]"))
+((comment) @docstring . (abstract_class_declaration) @doc.owner
+  (#match? @docstring "^/[*][*]"))
+(class_body
+  (comment) @docstring . (method_definition) @doc.owner
+  (#match? @docstring "^/[*][*]"))
+
+; --- TS type surface (ENH-008) ---
+; `interface Foo {}` -> Interface (the dominant way TS describes contracts).
+(interface_declaration
+  name: (type_identifier) @name) @def.interface
+
+; `enum E {}` and `const enum E {}` -> Class (no dedicated Enum kind; a named
+; nominal type with members). Both parse as enum_declaration with an identifier.
+(enum_declaration
+  name: (identifier) @name) @def.enum
+
+; `type ID = ...` -> TypeAlias.
+(type_alias_declaration
+  name: (type_identifier) @name) @def.type
+
+; --- value bindings (ENH-008, shared with JS) ---
+; `const f = (…) => …` / `const f = function () {}` -> Function (named from the
+; binding). Captured at any depth — these are genuine functions.
+(lexical_declaration
+  (variable_declarator
+    name: (identifier) @name
+    value: [(arrow_function) (function_expression)])) @def.function
+
+; module-level const data tables / const-object enums -> Variable. Scoped to the
+; top level (program / export) so locals inside function bodies don't inflate the
+; graph. Only object/array initializers — NOT call results: `const x =
+; require(...)` is an import binding (BUG-006), and call-bound public constants
+; (e.g. zod's `ZodIssueCode = arrayToEnum([...])`) stay findable via their
+; companion `type X = ...` alias, captured above as a TypeAlias.
+(program
+  (lexical_declaration
+    (variable_declarator
+      name: (identifier) @name
+      value: [(object) (array)])) @def.variable)
+(program
+  (export_statement
+    (lexical_declaration
+      (variable_declarator
+        name: (identifier) @name
+        value: [(object) (array)])) @def.variable))
+
+; --- imports ---
+; `import { a, b } from "./mod"` -> module (relative path) + bound names
+(import_statement
+  (import_clause (named_imports (import_specifier name: (identifier) @import.name)))
+  source: (string (string_fragment) @import.module)) @import
+
+; `import * as ns from "./mod"` -> the namespace alias binds the whole module, so
+; `ns.foo()` and a qualified base `extends ns.Base` resolve to its exports (BUG-006).
+(import_statement
+  (import_clause (namespace_import (identifier) @import.default))
+  source: (string (string_fragment) @import.module)) @import

agentforge_graph/ingest/pipeline.py

@@ -0,0 +1,134 @@
+"""``IngestPipeline`` — drives the two passes over a whole repo.
+
+Extraction is CPU-bound and file-isolated, so files are parsed on a thread
+pool with bounded concurrency; the store serializes its own writes. A fresh
+``TreeSitterExtractor`` is built inside each worker thread because a
+tree-sitter ``Parser`` is not safe to share across threads (the grammar
+itself is cached, so only the lightweight parser/query objects are rebuilt).
+After all files are upserted, the resolver runs once.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from agentforge_graph.core import FileSubgraph, GraphStore, SourceFile
+from agentforge_graph.frameworks import FrameworkExtractor
+
+from .extractor import TreeSitterExtractor
+from .pack import LanguagePack, PackRegistry
+from .report import IndexReport
+from .resolver import ImportResolver
+from .source import RepoSource, read_go_module
+
+
+def _extract_one(
+    pack: LanguagePack,
+    repo: str,
+    commit: str,
+    sf: SourceFile,
+    frameworks: FrameworkExtractor | None,
+) -> tuple[FileSubgraph, int]:
+    # Built and used entirely within the worker thread (parser is not shareable).
+    sg = TreeSitterExtractor(pack, repo, commit).extract(sf)
+    unresolved = 0
+    if frameworks is not None and frameworks.active:
+        facts = frameworks.extract(sf, repo, commit)  # feat-011: routes/etc.
+        unresolved = facts.unresolved
+        if facts.nodes or facts.edges:
+            sg = sg.model_copy(
+                update={"nodes": [*sg.nodes, *facts.nodes], "edges": [*sg.edges, *facts.edges]}
+            )
+    return sg, unresolved
+
+
+class IngestPipeline:
+    def __init__(
+        self,
+        repo: str,
+        commit: str = "",
+        concurrency: int = 8,
+        frameworks: FrameworkExtractor | None = None,
+    ) -> None:
+        self.repo = repo
+        self.commit = commit
+        self.concurrency = concurrency
+        self.frameworks = frameworks
+
+    async def run(
+        self,
+        source: RepoSource,
+        store: GraphStore,
+        registry: PackRegistry,
+        paths: set[str] | None = None,
+    ) -> IndexReport:
+        """Extract + upsert each file, then resolve. When ``paths`` is given,
+        only those files are (re)extracted (feat-004 incremental scope); the
+        resolver is **not** run here — incremental refresh owns scoped
+        re-resolution. ``paths is None`` is the full-index path (resolve runs).
+        Active framework packs (feat-011) emit extra nodes/edges merged into
+        each file's subgraph, so they ride the same upsert + incrementality."""
+        report = IndexReport()
+        sem = asyncio.Semaphore(self.concurrency)
+
+        async def _do(sf: SourceFile) -> tuple[FileSubgraph, int] | None:
+            pack = registry.for_slug(sf.language)
+            if pack is None:
+                return None
+            async with sem:
+                result = await asyncio.to_thread(
+                    _extract_one, pack, self.repo, self.commit, sf, self.frameworks
+                )
+            await store.upsert(result[0])
+            return result
+
+        files = (sf for sf in source.iter_files(registry) if paths is None or sf.path in paths)
+        results = await asyncio.gather(*[_do(sf) for sf in files])
+
+        for result in results:
+            if result is None:
+                continue
+            sg, unresolved = result
+            report.files_indexed += 1
+            report.nodes += len(sg.nodes)
+            report.edges += len(sg.edges)
+            report.framework_unresolved += unresolved
+            for n in sg.nodes:
+                report.by_node_kind[n.kind.value] = report.by_node_kind.get(n.kind.value, 0) + 1
+            for e in sg.edges:
+                report.by_edge_kind[e.kind.value] = report.by_edge_kind.get(e.kind.value, 0) + 1
+        report.skipped = list(source.skipped)
+        report.routes_extracted = report.by_node_kind.get("Route", 0)
+        report.models_extracted = report.by_node_kind.get("DataModel", 0)
+        report.services_extracted = report.by_node_kind.get("Service", 0)
+
+        if paths is not None:
+            # Scoped (incremental) extract: the caller re-resolves with the
+            # right import-graph scope. Edge tallies come from that pass.
+            return report
+
+        stats = await ImportResolver(
+            registry, self.commit, go_module=read_go_module(source.root)
+        ).resolve(store)
+        report.resolve = stats
+        imports = stats.imports_resolved + stats.imports_external
+        report.by_edge_kind["IMPORTS"] = report.by_edge_kind.get("IMPORTS", 0) + imports
+        report.by_edge_kind["CALLS"] = report.by_edge_kind.get("CALLS", 0) + stats.refs_resolved
+        if stats.inherits_resolved:
+            report.by_edge_kind["INHERITS"] = (
+                report.by_edge_kind.get("INHERITS", 0) + stats.inherits_resolved
+            )
+        report.edges += imports + stats.refs_resolved + stats.inherits_resolved
+
+        # feat-011 pass-2: stitch ORM relationship/FK string targets into
+        # RELATES_TO edges (and future router-prefix composition).
+        if self.frameworks is not None and self.frameworks.active:
+            resolved, unresolved = await self.frameworks.resolve(store, self.commit)
+            if resolved:
+                report.relations_resolved = resolved
+                report.by_edge_kind["RELATES_TO"] = (
+                    report.by_edge_kind.get("RELATES_TO", 0) + resolved
+                )
+                report.edges += resolved
+            report.framework_unresolved += unresolved
+        return report

agentforge_graph/ingest/report.py

@@ -0,0 +1,84 @@
+"""Result types for an indexing run."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+
+class ResolveStats(BaseModel):
+    """Outcome of the pass-2 resolver."""
+
+    imports_resolved: int = 0  # IMPORTS edges to in-repo files
+    imports_external: int = 0  # IMPORTS edges to external (stdlib/third-party) packages
+    refs_resolved: int = 0  # CALLS edges created (unique match)
+    refs_unresolved: int = 0  # call sites with zero/ambiguous targets (recorded, not guessed)
+    inherits_resolved: int = 0  # INHERITS edges created (base class -> in-repo class)
+
+
+class RouteInfo(BaseModel):
+    """One extracted endpoint (feat-011), for ``CodeGraph.routes`` / ``ckg
+    routes`` / the ``ckg_routes`` tool."""
+
+    method: str
+    path: str
+    framework: str
+    handler: str  # handler symbol id (HANDLED_BY target)
+    file: str
+    line: int
+
+    def to_dict(self) -> dict[str, object]:
+        return self.model_dump()
+
+
+class ModelInfo(BaseModel):
+    """One extracted ORM data model (feat-011), for ``CodeGraph.models`` /
+    ``ckg models``."""
+
+    name: str  # table name when known, else the class name
+    table: str  # mapped table name ("" when not statically known)
+    framework: str
+    fields: list[str]  # mapped column/field names
+    relations: list[dict[str, str]]  # RELATES_TO out: {to, kind, via}
+    cls: str  # the underlying class symbol id
+    file: str
+    line: int
+
+    def to_dict(self) -> dict[str, object]:
+        return self.model_dump()
+
+
+class ServiceInfo(BaseModel):
+    """One DI-provided service (feat-011), for ``CodeGraph.services`` /
+    ``ckg services``."""
+
+    name: str  # the provider (dependency) name
+    framework: str
+    injected_into: list[str]  # consumer symbol ids the service is INJECTED_INTO
+    file: str
+    line: int
+
+    def to_dict(self) -> dict[str, object]:
+        return self.model_dump()
+
+
+class IndexReport(BaseModel):
+    """Summary of a full ``IngestPipeline.run`` / ``CodeGraph.index``."""
+
+    files_indexed: int = 0
+    nodes: int = 0
+    edges: int = 0
+    by_node_kind: dict[str, int] = Field(default_factory=dict)
+    by_edge_kind: dict[str, int] = Field(default_factory=dict)
+    skipped: list[str] = Field(default_factory=list)
+    resolve: ResolveStats = Field(default_factory=ResolveStats)
+    routes_extracted: int = 0  # feat-011: framework Route nodes emitted
+    models_extracted: int = 0  # feat-011: ORM DataModel nodes emitted
+    services_extracted: int = 0  # feat-011: DI Service nodes emitted
+    relations_resolved: int = 0  # feat-011: RELATES_TO edges from ORM relationship/FK targets
+    framework_unresolved: int = 0  # framework registrations seen but not extractable
+    decisions_indexed: int = 0  # feat-010: ADR Decision nodes
+    governs_resolved: int = 0  # GOVERNS edges from unambiguous ADR mentions
+    mentions_unresolved: int = 0  # ADR mentions seen but not linked
+    docs_indexed: int = 0  # feat-010: general doc files (doc_globs) ingested
+    describes_resolved: int = 0  # DESCRIBES edges from unambiguous doc mentions
+    commits_indexed: int = 0  # feat-010: commit messages ingested as DocChunks