codecortex-context-engine 0.1.0a1__py3-none-any.whl

codecortex/indexing/incremental.py

@@ -0,0 +1,163 @@
+"""Persistent incremental repository index."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+_EXCLUDED = {
+    ".git",
+    ".codecortex",
+    ".venv",
+    "venv",
+    "node_modules",
+    "dist",
+    "build",
+    "__pycache__",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class FileState:
+    digest: str
+    size: int
+    mtime_ns: int
+
+
+@dataclass(frozen=True, slots=True)
+class IndexStats:
+    tracked: int
+    added: tuple[str, ...]
+    changed: tuple[str, ...]
+    removed: tuple[str, ...]
+    unchanged: int
+    duration_ms: float
+
+    @property
+    def dirty(self) -> bool:
+        return bool(self.added or self.changed or self.removed)
+
+
+class IncrementalIndex:
+    """Track repository files by content hash and persist the result locally."""
+
+    VERSION = 1
+
+    def __init__(
+        self,
+        root: Path,
+        state_path: Path | None = None,
+        max_file_bytes: int = 4 * 1024 * 1024,
+    ) -> None:
+        self.root = root.resolve()
+        self.state_path = state_path or self.root / ".codecortex" / "index" / "manifest.json"
+        self.max_file_bytes = max_file_bytes
+
+    def _iter_files(self) -> list[Path]:
+        files: list[Path] = []
+        for path in self.root.rglob("*"):
+            if not path.is_file():
+                continue
+            relative = path.relative_to(self.root)
+            if any(part in _EXCLUDED for part in relative.parts):
+                continue
+            try:
+                if path.stat().st_size > self.max_file_bytes:
+                    continue
+            except OSError:
+                continue
+            files.append(path)
+        files.sort(key=lambda item: item.relative_to(self.root).as_posix())
+        return files
+
+    @staticmethod
+    def _digest(path: Path) -> str:
+        hasher = hashlib.blake2b(digest_size=20)
+        with path.open("rb") as handle:
+            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                hasher.update(chunk)
+        return hasher.hexdigest()
+
+    def _load(self) -> dict[str, FileState]:
+        try:
+            payload = json.loads(self.state_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            return {}
+        if payload.get("version") != self.VERSION:
+            return {}
+        result: dict[str, FileState] = {}
+        for name, value in payload.get("files", {}).items():
+            try:
+                result[name] = FileState(
+                    digest=str(value["digest"]),
+                    size=int(value["size"]),
+                    mtime_ns=int(value["mtime_ns"]),
+                )
+            except (KeyError, TypeError, ValueError):
+                continue
+        return result
+
+    def _save(self, files: dict[str, FileState]) -> None:
+        self.state_path.parent.mkdir(parents=True, exist_ok=True)
+        payload: dict[str, Any] = {
+            "version": self.VERSION,
+            "root": str(self.root),
+            "files": {name: asdict(state) for name, state in sorted(files.items())},
+        }
+        temp_path = self.state_path.with_suffix(".tmp")
+        temp_path.write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        os.replace(temp_path, self.state_path)
+
+    def refresh(self) -> IndexStats:
+        started = time.perf_counter()
+        previous = self._load()
+        current: dict[str, FileState] = {}
+        added: list[str] = []
+        changed: list[str] = []
+        unchanged = 0
+
+        for path in self._iter_files():
+            relative = path.relative_to(self.root).as_posix()
+            try:
+                stat = path.stat()
+            except OSError:
+                continue
+            old = previous.get(relative)
+            if old and old.size == stat.st_size and old.mtime_ns == stat.st_mtime_ns:
+                current[relative] = old
+                unchanged += 1
+                continue
+            try:
+                state = FileState(
+                    digest=self._digest(path),
+                    size=stat.st_size,
+                    mtime_ns=stat.st_mtime_ns,
+                )
+            except OSError:
+                continue
+            current[relative] = state
+            if old is None:
+                added.append(relative)
+            elif old.digest != state.digest:
+                changed.append(relative)
+            else:
+                unchanged += 1
+
+        removed = sorted(set(previous) - set(current))
+        self._save(current)
+        return IndexStats(
+            tracked=len(current),
+            added=tuple(added),
+            changed=tuple(changed),
+            removed=tuple(removed),
+            unchanged=unchanged,
+            duration_ms=(time.perf_counter() - started) * 1000,
+        )

codecortex/indexing/incremental_graph.py

@@ -0,0 +1,189 @@
+"""Incremental graph maintenance without full repository rebuilds."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from codecortex.indexing.graph import GraphEdge, GraphNode, ProjectGraph
+from codecortex.indexing.incremental import IncrementalIndex, IndexStats
+from codecortex.indexing.indexer import ProjectIndexer
+from codecortex.indexing.relationships import RelationshipExtractor
+from codecortex.indexing.resolution import CrossFileResolver
+from codecortex.symbols import SymbolProviderRegistry
+
+
+@dataclass(frozen=True, slots=True)
+class GraphUpdateStats:
+    index: IndexStats
+    full_rebuild: bool
+    nodes_before: int
+    nodes_after: int
+    edges_before: int
+    edges_after: int
+    files_reparsed: int
+
+
+class IncrementalGraphIndex:
+    """Persist and patch graph fragments for files reported dirty by the manifest."""
+
+    def __init__(self, root: Path, graph_path: Path | None = None) -> None:
+        self.root = root.resolve()
+        self.graph_path = graph_path or self.root / ".codecortex" / "index" / "graph.json"
+        self.manifest = IncrementalIndex(self.root)
+        self.symbols = SymbolProviderRegistry()
+        self.relationships = RelationshipExtractor()
+        self.resolver = CrossFileResolver()
+
+    def refresh(self) -> tuple[ProjectGraph, GraphUpdateStats]:
+        index_stats = self.manifest.refresh()
+        previous = ProjectGraph.load(self.graph_path)
+        before_nodes = len(previous.nodes)
+        before_edges = len(previous.edges)
+        if not previous.nodes:
+            graph = ProjectIndexer(self.root).build()
+            graph.save(self.graph_path)
+            return graph, GraphUpdateStats(
+                index=index_stats,
+                full_rebuild=True,
+                nodes_before=before_nodes,
+                nodes_after=len(graph.nodes),
+                edges_before=before_edges,
+                edges_after=len(graph.edges),
+                files_reparsed=index_stats.tracked,
+            )
+        dirty = set(index_stats.added) | set(index_stats.changed) | set(index_stats.removed)
+        if not dirty:
+            return previous, GraphUpdateStats(
+                index=index_stats,
+                full_rebuild=False,
+                nodes_before=before_nodes,
+                nodes_after=before_nodes,
+                edges_before=before_edges,
+                edges_after=before_edges,
+                files_reparsed=0,
+            )
+
+        removed_ids = {node.id for node in previous.nodes if node.path in dirty}
+        retained_nodes = [node for node in previous.nodes if node.id not in removed_ids]
+        retained_edges = [
+            edge
+            for edge in previous.edges
+            if edge.source not in removed_ids and edge.target not in removed_ids
+        ]
+        new_nodes, new_edges = self._fragments(sorted(set(index_stats.added) | set(index_stats.changed)), retained_nodes)
+        node_map = {node.id: node for node in retained_nodes}
+        node_map.update({node.id: node for node in new_nodes})
+        edge_map = {
+            (edge.source, edge.target, edge.kind): edge
+            for edge in retained_edges + new_edges
+            if edge.source != edge.target
+        }
+        graph = ProjectGraph(nodes=list(node_map.values()), edges=list(edge_map.values()))
+        graph.save(self.graph_path)
+        return graph, GraphUpdateStats(
+            index=index_stats,
+            full_rebuild=False,
+            nodes_before=before_nodes,
+            nodes_after=len(graph.nodes),
+            edges_before=before_edges,
+            edges_after=len(graph.edges),
+            files_reparsed=len(index_stats.added) + len(index_stats.changed),
+        )
+
+    def _fragments(
+        self,
+        paths: list[str],
+        retained_nodes: list[GraphNode],
+    ) -> tuple[list[GraphNode], list[GraphEdge]]:
+        nodes: list[GraphNode] = []
+        edges: list[GraphEdge] = []
+        names: dict[str, list[GraphNode]] = {}
+        for node in retained_nodes:
+            if node.kind not in {"file", "module", "reference"}:
+                names.setdefault(node.name, []).append(node)
+        sources: dict[str, str] = {}
+        local_symbols: dict[tuple[str, str], str] = {}
+
+        for relative in paths:
+            path = self.root / relative
+            if not path.is_file():
+                continue
+            file_id = f"file:{relative}"
+            nodes.append(
+                GraphNode(
+                    id=file_id,
+                    kind="file",
+                    name=path.name,
+                    path=relative,
+                    metadata={"extension": path.suffix.lower()},
+                )
+            )
+            if not self.symbols.supports(path):
+                continue
+            try:
+                source = path.read_text(encoding="utf-8")
+            except (OSError, UnicodeDecodeError):
+                continue
+            sources[relative] = source
+            for symbol in self.symbols.extract(path, source):
+                if symbol.kind in {"import", "export"}:
+                    continue
+                node = GraphNode(
+                    id=f"symbol:{relative}:{symbol.line}:{symbol.kind}:{symbol.name}",
+                    kind=symbol.kind,
+                    name=symbol.name,
+                    path=relative,
+                    line=symbol.line,
+                    metadata={"language": symbol.language, "container": symbol.container},
+                )
+                nodes.append(node)
+                names.setdefault(node.name, []).append(node)
+                local_symbols[(relative, node.name)] = node.id
+                edges.extend(
+                    [
+                        GraphEdge(source=file_id, target=node.id, kind="contains"),
+                        GraphEdge(source=file_id, target=node.id, kind="defines"),
+                    ]
+                )
+
+        existing_ids = {node.id for node in retained_nodes + nodes}
+        for relative, source in sources.items():
+            file_id = f"file:{relative}"
+            path = self.root / relative
+            for relation in self.relationships.extract(path, source):
+                source_id = local_symbols.get((relative, relation.source_symbol or ""), file_id)
+                resolution = self.resolver.resolve(
+                    relation.target,
+                    relative,
+                    names.get(relation.target, []),
+                    relation.kind,
+                )
+                if resolution.target_id:
+                    target_id = resolution.target_id
+                    metadata: dict[str, object] = {
+                        "resolution_confidence": round(resolution.confidence, 4),
+                        "ambiguity": round(resolution.ambiguity, 4),
+                        "candidate_count": len(resolution.candidates),
+                    }
+                else:
+                    prefix = "module" if relation.kind == "imports" else "reference"
+                    target_id = f"{prefix}:{relation.target}"
+                    metadata = {
+                        "resolution_confidence": 0.0,
+                        "ambiguity": 1.0,
+                        "candidate_count": 0,
+                    }
+                    if target_id not in existing_ids:
+                        existing_ids.add(target_id)
+                        nodes.append(GraphNode(id=target_id, kind=prefix, name=relation.target))
+                metadata["line"] = relation.line
+                edges.append(
+                    GraphEdge(
+                        source=source_id,
+                        target=target_id,
+                        kind=relation.kind,
+                        metadata=metadata,
+                    )
+                )
+        return nodes, edges

codecortex/indexing/indexer.py

@@ -0,0 +1,172 @@
+"""Build the repository knowledge graph."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from codecortex.indexing.graph import GraphEdge, GraphNode, ProjectGraph
+from codecortex.indexing.relationships import RelationshipExtractor
+from codecortex.indexing.resolution import CrossFileResolver
+from codecortex.symbols import SymbolProviderRegistry
+
+_EXCLUDED = {
+    ".git",
+    ".codecortex",
+    ".venv",
+    "venv",
+    "node_modules",
+    "dist",
+    "build",
+    "__pycache__",
+}
+
+
+class ProjectIndexer:
+    def __init__(self, root: Path, max_files: int = 5_000) -> None:
+        self.root = root.resolve()
+        self.max_files = max_files
+        self.symbols = SymbolProviderRegistry()
+        self.relationships = RelationshipExtractor()
+        self.resolver = CrossFileResolver()
+
+    def _files(self) -> list[Path]:
+        files: list[Path] = []
+        for path in self.root.rglob("*"):
+            if len(files) >= self.max_files:
+                break
+            if not path.is_file():
+                continue
+            relative = path.relative_to(self.root)
+            if any(part in _EXCLUDED for part in relative.parts):
+                continue
+            files.append(path)
+        return sorted(files)
+
+    def build(self) -> ProjectGraph:
+        files = self._files()
+        nodes: list[GraphNode] = []
+        edges: list[GraphEdge] = []
+        node_ids: set[str] = set()
+        names: dict[str, list[GraphNode]] = {}
+        file_sources: dict[Path, str] = {}
+        symbol_for_file: dict[tuple[str, str], str] = {}
+
+        for path in files:
+            relative = path.relative_to(self.root)
+            file_id = f"file:{relative.as_posix()}"
+            self._node(
+                nodes,
+                node_ids,
+                GraphNode(
+                    id=file_id,
+                    kind="file",
+                    name=relative.name,
+                    path=relative.as_posix(),
+                    metadata={"extension": relative.suffix.lower()},
+                ),
+            )
+            if not self.symbols.supports(path):
+                continue
+            try:
+                source = path.read_text(encoding="utf-8")
+            except (OSError, UnicodeDecodeError):
+                continue
+            file_sources[path] = source
+            for symbol in self.symbols.extract(path, source):
+                if symbol.kind in {"import", "export"}:
+                    continue
+                symbol_id = f"symbol:{relative.as_posix()}:{symbol.line}:{symbol.kind}:{symbol.name}"
+                node = GraphNode(
+                    id=symbol_id,
+                    kind=symbol.kind,
+                    name=symbol.name,
+                    path=relative.as_posix(),
+                    line=symbol.line,
+                    metadata={
+                        "language": symbol.language,
+                        "container": symbol.container,
+                    },
+                )
+                self._node(nodes, node_ids, node)
+                edges.append(GraphEdge(source=file_id, target=symbol_id, kind="contains"))
+                edges.append(GraphEdge(source=file_id, target=symbol_id, kind="defines"))
+                names.setdefault(symbol.name, []).append(node)
+                symbol_for_file[(relative.as_posix(), symbol.name)] = symbol_id
+
+        for path, source in file_sources.items():
+            relative = path.relative_to(self.root)
+            source_path = relative.as_posix()
+            file_id = f"file:{source_path}"
+            for relation in self.relationships.extract(path, source):
+                source_id = file_id
+                if relation.source_symbol:
+                    source_id = symbol_for_file.get(
+                        (source_path, relation.source_symbol),
+                        file_id,
+                    )
+                target_id, metadata = self._resolve_target(
+                    relation.target,
+                    relation.kind,
+                    source_path,
+                    names,
+                    nodes,
+                    node_ids,
+                )
+                metadata["line"] = relation.line
+                edges.append(
+                    GraphEdge(
+                        source=source_id,
+                        target=target_id,
+                        kind=relation.kind,
+                        metadata=metadata,
+                    )
+                )
+
+        unique_edges = {
+            (edge.source, edge.target, edge.kind): edge
+            for edge in edges
+            if edge.source != edge.target
+        }
+        return ProjectGraph(nodes=nodes, edges=list(unique_edges.values()))
+
+    @staticmethod
+    def _node(nodes: list[GraphNode], node_ids: set[str], node: GraphNode) -> None:
+        if node.id not in node_ids:
+            node_ids.add(node.id)
+            nodes.append(node)
+
+    def _resolve_target(
+        self,
+        target: str,
+        kind: str,
+        source_path: str,
+        names: dict[str, list[GraphNode]],
+        nodes: list[GraphNode],
+        node_ids: set[str],
+    ) -> tuple[str, dict[str, object]]:
+        candidates = names.get(target, [])
+        result = self.resolver.resolve(target, source_path, candidates, kind)
+        if result.target_id is not None:
+            return result.target_id, {
+                "resolution_confidence": round(result.confidence, 4),
+                "ambiguity": round(result.ambiguity, 4),
+                "candidate_count": len(result.candidates),
+                "candidates": [
+                    {
+                        "id": item.node_id,
+                        "score": round(item.score, 4),
+                        "reasons": list(item.reasons),
+                    }
+                    for item in result.candidates
+                ],
+            }
+        prefix = "module" if kind == "imports" else "reference"
+        target_id = f"{prefix}:{target}"
+        if target_id not in node_ids:
+            node_ids.add(target_id)
+            nodes.append(GraphNode(id=target_id, kind=prefix, name=target))
+        return target_id, {
+            "resolution_confidence": 0.0,
+            "ambiguity": 1.0,
+            "candidate_count": 0,
+        }

codecortex/indexing/relationships.py

@@ -0,0 +1,179 @@
+"""Extract dependency and call relationships from source files."""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True, slots=True)
+class Relationship:
+    kind: str
+    target: str
+    line: int
+    source_symbol: str | None = None
+
+
+class RelationshipExtractor:
+    _CALL_RE = re.compile(r"\b([A-Za-z_$][\w$]*)\s*\(")
+    _JS_IMPORT_RE = re.compile(
+        r"^\s*(?:import\s+(?:[^\n]+?\s+from\s+)?|require\s*\()[\"']([^\"']+)",
+        re.MULTILINE,
+    )
+    _GO_IMPORT_RE = re.compile(r"^\s*import\s+[\"']([^\"']+)[\"']", re.MULTILINE)
+    _RUST_USE_RE = re.compile(r"^\s*use\s+([^;]+);", re.MULTILINE)
+    _JVM_IMPORT_RE = re.compile(r"^\s*import\s+([A-Za-z_][\w.*]+)\s*;", re.MULTILINE)
+    _C_INCLUDE_RE = re.compile(r"^\s*#\s*include\s*[<\"]([^>\"]+)[>\"]", re.MULTILINE)
+    _PHP_USE_RE = re.compile(r"^\s*use\s+([^;]+);", re.MULTILINE)
+    _RUBY_REQUIRE_RE = re.compile(
+        r"^\s*require(?:_relative)?\s+[\"']([^\"']+)[\"']",
+        re.MULTILINE,
+    )
+    _EXTENDS_RE = re.compile(
+        r"\bclass\s+([A-Za-z_$]\w*)\s+extends\s+([A-Za-z_$]\w*)",
+        re.MULTILINE,
+    )
+    _IMPLEMENTS_RE = re.compile(
+        r"\bclass\s+([A-Za-z_$]\w*)[^\n{]*\bimplements\s+([A-Za-z_$][\w$.,\s]*)",
+        re.MULTILINE,
+    )
+    _CALL_KEYWORDS = {
+        "if",
+        "for",
+        "while",
+        "switch",
+        "catch",
+        "return",
+        "sizeof",
+        "typeof",
+        "function",
+        "def",
+        "class",
+        "new",
+    }
+
+    def extract(self, path: Path, source: str) -> list[Relationship]:
+        if path.suffix.lower() == ".py":
+            return self._python(source)
+        result = self._imports(path, source)
+        result.extend(self._inheritance(source))
+        result.extend(self._calls(source))
+        return self._dedupe(result)
+
+    @staticmethod
+    def _python(source: str) -> list[Relationship]:
+        try:
+            tree = ast.parse(source)
+        except SyntaxError:
+            return []
+        result: list[Relationship] = []
+        containers: list[str] = []
+
+        class Visitor(ast.NodeVisitor):
+            def visit_ClassDef(self, node: ast.ClassDef) -> None:
+                for base in node.bases:
+                    name = RelationshipExtractor._python_name(base)
+                    if name:
+                        result.append(Relationship("inherits", name, node.lineno, node.name))
+                containers.append(node.name)
+                self.generic_visit(node)
+                containers.pop()
+
+            def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+                containers.append(node.name)
+                self.generic_visit(node)
+                containers.pop()
+
+            visit_AsyncFunctionDef = visit_FunctionDef
+
+            def visit_Import(self, node: ast.Import) -> None:
+                for alias in node.names:
+                    result.append(Relationship("imports", alias.name, node.lineno))
+
+            def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+                if node.module:
+                    result.append(Relationship("imports", node.module, node.lineno))
+
+            def visit_Call(self, node: ast.Call) -> None:
+                name = RelationshipExtractor._python_name(node.func)
+                if name:
+                    result.append(
+                        Relationship(
+                            "calls",
+                            name.split(".")[-1],
+                            node.lineno,
+                            containers[-1] if containers else None,
+                        )
+                    )
+                self.generic_visit(node)
+
+        Visitor().visit(tree)
+        return RelationshipExtractor._dedupe(result)
+
+    @staticmethod
+    def _python_name(node: ast.AST) -> str | None:
+        if isinstance(node, ast.Name):
+            return node.id
+        if isinstance(node, ast.Attribute):
+            base = RelationshipExtractor._python_name(node.value)
+            return f"{base}.{node.attr}" if base else node.attr
+        return None
+
+    def _imports(self, path: Path, source: str) -> list[Relationship]:
+        suffix = path.suffix.lower()
+        pattern: re.Pattern[str] | None = None
+        if suffix in {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts"}:
+            pattern = self._JS_IMPORT_RE
+        elif suffix == ".go":
+            pattern = self._GO_IMPORT_RE
+        elif suffix == ".rs":
+            pattern = self._RUST_USE_RE
+        elif suffix in {".java", ".cs"}:
+            pattern = self._JVM_IMPORT_RE
+        elif suffix in {".c", ".h", ".cc", ".cpp", ".cxx", ".hpp"}:
+            pattern = self._C_INCLUDE_RE
+        elif suffix == ".php":
+            pattern = self._PHP_USE_RE
+        elif suffix == ".rb":
+            pattern = self._RUBY_REQUIRE_RE
+        if pattern is None:
+            return []
+        return [
+            Relationship("imports", match.group(1).strip(), source.count("\n", 0, match.start()) + 1)
+            for match in pattern.finditer(source)
+        ]
+
+    def _inheritance(self, source: str) -> list[Relationship]:
+        result = [
+            Relationship("inherits", match.group(2), source.count("\n", 0, match.start()) + 1, match.group(1))
+            for match in self._EXTENDS_RE.finditer(source)
+        ]
+        for match in self._IMPLEMENTS_RE.finditer(source):
+            line = source.count("\n", 0, match.start()) + 1
+            for target in match.group(2).split(","):
+                target = target.strip()
+                if target:
+                    result.append(Relationship("implements", target, line, match.group(1)))
+        return result
+
+    def _calls(self, source: str) -> list[Relationship]:
+        result: list[Relationship] = []
+        for match in self._CALL_RE.finditer(source):
+            name = match.group(1)
+            if name in self._CALL_KEYWORDS:
+                continue
+            result.append(Relationship("calls", name, source.count("\n", 0, match.start()) + 1))
+        return result
+
+    @staticmethod
+    def _dedupe(items: list[Relationship]) -> list[Relationship]:
+        seen: set[tuple[str, str, int, str | None]] = set()
+        result: list[Relationship] = []
+        for item in items:
+            key = (item.kind, item.target, item.line, item.source_symbol)
+            if key not in seen:
+                seen.add(key)
+                result.append(item)
+        return result