codecortex-context-engine 0.1.0a1__py3-none-any.whl

codecortex/pr_intelligence.py

@@ -0,0 +1,214 @@
+"""Pull-request change intelligence from Git diffs and the repository graph."""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from codecortex.indexing.graph import GraphNode, ProjectGraph
+from codecortex.indexing.impact import ImpactAnalyzer
+from codecortex.languages import LanguageRegistry
+
+_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(?P<start>\d+)(?:,(?P<count>\d+))? @@")
+
+
+@dataclass(frozen=True, slots=True)
+class LineRange:
+    start: int
+    end: int
+
+    def contains(self, line: int | None) -> bool:
+        return line is not None and self.start <= line <= self.end
+
+    def overlaps(self, other: LineRange) -> bool:
+        return self.start <= other.end and other.start <= self.end
+
+
+@dataclass(frozen=True, slots=True)
+class PRFileChange:
+    path: str
+    status: str
+    additions: int
+    deletions: int
+    changed_ranges: tuple[LineRange, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class ChangedSymbol:
+    node: GraphNode
+    direct_change: bool
+    impact_risk: float
+    affected_nodes: int
+    affected_tests: int
+
+
+@dataclass(frozen=True, slots=True)
+class PRReport:
+    base_ref: str
+    head_ref: str
+    files: tuple[PRFileChange, ...]
+    symbols: tuple[ChangedSymbol, ...]
+    affected_tests: tuple[str, ...]
+    risk_score: float
+    risk_level: str
+
+
+class PRIntelligence:
+    def __init__(self, root: Path, graph: ProjectGraph, timeout_seconds: float = 15.0) -> None:
+        self.root = root.resolve()
+        self.graph = graph
+        self.timeout_seconds = timeout_seconds
+        self.languages = LanguageRegistry()
+
+    def _git(self, *args: str) -> str:
+        try:
+            result = subprocess.run(
+                ["git", "-C", str(self.root), *args],
+                check=False,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=self.timeout_seconds,
+            )
+        except (OSError, subprocess.TimeoutExpired):
+            return ""
+        return result.stdout if result.returncode == 0 else ""
+
+    def analyze(self, base_ref: str, head_ref: str = "HEAD") -> PRReport:
+        files = self._file_changes(base_ref, head_ref)
+        changed_by_path = {change.path: change for change in files}
+        symbols: list[ChangedSymbol] = []
+        tests: set[str] = set()
+        analyzer = ImpactAnalyzer(self.graph)
+        ranges_by_path: dict[str, dict[tuple[str, int], LineRange]] = {}
+
+        for node in self.graph.nodes:
+            if not node.path or node.path not in changed_by_path:
+                continue
+            if node.kind in {"file", "module", "reference"} or node.line is None:
+                continue
+            change = changed_by_path[node.path]
+            symbol_ranges = ranges_by_path.setdefault(
+                node.path,
+                self._symbol_ranges(node.path),
+            )
+            symbol_range = symbol_ranges.get(
+                (node.name, node.line),
+                LineRange(node.line, node.line),
+            )
+            direct = not change.changed_ranges or any(
+                symbol_range.overlaps(changed) for changed in change.changed_ranges
+            )
+            if not direct:
+                continue
+            try:
+                impact = analyzer.analyze(node.name)
+                affected_nodes = len(impact.direct) + len(impact.indirect)
+                affected_tests = len(impact.affected_tests)
+                tests.update(
+                    item.node.path or item.node.name for item in impact.affected_tests
+                )
+                risk = impact.risk_score
+            except ValueError:
+                affected_nodes = 0
+                affected_tests = 0
+                risk = 0.0
+            symbols.append(
+                ChangedSymbol(
+                    node=node,
+                    direct_change=True,
+                    impact_risk=risk,
+                    affected_nodes=affected_nodes,
+                    affected_tests=affected_tests,
+                )
+            )
+
+        churn = sum(item.additions + item.deletions for item in files)
+        max_impact = max((item.impact_risk for item in symbols), default=0.0)
+        breadth = min(1.0, len(files) / 20)
+        churn_score = min(1.0, churn / 600)
+        risk_score = min(1.0, max_impact * 0.55 + breadth * 0.25 + churn_score * 0.20)
+        if risk_score >= 0.70:
+            level = "high"
+        elif risk_score >= 0.40:
+            level = "medium"
+        else:
+            level = "low"
+        symbols.sort(key=lambda item: (-item.impact_risk, item.node.path or "", item.node.line or 0))
+        return PRReport(
+            base_ref=base_ref,
+            head_ref=head_ref,
+            files=tuple(files),
+            symbols=tuple(symbols),
+            affected_tests=tuple(sorted(tests)),
+            risk_score=round(risk_score, 4),
+            risk_level=level,
+        )
+
+    def _symbol_ranges(self, relative: str) -> dict[tuple[str, int], LineRange]:
+        path = self.root / relative
+        try:
+            source = path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            return {}
+        units = sorted(
+            self.languages.parse(path, source),
+            key=lambda item: (item.line, item.name),
+        )
+        total_lines = max(1, len(source.splitlines()))
+        ranges: dict[tuple[str, int], LineRange] = {}
+        for index, unit in enumerate(units):
+            if unit.end_line is not None:
+                end = unit.end_line
+            else:
+                next_line = units[index + 1].line if index + 1 < len(units) else total_lines + 1
+                end = max(unit.line, next_line - 1)
+            ranges[(unit.name, unit.line)] = LineRange(unit.line, end)
+        return ranges
+
+    def _file_changes(self, base_ref: str, head_ref: str) -> list[PRFileChange]:
+        range_ref = f"{base_ref}...{head_ref}"
+        numstat = self._git("diff", "--numstat", range_ref)
+        statuses = self._git("diff", "--name-status", range_ref)
+        patch = self._git("diff", "--unified=0", "--no-color", range_ref)
+
+        stats: dict[str, tuple[int, int]] = {}
+        for line in numstat.splitlines():
+            parts = line.split("\t")
+            if len(parts) >= 3:
+                additions = int(parts[0]) if parts[0].isdigit() else 0
+                deletions = int(parts[1]) if parts[1].isdigit() else 0
+                stats[parts[-1]] = (additions, deletions)
+        status_map: dict[str, str] = {}
+        for line in statuses.splitlines():
+            parts = line.split("\t")
+            if len(parts) >= 2:
+                status_map[parts[-1]] = parts[0]
+        ranges: dict[str, list[LineRange]] = {}
+        current_path: str | None = None
+        for line in patch.splitlines():
+            if line.startswith("+++ b/"):
+                current_path = line[6:]
+                ranges.setdefault(current_path, [])
+                continue
+            match = _HUNK.match(line)
+            if current_path and match:
+                start = int(match.group("start"))
+                count = int(match.group("count") or "1")
+                if count > 0:
+                    ranges[current_path].append(LineRange(start, start + count - 1))
+
+        paths = sorted(set(stats) | set(status_map) | set(ranges))
+        return [
+            PRFileChange(
+                path=path,
+                status=status_map.get(path, "M"),
+                additions=stats.get(path, (0, 0))[0],
+                deletions=stats.get(path, (0, 0))[1],
+                changed_ranges=tuple(ranges.get(path, [])),
+            )
+            for path in paths
+        ]

codecortex/retrieval/__init__.py

@@ -0,0 +1,16 @@
+"""Semantic and hybrid retrieval."""
+
+from codecortex.retrieval.hybrid import HybridRetriever, RetrievalHit
+from codecortex.retrieval.index import SemanticDocument, SemanticIndex
+from codecortex.retrieval.providers import EmbeddingProvider, FeatureHashEmbeddingProvider
+from codecortex.retrieval.repository import RepositorySemanticIndex
+
+__all__ = [
+    "EmbeddingProvider",
+    "FeatureHashEmbeddingProvider",
+    "HybridRetriever",
+    "RepositorySemanticIndex",
+    "RetrievalHit",
+    "SemanticDocument",
+    "SemanticIndex",
+]

codecortex/retrieval/hybrid.py

@@ -0,0 +1,67 @@
+"""Hybrid retrieval combining vector, lexical, and structural priors."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from codecortex.retrieval.index import SemanticDocument, SemanticIndex
+
+
+@dataclass(frozen=True, slots=True)
+class RetrievalHit:
+    document: SemanticDocument
+    score: float
+    vector_score: float
+    lexical_score: float
+    structural_score: float
+
+
+class HybridRetriever:
+    def __init__(
+        self,
+        index: SemanticIndex,
+        vector_weight: float = 0.60,
+        lexical_weight: float = 0.30,
+        structural_weight: float = 0.10,
+    ) -> None:
+        total = vector_weight + lexical_weight + structural_weight
+        if total <= 0:
+            raise ValueError("retrieval weights must sum to a positive value")
+        self.index = index
+        self.vector_weight = vector_weight / total
+        self.lexical_weight = lexical_weight / total
+        self.structural_weight = structural_weight / total
+
+    def search(self, query: str, limit: int = 20) -> list[RetrievalHit]:
+        candidates = self.index.search(query, limit=max(limit * 4, 40))
+        query_terms = {term.lower() for term in re.findall(r"[A-Za-z_][\w.-]*", query)}
+        hits: list[RetrievalHit] = []
+        for match in candidates:
+            text_terms = {term.lower() for term in re.findall(r"[A-Za-z_][\w.-]*", match.document.text)}
+            lexical = len(query_terms & text_terms) / max(1, len(query_terms))
+            metadata = match.document.metadata
+            structural = 0.0
+            path = str(metadata.get("path", "")).lower()
+            symbol = str(metadata.get("symbol", "")).lower()
+            if any(term in path for term in query_terms):
+                structural += 0.45
+            if any(term == symbol for term in query_terms):
+                structural += 0.55
+            vector_score = (match.score + 1.0) / 2.0
+            score = (
+                self.vector_weight * vector_score
+                + self.lexical_weight * lexical
+                + self.structural_weight * min(1.0, structural)
+            )
+            hits.append(
+                RetrievalHit(
+                    document=match.document,
+                    score=score,
+                    vector_score=vector_score,
+                    lexical_score=lexical,
+                    structural_score=min(1.0, structural),
+                )
+            )
+        hits.sort(key=lambda item: (-item.score, item.document.id))
+        return hits[:limit]

codecortex/retrieval/index.py

@@ -0,0 +1,135 @@
+"""Persistent semantic vector index."""
+
+from __future__ import annotations
+
+import json
+import math
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from codecortex.retrieval.providers import EmbeddingProvider
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticDocument:
+    id: str
+    text: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticMatch:
+    document: SemanticDocument
+    score: float
+
+
+class SemanticIndex:
+    VERSION = 1
+
+    def __init__(self, provider: EmbeddingProvider, path: Path | None = None) -> None:
+        self.provider = provider
+        self.path = path
+        self._documents: dict[str, SemanticDocument] = {}
+        self._vectors: dict[str, list[float]] = {}
+        if path and path.exists():
+            self.load()
+
+    @property
+    def document_ids(self) -> set[str]:
+        return set(self._documents)
+
+    def upsert(self, documents: list[SemanticDocument]) -> None:
+        if not documents:
+            return
+        vectors = self.provider.embed([document.text for document in documents])
+        for document, vector in zip(documents, vectors, strict=True):
+            self._documents[document.id] = document
+            self._vectors[document.id] = vector
+        if self.path:
+            self.save()
+
+    def replace(self, documents: list[SemanticDocument]) -> None:
+        self._documents.clear()
+        self._vectors.clear()
+        self.upsert(documents)
+        if not documents and self.path:
+            self.save()
+
+    def delete(self, ids: set[str]) -> None:
+        for document_id in ids:
+            self._documents.pop(document_id, None)
+            self._vectors.pop(document_id, None)
+        if self.path:
+            self.save()
+
+    def search(self, query: str, limit: int = 20, min_score: float = -1.0) -> list[SemanticMatch]:
+        if not self._documents:
+            return []
+        query_vector = self.provider.embed([query])[0]
+        ranked: list[tuple[float, str]] = []
+        for document_id, vector in self._vectors.items():
+            score = self._cosine(query_vector, vector)
+            if score >= min_score:
+                ranked.append((score, document_id))
+        ranked.sort(key=lambda item: (-item[0], item[1]))
+        return [
+            SemanticMatch(document=self._documents[document_id], score=score)
+            for score, document_id in ranked[:limit]
+        ]
+
+    def save(self) -> None:
+        if self.path is None:
+            return
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "version": self.VERSION,
+            "provider": self.provider.name,
+            "dimensions": self.provider.dimensions,
+            "documents": {
+                document_id: asdict(document)
+                for document_id, document in sorted(self._documents.items())
+            },
+            "vectors": self._vectors,
+        }
+        temp = self.path.with_suffix(self.path.suffix + ".tmp")
+        temp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+        temp.replace(self.path)
+
+    def load(self) -> None:
+        if self.path is None:
+            return
+        try:
+            payload = json.loads(self.path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            return
+        if payload.get("version") != self.VERSION:
+            return
+        if payload.get("provider") != self.provider.name:
+            return
+        if int(payload.get("dimensions", -1)) != self.provider.dimensions:
+            return
+        documents = payload.get("documents", {})
+        vectors = payload.get("vectors", {})
+        self._documents = {
+            document_id: SemanticDocument(
+                id=str(value["id"]),
+                text=str(value["text"]),
+                metadata=dict(value.get("metadata", {})),
+            )
+            for document_id, value in documents.items()
+        }
+        self._vectors = {
+            document_id: [float(value) for value in vector]
+            for document_id, vector in vectors.items()
+            if document_id in self._documents
+        }
+
+    @staticmethod
+    def _cosine(left: list[float], right: list[float]) -> float:
+        if len(left) != len(right):
+            return -1.0
+        dot = sum(a * b for a, b in zip(left, right, strict=True))
+        left_norm = math.sqrt(sum(value * value for value in left)) or 1.0
+        right_norm = math.sqrt(sum(value * value for value in right)) or 1.0
+        return dot / (left_norm * right_norm)

codecortex/retrieval/providers.py

@@ -0,0 +1,67 @@
+"""Embedding provider contracts and local/optional implementations."""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+from typing import Protocol
+
+_TOKEN = re.compile(r"[A-Za-z_][A-Za-z0-9_.$:/-]*")
+
+
+class EmbeddingProvider(Protocol):
+    name: str
+    dimensions: int
+
+    def embed(self, texts: list[str]) -> list[list[float]]: ...
+
+
+class FeatureHashEmbeddingProvider:
+    """Deterministic zero-network fallback suitable for local code retrieval."""
+
+    name = "feature-hash-v1"
+
+    def __init__(self, dimensions: int = 384) -> None:
+        if dimensions < 64:
+            raise ValueError("dimensions must be >= 64")
+        self.dimensions = dimensions
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        return [self._embed_one(text) for text in texts]
+
+    def _embed_one(self, text: str) -> list[float]:
+        vector = [0.0] * self.dimensions
+        tokens = [token.lower() for token in _TOKEN.findall(text)]
+        features = tokens + [
+            f"{left}::{right}"
+            for left, right in zip(tokens, tokens[1:], strict=False)
+        ]
+        for feature in features:
+            digest = hashlib.blake2b(feature.encode("utf-8"), digest_size=8).digest()
+            raw = int.from_bytes(digest, "little")
+            index = raw % self.dimensions
+            sign = -1.0 if raw & 1 else 1.0
+            vector[index] += sign
+        norm = math.sqrt(sum(value * value for value in vector)) or 1.0
+        return [value / norm for value in vector]
+
+
+class SentenceTransformerEmbeddingProvider:
+    """Optional local neural provider loaded only when the semantic extra is installed."""
+
+    def __init__(self, model: str = "sentence-transformers/all-MiniLM-L6-v2") -> None:
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError as exc:
+            raise RuntimeError(
+                "Install CodeCortex with the 'semantic' extra to use neural embeddings."
+            ) from exc
+        self._model = SentenceTransformer(model)
+        self.name = f"sentence-transformer:{model}"
+        probe = self._model.encode(["codecortex"], normalize_embeddings=True)
+        self.dimensions = int(len(probe[0]))
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        vectors = self._model.encode(texts, normalize_embeddings=True)
+        return [[float(value) for value in row] for row in vectors]

codecortex/retrieval/repository.py

@@ -0,0 +1,94 @@
+"""Repository-to-semantic-index ingestion."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from codecortex.indexing.graph import ProjectGraph
+from codecortex.indexing.incremental_graph import IncrementalGraphIndex
+from codecortex.retrieval.hybrid import HybridRetriever, RetrievalHit
+from codecortex.retrieval.index import SemanticDocument, SemanticIndex
+from codecortex.retrieval.providers import EmbeddingProvider, FeatureHashEmbeddingProvider
+
+
+class RepositorySemanticIndex:
+    def __init__(
+        self,
+        root: Path,
+        provider: EmbeddingProvider | None = None,
+        index_path: Path | None = None,
+        max_snippet_chars: int = 3_000,
+    ) -> None:
+        self.root = root.resolve()
+        self.provider = provider or FeatureHashEmbeddingProvider()
+        self.index_path = index_path or self.root / ".codecortex" / "index" / "semantic.json"
+        self.index = SemanticIndex(self.provider, self.index_path)
+        self.max_snippet_chars = max_snippet_chars
+
+    def refresh(self, graph: ProjectGraph | None = None) -> int:
+        graph = graph or IncrementalGraphIndex(self.root).refresh()[0]
+        documents = [self._document(node, graph) for node in graph.nodes]
+        filtered = [document for document in documents if document is not None]
+        self.index.replace(filtered)
+        return len(filtered)
+
+    def search(self, query: str, limit: int = 20) -> list[RetrievalHit]:
+        if not self.index.document_ids:
+            self.refresh()
+        return HybridRetriever(self.index).search(query, limit)
+
+    def _document(self, node, graph: ProjectGraph) -> SemanticDocument | None:
+        if node.kind in {"module", "reference"}:
+            return None
+        metadata = {
+            "path": node.path or "",
+            "symbol": node.name if node.kind != "file" else "",
+            "kind": node.kind,
+            "line": node.line or 0,
+        }
+        structural = self._structural_context(node.id, graph)
+        if node.path:
+            path = self.root / node.path
+            snippet = self._snippet(path, node.line)
+        else:
+            snippet = ""
+        text = "\n".join(
+            part
+            for part in (
+                f"{node.kind} {node.name}",
+                f"path {node.path}" if node.path else "",
+                structural,
+                snippet,
+            )
+            if part
+        )
+        return SemanticDocument(id=node.id, text=text, metadata=metadata)
+
+    def _snippet(self, path: Path, line: int | None) -> str:
+        try:
+            source = path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            return ""
+        if line is None:
+            return source[: self.max_snippet_chars]
+        lines = source.splitlines()
+        start = max(0, line - 8)
+        end = min(len(lines), line + 20)
+        return "\n".join(lines[start:end])[: self.max_snippet_chars]
+
+    @staticmethod
+    def _structural_context(node_id: str, graph: ProjectGraph) -> str:
+        relations: list[str] = []
+        node_map = {node.id: node for node in graph.nodes}
+        for edge in graph.edges:
+            if edge.source == node_id:
+                target = node_map.get(edge.target)
+                if target:
+                    relations.append(f"{edge.kind} {target.name}")
+            elif edge.target == node_id:
+                source = node_map.get(edge.source)
+                if source:
+                    relations.append(f"used-by {source.name} via {edge.kind}")
+            if len(relations) >= 20:
+                break
+        return "\n".join(relations)

codecortex/router/__init__.py

@@ -0,0 +1,5 @@
+"""Adaptive request routing."""
+
+from codecortex.router.router import AdaptiveRouter
+
+__all__ = ["AdaptiveRouter"]

codecortex/router/router.py

@@ -0,0 +1,79 @@
+"""Deterministic adaptive router used by the orchestration core."""
+
+from __future__ import annotations
+
+from codecortex.core.models import AgentRequest, Capability, RequestKind, RoutePlan, RouteScore
+
+
+class AdaptiveRouter:
+    """Classify requests and produce an observable capability plan."""
+
+    _kind_terms: dict[RequestKind, tuple[str, ...]] = {
+        RequestKind.LOCATE: ("find", "where", "locate", "symbol", "definition", "reference"),
+        RequestKind.DEBUG: ("bug", "debug", "error", "fail", "crash", "race", "trace"),
+        RequestKind.REFACTOR: ("refactor", "rename", "extract", "move", "restructure"),
+        RequestKind.CHANGE: ("change", "edit", "implement", "add", "remove", "fix"),
+        RequestKind.REVIEW: ("review", "risk", "regression", "security", "inspect"),
+        RequestKind.EXPLAIN: ("explain", "why", "how", "architecture", "understand"),
+    }
+
+    def __init__(self, default_budget: int = 32_000) -> None:
+        self.default_budget = default_budget
+
+    def route(self, request: AgentRequest) -> RoutePlan:
+        text = request.query.lower()
+        kind = request.kind if request.kind != RequestKind.UNKNOWN else self._classify(text)
+
+        scores = {
+            Capability.REPOSITORY: 0.35,
+            Capability.SYMBOLS: 0.30,
+            Capability.CONTEXT: 0.45,
+            Capability.MEMORY: 0.20,
+            Capability.VALIDATION: 0.10,
+        }
+
+        if kind in {RequestKind.EXPLAIN, RequestKind.DEBUG, RequestKind.REVIEW}:
+            scores[Capability.REPOSITORY] += 0.45
+        if kind in {RequestKind.LOCATE, RequestKind.DEBUG, RequestKind.REFACTOR, RequestKind.CHANGE}:
+            scores[Capability.SYMBOLS] += 0.50
+        if kind in {RequestKind.DEBUG, RequestKind.REFACTOR, RequestKind.CHANGE, RequestKind.REVIEW}:
+            scores[Capability.VALIDATION] += 0.55
+        if any(term in text for term in ("history", "previous", "decision", "remember", "again")):
+            scores[Capability.MEMORY] += 0.65
+        if any(term in text for term in ("large", "logs", "context", "tokens", "many files")):
+            scores[Capability.CONTEXT] += 0.35
+
+        ranked = sorted(scores.items(), key=lambda item: item[1], reverse=True)
+        route_scores = [
+            RouteScore(
+                capability=capability,
+                score=min(score, 1.0),
+                reason=self._reason(capability, kind),
+            )
+            for capability, score in ranked
+        ]
+        selected = [item.capability for item in route_scores if item.score >= 0.50]
+        if not selected:
+            selected = [Capability.REPOSITORY]
+
+        return RoutePlan(
+            request_kind=kind,
+            scores=route_scores,
+            selected=selected,
+            context_budget=self.default_budget,
+        )
+
+    def _classify(self, text: str) -> RequestKind:
+        matches: list[tuple[int, RequestKind]] = []
+        for kind, terms in self._kind_terms.items():
+            count = sum(1 for term in terms if term in text)
+            if count:
+                matches.append((count, kind))
+        if not matches:
+            return RequestKind.EXPLAIN
+        matches.sort(key=lambda item: item[0], reverse=True)
+        return matches[0][1]
+
+    @staticmethod
+    def _reason(capability: Capability, kind: RequestKind) -> str:
+        return f"{capability.value} is relevant to a {kind.value} request"