codebase-intel 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

codebase_intel/__init__.py

@@ -1,3 +1,3 @@
 """Codebase Intelligence Platform — structured context for AI coding agents."""
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"

codebase_intel/analytics/benchmark.py

@@ -17,13 +17,15 @@ numbers every time for the same repo state.
 
 from __future__ import annotations
 
+import contextlib
 import logging
 import time
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
-from codebase_intel.analytics.tracker import AnalyticsTracker
+if TYPE_CHECKING:
+    from codebase_intel.analytics.tracker import AnalyticsTracker
 
 logger = logging.getLogger(__name__)
 
@@ -137,11 +139,13 @@ class BenchmarkReport:
         """Format as a markdown table for README/reports."""
         lines = [
             f"## Benchmark: {self.repo_name}",
-            f"",
-            f"**Graph:** {self.total_files} files → {self.total_nodes} nodes, {self.total_edges} edges (built in {self.build_time_ms:.0f}ms)",
-            f"",
-            f"| Scenario | Naive Tokens | Graph Tokens | Full Tokens | Reduction | Multiplier | Decisions | Contracts |",
-            f"|---|---:|---:|---:|---:|---:|---:|---:|",
+            "",
+            f"**Graph:** {self.total_files} files → {self.total_nodes} nodes, "
+            f"{self.total_edges} edges (built in {self.build_time_ms:.0f}ms)",
+            "",
+            "| Scenario | Naive Tokens | Graph Tokens | Full Tokens | Reduction | "
+            "Multiplier | Decisions | Contracts |",
+            "|---|---:|---:|---:|---:|---:|---:|---:|",
         ]
 
         for s in self.scenarios:
@@ -207,9 +211,7 @@ class BenchmarkRunner:
 
             # Run each scenario
             for scenario in scenarios:
-                result = await self._run_scenario(
-                    scenario, config, storage, engine
-                )
+                result = await self._run_scenario(scenario, config, storage, engine)
                 report.scenarios.append(result)
 
         # Record in tracker if provided
@@ -234,7 +236,7 @@ class BenchmarkRunner:
         engine: Any,
     ) -> ScenarioResult:
         """Run a single scenario in naive, graph, and full modes."""
-        from codebase_intel.orchestrator.assembler import estimate_tokens
+        from codebase_intel.core.tokens import estimate_tokens
 
         target_paths = [self._project_root / f for f in scenario.target_files]
         existing_paths = [p for p in target_paths if p.exists()]
@@ -253,29 +255,9 @@ class BenchmarkRunner:
         )
 
         # --- NAIVE: what an agent without context tools typically reads ---
-        # Agents usually read the target file + parent dir + any file they
-        # can grep for imports. We simulate: all files in parent dir + parent's
-        # parent dir (2 levels up from each target file).
-        naive_content = ""
-        naive_files: set[Path] = set()
-        code_extensions = {".py", ".ts", ".tsx", ".js", ".jsx", ".go", ".rs", ".java", ".rb"}
-        for fp in existing_paths:
-            for ancestor_dir in [fp.parent, fp.parent.parent]:
-                if not ancestor_dir.exists():
-                    continue
-                try:
-                    for child in ancestor_dir.iterdir():
-                        if child.is_file() and child.suffix in code_extensions and child not in naive_files:
-                            naive_files.add(child)
-                            try:
-                                naive_content += child.read_text(encoding="utf-8", errors="ignore")
-                            except OSError:
-                                pass
-                except OSError:
-                    pass
-
-        result.naive_tokens = estimate_tokens(naive_content)
-        result.naive_files = len(naive_files)
+        from codebase_intel.analytics.naive import estimate_naive_read_cost
+
+        result.naive_tokens, result.naive_files = estimate_naive_read_cost(existing_paths)
 
         # --- GRAPH: use code graph for relevant files ---
         start = time.monotonic()
@@ -287,10 +269,8 @@ class BenchmarkRunner:
         for node in graph_result.nodes:
             if node.file_path not in graph_files_set and node.file_path.exists():
                 graph_files_set.add(node.file_path)
-                try:
+                with contextlib.suppress(OSError):
                     graph_content += node.file_path.read_text(encoding="utf-8", errors="ignore")
-                except OSError:
-                    pass
 
         result.graph_tokens = estimate_tokens(graph_content)
         result.graph_files = len(graph_files_set)
@@ -299,9 +279,9 @@ class BenchmarkRunner:
         start = time.monotonic()
 
         from codebase_intel.contracts.registry import ContractRegistry
+        from codebase_intel.core.types import TokenBudget
         from codebase_intel.decisions.store import DecisionStore
         from codebase_intel.orchestrator.assembler import ContextAssembler
-        from codebase_intel.core.types import TokenBudget
 
         decision_store = DecisionStore(config.decisions, self._project_root)
         contract_registry = ContractRegistry(config.contracts, self._project_root)
@@ -317,16 +297,20 @@ class BenchmarkRunner:
         assembled = await assembler.assemble(
             task_description=scenario.task_description,
             file_paths=existing_paths,
-            budget=TokenBudget(total=8_000),  # Realistic agent budget — shows how well we prioritize
+            budget=TokenBudget(
+                total=8_000
+            ),  # Realistic agent budget — shows how well we prioritize
         )
 
         result.full_assembly_ms = (time.monotonic() - start) * 1000
         result.full_tokens = assembled.total_tokens
-        result.full_files = len({
-            item.metadata.get("file_path")
-            for item in assembled.items
-            if item.metadata.get("file_path")
-        })
+        result.full_files = len(
+            {
+                item.metadata.get("file_path")
+                for item in assembled.items
+                if item.metadata.get("file_path")
+            }
+        )
         result.decisions_surfaced = sum(
             1 for item in assembled.items if item.item_type == "decision"
         )
@@ -365,42 +349,50 @@ class BenchmarkRunner:
         # Core file (most nodes — likely a models or utils file)
         if len(files) >= 1:
             core_file = files[0][0]
-            scenarios.append(BenchmarkScenario(
-                name="Core module change",
-                task_description=f"Refactor the core module at {Path(core_file).name}",
-                target_files=[core_file],
-                expected_scope="wide",
-            ))
+            scenarios.append(
+                BenchmarkScenario(
+                    name="Core module change",
+                    task_description=f"Refactor the core module at {Path(core_file).name}",
+                    target_files=[core_file],
+                    expected_scope="wide",
+                )
+            )
 
         # Mid-level file
         mid_idx = len(files) // 3
         if len(files) > mid_idx:
             mid_file = files[mid_idx][0]
-            scenarios.append(BenchmarkScenario(
-                name="Feature module change",
-                task_description=f"Add a new endpoint to {Path(mid_file).name}",
-                target_files=[mid_file],
-                expected_scope="medium",
-            ))
+            scenarios.append(
+                BenchmarkScenario(
+                    name="Feature module change",
+                    task_description=f"Add a new endpoint to {Path(mid_file).name}",
+                    target_files=[mid_file],
+                    expected_scope="medium",
+                )
+            )
 
         # Leaf file (fewest nodes)
         if len(files) >= 3:
             leaf_file = files[-1][0]
-            scenarios.append(BenchmarkScenario(
-                name="Leaf file change",
-                task_description=f"Fix a bug in {Path(leaf_file).name}",
-                target_files=[leaf_file],
-                expected_scope="narrow",
-            ))
+            scenarios.append(
+                BenchmarkScenario(
+                    name="Leaf file change",
+                    task_description=f"Fix a bug in {Path(leaf_file).name}",
+                    target_files=[leaf_file],
+                    expected_scope="narrow",
+                )
+            )
 
         # Multi-file change
         if len(files) >= 5:
             multi_files = [files[0][0], files[len(files) // 2][0]]
-            scenarios.append(BenchmarkScenario(
-                name="Cross-module refactor",
-                task_description="Refactor shared types used across multiple modules",
-                target_files=multi_files,
-                expected_scope="wide",
-            ))
+            scenarios.append(
+                BenchmarkScenario(
+                    name="Cross-module refactor",
+                    task_description="Refactor shared types used across multiple modules",
+                    target_files=multi_files,
+                    expected_scope="wide",
+                )
+            )
 
         return scenarios

codebase_intel/analytics/eval.py

@@ -0,0 +1,288 @@
+"""Intent-graph eval harness — token AND correctness A/B against the v1 baseline.
+
+analytics/benchmark.py measures v1's code-graph pipeline: naive file reads vs
+graph-targeted vs full (graph + decisions + contracts) — tokens only. It has
+no v2 intent-graph scenario and no correctness signal, so the vision's central
+claim ("token AND correctness gains") stays unmeasured. This module closes
+that gap.
+
+Methodology — for each eligible intent node (ACTIVE, level COMPONENT or
+DECISION, a `why` with enough content beyond its own title):
+1. Synthesize a query from the node's WHY text with its own TITLE terms
+   removed. This forces a real query — the harness must find the node from
+   how its reasoning is described, not by echoing its name back at it. This
+   is the same regression shape validated in test_bootstrap.py's
+   TestTenCannedQuestions, generalized to run against any repo.
+2. Run IntentBootstrapper.bootstrap(query) and check whether the expected
+   node comes back — this is the correctness signal (accuracy@1 / accuracy@3).
+3. Compare token cost: the winning citation's own token cost vs
+   estimate_naive_read_cost() over the node's anchored files — the same
+   naive-cost methodology analytics/benchmark.py uses, so the two harnesses'
+   "naive" baseline means the same thing.
+
+Edge cases:
+- Empty or sparse graph: zero cases, explicit warning, never crashes —
+  the eval must degrade gracefully like every other v2 tool
+- Node has no anchors: naive_tokens is 0 for that case, excluded from the
+  reduction/multiplier averages (can't compute a ratio against zero)
+- Reproducibility: candidates are sorted by node_id before truncating to
+  max_cases, so the same graph state always yields the same report
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from codebase_intel.analytics.naive import estimate_naive_read_cost
+from codebase_intel.core.text import extract_terms
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from codebase_intel.analytics.tracker import AnalyticsTracker
+    from codebase_intel.intentgraph.bootstrap import IntentBootstrapper
+    from codebase_intel.intentgraph.models import IntentNode
+
+DEFAULT_MAX_CASES = 10
+DEFAULT_BUDGET_TOKENS = 2000
+MIN_DISTINGUISHING_TERMS = 3
+QUERY_TERM_COUNT = 6
+
+
+@dataclass
+class IntentEvalCase:
+    """One held-out question: can bootstrap find this node from its own why?"""
+
+    node_id: str
+    node_title: str
+    query: str
+    anchor_paths: list[Path]
+
+
+@dataclass
+class IntentEvalResult:
+    """Outcome of running one case through the bootstrapper."""
+
+    node_id: str
+    node_title: str
+    query: str
+    found: bool
+    rank: int | None
+    correct: bool  # rank == 1
+    top3: bool  # rank in {1, 2, 3}
+    naive_tokens: int
+    naive_files: int
+    citation_tokens: int  # tokens of just the winning citation
+    session_tokens: int  # tokens of the whole packed bootstrap response
+    coverage: str
+
+    @property
+    def reduction_pct(self) -> float:
+        if self.naive_tokens == 0 or self.citation_tokens == 0:
+            return 0.0
+        return (1 - self.citation_tokens / self.naive_tokens) * 100
+
+    @property
+    def multiplier(self) -> float:
+        if self.citation_tokens == 0:
+            return 0.0
+        return self.naive_tokens / self.citation_tokens
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "node_id": self.node_id,
+            "node_title": self.node_title,
+            "query": self.query,
+            "found": self.found,
+            "rank": self.rank,
+            "correct": self.correct,
+            "top3": self.top3,
+            "naive_tokens": self.naive_tokens,
+            "citation_tokens": self.citation_tokens,
+            "session_tokens": self.session_tokens,
+            "reduction_pct": round(self.reduction_pct, 1),
+            "multiplier": round(self.multiplier, 1),
+            "coverage": self.coverage,
+        }
+
+
+@dataclass
+class IntentEvalReport:
+    """Complete eval report for a project."""
+
+    repo_name: str
+    repo_path: str
+    total_cases: int = 0
+    results: list[IntentEvalResult] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def accuracy_at_1(self) -> float:
+        if not self.results:
+            return 0.0
+        return sum(1 for r in self.results if r.correct) / len(self.results)
+
+    @property
+    def accuracy_at_3(self) -> float:
+        if not self.results:
+            return 0.0
+        return sum(1 for r in self.results if r.top3) / len(self.results)
+
+    @property
+    def avg_reduction_pct(self) -> float:
+        vals = [
+            r.reduction_pct for r in self.results if r.naive_tokens > 0 and r.citation_tokens > 0
+        ]
+        return sum(vals) / max(len(vals), 1)
+
+    @property
+    def avg_multiplier(self) -> float:
+        vals = [r.multiplier for r in self.results if r.citation_tokens > 0]
+        return sum(vals) / max(len(vals), 1)
+
+    def format_table(self) -> str:
+        """Markdown table for README/reports."""
+        lines = [
+            f"## Intent eval: {self.repo_name}",
+            "",
+            f"**Cases:** {self.total_cases} · **Accuracy@1:** {self.accuracy_at_1:.0%} · "
+            f"**Accuracy@3:** {self.accuracy_at_3:.0%}",
+            "",
+        ]
+        if not self.results:
+            lines.append(f"_{self.warnings[0] if self.warnings else 'No eligible cases.'}_")
+            return "\n".join(lines)
+
+        lines.extend(
+            [
+                "| Query (from why, title-terms held out) | Found | Rank | Naive | "
+                "Citation | Reduction |",
+                "|---|---|---:|---:|---:|---:|",
+            ]
+        )
+        for r in self.results:
+            found_mark = "✓" if r.correct else ("~" if r.found else "✗")
+            lines.append(
+                f"| {r.query[:60]} | {found_mark} | {r.rank or '-'} | {r.naive_tokens:,} | "
+                f"{r.citation_tokens:,} | {r.reduction_pct:.0f}% |"
+            )
+
+        lines.append(
+            f"| **Average** | | | | | **{self.avg_reduction_pct:.0f}% "
+            f"({self.avg_multiplier:.1f}x)** |"
+        )
+        return "\n".join(lines)
+
+
+class IntentEvalRunner:
+    """Runs the intent-graph eval against a project's existing intent journal."""
+
+    def __init__(self, project_root: Path) -> None:
+        self._project_root = project_root
+
+    async def run(
+        self,
+        tracker: AnalyticsTracker | None = None,
+        max_cases: int = DEFAULT_MAX_CASES,
+        budget_tokens: int = DEFAULT_BUDGET_TOKENS,
+    ) -> IntentEvalReport:
+        from codebase_intel.core.config import ProjectConfig
+        from codebase_intel.intentgraph import (
+            IntentBootstrapper,
+            IntentGraphStore,
+            IntentJournal,
+            IntentLevel,
+        )
+
+        config = ProjectConfig(project_root=self._project_root)
+        journal = IntentJournal(config.intentgraph.journal_dir)
+        db_path = config.intentgraph.db_path
+
+        report = IntentEvalReport(
+            repo_name=self._project_root.name, repo_path=str(self._project_root)
+        )
+
+        async with IntentGraphStore.open(db_path) as store:
+            await store.materialize(journal)
+            eligible: list[IntentNode] = []
+            for level in (IntentLevel.COMPONENT, IntentLevel.DECISION):
+                eligible.extend(await store.nodes_at_level(level))
+
+        cases = self._build_cases(eligible, max_cases)
+        report.total_cases = len(cases)
+        if not cases:
+            report.warnings.append(
+                "no eligible intent nodes (need ACTIVE component/decision nodes with a "
+                "why that says more than the title) — run `codebase-intel why ...` first"
+            )
+            return report
+
+        bootstrapper = IntentBootstrapper(journal, db_path)
+        for case in cases:
+            report.results.append(await self._run_case(bootstrapper, case, budget_tokens))
+
+        if tracker:
+            tracker.record_intent_eval(
+                repo_name=report.repo_name,
+                repo_path=report.repo_path,
+                total_cases=report.total_cases,
+                accuracy_at_1=report.accuracy_at_1,
+                accuracy_at_3=report.accuracy_at_3,
+                avg_reduction_pct=report.avg_reduction_pct,
+                avg_multiplier=report.avg_multiplier,
+                results=[r.to_dict() for r in report.results],
+            )
+
+        return report
+
+    async def _run_case(
+        self, bootstrapper: IntentBootstrapper, case: IntentEvalCase, budget_tokens: int
+    ) -> IntentEvalResult:
+        boot = await bootstrapper.bootstrap(case.query, budget_tokens=budget_tokens)
+        relevant = [item for item in boot.items if item.role == "relevant"]
+        rank = next(
+            (idx + 1 for idx, item in enumerate(relevant) if item.node_id == case.node_id),
+            None,
+        )
+        citation = next((item for item in relevant if item.node_id == case.node_id), None)
+        naive_tokens, naive_files = estimate_naive_read_cost(case.anchor_paths)
+
+        return IntentEvalResult(
+            node_id=case.node_id,
+            node_title=case.node_title,
+            query=case.query,
+            found=rank is not None,
+            rank=rank,
+            correct=rank == 1,
+            top3=rank is not None and rank <= 3,
+            naive_tokens=naive_tokens,
+            naive_files=naive_files,
+            citation_tokens=citation.tokens if citation else 0,
+            session_tokens=boot.total_tokens,
+            coverage=boot.coverage,
+        )
+
+    def _build_cases(self, nodes: list[IntentNode], max_cases: int) -> list[IntentEvalCase]:
+        from codebase_intel.intentgraph import IntentNodeStatus
+
+        candidates: list[IntentEvalCase] = []
+        for node in nodes:
+            if node.status != IntentNodeStatus.ACTIVE:
+                continue
+            title_terms = extract_terms(node.title)
+            why_terms = sorted(extract_terms(node.why) - title_terms)
+            if len(why_terms) < MIN_DISTINGUISHING_TERMS:
+                continue
+            query = "why " + " ".join(why_terms[:QUERY_TERM_COUNT])
+            candidates.append(
+                IntentEvalCase(
+                    node_id=node.node_id,
+                    node_title=node.title,
+                    query=query,
+                    anchor_paths=[a.file_path for a in node.anchors],
+                )
+            )
+
+        candidates.sort(key=lambda c: c.node_id)  # deterministic across runs
+        return candidates[:max_cases]

codebase_intel/analytics/feedback.py

@@ -24,12 +24,14 @@ from __future__ import annotations
 import json
 import sqlite3
 from datetime import UTC, datetime
-from enum import Enum
-from pathlib import Path
-from typing import Any
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
 
+if TYPE_CHECKING:
+    from pathlib import Path
 
-class FeedbackType(str, Enum):
+
+class FeedbackType(StrEnum):
     """How the agent's output was received."""
 
     ACCEPTED = "accepted"  # Output used as-is or with minor edits
@@ -38,7 +40,7 @@ class FeedbackType(str, Enum):
     PARTIAL = "partial"  # Some parts used, some discarded
 
 
-class RejectionReason(str, Enum):
+class RejectionReason(StrEnum):
     """Why the output was rejected — maps to improvement actions."""
 
     WRONG_PATTERN = "wrong_pattern"  # Used a pattern the project doesn't follow
@@ -207,7 +209,8 @@ class FeedbackTracker:
             """
             SELECT
                 COUNT(*) as total,
-                SUM(CASE WHEN feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END) as accepted
+                SUM(CASE WHEN feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END)
+                    as accepted
             FROM feedback_events
             WHERE timestamp >= datetime('now', '-7 days')
             """
@@ -261,12 +264,14 @@ class FeedbackTracker:
             # Map rejection reasons to actionable suggestions
             suggestion = self._suggestion_for_reason(reason)
 
-            reasons.append({
-                "reason": reason,
-                "count": count,
-                "percentage": round(pct, 1),
-                "suggestion": suggestion,
-            })
+            reasons.append(
+                {
+                    "reason": reason,
+                    "count": count,
+                    "percentage": round(pct, 1),
+                    "suggestion": suggestion,
+                }
+            )
 
         return {
             "total_rejections": total_rejections,
@@ -306,7 +311,8 @@ class FeedbackTracker:
             """
             SELECT
                 COUNT(DISTINCT cs.session_id) as total,
-                SUM(CASE WHEN fe.feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END) as accepted
+                SUM(CASE WHEN fe.feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END)
+                    as accepted
             FROM context_sessions cs
             JOIN feedback_events fe ON cs.session_id = fe.session_id
             WHERE cs.decisions_provided > 0
@@ -318,7 +324,8 @@ class FeedbackTracker:
             """
             SELECT
                 COUNT(DISTINCT cs.session_id) as total,
-                SUM(CASE WHEN fe.feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END) as accepted
+                SUM(CASE WHEN fe.feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END)
+                    as accepted
             FROM context_sessions cs
             JOIN feedback_events fe ON cs.session_id = fe.session_id
             WHERE cs.decisions_provided = 0
@@ -330,7 +337,8 @@ class FeedbackTracker:
             """
             SELECT
                 COUNT(DISTINCT cs.session_id) as total,
-                SUM(CASE WHEN fe.feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END) as accepted
+                SUM(CASE WHEN fe.feedback_type IN ('accepted', 'modified') THEN 1 ELSE 0 END)
+                    as accepted
             FROM context_sessions cs
             JOIN feedback_events fe ON cs.session_id = fe.session_id
             WHERE cs.contracts_provided > 0
@@ -387,13 +395,15 @@ class FeedbackTracker:
             suggestion = self._suggestion_for_reason(reason.value)
             conn.execute(
                 """
-                INSERT INTO learning_insights (timestamp, insight_type, description, confidence, action_taken)
+                INSERT INTO learning_insights
+                    (timestamp, insight_type, description, confidence, action_taken)
                 VALUES (?, ?, ?, ?, ?)
                 """,
                 (
                     datetime.now(UTC).isoformat(),
                     "pattern_detected",
-                    f"Repeated rejection reason: {reason.value} ({count} times in 30 days). {suggestion}",
+                    f"Repeated rejection reason: {reason.value} "
+                    f"({count} times in 30 days). {suggestion}",
                     min(0.9, 0.3 + count * 0.1),
                     None,
                 ),
@@ -447,9 +457,7 @@ class FeedbackTracker:
             wod_rate = (without_dec["accepted"] or 0) / wod_total * 100
             diff = wd_rate - wod_rate
             if diff > 5:
-                parts.append(
-                    f"Sessions with decisions had {diff:.0f}% higher acceptance rate."
-                )
+                parts.append(f"Sessions with decisions had {diff:.0f}% higher acceptance rate.")
 
         wc_total = with_contracts["total"] or 0
         if wc_total >= 5:
@@ -483,7 +491,8 @@ class FeedbackTracker:
 
         conn.execute(
             """
-            INSERT INTO acceptance_trend (date, total_sessions, accepted, modified, rejected, partial, acceptance_rate)
+            INSERT INTO acceptance_trend
+                (date, total_sessions, accepted, modified, rejected, partial, acceptance_rate)
             VALUES (?, ?, ?, ?, ?, ?, ?)
             ON CONFLICT(date) DO UPDATE SET
                 total_sessions=excluded.total_sessions,
@@ -491,6 +500,14 @@ class FeedbackTracker:
                 rejected=excluded.rejected, partial=excluded.partial,
                 acceptance_rate=excluded.acceptance_rate
             """,
-            (today, total, row["accepted"] or 0, row["modified"] or 0, row["rejected"] or 0, row["partial"] or 0, round(rate, 1)),
+            (
+                today,
+                total,
+                row["accepted"] or 0,
+                row["modified"] or 0,
+                row["rejected"] or 0,
+                row["partial"] or 0,
+                round(rate, 1),
+            ),
         )
         conn.commit()