codegraph-brain 0.6.0__py3-none-any.whl

cgis/guardian/bench.py

@@ -0,0 +1,199 @@
+"""Benchmark ground truth, matching, and scoring (spec §3.1-3.3).
+
+Matching is deterministic and pure so the scorer can be unit-tested without
+any LLM: a prediction matches a ground-truth entry iff same file AND (line
+within the entry's range, or the entry has no range). Greedy by descending
+prediction confidence; each entry matches at most once.
+"""
+
+import statistics
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, Field, model_validator
+
+from cgis.guardian.findings import Category, Finding, Severity
+
+# Re-exported: the recording format moved to its own module when production
+# became a second consumer (#279); bench remains the historical import site.
+from cgis.guardian.recording import FinderRecording as FinderRecording  # noqa: PLC0414
+from cgis.guardian.recording import load_finder_recording as load_finder_recording  # noqa: PLC0414
+from cgis.guardian.recording import save_finder_recording as save_finder_recording  # noqa: PLC0414
+from cgis.guardian.skeptic import visible_findings
+
+
+class GroundTruthEntry(BaseModel, frozen=True):
+    """One curated real finding for a benchmark PR."""
+
+    id: str
+    file: str
+    lines: tuple[int, int] | None = None
+    severity: Severity
+    category: Category
+    summary: str
+    source: Literal["gemini", "sonar", "fix-commit", "human"]
+
+    @model_validator(mode="after")
+    def _lines_ordered(self) -> "GroundTruthEntry":
+        """Reject inverted ranges — a transposed YAML range would silently never match."""
+        if self.lines is not None and self.lines[0] > self.lines[1]:
+            _msg = f"lines range is inverted: {self.lines}"
+            raise ValueError(_msg)
+        return self
+
+
+class AmbiguousEntry(BaseModel, frozen=True):
+    """A debatable suggestion — neither a miss nor noise (spec §3.1)."""
+
+    file: str
+    summary: str
+
+
+class GroundTruth(BaseModel, frozen=True):
+    """Curated findings for one merged PR."""
+
+    pr: int
+    base: str
+    head: str
+    findings: list[GroundTruthEntry]
+    ambiguous: list[AmbiguousEntry] = Field(default_factory=list)
+
+
+class MatchResult(BaseModel, frozen=True):
+    """Outcome of matching predictions against one PR's ground truth."""
+
+    matched: dict[str, int]  # ground-truth id -> prediction index
+    missed: list[str]  # ground-truth ids with no match
+    noise: list[int]  # prediction indices matching nothing
+    ambiguous_hits: list[int]  # prediction indices on ambiguous files
+    total_predictions: int  # len(predictions) passed to match_findings
+
+
+class BenchScore(BaseModel, frozen=True):
+    """Per-PR metrics derived from a MatchResult (spec §3.3)."""
+
+    recall: float
+    precision: float
+    noise: int
+    missed: list[str]
+
+
+def load_ground_truth(path: Path) -> GroundTruth:
+    """Load and validate one benchmarks/guardian/pr-N.yaml file."""
+    return GroundTruth.model_validate(yaml.safe_load(path.read_text(encoding="utf-8")))
+
+
+def _entry_accepts(entry: GroundTruthEntry, prediction: Finding) -> bool:
+    """File must match; a lines range additionally requires the line within it."""
+    if entry.file != prediction.file:
+        return False
+    if entry.lines is None:
+        return True
+    return prediction.line is not None and entry.lines[0] <= prediction.line <= entry.lines[1]
+
+
+def match_findings(predictions: Sequence[Finding], truth: GroundTruth) -> MatchResult:
+    """Match predictions to ground truth: greedy by descending confidence.
+
+    Ties broken by original sequence position (stable sort).
+    """
+    matched: dict[str, int] = {}
+    noise: list[int] = []
+    ambiguous_hits: list[int] = []
+    ambiguous_files = {a.file for a in truth.ambiguous}
+    order = sorted(range(len(predictions)), key=lambda i: -predictions[i].confidence)
+    for i in order:
+        prediction = predictions[i]
+        hit = next(
+            (e.id for e in truth.findings if e.id not in matched and _entry_accepts(e, prediction)),
+            None,
+        )
+        if hit is not None:
+            matched[hit] = i
+        elif prediction.file in ambiguous_files:
+            ambiguous_hits.append(i)
+        else:
+            noise.append(i)
+    missed = [e.id for e in truth.findings if e.id not in matched]
+    return MatchResult(
+        matched=matched,
+        missed=missed,
+        noise=sorted(noise),
+        ambiguous_hits=sorted(ambiguous_hits),
+        total_predictions=len(predictions),
+    )
+
+
+def score(match: MatchResult, truth: GroundTruth) -> BenchScore:
+    """Compute recall / precision / noise for one PR.
+
+    Precision is TP / (TP + FP): ambiguous hits are excluded from the
+    denominator — they are neither correct nor noise (spec §3.1), so they
+    must not depress precision.
+    """
+    gt_total = len(truth.findings)
+    recall = len(match.matched) / gt_total if gt_total else 1.0
+    relevant = len(match.matched) + len(match.noise)
+    precision = len(match.matched) / relevant if relevant else 1.0
+    return BenchScore(
+        recall=recall, precision=precision, noise=len(match.noise), missed=match.missed
+    )
+
+
+def score_separation(gt_scores: Sequence[int], noise_scores: Sequence[int]) -> float | None:
+    """Median impact_score of GT-matching findings minus that of the rest (#246 §4.3).
+
+    This is the gate metric for the per-finding skeptic: it answers whether the
+    skeptic RANKS, which noise counts alone cannot. A flat distribution scores 0
+    even when noise happens to fall.
+
+    None when either population is empty — with nothing to compare, the run
+    cannot answer the question. Callers pool across benchmark PRs before
+    calling: individual PRs carry 1-6 GT matches, too few for a median to mean
+    anything.
+    """
+    if not gt_scores or not noise_scores:
+        return None
+    return statistics.median(gt_scores) - statistics.median(noise_scores)
+
+
+def killed_ground_truth(
+    findings: Sequence[Finding], truth: GroundTruth, threshold: int = 0
+) -> list[str]:
+    """Ground-truth ids the finder matched but the skeptic then made invisible (#270).
+
+    "Before" is every finding the finder produced, judged or not; "after" is what
+    a human would actually read — refuted findings dropped, and anything scored
+    below ``threshold`` hidden. A finding hidden by the threshold is exactly as
+    invisible as a refuted one, so both count.
+
+    This is the failure mode the benchmark exists to catch, and it cannot be
+    derived from ``matched_gt``: that flag is computed on the post-skeptic
+    visible list, so a refuted finding can never carry it and the kill is
+    invisible by construction.
+
+    Ids, not a count: which ground truth was lost is the actionable part.
+    """
+    before = set(match_findings(list(findings), truth).matched)
+    after = set(match_findings(visible_findings(findings, threshold), truth).matched)
+    return sorted(before - after)
+
+
+def annotate_matches(
+    findings: Sequence[Finding], visible: Sequence[Finding], match: MatchResult
+) -> list[dict[str, object]]:
+    """Dump each finding with a ``matched_gt`` flag for the benchmark record (#246 §4.2).
+
+    ``match`` indexes into ``visible`` (what the matcher scored), while the
+    record keeps EVERY finding including the ones the skeptic hid — otherwise
+    "noise fell" would be indistinguishable from "we went blind". Identity, not
+    equality, maps a finding back to its matcher position: two findings can be
+    field-identical, and only one of them may have matched.
+    """
+    position = {id(f): i for i, f in enumerate(visible)}
+    matched_positions = set(match.matched.values())
+    return [
+        {**f.model_dump(), "matched_gt": position.get(id(f)) in matched_positions} for f in findings
+    ]

cgis/guardian/chunked.py

@@ -0,0 +1,240 @@
+"""Chunked review: per-chunk finder passes behind GUARDIAN_FEATURES=chunked.
+
+Slice 2 of #154 (spec: 2026-06-11-guardian-chunked-review-design.md). The
+finder LGTMs large PRs (attention dilution); each chunk gets a small,
+complete world instead — its own diff, full files, and impact graph.
+"""
+
+import structlog
+from pydantic import BaseModel
+
+from cgis.guardian.chunker import Chunk, build_chunks, split_diff_by_file
+from cgis.guardian.collector import ContextCollector
+from cgis.guardian.core import GuardianReviewer, finder_pass
+from cgis.guardian.findings import Finding, ReviewResult
+from cgis.guardian.providers.base import BaseProvider
+from cgis.guardian.skeptic import (
+    apply_judgements,
+    judge_all,
+    skeptic_status_for,
+)
+from cgis.storage.sqlite_store import SQLiteStore
+
+log = structlog.getLogger(__name__)
+
+MAX_CHUNKS = 8
+
+
+class RoutedReview(BaseModel, frozen=True):
+    """Review outcome plus chunk accounting (chunk_count=None = single-pass path)."""
+
+    result: ReviewResult
+    chunk_count: int | None = None
+
+
+def _cap_chunks(chunks: list[Chunk]) -> list[Chunk]:
+    """Bound API calls (spec §4.3): keep the MAX_CHUNKS-1 largest, merge the rest.
+
+    The overflow chunk goes last; kept chunks stay in slice-1 order (sorted
+    by first file). Ties in size break by first file name — deterministic.
+    """
+    if len(chunks) <= MAX_CHUNKS:
+        return chunks
+    ranked = sorted(chunks, key=lambda c: (-len(c.diff), c.files[0]))
+    keep, rest = ranked[: MAX_CHUNKS - 1], ranked[MAX_CHUNKS - 1 :]
+    rest_sorted = sorted(rest, key=lambda c: c.files[0])
+    overflow = Chunk(
+        files=tuple(sorted({f for c in rest_sorted for f in c.files})),
+        diff="".join(c.diff for c in rest_sorted),
+    )
+    log.warning("Chunk cap hit; smallest chunks merged.", merged=len(rest), cap=MAX_CHUNKS)
+    return [*sorted(keep, key=lambda c: c.files[0]), overflow]
+
+
+def _normalize_path(path: str) -> str:
+    """Strip LLM path artifacts (leading ./, diff-header a/ b/ prefixes).
+
+    Used only as a fallback after an exact match fails, so a real directory
+    literally named `a/` or `b/` can never be corrupted by the stripping
+    (gemini review, PR #159).
+    """
+    p = path.removeprefix("./")
+    if p.startswith(("a/", "b/")):
+        p = p[2:]
+    return p
+
+
+def _chunk_survivors(chunk: Chunk, findings: list[Finding]) -> list[Finding]:
+    """Keep findings inside the chunk's files; drop out-of-chunk hallucinations.
+
+    Exact match first; then a normalized fallback so an LLM path artifact
+    ("./x.py", "a/x.py") doesn't drop a real finding. Fallback survivors are
+    canonicalized so inline-comment anchoring still works downstream.
+    """
+    allowed = set(chunk.files)
+    survivors: list[Finding] = []
+    for finding in findings:
+        if finding.file in allowed:
+            survivors.append(finding)
+            continue
+        normalized = _normalize_path(finding.file)
+        if normalized in allowed:
+            survivors.append(finding.model_copy(update={"file": normalized}))
+            continue
+        log.warning("Out-of-chunk finding dropped.", file=finding.file, title=finding.title)
+    return survivors
+
+
+def _dedup(findings: list[Finding]) -> list[Finding]:
+    """Drop duplicate (file, line, category) findings, keeping the higher confidence.
+
+    Cross-chunk duplicates are impossible after the per-chunk file filter
+    (chunks partition files) — this is insurance against intra-pass
+    duplicates. First-occurrence order is preserved.
+    """
+    best: dict[tuple[str, int | None, str], Finding] = {}
+    order: list[tuple[str, int | None, str]] = []
+    for finding in findings:
+        key = (finding.file, finding.line, finding.category)
+        if key not in best:
+            best[key] = finding
+            order.append(key)
+        elif finding.confidence > best[key].confidence:
+            best[key] = finding
+    return [best[k] for k in order]
+
+
+async def _single_pass(
+    provider: BaseProvider,
+    collector: ContextCollector,
+    skeptic_provider: BaseProvider | None,
+) -> RoutedReview:
+    """Run the unchunked reviewer.
+
+    Shared by the routing default and the no-reviewable-files fallback so the
+    two cannot drift apart (#277). chunk_count is None, which is the recorded
+    marker for "this review did not chunk".
+    """
+    reviewer = GuardianReviewer(
+        provider=provider,
+        context_collector=collector,
+        skeptic_provider=skeptic_provider,
+    )
+    return RoutedReview(result=await reviewer.run_review(), chunk_count=None)
+
+
+async def run_chunked_review(
+    *,
+    provider: BaseProvider,
+    collector: ContextCollector,
+    skeptic_provider: BaseProvider | None,
+) -> RoutedReview:
+    """Per-chunk finder passes -> filter -> merge -> dedup -> one skeptic pass.
+
+    Degradations (spec §5): a chunk whose finder call raises or never parses
+    contributes zero findings and a ⚠ bullet; parse_failed on the merged
+    result only when EVERY chunk failed; skeptic failure returns the merged
+    findings unverified.
+    """
+    diff = collector.get_git_diff()
+    if collector.db_path is None:  # routed guard (§4.1) — belt and braces
+        _msg = "run_chunked_review requires a graph DB"
+        raise RuntimeError(_msg)
+    with SQLiteStore(str(collector.db_path)) as store:
+        chunks = build_chunks(diff, store, source_root=collector.source_root)
+    if not chunks:
+        if not split_diff_by_file(diff):
+            return RoutedReview(
+                result=ReviewResult(findings=[], summary="Empty diff — nothing to review."),
+                chunk_count=0,
+            )
+        # Blocks exist but none are reviewable source: a docs-only PR. Single
+        # pass reviews it today, so returning "nothing to review" here would be
+        # a silent regression — exactly the invisible-skip failure #277 is about.
+        log.info("No reviewable source in the diff; falling back to single pass.")
+        return await _single_pass(provider, collector, skeptic_provider)
+    chunks = _cap_chunks(chunks)
+
+    bullets: list[str] = []
+    kept: list[Finding] = []
+    finding_contexts: list[dict[str, str]] = []
+    failed = 0
+    for chunk in chunks:
+        label = ", ".join(chunk.files)
+        try:
+            # Context collection is inside the guard on purpose: it opens the
+            # graph DB per chunk, and a flaky DB must cost one chunk, not the
+            # whole review (opus quality review on slice 2).
+            context = collector.collect_for_chunk(chunk)
+            result = await finder_pass(provider, context)
+        except Exception:
+            log.warning(
+                "Chunk finder call failed; chunk skipped.",
+                files=chunk.files,
+                exc_info=True,
+            )
+            failed += 1
+            bullets.append(f"- [{label}]: ⚠ finder call failed")
+            continue
+        if result.parse_failed:
+            failed += 1
+            bullets.append(f"- [{label}]: ⚠ finder output unparsable")
+            continue
+        survivors = _chunk_survivors(chunk, result.findings)
+        if survivors:
+            finding_contexts.append(context)
+        kept.extend(survivors)
+        bullets.append(f"- [{label}]: {result.summary}")
+
+    merged = ReviewResult(
+        findings=_dedup(kept),
+        summary="\n".join(bullets),
+        parse_failed=failed == len(chunks),
+    )
+    if skeptic_provider is None or not merged.findings or merged.parse_failed:
+        return RoutedReview(result=merged, chunk_count=len(chunks))
+
+    # Judgement context is limited to chunks that produced findings — not the
+    # full PR diff (spec §4.5); judge_all narrows further, per finding's file.
+    skeptic_context = {
+        "diff": "\n".join(c["diff"] for c in finding_contexts),
+        "full_files": "\n\n".join(c["full_files"] for c in finding_contexts if "full_files" in c),
+    }
+    judgements = await judge_all(skeptic_provider, merged.findings, skeptic_context["diff"])
+    judged = sum(1 for j in judgements if j is not None)
+    if judged == 0:
+        log.warning("Every skeptic judgement failed; returning unverified findings.")
+    return RoutedReview(
+        result=merged.model_copy(
+            update={
+                "findings": apply_judgements(merged.findings, judgements),
+                "skeptic_status": skeptic_status_for(judged, len(judgements)),
+                "skeptic_judged": judged,
+                "skeptic_total": len(judgements),
+            }
+        ),
+        chunk_count=len(chunks),
+    )
+
+
+async def run_review_routed(
+    *,
+    provider: BaseProvider,
+    collector: ContextCollector,
+    skeptic_provider: BaseProvider | None,
+) -> RoutedReview:
+    """Single entry point for runner and bench: chunked vs single-pass (spec §4.1).
+
+    chunked without a graph DB falls back to single pass: build_chunks would
+    degrade to all-isolated chunks = one API call per file with zero
+    connectivity benefit — strictly worse than the status quo.
+    """
+    chunked = "chunked" in collector.features
+    if chunked and (collector.db_path is None or not collector.db_path.exists()):
+        log.warning("chunked requested but no graph DB; falling back to single pass.")
+        chunked = False
+    if not chunked:
+        return await _single_pass(provider, collector, skeptic_provider)
+    return await run_chunked_review(
+        provider=provider, collector=collector, skeptic_provider=skeptic_provider
+    )

cgis/guardian/chunker.py

@@ -0,0 +1,148 @@
+"""Connected-subgraph chunking of a PR diff (spec: 2026-06-11-guardian-chunker-design.md).
+
+Slice 1 of #154: pure logic, no LLM calls; not wired into the review loop yet.
+"""
+
+import re
+
+import structlog
+from pydantic import BaseModel
+
+from cgis.core.models import EdgeType
+
+# Re-exported: the per-file split moved to the pure diff leaf next to the other
+# unified-diff parsers, since the skeptic pass needs it too (#246 plan T3).
+from cgis.guardian.diff_index import split_diff_by_file as split_diff_by_file  # noqa: PLC0414
+from cgis.storage.sqlite_store import RAW_CALL_PREFIX, SQLiteStore
+
+log = structlog.getLogger(__name__)
+
+
+class Chunk(BaseModel, frozen=True):
+    """One connected group of changed files and its slice of the diff."""
+
+    files: tuple[str, ...]
+    diff: str
+
+
+def build_chunks(
+    diff_text: str,
+    store: SQLiteStore | None,
+    source_root: str = "",
+    reviewable_suffixes: tuple[str, ...] = (".py",),
+) -> list[Chunk]:
+    """Group changed files into connected-component chunks via IMPORTS/CALLS.
+
+    Only files ending in ``reviewable_suffixes`` are chunked. A markdown or lock
+    file would otherwise cost one finder call and receive no context at all —
+    ``collect_for_chunk`` narrows to .py before assembling one (#277). An
+    all-filtered diff returns [], which the caller distinguishes from an empty
+    diff and answers with a single-pass fallback.
+
+    Degrades honestly: no store / file absent from the graph / store errors →
+    isolated single-file chunks, never worse than the unchunked status quo.
+    Deterministic: files sorted inside a chunk, chunks sorted by first file.
+    """
+    blocks = {
+        path: block
+        for path, block in split_diff_by_file(diff_text).items()
+        if path.endswith(reviewable_suffixes)
+    }
+    if not blocks:
+        return []
+    files = sorted(blocks)
+    parent: dict[str, str] = {f: f for f in files}
+
+    def find(x: str) -> str:
+        """Find root of union-find tree with path halving."""
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]  # path halving
+            x = parent[x]
+        return x
+
+    def union(a: str, b: str) -> None:
+        """Merge two sets in the union-find structure."""
+        parent[find(a)] = find(b)
+
+    for a, b in _graph_pairs(store, set(files), source_root):
+        union(a, b)
+
+    for test_file, impl_file in _test_pairs(files):
+        union(test_file, impl_file)
+
+    groups: dict[str, list[str]] = {}
+    for f in files:  # files is sorted → each group list is sorted
+        groups.setdefault(find(f), []).append(f)
+    return [
+        Chunk(files=tuple(group), diff="".join(blocks[f] for f in group))
+        for group in sorted(groups.values())
+    ]
+
+
+_CHUNK_EDGE_TYPES = frozenset({EdgeType.IMPORTS, EdgeType.CALLS})
+
+
+def _graph_pairs(
+    store: SQLiteStore | None, changed: set[str], source_root: str
+) -> list[tuple[str, str]]:
+    """File pairs joined by an IMPORTS/CALLS edge with both endpoints changed.
+
+    Graph paths are normalized with source_root (collector convention,
+    fix 48790da). Any store failure degrades to no pairs — the chunker sits
+    on the review path and must not take guardian down.
+    """
+    if store is None:
+        return []
+    prefix = f"{source_root}/" if source_root else ""
+    try:
+        # Load only the changed files' nodes and their outgoing edges instead
+        # of the whole graph: any qualifying edge has BOTH endpoints changed,
+        # so its source is always among these nodes (gemini review, PR #157).
+        fqn_to_file = {
+            node.id: prefix + node.file_path
+            for path in changed
+            if not prefix or path.startswith(prefix)
+            for node in store.get_nodes_by_file(path.removeprefix(prefix))
+        }
+        pairs: list[tuple[str, str]] = []
+        for edge in store.get_outgoing_edges_batch(list(fqn_to_file)):
+            if edge.type not in _CHUNK_EDGE_TYPES or edge.target.startswith(RAW_CALL_PREFIX):
+                continue
+            src = fqn_to_file.get(edge.source)
+            dst = fqn_to_file.get(edge.target)
+            if src and dst and src != dst:
+                pairs.append((src, dst))
+    except Exception:
+        log.warning("Graph connectivity skipped; falling back to isolated chunks.", exc_info=True)
+        return []
+    return pairs
+
+
+_TEST_FILE_RE = re.compile(r"^tests/(?:.+/)?test_(?P<name>[^/]+)\.py$")
+
+
+def _test_pairs(files: list[str]) -> list[tuple[str, str]]:
+    """Pair each changed tests/**/test_X.py with its unique implementation file.
+
+    Candidate = changed non-test .py whose path normalized to underscores
+    (src/cgis/guardian/core.py -> src_cgis_guardian_core) equals X or ends
+    with "_X". The underscore boundary stops cross-word bleed (test_core.py
+    must not capture score.py); same-suffix module names can still collide
+    (test_index.py would match diff_index.py) — accepted per spec §4.2.5,
+    and the unique-candidate rule below limits the blast radius. Zero or
+    several candidates -> the test stays isolated.
+    """
+    impl = [f for f in files if f.endswith(".py") and not _TEST_FILE_RE.match(f)]
+    norm = {f: f.removesuffix(".py").replace("/", "_") for f in impl}
+    pairs: list[tuple[str, str]] = []
+    for f in files:
+        match = _TEST_FILE_RE.match(f)
+        if not match:
+            continue
+        name = match.group("name")
+        candidates = [i for i in impl if norm[i] == name or norm[i].endswith("_" + name)]
+        if len(candidates) == 1:
+            pairs.append((f, candidates[0]))
+        elif candidates:
+            log.debug("Ambiguous test pairing; left isolated.", test=f, candidates=candidates)
+    return pairs