code-review-graph-codeblackwell 2.3.6.post1__py3-none-any.whl

code_review_graph/eval/benchmarks/impact_accuracy.py

@@ -0,0 +1,220 @@
+"""Impact accuracy benchmark: measures precision/recall of change impact analysis.
+
+Two ground-truth modes are emitted side by side (``ground_truth_mode`` column):
+
+- **graph-derived (circular — upper bound)** — the historical mode. Ground
+  truth is the changed files plus files with CALLS/IMPORTS_FROM edges into
+  them, i.e. derived from the same graph the predictor traverses. Recall in
+  this mode is an upper bound by construction, not independent evidence.
+- **co-change (same commit, seed excluded)** — the honest mode. The predictor
+  is seeded with a single changed file and graded against the *other* files
+  the author actually touched in the same commit. The ground truth comes from
+  git history, not from the graph.
+
+Failure semantics: if ``analyze_changes`` throws, the row is recorded with
+``status="error"`` and empty metric fields — it stays in the CSV but is
+excluded from aggregates. (Previously a failure silently set
+``predicted = set(changed)``, guaranteeing a fake recall of 1.0.)
+"""
+
+from __future__ import annotations
+
+import logging
+import statistics
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+MODE_GRAPH_DERIVED = "graph-derived (circular — upper bound)"
+MODE_CO_CHANGE = "co-change (same commit, seed excluded)"
+
+
+def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
+    """Get list of changed files for a commit."""
+    result = subprocess.run(
+        ["git", "diff", "--name-only", f"{sha}~1", sha],
+        cwd=str(repo_path),
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        result = subprocess.run(
+            ["git", "diff", "--name-only", "HEAD~1", "HEAD"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+    return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
+
+
+def _files_from_analysis(analysis: dict) -> set[str]:
+    """Extract predicted file paths from an ``analyze_changes`` result."""
+    predicted: set[str] = set()
+    for f in analysis.get("changed_functions", []):
+        if isinstance(f, dict) and "file_path" in f:
+            predicted.add(f["file_path"])
+        elif isinstance(f, dict) and "file" in f:
+            predicted.add(f["file"])
+    for flow in analysis.get("affected_flows", []):
+        if isinstance(flow, dict):
+            for node in flow.get("nodes", []):
+                if isinstance(node, dict) and "file_path" in node:
+                    predicted.add(node["file_path"])
+    return predicted
+
+
+def _graph_neighbor_files(store, files: list[str]) -> set[str]:
+    """Files with CALLS/IMPORTS_FROM edges into any node of *files* (one hop)."""
+    out: set[str] = set()
+    for f in files:
+        for node in store.get_nodes_by_file(f):
+            for edge in store.get_edges_by_target(node.qualified_name):
+                if edge.kind in ("CALLS", "IMPORTS_FROM"):
+                    src_qual = edge.source_qualified
+                    src_file = src_qual.split("::")[0] if "::" in src_qual else ""
+                    if src_file:
+                        out.add(src_file)
+    return out
+
+
+def _base_row(repo: str, sha: str, mode: str, seed: str) -> dict:
+    return {
+        "repo": repo,
+        "commit": sha,
+        "ground_truth_mode": mode,
+        "seed_file": seed,
+        "predicted_files": "",
+        "actual_files": "",
+        "true_positives": "",
+        "precision": "",
+        "recall": "",
+        "f1": "",
+        "status": "ok",
+        "error": "",
+    }
+
+
+def _scored_row(
+    repo: str, sha: str, mode: str, seed: str,
+    predicted: set[str], actual: set[str],
+) -> dict:
+    tp = len(predicted & actual)
+    precision = tp / max(len(predicted), 1)
+    recall = tp / max(len(actual), 1)
+    f1 = 2 * precision * recall / max(precision + recall, 0.001)
+    row = _base_row(repo, sha, mode, seed)
+    row.update({
+        "predicted_files": len(predicted),
+        "actual_files": len(actual),
+        "true_positives": tp,
+        "precision": round(precision, 3),
+        "recall": round(recall, 3),
+        "f1": round(f1, 3),
+    })
+    return row
+
+
+def _error_row(repo: str, sha: str, mode: str, seed: str, exc: Exception) -> dict:
+    row = _base_row(repo, sha, mode, seed)
+    row["status"] = "error"
+    row["error"] = str(exc)[:200]
+    return row
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run impact accuracy benchmark (both ground-truth modes)."""
+    from code_review_graph.changes import analyze_changes
+
+    results = []
+    repo = config["name"]
+    for tc in config.get("test_commits", []):
+        sha = tc["sha"]
+        changed = _get_changed_files(repo_path, sha)
+        if not changed:
+            continue
+
+        # --- Mode 1: graph-derived ground truth (circular — upper bound) ---
+        try:
+            analysis = analyze_changes(
+                store, changed, repo_root=str(repo_path), base=sha + "~1",
+            )
+        except Exception as exc:
+            # Old behaviour set predicted = set(changed) here, which
+            # guarantees recall 1.0 on a *failed* run. Mark failed instead.
+            logger.warning("analyze_changes failed on %s: %s", sha, exc)
+            results.append(_error_row(repo, sha, MODE_GRAPH_DERIVED, "", exc))
+            analysis = None
+
+        if analysis is not None:
+            predicted = set(changed) | _files_from_analysis(analysis)
+            actual = set(changed) | _graph_neighbor_files(store, changed)
+            results.append(
+                _scored_row(repo, sha, MODE_GRAPH_DERIVED, "", predicted, actual)
+            )
+
+        # --- Mode 2: co-change ground truth (honest) ---
+        # Seed the predictor with a single changed file and grade against
+        # the other files the author touched in the same commit. Note the
+        # seed analysis deliberately gets no repo_root/diff: it must only
+        # see the seed file, never the full commit diff.
+        seed = sorted(changed)[0]
+        co_actual = set(changed) - {seed}
+        if not co_actual:
+            row = _base_row(repo, sha, MODE_CO_CHANGE, seed)
+            row["status"] = "skipped"
+            row["error"] = "single-file commit: no co-changed files to grade against"
+            results.append(row)
+            continue
+
+        try:
+            seed_analysis = analyze_changes(store, [seed])
+        except Exception as exc:
+            logger.warning("analyze_changes (seed=%s) failed on %s: %s", seed, sha, exc)
+            results.append(_error_row(repo, sha, MODE_CO_CHANGE, seed, exc))
+            continue
+
+        co_predicted = _files_from_analysis(seed_analysis)
+        co_predicted |= _graph_neighbor_files(store, [seed])
+        co_predicted.discard(seed)
+        results.append(
+            _scored_row(repo, sha, MODE_CO_CHANGE, seed, co_predicted, co_actual)
+        )
+
+    return results
+
+
+def aggregate(results: list[dict]) -> dict:
+    """Per-mode means over successful rows only.
+
+    Error/skipped rows stay in the CSV but never contribute to a number.
+    """
+    out: dict = {
+        "total_rows": len(results),
+        "error_rows": sum(1 for r in results if r.get("status") == "error"),
+        "skipped_rows": sum(1 for r in results if r.get("status") == "skipped"),
+    }
+    for key, mode in (
+        ("graph_derived", MODE_GRAPH_DERIVED),
+        ("co_change", MODE_CO_CHANGE),
+    ):
+        rows = [
+            r for r in results
+            if r.get("ground_truth_mode") == mode and r.get("status") == "ok"
+        ]
+        out[key] = {
+            "ok_rows": len(rows),
+            "mean_precision": (
+                round(statistics.mean(float(r["precision"]) for r in rows), 3)
+                if rows else None
+            ),
+            "mean_recall": (
+                round(statistics.mean(float(r["recall"]) for r in rows), 3)
+                if rows else None
+            ),
+            "mean_f1": (
+                round(statistics.mean(float(r["f1"]) for r in rows), 3)
+                if rows else None
+            ),
+        }
+    return out

code_review_graph/eval/benchmarks/multi_hop_retrieval.py

@@ -0,0 +1,125 @@
+"""Multi-hop retrieval benchmark.
+
+Tests a two-step tool chain that mimics how an LLM agent actually uses the
+graph for complex tasks:
+
+  1. ``hybrid_search(nl_query)`` to find a starting anchor from a natural-
+     language question.
+  2. ``query_graph(pattern, target=anchor)`` to traverse one hop along the
+     requested edge kind (callers_of / callees_of / tests_for / ...).
+
+For each task the benchmark records:
+
+- ``anchor_found`` — did semantic search return a node whose qualified_name
+  ends with the expected suffix in the top-K?
+- ``anchor_rank`` — index in the search result list (lower is better).
+- ``neighbor_count`` — number of neighbors returned by the traversal.
+- ``neighbor_recall`` — fraction of ``expected_neighbor_names`` that appear
+  among the neighbor names.
+- ``score`` — ``int(anchor_found) * neighbor_recall``. Range 0–1.
+
+Tasks are defined per-config under ``multi_hop_tasks:`` in
+``code_review_graph/eval/configs/*.yaml``. See
+``docs/REPRODUCING.md`` for the schema and the curated canonical task set.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def _name_set(rows: list[dict[str, Any]]) -> set[str]:
+    out: set[str] = set()
+    for r in rows:
+        name = (r.get("name") or "").lower()
+        if name:
+            out.add(name)
+    return out
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run the multi-hop retrieval benchmark for one repo."""
+    # Imports are local so an import-time failure in one optional benchmark
+    # does not poison the whole runner.
+    from code_review_graph.search import hybrid_search
+    from code_review_graph.tools.query import query_graph
+
+    repo_root = str(repo_path)
+    results: list[dict] = []
+
+    for task in config.get("multi_hop_tasks", []):
+        task_id = task["id"]
+        nl_query = task["nl_query"]
+        suffix = task["anchor_qualified_suffix"].lower()
+        traversal = task.get("traversal_pattern", "callers_of")
+        expected = [e.lower() for e in task.get("expected_neighbor_names", [])]
+        k = int(task.get("k", 10))
+
+        # Step 1 — semantic search
+        try:
+            hits = hybrid_search(store, nl_query, limit=k)
+        except Exception as exc:  # noqa: BLE001 — benchmark must not abort the runner
+            logger.warning("hybrid_search failed on %s: %s", task_id, exc)
+            hits = []
+
+        anchor = None
+        anchor_rank = -1
+        for i, h in enumerate(hits):
+            qn = (h.get("qualified_name") or "").lower()
+            if qn.endswith(suffix):
+                anchor = h
+                anchor_rank = i
+                break
+
+        if anchor is None:
+            results.append({
+                "repo": config["name"],
+                "task_id": task_id,
+                "nl_query": nl_query,
+                "anchor_found": False,
+                "anchor_rank": -1,
+                "neighbor_count": 0,
+                "expected_count": len(expected),
+                "matched_count": 0,
+                "neighbor_recall": 0.0,
+                "score": 0.0,
+            })
+            continue
+
+        # Step 2 — single-hop graph traversal from the anchor
+        try:
+            trav = query_graph(
+                pattern=traversal,
+                target=anchor["qualified_name"],
+                repo_root=repo_root,
+                detail_level="standard",
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "query_graph(%s) failed on %s: %s", traversal, task_id, exc,
+            )
+            trav = {}
+
+        rows = trav.get("data") or trav.get("results") or []
+        names = _name_set(rows)
+        matched = sum(1 for e in expected if e in names)
+        recall = matched / len(expected) if expected else 0.0
+
+        results.append({
+            "repo": config["name"],
+            "task_id": task_id,
+            "nl_query": nl_query,
+            "anchor_found": True,
+            "anchor_rank": anchor_rank,
+            "neighbor_count": len(rows),
+            "expected_count": len(expected),
+            "matched_count": matched,
+            "neighbor_recall": round(recall, 3),
+            "score": round(recall, 3),
+        })
+
+    return results

code_review_graph/eval/benchmarks/search_quality.py

@@ -0,0 +1,59 @@
+"""Search quality benchmark: measures search result ranking via MRR."""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run search quality benchmark."""
+    results = []
+    for sq in config.get("search_queries", []):
+        query = sq["query"]
+        expected = sq["expected"]
+
+        try:
+            from code_review_graph.search import hybrid_search
+            search_results = hybrid_search(store, query, limit=20)
+        except (ImportError, sqlite3.OperationalError) as exc:
+            logger.debug("hybrid_search unavailable, using fallback: %s", exc)
+            # Fallback to basic search
+            search_results = [
+                {"qualified_name": n.qualified_name}
+                for n in store.search_nodes(query, limit=20)
+            ]
+
+        rank = 0
+        for i, r in enumerate(search_results):
+            if isinstance(r, dict):
+                qn = r.get("qualified_name", "")
+            elif hasattr(r, "qualified_name"):
+                qn = r.qualified_name
+            else:
+                qn = ""
+            qn_lower = qn.lower()
+            exp_lower = expected.lower()
+            # Match if expected is substring of qn, qn is substring of expected,
+            # or the name part after :: matches
+            exp_name = expected.rsplit("::", 1)[-1] if "::" in expected else expected
+            qn_name = qn.rsplit("::", 1)[-1] if "::" in qn else qn
+            if (
+                exp_lower in qn_lower
+                or qn_lower in exp_lower
+                or exp_name.lower() == qn_name.lower()
+            ):
+                rank = i + 1
+                break
+
+        results.append({
+            "repo": config["name"],
+            "query": query,
+            "expected": expected,
+            "rank": rank,
+            "reciprocal_rank": round(1.0 / rank if rank > 0 else 0.0, 3),
+        })
+    return results

code_review_graph/eval/benchmarks/token_efficiency.py

@@ -0,0 +1,143 @@
+"""Token efficiency benchmark: compares naive, standard, and graph-based token counts.
+
+Failure semantics: if ``get_review_context`` throws, the row is recorded with
+``status="error"`` and empty metric fields. It stays in the CSV for forensics
+but is excluded from every aggregate — a failed tool call is not a
+measurement. (Previously a failure silently produced ``graph_tokens=0`` and
+``ratio = naive / 1``, inflating the results.)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import statistics
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def _count_tokens(text: str) -> int:
+    """Approximate token count (1 token ~ 4 chars)."""
+    return len(text) // 4
+
+
+def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
+    """Get list of changed files for a commit."""
+    result = subprocess.run(
+        ["git", "diff", "--name-only", f"{sha}~1", sha],
+        cwd=str(repo_path),
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        # Fallback: diff against parent
+        result = subprocess.run(
+            ["git", "diff", "--name-only", "HEAD~1", "HEAD"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+    return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
+
+
+def _count_file_tokens(repo_path: Path, files: list[str]) -> int:
+    """Count tokens from full file contents (naive approach)."""
+    total = 0
+    for f in files:
+        fp = repo_path / f
+        if fp.is_file():
+            try:
+                total += _count_tokens(fp.read_text(encoding="utf-8", errors="replace"))
+            except OSError:
+                pass
+    return total
+
+
+def _count_diff_tokens(repo_path: Path, sha: str) -> int:
+    """Count tokens from git diff output (standard approach)."""
+    result = subprocess.run(
+        ["git", "diff", f"{sha}~1", sha],
+        cwd=str(repo_path),
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        result = subprocess.run(
+            ["git", "diff", "HEAD~1", "HEAD"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+    return _count_tokens(result.stdout)
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run token efficiency benchmark."""
+    results = []
+    for tc in config.get("test_commits", []):
+        changed = _get_changed_files(repo_path, tc["sha"])
+        if not changed:
+            continue
+
+        naive_tokens = _count_file_tokens(repo_path, changed)
+        standard_tokens = _count_diff_tokens(repo_path, tc["sha"])
+
+        row: dict = {
+            "repo": config["name"],
+            "commit": tc["sha"],
+            "description": tc.get("description", ""),
+            "changed_files": len(changed),
+            "naive_tokens": naive_tokens,
+            "standard_tokens": standard_tokens,
+            "graph_tokens": "",
+            "naive_to_graph_ratio": "",
+            "standard_to_graph_ratio": "",
+            "status": "ok",
+            "error": "",
+        }
+
+        # Graph-based: use get_review_context
+        try:
+            from code_review_graph.tools import get_review_context
+            ctx = get_review_context(
+                changed_files=changed, repo_root=str(repo_path)
+            )
+            graph_tokens = _count_tokens(json.dumps(ctx))
+        except Exception as exc:
+            # A failed tool call is not a measurement. Recording
+            # graph_tokens=0 used to turn this into ratio = naive/1 — a
+            # huge fake win. Mark the row failed; aggregate() excludes it.
+            logger.warning("get_review_context failed on %s: %s", tc["sha"], exc)
+            row["status"] = "error"
+            row["error"] = str(exc)[:200]
+            results.append(row)
+            continue
+
+        row["graph_tokens"] = graph_tokens
+        row["naive_to_graph_ratio"] = round(naive_tokens / max(graph_tokens, 1), 1)
+        row["standard_to_graph_ratio"] = round(standard_tokens / max(graph_tokens, 1), 1)
+        results.append(row)
+    return results
+
+
+def aggregate(results: list[dict]) -> dict:
+    """Aggregate token-efficiency rows, excluding failed measurements.
+
+    Rows with ``status != "ok"`` stay in the CSV for forensics but must not
+    contribute to any headline number.
+    """
+    ok = [r for r in results if r.get("status") == "ok"]
+    ratios = [float(r["naive_to_graph_ratio"]) for r in ok]
+    return {
+        "total_rows": len(results),
+        "ok_rows": len(ok),
+        "error_rows": sum(1 for r in results if r.get("status") == "error"),
+        "median_naive_to_graph_ratio": (
+            round(statistics.median(ratios), 1) if ratios else None
+        ),
+        "mean_naive_to_graph_ratio": (
+            round(statistics.mean(ratios), 1) if ratios else None
+        ),
+    }

code_review_graph/eval/configs/code-review-graph.yaml

@@ -0,0 +1,50 @@
+name: code-review-graph
+url: https://github.com/tirth8205/code-review-graph
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor. (This config replaces
+# the historical "nextjs" entry, which used the same URL but mis-labelled the
+# target as a Next.js monorepo.)
+commit: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
+language: python
+size_category: medium
+
+test_commits:
+  - sha: 528801f841e519567ef54d6e52e9b9831d162e1b
+    description: "feat: add multi-platform MCP server installation support"
+    changed_files: 3
+  - sha: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
+    description: "feat: add Google Antigravity platform support for MCP install"
+    changed_files: 2
+
+entry_points:
+  - "code_review_graph/cli.py::cli"
+  - "code_review_graph/main.py::main"
+
+search_queries:
+  - query: "GraphStore nodes"
+    expected: "code_review_graph/graph.py::GraphStore"
+  - query: "parse AST"
+    expected: "code_review_graph/parser.py::CodeParser"
+  - query: "full build"
+    expected: "code_review_graph/incremental.py::full_build"
+
+multi_hop_tasks:
+  - id: crg-parse-file-callers
+    nl_query: "Who invokes the parser entry point on a single source file"
+    anchor_qualified_suffix: "code_review_graph/parser.py::codeparser.parse_file"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["setup_method"]
+    k: 10
+  - id: crg-upsert-node-callers
+    nl_query: "Where the graph store inserts or updates a node"
+    anchor_qualified_suffix: "code_review_graph/graph.py::graphstore.upsert_node"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["store_file_nodes_edges"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does GraphStore upsert_node store a node"
+  - "Where does full_build parse the repository"
+  - "How does hybrid_search rank search results"

code_review_graph/eval/configs/express.yaml

@@ -0,0 +1,45 @@
+name: express
+url: https://github.com/expressjs/express
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
+language: javascript
+size_category: small
+
+test_commits:
+  - sha: 925a1dff1e42f1b393c977b8b77757fcf633e09f
+    description: "fix: bump qs minimum to ^6.14.2 for CVE-2026-2391"
+    changed_files: 1
+  - sha: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
+    description: "test: include edge case tests for res.type()"
+    changed_files: 1
+
+entry_points:
+  - "lib/application.js::app.handle"
+  - "lib/express.js::createApplication"
+
+search_queries:
+  - query: "app handle"
+    expected: "lib/application.js::app"
+  - query: "response send"
+    expected: "lib/response.js::res"
+  - query: "request"
+    expected: "lib/request.js::req"
+
+# Express has only one task — JS modules use prototypes + module.exports
+# heavily, so most "method" callers are not represented as proper Function
+# edges in the graph. createApplication is the cleanest anchor.
+multi_hop_tasks:
+  - id: express-create-application-callees
+    nl_query: "What express does when constructing an application"
+    anchor_qualified_suffix: "lib/express.js::createapplication"
+    traversal_pattern: callees_of
+    expected_neighbor_names: ["mixin", "create", "init"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does app.handle process the middleware stack"
+  - "Where does res.send write the response body"
+  - "How does createApplication initialize an app"

code_review_graph/eval/configs/fastapi.yaml

@@ -0,0 +1,48 @@
+name: fastapi
+url: https://github.com/tiangolo/fastapi
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
+language: python
+size_category: medium
+
+test_commits:
+  - sha: fa3588c38c7473aca7536b12d686102de4b0f407
+    description: "Fix typo for client_secret in OAuth2 form docstrings"
+    changed_files: 1
+  - sha: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
+    description: "Exclude spam comments from statistics in scripts/people.py"
+    changed_files: 1
+
+entry_points:
+  - "fastapi/applications.py::FastAPI"
+  - "fastapi/routing.py::APIRouter"
+
+search_queries:
+  - query: "FastAPI application"
+    expected: "fastapi/applications.py::FastAPI"
+  - query: "APIRoute routing"
+    expected: "fastapi/routing.py::APIRoute"
+  - query: "Depends injection"
+    expected: "fastapi/params.py::Depends"
+
+multi_hop_tasks:
+  - id: fastapi-route-handler-callers
+    nl_query: "How fastapi binds a route handler to an APIRoute"
+    anchor_qualified_suffix: "fastapi/routing.py::apiroute.get_route_handler"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["__init__"]
+    k: 10
+  - id: fastapi-get-dependant-callers
+    nl_query: "Where fastapi resolves dependency declarations into a tree"
+    anchor_qualified_suffix: "fastapi/dependencies/utils.py::get_dependant"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["get_parameterless_sub_dependant", "solve_dependencies"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does include_router register routes on the application"
+  - "Where does APIRoute build its route handler"
+  - "How does solve_dependencies resolve Depends parameters"