code-review-graph-codeblackwell 2.3.6.post1__py3-none-any.whl

code_review_graph/eval/configs/flask.yaml

@@ -0,0 +1,50 @@
+name: flask
+url: https://github.com/pallets/flask
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
+language: python
+size_category: small
+
+test_commits:
+  - sha: fbb6f0bc4c60a0bada0e03c3480d0ccf30a3c1df
+    description: "all teardown callbacks are called despite errors"
+    changed_files: 10
+  - sha: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
+    description: "document that headers must be set before streaming"
+    changed_files: 4
+
+entry_points:
+  - "src/flask/app.py::Flask.wsgi_app"
+  - "src/flask/sansio/app.py::App.add_url_rule"
+
+search_queries:
+  - query: "Flask wsgi"
+    expected: "src/flask/app.py::Flask"
+  - query: "AppContext globals"
+    expected: "src/flask/ctx.py::AppContext"
+  - query: "create logger"
+    expected: "src/flask/logging.py::create_logger"
+
+# Multi-hop retrieval tasks (semantic_search → query_graph one-hop)
+# See docs/REPRODUCING.md for the schema.
+multi_hop_tasks:
+  - id: flask-dispatch-callers
+    nl_query: "Where Flask dispatches HTTP requests"
+    anchor_qualified_suffix: "src/flask/app.py::flask.dispatch_request"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["full_dispatch_request"]
+    k: 10
+  - id: flask-exception-callers
+    nl_query: "Where Flask handles uncaught exceptions"
+    anchor_qualified_suffix: "src/flask/app.py::flask.handle_exception"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["wsgi_app"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does dispatch_request route an incoming HTTP request"
+  - "Where is the AppContext pushed and popped"
+  - "How does create_logger configure application logging"

code_review_graph/eval/configs/gin.yaml

@@ -0,0 +1,51 @@
+name: gin
+url: https://github.com/gin-gonic/gin
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
+language: go
+size_category: small
+
+test_commits:
+  - sha: 052d1a79aafe3f04078a2716f8e77d4340308383
+    description: "feat(render): add PDF renderer and tests"
+    changed_files: 5
+  - sha: 472d086af2acd924cb4b9d7be0525f7d790f69bc
+    description: "fix(tree): panic in findCaseInsensitivePathRec with RedirectFixedPath"
+    changed_files: 2
+  - sha: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
+    description: "fix(render): write content length in Data.Render"
+    changed_files: 2
+
+entry_points:
+  - "gin.go::Engine"
+  - "routergroup.go::RouterGroup"
+
+search_queries:
+  - query: "Engine ServeHTTP"
+    expected: "gin.go::Engine"
+  - query: "Context request"
+    expected: "context.go::Context"
+  - query: "node tree"
+    expected: "tree.go::node"
+
+multi_hop_tasks:
+  - id: gin-serve-http-callees
+    nl_query: "What does the gin engine do when serving an HTTP request"
+    anchor_qualified_suffix: "gin.go::engine.servehttp"
+    traversal_pattern: callees_of
+    expected_neighbor_names: ["reset"]
+    k: 10
+  - id: gin-context-next-callers
+    nl_query: "Who advances the gin middleware chain via Context.Next"
+    anchor_qualified_suffix: "context.go::context.next"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["handleHTTPRequest", "serveError"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does Engine.ServeHTTP route an incoming request"
+  - "Where does Context.Next advance the middleware chain"
+  - "How does the node tree match wildcard routes"

code_review_graph/eval/configs/httpx.yaml

@@ -0,0 +1,48 @@
+name: httpx
+url: https://github.com/encode/httpx
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: b55d4635701d9dc22928ee647880c76b078ba3f2
+language: python
+size_category: small
+
+test_commits:
+  - sha: ae1b9f66238f75ced3ced5e4485408435de10768
+    description: "Expose FunctionAuth in __all__"
+    changed_files: 3
+  - sha: b55d4635701d9dc22928ee647880c76b078ba3f2
+    description: "Upgrade Python type checker mypy"
+    changed_files: 4
+
+entry_points:
+  - "httpx/_client.py::Client"
+  - "httpx/_client.py::AsyncClient"
+
+search_queries:
+  - query: "Client request"
+    expected: "httpx/_client.py::Client"
+  - query: "Response headers"
+    expected: "httpx/_models.py::Response"
+  - query: "BaseClient"
+    expected: "httpx/_client.py::BaseClient"
+
+multi_hop_tasks:
+  - id: httpx-client-request-callers
+    nl_query: "Which HTTP verbs route through the httpx Client.request"
+    anchor_qualified_suffix: "httpx/_client.py::client.request"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["get", "options", "head", "post", "put", "patch"]
+    k: 10
+  - id: httpx-async-request-tests
+    nl_query: "Tests covering the httpx async client request method"
+    anchor_qualified_suffix: "httpx/_client.py::asyncclient.request"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["test_raise_for_status"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does Client.request send an HTTP request"
+  - "Where are Response headers parsed and decoded"
+  - "How does BaseClient build request URLs"

code_review_graph/eval/reporter.py

@@ -0,0 +1,301 @@
+"""Markdown report generator for evaluation benchmark results.
+
+Takes a list of benchmark result dicts and produces a formatted markdown table
+suitable for inclusion in documentation or CI output.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Any
+
+
+def generate_markdown_report(results: list[dict[str, Any]]) -> str:
+    """Generate a markdown report from benchmark results.
+
+    Each result dict should contain at minimum a ``benchmark`` key identifying
+    the benchmark name, plus any metric keys (e.g. ``ratio``,
+    ``reduction_percent``, ``mrr``, ``precision``, ``recall``, ``f1``).
+
+    Args:
+        results: List of result dicts from benchmark runs.
+
+    Returns:
+        A markdown string containing a summary table and per-benchmark details.
+    """
+    if not results:
+        return "# Evaluation Report\n\nNo benchmark results to report.\n"
+
+    lines: list[str] = []
+    lines.append("# Evaluation Report")
+    lines.append("")
+
+    # Collect all metric keys across results (excluding 'benchmark')
+    all_keys: list[str] = []
+    seen: set[str] = set()
+    for r in results:
+        for k in r:
+            if k != "benchmark" and k not in seen:
+                all_keys.append(k)
+                seen.add(k)
+
+    # Summary table
+    lines.append("## Summary")
+    lines.append("")
+
+    header = "| Benchmark | " + " | ".join(all_keys) + " |"
+    separator = "| --- | " + " | ".join("---" for _ in all_keys) + " |"
+    lines.append(header)
+    lines.append(separator)
+
+    for r in results:
+        name = r.get("benchmark", "unknown")
+        values = [str(r.get(k, "-")) for k in all_keys]
+        lines.append(f"| {name} | " + " | ".join(values) + " |")
+
+    lines.append("")
+
+    # Per-benchmark detail sections
+    lines.append("## Details")
+    lines.append("")
+    for r in results:
+        name = r.get("benchmark", "unknown")
+        lines.append(f"### {name}")
+        lines.append("")
+        for k in all_keys:
+            v = r.get(k, "-")
+            lines.append(f"- **{k}**: {v}")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _read_csvs(results_dir: Path, prefix: str) -> list[dict[str, str]]:
+    """Read all CSV files matching a prefix from the results directory."""
+    rows: list[dict[str, str]] = []
+    for p in sorted(results_dir.glob(f"*_{prefix}_*.csv")):
+        with open(p, newline="") as f:
+            reader = csv.DictReader(f)
+            rows.extend(reader)
+    return rows
+
+
+def _md_table(headers: list[str], rows: list[list[str]]) -> str:
+    """Build a markdown table from headers and rows."""
+    lines = []
+    lines.append("| " + " | ".join(headers) + " |")
+    lines.append("| " + " | ".join("---" for _ in headers) + " |")
+    for row in rows:
+        lines.append("| " + " | ".join(row) + " |")
+    return "\n".join(lines)
+
+
+def generate_full_report(results_dir: str | Path) -> str:
+    """Generate a full markdown evaluation report from CSV result files.
+
+    Reads all CSV files in *results_dir*, groups them by benchmark type,
+    and produces a markdown report with methodology notes and per-benchmark
+    result tables.
+
+    Args:
+        results_dir: Directory containing CSV result files.
+
+    Returns:
+        Markdown string with the full report.
+    """
+    results_dir = Path(results_dir)
+    lines: list[str] = []
+    lines.append("# Evaluation Report")
+    lines.append("")
+    lines.append("## Methodology")
+    lines.append("")
+    lines.append("Benchmarks are run against real open-source repositories.")
+    lines.append("Token counts use a consistent `len(text) // 4` approximation.")
+    lines.append(
+        "Impact accuracy reports two ground-truth modes: "
+        "graph-derived (circular — upper bound) and co-change "
+        "(files co-changed in the same commit, seed excluded)."
+    )
+    lines.append(
+        "Rows with `status=error` are kept for forensics but excluded "
+        "from all aggregates."
+    )
+    lines.append("")
+
+    benchmark_types = [
+        "token_efficiency",
+        "impact_accuracy",
+        "agent_baseline",
+        "flow_completeness",
+        "search_quality",
+        "build_performance",
+        "multi_hop_retrieval",
+    ]
+
+    for btype in benchmark_types:
+        rows = _read_csvs(results_dir, btype)
+        if not rows:
+            continue
+
+        title = btype.replace("_", " ").title()
+        lines.append(f"## {title}")
+        lines.append("")
+
+        headers = list(rows[0].keys())
+        table_rows = [[r.get(h, "-") for h in headers] for r in rows]
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    if len(lines) <= 6:
+        lines.append("No benchmark results found.")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def generate_readme_tables(results_dir: str | Path) -> str:
+    """Generate concise README-ready tables from CSV result files.
+
+    Produces three tables:
+    - Table A: Token Efficiency
+    - Table B: Accuracy & Quality
+    - Table C: Performance
+
+    Args:
+        results_dir: Directory containing CSV result files.
+
+    Returns:
+        Markdown string with the three tables.
+    """
+    results_dir = Path(results_dir)
+    lines: list[str] = []
+
+    # Table A: Token Efficiency
+    te_rows = _read_csvs(results_dir, "token_efficiency")
+    if te_rows:
+        lines.append("### Token Efficiency")
+        lines.append("")
+        headers = [
+            "Repo", "Files", "Naive Tokens", "Standard Tokens",
+            "Graph Tokens", "Naive/Graph", "Std/Graph",
+        ]
+        table_rows = []
+        for r in te_rows:
+            table_rows.append([
+                r.get("repo", "-"),
+                r.get("changed_files", "-"),
+                r.get("naive_tokens", "-"),
+                r.get("standard_tokens", "-"),
+                r.get("graph_tokens", "-"),
+                r.get("naive_to_graph_ratio", "-"),
+                r.get("standard_to_graph_ratio", "-"),
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    # Table B: Accuracy & Quality
+    ia_rows = _read_csvs(results_dir, "impact_accuracy")
+    fc_rows = _read_csvs(results_dir, "flow_completeness")
+    sq_rows = _read_csvs(results_dir, "search_quality")
+
+    if ia_rows or fc_rows or sq_rows:
+        lines.append("### Accuracy & Quality")
+        lines.append("")
+        headers = ["Repo", "Impact F1 (graph-derived)", "Flow Recall", "Search MRR"]
+        # Build a per-repo summary
+        repo_data: dict[str, dict[str, object]] = {}
+        mrr_accum: dict[str, list[float]] = {}
+        f1_accum: dict[str, list[float]] = {}
+        for r in ia_rows:
+            # Failed rows are kept in the CSV for forensics but must never
+            # contribute to a headline number; co-change rows are a
+            # different metric and get their own reporting.
+            if r.get("status", "ok") not in ("", "ok"):
+                continue
+            mode = r.get("ground_truth_mode", "")
+            if mode and not mode.startswith("graph-derived"):
+                continue
+            repo = r.get("repo", "?")
+            repo_data.setdefault(repo, {})
+            try:
+                f1_accum.setdefault(repo, []).append(float(r.get("f1", "")))
+            except (ValueError, TypeError):
+                pass
+        for r in fc_rows:
+            repo_data.setdefault(r.get("repo", "?"), {})["recall"] = r.get("recall", "-")
+        for r in sq_rows:
+            repo = r.get("repo", "?")
+            repo_data.setdefault(repo, {})
+            try:
+                mrr_accum.setdefault(repo, []).append(float(r.get("reciprocal_rank", 0)))
+            except (ValueError, TypeError):
+                pass
+
+        table_rows = []
+        for repo, d in sorted(repo_data.items()):
+            mrr_vals = mrr_accum.get(repo, [])
+            mrr = (
+                str(round(sum(mrr_vals) / len(mrr_vals), 3))
+                if mrr_vals
+                else "-"
+            )
+            f1_vals = f1_accum.get(repo, [])
+            f1 = (
+                str(round(sum(f1_vals) / len(f1_vals), 3))
+                if f1_vals
+                else "-"
+            )
+            table_rows.append([
+                repo,
+                f1,
+                str(d.get("recall", "-")),
+                mrr,
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    # Table B2: Agent Baseline (grep top-k vs graph query)
+    ab_rows = _read_csvs(results_dir, "agent_baseline")
+    if ab_rows:
+        lines.append("### Agent Baseline (grep top-k vs graph query)")
+        lines.append("")
+        headers = [
+            "Repo", "Question", "Baseline Tokens", "Graph Tokens",
+            "Baseline/Graph", "Status",
+        ]
+        table_rows = []
+        for r in ab_rows:
+            table_rows.append([
+                r.get("repo", "-"),
+                r.get("question", "-"),
+                r.get("baseline_tokens", "-"),
+                r.get("graph_tokens", "-"),
+                r.get("baseline_to_graph_ratio", "-"),
+                r.get("status", "ok") or "ok",
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    # Table C: Performance
+    bp_rows = _read_csvs(results_dir, "build_performance")
+    if bp_rows:
+        lines.append("### Performance")
+        lines.append("")
+        headers = ["Repo", "Files", "Nodes", "Flow Det. (s)", "Search (ms)"]
+        table_rows = []
+        for r in bp_rows:
+            table_rows.append([
+                r.get("repo", "-"),
+                r.get("file_count", "-"),
+                r.get("node_count", "-"),
+                r.get("flow_detection_seconds", "-"),
+                r.get("search_avg_ms", "-"),
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    if not lines:
+        return "No benchmark results found.\n"
+
+    return "\n".join(lines)

code_review_graph/eval/runner.py

@@ -0,0 +1,211 @@
+"""Evaluation runner: orchestrates benchmark execution across repositories."""
+
+from __future__ import annotations
+
+import csv
+import logging
+import subprocess
+from datetime import date
+from pathlib import Path
+
+try:
+    import yaml  # type: ignore[import-untyped]
+except ImportError:
+    yaml = None  # type: ignore[assignment]
+
+from code_review_graph.eval.benchmarks import (
+    agent_baseline,
+    build_performance,
+    flow_completeness,
+    impact_accuracy,
+    multi_hop_retrieval,
+    search_quality,
+    token_efficiency,
+)
+
+logger = logging.getLogger(__name__)
+
+BENCHMARK_REGISTRY = {
+    "token_efficiency": token_efficiency.run,
+    "impact_accuracy": impact_accuracy.run,
+    "flow_completeness": flow_completeness.run,
+    "search_quality": search_quality.run,
+    "build_performance": build_performance.run,
+    "multi_hop_retrieval": multi_hop_retrieval.run,
+    "agent_baseline": agent_baseline.run,
+}
+
+CONFIGS_DIR = Path(__file__).parent / "configs"
+DEFAULT_OUTPUT = Path("evaluate/results")
+DEFAULT_REPOS = Path("evaluate/test_repos")
+
+
+def _require_yaml():
+    if yaml is None:
+        raise ImportError("pyyaml is required: pip install code-review-graph[eval]")
+
+
+def load_config(name: str) -> dict:
+    """Load a single benchmark config by name."""
+    _require_yaml()
+    path = CONFIGS_DIR / f"{name}.yaml"
+    with open(path) as f:
+        return yaml.safe_load(f)
+
+
+def load_all_configs() -> list[dict]:
+    """Load all benchmark configs from the configs directory."""
+    _require_yaml()
+    configs = []
+    for p in sorted(CONFIGS_DIR.glob("*.yaml")):
+        with open(p) as f:
+            configs.append(yaml.safe_load(f))
+    return configs
+
+
+def clone_or_update(config: dict, repos_dir: Path | None = None) -> Path:
+    """Clone or update a repository at the config's pinned ``commit`` SHA.
+
+    Full clones (no ``--depth``) are required: the pinned ``test_commits`` are
+    often older than any reasonable shallow-clone window, and a missed SHA
+    used to silently fall back to ``git diff HEAD~1 HEAD`` — producing
+    benchmark numbers tied to whatever upstream HEAD looked like that day.
+
+    Every subprocess call's exit status is checked; failures raise
+    ``RuntimeError`` so reproducibility issues surface immediately instead of
+    yielding garbage results.
+    """
+    repos_dir = repos_dir or DEFAULT_REPOS
+    repos_dir.mkdir(parents=True, exist_ok=True)
+    repo_path = repos_dir / config["name"]
+
+    if repo_path.exists():
+        proc = subprocess.run(
+            ["git", "fetch", "--all", "--tags"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"git fetch failed in {repo_path}: {proc.stderr.strip()}"
+            )
+    else:
+        proc = subprocess.run(
+            ["git", "clone", config["url"], str(repo_path)],
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"git clone failed for {config['url']}: {proc.stderr.strip()}"
+            )
+
+    commit = config.get("commit", "HEAD")
+    if commit != "HEAD":
+        proc = subprocess.run(
+            ["git", "checkout", commit],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"git checkout {commit} failed in {repo_path}: "
+                f"{proc.stderr.strip()}"
+            )
+
+    return repo_path
+
+
+def write_csv(results: list[dict], path: Path) -> None:
+    """Write benchmark results to a CSV file."""
+    if not results:
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fieldnames = list(results[0].keys())
+    with open(path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(results)
+
+
+def run_eval(
+    repos: list[str] | None = None,
+    benchmarks: list[str] | None = None,
+    output_dir: str | Path | None = None,
+) -> dict[str, list[dict]]:
+    """Run evaluation benchmarks across repositories.
+
+    Args:
+        repos: List of repo config names to evaluate (None = all).
+        benchmarks: List of benchmark names to run (None = all).
+        output_dir: Directory for CSV output files.
+
+    Returns:
+        Dict mapping ``{repo}_{benchmark}`` to list of result dicts.
+    """
+    output_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if repos:
+        configs = [load_config(r) for r in repos]
+    else:
+        configs = load_all_configs()
+
+    benchmark_names = benchmarks or list(BENCHMARK_REGISTRY.keys())
+    all_results: dict[str, list[dict]] = {}
+    today = date.today().isoformat()
+
+    for config in configs:
+        name = config["name"]
+        logger.info("Evaluating %s...", name)
+
+        # Resolve the repo path to an absolute Path before handing it to
+        # full_build / get_db_path so the stored qualified_names match what
+        # the CLI/MCP layer produces (those paths go through _get_store ->
+        # _validate_repo_root which .resolve()s). Without this, a later
+        # ``code-review-graph update --repo <relative>`` writes the same
+        # function under a new absolute-prefixed qualified_name, leaving the
+        # graph with duplicate nodes for the same source location.
+        repo_path = clone_or_update(config).resolve()
+
+        # Build graph
+        from code_review_graph.graph import GraphStore
+        from code_review_graph.incremental import full_build, get_db_path
+        from code_review_graph.postprocessing import run_post_processing
+
+        db_path = get_db_path(repo_path)
+        store = GraphStore(db_path)
+
+        full_build(repo_path, store)
+        # full_build is the parsing-only primitive; the higher-level CLI/MCP
+        # wrappers run postprocessing on top. The eval framework bypasses
+        # those, so call it directly here. Without this, FTS5 stays empty
+        # and downstream benchmarks (token_efficiency, search_quality)
+        # silently produce useless results. See: search.rebuild_fts_index.
+        pp_result = run_post_processing(store)
+        for warning in pp_result.get("warnings", []):
+            logger.warning("  postprocessing: %s", warning)
+
+        for bench_name in benchmark_names:
+            if bench_name not in BENCHMARK_REGISTRY:
+                logger.warning("Unknown benchmark: %s", bench_name)
+                continue
+
+            logger.info("  Running %s...", bench_name)
+            try:
+                bench_fn = BENCHMARK_REGISTRY[bench_name]
+                results = bench_fn(repo_path, store, config)
+
+                key = f"{name}_{bench_name}"
+                all_results[key] = results
+                write_csv(results, output_dir / f"{key}_{today}.csv")
+                logger.info("  %s: %d result(s)", bench_name, len(results))
+            except Exception as e:
+                logger.error("  %s failed: %s", bench_name, e)
+                all_results[f"{name}_{bench_name}"] = []
+
+        store.close()
+
+    return all_results