semantic-code-review 0.24.2__py3-none-any.whl

semantic_code_review/augment/hunks.py

@@ -0,0 +1,264 @@
+"""Per-hunk pass: intent + segments + smells + context + refs.
+
+Fused into a single call per hunk for v1. The system prompt frames the
+job as comprehension-first; smells are secondary.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from pydantic_ai import CachePoint
+from pydantic_ai.messages import UserContent
+
+from ..augment.schemas import (
+    AnnotatedDiff,
+    AnnotatedFile,
+    AnnotatedHunk,
+    FoldDescription,
+    HunkAnnotations,
+    LineNote,
+    Overview,
+    ParsedHunk,
+    Ref,
+    Segment,
+    Smell,
+)
+from ..cache.store import CacheStore
+from .agents import Client, make_hunk_agent
+from .pass_ import PassMeta, run_pass
+from .prompts import HUNK_SYSTEM
+from .tools import TOOL_FUNCTIONS, RepoTools
+
+log = logging.getLogger(__name__)
+
+_HUNK = PassMeta(
+    name="hunk",
+    submit_tool="submit_annotations",
+    tool_names=tuple(fn.__name__ for fn in TOOL_FUNCTIONS),
+)
+
+
+# Anthropic prompt-caching settings applied to every per-hunk call.
+#
+# The cacheable prefix on this pass is `[system prompt] + [tool defs] +
+# [overview] + [file summary]`. The first two stay byte-identical for
+# every hunk in the run, so caching them buys cross-hunk reuse on a
+# multi-hunk PR. The CachePoint markers in `format_hunk_prompt` then
+# split the user prompt so within-file (overview + summary cached) and
+# within-PR (overview cached) prefixes are reused too.
+#
+# AnthropicModelSettings keys are silently ignored by non-Anthropic
+# backends (TypedDict total=False), so this is safe to apply
+# unconditionally — Google + the CLI drivers see them as no-ops.
+_HUNK_CACHE_SETTINGS: dict[str, Any] = {
+    "anthropic_cache_instructions": True,  # system prompt block
+    "anthropic_cache_tool_definitions": True,  # tools/<RepoTools>
+}
+
+
+def format_hunk_prompt(
+    fp: AnnotatedFile,
+    hunk: AnnotatedHunk,
+    overview_json: str,
+    file_summary: str,
+) -> list[UserContent]:
+    """Assemble the user-prompt blocks for one hunk call.
+
+    Returns a `UserContent` list with `CachePoint` markers between the
+    cacheable prefix sections (overview, file summary) and the
+    per-hunk text. pydantic-ai's Anthropic adapter translates each
+    `CachePoint` into a `cache_control: ephemeral` annotation on the
+    preceding text block; non-supporting providers filter the markers
+    out and concatenate the text blocks. Fold-region summaries are not
+    produced here — the review server fires a focused call on first
+    fold-close; see :mod:`semantic_code_review.augment.fold_summary`.
+    """
+    hunk_text = (
+        f"# File\npath: {fp.path}\nlang: {fp.ann.lang or ''}\n\n# Hunk\n{hunk.parsed.header}\n{hunk.parsed.body}"
+    )
+    return [
+        f"# PR overview\n{overview_json}",
+        CachePoint(),
+        f"# File summary\n{file_summary}",
+        CachePoint(),
+        hunk_text,
+    ]
+
+
+def _hunk_trace_path(
+    trace_dir: Path | None,
+    fp: AnnotatedFile,
+    hunk: AnnotatedHunk,
+) -> Path | None:
+    if trace_dir is None:
+        return None
+    safe_file = fp.path.replace("/", "_")
+    safe_hunk = (
+        hunk.parsed.header.replace(" ", "_").replace("@", "").replace(",", "_").replace("+", "p").replace("-", "m")
+    )
+    return trace_dir / f"hunk-{safe_file}-{safe_hunk[:40]}.json"
+
+
+async def run_hunk_pass(
+    client: Client,
+    *,
+    fp: AnnotatedFile,
+    hunk: AnnotatedHunk,
+    overview_json: str,
+    file_summary: str,
+    repo_tools: RepoTools,
+    model: str,
+    cache: CacheStore | None = None,
+    trace_dir: Path | None = None,
+) -> dict[str, Any]:
+    payload = await run_pass(
+        _HUNK,
+        client=client,
+        agent=make_hunk_agent(client.model),
+        user_content=format_hunk_prompt(fp, hunk, overview_json, file_summary),
+        system=HUNK_SYSTEM,
+        model=model,
+        cache_inputs=(
+            overview_json,
+            file_summary,
+            fp.path,
+            hunk.parsed.header,
+            hunk.parsed.body,
+        ),
+        deps=repo_tools,
+        model_settings=_HUNK_CACHE_SETTINGS,
+        cache=cache,
+        trace_path=_hunk_trace_path(trace_dir, fp, hunk),
+        cache_request={
+            "file": fp.path,
+            "header": hunk.parsed.header,
+            "body_len": len(hunk.parsed.body),
+        },
+    )
+    assert payload is not None  # `_HUNK.swallow_errors` is false
+    return payload
+
+
+def build_hunk_annotations(parsed: ParsedHunk, submit_args: dict[str, Any]) -> HunkAnnotations:
+    """Validate a submit_annotations payload against `parsed` and return
+    a `HunkAnnotations` record.
+
+    Drops segments/fold_descriptions outside the hunk's post-image range
+    or overlapping a previously-kept segment — the LLM occasionally emits
+    pre-image line numbers or off-by-a-few ranges.
+    """
+    hunk_end = parsed.new_start + parsed.new_count - 1
+
+    segments: list[Segment] = []
+    last_end = parsed.new_start - 1
+    for seg in submit_args.get("segments") or []:
+        try:
+            start = int(seg["new_start"])
+            count = int(seg["new_count"])
+        except (KeyError, TypeError, ValueError):
+            log.warning("hunk %s: malformed segment %r — dropped", parsed.header, seg)
+            continue
+        end = start + count - 1
+        if count <= 0 or start < parsed.new_start or end > hunk_end:
+            log.warning(
+                "hunk %s: segment +%d..+%d outside range +%d..+%d — dropped",
+                parsed.header,
+                start,
+                end,
+                parsed.new_start,
+                hunk_end,
+            )
+            continue
+        if start <= last_end:
+            log.warning(
+                "hunk %s: segment +%d..+%d overlaps previous (ends +%d) — dropped",
+                parsed.header,
+                start,
+                end,
+                last_end,
+            )
+            continue
+        segments.append(
+            Segment(
+                new_start=start,
+                new_count=count,
+                intent=seg.get("intent", "") or "",
+                smells=[_smell(s) for s in seg.get("smells") or []],
+                context=seg.get("context", "") or "",
+                refs=[Ref(**_ref(r)) for r in seg.get("refs") or []],
+            )
+        )
+        last_end = end
+
+    fold_descriptions: list[FoldDescription] = []
+    for fd in submit_args.get("fold_descriptions") or []:
+        try:
+            start = int(fd["new_start"])
+            count = int(fd["new_count"])
+        except (KeyError, TypeError, ValueError):
+            log.warning("hunk %s: malformed fold_description %r — dropped", parsed.header, fd)
+            continue
+        end = start + count - 1
+        if count <= 0 or start < parsed.new_start or end > hunk_end:
+            log.warning(
+                "hunk %s: fold +%d..+%d outside range — dropped",
+                parsed.header,
+                start,
+                end,
+            )
+            continue
+        summary = (fd.get("summary") or "").strip()
+        if not summary:
+            continue
+        fold_descriptions.append(FoldDescription(right_start=start, right_end=end, summary=summary))
+
+    line_notes = [
+        LineNote(**ln) for ln in submit_args.get("line_notes") or [] if _line_in_hunk(int(ln["line"]), parsed)
+    ]
+
+    return HunkAnnotations(
+        intent=submit_args.get("intent", "") or "",
+        context=submit_args.get("context", "") or "",
+        confidence=submit_args.get("confidence"),
+        smells=[_smell(s) for s in submit_args.get("smells") or []],
+        refs=[Ref(**_ref(r)) for r in submit_args.get("refs") or []],
+        line_notes=line_notes,
+        segments=segments,
+        fold_descriptions=fold_descriptions,
+    )
+
+
+def apply_hunk_annotations(hunk: AnnotatedHunk, submit_args: dict[str, Any]) -> AnnotatedHunk:
+    """Return a new AnnotatedHunk with `ann` set from `submit_args`."""
+    return hunk.model_copy(update={"ann": build_hunk_annotations(hunk.parsed, submit_args)})
+
+
+def _line_in_hunk(line: int, parsed: ParsedHunk) -> bool:
+    return parsed.new_start <= line <= parsed.new_start + parsed.new_count - 1
+
+
+def _smell(d: dict[str, Any]) -> Smell:
+    return Smell(tag=d.get("tag", ""), note=d.get("note", "") or "")
+
+
+def _ref(d: dict[str, Any]) -> dict[str, Any]:
+    return {"path": d["path"], "line": int(d["line"]), "reason": d.get("reason", "") or ""}
+
+
+def overview_to_prompt_json(diff: AnnotatedDiff) -> str:
+    """Serialize the overview into a compact JSON string for the hunk prompt."""
+    if not isinstance(diff.overview, Overview):
+        return "{}"
+    payload = {
+        "summary": diff.overview.summary,
+        "symbols_added": [s.model_dump() for s in diff.overview.symbols_added],
+        "symbols_modified": [s.model_dump() for s in diff.overview.symbols_modified],
+        "symbols_removed": [s.model_dump() for s in diff.overview.symbols_removed],
+        "callgraph_edges": [e.model_dump(by_alias=True) for e in diff.overview.callgraph_edges],
+        "themes": list(diff.overview.themes),
+    }
+    return json.dumps(payload, ensure_ascii=False)

semantic_code_review/augment/mcp_server.py

@@ -0,0 +1,156 @@
+"""Minimal stdio MCP server exposing `RepoTools` to `claude -p`.
+
+Used by `ClaudeCLIClient` when a RepoTools instance is available: we
+inject this server via `claude -p --mcp-config ... --strict-mcp-config`
+so the agentic loop can still `read_file`, `grep`, `list_dir`,
+`read_file_at`, and `git_log` against the run's worktrees — matching
+the behaviour of the API backend's in-process tool loop.
+
+Protocol: JSON-RPC 2.0 over newline-delimited JSON on stdio, per the
+MCP stdio transport spec. Only a subset is implemented:
+
+  - initialize               (request)
+  - notifications/initialized (notification, no response)
+  - tools/list               (request)
+  - tools/call               (request)
+
+Anything else returns a JSON-RPC method-not-found error. No resources,
+prompts, or sampling — we don't need them for read-only repo access.
+
+Run directly::
+
+    python -m semantic_code_review.augment.mcp_server \
+        --head-worktree <path> --repo-git <path> \
+        --base-sha <sha> --head-sha <sha>
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+from .tools import RepoTools, mcp_dispatch, mcp_tool_schemas
+
+log = logging.getLogger("scr.mcp_server")
+
+
+PROTOCOL_VERSION = "2025-06-18"
+SERVER_NAME = "scr"
+SERVER_VERSION = "0.1.0"
+
+
+def _make_error(req_id: Any, code: int, message: str) -> dict[str, Any]:
+    return {"jsonrpc": "2.0", "id": req_id, "error": {"code": code, "message": message}}
+
+
+def _make_result(req_id: Any, result: Any) -> dict[str, Any]:
+    return {"jsonrpc": "2.0", "id": req_id, "result": result}
+
+
+def _handle(req: dict[str, Any], repo_tools: RepoTools) -> dict[str, Any] | None:
+    """Return a response dict, or None for notifications."""
+    method = req.get("method", "")
+    req_id = req.get("id")
+    params = req.get("params") or {}
+    is_notification = "id" not in req
+
+    if method == "initialize":
+        return _make_result(
+            req_id,
+            {
+                "protocolVersion": PROTOCOL_VERSION,
+                "capabilities": {"tools": {"listChanged": False}},
+                "serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION},
+            },
+        )
+
+    if method == "notifications/initialized":
+        # Notification: no response.
+        return None
+
+    if method == "tools/list":
+        return _make_result(req_id, {"tools": mcp_tool_schemas()})
+
+    if method == "tools/call":
+        name = params.get("name", "")
+        args = params.get("arguments") or {}
+        try:
+            output = mcp_dispatch(repo_tools, name, args)
+            return _make_result(
+                req_id,
+                {
+                    "content": [{"type": "text", "text": output}],
+                    "isError": output.startswith("error:"),
+                },
+            )
+        except Exception as e:  # noqa: BLE001
+            return _make_result(
+                req_id,
+                {
+                    "content": [{"type": "text", "text": f"error: {e}"}],
+                    "isError": True,
+                },
+            )
+
+    if is_notification:
+        # Drop unknown notifications silently.
+        return None
+
+    return _make_error(req_id, -32601, f"method not found: {method}")
+
+
+def serve(repo_tools: RepoTools, stdin: Any = None, stdout: Any = None) -> None:
+    """Loop over newline-delimited JSON-RPC on stdio until EOF."""
+    stdin = stdin or sys.stdin
+    stdout = stdout or sys.stdout
+    for raw in stdin:
+        line = raw.strip()
+        if not line:
+            continue
+        try:
+            req = json.loads(line)
+        except json.JSONDecodeError as e:
+            log.warning("dropping malformed JSON-RPC line: %s", e)
+            continue
+        try:
+            response = _handle(req, repo_tools)
+        except Exception as e:
+            log.exception("handler crashed on request %s", req.get("method"))
+            response = _make_error(req.get("id"), -32603, f"internal error: {e}")
+        if response is not None:
+            stdout.write(json.dumps(response, ensure_ascii=False) + "\n")
+            stdout.flush()
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0])
+    parser.add_argument("--head-worktree", required=True, type=Path)
+    parser.add_argument("--repo-git", required=True, type=Path)
+    parser.add_argument("--base-sha", default="")
+    parser.add_argument("--head-sha", default="")
+    parser.add_argument("--log-file", type=Path, default=None)
+    args = parser.parse_args(argv)
+
+    if args.log_file is not None:
+        logging.basicConfig(
+            filename=str(args.log_file),
+            level=logging.INFO,
+            format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        )
+
+    repo_tools = RepoTools(
+        head_worktree=args.head_worktree,
+        repo_git=args.repo_git,
+        base_sha=args.base_sha,
+        head_sha=args.head_sha,
+    )
+    serve(repo_tools)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

semantic_code_review/augment/overview.py

@@ -0,0 +1,233 @@
+"""Overview pass: one call per PR producing the PR-level summary.
+
+Input: PR metadata + diffstat + per-file hunk headers (bodies omitted
+to save tokens). Output: the `Overview` object plus per-file summary
+text and optional `lang` override that populate `FileAnnotations` fields.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from ..augment.schemas import (
+    AnnotatedDiff,
+    AnnotatedFile,
+    FileSymbols,
+    Overview,
+    OverviewEdge,
+    OverviewGroup,
+    OverviewGroupMember,
+    OverviewSymbol,
+)
+from ..cache.store import CacheStore
+from ..structural import SymbolDelta
+from .agents import Client, make_overview_agent
+from .pass_ import PassMeta, run_pass
+from .prompts import OVERVIEW_SYSTEM
+
+log = logging.getLogger(__name__)
+
+
+_OVERVIEW = PassMeta(name="overview", submit_tool="submit_overview")
+
+
+def format_overview_prompt(
+    diff: AnnotatedDiff,
+    meta: dict[str, Any],
+    delta: SymbolDelta | None = None,
+) -> str:
+    """Produce the user-message text for the overview call.
+
+    `delta`, when present and non-empty, seeds the prompt with the
+    deterministic tree-sitter symbol delta (ADR 0001) so the model
+    reports `symbols_*` from ground truth rather than guessing from hunk
+    headers. An absent or empty delta (no supported-language change)
+    appends nothing — the prompt is byte-identical to the pre-seed form.
+    """
+    parts: list[str] = []
+    title = meta.get("title", "")
+    body = (meta.get("body") or "").strip()
+    parts.append(f"# PR\ntitle: {title}\n")
+    if body:
+        # Trim body — overview doesn't need the full novel.
+        if len(body) > 4000:
+            body = body[:4000] + "\n... [PR body truncated for brevity] ..."
+        parts.append(f"body:\n{body}\n")
+
+    parts.append("# Diffstat")
+    for f in diff.files:
+        adds = sum(sum(1 for ln in h.parsed.body.splitlines() if ln.startswith("+")) for h in f.hunks)
+        dels = sum(sum(1 for ln in h.parsed.body.splitlines() if ln.startswith("-")) for h in f.hunks)
+        parts.append(f"  {f.path}  +{adds} -{dels}  ({len(f.hunks)} hunks)")
+
+    # Each hunk header is prefixed with its 0-based `hunk_index` within
+    # the file, so the model can cite `{path, hunk_index}` from the
+    # `groups` output unambiguously.
+    parts.append("\n# Hunk headers")
+    for f in diff.files:
+        parts.append(f"{f.path}")
+        for i, h in enumerate(f.hunks):
+            parts.append(f"  [{i}] {h.parsed.header}")
+
+    seed = _format_symbol_seed(delta)
+    if seed:
+        parts.append(seed)
+
+    return "\n".join(parts) + "\n"
+
+
+def _format_symbol_seed(delta: SymbolDelta | None) -> str:
+    """Render the deterministic symbol delta as a prompt section, or `""`.
+
+    Returns the empty string when there's nothing to seed (no delta, or
+    every bucket empty) so callers can keep the prompt byte-identical to
+    the unseeded form for all-unsupported-language diffs.
+    """
+    if delta is None:
+        return ""
+    buckets = (("added", delta.added), ("modified", delta.modified), ("removed", delta.removed))
+    if not any(items for _, items in buckets):
+        return ""
+    lines = [
+        "\n# Symbols changed (deterministic — tree-sitter, not your inference)",
+        "These are the exact definitions that changed between base and head, by "
+        "name and kind. Populate `symbols_added` / `symbols_modified` / "
+        "`symbols_removed` from THIS list verbatim — one entry per line below, "
+        "using its `qualified_name` as the symbol `name`. Do not add, drop, or "
+        "rename entries; languages absent here are simply not yet parseable.",
+    ]
+    for label, items in buckets:
+        lines.append(f"{label}:")
+        if not items:
+            lines.append("  (none)")
+            continue
+        for c in items:
+            lines.append(f"  {c.kind} {c.qualified_name}  ({c.path})")
+    return "\n".join(lines)
+
+
+async def run_overview_pass(
+    client: Client,
+    *,
+    diff: AnnotatedDiff,
+    meta: dict[str, Any],
+    model: str,
+    delta: SymbolDelta | None = None,
+    cache: CacheStore | None = None,
+    trace_dir: Path | None = None,
+) -> dict[str, Any]:
+    """Run the overview call. Returns the raw submit_args from the model.
+
+    `delta`, when present, seeds the prompt with the deterministic
+    structural symbol delta (ADR 0001 Slice 3). It feeds the prompt text,
+    so the cache key (which hashes `user_text`) tracks it automatically.
+    """
+    user_text = format_overview_prompt(diff, meta, delta)
+    payload = await run_pass(
+        _OVERVIEW,
+        client=client,
+        agent=make_overview_agent(client.model),
+        user_content=user_text,
+        system=OVERVIEW_SYSTEM,
+        model=model,
+        cache_inputs=(user_text,),
+        cache=cache,
+        trace_path=(trace_dir / "overview.json") if trace_dir is not None else None,
+        cache_request={"system": OVERVIEW_SYSTEM, "user": user_text},
+    )
+    # `_OVERVIEW.swallow_errors` is false, so `payload` is never None here.
+    assert payload is not None
+    return payload
+
+
+def apply_overview_to_diff(diff: AnnotatedDiff, submit_args: dict[str, Any]) -> AnnotatedDiff:
+    """Fold a submit_overview payload into an AnnotatedDiff. Returns a new
+    AnnotatedDiff; `diff` is not mutated.
+
+    Per-file fields named in the submission overwrite existing
+    `FileAnnotations.summary`/`lang`/`symbols`; files not named are
+    untouched (preserving e.g. the `GENERATED` role pre-set by the
+    pipeline for skipped files).
+    """
+    overview = Overview(
+        summary=submit_args.get("summary", ""),
+        symbols_added=[OverviewSymbol(**s) for s in submit_args.get("symbols_added", [])],
+        symbols_modified=[OverviewSymbol(**s) for s in submit_args.get("symbols_modified", [])],
+        symbols_removed=[OverviewSymbol(**s) for s in submit_args.get("symbols_removed", [])],
+        callgraph_edges=[OverviewEdge.model_validate(e) for e in submit_args.get("callgraph_edges", [])],
+        themes=list(submit_args.get("themes", [])),
+        groups=_resolve_groups(diff, submit_args.get("groups") or []),
+    )
+    by_path = {f["path"]: f for f in submit_args.get("files", [])}
+    new_files: list[AnnotatedFile] = []
+    for fp in diff.files:
+        entry = by_path.get(fp.path)
+        if entry is None:
+            new_files.append(fp)
+            continue
+        sym = entry.get("symbols")
+        ann = fp.ann.model_copy(
+            update={
+                "summary": entry.get("summary", ""),
+                **({"lang": entry["lang"]} if entry.get("lang") else {}),
+                **(
+                    {
+                        "symbols": FileSymbols(
+                            added=list(sym.get("added", [])),
+                            modified=list(sym.get("modified", [])),
+                            removed=list(sym.get("removed", [])),
+                        )
+                    }
+                    if isinstance(sym, dict)
+                    else {}
+                ),
+            }
+        )
+        new_files.append(fp.model_copy(update={"ann": ann}))
+    return diff.model_copy(update={"overview": overview, "files": new_files})
+
+
+def _resolve_groups(diff: AnnotatedDiff, raw_groups: list[dict[str, Any]]) -> list[OverviewGroup]:
+    """Build OverviewGroup instances from raw submit_overview payload.
+
+    Members whose (path, hunk_index) don't resolve to a real hunk in
+    the diff are dropped with a warning. A group whose members all get
+    dropped is itself dropped.
+    """
+    hunks_per_path: dict[str, int] = {fp.path: len(fp.hunks) for fp in diff.files}
+
+    out: list[OverviewGroup] = []
+    for raw in raw_groups:
+        title = (raw.get("title") or "").strip()
+        if not title:
+            continue
+        rationale = (raw.get("rationale") or "").strip()
+        members: list[OverviewGroupMember] = []
+        for m in raw.get("members") or []:
+            try:
+                path = str(m["path"])
+                idx = int(m["hunk_index"])
+            except (KeyError, TypeError, ValueError):
+                log.warning("group %r: malformed member %r — dropped", title, m)
+                continue
+            n = hunks_per_path.get(path)
+            if n is None:
+                log.warning("group %r: path %r not in diff — dropped", title, path)
+                continue
+            if idx < 0 or idx >= n:
+                log.warning(
+                    "group %r: hunk_index %d out of range for %s (n=%d) — dropped",
+                    title,
+                    idx,
+                    path,
+                    n,
+                )
+                continue
+            members.append(OverviewGroupMember(path=path, hunk_index=idx))
+        if not members:
+            log.warning("group %r: no valid members — dropped", title)
+            continue
+        out.append(OverviewGroup(title=title, rationale=rationale, members=members))
+    return out