diffbutler 0.5.0__py3-none-any.whl

diffbutler/reviewers/extraction.py

@@ -0,0 +1,198 @@
+"""Declarative offline extraction of backend envelopes; no backend flag assumptions."""
+
+import json
+from dataclasses import dataclass
+from typing import cast
+
+from diffbutler.reviewers.base import ReviewOutput
+
+
+def _pairs(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    result: dict[str, object] = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError("duplicate key")
+        result[key] = value
+    return result
+
+
+def _constant(value: str) -> object:
+    raise ValueError("nonfinite JSON number")
+
+
+def _load(text: str) -> object:
+    return json.loads(text, object_pairs_hook=_pairs, parse_constant=_constant)
+
+
+def _at(value: object, path: tuple[str, ...]) -> object:
+    for part in path:
+        if not isinstance(value, dict):
+            return None
+        value = cast(dict[str, object], value).get(part)
+    return value
+
+
+@dataclass(frozen=True)
+class EnvelopeFormat:
+    text_path: tuple[str, ...] = ("result",)
+    session_path: tuple[str, ...] | None = ("session_id",)
+    error_path: tuple[str, ...] | None = ("is_error",)
+
+
+@dataclass(frozen=True)
+class StreamFormat:
+    """Only complete assistant messages are selected, never deltas or tool output.
+
+    Required successful terminal events distinguish a complete turn from a
+    truncated stream. Selectors describe a verified backend protocol; these
+    defaults are a synthetic testing protocol, not a Codex/Claude preset.
+    """
+
+    type_path: tuple[str, ...] = ("type",)
+    assistant_type: str = "assistant"
+    text_path: tuple[str, ...] = ("text",)
+    session_type: str = "session"
+    session_path: tuple[str, ...] = ("id",)
+    complete_type: str = "complete"
+    failure_types: tuple[str, ...] = ("error", "failed")
+    assistant_filter_path: tuple[str, ...] | None = None
+    assistant_filter_value: str | None = None
+    require_session: bool = False
+    session_filter_path: tuple[str, ...] | None = None
+    session_filter_value: str | None = None
+    terminal_text_path: tuple[str, ...] | None = None
+    terminal_success_path: tuple[str, ...] | None = None
+    terminal_success_value: object = None
+    terminal_error_path: tuple[str, ...] | None = None
+
+    def __post_init__(self) -> None:
+        types = (self.assistant_type, self.session_type, self.complete_type, *self.failure_types)
+        if not all(types) or len(types) != len(set(types)):
+            raise ValueError("stream event types must be nonempty and distinct")
+
+
+type OutputFormat = EnvelopeFormat | StreamFormat
+
+
+def backend_failure_reason(output: ReviewOutput, format: OutputFormat | None) -> str | None:
+    """Recognize known failures without copying private backend text into diagnostics."""
+    messages = [output.stderr.decode("utf-8", errors="replace")]
+    if isinstance(format, StreamFormat):
+        for line in output.stdout.decode("utf-8", errors="replace").splitlines():
+            try:
+                event = _load(line)
+            except ValueError, RecursionError:
+                continue
+            if _at(event, format.type_path) in format.failure_types:
+                for path in (("message",), ("error", "message")):
+                    message = _at(event, path)
+                    if isinstance(message, str):
+                        messages.append(message)
+    if any("thread already has an active writer" in " ".join(m.split()) for m in messages):
+        return (
+            "Codex conversation is busy (thread already has an active writer). "
+            "Exit any other session using this conversation, wait briefly, then retry explicitly."
+        )
+    return None
+
+
+@dataclass(frozen=True)
+class ExtractedOutput:
+    """Raw output is never replaced by extracted assistant text."""
+
+    output: ReviewOutput
+    text: bytes | None
+    backend_session_id: str | None
+    error: str | None = None
+
+
+def _identity(value: object, previous: str | None) -> str:
+    if (
+        not isinstance(value, str)
+        or not value.strip()
+        or len(value) > 1024
+        or any(ord(char) < 32 or 127 <= ord(char) <= 159 for char in value)
+        or previous not in (None, value)
+    ):
+        raise ValueError("invalid or conflicting session identity")
+    return value
+
+
+def extract_output(output: ReviewOutput, format: OutputFormat) -> ExtractedOutput:
+    """Fail the entire capture on malformed/truncated/error events, retaining raw bytes.
+
+    No progress text is returned: tool events and earlier messages can contain
+    secrets. Final assistant text is validated and retained by the review parser.
+    """
+    session_id = output.backend_session_id
+    try:
+        if session_id is not None:
+            _identity(session_id, None)
+        text = output.stdout.decode("utf-8")
+        if isinstance(format, EnvelopeFormat):
+            root = _load(text)
+            if not isinstance(root, dict):
+                raise ValueError
+            root = cast(dict[str, object], root)
+            answer = _at(root, format.text_path)
+            if format.error_path is not None and _at(root, format.error_path) not in (None, False):
+                raise ValueError
+            if format.session_path is not None:
+                value = _at(root, format.session_path)
+                if value is not None:
+                    session_id = _identity(value, session_id)
+        else:
+            answer: object = None
+            complete = False
+            for line in text.splitlines():
+                if not line.strip():
+                    continue
+                if complete:
+                    raise ValueError
+                event = _load(line)
+                kind = _at(event, format.type_path)
+                if not isinstance(event, dict) or not isinstance(kind, str):
+                    raise ValueError
+                event = cast(dict[str, object], event)
+                if kind in format.failure_types:
+                    raise ValueError
+                if kind == format.session_type and (
+                    format.session_filter_path is None
+                    or _at(event, format.session_filter_path) == format.session_filter_value
+                ):
+                    session_id = _identity(_at(event, format.session_path), session_id)
+                elif (
+                    format.terminal_text_path is None
+                    and kind == format.assistant_type
+                    and (
+                        format.assistant_filter_path is None
+                        or _at(event, format.assistant_filter_path) == format.assistant_filter_value
+                    )
+                ):
+                    answer = _at(event, format.text_path)
+                    if not isinstance(answer, str):
+                        raise ValueError
+                elif kind == format.complete_type:
+                    if format.terminal_success_path is not None and (
+                        _at(event, format.terminal_success_path) != format.terminal_success_value
+                    ):
+                        raise ValueError
+                    if format.terminal_error_path is not None and (
+                        _at(event, format.terminal_error_path) is not False
+                    ):
+                        raise ValueError
+                    if format.terminal_text_path is not None:
+                        answer = _at(event, format.terminal_text_path)
+                        session_id = _identity(_at(event, format.session_path), session_id)
+                    complete = True
+            if not complete or (format.require_session and session_id is None):
+                raise ValueError
+        if not isinstance(answer, str) or not answer.strip():
+            raise ValueError
+        if output.exit_code != 0:
+            return ExtractedOutput(output, None, session_id, "reviewer exited unsuccessfully")
+        return ExtractedOutput(output, answer.encode("utf-8"), session_id)
+    except ValueError, RecursionError:
+        return ExtractedOutput(
+            output, None, None, backend_failure_reason(output, format) or "invalid backend output"
+        )

diffbutler/reviewers/native.py

@@ -0,0 +1,238 @@
+"""Native agent launch contract. Sandboxing belongs to the installed agent CLI."""
+
+import asyncio
+import json
+import logging
+import os
+import signal
+import tomllib
+from collections.abc import Awaitable, Callable, Mapping
+from contextlib import suppress
+from dataclasses import dataclass
+from importlib.resources import files
+from pathlib import Path
+from typing import cast
+from uuid import UUID
+
+from diffbutler.reviewers.base import ReviewOutput
+from diffbutler.reviewers.extraction import StreamFormat
+
+CODEX_OUTPUT = StreamFormat(
+    assistant_type="item.completed",
+    text_path=("item", "text"),
+    session_type="thread.started",
+    session_path=("thread_id",),
+    complete_type="turn.completed",
+    failure_types=("error", "turn.failed"),
+    assistant_filter_path=("item", "type"),
+    assistant_filter_value="agent_message",
+    require_session=True,
+)
+
+
+CLAUDE_OUTPUT = StreamFormat(
+    assistant_type="assistant",  # Only the final result is an answer.
+    session_type="system",
+    session_path=("session_id",),
+    session_filter_path=("subtype",),
+    session_filter_value="init",
+    complete_type="result",
+    terminal_text_path=("result",),
+    terminal_success_path=("subtype",),
+    terminal_success_value="success",
+    terminal_error_path=("is_error",),
+    require_session=True,
+)
+
+
+class NativeError(RuntimeError):
+    """Fixed local diagnostic without captured agent text or credentials."""
+
+
+@dataclass(frozen=True)
+class NativePreset:
+    background: tuple[str, ...]
+    background_resume: tuple[str, ...]
+    foreground: tuple[str, ...]
+    name: str = "codex"
+
+    @property
+    def output_format(self) -> StreamFormat:
+        return CLAUDE_OUTPUT if self.name == "claude" else CODEX_OUTPUT
+
+    def command_at(self, command: tuple[str, ...], cwd: Path) -> tuple[str, ...]:
+        return (*command, "-C", str(cwd)) if self.name == "codex" else command
+
+    def argv(
+        self,
+        command: tuple[str, ...],
+        session_id: str | None,
+        *,
+        foreground: bool = False,
+        session_name: str | None = None,
+    ) -> tuple[str, ...]:
+        if session_id is not None:
+            try:
+                UUID(session_id)
+            except ValueError:
+                raise NativeError("Invalid native conversation identity") from None
+        if foreground and session_id is None:
+            raise NativeError("No saved native conversation to resume")
+        arguments = (
+            self.foreground
+            if foreground
+            else (self.background_resume if session_id else self.background)
+        )
+        # Opaque identities occupy a single argv element, never a shell/template.
+        return (
+            *command,
+            *(("--name", session_name) if self.name == "claude" and session_name else ()),
+            *((session_id or "") if arg == "{session_id}" else arg for arg in arguments),
+        )
+
+
+def codex_preset() -> NativePreset:
+    return native_preset("codex")
+
+
+def native_preset(name: str) -> NativePreset:
+    if name not in {"codex", "claude"}:
+        raise NativeError("Unsupported native reviewer preset")
+    data = tomllib.loads(files("diffbutler.reviewers").joinpath("presets.toml").read_text())[name]
+    return NativePreset(
+        tuple(data["background"]),
+        tuple(data["background_resume"]),
+        tuple(data["foreground"]),
+        name,
+    )
+
+
+async def stop_process_group(pid: int, *, grace_s: float = 2.0) -> None:
+    """Let owned agents release native resources before forcing remaining members out."""
+    try:
+        os.killpg(pid, signal.SIGTERM)
+    except ProcessLookupError:
+        return
+    try:
+        async with asyncio.timeout(grace_s):
+            while True:
+                await asyncio.sleep(0.05)
+                try:
+                    os.killpg(pid, 0)
+                except ProcessLookupError:
+                    return
+    except TimeoutError:
+        pass
+    finally:
+        with suppress(ProcessLookupError):
+            os.killpg(pid, signal.SIGKILL)
+
+
+class CapturedProcess:
+    """Own a trusted CLI's process group and captured pipes, including spawn races.
+
+    Deliberately detached processes are outside this native CLI supervision model.
+    EOF never releases ownership: stop_and_wait must first stop and reap the group.
+    """
+
+    def __init__(self) -> None:
+        self.spawn: asyncio.Task[asyncio.subprocess.Process] | None = None
+        self.process: asyncio.subprocess.Process | None = None
+        self.readers: list[asyncio.Task[bytes]] = []
+        self.stdout = bytearray()
+        self.stderr = bytearray()
+
+    async def capture(
+        self,
+        argv: tuple[str, ...],
+        *,
+        cwd: Path,
+        env: Mapping[str, str],
+        input: bytes,
+        checkpoint: Callable[[str], Awaitable[None]] | None = None,
+        output_format: StreamFormat = CODEX_OUTPUT,
+    ) -> ReviewOutput:
+        if self.spawn is not None:
+            raise NativeError("A process execution cannot be reused")
+        self.spawn = asyncio.create_task(
+            asyncio.create_subprocess_exec(
+                *argv,
+                cwd=cwd,
+                env=env,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                start_new_session=True,
+            )
+        )
+        self.process = await asyncio.shield(self.spawn)
+        process = self.process
+        assert process is not None
+        assert (
+            process.stdin is not None and process.stdout is not None and process.stderr is not None
+        )
+
+        async def collect(stream: asyncio.StreamReader, *, events: bool) -> bytes:
+            result = self.stdout if events else self.stderr
+            pending = bytearray()
+            while chunk := await stream.read(65536):
+                result.extend(chunk)
+                if events and checkpoint is not None:
+                    pending.extend(chunk)
+                    while b"\n" in pending:
+                        line, _, rest = pending.partition(b"\n")
+                        pending = bytearray(rest)
+                        try:
+                            event = cast(object, json.loads(line))
+                        except ValueError:
+                            continue  # Strict final extraction rejects malformed output.
+                        if isinstance(event, dict):
+                            fields = cast(dict[str, object], event)
+                            if fields.get("type") == output_format.session_type and (
+                                output_format.session_filter_path is None
+                                or fields.get(output_format.session_filter_path[0])
+                                == output_format.session_filter_value
+                            ):
+                                identity = fields.get(output_format.session_path[0])
+                                if not isinstance(identity, str):
+                                    raise NativeError("Invalid native conversation identity")
+                                await checkpoint(identity)
+            return bytes(result)
+
+        self.readers = [
+            asyncio.create_task(collect(process.stdout, events=True)),
+            asyncio.create_task(collect(process.stderr, events=False)),
+        ]
+        try:
+            process.stdin.write(input)
+            await process.stdin.drain()
+        except BrokenPipeError, ConnectionResetError:
+            pass
+        finally:
+            process.stdin.close()
+        stdout, stderr = await asyncio.gather(*self.readers)
+        code = await process.wait()
+        logging.getLogger(__name__).info("Background agent exited code=%d", code)
+        return ReviewOutput(stdout, stderr, code)
+
+    async def stop_and_wait(self) -> None:
+        if self.spawn is None:
+            return
+        try:
+            self.process = await asyncio.shield(self.spawn)
+        except OSError:
+            return  # Spawn failed; no process exists.
+        process = self.process
+        assert process is not None
+        await stop_process_group(process.pid)
+        for reader in self.readers:
+            reader.cancel()
+        await asyncio.gather(*self.readers, return_exceptions=True)
+
+        async def drain(stream: asyncio.StreamReader | None, target: bytearray) -> None:
+            if stream is not None:
+                while chunk := await stream.read(65536):
+                    target.extend(chunk)
+
+        await asyncio.gather(drain(process.stdout, self.stdout), drain(process.stderr, self.stderr))
+        await process.wait()

diffbutler/reviewers/parsing.py

@@ -0,0 +1,247 @@
+"""Parse captured review text without executing a backend or changing queue state."""
+
+import json
+from dataclasses import dataclass, replace
+from typing import Annotated, Literal, cast
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from diffbutler.domain.review import Side, SourceFile
+from diffbutler.reviewers.base import ReviewOutput
+from diffbutler.reviewers.extraction import OutputFormat, backend_failure_reason, extract_output
+
+Text = Annotated[str, Field(min_length=1, pattern=r"\S")]
+Line = Annotated[int, Field(gt=0)]
+Severity = Literal["blocking", "important", "minor", "nit", "question"]
+Category = Literal["correctness", "security", "performance", "api", "tests", "style", "docs"]
+
+
+class _Record(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, extra="forbid")
+
+
+class Location(_Record):
+    path: Text
+    line: Line
+
+    @field_validator("path")
+    @classmethod
+    def relative_path(cls, value: str) -> str:
+        SourceFile(value, Side.HEAD)
+        return value
+
+
+class LookHere(Location):
+    why: Text
+
+
+class Orientation(_Record):
+    intent: Text
+    shape: Text
+    look_here: tuple[LookHere, ...]
+    low_risk: tuple[Text, ...]
+    unverified: tuple[Text, ...]
+
+
+class Finding(Location):
+    start_line: Line | None = None
+    side: Literal["LEFT", "RIGHT"]
+    severity: Severity
+    category: Category
+    title: Text
+    body: Text
+    suggestion: str | None = None
+    confidence: float = Field(ge=0, le=1, allow_inf_nan=False)
+
+    @model_validator(mode="after")
+    def ordered_range(self) -> Finding:
+        if self.start_line is not None and self.start_line > self.line:
+            raise ValueError("finding range is reversed")
+        return self
+
+
+class ReviewDocument(_Record):
+    orientation: Orientation
+    findings: tuple[Finding, ...]
+
+
+class DraftReview(_Record):
+    """Human-readable analysis retained independently of raw backend transcripts."""
+
+    markdown: Text
+
+    @property
+    def no_findings(self) -> bool:
+        """Only an explicit, unambiguous leading outcome establishes a clean review."""
+        lines = self.markdown.strip().splitlines()
+        outcomes = [line for line in lines if line.strip().startswith("Review outcome:")]
+        return lines[0] == "Review outcome: no_findings" and len(outcomes) == 1
+
+
+type ResponseFormat = Literal["json", "markdown"]
+
+
+@dataclass(frozen=True)
+class ParsedReview:
+    """Raw bytes remain available on failure; findings are never silently dropped.
+
+    Locations are only structurally valid. DiffIndex validation is still required
+    before any anchor can be considered commentable.
+    """
+
+    output: ReviewOutput
+    document: ReviewDocument | None
+    tier: Literal["json", "fence", "object", "markdown"] | None
+    notes: tuple[str, ...] = ()
+    error: str | None = None
+    draft: DraftReview | None = None
+
+
+class _InvalidJSON(ValueError):
+    pass
+
+
+def _object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    result: dict[str, object] = {}
+    for key, value in pairs:
+        if key in result:
+            raise _InvalidJSON("duplicate key")
+        result[key] = value
+    return result
+
+
+def _constant(value: str) -> object:
+    raise _InvalidJSON("nonfinite number")
+
+
+def _load(text: str) -> object:
+    return json.loads(text, object_pairs_hook=_object, parse_constant=_constant)
+
+
+def _outer_object(text: str) -> str | None:
+    """Extract the first outer object, respecting quoted braces and escapes."""
+    start = text.find("{")
+    if start < 0:
+        return None
+    depth = 0
+    quoted = escaped = False
+    for offset in range(start, len(text)):
+        char = text[offset]
+        if quoted:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                quoted = False
+        elif char == '"':
+            quoted = True
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : offset + 1]
+    return None
+
+
+def _last_fence(text: str) -> str | None:
+    # Scan once, including an unfinished final fence; never select an older answer.
+    lines: list[str] | None = None
+    candidate: str | None = None
+    for line in text.splitlines(keepends=True):
+        marker = line.rstrip(" \t\r\n")
+        if marker == "```json":
+            lines = []
+        elif marker == "```" and lines is not None:
+            candidate = "".join(lines)
+            lines = None
+        elif lines is not None:
+            lines.append(line)
+    if lines is not None:
+        raise _InvalidJSON("unfinished final fence")
+    return candidate
+
+
+def _normalize(value: object) -> tuple[object, tuple[str, ...]]:
+    notes: list[str] = []
+    if isinstance(value, dict):
+        root = cast(dict[str, object], value)
+        findings = root.get("findings")
+        if isinstance(findings, list):
+            for index, item in enumerate(cast(list[object], findings)):
+                if not isinstance(item, dict):
+                    continue
+                finding = cast(dict[str, object], item)
+                for key, choices, fallback in (
+                    ("severity", {"blocking", "important", "minor", "nit", "question"}, "minor"),
+                    (
+                        "category",
+                        {"correctness", "security", "performance", "api", "tests", "style", "docs"},
+                        "style",
+                    ),
+                ):
+                    label = finding.get(key)
+                    if isinstance(label, str) and label not in choices:
+                        finding[key] = fallback
+                        notes.append(f"finding {index + 1}: unknown {key} normalized to {fallback}")
+    return cast(object, value), tuple(notes)
+
+
+def parse_review(
+    output: ReviewOutput,
+    *,
+    output_format: OutputFormat | None = None,
+    response_format: ResponseFormat = "json",
+) -> ParsedReview:
+    """Parse assistant text or a declaratively selected backend envelope/stream.
+
+    Markdown is retained as written, without inferring issue counts or severity.
+    Legacy JSON schema errors fail the entire result.
+    Diagnostics contain no backend text; raw stdout/stderr stay in the result.
+    """
+    if output.exit_code != 0:
+        return ParsedReview(
+            output,
+            None,
+            None,
+            error=backend_failure_reason(output, output_format)
+            or (
+                f"reviewer exited unsuccessfully (exit code {output.exit_code}); "
+                "inspect captured output"
+            ),
+        )
+    payload = output.stdout
+    if output_format is not None:
+        extracted = extract_output(output, output_format)
+        output = replace(output, backend_session_id=extracted.backend_session_id)
+        if extracted.error is not None or extracted.text is None:
+            return ParsedReview(output, None, None, error=extracted.error)
+        payload = extracted.text
+    try:
+        text = payload.decode("utf-8")
+    except UnicodeDecodeError:
+        return ParsedReview(output, None, None, error="output is not UTF-8")
+    if response_format == "markdown":
+        if not text.strip():
+            return ParsedReview(output, None, None, error="empty review response")
+        return ParsedReview(output, None, "markdown", draft=DraftReview(markdown=text))
+    tier: Literal["json", "fence", "object"] = "json"
+    try:
+        try:
+            value = _load(text)
+        except json.JSONDecodeError:
+            candidate = _last_fence(text)
+            tier = "fence"
+            try:
+                value = _load(candidate) if candidate is not None else _load("")
+            except json.JSONDecodeError:
+                # Do not recover an earlier clean answer from a malformed final fence.
+                candidate = _outer_object(candidate if candidate is not None else text)
+                tier = "object"
+                value = _load(candidate or "")
+        value, notes = _normalize(value)
+        document = ReviewDocument.model_validate_json(json.dumps(value))
+    except ValueError, RecursionError:
+        return ParsedReview(output, None, None, error="invalid review document")
+    return ParsedReview(output, document, tier, notes)

diffbutler/reviewers/presets.toml

@@ -0,0 +1,12 @@
+# Native CLI arguments; command assembly is covered by offline acceptance tests.
+[codex]
+background = ["-a", "never", "-s", "read-only", "exec", "--json", "--skip-git-repo-check", "-"]
+background_resume = ["-a", "never", "-s", "read-only", "exec", "resume", "--json", "--skip-git-repo-check", "{session_id}", "-"]
+foreground = ["-a", "on-request", "resume", "{session_id}"]
+
+# Bash uses native auto permission checks; do not blanket-allow it.
+# Keep normal user configuration; override tools and permissions per launch mode.
+[claude]
+background = ["--print", "--output-format", "stream-json", "--verbose", "--tools", "Read,Glob,Grep,Bash", "--allowedTools", "Read,Glob,Grep", "--permission-mode", "auto", "--permission-prompts", "none"]
+background_resume = ["--print", "--output-format", "stream-json", "--verbose", "--tools", "Read,Glob,Grep,Bash", "--allowedTools", "Read,Glob,Grep", "--permission-mode", "auto", "--permission-prompts", "none", "--resume", "{session_id}"]
+foreground = ["--tools", "default", "--permission-mode", "auto", "--resume", "{session_id}"]