repro-engine 1.2.0__py3-none-any.whl

repro/__init__.py

@@ -0,0 +1,8 @@
+"""Repro engine."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("repro-engine")
+except PackageNotFoundError:  # imported from a source tree that was never installed
+    __version__ = "0+unknown"

repro/__main__.py

@@ -0,0 +1,3 @@
+from repro.cli import app
+
+app(prog_name="repro")

repro/agents/__init__.py

@@ -0,0 +1,67 @@
+"""Reviewer agents: the LLM backend, its lockdown and dispatch (Stage G; ADR-0007).
+
+``LLMBackend`` is the one way the engine reaches a model; ``AgentSdkBackend`` is its default,
+the Claude Agent SDK under the lockdown in ``lockdown``; ``OpenAICompatibleBackend`` reaches
+Groq or a local server (ADR-0016); and ``CassetteBackend`` records calls and replays them
+offline. Prompts are versioned files (``prompts``). ``run_agent`` runs a call in one of a run's
+dispatch slots, within its ``RunLedger``, and reports it as events.
+``EvidenceTools`` are what agents read: the run's evidence and the change's files (G03).
+"""
+
+from repro.agents.backend import (
+    AgentRequest,
+    AgentResult,
+    Budget,
+    LLMBackend,
+    OnProgress,
+    Outcome,
+    Progress,
+    ReproTool,
+    Role,
+    Usage,
+)
+from repro.agents.cassette import (
+    CassetteBackend,
+    CassetteError,
+    MissingCassetteError,
+    StaleCassetteError,
+)
+from repro.agents.cost import RunLedger
+from repro.agents.dispatch import SLOTS, Dispatch, concurrency_for, default_concurrency
+from repro.agents.events import run_agent
+from repro.agents.evidence import EvidenceView
+from repro.agents.openai_compat import Endpoint, OpenAICompatibleBackend
+from repro.agents.sdk import AgentSdkBackend
+from repro.agents.sources import RepoSources, SourceError
+from repro.agents.tools import EvidenceTools, ToolInputError
+
+__all__ = [
+    "SLOTS",
+    "AgentRequest",
+    "AgentResult",
+    "AgentSdkBackend",
+    "Budget",
+    "CassetteBackend",
+    "CassetteError",
+    "Dispatch",
+    "Endpoint",
+    "EvidenceTools",
+    "EvidenceView",
+    "LLMBackend",
+    "MissingCassetteError",
+    "OnProgress",
+    "OpenAICompatibleBackend",
+    "Outcome",
+    "Progress",
+    "RepoSources",
+    "ReproTool",
+    "Role",
+    "RunLedger",
+    "SourceError",
+    "StaleCassetteError",
+    "ToolInputError",
+    "Usage",
+    "concurrency_for",
+    "default_concurrency",
+    "run_agent",
+]

repro/agents/backend.py

@@ -0,0 +1,129 @@
+"""The interface every LLM call goes through (G01; ADR-0007, ARCHITECTURE §10.5).
+
+An agent call is a request (a role, a system prompt, the user's text, the read-only tools it
+may call, an optional output schema, a model, the prompt's version and a budget) and a result
+(the structured output or text, how the run ended, its usage). The default implementation is
+the Claude Agent SDK (``repro.agents.sdk``); cassettes (``repro.agents.cassette``) and tests
+implement the same protocol, so the stages that use agents never depend on how the model is
+reached.
+
+``on_progress`` reports what the agent is doing while it runs (thinking, a tool call): the
+``agent.state`` events the cockpit's dispatch and workspace panels draw (``repro.agents.events``).
+"""
+
+import re
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any, Final, Literal, Protocol
+
+Role = Literal["read", "build", "review", "gap", "probe", "qa"]
+#: How a run ended: ``ok`` with its output; or why there is none.
+Outcome = Literal["ok", "max_turns", "max_budget", "timeout", "invalid_output", "error"]
+
+
+@dataclass(frozen=True, slots=True)
+class Budget:
+    """What one agent call may spend; every call has one (ADR-0007)."""
+
+    max_turns: int = 8
+    max_budget_usd: float = 1.0
+    timeout_s: float = 600.0
+
+    def __post_init__(self) -> None:
+        if self.max_turns < 1 or self.max_budget_usd <= 0 or self.timeout_s <= 0:
+            raise ValueError("a budget's limits must be positive")
+
+
+#: A tool's name: words of lowercase letters and digits joined by single underscores.
+TOOL_NAME: Final = re.compile(r"(?=.{1,48}$)[a-z][a-z0-9]*(?:_[a-z0-9]+)*")
+#: A prompt's version: its file's name and version number, as ``reviewer/3`` (G02).
+PROMPT_VERSION: Final = re.compile(r"[a-z][a-z0-9_-]{0,47}/[1-9][0-9]{0,5}")
+#: A tool handler: its validated input, then the text it returns to the model.
+ToolHandler = Callable[[Mapping[str, Any]], Awaitable[str]]
+
+
+@dataclass(frozen=True, slots=True)
+class ReproTool:
+    """A read-only, in-process tool the agent may call (``mcp__repro__<name>``)."""
+
+    name: str
+    description: str
+    input_schema: Mapping[str, Any]
+    handler: ToolHandler
+
+    def __post_init__(self) -> None:
+        if not TOOL_NAME.fullmatch(self.name):
+            raise ValueError(f"tool names are lower_snake_case, up to 48 characters: {self.name!r}")
+
+
+@dataclass(frozen=True, slots=True)
+class AgentRequest:
+    role: Role
+    system_prompt: str
+    prompt: str
+    model: str
+    #: The version of the prompt the call was built from (``repro.agents.prompts``).
+    prompt_version: str
+    budget: Budget = field(default_factory=Budget)
+    tools: Sequence[ReproTool] = ()
+    #: JSON Schema (draft-07, ``$ref``-free) the final answer must satisfy, or None for text.
+    output_schema: Mapping[str, Any] | None = None
+    effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None
+
+    def __post_init__(self) -> None:
+        if not PROMPT_VERSION.fullmatch(self.prompt_version):
+            raise ValueError(f"a prompt version is name/number: {self.prompt_version!r}")
+
+
+@dataclass(frozen=True, slots=True)
+class Usage:
+    """Client-side estimates, never billing data (research/04 §1.8)."""
+
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_tokens: int = 0
+    cost_usd: float = 0.0
+    #: Whether ``cost_usd`` rests on a known price; False: the tokens are counted, the cost is
+    #: unknown (0 here, and no ``cost.estimate``; G10, ADR-0016).
+    priced: bool = True
+
+
+@dataclass(frozen=True, slots=True)
+class AgentResult:
+    outcome: Outcome
+    #: The schema-valid answer (with an ``output_schema``), else None.
+    structured: Any = None
+    text: str = ""
+    turns: int = 0
+    usage: Usage = field(default_factory=Usage)
+    #: Why there is no output, in a line (empty when ``ok``).
+    detail: str = ""
+    #: Tool calls the model made, in order: (tool name, whether it was allowed to run).
+    tool_calls: tuple[tuple[str, bool], ...] = ()
+    #: ``usage`` per model the call used (subagents included, research/04 §1.8).
+    by_model: Mapping[str, Usage] = field(default_factory=dict)
+    #: Served from a cassette: ``usage`` is what the call cost when it was recorded.
+    replayed: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class Progress:
+    """What an agent is doing now (``agent.state``'s ``thinking`` and ``tool_call``)."""
+
+    state: Literal["thinking", "tool_call"]
+    #: The model turn it happened in, from 1 (a turn is one model response).
+    turn: int
+    tool: str | None = None
+    detail: str | None = None
+
+
+OnProgress = Callable[[Progress], None]
+
+
+class LLMBackend(Protocol):
+    """Runs one agent call to its end; never raises for what the model does (the result says
+    so), only for a backend that cannot run at all."""
+
+    async def run(
+        self, request: AgentRequest, on_progress: OnProgress | None = None
+    ) -> AgentResult: ...

repro/agents/bias.py

@@ -0,0 +1,109 @@
+"""Bias controls for the reviewers' verdicts (G08; ARCHITECTURE §10.4, research/04 §2.3).
+
+- **Order swap.** LLM judges favour a position: in MT-Bench, GPT-4's verdicts survived swapping
+  the order of what it compared in 65 % of cases. A ``contradicted`` verdict flags the change,
+  so the reviewer is asked again with the evidence and the other claims in the opposite order
+  (``repro.agents.reviewer``'s ``reverse``); unless both orders give it, the verdict degrades
+  to ``unverifiable`` (``degraded``). The ``agent_order_swap`` setting can ask for the second
+  order on every verdict, which is how the flip rate is measured, or not at all.
+- **Panel.** With ``agent_panel``, three reviewers judge each claim (their models in the
+  setting's order, the second with the order reversed) and a verdict stands when two of them
+  give it (``majority``); otherwise it is ``unverifiable``.
+- **Judge ≠ author.** Judges favour their own generations (Panickssery et al.). A change's
+  commits and PR text often name the model that wrote it, as Claude Code's
+  ``Co-Authored-By: Claude …`` trailer does; when every reviewer is of that model's family,
+  the card says so (``judge_is_author``). The families are read from model names only
+  (``family``), never from people's names or addresses.
+
+Every verdict records how many judgements it rests on and how many of them gave it
+(``agreement``).
+"""
+
+import re
+from collections import Counter
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any, Final
+
+#: Model-name tokens and the family (the developer of the model) each belongs to.
+FAMILIES: Final[tuple[tuple[str, str], ...]] = (
+    ("claude", "anthropic"),
+    ("gpt", "openai"),
+    ("codex", "openai"),
+    ("chatgpt", "openai"),
+    ("gemini", "google"),
+    ("qwen", "alibaba"),
+    ("llama", "meta"),
+    ("deepseek", "deepseek"),
+    ("mistral", "mistral"),
+)
+#: The lines that name a change's author: a co-author trailer, a "generated with" line.
+_AUTHOR_LINE: Final = re.compile(
+    r"^\s*(?:co-authored-by:|.*\bgenerated (?:with|by)\b)(?P<who>.*)$", re.IGNORECASE | re.MULTILINE
+)
+MAX_RATIONALE: Final = 4000
+
+
+def family(model: str) -> str | None:
+    """The family a model name belongs to (``openai/gpt-oss-120b``: openai), if known."""
+    words = re.split(r"[^a-z0-9]+", model.lower())
+    for token, name in FAMILIES:  # the token, or the token and a version (qwen3, gpt4o)
+        if any(re.fullmatch(rf"{token}(?:\d[\da-z]*)?", word) for word in words):
+            return name
+    return None
+
+
+def author_families(texts: Iterable[str]) -> set[str]:
+    """The model families the texts name as the change's authors (co-author trailers and
+    "generated with" lines only)."""
+    found = set()
+    for text in texts:
+        for match in _AUTHOR_LINE.finditer(text):
+            named = family(match.group("who"))
+            if named is not None:
+                found.add(named)
+    return found
+
+
+def judge_is_author(models: Sequence[str], authors: set[str]) -> str | None:
+    """The family every judge shares with the change's authors, or None."""
+    families = {family(model) for model in models}
+    if len(families) == 1:
+        (shared,) = families
+        if shared is not None and shared in authors:
+            return shared
+    return None
+
+
+def agreeing(verdict: Mapping[str, Any], runs: int, agree: int) -> dict[str, Any]:
+    """``verdict`` with how many judgements it rests on and how many of them gave it."""
+    return {**verdict, "agreement": {"runs": runs, "agreeing": agree}}
+
+
+def degraded(first: Mapping[str, Any], why: str, runs: int, agree: int) -> dict[str, Any]:
+    """``first`` kept as ``unverifiable``: its judges did not agree (``why``). It still cites
+    and quotes what the first judgement did."""
+    rationale = f"Not kept as {first['verdict']}: {why}. The first judgement: {first['rationale']}"
+    return {
+        **first,
+        "verdict": "unverifiable",
+        "rationale": rationale[:MAX_RATIONALE],
+        "agreement": {"runs": runs, "agreeing": agree},
+    }
+
+
+def majority(verdicts: Sequence[Mapping[str, Any]], runs: int) -> dict[str, Any] | None:
+    """The first verdict whose value at least two of ``verdicts`` give, with its agreement;
+    None when no two agree."""
+    counts = Counter(v["verdict"] for v in verdicts)
+    for verdict in verdicts:
+        if counts[verdict["verdict"]] >= 2:  # noqa: PLR2004 - two of three
+            return agreeing(verdict, runs, counts[verdict["verdict"]])
+    return None
+
+
+def described(verdicts: Sequence[tuple[str, Mapping[str, Any] | str]]) -> str:
+    """What each judge answered, for a rationale: ``m1: supported; m2: rejected (…)``."""
+    return "; ".join(
+        f"{model}: {answer['verdict'] if isinstance(answer, Mapping) else answer}"
+        for model, answer in verdicts
+    )