evalrx 0.1.2__py3-none-any.whl

evalrx/agent_runtime/factory.py

@@ -0,0 +1,102 @@
+"""Sandbox factory — create the appropriate sandbox backend from config.
+
+Mirrors ``researchclaw/experiment/factory.py``.
+
+Supported modes:
+    ``"subprocess"`` (default) — :class:`ExperimentSandbox` running in a local
+        subprocess.  No extra dependencies required.
+    ``"docker"`` — Docker-isolated sandbox.  Falls back to the subprocess
+        backend when Docker is not available or the DockerSandbox import fails.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from evalrx.agent_runtime.sandbox import ExperimentSandbox, SandboxProtocol
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Configuration dataclasses
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class SandboxConfig:
+    """Configuration for the subprocess sandbox backend.
+
+    Attributes:
+        python_path: Python executable to use.  Empty string → ``sys.executable``.
+        timeout_sec: Default timeout for sandbox runs (seconds).
+    """
+
+    python_path: str = ""
+    timeout_sec: int = 60
+
+
+@dataclass(frozen=True)
+class SandboxFactoryConfig:
+    """Top-level factory configuration.
+
+    Attributes:
+        mode:    Backend to use: ``"subprocess"`` (default) or ``"docker"``.
+        sandbox: Settings for the subprocess backend.
+    """
+
+    mode: str = "subprocess"
+    sandbox: SandboxConfig = field(default_factory=SandboxConfig)
+
+
+# ---------------------------------------------------------------------------
+# Factory
+# ---------------------------------------------------------------------------
+
+
+def create_sandbox(config: SandboxFactoryConfig, workdir: Path) -> SandboxProtocol:
+    """Instantiate the sandbox backend specified by *config.mode*.
+
+    Args:
+        config:  Factory configuration.
+        workdir: Directory where the sandbox writes scripts and project copies.
+
+    Returns:
+        A :class:`~evalrx.agent_runtime.sandbox.SandboxProtocol` implementation.
+        Falls back to :class:`ExperimentSandbox` if the requested backend is
+        unavailable.
+    """
+    if config.mode == "docker":
+        sandbox = _try_create_docker_sandbox(workdir)
+        if sandbox is not None:
+            return sandbox
+        logger.warning(
+            "Docker sandbox unavailable — falling back to subprocess sandbox"
+        )
+
+    python_path = config.sandbox.python_path or None
+    return ExperimentSandbox(workdir=workdir, python_path=python_path)
+
+
+def _try_create_docker_sandbox(workdir: Path) -> SandboxProtocol | None:
+    """Attempt to create a DockerSandbox; return None if not available."""
+    try:
+        import subprocess as _sp
+        result = _sp.run(
+            ["docker", "info"],
+            capture_output=True,
+            timeout=5,
+            check=False,
+        )
+        if result.returncode != 0:
+            logger.debug("Docker not running (docker info returned %s)", result.returncode)
+            return None
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("Docker check failed: %s", exc)
+        return None
+
+    # DockerSandbox is not yet implemented in evalrx — stub falls through.
+    logger.debug("Docker is available but DockerSandbox is not yet implemented")
+    return None

evalrx/agent_runtime/json_shape.py

@@ -0,0 +1,44 @@
+"""Minimal JSON schema validator shared by anything that parses LLM JSON output.
+
+No external deps (not even ``jsonschema``) — only handles type/required/
+minLength, which is enough to catch the common LLM mistakes (missing field,
+empty string, wrong top-level type). Used by M3's hypothesis parser, the
+agentic loop's action parser, and failure-mode cluster naming.
+"""
+
+from __future__ import annotations
+
+
+def validate_json_shape(data: object, schema: dict) -> list[str]:
+    """Returns a list of error strings; empty means valid."""
+    errors: list[str] = []
+
+    def _check(node: object, s: dict, path: str) -> None:
+        expected_type = s.get("type")
+        if expected_type == "array":
+            if not isinstance(node, list):
+                errors.append(f"{path}: expected array, got {type(node).__name__}")
+                return
+            item_schema = s.get("items", {})
+            for i, item in enumerate(node):
+                _check(item, item_schema, f"{path}[{i}]")
+        elif expected_type == "object":
+            if not isinstance(node, dict):
+                errors.append(f"{path}: expected object, got {type(node).__name__}")
+                return
+            for req in s.get("required", []):
+                if req not in node:
+                    errors.append(f"{path}: missing required field '{req}'")
+            for prop, prop_schema in s.get("properties", {}).items():
+                if prop in node:
+                    _check(node[prop], prop_schema, f"{path}.{prop}")
+        elif expected_type == "string":
+            if not isinstance(node, str):
+                errors.append(f"{path}: expected string, got {type(node).__name__}")
+            elif "minLength" in s and len(node) < s["minLength"]:
+                errors.append(
+                    f"{path}: string too short ({len(node)} < {s['minLength']})"
+                )
+
+    _check(data, schema, "$")
+    return errors

evalrx/agent_runtime/judges/__init__.py

@@ -0,0 +1,28 @@
+"""CLI-backed judge model wrappers."""
+
+from evalrx.agent_runtime.judges.agy import AgyModel, scan_agy_log
+from evalrx.agent_runtime.judges.autodetect import (
+    DEFAULT_AGY_CANDIDATES,
+    DEFAULT_CLAUDE_CANDIDATES,
+    ResolvedJudge,
+    pick_agy_model,
+    pick_claude_model,
+    pick_live_model,
+    resolve_cli_judge,
+)
+from evalrx.agent_runtime.judges.claude import ClaudeModel
+from evalrx.agent_runtime.judges.codex import CodexModel
+
+__all__ = [
+    "AgyModel",
+    "ClaudeModel",
+    "CodexModel",
+    "scan_agy_log",
+    "DEFAULT_AGY_CANDIDATES",
+    "DEFAULT_CLAUDE_CANDIDATES",
+    "ResolvedJudge",
+    "pick_agy_model",
+    "pick_claude_model",
+    "pick_live_model",
+    "resolve_cli_judge",
+]

evalrx/agent_runtime/judges/agy.py

@@ -0,0 +1,179 @@
+"""Antigravity CLI wrapped as a text-generation judge model."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import tempfile
+import warnings
+from pathlib import Path
+
+AGY_ERROR_MARKERS = (
+    "RESOURCE_EXHAUSTED",
+    "code 429",
+    "quota",
+    "ineligible",
+    "PERMISSION_DENIED",
+    "UNAUTHENTICATED",
+    "exhausted",
+)
+
+
+def safe_unlink(path: str) -> None:
+    try:
+        os.unlink(path)
+    except OSError:
+        pass
+
+
+def scan_agy_log(path: str) -> str:
+    """Return the last agy-log error line, or ``""`` if none."""
+    try:
+        with open(path, encoding="utf-8", errors="replace") as fh:
+            lines = fh.readlines()[-300:]
+    except OSError:
+        return ""
+    for line in reversed(lines):
+        if any(marker.lower() in line.lower() for marker in AGY_ERROR_MARKERS):
+            msg = line.strip()
+            idx = msg.rfind("] ")
+            return (msg[idx + 2:] if idx != -1 else msg)[:240]
+    return ""
+
+
+class AgyModel:
+    """agy CLI wrapped as a judge model."""
+
+    key = "agy"
+
+    def __init__(
+        self,
+        binary_path: str = "",
+        timeout_sec: int = 120,
+        model: str = "",
+    ) -> None:
+        from evalrx.core.capability import Capability
+
+        binary = binary_path or shutil.which("agy") or ""
+        if not binary or not os.path.isfile(binary) or not os.access(binary, os.X_OK):
+            raise RuntimeError(
+                "AgyModel: 'agy' binary not found or not executable. "
+                "Set AGY_PATH=$(which agy) and re-run, or pass binary_path= explicitly."
+            )
+        self._binary = binary
+        self._timeout_sec = timeout_sec
+        self._model = model
+        self.capabilities = frozenset({Capability.GENERATE})
+        self.modalities = frozenset({"text"})
+
+    # ``-p`` passes the prompt as a literal argv element. M2/M3 judge prompts
+    # in this framework routinely embed a full per-case evidence dump (every
+    # M1 analyzer's findings across every EXPLORE-split case) and can exceed
+    # the kernel's argv+envp size limit -- observed in practice as
+    # ``OSError: [Errno 7] Argument list too long`` on exec, which the caller
+    # (StatsAnalysisAgent/DiagnosisAgent) then silently treats as "judge
+    # returned nothing" and falls back to a threshold-only conclusion / zero
+    # hypotheses. Conservative threshold: well under any real ARG_MAX
+    # (2 MiB is typical on Linux, shared with the environment).
+    _LARGE_PROMPT_BYTES = 60_000
+
+    def generate(
+        self,
+        inputs: object,
+        *,
+        images: "list[Path] | None" = None,
+        **kwargs: object,
+    ) -> str:
+        """Run ``agy -p <inputs>`` and return the text response."""
+        fd, log_path = tempfile.mkstemp(prefix="agy_", suffix=".log")
+        os.close(fd)
+        workspace_dir: str | None = None
+        try:
+            prompt_text = str(inputs)
+            img_names: list[str] = []
+            if images:
+                valid = [p for p in images if isinstance(p, pathlib.Path) and p.exists()]
+                if valid:
+                    workspace_dir = workspace_dir or tempfile.mkdtemp(prefix="agy_ws_")
+                    for path in valid:
+                        shutil.copy2(path, pathlib.Path(workspace_dir) / path.name)
+                    img_names = [path.name for path in valid]
+
+            # Above the threshold, spill the prompt to a file in the same
+            # isolated per-call workspace instead of passing it inline --
+            # this exposes nothing the inline prompt didn't already contain,
+            # it just relocates it out of argv.
+            if len(prompt_text.encode("utf-8", errors="replace")) > self._LARGE_PROMPT_BYTES:
+                workspace_dir = workspace_dir or tempfile.mkdtemp(prefix="agy_ws_")
+                prompt_path = pathlib.Path(workspace_dir) / "prompt.txt"
+                prompt_path.write_text(prompt_text, encoding="utf-8")
+                cli_prompt = (
+                    "Your full task instructions are in the file `prompt.txt` "
+                    "in this workspace (too large to pass inline). Read it "
+                    "completely and respond exactly as it instructs -- do not "
+                    "summarize, truncate, or skip any part of it."
+                )
+            else:
+                cli_prompt = prompt_text
+            if img_names:
+                names = ", ".join(img_names)
+                cli_prompt = f"Images available in workspace: {names}\n\n{cli_prompt}"
+
+            cmd = [
+                self._binary,
+                "-p",
+                cli_prompt,
+                # A judge receives the evidence explicitly in its prompt. It
+                # must not inspect repository files, benchmark manifests, or
+                # held-out labels through terminal tools.
+                "--sandbox",
+                "--dangerously-skip-permissions",
+                "--log-file",
+                log_path,
+            ]
+            if workspace_dir:
+                cmd += ["--add-dir", workspace_dir]
+            if self._model:
+                cmd += ["--model", self._model]
+
+            try:
+                proc = subprocess.run(
+                    cmd,
+                    capture_output=True,
+                    timeout=self._timeout_sec,
+                    text=True,
+                    encoding="utf-8",
+                    errors="replace",
+                    env={**os.environ},
+                )
+            except subprocess.TimeoutExpired as exc:
+                raise RuntimeError(
+                    f"AgyModel: agy timed out after {self._timeout_sec}s"
+                ) from exc
+
+            output = proc.stdout.strip()
+            if proc.returncode != 0 and not output:
+                reason = scan_agy_log(log_path) or (proc.stderr or "").strip()[:240]
+                raise RuntimeError(f"AgyModel: agy exited {proc.returncode}: {reason}")
+
+            output = re.sub(r"<think>.*?</think>", "", output, flags=re.DOTALL).strip()
+            if not output:
+                reason = scan_agy_log(log_path)
+                warnings.warn(
+                    "AgyModel: agy returned an empty response"
+                    + (f" -- {reason}" if reason else "")
+                    + ". agy is likely rate-limited or quota-exhausted; the caller "
+                    "will fall back to a non-LLM path.",
+                    stacklevel=2,
+                )
+            return output
+        finally:
+            safe_unlink(log_path)
+            if workspace_dir:
+                shutil.rmtree(workspace_dir, ignore_errors=True)
+
+    def __repr__(self) -> str:
+        return f"AgyModel(binary={self._binary!r})"

evalrx/agent_runtime/judges/autodetect.py

@@ -0,0 +1,135 @@
+"""Judge liveness-probe autodetection: find a CLI judge that actually responds.
+
+A quota-exhausted CLI model often exits 0 with an empty response rather than
+an error, so a tiny generation probe is the only reliable availability check.
+Promoted from the ``_pick_agy_model``/``_pick_claude_model``/``_resolve_judge``
+helpers duplicated across the ``VLDiagnoseLoop`` examples (``examples/m1_m3/*``,
+``examples/m1_m5/*``).
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass
+from typing import Any, Callable, Sequence
+
+from evalrx.agent_runtime.judges.agy import AgyModel
+from evalrx.agent_runtime.judges.claude import ClaudeModel
+
+_DEFAULT_PROBE_PROMPT = "Reply with exactly the word OK"
+
+# Probe order: cheap/fast candidates first; quotas are per-model and reset on
+# independent clocks, so whichever responds first is "the available version".
+DEFAULT_AGY_CANDIDATES: tuple[str, ...] = (
+    "Gemini 3.1 Pro (Low)",
+    "Claude Sonnet 4.6 (Thinking)",
+    "GPT-OSS 120B (Medium)",
+    "Gemini 3.5 Flash (Low)",
+    "Claude Opus 4.6 (Thinking)",
+)
+# Claude judge candidates: the user's session model first (Fable), then
+# cheaper aliases.
+DEFAULT_CLAUDE_CANDIDATES: tuple[str, ...] = ("claude-fable-5", "sonnet", "haiku")
+
+
+def pick_live_model(
+    factory: "Callable[[str], Any]",
+    candidates: "Sequence[str]",
+    *,
+    probe_prompt: str = _DEFAULT_PROBE_PROMPT,
+) -> str:
+    """Return the first candidate name whose model actually answers *probe_prompt*.
+
+    *factory* builds a model from a candidate name (e.g. ``lambda name:
+    AgyModel(model=name, timeout_sec=60)``). A quota-exhausted model typically
+    returns an empty string rather than raising, so an empty response is
+    treated the same as a failed candidate. Raises ``RuntimeError`` (with every
+    attempt's failure reason) when all candidates are dead.
+    """
+    errors: list[str] = []
+    for name in candidates:
+        try:
+            probe = factory(name)
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")  # quota warnings are expected here
+                out = probe.generate(probe_prompt)
+            if str(out).strip():
+                return name
+            errors.append(f"{name!r}: empty response (likely quota-exhausted)")
+        except RuntimeError as exc:
+            errors.append(f"{name!r}: {exc}")
+    raise RuntimeError(
+        f"no model responded to the availability probe ({len(candidates)} tried): "
+        + "; ".join(errors)
+    )
+
+
+def pick_agy_model(
+    candidates: "Sequence[str]" = DEFAULT_AGY_CANDIDATES, *, timeout_sec: int = 60,
+) -> str:
+    """Return the first agy model name that answers a probe prompt."""
+    return pick_live_model(
+        lambda name: AgyModel(model=name, timeout_sec=timeout_sec), candidates,
+    )
+
+
+def pick_claude_model(
+    candidates: "Sequence[str]" = DEFAULT_CLAUDE_CANDIDATES, *, timeout_sec: int = 90,
+) -> str:
+    """Return the first claude model name that answers a probe prompt."""
+    return pick_live_model(
+        lambda name: ClaudeModel(model=name, timeout_sec=timeout_sec), candidates,
+    )
+
+
+@dataclass(frozen=True)
+class ResolvedJudge:
+    """A CLI judge resolved by :func:`resolve_cli_judge`, with its provenance."""
+
+    judge: Any
+    provider: str  # "agy" | "claude"
+    model: str
+
+
+def resolve_cli_judge(
+    provider: str = "auto",
+    model: str = "auto",
+    *,
+    effort: str = "",
+    timeout_sec: int = 240,
+) -> ResolvedJudge:
+    """Resolve a live CLI judge.
+
+    ``provider="auto"`` tries agy first (its quota errors fail fast), then
+    claude. ``model="auto"`` probes :data:`DEFAULT_AGY_CANDIDATES` /
+    :data:`DEFAULT_CLAUDE_CANDIDATES` for a live one; an explicit model name
+    is used directly (constructed, not probed). Raises ``RuntimeError``
+    (aggregating every attempt's failure) when nothing is available; raises
+    ``ValueError`` for an unrecognized *provider*.
+    """
+
+    def _try_agy() -> ResolvedJudge:
+        name = pick_agy_model() if model == "auto" else model
+        return ResolvedJudge(AgyModel(model=name, timeout_sec=timeout_sec), "agy", name)
+
+    def _try_claude() -> ResolvedJudge:
+        name = pick_claude_model() if model == "auto" else model
+        return ResolvedJudge(
+            ClaudeModel(model=name, timeout_sec=timeout_sec, effort=effort), "claude", name,
+        )
+
+    try:
+        chain = {"agy": (_try_agy,), "claude": (_try_claude,),
+                 "auto": (_try_agy, _try_claude)}[provider]
+    except KeyError:
+        raise ValueError(
+            f"resolve_cli_judge: unknown provider {provider!r} (agy/claude/auto)"
+        ) from None
+
+    errors: list[str] = []
+    for attempt in chain:
+        try:
+            return attempt()
+        except RuntimeError as exc:
+            errors.append(str(exc))
+    raise RuntimeError("no CLI judge available: " + "; ".join(errors))

evalrx/agent_runtime/judges/claude.py

@@ -0,0 +1,159 @@
+"""Claude Code CLI wrapped as a text-generation judge model."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import tempfile
+import warnings
+from pathlib import Path
+
+#: The Claude Code CLI's limit notices (plan / session / spend / usage), which
+#: arrive on stdout with exit 0 exactly like a real answer.
+_LIMIT_RE = re.compile(
+    r"hit your (monthly |weekly |session )?(spend|usage|rate) limit|"
+    r"(spend|usage) limit (reached|exceeded)|out of (extra )?usage|"
+    r"limit resets", re.I)
+
+
+class ClaudeModel:
+    """Claude Code CLI wrapped as a judge model."""
+
+    key = "claude"
+
+    def __init__(
+        self,
+        binary_path: str = "",
+        timeout_sec: int = 240,
+        model: str = "",
+        effort: str = "",
+    ) -> None:
+        from evalrx.core.capability import Capability
+
+        binary = binary_path or shutil.which("claude") or ""
+        if not binary or not os.path.isfile(binary) or not os.access(binary, os.X_OK):
+            raise RuntimeError(
+                "ClaudeModel: 'claude' binary not found or not executable. "
+                "Set CLAUDE_PATH=$(which claude) and re-run, or pass "
+                "binary_path= explicitly."
+            )
+        self._binary = binary
+        self._timeout_sec = timeout_sec
+        self._model = model
+        self._effort = effort
+        self.capabilities = frozenset({Capability.GENERATE})
+        self.modalities = frozenset({"text"})
+
+    def generate(
+        self,
+        inputs: object,
+        *,
+        images: "list[Path] | None" = None,
+        **kwargs: object,
+    ) -> str:
+        """Run ``claude -p`` with the prompt on STDIN and return the response.
+
+        The prompt must not be an argv entry. Linux caps a SINGLE argument at
+        ``MAX_ARG_STRLEN`` = 32 pages = 131,072 bytes — a limit ``ulimit -s`` and
+        ``ARG_MAX`` do not describe and no configuration raises — and execve
+        fails outright with ``OSError: [Errno 7] Argument list too long``.
+
+        M2's prompt is the analyzers' findings JSON, so its size grows with how
+        many analyzers M1 selected. Measured on qwen3.5-2b / bbh_word_sorting:
+        9 analyzers produced 127,856 bytes and worked, 12 analyzers produced
+        ~137,600 and did not. The size that fails is therefore a property of one
+        judge's analyzer selection, which is exactly the thing that varies run to
+        run — the same batch, the same code, and one run crosses the cliff.
+
+        Stdin has no equivalent limit; 144,050 bytes round-trips fine.
+        """
+        img_dir: str | None = None
+        try:
+            prompt_text = str(inputs)
+            if images:
+                valid = [p for p in images if isinstance(p, pathlib.Path) and p.exists()]
+                if valid:
+                    img_dir = tempfile.mkdtemp(prefix="claude_imgs_")
+                    for path in valid:
+                        shutil.copy2(path, pathlib.Path(img_dir) / path.name)
+                    names = ", ".join(path.name for path in valid)
+                    prompt_text = f"Images available in workspace: {names}\n\n{prompt_text}"
+
+            # A judge generates text; it does not act. Asking for the blanket
+            # permission bypass was more than that needs, and it made the judge
+            # unrunnable anywhere nothing wraps it: the CLI refuses
+            # `--dangerously-skip-permissions` under root unless IS_SANDBOX=1,
+            # which our compose files set (see _common/compose/base.yml) and a
+            # bare root box — Colab, a plain container, CI — does not. There
+            # every judge call exited 1 before reaching the model.
+            #
+            # `--tools ""` is the accurate request for a text completion: no
+            # tools, therefore nothing to permission. An image-bearing call is
+            # the one exception — the figures are staged in a temp dir and the
+            # model has to Read them — so that call allows exactly Read, scoped
+            # to the directory just added, and nothing else.
+            cmd = [
+                self._binary,
+                "-p",
+                "--output-format",
+                "text",
+            ]
+            if img_dir:
+                cmd += ["--add-dir", img_dir, "--allowed-tools", "Read"]
+            else:
+                cmd += ["--tools", ""]
+            if self._model:
+                cmd += ["--model", self._model]
+            if self._effort:
+                cmd += ["--effort", self._effort]
+
+            try:
+                proc = subprocess.run(
+                    cmd,
+                    input=prompt_text,
+                    capture_output=True,
+                    timeout=self._timeout_sec,
+                    text=True,
+                    encoding="utf-8",
+                    errors="replace",
+                    env={**os.environ},
+                )
+            except subprocess.TimeoutExpired as exc:
+                raise RuntimeError(
+                    f"ClaudeModel: claude timed out after {self._timeout_sec}s"
+                ) from exc
+
+            output = (proc.stdout or "").strip()
+            # The CLI reports an exhausted plan/session/spend limit as a one-line
+            # *answer* on stdout with exit 0 ("You've hit your monthly spend
+            # limit ..."). Returned as a judge reply it parses to zero
+            # hypotheses and the run "finishes" with an empty diagnosis
+            # (qwen2-audio/mmau 2026-09-07, all three demo128 cells
+            # 2026-09-09). Fail loudly instead.
+            if len(output) < 400 and _LIMIT_RE.search(output):
+                raise RuntimeError(f"ClaudeModel: account limit reached, not a judge answer: {output!r}")
+            if proc.returncode != 0 and not output:
+                reason = (proc.stderr or "").strip()[:240]
+                raise RuntimeError(
+                    f"ClaudeModel: claude exited {proc.returncode}: {reason}"
+                )
+            if not output:
+                warnings.warn(
+                    "ClaudeModel: claude returned an empty response -- likely "
+                    "rate-limited or out of quota; the caller will fall back "
+                    "to a non-LLM path.",
+                    stacklevel=2,
+                )
+            return output
+        finally:
+            if img_dir:
+                shutil.rmtree(img_dir, ignore_errors=True)
+
+    def __repr__(self) -> str:
+        return (
+            f"ClaudeModel(binary={self._binary!r}, model={self._model!r}, "
+            f"effort={self._effort!r})"
+        )