evalrx 0.1.2__py3-none-any.whl

evalrx/agent_runtime/judges/codex.py

@@ -0,0 +1,120 @@
+"""OpenAI Codex CLI wrapped as a text-generation judge model.
+
+The judge receives only the prompt placed in an isolated temporary directory.
+It uses ``codex exec`` in read-only mode; coding-agent uses elsewhere in the
+pipeline are configured independently through :class:`CliAgentConfig`.
+"""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import shutil
+import subprocess
+import tempfile
+import warnings
+from pathlib import Path
+
+
+class CodexModel:
+    """Codex CLI wrapped behind EvalRX' ``generate`` model protocol."""
+
+    key = "codex"
+
+    def __init__(
+        self,
+        binary_path: str = "",
+        timeout_sec: int = 300,
+        model: str = "gpt-5.6-terra",
+        effort: str = "",
+    ) -> None:
+        from evalrx.core.capability import Capability
+
+        binary = binary_path or shutil.which("codex") or ""
+        if not binary or not os.path.isfile(binary) or not os.access(binary, os.X_OK):
+            raise RuntimeError(
+                "CodexModel: 'codex' binary not found or not executable. "
+                "Set CODEX_PATH=$(which codex) and re-run, or pass binary_path= explicitly."
+            )
+        self._binary = binary
+        self._timeout_sec = timeout_sec
+        self._model = model
+        self._effort = effort
+        self.capabilities = frozenset({Capability.GENERATE})
+        self.modalities = frozenset({"text"})
+
+    def generate(
+        self,
+        inputs: object,
+        *,
+        images: "list[Path] | None" = None,
+        **kwargs: object,
+    ) -> str:
+        """Run an isolated, non-interactive Codex turn and return its answer.
+
+        The prompt is passed over stdin, never as an argv entry, so an M2
+        evidence table cannot hit Linux's per-argument size limit.  Images are
+        copied into the isolated workdir and named in the prompt; this mirrors
+        the existing CLI judge contract without exposing the repository.
+        """
+        workspace = tempfile.mkdtemp(prefix="codex_judge_")
+        output_path = Path(workspace) / "answer.txt"
+        try:
+            prompt_text = str(inputs)
+            valid = [path for path in images or [] if isinstance(path, pathlib.Path) and path.exists()]
+            if valid:
+                names: list[str] = []
+                for path in valid:
+                    destination = Path(workspace) / path.name
+                    shutil.copy2(path, destination)
+                    names.append(destination.name)
+                prompt_text = f"Images available in workspace: {', '.join(names)}\n\n{prompt_text}"
+
+            cmd = [
+                self._binary,
+                "exec",
+                "-",
+                "--sandbox", "read-only",
+                "--skip-git-repo-check",
+                "--ephemeral",
+                "--output-last-message", str(output_path),
+                "-C", workspace,
+            ]
+            if self._model:
+                cmd += ["--model", self._model]
+            if self._effort:
+                cmd += ["-c", f'model_reasoning_effort="{self._effort}"']
+            try:
+                proc = subprocess.run(
+                    cmd,
+                    input=prompt_text,
+                    capture_output=True,
+                    timeout=self._timeout_sec,
+                    text=True,
+                    encoding="utf-8",
+                    errors="replace",
+                    env={**os.environ},
+                )
+            except subprocess.TimeoutExpired as exc:
+                raise RuntimeError(
+                    f"CodexModel: codex timed out after {self._timeout_sec}s"
+                ) from exc
+
+            output = output_path.read_text(encoding="utf-8", errors="replace").strip() if output_path.is_file() else ""
+            if proc.returncode != 0 and not output:
+                reason = (proc.stderr or proc.stdout or "").strip()[-500:]
+                raise RuntimeError(f"CodexModel: codex exited {proc.returncode}: {reason}")
+            if not output:
+                warnings.warn(
+                    "CodexModel: codex returned an empty response; the caller will fall back to a non-LLM path.",
+                    stacklevel=2,
+                )
+            return output
+        finally:
+            shutil.rmtree(workspace, ignore_errors=True)
+
+    def __repr__(self) -> str:
+        return (
+            f"CodexModel(binary={self._binary!r}, model={self._model!r}, "
+            f"effort={self._effort!r})"
+        )

evalrx/agent_runtime/providers/__init__.py

@@ -0,0 +1,21 @@
+"""CLI coding-provider adapters."""
+
+from evalrx.agent_runtime.providers.antigravity import AntigravityAgent
+from evalrx.agent_runtime.providers.base import CliAgentBase
+from evalrx.agent_runtime.providers.claude_code import ClaudeCodeAgent
+from evalrx.agent_runtime.providers.codex import CodexAgent
+from evalrx.agent_runtime.providers.gemini_cli import GeminiCliAgent
+from evalrx.agent_runtime.providers.kimi_cli import KimiCliAgent
+from evalrx.agent_runtime.providers.opencode import OpenCodeAgent
+from evalrx.agent_runtime.providers.registry import create_cli_agent
+
+__all__ = [
+    "AntigravityAgent",
+    "CliAgentBase",
+    "ClaudeCodeAgent",
+    "CodexAgent",
+    "GeminiCliAgent",
+    "KimiCliAgent",
+    "OpenCodeAgent",
+    "create_cli_agent",
+]

evalrx/agent_runtime/providers/antigravity.py

@@ -0,0 +1,31 @@
+"""Antigravity CLI coding-provider adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalrx.agent_runtime.providers.base import CliAgentBase
+
+
+class AntigravityAgent(CliAgentBase):
+    """Antigravity CLI backend (``agy -p``)."""
+
+    _provider_name = "antigravity"
+
+    def _build_cmd(self, prompt: str, workdir: Path) -> list[str]:
+        cmd = [
+            self._binary,
+            "-p",
+            prompt,
+            # Limit terminal/file access to the explicitly added workdir. A
+            # repair author must never inspect benchmark manifests, gold labels,
+            # or confirmation outputs while selecting a candidate.
+            "--sandbox",
+            "--dangerously-skip-permissions",
+            "--add-dir",
+            str(workdir),
+        ]
+        if self._model:
+            cmd += ["--model", self._model]
+        cmd.extend(self._extra_args)
+        return cmd

evalrx/agent_runtime/providers/base.py

@@ -0,0 +1,145 @@
+"""Base class for CLI coding-provider adapters."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from evalrx.agent_runtime.cli_runtime import ProcessRun, SubprocessRunner, collect_py_files
+from evalrx.agent_runtime.cli_transcript import RAW_OUTPUT_CAP
+from evalrx.agent_runtime.cli_types import CliAgentResult
+from evalrx.agent_runtime.skill_audit import build_agent_audit
+from evalrx.agent_runtime.skills.installer import SkillInstaller
+
+logger = logging.getLogger(__name__)
+
+
+class CliAgentBase:
+    """Subprocess runner shared by all CLI coding providers."""
+
+    _provider_name: str = "unknown"
+
+    # Every provider's ``_build_cmd`` places *prompt* directly in argv (``-p
+    # <prompt>`` / ``exec <prompt>`` / ``--message <prompt>``). This
+    # framework's prompts (a full M1 evidence dump, an L1/L2 fix-candidate
+    # brief with per-case tables, ...) can exceed the kernel's argv+envp size
+    # limit -- observed in practice as ``OSError: [Errno 7] Argument list
+    # too long`` on exec. Above this conservative threshold, ``run()`` spills
+    # the prompt to a file in *workdir* instead of passing it inline.
+    _LARGE_PROMPT_BYTES = 60_000
+
+    def __init__(
+        self,
+        binary_path: str,
+        model: str = "",
+        max_budget_usd: float = 5.0,
+        timeout_sec: int = 600,
+        extra_args: list[str] | None = None,
+        skills: list[str] | None = None,
+        allow_skills: bool = False,
+    ) -> None:
+        self._binary = binary_path
+        self._model = model
+        self._max_budget_usd = max_budget_usd
+        self._timeout_sec = timeout_sec
+        self._extra_args: list[str] = extra_args or []
+        self._skills: list[str] = list(skills or [])
+        self._allow_skills: bool = bool(allow_skills or self._skills)
+        self._runner = SubprocessRunner()
+        self._skill_installer = self._make_skill_installer(self._skills)
+
+    def _make_skill_installer(self, skills: list[str]) -> SkillInstaller:
+        return SkillInstaller(skills)
+
+    def _install_skills(self, workdir: Path) -> None:
+        """Install configured skills into the provider-visible workspace."""
+        self._skill_installer.install(workdir)
+
+    def run(
+        self,
+        prompt: str,
+        workdir: Path,
+        timeout_sec: int | None = None,
+    ) -> CliAgentResult:
+        """Invoke the CLI provider with *prompt* and return collected files."""
+        timeout = timeout_sec if timeout_sec is not None else self._timeout_sec
+        workdir.mkdir(parents=True, exist_ok=True)
+        self._install_skills(workdir)
+        cli_prompt = prompt
+        if len(prompt.encode("utf-8", errors="replace")) > self._LARGE_PROMPT_BYTES:
+            (workdir / "prompt.txt").write_text(prompt, encoding="utf-8")
+            cli_prompt = (
+                "Your full task instructions are in the file `prompt.txt` in "
+                "this workspace (too large to pass inline). Read it "
+                "completely and follow it exactly -- do not summarize, "
+                "truncate, or skip any part of it."
+            )
+        cmd = self._build_cmd(cli_prompt, workdir)
+        logger.debug("%s: running %s", self._provider_name, cmd[0])
+
+        run = self._run_subprocess(cmd, workdir, timeout)
+        return self._build_result(workdir, run)
+
+    def _build_cmd(self, prompt: str, workdir: Path) -> list[str]:  # pragma: no cover
+        raise NotImplementedError
+
+    def _postprocess_output(self, stdout: str) -> tuple[str, dict | None]:
+        return stdout[:RAW_OUTPUT_CAP], None
+
+    def _run_subprocess(self, cmd: list[str], workdir: Path, timeout_sec: int) -> ProcessRun:
+        return self._runner.run(cmd, workdir, timeout_sec)
+
+    def _build_result(self, workdir: Path, run: ProcessRun) -> CliAgentResult:
+        files = collect_py_files(workdir)
+        error: str | None = None
+        if run.timed_out:
+            error = f"[TIMEOUT] agent killed after {run.elapsed_sec:.0f}s"
+        elif run.returncode != 0 and not files:
+            error = f"Exited {run.returncode}: {run.stderr[:500]}"
+
+        # Persist the UNTRUNCATED raw agent stream before any rendering caps
+        # apply — this is the most detailed audit layer (every tool call and
+        # tool result). The truncated rendering stays in CliAgentResult.raw_output
+        # for UIs; the full stream is durable evidence in the workdir.
+        raw_stream_path = ""
+        if run.stdout:
+            try:
+                stream_file = workdir / "agent_raw_stream.txt"
+                stream_file.write_text(run.stdout, encoding="utf-8")
+                raw_stream_path = "agent_raw_stream.txt"
+            except OSError:
+                logger.debug("could not persist raw agent stream for %s", self._provider_name)
+
+        raw_output, usage = self._postprocess_output(run.stdout)
+        audit = build_agent_audit(
+            provider=self._provider_name,
+            skills=self._skills,
+            workdir=workdir,
+            stdout=run.stdout,
+            stderr=run.stderr,
+            returncode=run.returncode,
+            timed_out=run.timed_out,
+            elapsed_sec=run.elapsed_sec,
+            files=files,
+            error=error,
+        )
+        if audit is not None and raw_stream_path:
+            audit.setdefault("execution", {})["raw_stream"] = raw_stream_path
+        logger.debug(
+            "%s: rc=%d files=%s elapsed=%.1fs timed_out=%s",
+            self._provider_name,
+            run.returncode,
+            list(files),
+            run.elapsed_sec,
+            run.timed_out,
+        )
+        return CliAgentResult(
+            files=files,
+            provider_name=self._provider_name,
+            elapsed_sec=run.elapsed_sec,
+            raw_output=raw_output,
+            usage=usage,
+            error=error,
+            audit=audit,
+            raw_stream_path=raw_stream_path,
+        )

evalrx/agent_runtime/providers/claude_code.py

@@ -0,0 +1,49 @@
+"""Claude Code CLI coding-provider adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalrx.agent_runtime.cli_transcript import render_claude_stream
+from evalrx.agent_runtime.providers.base import CliAgentBase
+
+
+class ClaudeCodeAgent(CliAgentBase):
+    """Claude Code CLI backend (``claude -p``)."""
+
+    _provider_name = "claude_code"
+
+    def _build_cmd(self, prompt: str, workdir: Path) -> list[str]:
+        # No --dangerously-skip-permissions. The narrow grant below already says
+        # exactly what this agent may do, so the blanket flag added nothing --
+        # and it is refused outright under root unless something sets
+        # IS_SANDBOX=1, which our compose files do and a bare root box (Colab, a
+        # plain container, CI) does not. There the coder exited in 0.7s having
+        # written no files, ExperimentWriter aborted for want of a .py, and every
+        # L2-and-above repair candidate was dead on arrival while the log said
+        # only "CLI agent produced no .py files".
+        #
+        # Same reasoning as the judge in agent_runtime/judges/claude.py: ask for
+        # the tools the work needs, not for every permission there is.
+        allowed = "Bash Edit Write Read" + (" Skill" if self._allow_skills else "")
+        cmd = [
+            self._binary,
+            "-p",
+            prompt,
+            "--output-format",
+            "stream-json",
+            "--verbose",
+            "--allowed-tools",
+            allowed,
+            "--add-dir",
+            str(workdir),
+        ]
+        if self._model:
+            cmd += ["--model", self._model]
+        if self._max_budget_usd:
+            cmd += ["--max-budget-usd", str(self._max_budget_usd)]
+        cmd.extend(self._extra_args)
+        return cmd
+
+    def _postprocess_output(self, stdout: str) -> tuple[str, dict | None]:
+        return render_claude_stream(stdout)

evalrx/agent_runtime/providers/codex.py

@@ -0,0 +1,37 @@
+"""OpenAI Codex CLI coding-provider adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalrx.agent_runtime.providers.base import CliAgentBase
+from evalrx.agent_runtime.skills.installer import CodexSkillInstaller, SkillInstaller
+
+
+class CodexAgent(CliAgentBase):
+    """OpenAI Codex CLI backend (``codex exec``)."""
+
+    _provider_name = "codex"
+
+    def _make_skill_installer(self, skills: list[str]) -> SkillInstaller:
+        return CodexSkillInstaller(skills)
+
+    def _build_cmd(self, prompt: str, workdir: Path) -> list[str]:
+        cmd = [
+            self._binary,
+            "exec",
+            prompt,
+            # Generated-code workspaces are intentionally isolated artifact
+            # directories, not Git checkouts.  Codex otherwise exits before
+            # doing any work with "Not inside a trusted directory".
+            "--skip-git-repo-check",
+            "--sandbox",
+            "workspace-write",
+            "--json",
+            "-C",
+            str(workdir),
+        ]
+        if self._model:
+            cmd += ["-m", self._model]
+        cmd.extend(self._extra_args)
+        return cmd

evalrx/agent_runtime/providers/gemini_cli.py

@@ -0,0 +1,26 @@
+"""Gemini CLI coding-provider adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalrx.agent_runtime.providers.base import CliAgentBase
+
+
+class GeminiCliAgent(CliAgentBase):
+    """Gemini CLI backend (``gemini -p``)."""
+
+    _provider_name = "gemini_cli"
+
+    def _build_cmd(self, prompt: str, workdir: Path) -> list[str]:
+        cmd = [
+            self._binary,
+            "-p",
+            prompt,
+            "--cwd",
+            str(workdir),
+        ]
+        if self._model:
+            cmd += ["--model", self._model]
+        cmd.extend(self._extra_args)
+        return cmd

evalrx/agent_runtime/providers/kimi_cli.py

@@ -0,0 +1,27 @@
+"""Kimi CLI coding-provider adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalrx.agent_runtime.providers.base import CliAgentBase
+
+
+class KimiCliAgent(CliAgentBase):
+    """Kimi CLI backend (``kimi chat``)."""
+
+    _provider_name = "kimi_cli"
+
+    def _build_cmd(self, prompt: str, workdir: Path) -> list[str]:
+        cmd = [
+            self._binary,
+            "chat",
+            "--message",
+            prompt,
+            "--workdir",
+            str(workdir),
+        ]
+        if self._model:
+            cmd += ["--model", self._model]
+        cmd.extend(self._extra_args)
+        return cmd

evalrx/agent_runtime/providers/opencode.py

@@ -0,0 +1,27 @@
+"""OpenCode CLI coding-provider adapter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalrx.agent_runtime.providers.base import CliAgentBase
+
+
+class OpenCodeAgent(CliAgentBase):
+    """OpenCode CLI backend (``opencode run``)."""
+
+    _provider_name = "opencode"
+
+    def _build_cmd(self, prompt: str, workdir: Path) -> list[str]:
+        cmd = [
+            self._binary,
+            "run",
+            "--message",
+            prompt,
+            "--cwd",
+            str(workdir),
+        ]
+        if self._model:
+            cmd += ["--model", self._model]
+        cmd.extend(self._extra_args)
+        return cmd

evalrx/agent_runtime/providers/registry.py

@@ -0,0 +1,58 @@
+"""Provider registry and factory for CLI coding agents."""
+
+from __future__ import annotations
+
+import shutil
+
+from evalrx.agent_runtime.cli_types import BINARY_DEFAULTS, CliAgentConfig
+from evalrx.agent_runtime.providers.antigravity import AntigravityAgent
+from evalrx.agent_runtime.providers.base import CliAgentBase
+from evalrx.agent_runtime.providers.claude_code import ClaudeCodeAgent
+from evalrx.agent_runtime.providers.codex import CodexAgent
+from evalrx.agent_runtime.providers.gemini_cli import GeminiCliAgent
+from evalrx.agent_runtime.providers.kimi_cli import KimiCliAgent
+from evalrx.agent_runtime.providers.opencode import OpenCodeAgent
+
+PROVIDER_CLASSES: dict[str, type[CliAgentBase]] = {
+    "claude_code": ClaudeCodeAgent,
+    "codex": CodexAgent,
+    "opencode": OpenCodeAgent,
+    "gemini_cli": GeminiCliAgent,
+    "kimi_cli": KimiCliAgent,
+    "antigravity": AntigravityAgent,
+}
+
+
+def create_cli_agent(config: CliAgentConfig) -> CliAgentBase:
+    """Instantiate the appropriate CLI coding provider for *config*."""
+    provider = config.provider
+
+    if provider == "llm":
+        raise ValueError(
+            "'llm' is not a CLI provider. Use ExperimentWriter directly "
+            "(leave cli_agent=None or CliAgentConfig(provider='llm'))."
+        )
+
+    cls = PROVIDER_CLASSES.get(provider)
+    if cls is None:
+        raise ValueError(
+            f"Unknown CLI provider: {provider!r}. Valid: {sorted(PROVIDER_CLASSES)}"
+        )
+
+    binary = config.binary_path or shutil.which(BINARY_DEFAULTS[provider]) or ""
+    if not binary:
+        raise RuntimeError(
+            f"CLI agent binary for {provider!r} not found in PATH. "
+            f"Install '{BINARY_DEFAULTS[provider]}' or pass "
+            f"CliAgentConfig(binary_path='/path/to/{BINARY_DEFAULTS[provider]}')."
+        )
+
+    return cls(
+        binary_path=binary,
+        model=config.model,
+        max_budget_usd=config.max_budget_usd,
+        timeout_sec=config.timeout_sec,
+        extra_args=list(config.extra_args),
+        skills=list(config.skills),
+        allow_skills=config.allow_skills,
+    )