evalrx 0.1.2__py3-none-any.whl

evalrx/agent_runtime/_docker_runner.py

@@ -0,0 +1,89 @@
+"""Docker worker — runs inside the analyzer container.
+
+Invoked as ``python -m evalrx.agent_runtime._docker_runner``.
+
+Reads a JSON payload from stdin::
+
+    {
+        "analyzer": "self_consistency",
+        "params":   {"n": 3},
+        "cases":    [{"id": "...", "prompt": "...", "label": null, "metadata": {}}],
+        "model_env": "GEMINI_API_KEY"
+    }
+
+Writes the analyzer findings as JSON to stdout::
+
+    {"findings": {...}}
+
+The containerised model is constructed from the ``model_env`` key: the value
+names the environment variable that holds the API key.  Currently supports
+Gemini (``GEMINI_API_KEY``) and OpenAI-compatible (``OPENAI_API_KEY``) keys.
+An unrecognised or missing key produces a ``{"error": "..."}`` response.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+
+def _build_model(model_env: str):
+    key = os.getenv(model_env, "")
+    if not key:
+        raise ValueError(
+            f"Environment variable '{model_env}' is not set inside the container."
+        )
+    if "GEMINI" in model_env.upper():
+        from evalrx.models.blackbox.gemini import GeminiModel
+
+        return GeminiModel(api_key=key)
+    raise ValueError(
+        f"No model factory for env var '{model_env}'. "
+        "Add a case to evalrx/agent_runtime/_docker_runner.py."
+    )
+
+
+def _build_cases(raw_cases: list[dict]):
+    from evalrx.core.case import CaseBatch, FailureCase, Inputs, Label
+
+    cases = []
+    for rec in raw_cases:
+        label = None
+        raw_label = rec.get("label")
+        if raw_label is not None:
+            try:
+                label = Label(raw_label)
+            except ValueError:
+                pass
+        cases.append(
+            FailureCase(
+                inputs=Inputs(prompt=rec.get("prompt", "")),
+                label=label,
+                metadata=rec.get("metadata", {}),
+            )
+        )
+    return CaseBatch(cases)
+
+
+def main() -> None:
+    payload = json.loads(sys.stdin.read())
+    try:
+        model = _build_model(payload.get("model_env", "GEMINI_API_KEY"))
+        cases = _build_cases(payload.get("cases", []))
+        analyzer_name = payload["analyzer"]
+        params = payload.get("params", {})
+
+        from evalrx.core.registry import registry
+
+        cls = registry.analyzers.get(analyzer_name)
+        analyzer = cls(**params)
+        result = analyzer.run(model, cases)
+        print(json.dumps({"findings": result.findings}, default=str))
+    except Exception as exc:
+        print(json.dumps({"error": str(exc), "findings": {}}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

evalrx/agent_runtime/cli_runtime.py

@@ -0,0 +1,103 @@
+"""Runtime primitives for CLI coding-agent adapters."""
+
+from __future__ import annotations
+
+import os
+import signal
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class ProcessRun:
+    """Completed subprocess execution."""
+
+    returncode: int
+    stdout: str
+    stderr: str
+    elapsed_sec: float
+    timed_out: bool = False
+
+
+def to_text(value: str | bytes | None) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    return value
+
+
+def collect_py_files(workdir: Path) -> dict[str, str]:
+    """Read top-level Python files produced by a coding agent."""
+    files: dict[str, str] = {}
+    for pyfile in sorted(workdir.glob("*.py")):
+        if pyfile.name.startswith(("_codex_", "_agent_")):
+            continue
+        try:
+            files[pyfile.name] = pyfile.read_text(encoding="utf-8")
+        except OSError:
+            pass
+    return files
+
+
+class SubprocessRunner:
+    """Run CLI agents in a workdir with timeout and environment normalization."""
+
+    def run(self, cmd: list[str], workdir: Path, timeout_sec: int) -> ProcessRun:
+        workdir.mkdir(parents=True, exist_ok=True)
+        start = time.monotonic()
+        timed_out = False
+
+        env = {**os.environ}
+        bindir = os.path.dirname(sys.executable)
+        if bindir:
+            env["PATH"] = bindir + os.pathsep + env.get("PATH", "")
+
+        proc = subprocess.Popen(
+            cmd,
+            # ``codex exec`` accepts a one-shot prompt on argv but keeps
+            # reading stdin for a possible follow-up turn.  The runner has no
+            # interactive continuation protocol, so leave stdin at EOF.  If
+            # inherited from the parent it remains open and Codex idles until
+            # the wall-clock timeout, withholding its generated artifacts.
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=workdir,
+            env=env,
+            start_new_session=True,
+        )
+
+        try:
+            stdout_bytes, stderr_bytes = proc.communicate(timeout=timeout_sec)
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+            except OSError:
+                pass
+            try:
+                stdout_bytes, stderr_bytes = proc.communicate(timeout=5)
+            except subprocess.TimeoutExpired:
+                try:
+                    os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+                except OSError:
+                    pass
+                stdout_bytes, stderr_bytes = b"", b""
+                try:
+                    proc.communicate(timeout=5)
+                except Exception:  # noqa: BLE001
+                    pass
+
+        elapsed = time.monotonic() - start
+        rc = proc.returncode if proc.returncode is not None else -1
+        return ProcessRun(
+            returncode=rc,
+            stdout=to_text(stdout_bytes),
+            stderr=to_text(stderr_bytes),
+            elapsed_sec=elapsed,
+            timed_out=timed_out,
+        )

evalrx/agent_runtime/cli_transcript.py

@@ -0,0 +1,138 @@
+"""Transcript rendering for CLI coding-agent event streams."""
+
+from __future__ import annotations
+
+import json
+
+RAW_OUTPUT_CAP = 3000
+STREAM_TEXT_CAP = 4000
+STREAM_TOOL_INPUT_CAP = 2000
+STREAM_TOOL_RESULT_CAP = 2000
+STREAM_FALLBACK_CAP = 8000
+
+
+def trunc(text: str, cap: int) -> str:
+    text = text or ""
+    if len(text) <= cap:
+        return text
+    return text[:cap] + f"\n... [truncated {len(text) - cap} chars]"
+
+
+def _blocks_text(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        out: list[str] = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                out.append(str(block.get("text", "")))
+            elif isinstance(block, str):
+                out.append(block)
+        return "\n".join(out)
+    return ""
+
+
+def _summarise_tool_use(name: str, inp: object) -> str:
+    if not isinstance(inp, dict):
+        return trunc(str(inp), STREAM_TOOL_INPUT_CAP)
+    if name == "Bash":
+        return trunc("$ " + str(inp.get("command", "")), STREAM_TOOL_INPUT_CAP)
+    if name == "Write":
+        content_len = len(str(inp.get("content", "")))
+        return f"write {inp.get('file_path', '?')} ({content_len} chars)"
+    if name == "Edit":
+        return f"edit {inp.get('file_path', '?')}"
+    if name == "Read":
+        span = ""
+        if inp.get("offset") is not None or inp.get("limit") is not None:
+            span = f" [offset={inp.get('offset')}, limit={inp.get('limit')}]"
+        return f"read {inp.get('file_path', '?')}{span}"
+    try:
+        return trunc(json.dumps(inp, default=str), STREAM_TOOL_INPUT_CAP)
+    except (TypeError, ValueError):
+        return trunc(str(inp), STREAM_TOOL_INPUT_CAP)
+
+
+def render_claude_stream(stdout: str) -> tuple[str, dict | None]:
+    """Render ``claude -p --output-format stream-json`` into a coding trajectory."""
+    events: list[dict] = []
+    for line in stdout.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            event = json.loads(line)
+        except (ValueError, TypeError):
+            continue
+        if isinstance(event, dict):
+            events.append(event)
+    if not events:
+        return trunc(stdout, STREAM_FALLBACK_CAP), None
+
+    out: list[str] = []
+    usage: dict | None = None
+    tool_seq: dict[str, int] = {}
+    step = 0
+
+    for event in events:
+        etype = event.get("type")
+        if etype == "system" and event.get("subtype") == "init":
+            tools = event.get("tools") or []
+            out.append(
+                f"=== session start | model={event.get('model', '?')} | "
+                f"tools={','.join(map(str, tools)) if tools else '-'} ==="
+            )
+        elif etype == "assistant":
+            for block in (event.get("message") or {}).get("content") or []:
+                if not isinstance(block, dict):
+                    continue
+                if block.get("type") == "text":
+                    txt = str(block.get("text", "")).strip()
+                    if txt:
+                        out.append(f"[assistant] {trunc(txt, STREAM_TEXT_CAP)}")
+                elif block.get("type") == "tool_use":
+                    step += 1
+                    if block.get("id"):
+                        tool_seq[block["id"]] = step
+                    name = str(block.get("name", "tool"))
+                    summary = _summarise_tool_use(name, block.get("input"))
+                    out.append(f"[#{step} {name}] {summary}")
+        elif etype == "user":
+            for block in (event.get("message") or {}).get("content") or []:
+                if isinstance(block, dict) and block.get("type") == "tool_result":
+                    n = tool_seq.get(block.get("tool_use_id"), "?")
+                    err = " ERROR" if block.get("is_error") else ""
+                    out.append(
+                        f"[#{n} result{err}] "
+                        f"{trunc(_blocks_text(block.get('content')), STREAM_TOOL_RESULT_CAP)}"
+                    )
+        elif etype == "result":
+            token_usage = event.get("usage") or {}
+            usage = {
+                "cost_usd": event.get("total_cost_usd"),
+                "num_turns": event.get("num_turns"),
+                "duration_ms": event.get("duration_ms"),
+                "input_tokens": token_usage.get("input_tokens"),
+                "output_tokens": token_usage.get("output_tokens"),
+                "cache_read_input_tokens": token_usage.get("cache_read_input_tokens"),
+                "cache_creation_input_tokens": token_usage.get("cache_creation_input_tokens"),
+            }
+            final = str(event.get("result", "")).strip()
+            if final:
+                out.append(f"[final] {trunc(final, STREAM_TEXT_CAP)}")
+            cost = usage["cost_usd"]
+            dur = event.get("duration_ms")
+            out.append(
+                "=== result: {sub} | turns={turns} | cost={cost} | "
+                "tokens in={inp} out={outp} (cache_read={cache}) | wall={wall} ===".format(
+                    sub=event.get("subtype", "?"),
+                    turns=usage["num_turns"],
+                    cost=(f"${cost:.4f}" if isinstance(cost, (int, float)) else "n/a"),
+                    inp=usage["input_tokens"],
+                    outp=usage["output_tokens"],
+                    cache=usage["cache_read_input_tokens"],
+                    wall=(f"{dur / 1000:.1f}s" if isinstance(dur, (int, float)) else "?"),
+                )
+            )
+
+    return "\n".join(out), usage

evalrx/agent_runtime/cli_types.py

@@ -0,0 +1,68 @@
+"""Public types and provider metadata for CLI coding agents."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+VALID_PROVIDERS = frozenset(
+    {"llm", "claude_code", "codex", "opencode", "gemini_cli", "kimi_cli", "antigravity"}
+)
+
+BINARY_DEFAULTS: dict[str, str] = {
+    "claude_code": "claude",
+    "codex": "codex",
+    "opencode": "opencode",
+    "gemini_cli": "gemini",
+    "kimi_cli": "kimi",
+    "antigravity": "agy",
+}
+
+
+@dataclass(frozen=True)
+class CliAgentConfig:
+    """Configuration for a CLI-based coding-agent backend.
+
+    Args:
+        provider: Which CLI agent to use. ``"llm"`` means no CLI agent.
+        binary_path: Explicit path to the CLI binary. Auto-detected when empty.
+        model: Model override flag forwarded to the binary.
+        max_budget_usd: Spend cap forwarded to providers that support it.
+        timeout_sec: Hard wall-clock limit for the agent subprocess.
+        extra_args: Additional flags appended verbatim to the CLI command.
+        skills: Paths to Agent-Skill directories containing ``SKILL.md``.
+        allow_skills: Enable provider-native skill invocation where supported.
+    """
+
+    provider: str = "llm"
+    binary_path: str = ""
+    model: str = ""
+    max_budget_usd: float = 5.0
+    timeout_sec: int = 600
+    extra_args: tuple[str, ...] = ()
+    skills: tuple[str, ...] = ()
+    allow_skills: bool = False
+
+    @property
+    def skills_enabled(self) -> bool:
+        return self.allow_skills or bool(self.skills)
+
+
+@dataclass
+class CliAgentResult:
+    """Output of one CLI agent invocation."""
+
+    files: dict[str, str]
+    provider_name: str
+    elapsed_sec: float
+    raw_output: str = ""
+    usage: dict | None = None
+    error: str | None = None
+    audit: dict | None = None
+    # Path (inside the invocation workdir) of the UNTRUNCATED raw agent
+    # stream — ``raw_output`` above is a capped rendering for UIs; the full
+    # stream (every tool call and result) is persisted here for debugging.
+    raw_stream_path: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None and bool(self.files)

evalrx/agent_runtime/codegen/__init__.py

@@ -0,0 +1,5 @@
+"""Shared code-generation orchestration helpers."""
+
+from evalrx.agent_runtime.codegen.runner import CodegenCodeResult, CodegenRunner
+
+__all__ = ["CodegenCodeResult", "CodegenRunner"]

evalrx/agent_runtime/codegen/runner.py

@@ -0,0 +1,94 @@
+"""Shared runner for CLI coding-provider invocations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from evalrx.agent_runtime.cli_types import CliAgentConfig, CliAgentResult
+
+
+@dataclass
+class CodegenCodeResult:
+    """Selected source file plus the underlying CLI run metadata."""
+
+    code: str
+    raw_output: str
+    usage: dict | None = None
+    files: dict[str, str] = field(default_factory=dict)
+    error: str | None = None
+    elapsed_sec: float = 0.0
+    audit: dict | None = None
+    # Path (relative to the invocation workdir) of the UNTRUNCATED raw agent
+    # stream persisted by the provider base class.
+    raw_stream_path: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return bool(self.code) and self.error is None
+
+
+def _select_py_file(
+    files: dict[str, str],
+    *,
+    preferred: tuple[str, ...] = (),
+) -> str:
+    py_files = {name: body for name, body in files.items() if name.endswith(".py")}
+    for name in preferred:
+        if name in py_files:
+            return py_files[name]
+    if not py_files:
+        return ""
+    return max(py_files.values(), key=len)
+
+
+class CodegenRunner:
+    """Run a configured CLI coding provider and select generated code files."""
+
+    def __init__(self, cli_config: CliAgentConfig) -> None:
+        self._cli_config = cli_config
+
+    @property
+    def provider(self) -> str:
+        return self._cli_config.provider
+
+    def run(
+        self,
+        prompt: str,
+        *,
+        workdir: Path,
+        timeout_sec: int | None = None,
+    ) -> CliAgentResult:
+        # Lazy import: this is the sole call site, so importing here avoids
+        # pulling every provider module in at CodegenRunner import time.
+        from evalrx.agent_runtime.providers.registry import create_cli_agent
+
+        agent = create_cli_agent(self._cli_config)
+        return agent.run(prompt, workdir=workdir, timeout_sec=timeout_sec)
+
+    def write_code(
+        self,
+        prompt: str,
+        *,
+        workdir: Path,
+        timeout_sec: int | None = None,
+        preferred_filenames: tuple[str, ...] = (),
+        include_error_in_raw: bool = False,
+    ) -> CodegenCodeResult:
+        result = self.run(prompt, workdir=workdir, timeout_sec=timeout_sec)
+        raw_output = result.raw_output or (
+            (result.error or "") if include_error_in_raw else ""
+        )
+        code = _select_py_file(result.files, preferred=preferred_filenames)
+        if not result.ok:
+            code = ""
+        return CodegenCodeResult(
+            code=code,
+            raw_output=raw_output,
+            usage=result.usage,
+            files=result.files,
+            error=result.error,
+            elapsed_sec=result.elapsed_sec,
+            audit=result.audit,
+            raw_stream_path=result.raw_stream_path,
+        )

evalrx/agent_runtime/experiment_harness.py

@@ -0,0 +1,117 @@
+"""Experiment harness — immutable evaluation infrastructure.
+
+This file is injected into the sandbox project directory at execution time.
+The LLM-generated experiment code should import and use this harness for:
+- Time budget management (should_stop)
+- Metric reporting (report_metric)
+- Result finalization (finalize)
+- NaN/divergence detection (built-in)
+
+This file is NOT editable by the LLM agent — it provides a trust boundary
+for metric reporting, mirroring AutoResearchClaw's immutable harness pattern.
+"""
+
+import json
+import math
+import sys
+import time
+
+
+class ExperimentHarness:
+    """Immutable experiment infrastructure for time and metric management."""
+
+    def __init__(self, time_budget: int = 120):
+        self._start = time.time()
+        self._time_budget = max(1, int(time_budget))
+        self._metrics: dict[str, float] = {}
+        self._partial_results: list[dict[str, object]] = []
+        self._step_count = 0
+        self._nan_count = 0
+
+    @property
+    def elapsed(self) -> float:
+        """Seconds elapsed since harness creation."""
+        return time.time() - self._start
+
+    @property
+    def progress(self) -> float:
+        """Fraction of time budget used (0.0 to 1.0)."""
+        return min(self.elapsed / self._time_budget, 1.0)
+
+    def should_stop(self) -> bool:
+        """Return True if approaching 80% of time budget."""
+        return self.elapsed >= self._time_budget * 0.8
+
+    def check_value(self, value: float, name: str = "metric") -> bool:
+        """Return True if value is finite. Log warning and count NaN/Inf."""
+        if math.isnan(value) or math.isinf(value):
+            self._nan_count += 1
+            print(
+                f"WARNING: {name} = {value} (non-finite, skipped)",
+                file=sys.stderr,
+            )
+            if self._nan_count >= 5:
+                print(
+                    "FAIL: Too many NaN/Inf values detected. "
+                    "Stopping experiment early.",
+                    file=sys.stderr,
+                )
+                self.finalize()
+                sys.exit(1)
+            return False
+        return True
+
+    def report_metric(self, name: str, value: float) -> None:
+        """Report a metric value. Validates and prints in standard format.
+
+        Non-finite values (NaN, Inf) are rejected and logged as warnings.
+        """
+        if not isinstance(value, (int, float)):
+            try:
+                value = float(value)
+            except (TypeError, ValueError):
+                print(f"WARNING: Cannot convert {name}={value!r} to float", file=sys.stderr)
+                return
+
+        if not self.check_value(value, name):
+            return
+
+        self._metrics[name] = value
+        print(f"{name}: {value}")
+
+    def log_result(self, result_dict: dict[str, object]) -> None:
+        """Log a structured result row (e.g., per-condition results)."""
+        self._partial_results.append(result_dict)
+
+    def finalize(self) -> None:
+        """Write results.json with all reported metrics and partial results."""
+        output: dict[str, object] = {
+            "metrics": self._metrics,
+            "elapsed_sec": round(self.elapsed, 2),
+            "time_budget_sec": self._time_budget,
+            "steps_completed": self._step_count,
+            "nan_count": self._nan_count,
+        }
+        if self._partial_results:
+            output["results"] = self._partial_results
+
+        try:
+            with open("results.json", "w", encoding="utf-8") as f:
+                json.dump(output, f, indent=2, default=str)
+        except OSError as e:
+            print(f"WARNING: Could not write results.json: {e}", file=sys.stderr)
+
+    def step(self) -> None:
+        """Increment step counter. Call this once per experiment step."""
+        self._step_count += 1
+
+
+_default_harness: ExperimentHarness | None = None
+
+
+def get_harness(time_budget: int = 120) -> ExperimentHarness:
+    """Get or create the default experiment harness."""
+    global _default_harness  # noqa: PLW0603
+    if _default_harness is None:
+        _default_harness = ExperimentHarness(time_budget)
+    return _default_harness