evalpilot 0.1.0__py3-none-any.whl

eval_agent/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

eval_agent/adapters/__init__.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from pathlib import Path
+from eval_agent.models import Task, Trajectory
+
+
+class AgentAdapter(ABC):
+    """Narrow interface: drive the agent under test for one task.
+
+    The framework only ever calls .run(); how the adapter invokes the
+    underlying agent (subprocess, HTTP, in-process function) is internal.
+    """
+
+    name: str = "agent"
+
+    @abstractmethod
+    def run(self, task: Task, workspace: Path) -> Trajectory:
+        ...
+
+
+def build_adapter(config: dict) -> AgentAdapter:
+    """Factory: pick an adapter implementation from a config dict.
+
+    config["type"] in {"function", "cli", "http", "webui"}.
+    """
+    atype = config["type"]
+    if atype == "cli":
+        from eval_agent.adapters.cli import CLIAgentAdapter
+        return CLIAgentAdapter.from_config(config)
+    if atype == "http":
+        from eval_agent.adapters.http import HTTPAgentAdapter
+        return HTTPAgentAdapter.from_config(config)
+    if atype == "function":
+        from eval_agent.adapters.function import FunctionAgentAdapter
+        return FunctionAgentAdapter.from_config(config)
+    if atype == "webui":
+        from eval_agent.adapters.webui import WebUIAgentAdapter
+        return WebUIAgentAdapter.from_config(config)
+    raise ValueError(f"unknown adapter type: {atype}")

eval_agent/adapters/cli.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+import json
+import shlex
+import subprocess
+import time
+from pathlib import Path
+from eval_agent.adapters import AgentAdapter
+from eval_agent.models import Task, Trajectory, Step, Usage
+
+
+class CLIAgentAdapter(AgentAdapter):
+    """Drives a command-line agent that emits a JSONL stream on stdout.
+
+    command_template may contain {prompt}; it is substituted (shell-quoted)
+    with the task prompt. The process runs with cwd=workspace.
+    """
+
+    name = "cli-agent"
+
+    def __init__(self, command_template: str, name: str = "cli-agent"):
+        self.command_template = command_template
+        self.name = name
+
+    @classmethod
+    def from_config(cls, config: dict) -> "CLIAgentAdapter":
+        return cls(command_template=config["command"],
+                   name=config.get("name", "cli-agent"))
+
+    def run(self, task: Task, workspace: Path) -> Trajectory:
+        cmd = self.command_template.format(prompt=shlex.quote(task.prompt))
+        steps: list[Step] = []
+        final_output = ""
+        total_tokens = 0
+        start = time.monotonic()
+        try:
+            proc = subprocess.run(
+                cmd, shell=True, cwd=str(workspace),
+                capture_output=True, text=True,
+                timeout=task.budget.timeout_seconds,
+            )
+        except subprocess.TimeoutExpired:
+            return Trajectory(
+                steps=steps,
+                usage=Usage(wall_seconds=time.monotonic() - start,
+                            num_steps=len(steps)),
+                status="timeout",
+                error="process exceeded timeout",
+            )
+
+        for line in proc.stdout.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            t = obj.get("type")
+            if t == "tool_use":
+                steps.append(Step(index=len(steps), kind="tool_call",
+                                  name=obj.get("name"),
+                                  payload=obj.get("input", {})))
+            elif t == "text":
+                steps.append(Step(index=len(steps), kind="message",
+                                  payload={"text": obj.get("text", "")}))
+            elif t == "result":
+                final_output = obj.get("output", "")
+                total_tokens = obj.get("usage", {}).get("total_tokens", 0)
+
+        status = "success" if proc.returncode == 0 else "crash"
+        return Trajectory(
+            steps=steps,
+            final_output=final_output,
+            usage=Usage(total_tokens=total_tokens,
+                        wall_seconds=time.monotonic() - start,
+                        num_steps=len(steps)),
+            status=status,
+            error=None if status == "success" else proc.stderr[-2000:],
+        )

eval_agent/adapters/function.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+import importlib
+import time
+from pathlib import Path
+from typing import Callable, Optional
+from eval_agent.adapters import AgentAdapter
+from eval_agent.models import Task, Trajectory, Step, Usage
+
+
+class FunctionAgentAdapter(AgentAdapter):
+    """Drives an in-process Python callable.
+
+    The callable signature is fn(task, workspace, record) where `record`
+    is record(kind, name, payload) and it may return a dict with optional
+    keys: final_output (str), tokens (int), retries (int).
+    """
+
+    name = "function-agent"
+
+    def __init__(self, fn: Callable, name: Optional[str] = None):
+        self.fn = fn
+        if name:
+            self.name = name
+
+    @classmethod
+    def from_config(cls, config: dict) -> "FunctionAgentAdapter":
+        # config["entrypoint"] = "module.path:callable"
+        module_path, attr = config["entrypoint"].split(":")
+        fn = getattr(importlib.import_module(module_path), attr)
+        return cls(fn=fn, name=config.get("name", "function-agent"))
+
+    def run(self, task: Task, workspace: Path) -> Trajectory:
+        steps: list[Step] = []
+
+        def record(kind: str, name=None, payload=None):
+            steps.append(Step(index=len(steps), kind=kind, name=name,
+                              payload=payload or {}))
+
+        start = time.monotonic()
+        try:
+            ret = self.fn(task, str(workspace), record) or {}
+            elapsed = time.monotonic() - start
+            return Trajectory(
+                steps=steps,
+                final_output=ret.get("final_output", ""),
+                usage=Usage(
+                    total_tokens=ret.get("tokens", 0),
+                    wall_seconds=elapsed,
+                    num_steps=len(steps),
+                    num_retries=ret.get("retries", 0),
+                ),
+                status="success",
+            )
+        except Exception as e:
+            elapsed = time.monotonic() - start
+            return Trajectory(
+                steps=steps,
+                usage=Usage(wall_seconds=elapsed, num_steps=len(steps)),
+                status="crash",
+                error=f"{type(e).__name__}: {e}",
+            )

eval_agent/adapters/http.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+import json as jsonlib
+import time
+from pathlib import Path
+from typing import Callable, Optional
+from eval_agent.adapters import AgentAdapter
+from eval_agent.models import Task, Trajectory, Step, Usage
+
+
+def _default_post(url: str, json_body: dict, timeout: Optional[float]):
+    import urllib.request
+    data = jsonlib.dumps(json_body).encode()
+    req = urllib.request.Request(url, data=data,
+                                 headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return jsonlib.loads(resp.read().decode())
+
+
+class HTTPAgentAdapter(AgentAdapter):
+    """Drives an OpenAI-style chat endpoint.
+
+    `post(url, json_body, timeout) -> dict` is injectable for testing.
+    """
+
+    name = "http-agent"
+
+    def __init__(self, url: str, post: Callable = _default_post,
+                 name: str = "http-agent"):
+        self.url = url
+        self.post = post
+        self.name = name
+
+    @classmethod
+    def from_config(cls, config: dict) -> "HTTPAgentAdapter":
+        return cls(url=config["url"], name=config.get("name", "http-agent"))
+
+    def run(self, task: Task, workspace: Path) -> Trajectory:
+        body = {"messages": [{"role": "user", "content": task.prompt}]}
+        start = time.monotonic()
+        try:
+            resp = self.post(self.url, body, task.budget.timeout_seconds)
+        except Exception as e:
+            return Trajectory(
+                usage=Usage(wall_seconds=time.monotonic() - start),
+                status="crash",
+                error=f"{type(e).__name__}: {e}",
+            )
+
+        try:
+            message = resp["choices"][0]["message"]
+            steps: list[Step] = []
+            text = message.get("content") or ""
+            if text:
+                steps.append(Step(index=len(steps), kind="message",
+                                  payload={"text": text}))
+            for tc in message.get("tool_calls", []) or []:
+                fn = tc.get("function", {})
+                try:
+                    args = jsonlib.loads(fn.get("arguments", "{}"))
+                except jsonlib.JSONDecodeError:
+                    args = {"raw": fn.get("arguments")}
+                steps.append(Step(index=len(steps), kind="tool_call",
+                                  name=fn.get("name"), payload=args))
+        except Exception as e:
+            return Trajectory(
+                usage=Usage(wall_seconds=time.monotonic() - start),
+                status="crash",
+                error=f"{type(e).__name__}: {e}",
+            )
+
+        return Trajectory(
+            steps=steps,
+            final_output=text,
+            usage=Usage(total_tokens=resp.get("usage", {}).get("total_tokens", 0),
+                        wall_seconds=time.monotonic() - start,
+                        num_steps=len(steps)),
+            status="success",
+        )

eval_agent/adapters/webui.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+import time
+from pathlib import Path
+from typing import Callable, Optional
+from eval_agent.adapters import AgentAdapter
+from eval_agent.models import Task, Trajectory, Step, Usage
+
+
+class WebUIAgentAdapter(AgentAdapter):
+    """Drives a browser-based agent: type prompt, wait for reply, scrape it.
+
+    All page operations go through an injected BrowserDriver factory, so the
+    adapter is fully unit-testable offline. clock/sleep are injectable so the
+    text-stabilization loop is deterministic in tests.
+    """
+
+    name = "webui-agent"
+
+    def __init__(
+        self,
+        url: str,
+        input_selector: str,
+        output_selector: str,
+        *,
+        submit: str = "enter",
+        storage_state: Optional[str] = None,
+        headless: bool = True,
+        stable_ms: int = 1500,
+        done_selector: Optional[str] = None,
+        done_timeout: Optional[float] = None,
+        poll_interval: float = 0.25,
+        name: str = "webui-agent",
+        driver_factory: Optional[Callable[[], "BrowserDriver"]] = None,
+        clock: Callable[[], float] = time.monotonic,
+        sleep: Callable[[float], None] = time.sleep,
+    ):
+        self.url = url
+        self.input_selector = input_selector
+        self.output_selector = output_selector
+        self.submit = submit
+        self.storage_state = storage_state
+        self.headless = headless
+        self.stable_ms = stable_ms
+        self.done_selector = done_selector
+        self.done_timeout = done_timeout
+        self.poll_interval = poll_interval
+        self.name = name
+        self.driver_factory = driver_factory or (lambda: PlaywrightDriver())
+        self.clock = clock
+        self.sleep = sleep
+
+    @classmethod
+    def from_config(cls, config: dict) -> "WebUIAgentAdapter":
+        done = config.get("done") or {}
+        return cls(
+            url=config["url"],
+            input_selector=config["input_selector"],
+            output_selector=config["output_selector"],
+            submit=config.get("submit", "enter"),
+            storage_state=config.get("storage_state"),
+            headless=config.get("headless", True),
+            stable_ms=done.get("stable_ms", 1500),
+            done_selector=done.get("selector"),
+            done_timeout=done.get("timeout_seconds"),
+            name=config.get("name", "webui-agent"),
+        )
+
+    def run(self, task: Task, workspace: Path) -> Trajectory:
+        driver = self.driver_factory()
+        start = self.clock()
+        timeout = self.done_timeout or task.budget.timeout_seconds or 120.0
+        try:
+            try:
+                driver.open(self.url, self.storage_state, self.headless)
+                driver.wait_for(self.input_selector, timeout)
+                driver.fill(self.input_selector, task.prompt)
+                if self.submit == "enter":
+                    driver.press_enter(self.input_selector)
+                else:
+                    driver.click(self.submit)
+                try:
+                    output = self._wait_for_done(driver, timeout)
+                except TimeoutError as e:
+                    partial = e.args[0] if e.args else ""
+                    return Trajectory(
+                        steps=[Step(index=0, kind="message",
+                                    payload={"text": task.prompt})],
+                        final_output=partial,
+                        usage=Usage(wall_seconds=self.clock() - start,
+                                    num_steps=1),
+                        status="timeout",
+                        error=f"agent reply did not stabilize within {timeout}s",
+                    )
+            except Exception as e:
+                return Trajectory(
+                    usage=Usage(wall_seconds=self.clock() - start),
+                    status="crash",
+                    error=f"{type(e).__name__}: {e}",
+                )
+        finally:
+            try:
+                driver.close()
+            except Exception:
+                pass
+        return Trajectory(
+            steps=[Step(index=0, kind="message", payload={"text": task.prompt})],
+            final_output=output,
+            usage=Usage(wall_seconds=self.clock() - start, num_steps=1),
+            status="success",
+        )
+
+    def _wait_for_done(self, driver, timeout: float) -> str:
+        """Poll output_selector; done when text is non-empty, unchanged for
+        stable_ms, and (if configured) done_selector is present. Raises
+        TimeoutError(last_text) if the deadline passes first."""
+        start = self.clock()
+        last_text = ""
+        last_change = start
+        while self.clock() - start < timeout:
+            text = driver.read_text(self.output_selector)
+            now = self.clock()
+            if text != last_text:
+                last_text = text
+                last_change = now
+            done_signal = (self.done_selector is None) or driver.is_present(
+                self.done_selector
+            )
+            if last_text and done_signal and (now - last_change) * 1000 >= self.stable_ms:
+                return last_text
+            self.sleep(self.poll_interval)
+        raise TimeoutError(last_text)
+
+
+class BrowserDriver:
+    """Primitive page operations. No evaluation logic lives here."""
+
+    def open(self, url, storage_state, headless): raise NotImplementedError
+    def wait_for(self, selector, timeout): raise NotImplementedError
+    def fill(self, selector, text): raise NotImplementedError
+    def press_enter(self, selector): raise NotImplementedError
+    def click(self, selector): raise NotImplementedError
+    def read_text(self, selector) -> str: raise NotImplementedError
+    def is_present(self, selector) -> bool: raise NotImplementedError
+    def close(self): raise NotImplementedError
+
+
+class PlaywrightDriver(BrowserDriver):
+    """Real driver. Imports playwright lazily so the module loads without it."""
+
+    def __init__(self):
+        self._pw = None
+        self._browser = None
+        self._context = None
+        self._page = None
+
+    def open(self, url, storage_state, headless):
+        try:
+            from playwright.sync_api import sync_playwright
+        except ImportError as e:
+            raise RuntimeError(
+                "WebUI adapter needs Playwright. Install with "
+                "'pip install eval-agent[webui]' then run "
+                "'playwright install chromium'."
+            ) from e
+        self._pw = sync_playwright().start()
+        self._browser = self._pw.chromium.launch(headless=headless)
+        ctx_kwargs = {"storage_state": storage_state} if storage_state else {}
+        self._context = self._browser.new_context(**ctx_kwargs)
+        self._page = self._context.new_page()
+        self._page.goto(url)
+
+    def wait_for(self, selector, timeout):
+        self._page.wait_for_selector(selector, timeout=timeout * 1000)
+
+    def fill(self, selector, text):
+        self._page.fill(selector, text)
+
+    def press_enter(self, selector):
+        self._page.press(selector, "Enter")
+
+    def click(self, selector):
+        self._page.click(selector)
+
+    def read_text(self, selector) -> str:
+        el = self._page.query_selector(selector)
+        return el.inner_text() if el else ""
+
+    def is_present(self, selector) -> bool:
+        return self._page.query_selector(selector) is not None
+
+    def close(self):
+        if self._context:
+            self._context.close()
+        if self._browser:
+            self._browser.close()
+        if self._pw:
+            self._pw.stop()

eval_agent/aggregate.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+from typing import Any
+from eval_agent.models import Score, TaskResult
+
+DEFAULT_WEIGHTS: dict[str, float] = {
+    "checkpoint": 3,
+    "llm_judge": 2,
+    "trajectory": 1,
+    "efficiency": 0,
+}
+
+
+def _weight_for(name: str, task_weights: dict, suite_weights: dict) -> float:
+    if name in task_weights:
+        return task_weights[name]
+    if name in suite_weights:
+        return suite_weights[name]
+    return DEFAULT_WEIGHTS.get(name, 1.0)
+
+
+def aggregate_task(scores: list[Score], task_weights: dict,
+                   suite_weights: dict) -> float:
+    num = 0.0
+    den = 0.0
+    for s in scores:
+        if not s.available:
+            continue
+        w = _weight_for(s.scorer_name, task_weights, suite_weights)
+        if w <= 0:
+            continue
+        num += s.value * w
+        den += w
+    return num / den if den else 0.0
+
+
+def apply_weights(scores: list[Score], task_weights: dict,
+                  suite_weights: dict) -> list[Score]:
+    """Return scores with their effective weight filled in (for reporting)."""
+    out = []
+    for s in scores:
+        s.weight = _weight_for(s.scorer_name, task_weights, suite_weights)
+        out.append(s)
+    return out
+
+
+def aggregate_run(task_results: list[TaskResult]) -> dict[str, Any]:
+    n = len(task_results)
+    if n == 0:
+        return {"num_tasks": 0, "mean_score": 0.0, "num_success": 0}
+    mean = sum(tr.aggregate_score for tr in task_results) / n
+    num_success = sum(1 for tr in task_results
+                      if tr.trajectory.status == "success")
+    return {
+        "num_tasks": n,
+        "mean_score": mean,
+        "num_success": num_success,
+    }

eval_agent/cli.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+import json
+import os
+import sys
+from pathlib import Path
+import click
+from eval_agent.loader import load_suite, load_agent_config
+from eval_agent.adapters import build_adapter
+from eval_agent.scorers import build_scorers
+from eval_agent.runner import Runner
+from eval_agent.reporter import write_json, write_html, write_index
+from eval_agent.models import RunResult
+
+
+def _stdin_isatty() -> bool:
+    return sys.stdin.isatty()
+
+
+@click.group()
+def cli():
+    """eval-agent: evaluate agents against task suites."""
+
+
+@cli.command()
+@click.option("--suite", "suite_path", default=None)
+@click.option("--agent", "agent_path", default=None)
+@click.option("--out", "out_dir", default="runs")
+@click.option("--run-id", "run_id", default=None)
+@click.option("--no-llm-judge", is_flag=True, default=False,
+              help="Disable the LLM judge scorer (avoids API calls).")
+def run(suite_path, agent_path, out_dir, run_id, no_llm_judge):
+    """Run a suite against an agent; write JSON + HTML.
+
+    With both --suite and --agent, runs non-interactively. If either is missing
+    and stdin is a TTY, launches an interactive wizard to build the agent and
+    tasks. Missing args without a TTY is an error (never hangs)."""
+    wiz_result = None
+    if suite_path and agent_path:
+        suite = load_suite(suite_path)
+        agent_cfg = load_agent_config(agent_path)
+        judge_enabled = not no_llm_judge
+    elif _stdin_isatty():
+        from eval_agent import wizard
+        wiz_result = wizard.run_wizard()
+        suite = wiz_result.suite
+        agent_cfg = wiz_result.agent_config
+        judge_enabled = wiz_result.uses_rubric and not no_llm_judge
+        if wiz_result.uses_rubric and not os.environ.get("ANTHROPIC_API_KEY"):
+            click.echo("warning: 有任务用了 rubric 但未设置 ANTHROPIC_API_KEY,"
+                       "LLM judge 将无法评分。")
+    else:
+        raise click.UsageError(
+            "Missing option '--suite' and/or '--agent' (non-interactive stdin).")
+
+    adapter = build_adapter(agent_cfg)
+    scorers = build_scorers({"llm_judge": {"enabled": judge_enabled}})
+
+    if run_id is None:
+        run_id = f"{suite.name}-{adapter.name}"
+
+    runner = Runner(adapter=adapter, scorers=scorers)
+    result = runner.run(suite, run_id=run_id)
+
+    run_out = str(Path(out_dir) / run_id)
+    write_json(result, run_out)
+    write_html([result], run_out)
+    write_index(out_dir)
+    click.echo(f"Run complete: {run_out}  mean_score="
+               f"{result.stats.get('mean_score', 0.0):.3f}")
+
+    if wiz_result is not None and click.confirm("保存这次配置以便下次复用?",
+                                                default=False):
+        from eval_agent import wizard
+        agent_path_out, suite_path_out = wizard.save_wizard_output(wiz_result)
+        click.echo(f"已写入 {agent_path_out} 和 {suite_path_out}")
+
+
+@cli.command()
+@click.option("--out", "out_dir", default="runs")
+def index(out_dir):
+    """Write an index.html overview of all runs under --out."""
+    path = write_index(out_dir)
+    click.echo(f"Index written: {path}")
+
+
+@cli.command()
+@click.option("--run", "run_dir", required=True)
+def report(run_dir):
+    """Re-render report.html from an existing result.json."""
+    data = json.loads((Path(run_dir) / "result.json").read_text())
+    rr = RunResult.model_validate(data)
+    write_html([rr], run_dir)
+    click.echo(f"Report written: {Path(run_dir) / 'report.html'}")
+
+
+@cli.command()
+@click.argument("run_dirs", nargs=-1, required=True)
+@click.option("--out", "out_dir", default="runs/compare")
+def compare(run_dirs, out_dir):
+    """Render a combined leaderboard across multiple runs."""
+    runs = []
+    for d in run_dirs:
+        data = json.loads((Path(d) / "result.json").read_text())
+        runs.append(RunResult.model_validate(data))
+    write_html(runs, out_dir)
+    click.echo(f"Comparison written: {Path(out_dir) / 'report.html'}")

eval_agent/examples/echo.py

@@ -0,0 +1,11 @@
+"""A trivial example agent for smoke-testing the framework end-to-end.
+It writes the prompt into output.txt and reports done."""
+from __future__ import annotations
+from pathlib import Path
+
+
+def run(task, workspace, record):
+    record(kind="tool_call", name="write_file", payload={"path": "output.txt"})
+    Path(workspace, "output.txt").write_text(task.prompt)
+    record(kind="message", payload={"text": "done"})
+    return {"final_output": "done", "tokens": len(task.prompt)}

eval_agent/loader.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+import yaml
+from eval_agent.models import TaskSuite, Task, Checkpoint, Budget
+
+
+def _parse_checkpoint(raw: dict) -> Checkpoint:
+    raw = dict(raw)
+    ctype = raw.pop("type")
+    return Checkpoint(type=ctype, args=raw)
+
+
+def _parse_task(raw: dict) -> Task:
+    raw = dict(raw)
+    raw["checkpoints"] = [_parse_checkpoint(c)
+                          for c in raw.get("checkpoints", [])]
+    if "budget" in raw:
+        raw["budget"] = Budget(**raw["budget"])
+    return Task(**raw)
+
+
+def load_suite(path: str) -> TaskSuite:
+    with open(path) as f:
+        data = yaml.safe_load(f)
+    tasks = [_parse_task(t) for t in data.get("tasks", [])]
+    return TaskSuite(
+        name=data["name"],
+        tasks=tasks,
+        default_weights=data.get("default_weights", {}),
+        meta=data.get("meta", {}),
+    )
+
+
+def load_agent_config(path: str) -> dict:
+    with open(path) as f:
+        return yaml.safe_load(f)