evalpilot 0.1.0__py3-none-any.whl

eval_agent/models.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+from typing import Any, Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class Budget(BaseModel):
+    max_tokens: Optional[int] = None
+    max_steps: Optional[int] = None
+    timeout_seconds: Optional[float] = None
+
+
+class Checkpoint(BaseModel):
+    type: str
+    args: dict[str, Any] = Field(default_factory=dict)
+
+
+class Task(BaseModel):
+    id: str
+    prompt: str
+    setup: dict[str, Any] = Field(default_factory=dict)
+    checkpoints: list[Checkpoint] = Field(default_factory=list)
+    rubric: Optional[str] = None
+    budget: Budget = Field(default_factory=Budget)
+    weights: dict[str, float] = Field(default_factory=dict)
+    tags: list[str] = Field(default_factory=list)
+
+
+class TaskSuite(BaseModel):
+    name: str
+    tasks: list[Task]
+    default_weights: dict[str, float] = Field(default_factory=dict)
+    meta: dict[str, Any] = Field(default_factory=dict)
+
+
+class Step(BaseModel):
+    index: int
+    kind: Literal["tool_call", "message", "result"]
+    name: Optional[str] = None
+    payload: dict[str, Any] = Field(default_factory=dict)
+
+
+class Usage(BaseModel):
+    total_tokens: int = 0
+    wall_seconds: float = 0.0
+    num_steps: int = 0
+    num_retries: int = 0
+
+
+class Trajectory(BaseModel):
+    steps: list[Step] = Field(default_factory=list)
+    final_output: str = ""
+    usage: Usage = Field(default_factory=Usage)
+    final_state: Optional[str] = None  # path to workspace snapshot
+    status: Literal["success", "timeout", "crash"] = "success"
+    error: Optional[str] = None
+
+
+class Score(BaseModel):
+    scorer_name: str
+    value: float
+    weight: float = 1.0
+    reason: str = ""
+    available: bool = True
+
+
+class TaskResult(BaseModel):
+    task_id: str
+    agent_name: str
+    trajectory: Trajectory
+    scores: list[Score] = Field(default_factory=list)
+    aggregate_score: float = 0.0
+
+
+class RunResult(BaseModel):
+    run_id: str
+    suite_name: str
+    task_results: list[TaskResult] = Field(default_factory=list)
+    stats: dict[str, Any] = Field(default_factory=dict)
+    meta: dict[str, Any] = Field(default_factory=dict)

eval_agent/reporter.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+import json
+import time
+from pathlib import Path
+from jinja2 import Environment, PackageLoader, select_autoescape
+from eval_agent.models import RunResult
+
+
+def write_json(run_result: RunResult, out_dir: str) -> str:
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    path = out / "result.json"
+    path.write_text(run_result.model_dump_json(indent=2))
+    return str(path)
+
+
+def write_html(run_results: list[RunResult], out_dir: str) -> str:
+    env = Environment(
+        loader=PackageLoader("eval_agent", "templates"),
+        autoescape=select_autoescape(["html"]),
+    )
+    template = env.get_template("report.html.j2")
+    html = template.render(runs=run_results)
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    path = out / "report.html"
+    path.write_text(html)
+    return str(path)
+
+
+def _scan_runs(out_dir: str) -> list[dict]:
+    """Read each <out_dir>/*/result.json into a row dict, newest first.
+    Unreadable/malformed result files are skipped."""
+    base = Path(out_dir)
+    rows: list[dict] = []
+    for result_path in sorted(base.glob("*/result.json")):
+        try:
+            data = json.loads(result_path.read_text())
+        except Exception:
+            continue
+        stats = data.get("stats", {})
+        mtime = result_path.stat().st_mtime
+        rows.append({
+            "run_id": data.get("run_id", result_path.parent.name),
+            "agent": data.get("meta", {}).get("agent", ""),
+            "suite": data.get("suite_name", ""),
+            "mean_score": stats.get("mean_score", 0.0),
+            "num_tasks": stats.get("num_tasks", 0),
+            "num_success": stats.get("num_success", 0),
+            "mtime": mtime,
+            "time": time.strftime("%m-%d %H:%M", time.localtime(mtime)),
+        })
+    rows.sort(key=lambda r: r["mtime"], reverse=True)
+    return rows
+
+
+def write_index(out_dir: str) -> str:
+    """Render <out_dir>/index.html listing all runs found under out_dir."""
+    rows = _scan_runs(out_dir)
+    env = Environment(
+        loader=PackageLoader("eval_agent", "templates"),
+        autoescape=select_autoescape(["html"]),
+    )
+    template = env.get_template("index.html.j2")
+    html = template.render(rows=rows)
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    path = out / "index.html"
+    path.write_text(html)
+    return str(path)

eval_agent/runner.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+import shutil
+from eval_agent.adapters import AgentAdapter
+from eval_agent.scorers import Scorer
+from eval_agent.sandbox import Sandbox
+from eval_agent.aggregate import aggregate_task, aggregate_run, apply_weights
+from eval_agent.models import TaskSuite, Task, TaskResult, RunResult
+
+
+class Runner:
+    def __init__(self, adapter: AgentAdapter, scorers: list[Scorer]):
+        self.adapter = adapter
+        self.scorers = scorers
+
+    def run(self, suite: TaskSuite, run_id: str) -> RunResult:
+        results = [self._run_task(t, suite) for t in suite.tasks]
+        stats = aggregate_run(results)
+        return RunResult(run_id=run_id, suite_name=suite.name,
+                         task_results=results, stats=stats,
+                         meta={"agent": self.adapter.name})
+
+    def _run_task(self, task: Task, suite: TaskSuite) -> TaskResult:
+        with Sandbox.create(task.setup) as ws:
+            trajectory = self.adapter.run(task, ws.path)
+            trajectory.final_state = ws.snapshot()
+
+        snapshot_path = trajectory.final_state
+        try:
+            scores = [s.score(task, trajectory) for s in self.scorers]
+            scores = apply_weights(scores, task.weights, suite.default_weights)
+            agg = aggregate_task(scores, task.weights, suite.default_weights)
+            return TaskResult(task_id=task.id, agent_name=self.adapter.name,
+                              trajectory=trajectory, scores=scores,
+                              aggregate_score=agg)
+        finally:
+            if snapshot_path:
+                shutil.rmtree(snapshot_path, ignore_errors=True)

eval_agent/sandbox.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+class Sandbox:
+    """Isolated per-task workspace. Use as a context manager.
+
+    setup schema:
+      {"files": {"<relpath>": "<content>", ...}}
+    """
+
+    def __init__(self, path: Path):
+        self.path = path
+        self._snapshot_dir: Path | None = None
+
+    @classmethod
+    def create(cls, setup: dict[str, Any]) -> "Sandbox":
+        path = Path(tempfile.mkdtemp(prefix="evalagent_ws_"))
+        sb = cls(path)
+        sb._apply_setup(setup or {})
+        return sb
+
+    def _apply_setup(self, setup: dict[str, Any]) -> None:
+        files = setup.get("files", {})
+        for relpath, content in files.items():
+            target = self.path / relpath
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(content)
+
+    def snapshot(self) -> str:
+        """Copy current workspace to a persistent temp dir; return its path."""
+        self._snapshot_dir = Path(tempfile.mkdtemp(prefix="evalagent_snap_"))
+        shutil.copytree(self.path, self._snapshot_dir, dirs_exist_ok=True)
+        return str(self._snapshot_dir)
+
+    def __enter__(self) -> "Sandbox":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        shutil.rmtree(self.path, ignore_errors=True)

eval_agent/scorers/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from eval_agent.models import Task, Trajectory, Score
+
+
+class Scorer(ABC):
+    """Read-only grader. Reads a Trajectory, returns a Score. Never mutates."""
+
+    name: str = "scorer"
+
+    @abstractmethod
+    def score(self, task: Task, trajectory: Trajectory) -> Score:
+        ...
+
+
+def build_scorers(config: dict | None = None) -> list[Scorer]:
+    """Factory: instantiate the default set of scorers.
+
+    config may carry per-scorer options (e.g. llm_judge model). For the
+    default build we instantiate all four.
+    """
+    config = config or {}
+    from eval_agent.scorers.checkpoint import CheckpointScorer
+    from eval_agent.scorers.trajectory import TrajectoryScorer
+    from eval_agent.scorers.efficiency import EfficiencyScorer
+    scorers: list[Scorer] = [
+        CheckpointScorer(),
+        TrajectoryScorer(),
+        EfficiencyScorer(),
+    ]
+    if config.get("llm_judge", {}).get("enabled", True):
+        from eval_agent.scorers.llm_judge import LLMJudgeScorer
+        scorers.append(LLMJudgeScorer(**config.get("llm_judge", {}).get("kwargs", {})))
+    return scorers

eval_agent/scorers/checkpoint.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+import json
+import re
+import subprocess
+from pathlib import Path
+from eval_agent.scorers import Scorer
+from eval_agent.models import Task, Trajectory, Score, Checkpoint
+
+
+class CheckpointScorer(Scorer):
+    name = "checkpoint"
+
+    def score(self, task: Task, trajectory: Trajectory) -> Score:
+        if not task.checkpoints:
+            return Score(scorer_name=self.name, value=0.0, reason="no checkpoints",
+                         available=False)
+        root = Path(trajectory.final_state) if trajectory.final_state else None
+        passed = 0
+        details = []
+        for cp in task.checkpoints:
+            ok = self._check(cp, root, trajectory)
+            passed += int(ok)
+            details.append(f"{cp.type}={'ok' if ok else 'fail'}")
+        value = passed / len(task.checkpoints)
+        return Score(scorer_name=self.name, value=value,
+                     reason=f"{passed}/{len(task.checkpoints)} passed | " +
+                            ", ".join(details))
+
+    def _check(self, cp: Checkpoint, root: Path | None,
+               trajectory: Trajectory) -> bool:
+        try:
+            a = cp.args
+            if cp.type == "file_exists":
+                return root is not None and (root / a["path"]).exists()
+            if cp.type == "file_contains":
+                p = root / a["path"] if root else None
+                return bool(p and p.exists() and a["pattern"] in p.read_text())
+            if cp.type == "command_passes":
+                cwd = str(root) if root else None
+                r = subprocess.run(a["cmd"], shell=True, cwd=cwd,
+                                   capture_output=True)
+                return r.returncode == 0
+            if cp.type == "json_equals":
+                p = root / a["path"] if root else None
+                if not (p and p.exists()):
+                    return False
+                data = json.loads(p.read_text())
+                for key in a["field"].split("."):
+                    data = data.get(key) if isinstance(data, dict) else None
+                return data == a["value"]
+            if cp.type == "output_contains":
+                out = trajectory.final_output or ""
+                text = a["text"]
+                if a.get("case_sensitive", False):
+                    return text in out
+                return text.lower() in out.lower()
+            if cp.type == "output_matches":
+                return re.search(a["pattern"], trajectory.final_output or "") is not None
+            if cp.type == "custom":
+                cwd = str(root) if root else None
+                r = subprocess.run(a["script"], shell=True, cwd=cwd,
+                                   capture_output=True)
+                return r.returncode == 0
+            return False
+        except Exception:
+            return False

eval_agent/scorers/efficiency.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+from eval_agent.scorers import Scorer
+from eval_agent.models import Task, Trajectory, Score
+
+
+class EfficiencyScorer(Scorer):
+    name = "efficiency"
+
+    def score(self, task: Task, trajectory: Trajectory) -> Score:
+        u = trajectory.usage
+        b = task.budget
+        raw = (f"tokens={u.total_tokens}, steps={u.num_steps}, "
+               f"wall={u.wall_seconds:.2f}s, retries={u.num_retries}")
+        if b.max_tokens:
+            value = max(0.0, min(1.0, 1 - u.total_tokens / b.max_tokens))
+            return Score(scorer_name=self.name, value=value, weight=0.0,
+                         reason=raw, available=True)
+        return Score(scorer_name=self.name, value=0.0, weight=0.0,
+                     reason=raw + " (no token budget set)", available=False)

eval_agent/scorers/llm_judge.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+import json
+from typing import Callable, Optional
+from eval_agent.scorers import Scorer
+from eval_agent.models import Task, Trajectory, Score
+
+_SYSTEM = (
+    "You are a strict evaluation judge. Given a rubric and an agent's output, "
+    "return ONLY a JSON object: {\"score\": <float 0-1>, \"reasons\": [<str>, ...]}. "
+    "Anchor the score to the rubric's stated criteria."
+)
+
+
+def _anthropic_complete(model: str) -> Callable[[str, str], str]:
+    def complete(system: str, user: str) -> str:
+        import anthropic
+        client = anthropic.Anthropic()
+        resp = client.messages.create(
+            model=model, max_tokens=1024, system=system,
+            messages=[{"role": "user", "content": user}],
+        )
+        return resp.content[0].text
+    return complete
+
+
+class LLMJudgeScorer(Scorer):
+    name = "llm_judge"
+
+    def __init__(self, complete: Optional[Callable[[str, str], str]] = None,
+                 model: str = "claude-opus-4-8"):
+        self.complete = complete or _anthropic_complete(model)
+
+    def score(self, task: Task, trajectory: Trajectory) -> Score:
+        if not task.rubric:
+            return Score(scorer_name=self.name, value=0.0,
+                         reason="no rubric provided", available=False)
+        user = (f"RUBRIC:\n{task.rubric}\n\n"
+                f"AGENT OUTPUT:\n{trajectory.final_output}\n\n"
+                f"Return the JSON now.")
+        raw = self.complete(_SYSTEM, user)
+        try:
+            data = json.loads(self._extract_json(raw))
+            value = float(data["score"])
+            reasons = data.get("reasons", [])
+            return Score(scorer_name=self.name, value=value,
+                         reason="; ".join(reasons))
+        except (json.JSONDecodeError, KeyError, ValueError, TypeError):
+            return Score(scorer_name=self.name, value=0.0,
+                         reason=f"could not parse judge output: {raw[:200]}")
+
+    @staticmethod
+    def _extract_json(text: str) -> str:
+        start = text.find("{")
+        end = text.rfind("}")
+        if start == -1 or end == -1:
+            return text
+        return text[start:end + 1]

eval_agent/scorers/trajectory.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+import json
+from eval_agent.scorers import Scorer
+from eval_agent.models import Task, Trajectory, Score
+
+
+class TrajectoryScorer(Scorer):
+    """Rule-based process quality. Degrades to unavailable for black-box
+    agents that expose no steps."""
+
+    name = "trajectory"
+
+    def score(self, task: Task, trajectory: Trajectory) -> Score:
+        steps = trajectory.steps
+        if not steps:
+            return Score(scorer_name=self.name, value=0.0,
+                         reason="no steps recorded (black-box agent)",
+                         available=False)
+
+        penalties = []
+        value = 1.0
+
+        repeats = self._max_consecutive_repeats(trajectory)
+        if repeats >= 3:
+            value -= 0.4
+            penalties.append(f"repeated identical action x{repeats}")
+
+        if task.budget.max_steps and len(steps) > task.budget.max_steps:
+            value -= 0.3
+            penalties.append(
+                f"exceeded max_steps ({len(steps)}>{task.budget.max_steps})")
+
+        value = max(0.0, value)
+        reason = "clean run" if not penalties else "; ".join(penalties)
+        return Score(scorer_name=self.name, value=value, reason=reason)
+
+    @staticmethod
+    def _max_consecutive_repeats(trajectory: Trajectory) -> int:
+        best = 1
+        run = 1
+        prev = None
+        for s in trajectory.steps:
+            key = (s.name, json.dumps(s.payload, sort_keys=True))
+            if key == prev:
+                run += 1
+                best = max(best, run)
+            else:
+                run = 1
+            prev = key
+        return best

eval_agent/templates/index.html.j2

@@ -0,0 +1,73 @@
+<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>eval-agent runs</title>
+  <style>
+    body { font-family: system-ui, sans-serif; margin: 2rem; }
+    table { border-collapse: collapse; width: 100%; }
+    th, td { border: 1px solid #ccc; padding: 6px 10px; text-align: left; }
+    th { background: #f3f3f3; cursor: pointer; user-select: none; }
+    tr.run-row:hover { background: #f9f9f9; cursor: pointer; }
+    .empty { color: #777; }
+  </style>
+</head>
+<body>
+  <h1>eval-agent runs</h1>
+  {% if rows %}
+  <table id="runs">
+    <thead>
+      <tr>
+        <th data-type="text">run-id</th>
+        <th data-type="text">agent</th>
+        <th data-type="text">suite</th>
+        <th data-type="num">score</th>
+        <th data-type="num">tasks</th>
+        <th data-type="num">ok</th>
+        <th data-type="text">time</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for r in rows %}
+      <tr class="run-row" data-href="{{ r.run_id }}/report.html">
+        <td>{{ r.run_id }}</td>
+        <td>{{ r.agent }}</td>
+        <td>{{ r.suite }}</td>
+        <td>{{ "%.3f"|format(r.mean_score) }}</td>
+        <td>{{ r.num_tasks }}</td>
+        <td>{{ r.num_success }}</td>
+        <td>{{ r.time }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  <script>
+    document.querySelectorAll("tr.run-row").forEach(function (row) {
+      row.addEventListener("click", function () {
+        window.location.href = row.dataset.href;
+      });
+    });
+    document.querySelectorAll("#runs thead th").forEach(function (th, idx) {
+      th.addEventListener("click", function () {
+        var tbody = document.querySelector("#runs tbody");
+        var rows = Array.prototype.slice.call(tbody.querySelectorAll("tr"));
+        var numeric = th.dataset.type === "num";
+        var asc = th.dataset.asc !== "true";
+        th.dataset.asc = asc;
+        rows.sort(function (a, b) {
+          var x = a.children[idx].textContent.trim();
+          var y = b.children[idx].textContent.trim();
+          if (numeric) { x = parseFloat(x); y = parseFloat(y); }
+          if (x < y) return asc ? -1 : 1;
+          if (x > y) return asc ? 1 : -1;
+          return 0;
+        });
+        rows.forEach(function (r) { tbody.appendChild(r); });
+      });
+    });
+  </script>
+  {% else %}
+  <p class="empty">暂无评测。先跑一次:<code>eval-agent run</code></p>
+  {% endif %}
+</body>
+</html>

eval_agent/templates/report.html.j2

@@ -0,0 +1,68 @@
+<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Eval-Agent Report</title>
+  <style>
+    body { font-family: system-ui, sans-serif; margin: 2rem; }
+    table { border-collapse: collapse; margin-bottom: 2rem; }
+    th, td { border: 1px solid #ccc; padding: 6px 10px; text-align: left; }
+    th { background: #f3f3f3; }
+    .task { margin: 1rem 0; padding: 1rem; border: 1px solid #ddd; }
+    .reason { color: #555; font-size: 0.9em; }
+    .status-crash, .status-timeout { color: #b00; }
+  </style>
+</head>
+<body>
+  <h1>Eval-Agent Report</h1>
+
+  <h2>Leaderboard</h2>
+  <table>
+    <tr><th>Agent</th><th>Suite</th><th>Tasks</th><th>Success</th><th>Mean score</th></tr>
+    {% for rr in runs %}
+    {% set agent_label = rr.meta.get("agent") or (rr.task_results[0].agent_name if rr.task_results else rr.run_id) %}
+    <tr>
+      <td>{{ agent_label }}</td>
+      <td>{{ rr.suite_name }}</td>
+      <td>{{ rr.stats.get("num_tasks", 0) }}</td>
+      <td>{{ rr.stats.get("num_success", 0) }}</td>
+      <td>{{ "%.3f"|format(rr.stats.get("mean_score", 0.0)) }}</td>
+    </tr>
+    {% endfor %}
+  </table>
+
+  {% for rr in runs %}
+  {% set agent_label = rr.meta.get("agent") or (rr.task_results[0].agent_name if rr.task_results else rr.run_id) %}
+  <h2>{{ agent_label }} — task details</h2>
+  {% for tr in rr.task_results %}
+  <div class="task">
+    <h3>{{ tr.task_id }}
+      <span class="status-{{ tr.trajectory.status }}">[{{ tr.trajectory.status }}]</span>
+      — score {{ "%.3f"|format(tr.aggregate_score) }}
+    </h3>
+    <table>
+      <tr><th>Scorer</th><th>Value</th><th>Weight</th><th>Reason</th></tr>
+      {% for s in tr.scores %}
+      <tr>
+        <td>{{ s.scorer_name }}{% if not s.available %} (n/a){% endif %}</td>
+        <td>{{ "%.3f"|format(s.value) }}</td>
+        <td>{{ s.weight }}</td>
+        <td class="reason">{{ s.reason }}</td>
+      </tr>
+      {% endfor %}
+    </table>
+    <details>
+      <summary>Trajectory ({{ tr.trajectory.steps|length }} steps,
+        {{ tr.trajectory.usage.total_tokens }} tokens)</summary>
+      <ol>
+        {% for step in tr.trajectory.steps %}
+        <li>{{ step.kind }}{% if step.name %}: {{ step.name }}{% endif %}
+          — {{ step.payload }}</li>
+        {% endfor %}
+      </ol>
+    </details>
+  </div>
+  {% endfor %}
+  {% endfor %}
+</body>
+</html>