aeval-framework 0.1.0__py3-none-any.whl

agent_eval/examples/mock_runner.py

@@ -0,0 +1,195 @@
+"""
+Mock AgentRunner for testing and demonstration.
+
+Simulates an agent by returning predefined results, with per-task scripted
+behaviors for exercising the framework's failure paths.
+
+Usage:
+    from agent_eval.examples.mock_runner import MockAgentRunner
+
+    # Random behavior (demo)
+    runner = EvalRunner(agent_runner=MockAgentRunner())
+
+    # Scripted behavior (tests): each task consumes its behavior list in
+    # order across calls; the last entry repeats once exhausted.
+    agent = MockAgentRunner(
+        latency_range=(0.0, 0.01),
+        script={
+            "task_ok": ["success"],
+            "task_flaky": ["transient", "transient", "success"],
+            "task_dead": ["failure"],
+            "task_slow": ["timeout"],
+        },
+    )
+"""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import uuid
+from typing import Any
+
+from agent_eval.core.contract import TransientError
+from agent_eval.core.types import EvalTask
+
+
+class MockAgentRunner:
+    """
+    模拟 AgentRunner。
+
+    用于测试和演示框架功能，无需真实 Agent 系统。
+    支持脚本化场景: success / failure / transient / timeout。
+    """
+
+    def __init__(
+        self,
+        success_rate: float = 0.7,
+        latency_range: tuple[float, float] = (0.1, 0.5),
+        script: dict[str, list[str]] | None = None,
+        timeout_duration: float = 10.0,
+    ):
+        """
+        Args:
+            success_rate: 随机模式下的模拟成功率 (0.0-1.0)
+            latency_range: 模拟延迟范围 (秒)
+            script: task_id → 行为序列 ("success"|"failure"|"transient"|"timeout"),
+                    逐次调用消耗, 耗尽后重复最后一项
+            timeout_duration: "timeout" 行为的挂起时长 (秒),
+                              配合 EvalRunner(per_trial_timeout=...) 触发超时
+        """
+        self.success_rate = success_rate
+        self.latency_range = latency_range
+        self.script = script or {}
+        self.timeout_duration = timeout_duration
+        self.call_counts: dict[str, int] = {}
+
+    def _next_behavior(self, task_id: str, call_index: int) -> str | None:
+        """取该 task 指定调用的脚本行为 (无脚本返回 None = 随机模式)"""
+        behaviors = self.script.get(task_id)
+        if not behaviors:
+            return None
+        return behaviors[min(call_index, len(behaviors) - 1)]
+
+    async def run(
+        self,
+        task: EvalTask,
+    ) -> tuple[str, list[dict[str, Any]], dict[str, Any]]:
+        """
+        模拟 Agent 执行。
+
+        Returns:
+            trace_id, transcript, outcome
+        """
+        index = self.call_counts.get(task.id, 0)
+        self.call_counts[task.id] = index + 1
+        behavior = self._next_behavior(task.id, index)
+
+        # 超时场景: 长时间挂起, 由框架的 per_trial_timeout 打断
+        if behavior == "timeout":
+            await asyncio.sleep(self.timeout_duration)
+
+        # 模拟延迟
+        latency = random.uniform(*self.latency_range)
+        await asyncio.sleep(latency)
+
+        # 瞬态错误场景: 框架按指数退避重试
+        if behavior == "transient":
+            raise TransientError(
+                f"Mock transient failure for {task.id} (call {index + 1})"
+            )
+
+        # 生成 trace_id (编码 task id, 便于 MockTraceProvider 关联 spans)
+        trace_id = f"trace_{task.id}_{uuid.uuid4().hex[:8]}"
+
+        # 构建 transcript
+        transcript = [
+            {
+                "role": "user",
+                "content": task.prompt,
+            },
+            {
+                "role": "assistant",
+                "content": f"Mock response for task: {task.id}",
+            },
+        ]
+
+        # 构建 outcome (模拟成功/失败)
+        if behavior == "success":
+            success = True
+        elif behavior == "failure":
+            success = False
+        else:
+            success = random.random() < self.success_rate
+
+        outcome: dict[str, Any] = {
+            "success": success,
+            "files": {
+                "output.py": f"# Mock output for {task.id}\ndef hello(): pass\n",
+            },
+            "artifacts": [
+                {
+                    "type": "code_file",
+                    "id": f"art_{uuid.uuid4().hex[:8]}",
+                    "content": f"# Generated code for {task.id}",
+                }
+            ] if success else [],
+        }
+
+        return trace_id, transcript, outcome
+
+
+class MockTraceProvider:
+    """模拟 TraceProvider
+
+    可选按 task id 关联 span 数据: trace_id 形如 "trace_{task_id}_{suffix}"
+    时返回 spans_by_task[task_id] (若已配置), 否则返回默认 spans。
+    """
+
+    def __init__(
+        self,
+        spans_by_task: dict[str, list[dict[str, Any]]] | None = None,
+        default_spans: list[dict[str, Any]] | None = None,
+    ):
+        self.spans_by_task = spans_by_task or {}
+        self.default_spans = default_spans or self._build_default_spans()
+
+    @staticmethod
+    def _build_default_spans() -> list[dict[str, Any]]:
+        return [
+            {
+                "name": "agent.turn",
+                "attributes": {
+                    "agenthub.total_tokens": 150,
+                },
+                "start_time": "2026-08-29T10:00:00Z",
+                "end_time": "2026-08-29T10:00:01Z",
+                "status": {"status_code": "OK"},
+            },
+            {
+                "name": "tool.call",
+                "attributes": {
+                    "agenthub.tool_name": "fs_write",
+                    "agenthub.success": True,
+                },
+                "start_time": "2026-08-29T10:00:01Z",
+                "end_time": "2026-08-29T10:00:02Z",
+                "status": {"status_code": "OK"},
+            },
+        ]
+
+    async def get_spans(self, trace_id: str) -> list[dict[str, Any]]:
+        """返回模拟 span 数据 (优先按 task id 匹配)"""
+        if trace_id.startswith("trace_"):
+            remainder = trace_id[len("trace_"):]
+            for task_id, spans in self.spans_by_task.items():
+                if remainder == task_id or remainder.startswith(f"{task_id}_"):
+                    return spans
+        return self.default_spans
+
+    async def get_trace_ids(
+        self,
+        filters: dict[str, Any] | None = None,
+        limit: int = 100,
+    ) -> list[str]:
+        return [f"trace_mock_{i}" for i in range(min(limit, 5))]

agent_eval/graders/__init__.py

@@ -0,0 +1,91 @@
+"""
+Built-in graders for the Aeval evaluation framework.
+
+Provides 9 built-in graders covering agent eval scenarios:
+- code_based: Deterministic checks (string/regex matching)
+- model_based: LLM-as-Judge
+- state_check: Environment state verification
+- tool_calls: Tool call validation
+- transcript: Transcript analysis (turns/tokens/redundancy)
+- artifact_check: Artifact verification
+- human: Human expert scoring (pending semantics, async score submission)
+- step_level: Step-level evaluation (expected_trace comparison)
+- metric: LLM 输出质量指标分发 (按 config.metric_name 路由到 metrics 注册表)
+
+Usage:
+    from agent_eval.graders import DEFAULT_GRADERS, get_grader_catalog
+
+    runner = EvalRunner(
+        agent_runner=my_runner,
+        graders=DEFAULT_GRADERS,  # Use all built-in graders
+    )
+
+    # API listing (name/type/description)
+    catalog = get_grader_catalog()
+"""
+
+from typing import Any
+
+from agent_eval.core.types import GraderType
+from agent_eval.graders.artifact_check import ArtifactCheckGrader
+from agent_eval.graders.code_based import CodeBasedGrader
+from agent_eval.graders.human import HumanGrader
+from agent_eval.graders.metric import MetricGrader
+from agent_eval.graders.model_based import ModelBasedGrader
+from agent_eval.graders.state_check import StateCheckGrader
+from agent_eval.graders.step_level import StepLevelGrader
+from agent_eval.graders.tool_calls import ToolCallsGrader
+from agent_eval.graders.transcript import TranscriptGrader
+
+# 注册表: name → {grader 实例, 类型, 描述} (供 API 列举与默认装配)
+GRADER_REGISTRY: dict[str, dict[str, Any]] = {}
+
+
+def _register(grader: Any, grader_type: GraderType, description: str) -> None:
+    GRADER_REGISTRY[grader.name] = {
+        "grader": grader,
+        "type": grader_type,
+        "description": description,
+    }
+
+
+_register(CodeBasedGrader(), GraderType.CODE, "确定性评分：字符串/正则/精确匹配检查")
+_register(ModelBasedGrader(), GraderType.MODEL, "LLM-as-Judge 评分")
+_register(StateCheckGrader(), GraderType.STATE, "环境状态检查")
+_register(ToolCallsGrader(), GraderType.TOOL_CALLS, "工具调用验证 (必须/禁止调用)")
+_register(TranscriptGrader(), GraderType.TRANSCRIPT, "转录记录分析 (轮次/Token 冗余)")
+_register(ArtifactCheckGrader(), GraderType.ARTIFACT, "产物检查 (类型/内容正则)")
+_register(HumanGrader(), GraderType.CUSTOM, "人工评分 (pending 语义, 异步回传)")
+_register(StepLevelGrader(), GraderType.CUSTOM, "步骤级评估 (expected_trace 对照)")
+_register(MetricGrader(), GraderType.METRIC, "LLM 输出质量指标 (按 metric_name 从注册表分发)")
+
+# 默认内置 grader 实例列表
+DEFAULT_GRADERS = [entry["grader"] for entry in GRADER_REGISTRY.values()]
+
+
+def get_grader_catalog() -> list[dict[str, str]]:
+    """列出可用 grader (name/type/description), 供 GET /graders 使用"""
+    return [
+        {
+            "name": name,
+            "type": entry["type"].value,
+            "description": entry["description"],
+        }
+        for name, entry in GRADER_REGISTRY.items()
+    ]
+
+
+__all__ = [
+    "DEFAULT_GRADERS",
+    "GRADER_REGISTRY",
+    "get_grader_catalog",
+    "CodeBasedGrader",
+    "ModelBasedGrader",
+    "StateCheckGrader",
+    "ToolCallsGrader",
+    "TranscriptGrader",
+    "ArtifactCheckGrader",
+    "HumanGrader",
+    "StepLevelGrader",
+    "MetricGrader",
+]

agent_eval/graders/artifact_check.py

@@ -0,0 +1,114 @@
+"""
+Artifact-check grader — artifact verification.
+
+Validates artifacts produced by the agent:
+- Artifact exists
+- Artifact type matches expected
+- Artifact content matches regex pattern
+
+Config schema:
+    {
+        "expected_type": "code_file",
+        "content_regex": "def \\w+\\(",
+        "threshold": 1.0
+    }
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+
+class ArtifactCheckGrader:
+    """产物检查评分器"""
+
+    name = "artifact_check"
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+        expected_type = config.get("expected_type")
+        content_regex = config.get("content_regex")
+        threshold = config.get("threshold", 1.0)
+
+        # 从 outcome 或 spans 提取产物
+        artifacts = self._extract_artifacts(trial, spans)
+
+        if not artifacts:
+            return GraderResult(
+                grader_name=self.name,
+                grader_type=GraderType.ARTIFACT,
+                score=0.0,
+                passed=False,
+                explanation="No artifacts produced",
+            )
+
+        # 检查类型
+        if expected_type:
+            types = [a.get("type", "") for a in artifacts]
+            if expected_type not in types:
+                return GraderResult(
+                    grader_name=self.name,
+                    grader_type=GraderType.ARTIFACT,
+                    score=0.0,
+                    passed=False,
+                    explanation=(
+                        f"Expected type '{expected_type}', "
+                        f"got {types}"
+                    ),
+                    details={"artifacts": artifacts},
+                )
+
+        # 检查内容
+        if content_regex:
+            contents = [a.get("content", "") for a in artifacts]
+            content_match = any(re.search(content_regex, c) for c in contents)
+            if not content_match:
+                return GraderResult(
+                    grader_name=self.name,
+                    grader_type=GraderType.ARTIFACT,
+                    score=0.3,
+                    passed=threshold <= 0.3,
+                    explanation=f"Content does not match pattern: {content_regex}",
+                    details={"artifacts": artifacts},
+                )
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.ARTIFACT,
+            score=1.0,
+            passed=True,
+            explanation=f"Artifact check passed: {len(artifacts)} artifact(s)",
+            details={"artifacts": artifacts},
+        )
+
+    def _extract_artifacts(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+    ) -> list[dict[str, Any]]:
+        """从 outcome 或 spans 中提取产物"""
+        # 优先从 outcome 获取
+        artifacts = trial.outcome.get("artifacts", [])
+        if artifacts:
+            return artifacts
+
+        # 从 spans 提取
+        return [
+            {
+                "type": span.get("attributes", {}).get("agenthub.artifact_type", ""),
+                "id": span.get("attributes", {}).get("agenthub.artifact_id", ""),
+                "content": span.get("attributes", {}).get("agenthub.content", ""),
+            }
+            for span in spans
+            if "artifact.create" in span.get("name", "")
+        ]

agent_eval/graders/code_based.py

@@ -0,0 +1,101 @@
+"""
+Code-based grader — deterministic scoring via string/regex matching.
+
+Supports:
+- contains: substring match
+- not_contains: substring absence
+- regex: regular expression match
+- exact: exact string equality
+
+Config schema:
+    {
+        "checks": [
+            {"type": "contains", "value": "def hello", "target": "transcript"},
+            {"type": "regex", "value": "class \\w+:", "target": "outcome"},
+        ],
+        "threshold": 1.0  # fraction of checks that must pass
+    }
+
+Target can be: "transcript" | "outcome" | "spans"
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+
+class CodeBasedGrader:
+    """通用确定性评分器"""
+
+    name = "code_based"
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+        checks = config.get("checks", [])
+        threshold = config.get("threshold", 1.0)
+
+        if not checks:
+            return GraderResult(
+                grader_name=self.name,
+                grader_type=GraderType.CODE,
+                score=1.0,
+                passed=True,
+                explanation="No checks configured, auto-pass",
+            )
+
+        passed_count = 0
+        details: list[dict[str, Any]] = []
+
+        for check in checks:
+            check_type = check.get("type", "contains")
+            target = check.get("target", "transcript")
+            value = check.get("value", "")
+
+            # 获取目标文本
+            if target == "transcript":
+                text = json.dumps(trial.transcript, ensure_ascii=False)
+            elif target == "outcome":
+                text = json.dumps(trial.outcome, ensure_ascii=False)
+            elif target == "spans":
+                text = json.dumps(spans, ensure_ascii=False)
+            else:
+                text = ""
+
+            # 执行检查
+            if check_type == "contains":
+                ok = value in text
+            elif check_type == "not_contains":
+                ok = value not in text
+            elif check_type == "regex":
+                ok = bool(re.search(value, text))
+            elif check_type == "exact":
+                ok = value == text
+            else:
+                ok = False
+
+            if ok:
+                passed_count += 1
+            details.append({"check": check, "passed": ok})
+
+        total = len(checks)
+        score = passed_count / total if total > 0 else 1.0
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.CODE,
+            score=score,
+            passed=score >= threshold,
+            explanation=f"{passed_count}/{total} checks passed",
+            details={"checks": details},
+        )

agent_eval/graders/human.py

@@ -0,0 +1,77 @@
+"""
+Human grader — routes scoring to human experts with pending semantics.
+
+Semantics (design decision D5): ``grade()`` returns IMMEDIATELY with a pending
+result (score=0, passed=False, ``details.status="pending"``, confidence=0) and
+persists a score request to Storage. The run completes normally; pending
+trials are listed separately in the summary and excluded from pass rates.
+Scores come back later via ``POST /api/eval/runs/{run_id}/human-scores``.
+
+Config schema:
+    {
+        "threshold": 0.7,       # optional, defaults to task.score_threshold
+        "instructions": "...",  # optional guidance shown to the reviewer
+    }
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+
+class HumanGrader:
+    """
+    人工评分器 — pending 语义, 不阻塞 run 完成。
+
+    可选注入 Storage (EvalRunner 构造时自动注入): 评分请求通过
+    ``save_human_score_request`` 落库, 供 Dashboard (change ②) 拉取。
+    """
+
+    name = "human"
+
+    def __init__(self, storage: Any | None = None):
+        self.storage = storage
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+
+        request: dict[str, Any] = {
+            "run_id": context.run_id if context else "",
+            "task_id": task.id,
+            "trial_index": trial.trial_index,
+            "grader_name": self.name,
+            "prompt": task.prompt,
+            "instructions": config.get("instructions", ""),
+            "transcript": trial.transcript,
+            "outcome": trial.outcome,
+            "created_at": time.time() * 1000,
+        }
+
+        # 评分请求写入 Storage (自定义 Storage 未实现该可选方法时跳过)
+        if self.storage is not None:
+            save = getattr(self.storage, "save_human_score_request", None)
+            if save is not None:
+                await save(request)
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.CUSTOM,
+            score=0.0,
+            passed=False,
+            explanation="等待人工评分",
+            details={
+                "status": "pending",
+                "request": request,
+            },
+            confidence=0.0,
+        )