aeval-framework 0.1.0__py3-none-any.whl

agent_eval/graders/metric.py

@@ -0,0 +1,142 @@
+"""
+Metric grader — the built-in dispatcher from grader configs to Metric instances.
+
+Task grader configs with `type: metric` route through this grader (D1):
+- `name: metric` + `config.metric_name` (single dispatcher per task), or
+- `name: <metric_name>` + `type: metric` (several metrics per task — the
+  runner falls back to this dispatcher for unregistered metric-type configs)
+
+The metric registry and LLM function are injected by EvalRunner
+(metrics_registry=..., llm_fn=...). Unregistered metric names score 0 with
+an explicit reason; missing LLM configuration surfaces a clear config error
+result instead of crashing the run.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import (
+    EvalTask,
+    GraderConfig,
+    GraderResult,
+    GraderType,
+    TrialResult,
+)
+from agent_eval.metrics.base import Metric, MetricError
+from agent_eval.metrics.llm_judge import LLMFn, LLMJudgeError, LLMNotConfiguredError
+
+# 计算失败 (配置/解析/调用) 映射为 0 分结果, 不 crash run (D2)
+_CALC_ERRORS = (LLMNotConfiguredError, LLMJudgeError, MetricError)
+
+
+class MetricGrader:
+    """按 config.metric_name 从注入注册表分发到对应 Metric 计算"""
+
+    name = "metric"
+
+    def __init__(
+        self,
+        metrics_registry: dict[str, Metric] | None = None,
+        llm_fn: LLMFn | None = None,
+    ):
+        # 由 EvalRunner 组合根覆盖注入 (与 storage 注入同一模式)
+        self.metrics_registry: dict[str, Metric] = dict(metrics_registry or {})
+        self.llm_fn = llm_fn
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = self._active_config(task, context)
+        metric_name = self._metric_name(config)
+
+        if not metric_name:
+            return self._result(config, 0.0, False, "metric grader 缺少 config.metric_name")
+
+        metric = self.metrics_registry.get(metric_name)
+        if metric is None:
+            return self._result(
+                config, 0.0, False, f"未知指标: {metric_name} (未在 metrics_registry 注册)"
+            )
+
+        kwargs = self._measure_kwargs(config, trial)
+        try:
+            result = await metric.measure(**kwargs)
+        except _CALC_ERRORS as e:
+            return self._result(config, 0.0, False, f"配置/计算错误: {e}")
+        except Exception as e:  # noqa: BLE001 — 指标实现方错误同样不 crash run
+            return self._result(config, 0.0, False, f"指标计算异常: {e}")
+
+        threshold = float(config.config.get("threshold", metric.threshold))
+        return self._result(
+            config,
+            result.score,
+            result.score >= threshold,
+            result.reason,
+            details={
+                **result.details,
+                "metric": result.name,
+                "metric_threshold": result.threshold,
+                "grader_threshold": threshold,
+            },
+        )
+
+    # ── Config resolution ────────────────────────────────────────────────
+
+    @staticmethod
+    def _active_config(task: EvalTask, context: EvalContext | None) -> GraderConfig:
+        """当前生效的 metric 配置: runner 传入优先, 否则取名为 metric 的配置"""
+        if context is not None and context.grader_config is not None:
+            return context.grader_config
+        for g in task.graders:
+            if g.type == GraderType.METRIC:
+                return g
+        raise ValueError(f"task '{task.id}' has no metric grader config")
+
+    @staticmethod
+    def _metric_name(config: GraderConfig) -> str:
+        """metric_name 显式配置优先; 命名分发型 (name=指标名) 取配置名"""
+        explicit = config.config.get("metric_name")
+        if explicit:
+            return str(explicit)
+        if config.name != MetricGrader.name:
+            return config.name
+        return ""
+
+    @staticmethod
+    def _measure_kwargs(config: GraderConfig, trial: TrialResult) -> dict[str, Any]:
+        """从 trial transcript 与 grader config 提取 measure() 入参"""
+        first = trial.transcript[0] if trial.transcript else {}
+        last = trial.transcript[-1] if trial.transcript else {}
+        prompt = first.get("content", "") if isinstance(first, dict) else ""
+        output = last.get("content", "") if isinstance(last, dict) else ""
+
+        return {
+            "input": prompt,
+            "actual_output": output,
+            "expected_output": config.config.get("expected_output"),
+            "context": config.config.get("context"),
+            "retrieval_context": config.config.get("retrieval_context"),
+        }
+
+    @staticmethod
+    def _result(
+        config: GraderConfig,
+        score: float,
+        passed: bool,
+        explanation: str,
+        details: dict[str, Any] | None = None,
+    ) -> GraderResult:
+        return GraderResult(
+            grader_name=config.name,
+            grader_type=GraderType.METRIC,
+            score=max(0.0, min(1.0, score)),
+            passed=passed,
+            explanation=explanation,
+            details=details or {},
+        )

agent_eval/graders/model_based.py

@@ -0,0 +1,179 @@
+"""
+Model-based grader — LLM-as-Judge scoring.
+
+Sends the trial transcript and a rubric to an LLM, which returns
+a score between 0 and 1.
+
+Config schema:
+    {
+        "rubric": "The response must contain...",
+        "dimensions": ["correctness", "completeness"],
+        "threshold": 0.7,
+        "model": "gpt-4o-mini",  # optional
+    }
+
+Requires either:
+- A configured llm_fn callback, or
+- An API key in the environment (OPENAI_API_KEY, etc.)
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+from collections.abc import Callable
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+# Type alias for LLM function: (system_prompt, user_message) -> str
+LLMFn = Callable[[str, str], str]
+
+
+class ModelBasedGrader:
+    """LLM-as-Judge 评分器"""
+
+    name = "model_based"
+
+    def __init__(self, llm_fn: LLMFn | None = None):
+        """
+        Args:
+            llm_fn: (system_prompt, user_message) → response text.
+                    If None, uses OpenAI API with OPENAI_API_KEY.
+        """
+        self._llm_fn = llm_fn
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+        rubric = config.get("rubric", "")
+        dimensions = config.get("dimensions", ["quality"])
+        threshold = config.get("threshold", 0.7)
+
+        # Build judge prompt
+        prompt = self._build_prompt(trial, rubric, dimensions)
+
+        # Call LLM
+        llm_fn = self._llm_fn or self._default_llm_fn(config)
+        try:
+            raw = llm_fn("You are an evaluation expert.", prompt)
+        except Exception as e:
+            return GraderResult(
+                grader_name=self.name,
+                grader_type=GraderType.MODEL,
+                score=0.0,
+                passed=False,
+                explanation=f"LLM call failed: {e}",
+            )
+
+        # Parse scores
+        scores = self._parse_scores(raw, dimensions)
+        avg_score = sum(scores.values()) / len(scores) if scores else 0.0
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.MODEL,
+            score=avg_score,
+            passed=avg_score >= threshold,
+            explanation=f"LLM Judge scores: {scores}",
+            details={"dimensions": scores, "raw_response": raw},
+        )
+
+    def _build_prompt(
+        self,
+        trial: TrialResult,
+        rubric: str,
+        dimensions: list[str],
+    ) -> str:
+        """构建 judge prompt"""
+        input_msg = trial.transcript[0] if trial.transcript else "N/A"
+        output_msg = trial.transcript[-1] if trial.transcript else "N/A"
+
+        # 提取工具调用摘要
+        tools_used = list(set(
+            msg.get("tool_name", "")
+            for msg in trial.transcript
+            if msg.get("role") == "tool_call"
+        ))
+
+        dims_json = ", ".join(f'"{d}": 0.0' for d in dimensions)
+
+        return f"""请根据以下评分标准对 Agent 表现进行评分。
+
+## 评分标准
+{rubric}
+
+## 评分维度
+{", ".join(dimensions)}
+
+## Agent 执行记录
+- 输入: {input_msg}
+- 输出: {output_msg}
+- 使用的工具: {tools_used or "无"}
+
+请以 JSON 格式返回各维度评分 (0.0-1.0):
+```json
+{{{dims_json}}}
+```"""
+
+    def _parse_scores(self, raw: str, dimensions: list[str]) -> dict[str, float]:
+        """从 LLM 响应中解析分数"""
+        scores: dict[str, float] = {}
+
+        # 尝试提取 JSON
+        try:
+            # 查找 JSON 块
+            json_start = raw.find("{")
+            json_end = raw.rfind("}") + 1
+            if json_start >= 0 and json_end > json_start:
+                json_str = raw[json_start:json_end]
+                parsed = json.loads(json_str)
+                for dim in dimensions:
+                    if dim in parsed:
+                        with contextlib.suppress(ValueError, TypeError):
+                            scores[dim] = float(parsed[dim])
+        except json.JSONDecodeError:
+            pass
+
+        # 如果解析失败，给所有维度默认分
+        if not scores:
+            for dim in dimensions:
+                scores[dim] = 0.5
+
+        return scores
+
+    def _default_llm_fn(self, config: dict[str, Any]) -> LLMFn:
+        """创建默认的 LLM 调用函数"""
+        model = config.get("model", "gpt-4o-mini")
+
+        def call_llm(system: str, user: str) -> str:
+            try:
+                from openai import OpenAI
+
+                client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+                resp = client.chat.completions.create(
+                    model=model,
+                    messages=[
+                        {"role": "system", "content": system},
+                        {"role": "user", "content": user},
+                    ],
+                    temperature=0.0,
+                    max_tokens=500,
+                )
+                return resp.choices[0].message.content or ""
+            except ImportError as e:
+                raise RuntimeError(
+                    "openai package not installed. "
+                    "Install with: pip install openai"
+                ) from e
+            except Exception as e:
+                raise RuntimeError(f"OpenAI API call failed: {e}") from e
+
+        return call_llm

agent_eval/graders/state_check.py

@@ -0,0 +1,106 @@
+"""
+State-check grader — environment state verification.
+
+Validates the environment state after a trial:
+- file_exists: file is present
+- file_contains: file content contains a substring
+- db_record: database record matches criteria
+- custom: custom check function
+
+Config schema:
+    {
+        "expectations": [
+            {"type": "file_exists", "path": "output.py"},
+            {"type": "file_contains", "path": "output.py", "value": "def main"},
+            {"type": "db_record", "table": "users", "match": {"id": 1}},
+        ],
+        "threshold": 1.0
+    }
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+
+class StateCheckGrader:
+    """环境状态检查评分器"""
+
+    name = "state_check"
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+        expectations = config.get("expectations", [])
+        threshold = config.get("threshold", 1.0)
+
+        if not expectations:
+            return GraderResult(
+                grader_name=self.name,
+                grader_type=GraderType.STATE,
+                score=1.0,
+                passed=True,
+                explanation="No expectations configured, auto-pass",
+            )
+
+        passed_count = 0
+        details: list[dict[str, Any]] = []
+
+        for exp in expectations:
+            exp_type = exp.get("type", "file_exists")
+            ok = False
+
+            if exp_type == "file_exists":
+                files = trial.outcome.get("files", {})
+                ok = exp["path"] in files
+
+            elif exp_type == "file_contains":
+                files = trial.outcome.get("files", {})
+                content = files.get(exp["path"], "")
+                ok = exp["value"] in content
+
+            elif exp_type == "file_regex":
+                files = trial.outcome.get("files", {})
+                content = files.get(exp["path"], "")
+                ok = bool(re.search(exp["value"], content))
+
+            elif exp_type == "db_record":
+                records = trial.outcome.get("db_records", [])
+                match_criteria = exp.get("match", {})
+                ok = any(
+                    all(r.get(k) == v for k, v in match_criteria.items())
+                    for r in records
+                )
+
+            elif exp_type == "no_conflict_markers":
+                files = trial.outcome.get("files", {})
+                content = files.get(exp["path"], "")
+                ok = "<<<<<<<" not in content and ">>>>>>>" not in content
+
+            else:
+                ok = False
+
+            if ok:
+                passed_count += 1
+            details.append({"expectation": exp, "passed": ok})
+
+        total = len(expectations)
+        score = passed_count / total if total > 0 else 1.0
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.STATE,
+            score=score,
+            passed=score >= threshold,
+            explanation=f"{passed_count}/{total} state checks passed",
+            details={"expectations": details},
+        )

agent_eval/graders/step_level.py

@@ -0,0 +1,116 @@
+"""
+Step-level grader — compares the agent's tool-call sequence against an
+expected trace (design decision D6, first version: exact index-by-index
+comparison only).
+
+From the trace spans, extracts the sequence of ``tool.call`` steps (tool name
+from span attributes, falling back to the span name), aligns it with the
+task's ``expected_trace`` config by index, reports the first wrong step and
+scores ``correct_steps / total_steps``.
+
+Config schema:
+    {
+        "expected_trace": ["fs_read", "fs_write", "bash"],  # required
+        "threshold": 0.7,  # optional, pass threshold
+    }
+
+When ``expected_trace`` is not configured the grader auto-passes (nothing to
+compare against), mirroring code_based's behavior with no checks.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+
+class StepLevelGrader:
+    """步骤级评估 — expected_trace 按索引对照, 定位首个错误步骤"""
+
+    name = "step_level"
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+        expected = config.get("expected_trace")
+        threshold = config.get("threshold", 0.7)
+
+        actual = self._extract_steps(spans)
+
+        if not expected:
+            return GraderResult(
+                grader_name=self.name,
+                grader_type=GraderType.CUSTOM,
+                score=1.0,
+                passed=True,
+                explanation="No expected_trace configured, auto-pass",
+                details={"actual_steps": actual},
+            )
+
+        total = len(expected)
+        step_details: list[dict[str, Any]] = []
+        first_error: int | None = None
+
+        for i in range(total):
+            expected_step = expected[i]
+            actual_step = actual[i] if i < len(actual) else None
+            ok = actual_step == expected_step
+            if not ok and first_error is None:
+                first_error = i
+            step_details.append({
+                "index": i,
+                "expected": expected_step,
+                "actual": actual_step,
+                "correct": ok,
+            })
+
+        correct_count = sum(1 for s in step_details if s["correct"])
+        score = correct_count / total if total > 0 else 1.0
+
+        explanation = f"{correct_count}/{total} steps correct"
+        if first_error is not None:
+            explanation += (
+                f"; first error at step {first_error}: "
+                f"expected '{expected[first_error]}', "
+                f"got '{actual[first_error] if first_error < len(actual) else None}'"
+            )
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.CUSTOM,
+            score=score,
+            passed=score >= threshold,
+            explanation=explanation,
+            details={
+                "steps": step_details,
+                "first_error_step": first_error,
+                "actual_steps": actual,
+                "extra_steps": (
+                    actual[total:] if len(actual) > total else []
+                ),
+            },
+        )
+
+    @staticmethod
+    def _extract_steps(spans: list[dict[str, Any]]) -> list[str]:
+        """从 spans 提取 tool.call 步骤序列 (工具名, 回退到 span 名称)"""
+        steps: list[str] = []
+        for span in spans:
+            name = span.get("name", "")
+            if "tool.call" not in name and "tool_call" not in name:
+                continue
+            attrs = span.get("attributes", {}) or {}
+            tool_name = (
+                attrs.get("agenthub.tool_name")
+                or attrs.get("tool_name")
+                or name
+            )
+            steps.append(str(tool_name))
+        return steps

agent_eval/graders/tool_calls.py

@@ -0,0 +1,102 @@
+"""
+Tool-calls grader — tool call validation.
+
+Validates that the agent:
+- Called required tools
+- Did not use forbidden tools
+- (Optional) Called tools in a specific order
+
+Config schema:
+    {
+        "required_tools": ["fs_read", "fs_write"],
+        "forbidden_tools": ["dangerous_tool"],
+        "threshold": 1.0
+    }
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+
+class ToolCallsGrader:
+    """工具调用验证评分器"""
+
+    name = "tool_calls"
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+        required_tools = config.get("required_tools", [])
+        forbidden_tools = config.get("forbidden_tools", [])
+        threshold = config.get("threshold", 1.0)
+
+        # 从 spans 提取工具调用
+        tool_calls = self._extract_tool_calls(spans)
+        used_tools = [tc["name"] for tc in tool_calls]
+
+        # 检查必须使用的工具
+        missing = [t for t in required_tools if t not in used_tools]
+        # 检查禁止使用的工具
+        violated = [t for t in forbidden_tools if t in used_tools]
+
+        # 计算分数
+        if required_tools:
+            found = len(required_tools) - len(missing)
+            score = found / len(required_tools)
+        else:
+            score = 1.0
+
+        # 违反禁止工具则直接 0 分
+        if violated:
+            score = 0.0
+
+        passed = score >= threshold
+
+        # 构建解释
+        parts = []
+        if used_tools:
+            parts.append(f"Used: {used_tools}")
+        if missing:
+            parts.append(f"Missing required: {missing}")
+        if violated:
+            parts.append(f"Violated forbidden: {violated}")
+        explanation = "; ".join(parts) if parts else "No tool calls checked"
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.TOOL_CALLS,
+            score=score,
+            passed=passed,
+            explanation=explanation,
+            details={
+                "tool_calls": tool_calls,
+                "used_tools": used_tools,
+                "missing": missing,
+                "violated": violated,
+            },
+        )
+
+    def _extract_tool_calls(self, spans: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """从 spans 中提取工具调用"""
+        tool_calls = []
+        for span in spans:
+            name = span.get("name", "")
+            attrs = span.get("attributes", {})
+
+            if "tool.call" in name or "tool_call" in name:
+                tool_calls.append({
+                    "name": attrs.get("agenthub.tool_name")
+                        or attrs.get("tool.name", ""),
+                    "success": attrs.get("agenthub.success", True),
+                })
+
+        return tool_calls