aeval-framework 0.1.0__py3-none-any.whl

agent_eval/graders/transcript.py

@@ -0,0 +1,86 @@
+"""
+Transcript grader — transcript analysis.
+
+Analyzes the efficiency of an agent's execution:
+- Number of turns vs limit
+- Token usage vs limit
+- Tool call redundancy
+
+Config schema:
+    {
+        "max_turns": 20,
+        "max_tokens": 10000,
+        "threshold": 0.5
+    }
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agent_eval.core.contract import EvalContext
+from agent_eval.core.types import EvalTask, GraderResult, GraderType, TrialResult
+
+
+class TranscriptGrader:
+    """转录记录分析评分器"""
+
+    name = "transcript"
+
+    async def grade(
+        self,
+        trial: TrialResult,
+        spans: list[dict[str, Any]],
+        task: EvalTask,
+        context: EvalContext | None = None,
+    ) -> GraderResult:
+        config = task.get_grader_config(self.name)
+        max_turns = config.get("max_turns", 20)
+        max_tokens = config.get("max_tokens", 10000)
+        threshold = config.get("threshold", 0.5)
+
+        # 从 metrics 获取实际值
+        n_turns = trial.metrics.get("n_turns", 0)
+        n_tokens = trial.metrics.get("n_total_tokens", 0)
+
+        # 计算分数 (线性衰减)
+        turns_score = max(0.0, 1.0 - n_turns / max_turns) if max_turns > 0 else 1.0
+        tokens_score = max(0.0, 1.0 - n_tokens / max_tokens) if max_tokens > 0 else 1.0
+
+        # 计算工具调用冗余度
+        redundancy = self._calc_redundancy(spans)
+
+        # 综合分数
+        score = (turns_score + tokens_score + (1.0 - redundancy)) / 3.0
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.TRANSCRIPT,
+            score=score,
+            passed=score >= threshold,
+            explanation=(
+                f"turns={n_turns}/{max_turns}, "
+                f"tokens={n_tokens}/{max_tokens}, "
+                f"redundancy={redundancy:.1%}"
+            ),
+            details={
+                "turns_score": turns_score,
+                "tokens_score": tokens_score,
+                "redundancy": redundancy,
+            },
+        )
+
+    def _calc_redundancy(self, spans: list[dict[str, Any]]) -> float:
+        """计算工具调用冗余度"""
+        tool_calls = [
+            span.get("attributes", {}).get("agenthub.tool_name", "")
+            for span in spans
+            if "tool.call" in span.get("name", "")
+        ]
+
+        if not tool_calls:
+            return 0.0
+
+        unique_calls = set(tool_calls)
+        # 冗余度 = 1 - (唯一调用数 / 总调用数)
+        return 1.0 - len(unique_calls) / len(tool_calls)

agent_eval/metrics/__init__.py

@@ -0,0 +1,110 @@
+"""LLM output quality metrics for the Aeval evaluation framework.
+
+Modules:
+    base          — MetricResult / Metric ABC / BaseLLMMetric / to_grader()
+    llm_judge     — LLMFn protocol + tolerant JSON parsing + retry
+    answer_relevancy / faithfulness / context_recall / context_precision — P0 metrics
+    synthetic_data — Golden + SyntheticDataGenerator (documents → dataset items)
+    batch_evaluation — BatchEvaluator (对已有输出批量打分, P1)
+    prompt_metric  — PromptMetric (Prompt 变体 A/B, P1)
+    report         — 批量/run 结果渲染为 Markdown/JSON 报告 (P1)
+    pytest_plugin  — pytest 集成 (fixtures + suite 门禁; 只依赖 pytest, 不在
+                     此处导入以保持框架可无 pytest 运行 — 用例侧按需注册)
+
+LLM functions are injected as protocols (async (system, user) -> str);
+no LLM SDK is bound in the framework core.
+"""
+
+from agent_eval.metrics.answer_relevancy import AnswerRelevancyMetric
+from agent_eval.metrics.base import (
+    BaseLLMMetric,
+    Metric,
+    MetricError,
+    MetricGraderAdapter,
+    MetricResult,
+)
+from agent_eval.metrics.batch_evaluation import (
+    BatchCaseResult,
+    BatchEvaluationRequest,
+    BatchEvaluationResult,
+    BatchEvaluator,
+    BatchMetricSummary,
+    BatchTestCase,
+    MetricScore,
+    UnknownMetricsError,
+)
+from agent_eval.metrics.context_precision import ContextPrecisionMetric
+from agent_eval.metrics.context_recall import ContextRecallMetric
+from agent_eval.metrics.faithfulness import FaithfulnessMetric
+from agent_eval.metrics.llm_judge import (
+    DEFAULT_MAX_RETRIES,
+    LLMFn,
+    LLMJudgeError,
+    LLMNotConfiguredError,
+    extract_json_object,
+    judge_json,
+)
+from agent_eval.metrics.prompt_metric import (
+    PromptComparisonResult,
+    PromptMetric,
+    PromptTemplateError,
+    PromptTrialDetail,
+    PromptVariant,
+)
+from agent_eval.metrics.report import render_batch_report, render_run_report
+from agent_eval.metrics.synthetic_data import Golden, SyntheticDataGenerator
+
+
+def build_default_metrics_registry(
+    llm_fn: LLMFn | None = None,
+    threshold: float = 0.5,
+) -> dict[str, Metric]:
+    """
+    构建 P0 指标注册表 (name → Metric), 供 EvalRunner(metrics_registry=...) 注入。
+
+    llm_fn 可为 None (metric grader 届时返回明确配置错误而非崩溃)。
+    """
+    metrics: list[Metric] = [
+        AnswerRelevancyMetric(llm_fn=llm_fn, threshold=threshold),
+        FaithfulnessMetric(llm_fn=llm_fn, threshold=threshold),
+        ContextRecallMetric(llm_fn=llm_fn, threshold=threshold),
+        ContextPrecisionMetric(llm_fn=llm_fn, threshold=threshold),
+    ]
+    return {m.name: m for m in metrics}
+
+
+__all__ = [
+    "Metric",
+    "MetricResult",
+    "MetricError",
+    "BaseLLMMetric",
+    "MetricGraderAdapter",
+    "LLMFn",
+    "LLMJudgeError",
+    "LLMNotConfiguredError",
+    "DEFAULT_MAX_RETRIES",
+    "judge_json",
+    "extract_json_object",
+    "AnswerRelevancyMetric",
+    "FaithfulnessMetric",
+    "ContextRecallMetric",
+    "ContextPrecisionMetric",
+    "Golden",
+    "SyntheticDataGenerator",
+    "build_default_metrics_registry",
+    "BatchTestCase",
+    "BatchEvaluationRequest",
+    "BatchEvaluationResult",
+    "BatchCaseResult",
+    "BatchMetricSummary",
+    "MetricScore",
+    "BatchEvaluator",
+    "UnknownMetricsError",
+    "PromptVariant",
+    "PromptTrialDetail",
+    "PromptComparisonResult",
+    "PromptMetric",
+    "PromptTemplateError",
+    "render_batch_report",
+    "render_run_report",
+]

agent_eval/metrics/answer_relevancy.py

@@ -0,0 +1,57 @@
+"""P0 metric — Answer Relevancy (回答与问题的相关度)."""
+
+from __future__ import annotations
+
+from agent_eval.metrics.base import BaseLLMMetric, MetricError, MetricResult
+
+
+class AnswerRelevancyMetric(BaseLLMMetric):
+    """
+    回答相关度: Agent 回答是否切题, 是否解决了用户的问题。
+
+    评分逻辑: LLM 从回答中提取独立陈述并逐一判断与问题的相关度,
+    输出平均相关度分数。
+    """
+
+    name = "answer_relevancy"
+
+    _SYSTEM_PROMPT = """你是一个评测专家。请评估 Agent 的回答与用户问题的相关度。
+
+步骤:
+1. 从 Agent 回答中提取所有独立陈述
+2. 对每个陈述, 判断它与用户问题的相关度 (0-1)
+3. 计算平均相关度分数
+
+以 JSON 格式返回:
+{
+  "statements": ["陈述1", "陈述2"],
+  "relevancies": [0.9, 0.3],
+  "score": 0.85,
+  "reason": "一句话理由"
+}"""
+
+    async def measure(
+        self,
+        input: str,
+        actual_output: str,
+        expected_output: str | None = None,
+        context: list[str] | None = None,
+        retrieval_context: list[str] | None = None,
+    ) -> MetricResult:
+        user_prompt = f"用户问题: {input}\n\nAgent 回答: {actual_output}"
+        data = await self._llm_judge(self._SYSTEM_PROMPT, user_prompt)
+
+        score = self._score_of(data)
+        if "score" not in data:
+            raise MetricError(f"judge response missing 'score': {data!r}")
+
+        return MetricResult(
+            name=self.name,
+            score=score,
+            reason=str(data.get("reason", "")),
+            details={
+                "statements": data.get("statements", []),
+                "relevancies": data.get("relevancies", []),
+            },
+            threshold=self.threshold,
+        )

agent_eval/metrics/base.py

@@ -0,0 +1,155 @@
+"""Metric protocol — MetricResult / Metric ABC / BaseLLMMetric / to_grader bridge.
+
+A Metric measures LLM output quality (score 0-1 + reason). `to_grader()`
+adapts a Metric into the Grader protocol so it can be injected directly into
+EvalRunner; the registry-dispatch path (type: metric grader config) lives in
+graders/metric_check.py.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from agent_eval.metrics.llm_judge import (
+    LLMFn,
+    LLMJudgeError,
+    LLMNotConfiguredError,
+    judge_json,
+)
+
+
+@dataclass
+class MetricResult:
+    """单个指标的计算结果"""
+
+    name: str  # 指标名称
+    score: float  # 分数 (0-1)
+    reason: str = ""  # 评分理由 (LLM Judge 生成)
+    details: dict[str, Any] = field(default_factory=dict)  # 中间数据 (statements/verdicts...)
+    threshold: float = 0.5  # 通过阈值
+    success: bool = False  # 是否通过
+
+    def __post_init__(self):
+        self.success = self.score >= self.threshold
+
+
+class MetricError(Exception):
+    """指标计算失败 (LLM 输出结构不符合预期等)。"""
+
+
+class Metric(ABC):
+    """指标基类 — 所有 LLM 输出质量指标的抽象"""
+
+    name: str = "base_metric"
+    threshold: float = 0.5
+
+    @abstractmethod
+    async def measure(
+        self,
+        input: str,
+        actual_output: str,
+        expected_output: str | None = None,
+        context: list[str] | None = None,
+        retrieval_context: list[str] | None = None,
+    ) -> MetricResult:
+        """
+        核心测量方法。
+
+        Args:
+            input: 用户输入/问题
+            actual_output: Agent 实际输出
+            expected_output: 期望输出 (可选)
+            context: 回答所依据的上下文 (RAG)
+            retrieval_context: 检索到的原始文档 (RAG)
+
+        Raises:
+            LLMNotConfiguredError: 未注入 LLM 函数 (明确配置错误, 非 0 分)
+            MetricError / LLMJudgeError: 计算失败
+        """
+        ...
+
+    def to_grader(self) -> MetricGraderAdapter:
+        """将 Metric 转换为 Grader 适配器, 融入评分流水线 (grader 类型 metric)"""
+        return MetricGraderAdapter(self)
+
+
+class BaseLLMMetric(Metric):
+    """基于 LLM Judge 的指标基类"""
+
+    def __init__(self, llm_fn: LLMFn | None = None, threshold: float = 0.5):
+        self.llm_fn = llm_fn
+        self.threshold = threshold
+
+    async def _llm_judge(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+    ) -> dict[str, Any]:
+        """调用 LLM 并解析结构化 JSON (容错 + 重试在 llm_judge.judge_json)"""
+        return await judge_json(self.llm_fn, system_prompt, user_prompt)
+
+    @staticmethod
+    def _score_of(data: dict[str, Any], fallback: float = 0.0) -> float:
+        """从 judge 输出中取 score 并夹取到 [0, 1]"""
+        try:
+            score = float(data.get("score", fallback))
+        except (TypeError, ValueError):
+            return fallback
+        return max(0.0, min(1.0, score))
+
+
+class MetricGraderAdapter:
+    """
+    Metric → Grader 协议适配器 (metric.to_grader() 产物)。
+
+    从 trial 提取 input/actual_output (transcript 首末条), 调用
+    metric.measure 映射为 GraderResult。grader name = metric.name,
+    task 配置须使用同名 grader (config 可覆盖 threshold)。
+    """
+
+    def __init__(self, metric: Metric):
+        self.metric = metric
+        self.name = metric.name
+
+    async def grade(self, trial, spans, task, context=None):
+        from agent_eval.core.types import GraderResult, GraderType
+
+        config = task.get_grader_config(self.name)
+        threshold = config.get("threshold", self.metric.threshold)
+
+        first = trial.transcript[0] if trial.transcript else {}
+        last = trial.transcript[-1] if trial.transcript else {}
+        prompt = first.get("content", "") if isinstance(first, dict) else ""
+        output = last.get("content", "") if isinstance(last, dict) else ""
+
+        try:
+            result = await self.metric.measure(
+                input=prompt,
+                actual_output=output,
+                expected_output=config.get("expected_output"),
+                context=config.get("context"),
+                retrieval_context=config.get("retrieval_context"),
+            )
+        except (LLMNotConfiguredError, LLMJudgeError, MetricError) as e:
+            return GraderResult(
+                grader_name=self.name,
+                grader_type=GraderType.METRIC,
+                score=0.0,
+                passed=False,
+                explanation=f"配置/计算错误: {e}",
+            )
+
+        return GraderResult(
+            grader_name=self.name,
+            grader_type=GraderType.METRIC,
+            score=result.score,
+            passed=result.score >= threshold,
+            explanation=result.reason,
+            details={
+                **result.details,
+                "metric": result.name,
+                "metric_threshold": result.threshold,
+            },
+        )

agent_eval/metrics/batch_evaluation.py

@@ -0,0 +1,267 @@
+"""Batch evaluation over pre-existing outputs — BatchEvaluator + request/result models.
+
+对**已有输出** (历史对话 / 日志, 非运行 Agent 产生) 的用例集批量计算指标:
+输入 (input/actual_output, 可选 expected_output/context/retrieval_context) 与
+指标名列表, 输出逐条结果 + 汇总 (各指标 avg/min/max、pass/fail 计数、pass_rate)。
+
+- 指标名解析前置: 任一未注册抛 UnknownMetricsError (携带无效名列表), 未发任何 LLM 调用
+- 单条异常隔离: 单条用例内指标报错记入该条 result, 不中断整批
+- 受限并发: asyncio.Semaphore (默认 4)
+- LLM 函数沿用注入约定 (LLMFn); 未注入走 require_llm_fn 明确配置错误, 非静默 0 分
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from pydantic import BaseModel, Field
+
+from agent_eval.metrics.base import Metric
+from agent_eval.metrics.llm_judge import LLMFn, require_llm_fn
+
+DEFAULT_CONCURRENCY = 4
+
+
+class UnknownMetricsError(Exception):
+    """请求中的指标名未注册 (携带无效名列表; 解析前置, 零 LLM 调用)。"""
+
+    def __init__(self, unknown: list[str]):
+        self.unknown = list(unknown)
+        super().__init__(f"Unknown metrics: {', '.join(self.unknown)}")
+
+
+class BatchTestCase(BaseModel):
+    """单条待评用例 — 已有输出 (非运行 Agent 产生)"""
+
+    input: str
+    actual_output: str
+    expected_output: str | None = None
+    context: list[str] | None = None
+    retrieval_context: list[str] | None = None
+
+
+class BatchEvaluationRequest(BaseModel):
+    """批量评测请求 (与 REST POST /api/eval/metrics/batch 请求体同构)"""
+
+    test_cases: list[BatchTestCase] = Field(default_factory=list)
+    metrics: list[str] = Field(
+        ..., min_length=1, description="要计算的指标名 (须经注册表解析)"
+    )
+    thresholds: dict[str, float] = Field(
+        default_factory=dict,
+        description="逐指标覆盖 Metric 默认阈值 (显式给出的优先生效)",
+    )
+
+
+class MetricScore(BaseModel):
+    """单条用例上单指标的结果 (异常隔离: error 非空时 score=0)"""
+
+    name: str
+    score: float
+    reason: str = ""
+    threshold: float = 0.5  # 实际使用的阈值 (thresholds 覆盖后)
+    success: bool = False
+    error: str | None = None
+
+
+class BatchCaseResult(BaseModel):
+    """单条用例的逐指标结果"""
+
+    index: int
+    input: str
+    scores: dict[str, MetricScore] = Field(default_factory=dict)
+    overall_pass: bool = False
+
+
+class BatchMetricSummary(BaseModel):
+    """单指标汇总 (threshold 记录实际使用的阈值)"""
+
+    avg: float = 0.0
+    min: float = 0.0
+    max: float = 0.0
+    pass_count: int = 0
+    fail_count: int = 0
+    pass_rate: float = 0.0
+    threshold: float = 0.5
+
+
+class BatchEvaluationResult(BaseModel):
+    """批量评测结果"""
+
+    results: list[BatchCaseResult] = Field(default_factory=list)
+    summary: dict[str, BatchMetricSummary] = Field(default_factory=dict)
+    pass_count: int = 0
+    fail_count: int = 0
+    pass_rate: float = 0.0
+
+
+class BatchEvaluator:
+    """
+    批量评测器 — 对已有输出用例集计算指标 (受限并发 + 单条异常隔离)。
+
+    用法:
+        evaluator = BatchEvaluator(metrics_registry, llm_fn=llm_fn, concurrency=4)
+        result = await evaluator.evaluate(request)
+
+    llm_fn 注入约定与 EvalRunner 一致: 注入注册表中未自行配置 llm_fn 的指标。
+    """
+
+    def __init__(
+        self,
+        metrics_registry: dict[str, Metric] | None,
+        llm_fn: LLMFn | None = None,
+        concurrency: int = DEFAULT_CONCURRENCY,
+    ):
+        self.metrics_registry = dict(metrics_registry or {})
+        self.llm_fn = llm_fn
+        self.concurrency = max(1, concurrency)
+        if self.llm_fn is not None:
+            for metric in self.metrics_registry.values():
+                if getattr(metric, "llm_fn", None) is None:
+                    metric.llm_fn = self.llm_fn
+
+    async def evaluate(
+        self,
+        request: BatchEvaluationRequest,
+    ) -> BatchEvaluationResult:
+        """
+        执行批量评测。
+
+        指标名解析前置 (未注册 → UnknownMetricsError, 零 LLM 调用);
+        逐条逐指标计算 (Semaphore 受限并发, 单条异常记入该条 result)。
+        """
+        metrics = self._resolve_metrics(request.metrics)
+        self._ensure_llm_configured(metrics)
+
+        thresholds = {
+            m.name: request.thresholds.get(m.name, m.threshold) for m in metrics
+        }
+        semaphore = asyncio.Semaphore(self.concurrency)
+
+        async def _score(
+            case_index: int,
+            case: BatchTestCase,
+            metric: Metric,
+        ) -> tuple[int, str, MetricScore]:
+            async with semaphore:
+                score = await self._measure_isolated(
+                    metric, thresholds[metric.name], case
+                )
+            return case_index, metric.name, score
+
+        pairs = await asyncio.gather(
+            *(
+                _score(i, case, m)
+                for i, case in enumerate(request.test_cases)
+                for m in metrics
+            )
+        )
+
+        by_case: list[dict[str, MetricScore]] = [{} for _ in request.test_cases]
+        for case_index, metric_name, score in pairs:
+            by_case[case_index][metric_name] = score
+
+        results = [
+            BatchCaseResult(
+                index=i,
+                input=case.input,
+                scores=by_case[i],
+                overall_pass=all(s.success for s in by_case[i].values()),
+            )
+            for i, case in enumerate(request.test_cases)
+        ]
+
+        pass_count = sum(1 for r in results if r.overall_pass)
+        total = len(results)
+        return BatchEvaluationResult(
+            results=results,
+            summary=self._compute_summary(results, thresholds),
+            pass_count=pass_count,
+            fail_count=total - pass_count,
+            pass_rate=pass_count / total if total else 0.0,
+        )
+
+    # ── 内部 ──────────────────────────────────────────────────────────────
+
+    def _resolve_metrics(self, names: list[str]) -> list[Metric]:
+        """解析指标名 → 实例; 任一未注册即抛 UnknownMetricsError (前置)。"""
+        unknown = sorted({n for n in names if n not in self.metrics_registry})
+        if unknown:
+            raise UnknownMetricsError(unknown)
+        # 保序去重 (同名指标重复请求只算一次)
+        seen: dict[str, Metric] = {}
+        for name in names:
+            seen.setdefault(name, self.metrics_registry[name])
+        return list(seen.values())
+
+    def _ensure_llm_configured(self, metrics: list[Metric]) -> None:
+        """请求到的指标必须拿到 LLM 函数 (自身持有或注入), 否则明确配置错误。"""
+        missing = [m.name for m in metrics if getattr(m, "llm_fn", None) is None]
+        if not missing:
+            return
+        # 走 require_llm_fn 的明确配置错误语义 (而非静默 0 分)
+        llm_fn = require_llm_fn(self.llm_fn)
+        for name in missing:
+            self.metrics_registry[name].llm_fn = llm_fn
+
+    @staticmethod
+    async def _measure_isolated(
+        metric: Metric,
+        threshold: float,
+        case: BatchTestCase,
+    ) -> MetricScore:
+        """单条单指标计算; 异常记入该条结果 (score=0 + error), 不中断整批。"""
+        try:
+            result = await metric.measure(
+                input=case.input,
+                actual_output=case.actual_output,
+                expected_output=case.expected_output,
+                context=case.context,
+                retrieval_context=case.retrieval_context,
+            )
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            return MetricScore(
+                name=metric.name,
+                score=0.0,
+                reason=f"指标计算失败: {e}",
+                threshold=threshold,
+                success=False,
+                error=str(e),
+            )
+        # success 按实际使用阈值重算 (MetricResult.success 用的是指标自带阈值)
+        return MetricScore(
+            name=metric.name,
+            score=result.score,
+            reason=result.reason,
+            threshold=threshold,
+            success=result.score >= threshold,
+        )
+
+    @staticmethod
+    def _compute_summary(
+        results: list[BatchCaseResult],
+        thresholds: dict[str, float],
+    ) -> dict[str, BatchMetricSummary]:
+        """按指标聚合 avg/min/max + pass/fail 计数 (记录实际使用阈值)。"""
+        by_metric: dict[str, list[MetricScore]] = {}
+        for r in results:
+            for name, s in r.scores.items():
+                by_metric.setdefault(name, []).append(s)
+
+        summary: dict[str, BatchMetricSummary] = {}
+        for name, scores in by_metric.items():
+            values = [s.score for s in scores]
+            pass_count = sum(1 for s in scores if s.success)
+            total = len(scores)
+            summary[name] = BatchMetricSummary(
+                avg=sum(values) / total,
+                min=min(values),
+                max=max(values),
+                pass_count=pass_count,
+                fail_count=total - pass_count,
+                pass_rate=pass_count / total if total else 0.0,
+                threshold=thresholds.get(name, 0.5),
+            )
+        return summary