aeval-framework 0.1.0__py3-none-any.whl

agent_eval/metrics/context_precision.py

@@ -0,0 +1,62 @@
+"""P0 metric — Context Precision (retrieval_context 中相关文档占比)."""
+
+from __future__ import annotations
+
+from agent_eval.metrics.base import BaseLLMMetric, MetricError, MetricResult
+
+
+class ContextPrecisionMetric(BaseLLMMetric):
+    """
+    上下文精确率: 检索到的文档中有多少真正与问题相关。
+
+    缺 retrieval_context 明确失败 (score=0 + 理由)。
+    """
+
+    name = "context_precision"
+
+    _SYSTEM_PROMPT = """你是一个信息检索评测专家。请评估检索上下文的精确率。
+
+步骤:
+1. 对检索上下文中的每个文档, 判断它是否对回答用户问题有用
+2. 计算精确率 = 有用文档数 / 总文档数
+
+以 JSON 格式返回:
+{
+  "documents": [{"index": 0, "relevant": true, "reason": "..."}],
+  "score": 0.67,
+  "reason": "一句话理由"
+}"""
+
+    async def measure(
+        self,
+        input: str,
+        actual_output: str,
+        expected_output: str | None = None,
+        context: list[str] | None = None,
+        retrieval_context: list[str] | None = None,
+    ) -> MetricResult:
+        docs = [d for d in (retrieval_context or []) if str(d).strip()]
+        if not docs:
+            return MetricResult(
+                name=self.name,
+                score=0.0,
+                reason="需要 retrieval_context 才能评估精确率",
+                details={"error": "missing_parameters"},
+                threshold=self.threshold,
+            )
+
+        doc_lines = "\n".join(f"[{i}] {doc}" for i, doc in enumerate(docs))
+        user_prompt = f"用户问题: {input}\n\n检索文档:\n{doc_lines}"
+        data = await self._llm_judge(self._SYSTEM_PROMPT, user_prompt)
+
+        if "score" not in data:
+            raise MetricError(f"judge response missing 'score': {data!r}")
+        score = self._score_of(data)
+
+        return MetricResult(
+            name=self.name,
+            score=score,
+            reason=str(data.get("reason", "")),
+            details={"documents": data.get("documents", [])},
+            threshold=self.threshold,
+        )

agent_eval/metrics/context_recall.py

@@ -0,0 +1,71 @@
+"""P0 metric — Context Recall (expected_output 信息点被 retrieval_context 覆盖率)."""
+
+from __future__ import annotations
+
+from agent_eval.metrics.base import BaseLLMMetric, MetricError, MetricResult
+
+
+class ContextRecallMetric(BaseLLMMetric):
+    """
+    上下文召回率: 检索到的文档是否包含回答所需的全部信息。
+
+    需要 expected_output (信息点来源) 与 retrieval_context (检索结果);
+    缺任一参数明确失败 (score=0 + 理由)。
+    """
+
+    name = "context_recall"
+
+    _SYSTEM_PROMPT = """你是一个信息检索评测专家。请评估检索上下文的召回率。
+
+步骤:
+1. 从期望回答中提取所有关键信息点 (facts/claims)
+2. 对每个信息点, 检查检索上下文中是否包含
+3. 计算召回率 = 被覆盖的信息点数 / 总信息点数
+
+以 JSON 格式返回:
+{
+  "information_points": ["信息点1", "信息点2"],
+  "covered": [true, false],
+  "score": 0.75,
+  "reason": "一句话理由"
+}"""
+
+    async def measure(
+        self,
+        input: str,
+        actual_output: str,
+        expected_output: str | None = None,
+        context: list[str] | None = None,
+        retrieval_context: list[str] | None = None,
+    ) -> MetricResult:
+        docs = [d for d in (retrieval_context or []) if str(d).strip()]
+        if not expected_output or not expected_output.strip() or not docs:
+            return MetricResult(
+                name=self.name,
+                score=0.0,
+                reason="需要 expected_output 和 retrieval_context 才能评估召回率",
+                details={"error": "missing_parameters"},
+                threshold=self.threshold,
+            )
+
+        retrieval_str = "\n---\n".join(str(d) for d in docs)
+        user_prompt = (
+            f"期望回答 (包含所有应覆盖的信息):\n{expected_output}\n\n"
+            f"检索上下文:\n{retrieval_str}"
+        )
+        data = await self._llm_judge(self._SYSTEM_PROMPT, user_prompt)
+
+        if "score" not in data:
+            raise MetricError(f"judge response missing 'score': {data!r}")
+        score = self._score_of(data)
+
+        return MetricResult(
+            name=self.name,
+            score=score,
+            reason=str(data.get("reason", "")),
+            details={
+                "information_points": data.get("information_points", []),
+                "covered": data.get("covered", []),
+            },
+            threshold=self.threshold,
+        )

agent_eval/metrics/faithfulness.py

@@ -0,0 +1,72 @@
+"""P0 metric — Faithfulness (回答忠于 context 的程度, 防幻觉)."""
+
+from __future__ import annotations
+
+from agent_eval.metrics.base import BaseLLMMetric, MetricError, MetricResult
+
+
+class FaithfulnessMetric(BaseLLMMetric):
+    """
+    忠实度: Agent 回答是否完全基于给定上下文, 无幻觉。
+
+    无 context 时明确失败 (score=0 + 明确理由), 不猜测 (spec 场景:
+    "对 faithfulness 计算但不提供 context → score=0 与'无上下文无法评估'")。
+    """
+
+    name = "faithfulness"
+
+    _SYSTEM_PROMPT = """你是一个事实核查专家。请评估 Agent 回答是否忠于给定上下文。
+
+步骤:
+1. 从 Agent 回答中提取所有事实性陈述
+2. 对每个陈述, 在上下文中查找支持证据
+3. 判断: supported / unsupported
+4. 计算忠实度 = 有支持的陈述数 / 总陈述数
+
+以 JSON 格式返回:
+{
+  "claims": ["陈述1", "陈述2"],
+  "verdicts": ["supported", "unsupported"],
+  "unsupported_claims": ["不被上下文支持的陈述"],
+  "score": 0.8,
+  "reason": "一句话理由 (标注不被支持的陈述)"
+}"""
+
+    async def measure(
+        self,
+        input: str,
+        actual_output: str,
+        expected_output: str | None = None,
+        context: list[str] | None = None,
+        retrieval_context: list[str] | None = None,
+    ) -> MetricResult:
+        context = [c for c in (context or []) if str(c).strip()]
+        if not context:
+            # 缺参明确失败路径: 返回 score=0 与明确理由 (不猜、不静默)
+            return MetricResult(
+                name=self.name,
+                score=0.0,
+                reason="无上下文，无法评估忠实度 — 需要提供 context 或 retrieval_context",
+                details={"error": "missing_context"},
+                threshold=self.threshold,
+            )
+
+        context_str = "\n---\n".join(str(c) for c in context)
+        user_prompt = f"上下文:\n{context_str}\n\nAgent 回答:\n{actual_output}"
+        data = await self._llm_judge(self._SYSTEM_PROMPT, user_prompt)
+
+        if "score" not in data:
+            raise MetricError(f"judge response missing 'score': {data!r}")
+        score = self._score_of(data)
+
+        return MetricResult(
+            name=self.name,
+            score=score,
+            reason=str(data.get("reason", "")),
+            details={
+                "claims": data.get("claims", []),
+                "verdicts": data.get("verdicts", []),
+                "unsupported_claims": data.get("unsupported_claims", []),
+            },
+            threshold=self.threshold,
+        )

agent_eval/metrics/llm_judge.py

@@ -0,0 +1,100 @@
+"""LLM Judge infrastructure — protocol-injected LLM function + tolerant JSON parsing.
+
+D2: the framework core only knows the LLMFn protocol
+(async (system_prompt, user_message) -> raw text); no LLM SDK is bound here.
+Assembly of a concrete implementation lives in eval_integration.config.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+# LLM 函数协议: (system_prompt, user_message) → raw text
+LLMFn = Callable[[str, str], Awaitable[str]]
+
+# 解析失败后的最大重试次数 (每次重试重新调用 LLM)
+DEFAULT_MAX_RETRIES = 2
+
+
+class LLMJudgeError(Exception):
+    """LLM Judge 调用/解析最终失败 (重试用尽)。"""
+
+
+class LLMNotConfiguredError(Exception):
+    """未注入 LLM 函数却调用了依赖 LLM 的指标 — 明确配置错误。"""
+
+
+def require_llm_fn(llm_fn: LLMFn | None) -> LLMFn:
+    """断言 LLM 函数已注入, 否则抛出明确配置错误 (而非静默 0 分)。"""
+    if llm_fn is None:
+        raise LLMNotConfiguredError(
+            "LLM function not configured — inject llm_fn (eval_integration "
+            "assembles one from AEVAL_JUDGE_* / eval LLM settings) or pass a "
+            "stub in tests."
+        )
+    return llm_fn
+
+
+def extract_json_object(raw: str) -> dict[str, Any] | None:
+    """
+    容错提取 LLM 输出中的 JSON 对象。
+
+    容忍 ```json 围栏、前后缀说明文本; 失败返回 None。
+    """
+    text = (raw or "").strip()
+    fence = re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL)
+    if fence:
+        text = fence.group(1).strip()
+
+    start = text.find("{")
+    end = text.rfind("}")
+    if start < 0 or end <= start:
+        return None
+    try:
+        data = json.loads(text[start : end + 1])
+    except json.JSONDecodeError:
+        return None
+    return data if isinstance(data, dict) else None
+
+
+async def judge_json(
+    llm_fn: LLMFn,
+    system_prompt: str,
+    user_prompt: str,
+    max_retries: int = DEFAULT_MAX_RETRIES,
+) -> dict[str, Any]:
+    """
+    调用 LLM 并解析 JSON 响应; 解析失败带完整上下文重试。
+
+    Raises:
+        LLMNotConfiguredError: llm_fn 为 None
+        LLMJudgeError: 重试用尽仍无法解析 (附最后一次原始输出)
+    """
+    require_llm_fn(llm_fn)
+
+    last_raw = ""
+    last_error: Exception | None = None
+    for attempt in range(max_retries + 1):
+        try:
+            last_raw = await llm_fn(system_prompt, user_prompt)
+        except Exception as e:
+            # 传输层错误同样重试 (judge 输出不稳定/网络抖动, design §Risks)
+            last_error = e
+            if attempt < max_retries:
+                await asyncio.sleep(0)
+                continue
+            raise LLMJudgeError(f"LLM call failed after {max_retries + 1} attempts: {e}") from e
+
+        parsed = extract_json_object(last_raw)
+        if parsed is not None:
+            return parsed
+        last_error = LLMJudgeError("response is not a parsable JSON object")
+
+    raise LLMJudgeError(
+        f"LLM judge failed after {max_retries + 1} attempts ({last_error}); "
+        f"last response: {last_raw[:200]!r}"
+    )

agent_eval/metrics/prompt_metric.py

@@ -0,0 +1,150 @@
+"""Prompt variant A/B — PromptMetric (纯 LLM 层, 不经 AgentRunner).
+
+对多个 Prompt 变体: 渲染模板 (str.format(**context)) → llm_fn 生成回答 →
+指标打分 → n_trials 取平均 → 声明胜者。不经过 AgentRunner — Agent 行为层的
+A/B 由 Dashboard run compare 覆盖, 本模块补 Prompt 层的快速对比通道。
+
+胜者判定 (v1 简化语义): 各指标平均分求和最大者; 结果对象保留逐 trial 明细,
+为 Phase 3 显著性检验预留。
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from agent_eval.metrics.base import Metric, MetricResult
+from agent_eval.metrics.llm_judge import LLMFn, require_llm_fn
+
+
+class PromptTemplateError(Exception):
+    """变体模板渲染失败 (str.format 缺 key / 格式错误) — 校验错误。"""
+
+
+@dataclass
+class PromptVariant:
+    """Prompt 变体 (template 经 str.format(**context) 渲染)"""
+
+    name: str
+    template: str
+
+
+@dataclass
+class PromptTrialDetail:
+    """单次试验明细 (Phase 3 显著性检验预留)"""
+
+    output: str
+    scores: dict[str, float] = field(default_factory=dict)  # metric name → score
+
+
+@dataclass
+class PromptComparisonResult:
+    """变体对比结果 (含逐 trial 明细与 winner 标注)"""
+
+    variant_name: str
+    metric_scores: dict[str, float] = field(default_factory=dict)  # n_trials 平均
+    trials: list[PromptTrialDetail] = field(default_factory=list)
+    winner: bool = False
+
+
+class PromptMetric:
+    """
+    Prompt A/B 对比 — 渲染 → 生成 → 打分 → n_trials 平均 → 声明胜者。
+
+    用法:
+        pm = PromptMetric(variants=[...], metrics=[...], llm_fn=llm_fn)
+        results = await pm.compare(context={"question": "..."}, n_trials=3)
+        winner = pm.declare_winner(results)
+
+    胜者判定 (v1): 指标平均分求和最大 (平分取声明序首个)。
+    变体模板作为 system prompt 传给 llm_fn (user 为空串, 与蓝图一致)。
+    """
+
+    def __init__(
+        self,
+        variants: list[PromptVariant],
+        metrics: list[Metric],
+        llm_fn: LLMFn | None = None,
+    ):
+        if not variants:
+            raise ValueError("PromptMetric requires at least one variant")
+        if not metrics:
+            raise ValueError("PromptMetric requires at least one metric")
+        self.variants = list(variants)
+        self.metrics = list(metrics)
+        self.llm_fn = llm_fn
+
+    async def compare(
+        self,
+        context: dict[str, Any],
+        n_trials: int = 3,
+    ) -> list[PromptComparisonResult]:
+        """
+        对比所有变体 (每个变体 n_trials 次试验取平均)。
+
+        Raises:
+            LLMNotConfiguredError: 未注入 llm_fn (明确配置错误)
+            PromptTemplateError: 模板渲染缺 key / 格式错误 (校验错误)
+            ValueError: n_trials < 1
+        """
+        llm_fn = require_llm_fn(self.llm_fn)
+        if n_trials < 1:
+            raise ValueError(f"n_trials must be >= 1, got {n_trials}")
+
+        results: list[PromptComparisonResult] = []
+        for variant in self.variants:
+            prompt = self._render(variant, context)
+            trials: list[PromptTrialDetail] = []
+            for _ in range(n_trials):
+                output = await llm_fn(prompt, "")
+                detail = PromptTrialDetail(output=output)
+                for metric in self.metrics:
+                    result: MetricResult = await metric.measure(
+                        input=prompt,
+                        actual_output=output,
+                    )
+                    detail.scores[metric.name] = result.score
+                trials.append(detail)
+
+            metric_scores = {
+                metric.name: sum(t.scores[metric.name] for t in trials) / len(trials)
+                for metric in self.metrics
+            }
+            results.append(PromptComparisonResult(
+                variant_name=variant.name,
+                metric_scores=metric_scores,
+                trials=trials,
+            ))
+
+        self.declare_winner(results)
+        return results
+
+    @staticmethod
+    def declare_winner(
+        results: list[PromptComparisonResult],
+    ) -> PromptComparisonResult:
+        """声明胜者 (v1 求和语义: 指标平均分求和最大; 平分取首个)。
+
+        原地标注: 恰有一个变体 winner=True (清除既有标注)。返回胜者。
+        """
+        if not results:
+            raise ValueError("no comparison results to declare a winner from")
+        winner = max(results, key=lambda r: sum(r.metric_scores.values()))
+        for r in results:
+            r.winner = r is winner
+        return winner
+
+    @staticmethod
+    def _render(variant: PromptVariant, context: dict[str, Any]) -> str:
+        """渲染变体模板; 缺 key / 格式错误转 PromptTemplateError (校验错误)。"""
+        try:
+            return variant.template.format(**context)
+        except KeyError as e:
+            raise PromptTemplateError(
+                f"variant '{variant.name}' template missing context key: {e} "
+                f"(template: {variant.template!r})"
+            ) from e
+        except (IndexError, ValueError) as e:
+            raise PromptTemplateError(
+                f"variant '{variant.name}' template render failed: {e}"
+            ) from e