aeval-framework 0.1.0__py3-none-any.whl

agent_eval/dataset/sources/regression.py

@@ -0,0 +1,201 @@
+"""Regression sample extraction — close the eval→failure→extract→re-eval loop.
+
+Extracts failed trials from a Run into regression items (D5):
+- one item per failing TASK (first failed trial provides trace_id reference)
+- prompt from the trial transcript's first message, falling back to the
+  suite task's prompt; trials without any prompt source are skipped
+- graders/env copied from the suite task so extracted items are re-runnable
+- merging into a dataset dedupes on normalized prompt so regression samples
+  don't balloon with repeated runs
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from agent_eval.core.types import EvalSuite, RunResult, TrialResult
+from agent_eval.dataset.models import EvalDataset, EvalDatasetItem, SourceType, now_ms
+
+# 回归样本数上限默认值 (D5)
+DEFAULT_MAX_ITEMS = 50
+
+
+def normalize_prompt(prompt: str) -> str:
+    """prompt 归一化 (去首尾空白 + 折叠连续空白), 用于去重比较"""
+    return re.sub(r"\s+", " ", prompt or "").strip()
+
+
+@dataclass
+class RegressionReport:
+    """提取/合入报告"""
+
+    run_id: str = ""
+    failed_trials: int = 0
+    extracted: int = 0
+    skipped: list[dict[str, Any]] = field(default_factory=list)
+    merged: int = 0
+    merged_skipped: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "failed_trials": self.failed_trials,
+            "extracted": self.extracted,
+            "skipped": self.skipped,
+            "merged": self.merged,
+            "merged_skipped": self.merged_skipped,
+        }
+
+
+class RegressionExtractor:
+    """从 Run 失败 trial 中提取回归样本"""
+
+    def __init__(self, max_items: int = DEFAULT_MAX_ITEMS):
+        self.max_items = max_items
+
+    def extract_from_run(
+        self,
+        run: RunResult,
+        suite: EvalSuite | None = None,
+        max_items: int | None = None,
+    ) -> tuple[list[EvalDatasetItem], RegressionReport]:
+        """
+        从一次 Run 的失败 trial 中提取回归条目。
+
+        同一 task 的多个失败 trial 只提取一条 (prompt 相同,
+        取首个失败 trial 的 trace_id 为引用)。
+
+        Args:
+            run: 评测运行结果
+            suite: 可选的源 Suite — 提供任务 prompt 兜底与 graders/env 复用
+            max_items: 本次提取上限 (默认取实例配置)
+
+        Returns:
+            (items, report)
+        """
+        cap = max_items if max_items is not None else self.max_items
+        suite_tasks = {t.id: t for t in suite.tasks} if suite else {}
+
+        report = RegressionReport(run_id=run.run_id)
+        items: list[EvalDatasetItem] = []
+
+        for task_id, trials in run.trials.items():
+            failed = [t for t in trials if not t.success]
+            report.failed_trials += len(failed)
+            if not failed:
+                continue
+
+            # 同一 task 的多个失败 trial 只提取一条 (prompt 相同, 取首个失败 trial)
+            first_failure: TrialResult = failed[0]
+            prompt = self._prompt_of(first_failure, suite_tasks.get(task_id))
+            if not prompt:
+                report.skipped.append({
+                    "task_id": task_id,
+                    "trial_index": first_failure.trial_index,
+                    "reason": "no prompt available (empty transcript, no suite task)",
+                })
+                continue
+
+            if len(items) >= cap:
+                report.skipped.append({
+                    "task_id": task_id,
+                    "trial_index": first_failure.trial_index,
+                    "reason": f"max_items={cap} reached",
+                })
+                continue
+
+            task = suite_tasks.get(task_id)
+            error_note = (
+                f" (error: {first_failure.error[:120]})"
+                if first_failure.error
+                else ""
+            )
+            items.append(
+                EvalDatasetItem(
+                    id=f"regression_{task_id}_{first_failure.trial_index}",
+                    prompt=prompt,
+                    description=f"Regression: {task_id}{error_note}",
+                    graders=list(task.graders) if task else [],
+                    env=dict(task.env) if task else {},
+                    metadata={
+                        "capabilities": [],
+                        "regression": {
+                            "run_id": run.run_id,
+                            "task_id": task_id,
+                            "trial_index": first_failure.trial_index,
+                            "error": first_failure.error,
+                        },
+                    },
+                    source_type=SourceType.REGRESSION,
+                    source_ref=first_failure.trace_id or run.run_id,
+                    created_at=now_ms(),
+                )
+            )
+
+        report.extracted = len(items)
+        return items, report
+
+    @staticmethod
+    def _prompt_of(trial: TrialResult, suite_task: Any = None) -> str:
+        """trial prompt: transcript 首条消息优先, suite 任务 prompt 兜底"""
+        if trial.transcript:
+            first = trial.transcript[0]
+            if isinstance(first, dict):
+                content = first.get("content", "")
+                if isinstance(content, str) and content.strip():
+                    return content
+        if suite_task is not None:
+            return suite_task.prompt or ""
+        return ""
+
+    def merge_into_dataset(
+        self,
+        dataset: EvalDataset,
+        items: list[EvalDatasetItem],
+    ) -> tuple[EvalDataset, RegressionReport]:
+        """
+        将提取的条目合入数据集 — 按 prompt 归一化去重。
+
+        数据集或新增条目中 prompt 归一化后重复的条目跳过 (记录于报告),
+        避免回归样本随 run 次数膨胀。
+
+        Returns:
+            (更新后的 dataset 副本, 报告)
+        """
+        report = RegressionReport(run_id="")
+        existing = {normalize_prompt(i.prompt) for i in dataset.items if i.prompt}
+
+        merged_items = list(dataset.items)
+        for raw in items:
+            # 容忍 dict 形态 (API/JSON 透传场景)
+            item = raw if isinstance(raw, EvalDatasetItem) else EvalDatasetItem(**raw)
+            key = normalize_prompt(item.prompt)
+            if not key or key in existing:
+                report.merged_skipped.append({
+                    "item_id": item.id,
+                    "reason": "duplicate prompt (normalized) already in dataset",
+                })
+                continue
+            existing.add(key)
+            merged_items.append(item)
+            report.merged += 1
+
+        # 数据集内 ID 冲突时重命名 (不同 run 的同名回归条目)
+        taken_ids = set()
+        updated = []
+        for item in merged_items:
+            if item.id in taken_ids:
+                suffix = 1
+                while f"{item.id}_v{suffix}" in taken_ids:
+                    suffix += 1
+                item = item.model_copy(update={"id": f"{item.id}_v{suffix}"})
+            taken_ids.add(item.id)
+            updated.append(item)
+
+        new_dataset = dataset.model_copy(update={
+            "items": updated,
+            "updated_at": now_ms(),
+        })
+        return new_dataset, report

agent_eval/dataset/sources/trace_mining.py

@@ -0,0 +1,277 @@
+"""Trace Mining source — mine eval items from real Agent traces.
+
+Three first-version strategies (D4):
+- failed_tasks:    traces containing error spans
+- long_running:    traces whose duration exceeds P90 × multiplier
+- diverse_sampling: deterministic hash-based uniform sampling
+
+The prompt comes from the ROOT span's input attribute; traces without one are
+counted as `skipped` in the mining report (never guessed). Every mined item
+keeps trace provenance (source_type=trace_mining, source_ref=trace_id).
+`user_dissatisfied` stays as an enum placeholder until a user-feedback
+data channel exists (design §18, first-version scope note).
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+from agent_eval.core.contract import TraceProvider
+from agent_eval.dataset.models import EvalDatasetItem, SourceType, now_ms
+
+
+class MiningStrategy(str, Enum):
+    """挖掘策略 (首版 3 个; user_dissatisfied 依赖用户反馈通道, 留枚举位)"""
+
+    FAILED_TASKS = "failed_tasks"
+    LONG_RUNNING = "long_running"
+    DIVERSE_SAMPLING = "diverse_sampling"
+    USER_DISSATISFIED = "user_dissatisfied"  # 首版未实现
+
+
+# 根 span input 属性的候选键 (Phoenix/OpenInference 惯例优先)
+_INPUT_ATTR_KEYS = ("input.value", "input", "agent.input", "llm.input_messages")
+
+
+class TraceMiner:
+    """从 TraceProvider 提供的 trace 中按策略挖掘评测条目"""
+
+    def __init__(
+        self,
+        trace_provider: TraceProvider,
+        long_running_multiplier: float = 2.0,
+    ):
+        """
+        Args:
+            trace_provider: trace 数据源 (get_trace_ids + get_spans)
+            long_running_multiplier: long_running 策略的 P90 倍数阈值
+        """
+        self.trace_provider = trace_provider
+        self.long_running_multiplier = max(1.0, long_running_multiplier)
+
+    async def mine(
+        self,
+        strategy: str | MiningStrategy,
+        filters: dict[str, Any] | None = None,
+        limit: int = 20,
+        candidate_limit: int = 100,
+    ) -> MiningReport:
+        """
+        执行挖掘。
+
+        Args:
+            strategy: 挖掘策略名
+            filters: 传给 get_trace_ids 的过滤条件
+            limit: 最多产出的条目数
+            candidate_limit: 最多检查的候选 trace 数 (先过滤再拉 spans, 控制成本)
+
+        Returns:
+            MiningReport: items + skipped + 统计
+        """
+        strategy_enum = MiningStrategy(strategy)
+        if strategy_enum == MiningStrategy.USER_DISSATISFIED:
+            raise NotImplementedError(
+                "user_dissatisfied mining requires a user-feedback data channel "
+                "(not available in the first version — see design §18 scope)"
+            )
+
+        trace_ids = await self.trace_provider.get_trace_ids(
+            filters=filters, limit=candidate_limit
+        )
+
+        # 先拉 spans, 再按策略筛选 (策略内限制候选数, 避免全量展开)
+        candidates: list[tuple[str, list[dict[str, Any]]]] = []
+        for trace_id in trace_ids:
+            spans = await self.trace_provider.get_spans(trace_id)
+            if spans:
+                candidates.append((trace_id, spans))
+
+        selected = self._select_by_strategy(strategy_enum, candidates, limit)
+
+        items: list[EvalDatasetItem] = []
+        skipped: list[dict[str, Any]] = []
+        for trace_id, spans in selected:
+            root = self._root_span(spans)
+            prompt = self._extract_prompt(root) if root is not None else None
+            if not prompt:
+                skipped.append({
+                    "trace_id": trace_id,
+                    "reason": "no user input found in root span",
+                })
+                continue
+            items.append(self._trace_to_item(trace_id, spans, strategy_enum, prompt))
+
+        return MiningReport(
+            strategy=strategy_enum.value,
+            candidates=len(trace_ids),
+            inspected=len(candidates),
+            items=items,
+            skipped=skipped,
+        )
+
+    # ── 策略筛选 ──
+
+    def _select_by_strategy(
+        self,
+        strategy: MiningStrategy,
+        candidates: list[tuple[str, list[dict[str, Any]]]],
+        limit: int,
+    ) -> list[tuple[str, list[dict[str, Any]]]]:
+        if strategy == MiningStrategy.FAILED_TASKS:
+            selected = [(tid, spans) for tid, spans in candidates if self._is_failed(spans)]
+        elif strategy == MiningStrategy.LONG_RUNNING:
+            selected = self._select_long_running(candidates)
+        else:  # DIVERSE_SAMPLING
+            selected = self._select_diverse([tid for tid, _ in candidates], limit)
+            selected = [(tid, spans) for tid, spans in candidates if tid in selected]
+
+        return selected[:limit]
+
+    def _select_long_running(
+        self,
+        candidates: list[tuple[str, list[dict[str, Any]]]],
+    ) -> list[tuple[str, list[dict[str, Any]]]]:
+        timed = [
+            (tid, spans, self._trace_duration(spans))
+            for tid, spans in candidates
+        ]
+        timed = [(tid, spans, d) for tid, spans, d in timed if d is not None]
+        if not timed:
+            return []
+        ordered = sorted(d for _, _, d in timed)
+        p90 = ordered[max(0, int(len(ordered) * 0.9) - 1)] if len(ordered) > 1 else ordered[0]
+        threshold = p90 * self.long_running_multiplier
+        return [(tid, spans) for tid, spans, d in timed if d > threshold]
+
+    def _select_diverse(self, trace_ids: list[str], limit: int) -> list[str]:
+        """按 trace_id 哈希排序均匀采样 (确定性, 与到达顺序无关)"""
+        ranked = sorted(trace_ids, key=lambda tid: hashlib.md5(tid.encode()).hexdigest())
+        return ranked[:limit]
+
+    # ── Trace 分析 ──
+
+    @staticmethod
+    def _is_failed(spans: list[dict[str, Any]]) -> bool:
+        for span in spans:
+            status = span.get("status") or {}
+            status_code = (
+                status.get("status_code")
+                if isinstance(status, dict)
+                else str(status)
+            )
+            if str(status_code).upper() in ("ERROR", "STATUS_CODE_ERROR", "2"):
+                return True
+            attrs = span.get("attributes") or {}
+            if attrs.get("error") is True or attrs.get("error.type"):
+                return True
+        return False
+
+    @staticmethod
+    def _trace_duration(spans: list[dict[str, Any]]) -> float | None:
+        """trace 时长 (span start/end 解析失败返回 None)"""
+        starts: list[float] = []
+        ends: list[float] = []
+        for span in spans:
+            s = TraceMiner._parse_time(span.get("start_time"))
+            e = TraceMiner._parse_time(span.get("end_time"))
+            if s is not None:
+                starts.append(s)
+            if e is not None:
+                ends.append(e)
+        if not starts or not ends:
+            return None
+        return max(0.0, max(ends) - min(starts))
+
+    @staticmethod
+    def _parse_time(value: Any) -> float | None:
+        if value is None or value == "":
+            return None
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return None
+
+    @staticmethod
+    def _root_span(spans: list[dict[str, Any]]) -> dict[str, Any] | None:
+        """根 span = 最早开始的 span (归一化 span 不含 parent_id)"""
+        if not spans:
+            return None
+        return min(
+            spans,
+            key=lambda s: TraceMiner._parse_time(s.get("start_time")) or 0.0,
+        )
+
+    @staticmethod
+    def _extract_prompt(root: dict[str, Any] | None) -> str | None:
+        """从根 span input 属性提取用户输入; 缺失返回 None (不猜)"""
+        if root is None:
+            return None
+        attrs = root.get("attributes") or {}
+        for key in _INPUT_ATTR_KEYS:
+            value = attrs.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+            if isinstance(value, list) and value:
+                # llm.input_messages: [{"content": ...}] 形态兜底
+                first = value[0]
+                if isinstance(first, dict):
+                    content = first.get("content")
+                    if isinstance(content, str) and content.strip():
+                        return content.strip()
+        return None
+
+    def _trace_to_item(
+        self,
+        trace_id: str,
+        spans: list[dict[str, Any]],
+        strategy: MiningStrategy,
+        prompt: str,
+    ) -> EvalDatasetItem:
+        root = self._root_span(spans) or {}
+        failed = self._is_failed(spans)
+        description = (
+            f"Mined [{strategy.value}]: {root.get('name', 'unknown span')}"
+            + (" (failed trace)" if failed else "")
+        )
+        return EvalDatasetItem(
+            id=f"mined_{strategy.value}_{trace_id[:12]}",
+            prompt=prompt,
+            description=description,
+            graders=[],
+            metadata={
+                "capabilities": [],
+                "mining": {
+                    "strategy": strategy.value,
+                    "span_count": len(spans),
+                    "failed": failed,
+                },
+            },
+            source_type=SourceType.TRACE_MINING,
+            source_ref=trace_id,
+            created_at=now_ms(),
+        )
+
+
+@dataclass
+class MiningReport:
+    """挖掘结果报告 — 条目 + skipped 明细 + 统计"""
+
+    strategy: str
+    candidates: int = 0
+    inspected: int = 0
+    items: list[EvalDatasetItem] = field(default_factory=list)
+    skipped: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "strategy": self.strategy,
+            "candidates": self.candidates,
+            "inspected": self.inspected,
+            "mined": len(self.items),
+            "skipped_count": len(self.skipped),
+            "skipped": self.skipped,
+            "item_ids": [i.id for i in self.items],
+        }