aeval-framework 0.1.0__py3-none-any.whl

agent_eval/dataset/quality.py

@@ -0,0 +1,194 @@
+"""Dataset quality checks and capability coverage analysis."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from agent_eval.dataset.models import EvalDataset
+from agent_eval.dataset.sources.regression import normalize_prompt
+
+# 单条 prompt 长度告警阈值 (字符)
+PROMPT_LENGTH_WARN = 10_000
+# 每个能力维度视为充分覆盖所需的最少条目数
+COVERAGE_FULL_ITEMS = 5.0
+
+
+@dataclass
+class QualityIssue:
+    """单条质量问题"""
+
+    code: str  # empty_prompt / missing_graders / duplicate_prompt / long_prompt / duplicate_item_id
+    severity: str  # error / warning
+    item_id: str = ""
+    message: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "code": self.code,
+            "severity": self.severity,
+            "item_id": self.item_id,
+            "message": self.message,
+        }
+
+
+@dataclass
+class QualityReport:
+    """数据集质量报告"""
+
+    total_items: int = 0
+    errors: list[QualityIssue] = field(default_factory=list)
+    warnings: list[QualityIssue] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return not self.errors
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "total_items": self.total_items,
+            "ok": self.ok,
+            "error_count": len(self.errors),
+            "warning_count": len(self.warnings),
+            "errors": [e.to_dict() for e in self.errors],
+            "warnings": [w.to_dict() for w in self.warnings],
+        }
+
+
+class DatasetQualityChecker:
+    """数据集质量检查 — errors 阻塞转换, warnings 仅提示"""
+
+    def check(self, dataset: EvalDataset) -> QualityReport:
+        from agent_eval.dataset.models import EvalDatasetItem
+
+        items = [
+            i if isinstance(i, EvalDatasetItem) else EvalDatasetItem(**i)
+            for i in dataset.items
+        ]
+        report = QualityReport(total_items=len(items))
+
+        seen_prompts: dict[str, str] = {}
+        seen_ids: dict[str, str] = {}
+
+        for item in items:
+            # 1. 空 prompt (error)
+            if not item.prompt.strip():
+                report.errors.append(QualityIssue(
+                    code="empty_prompt", severity="error", item_id=item.id,
+                    message=f"Item '{item.id}' has an empty prompt",
+                ))
+
+            # 2. 缺 graders (error — to_suite 无法执行)
+            if not item.graders:
+                report.errors.append(QualityIssue(
+                    code="missing_graders", severity="error", item_id=item.id,
+                    message=f"Item '{item.id}' has no graders configured",
+                ))
+
+            # 3. 重复 prompt (warning, 归一化比较)
+            if item.prompt.strip():
+                normalized = normalize_prompt(item.prompt)
+                if normalized in seen_prompts:
+                    report.warnings.append(QualityIssue(
+                        code="duplicate_prompt", severity="warning", item_id=item.id,
+                        message=(
+                            f"Item '{item.id}' duplicates prompt of "
+                            f"'{seen_prompts[normalized]}'"
+                        ),
+                    ))
+                else:
+                    seen_prompts[normalized] = item.id
+
+            # 4. 超长 prompt (warning)
+            if len(item.prompt) > PROMPT_LENGTH_WARN:
+                report.warnings.append(QualityIssue(
+                    code="long_prompt", severity="warning", item_id=item.id,
+                    message=(
+                        f"Item '{item.id}' prompt is very long "
+                        f"({len(item.prompt)} chars, warn threshold {PROMPT_LENGTH_WARN})"
+                    ),
+                ))
+
+            # 5. 重复条目 ID (error — to_suite 要求任务 ID 唯一)
+            if item.id in seen_ids:
+                report.errors.append(QualityIssue(
+                    code="duplicate_item_id", severity="error", item_id=item.id,
+                    message=f"Item id '{item.id}' is used more than once",
+                ))
+            else:
+                seen_ids[item.id] = item.id
+
+        return report
+
+
+@dataclass
+class CoverageReport:
+    """能力维度覆盖度报告"""
+
+    total_items: int = 0
+    untagged_items: int = 0
+    coverage: dict[str, float] = field(default_factory=dict)  # 维度 → 0-1
+    insufficient: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "total_items": self.total_items,
+            "untagged_items": self.untagged_items,
+            "coverage": self.coverage,
+            "insufficient": self.insufficient,
+        }
+
+
+class CoverageAnalyzer:
+    """
+    分析数据集对能力维度的覆盖度。
+
+    条目的能力维度标签取 metadata.capabilities (list[str])。
+    覆盖度 = min(1, 该维度条目数 / COVERAGE_FULL_ITEMS) — 前几个条目
+    贡献最大, 达到 COVERAGE_FULL_ITEMS 个即视为充分覆盖。
+    """
+
+    def __init__(self, full_items_per_dim: float = COVERAGE_FULL_ITEMS,
+                 insufficient_below: float = 0.6):
+        """
+        Args:
+            full_items_per_dim: 视为充分覆盖的条目数
+            insufficient_below: 覆盖度低于该值列入覆盖不足清单
+        """
+        self.full_items_per_dim = max(1.0, full_items_per_dim)
+        self.insufficient_below = insufficient_below
+
+    def analyze(
+        self,
+        dataset: EvalDataset,
+        expected_capabilities: list[str] | None = None,
+    ) -> CoverageReport:
+        """
+        Args:
+            dataset: 数据集
+            expected_capabilities: 额外要求覆盖的维度 (即使条目未标注也计入报告)
+        """
+        report = CoverageReport(total_items=len(dataset.items))
+        dim_counts: dict[str, int] = {}
+
+        for item in dataset.items:
+            caps = item.metadata.get("capabilities", [])
+            if not isinstance(caps, list) or not caps:
+                report.untagged_items += 1
+                continue
+            for cap in caps:
+                dim_counts[str(cap)] = dim_counts.get(str(cap), 0) + 1
+
+        dims = set(dim_counts) | set(expected_capabilities or [])
+        for dim in sorted(dims):
+            count = dim_counts.get(dim, 0)
+            coverage = min(1.0, count / self.full_items_per_dim)
+            report.coverage[dim] = round(coverage, 3)
+            if coverage < self.insufficient_below:
+                report.insufficient.append({
+                    "capability": dim,
+                    "item_count": count,
+                    "coverage": report.coverage[dim],
+                })
+
+        return report

agent_eval/dataset/sources/__init__.py

@@ -0,0 +1,45 @@
+"""Dataset sources — manual import, trace mining, LLM generation, regression."""
+
+from agent_eval.dataset.sources.llm_generator import (
+    DatasetGenerationError,
+    GenerationReport,
+    LLMDatasetGenerator,
+    LLMFn,
+)
+from agent_eval.dataset.sources.manual import (
+    DatasetImportError,
+    import_from_content,
+    import_from_json,
+    import_from_yaml,
+    parse_dataset_payload,
+)
+from agent_eval.dataset.sources.regression import (
+    DEFAULT_MAX_ITEMS,
+    RegressionExtractor,
+    RegressionReport,
+    normalize_prompt,
+)
+from agent_eval.dataset.sources.trace_mining import (
+    MiningReport,
+    MiningStrategy,
+    TraceMiner,
+)
+
+__all__ = [
+    "DatasetImportError",
+    "import_from_content",
+    "import_from_json",
+    "import_from_yaml",
+    "parse_dataset_payload",
+    "DatasetGenerationError",
+    "GenerationReport",
+    "LLMDatasetGenerator",
+    "LLMFn",
+    "DEFAULT_MAX_ITEMS",
+    "RegressionExtractor",
+    "RegressionReport",
+    "normalize_prompt",
+    "MiningReport",
+    "MiningStrategy",
+    "TraceMiner",
+]

agent_eval/dataset/sources/llm_generator.py

@@ -0,0 +1,219 @@
+"""LLM-assisted dataset generation — scenario description → batch items.
+
+The LLM function is injected as a protocol (`async (system, user) -> str`);
+the framework core binds to no LLM SDK. Generated items go through the SAME
+validation as manual import: every item must carry a prompt and grader
+config; invalid ones are rejected with explicit reasons.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from agent_eval.core.types import GraderConfig
+from agent_eval.dataset.models import (
+    DatasetError,
+    EvalDatasetItem,
+    SourceType,
+    now_ms,
+)
+
+# LLM 函数协议: (system_prompt, user_message) → raw text (D2)
+LLMFn = Callable[[str, str], Awaitable[str]]
+
+_ALLOWED_GRADER_TYPES = ("code", "model", "metric", "tool_calls", "transcript", "artifact")
+
+_GEN_SYSTEM_PROMPT = "You are an evaluation dataset designer. You output only valid JSON."
+
+_GEN_PROMPT = """根据以下场景描述，生成 {count} 个评测任务。
+
+场景: {scenario}
+能力维度: {capabilities}
+
+每个任务是一个 JSON 对象，包含:
+- "id": 唯一标识 (kebab-case, 如 "summarize-quarterly-report")
+- "description": 一句话描述
+- "prompt": 给 Agent 的完整指令
+- "capabilities": 该任务考察的能力维度标签列表 (从上面能力维度中选)
+- "graders": 评分器配置列表，每项形如 {{"type": "<type>", "name": "<name>", "config": {{...}}}}
+  - type 只能是: {grader_types}
+  - name 必须用框架内置名: model→"model_based", code→"code_based", metric→"metric",
+    tool_calls→"tool_calls", transcript→"transcript", artifact→"artifact_check"
+  - metric 类型请在 config 中加 "metric_name" (如 faithfulness) 与 "threshold"
+
+以 JSON 数组格式返回，不要输出数组以外的任何内容。"""
+
+
+class DatasetGenerationError(DatasetError):
+    """LLM 生成失败 (LLM 未配置 / 输出不可解析 / 无合法条目)。"""
+
+
+@dataclass
+class GenerationReport:
+    """生成结果 — 合法条目 + 被拒绝条目的明细"""
+
+    scenario: str = ""
+    requested: int = 0
+    items: list[EvalDatasetItem] = field(default_factory=list)
+    invalid: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "scenario": self.scenario,
+            "requested": self.requested,
+            "generated": len(self.items),
+            "invalid_count": len(self.invalid),
+            "invalid": self.invalid,
+            "item_ids": [i.id for i in self.items],
+        }
+
+
+class LLMDatasetGenerator:
+    """LLM 辅助生成评测数据集条目"""
+
+    def __init__(self, llm_fn: LLMFn | None = None):
+        self.llm_fn = llm_fn
+
+    async def generate(
+        self,
+        scenario: str,
+        capabilities: list[str] | None = None,
+        count: int = 5,
+        llm_fn: LLMFn | None = None,
+    ) -> GenerationReport:
+        """
+        按场景批量生成评测条目。
+
+        Args:
+            scenario: 场景描述
+            capabilities: 能力维度标签 (写入条目 metadata.capabilities)
+            count: 请求生成的条目数
+            llm_fn: 覆盖实例级 llm_fn
+
+        Raises:
+            DatasetGenerationError: 未注入 llm_fn / 输出不可解析 / 无一条合法
+        """
+        fn = llm_fn or self.llm_fn
+        if fn is None:
+            raise DatasetGenerationError(
+                "LLM function not configured — pass llm_fn to "
+                "LLMDatasetGenerator() or generate(). Metric-style judge "
+                "assembly lives in eval_integration.config."
+            )
+
+        capabilities = capabilities or []
+        report = GenerationReport(scenario=scenario, requested=count)
+        caps = ", ".join(capabilities) if capabilities else "(无)"
+
+        raw = await fn(
+            _GEN_SYSTEM_PROMPT,
+            _GEN_PROMPT.format(
+                count=count,
+                scenario=scenario,
+                capabilities=caps,
+                grader_types=", ".join(_ALLOWED_GRADER_TYPES),
+            ),
+        )
+
+        parsed = _extract_json_array(raw)
+        if parsed is None:
+            raise DatasetGenerationError(
+                f"LLM response is not a parsable JSON array (got: {raw[:200]!r}...)"
+            )
+
+        for idx, entry in enumerate(parsed):
+            try:
+                report.items.append(self._entry_to_item(entry, idx, capabilities, scenario))
+            except DatasetError as e:
+                report.invalid.append({"index": idx, "error": str(e)})
+
+        if not report.items:
+            detail = "; ".join(i["error"] for i in report.invalid) or "empty response"
+            raise DatasetGenerationError(
+                f"LLM generation produced no valid items — {detail}"
+            )
+        return report
+
+    def _entry_to_item(
+        self,
+        entry: Any,
+        idx: int,
+        capabilities: list[str],
+        scenario: str,
+    ) -> EvalDatasetItem:
+        if not isinstance(entry, dict):
+            raise DatasetError(f"item #{idx}: must be a JSON object")
+
+        prompt = entry.get("prompt")
+        if not isinstance(prompt, str) or not prompt.strip():
+            raise DatasetError(f"item #{idx}: missing required field 'prompt'")
+        raw_graders = entry.get("graders")
+        if not raw_graders:
+            raise DatasetError(f"item #{idx}: missing required field 'graders'")
+
+        graders = [self._parse_grader(g, idx) for g in raw_graders]
+
+        item_id = entry.get("id")
+        if not isinstance(item_id, str) or not item_id.strip():
+            item_id = f"gen-{idx}"
+        item_caps = entry.get("capabilities")
+        if not isinstance(item_caps, list) or not item_caps:
+            item_caps = list(capabilities)
+
+        return EvalDatasetItem(
+            id=item_id.strip(),
+            prompt=prompt.strip(),
+            description=str(entry.get("description", "")),
+            graders=graders,
+            metadata={
+                "capabilities": [str(c) for c in item_caps],
+                "scenario": scenario[:128],
+            },
+            source_type=SourceType.LLM_GENERATED,
+            source_ref=scenario[:128],
+            created_at=now_ms(),
+        )
+
+    @staticmethod
+    def _parse_grader(raw: Any, idx: int) -> GraderConfig:
+        from agent_eval.dataset.models import make_grader_config
+
+        if isinstance(raw, str):
+            return make_grader_config(raw)
+
+        if isinstance(raw, dict):
+            grader_type = raw.get("type")
+            if not grader_type:
+                raise DatasetError(f"item #{idx}: grader missing 'type': {raw!r}")
+            config = raw.get("config") or {}
+            if not isinstance(config, dict):
+                raise DatasetError(f"item #{idx}: grader 'config' must be a mapping")
+            name = raw.get("name")
+            return make_grader_config(
+                str(grader_type), name=str(name) if name else None, **config
+            )
+
+        raise DatasetError(f"item #{idx}: grader must be a string or object, got {raw!r}")
+
+
+def _extract_json_array(raw: str) -> list[Any] | None:
+    """从容错提取 LLM 输出中的 JSON 数组 (容忍代码围栏/前后缀文本)"""
+    text = raw.strip()
+    # 去掉 ```json ... ``` 围栏
+    fence = re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL)
+    if fence:
+        text = fence.group(1).strip()
+
+    start = text.find("[")
+    end = text.rfind("]")
+    if start < 0 or end <= start:
+        return None
+    try:
+        data = json.loads(text[start : end + 1])
+    except json.JSONDecodeError:
+        return None
+    return data if isinstance(data, list) else None

agent_eval/dataset/sources/manual.py

@@ -0,0 +1,172 @@
+"""Manual dataset import — YAML / JSON files with strict validation.
+
+Validation failure messages follow the change-① Suite loader convention
+(pydantic ValidationError wrapped with the source file path). Items missing
+a prompt or grader config are rejected with the specific item and field.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import ValidationError
+
+from agent_eval.dataset.models import (
+    DatasetError,
+    EvalDataset,
+    SourceType,
+    now_ms,
+)
+
+
+class DatasetImportError(DatasetError):
+    """数据集导入失败 (解析/校验), 错误信息含来源文件上下文。"""
+
+
+def _validate_items_or_raise(items: list[dict[str, Any]], source: str) -> None:
+    """条目级必填校验: prompt 与 graders 必填, 缺失时报具体条目与字段。"""
+    problems: list[str] = []
+    for idx, item in enumerate(items):
+        if not isinstance(item, dict):
+            problems.append(f"items[{idx}]: must be a mapping (got {type(item).__name__})")
+            continue
+        item_label = f"items[{idx}]" + (f" (id={item.get('id')!r})" if item.get("id") else "")
+        prompt = item.get("prompt")
+        if not isinstance(prompt, str) or not prompt.strip():
+            problems.append(f"{item_label}: missing required field 'prompt'")
+        if not item.get("graders"):
+            problems.append(f"{item_label}: missing required field 'graders'")
+    if problems:
+        raise DatasetImportError(
+            f"Dataset import failed for '{source}' — {len(problems)} item problem(s):\n"
+            + "\n".join(f"- {p}" for p in problems)
+        )
+
+
+def _fill_item_defaults(items: list[dict[str, Any]], source_type: SourceType) -> list[dict[str, Any]]:
+    """补齐条目溯源默认值 (source_type / created_at), 保留显式指定值。"""
+    filled = []
+    for item in items:
+        item = dict(item)
+        item.setdefault("source_type", source_type)
+        item.setdefault("created_at", now_ms())
+        filled.append(item)
+    return filled
+
+
+def parse_dataset_payload(
+    data: dict[str, Any],
+    source: str = "<payload>",
+    source_type: SourceType = SourceType.MANUAL,
+    source_ref: str = "",
+) -> EvalDataset:
+    """
+    将 dict 解析为 EvalDataset (与文件导入同一校验路径, API 复用)。
+
+    Args:
+        data: 数据集定义 (name/items[...]/tags/...)
+        source: 报错时展示的来源描述 (文件路径或 API)
+        source_type: 条目默认来源类型
+        source_ref: 条目默认来源引用
+
+    Raises:
+        DatasetImportError: 顶层不是 mapping / 条目缺 prompt 或 graders / 模型校验失败
+    """
+    if not isinstance(data, dict):
+        raise DatasetImportError(
+            f"Dataset payload '{source}' must contain a mapping "
+            f"(got {type(data).__name__})"
+        )
+
+    raw_items = data.get("items", [])
+    if not isinstance(raw_items, list):
+        raise DatasetImportError(f"Dataset payload '{source}': 'items' must be a list")
+
+    _validate_items_or_raise(raw_items, source)
+
+    payload = dict(data)
+    payload["items"] = _fill_item_defaults(raw_items, source_type)
+    if source_ref:
+        for item in payload["items"]:
+            item.setdefault("source_ref", source_ref)
+
+    try:
+        return EvalDataset(**payload)
+    except ValidationError as e:
+        raise DatasetImportError(
+            f"Dataset validation failed for '{source}':\n{e}"
+        ) from e
+
+
+def import_from_yaml(path: str | Path, source_type: SourceType = SourceType.MANUAL) -> EvalDataset:
+    """从 YAML 文件导入数据集 (对抗样本等手工构造场景传 adversarial)。"""
+    dataset_path = Path(path)
+
+    if not dataset_path.exists():
+        raise DatasetImportError(f"Dataset file not found: {dataset_path}")
+
+    try:
+        with open(dataset_path, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+    except OSError as e:
+        raise DatasetImportError(f"Cannot read dataset file '{dataset_path}': {e}") from e
+    except yaml.YAMLError as e:
+        raise DatasetImportError(f"Invalid YAML in dataset file '{dataset_path}': {e}") from e
+
+    return parse_dataset_payload(data, source=str(dataset_path), source_type=source_type)
+
+
+def import_from_json(path: str | Path, source_type: SourceType = SourceType.MANUAL) -> EvalDataset:
+    """从 JSON 文件导入数据集。"""
+    dataset_path = Path(path)
+
+    if not dataset_path.exists():
+        raise DatasetImportError(f"Dataset file not found: {dataset_path}")
+
+    try:
+        with open(dataset_path, encoding="utf-8") as f:
+            data = json.load(f)
+    except OSError as e:
+        raise DatasetImportError(f"Cannot read dataset file '{dataset_path}': {e}") from e
+    except json.JSONDecodeError as e:
+        raise DatasetImportError(f"Invalid JSON in dataset file '{dataset_path}': {e}") from e
+
+    return parse_dataset_payload(data, source=str(dataset_path), source_type=source_type)
+
+
+def import_from_content(
+    content: str,
+    format: str = "yaml",
+    source: str = "<inline>",
+    source_type: SourceType = SourceType.MANUAL,
+    source_ref: str = "",
+) -> EvalDataset:
+    """
+    从文本内容导入 (API 用): format 为 "yaml" 或 "json"。
+
+    Raises:
+        DatasetImportError: 格式不支持 / 语法错误 / 校验失败
+    """
+    fmt = (format or "yaml").lower()
+    if fmt not in ("yaml", "json"):
+        raise DatasetImportError(
+            f"Unsupported import format '{format}' (valid: yaml, json)"
+        )
+
+    if fmt == "json":
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError as e:
+            raise DatasetImportError(f"Invalid JSON in dataset content '{source}': {e}") from e
+    else:
+        try:
+            data = yaml.safe_load(content)
+        except yaml.YAMLError as e:
+            raise DatasetImportError(f"Invalid YAML in dataset content '{source}': {e}") from e
+
+    return parse_dataset_payload(
+        data, source=source, source_type=source_type, source_ref=source_ref
+    )