aeval-framework 0.1.0__py3-none-any.whl

agent_eval/core/suite.py

@@ -0,0 +1,60 @@
+"""
+Suite loading from YAML with strict validation.
+
+All validation rules (semver version, unique task ids, grader name format,
+weight >= 0, sample_count 1-10, name <= 128 chars, at least one task) live on
+the Pydantic models in core/types.py so both YAML loading and API JSON
+creation validate identically. This module wraps load/validation failures
+with file-path context.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+from pydantic import ValidationError
+
+from agent_eval.core.types import EvalSuite
+
+
+class SuiteLoadError(Exception):
+    """Suite YAML 加载或校验失败 (错误信息包含文件路径上下文)。"""
+
+
+def load_suite(path: str | Path) -> EvalSuite:
+    """
+    从 YAML 文件加载评测套件。
+
+    Args:
+        path: YAML 文件路径
+
+    Returns:
+        校验通过的 EvalSuite
+
+    Raises:
+        SuiteLoadError: 文件不存在 / YAML 语法错误 / 校验失败 (均含文件路径)
+    """
+    suite_path = Path(path)
+
+    if not suite_path.exists():
+        raise SuiteLoadError(f"Suite file not found: {suite_path}")
+
+    try:
+        with open(suite_path, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+    except OSError as e:
+        raise SuiteLoadError(f"Cannot read suite file '{suite_path}': {e}") from e
+    except yaml.YAMLError as e:
+        raise SuiteLoadError(f"Invalid YAML in suite file '{suite_path}': {e}") from e
+
+    if not isinstance(data, dict):
+        raise SuiteLoadError(
+            f"Suite file '{suite_path}' must contain a YAML mapping "
+            f"(got {type(data).__name__})"
+        )
+
+    try:
+        return EvalSuite(**data)
+    except ValidationError as e:
+        raise SuiteLoadError(f"Suite validation failed for '{suite_path}':\n{e}") from e

agent_eval/core/types.py

@@ -0,0 +1,227 @@
+"""
+Core data types for the Aeval evaluation framework.
+
+This module defines all the data models used throughout the framework:
+- Task definition layer: EvalTask, EvalSuite, GraderConfig
+- Run result layer: TrialResult, GraderResult, TaskSummary, RunSummary, RunResult
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+# ─── Task Definition Layer ───────────────────────────────────────────────────
+
+
+class GraderType(str, Enum):
+    """评分器类型枚举"""
+
+    CODE = "code"  # 确定性评分 (字符串匹配/正则/静态分析)
+    MODEL = "model"  # LLM Judge
+    STATE = "state"  # 环境状态检查
+    TOOL_CALLS = "tool_calls"  # 工具调用验证
+    TRANSCRIPT = "transcript"  # 转录记录分析
+    ARTIFACT = "artifact"  # 产物检查
+    METRIC = "metric"  # LLM 输出质量指标 (AnswerRelevancy/Faithfulness/...)
+    CUSTOM = "custom"  # 自定义
+
+
+class ScoreStrategy(str, Enum):
+    """评分聚合策略"""
+
+    ALL_PASS = "all_pass"  # 所有 grader 必须通过
+    WEIGHTED = "weighted"  # 加权平均
+    HYBRID = "hybrid"  # required 必须通过 + 非 required 加权
+
+
+class GraderConfig(BaseModel):
+    """单个评分器的配置"""
+
+    type: GraderType
+    name: str = Field(
+        ...,
+        min_length=1,
+        pattern=r"^[a-zA-Z][a-zA-Z0-9_-]*$",
+        description="评分器名称 (用于注册/查找)",
+    )
+    weight: float = Field(1.0, ge=0.0, description="权重 (用于加权评分)")
+    required: bool = Field(False, description="是否必须通过")
+    sample_count: int = Field(
+        1, ge=1, le=10, description="采样次数 (LLM Judge 多采样计算 confidence)"
+    )
+    dependencies: list[str] = Field(
+        default_factory=list,
+        description="依赖的其他 grader 名称 (拓扑排序, 依赖未通过则跳过)",
+    )
+    config: dict[str, Any] = Field(default_factory=dict, description="类型特定的配置")
+
+
+class EvalTask(BaseModel):
+    """单个评测任务"""
+
+    id: str = Field(..., min_length=1, description="唯一标识")
+    description: str = Field("", description="人类可读描述")
+    prompt: str = Field(..., description="给 Agent 的输入")
+    graders: list[GraderConfig] = Field(..., min_length=1, description="评分器列表")
+    env: dict[str, Any] = Field(default_factory=dict, description="环境参数 (透传给 AgentRunner)")
+    max_trials: int = Field(3, ge=1, description="默认 trial 数")
+    score_strategy: ScoreStrategy = Field(ScoreStrategy.HYBRID, description="评分聚合策略")
+    score_threshold: float = Field(0.7, ge=0.0, le=1.0, description="通过阈值 (用于 WEIGHTED/HYBRID)")
+    tracked_metrics: list[str] = Field(
+        default_factory=lambda: [
+            "n_turns",
+            "n_toolcalls",
+            "n_total_tokens",
+            "latency_ms",
+        ],
+        description="从 trace 提取的过程指标",
+    )
+
+    def get_grader_config(self, name: str) -> dict[str, Any]:
+        """获取指定名称的评分器配置"""
+        for g in self.graders:
+            if g.name == name:
+                return g.config
+        return {}
+
+
+class EvalSuite(BaseModel):
+    """评测套件 (一组任务)"""
+
+    name: str = Field(..., description="套件名称")
+    description: str = Field("", description="套件描述")
+    version: str = Field(
+        "1.0.0",
+        pattern=r"^\d+\.\d+\.\d+$",
+        description="语义化版本 (semver)",
+    )
+    tasks: list[EvalTask] = Field(..., min_length=1, description="任务列表")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="自定义元数据")
+
+    @field_validator("name")
+    @classmethod
+    def _validate_name(cls, v: str) -> str:
+        if not v or not v.strip():
+            raise ValueError("Suite name cannot be empty")
+        if len(v) > 128:
+            raise ValueError("Suite name too long (max 128 chars)")
+        return v
+
+    @model_validator(mode="after")
+    def _validate_task_ids_unique(self) -> EvalSuite:
+        ids = [t.id for t in self.tasks]
+        if len(ids) != len(set(ids)):
+            duplicates = sorted({x for x in ids if ids.count(x) > 1})
+            raise ValueError(f"Duplicate task IDs: {duplicates}")
+        return self
+
+    @classmethod
+    def from_yaml(cls, path: str) -> EvalSuite:
+        """从 YAML 文件加载评测套件 (含严格校验, 错误带文件路径上下文)"""
+        from agent_eval.core.suite import load_suite
+
+        return load_suite(path)
+
+
+# ─── Run Result Layer ─────────────────────────────────────────────────────────
+
+
+class GraderResult(BaseModel):
+    """单个评分器的评分结果"""
+
+    grader_name: str
+    grader_type: GraderType
+    score: float = Field(..., ge=0.0, le=1.0, description="评分 0.0-1.0")
+    passed: bool = Field(..., description="是否通过")
+    explanation: str = Field("", description="评分理由")
+    details: dict[str, Any] = Field(default_factory=dict, description="类型特定的详情")
+    confidence: float = Field(
+        1.0, ge=0.0, le=1.0, description="置信度 (多采样时 = 1 - 不确定性)"
+    )
+    uncertainty: float = Field(
+        0.0, ge=0.0, le=1.0, description="不确定性 (多采样极差的一半)"
+    )
+    sample_count: int = Field(1, ge=1, description="评分采样次数")
+    duration_ms: float = Field(0.0, description="评分耗时 (毫秒)")
+
+
+class TrialResult(BaseModel):
+    """单次 trial 的完整结果"""
+
+    trial_index: int = Field(..., ge=0, description="第几次 trial (0-based)")
+    trace_id: str = Field("", description="OTel trace ID")
+    success: bool = Field(True, description="最终是否成功")
+    grader_results: list[GraderResult] = Field(default_factory=list, description="各 grader 的评分")
+    metrics: dict[str, float] = Field(default_factory=dict, description="过程指标")
+    transcript: list[dict[str, Any]] = Field(default_factory=list, description="完整对话记录")
+    outcome: dict[str, Any] = Field(default_factory=dict, description="环境最终状态")
+    duration_ms: float = Field(0.0, description="总耗时 (毫秒)")
+    error: str | None = Field(None, description="错误信息 (如果失败)")
+
+    def avg_score(self) -> float:
+        """计算所有 grader 的平均分"""
+        if not self.grader_results:
+            return 0.0
+        return sum(r.score for r in self.grader_results) / len(self.grader_results)
+
+
+class TaskSummary(BaseModel):
+    """单个任务的汇总 (跨 trials)"""
+
+    task_id: str
+    task_description: str = ""
+    total_trials: int
+    pass_at_k: dict[int, float] = Field(default_factory=dict, description="{k: rate}")
+    pass_power_k: dict[int, float] = Field(default_factory=dict, description="{k: rate}")
+    avg_score: float = Field(0.0, description="所有 trial 的平均分")
+    avg_metrics: dict[str, float] = Field(default_factory=dict, description="平均过程指标")
+    failures: list[int] = Field(default_factory=list, description="失败的 trial 索引")
+    pending_trials: list[int] = Field(
+        default_factory=list, description="等待人工评分的 trial 索引 (不计入通过率)"
+    )
+    consistent: bool = Field(True, description="trial 间分数是否一致 (std < 0.2)")
+    score_std_dev: float = Field(0.0, description="trial 间分数标准差")
+
+
+class RunSummary(BaseModel):
+    """一次 suite 运行的汇总"""
+
+    total_tasks: int
+    total_trials: int
+    pass_at_k: dict[int, float] = Field(default_factory=dict, description="全局 pass@k")
+    pass_power_k: dict[int, float] = Field(default_factory=dict, description="全局 pass^k")
+    avg_score: float = Field(0.0, description="全局平均分")
+    avg_metrics: dict[str, float] = Field(default_factory=dict, description="全局平均指标")
+    task_summaries: list[TaskSummary] = Field(default_factory=list, description="每个任务的汇总")
+    failures: list[str] = Field(default_factory=list, description="未通过的任务 ID")
+    saturation: dict[str, Any] = Field(
+        default_factory=dict,
+        description="饱和度检测结果 (is_saturated/saturation_ratio/recommendation)",
+    )
+
+
+class RunResult(BaseModel):
+    """一次 suite 运行的完整结果"""
+
+    run_id: str = Field(default_factory=lambda: f"run_{uuid.uuid4().hex[:12]}")
+    suite_name: str = ""
+    status: Literal["pending", "running", "completed", "failed", "cancelled"] = "pending"
+    started_at: float = Field(default_factory=lambda: time.time() * 1000)
+    completed_at: float | None = None
+    trials: dict[str, list[TrialResult]] = Field(
+        default_factory=dict, description="task_id → trials"
+    )
+    summary: RunSummary | None = None
+    error: str | None = None
+
+    @property
+    def duration_ms(self) -> float | None:
+        """运行总耗时 (毫秒)"""
+        if self.completed_at is not None:
+            return self.completed_at - self.started_at
+        return None

agent_eval/dataset/__init__.py

@@ -0,0 +1,31 @@
+"""Dataset construction and management for the Aeval evaluation framework.
+
+Modules:
+    models  — EvalDataset / EvalDatasetItem with provenance + to_suite()
+    storage — DatasetStorage protocol + SQLite / Memory implementations
+    quality — DatasetQualityChecker + CoverageAnalyzer
+    version — DatasetVersionManager (semver bump + change log)
+    sources — manual import / trace mining / LLM generation / regression extract
+"""
+
+from agent_eval.dataset.models import (
+    DatasetError,
+    EvalDataset,
+    EvalDatasetItem,
+    SourceType,
+)
+from agent_eval.dataset.storage import (
+    DatasetStorage,
+    MemoryDatasetStorage,
+    SqliteDatasetStorage,
+)
+
+__all__ = [
+    "DatasetError",
+    "EvalDataset",
+    "EvalDatasetItem",
+    "SourceType",
+    "DatasetStorage",
+    "MemoryDatasetStorage",
+    "SqliteDatasetStorage",
+]

agent_eval/dataset/models.py

@@ -0,0 +1,199 @@
+"""
+Dataset data models — EvalDatasetItem / EvalDataset with provenance.
+
+A dataset is a curated pool of evaluation items. Each item carries provenance
+(source_type / source_ref) and capability metadata; `to_suite()` converts a
+dataset into an executable EvalSuite reusing the change-① Suite validation
+(single source of truth for validation rules).
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
+
+from agent_eval.core.types import EvalSuite, EvalTask, GraderConfig
+
+
+def now_ms() -> float:
+    """当前时间 (epoch 毫秒)"""
+    return time.time() * 1000
+
+
+def new_dataset_id() -> str:
+    return f"ds_{uuid.uuid4().hex[:12]}"
+
+
+class SourceType(str, Enum):
+    """数据集条目来源类型"""
+
+    MANUAL = "manual"  # 手动编写
+    TRACE_MINING = "trace_mining"  # 从真实 trace 挖掘
+    LLM_GENERATED = "llm_generated"  # LLM 辅助生成 (含合成数据)
+    ADVERSARIAL = "adversarial"  # 对抗样本 (手工构造, 手动的子类)
+    REGRESSION = "regression"  # 从 run 失败 trial 提取
+
+
+class DatasetError(Exception):
+    """数据集操作失败 (导入/转换/升版等)。"""
+
+
+class EvalDatasetItem(BaseModel):
+    """评测数据集中的单个条目 (含溯源)"""
+
+    id: str = Field(..., min_length=1, description="条目唯一标识 (数据集内)")
+    prompt: str = Field(..., description="给 Agent 的输入")
+    description: str = Field("", description="人类可读描述")
+    graders: list[GraderConfig] = Field(
+        default_factory=list,
+        description="评分器配置 (空 = 待补; to_suite 与质量检查会标出)",
+    )
+    env: dict[str, Any] = Field(default_factory=dict, description="环境参数 (透传)")
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="自定义元数据 (能力维度标签放 metadata.capabilities: list[str])",
+    )
+    source_type: SourceType = Field(SourceType.MANUAL, description="来源类型")
+    source_ref: str = Field("", description="来源引用 (trace_id / run_id / 场景等)")
+    created_at: float = Field(default_factory=now_ms, description="创建时间 (epoch ms)")
+
+
+class EvalDataset(BaseModel):
+    """评测数据集 — 一组相关的评测条目"""
+
+    id: str = Field(default_factory=new_dataset_id, description="唯一标识")
+    name: str = Field(..., min_length=1, description="数据集名称")
+    description: str = Field("", description="描述")
+    version: str = Field(
+        "1.0.0",
+        pattern=r"^\d+\.\d+\.\d+$",
+        description="语义化版本 (semver)",
+    )
+    tags: list[str] = Field(default_factory=list, description="标签 (分类/筛选)")
+    capability_map: dict[str, float] = Field(
+        default_factory=dict,
+        description="能力维度 → 覆盖度 (0-1), 由 CoverageAnalyzer 更新",
+    )
+    items: list[EvalDatasetItem] = Field(default_factory=list, description="条目列表")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="自定义元数据")
+    change_log: list[dict[str, Any]] = Field(
+        default_factory=list,
+        description="版本变更记录 [{version, change_type, note, at, item_count}]",
+    )
+    created_at: float = Field(default_factory=now_ms, description="创建时间 (epoch ms)")
+    updated_at: float = Field(default_factory=now_ms, description="更新时间 (epoch ms)")
+
+    @field_validator("name")
+    @classmethod
+    def _validate_name(cls, v: str) -> str:
+        if not v or not v.strip():
+            raise ValueError("Dataset name cannot be empty")
+        if len(v) > 128:
+            raise ValueError("Dataset name too long (max 128 chars)")
+        return v
+
+    @model_validator(mode="after")
+    def _validate_item_ids_unique(self) -> EvalDataset:
+        ids = [i.id for i in self.items]
+        if len(ids) != len(set(ids)):
+            duplicates = sorted({x for x in ids if ids.count(x) > 1})
+            raise ValueError(f"Duplicate item IDs: {duplicates}")
+        return self
+
+    def get_item(self, item_id: str) -> EvalDatasetItem | None:
+        """按 ID 获取条目"""
+        for item in self.items:
+            if item.id == item_id:
+                return item
+        return None
+
+    def to_suite(self, name: str | None = None) -> EvalSuite:
+        """
+        转换为可执行的评测 Suite。
+
+        条目 → 任务 (prompt/graders/env 直传); suite 元数据记录数据集
+        ID 与版本, 使 run 结果可关联回数据集版本。复用 EvalSuite 的
+        校验器 (任务 ID 唯一 / 至少一个任务 / grader 配置合法),
+        非法条目 (缺 prompt/graders、ID 重复等) 拒绝转换并给出明确错误。
+        """
+        try:
+            tasks = [
+                EvalTask(
+                    id=item.id,
+                    description=item.description,
+                    prompt=item.prompt,
+                    graders=item.graders,
+                    env=item.env,
+                )
+                for item in self.items
+            ]
+            return EvalSuite(
+                name=name or self.name,
+                description=self.description,
+                tasks=tasks,
+                metadata={
+                    **self.metadata,
+                    "dataset_id": self.id,
+                    "dataset_version": self.version,
+                },
+            )
+        except ValidationError as e:
+            raise DatasetError(
+                f"Dataset '{self.name}' (v{self.version}) → Suite conversion "
+                f"failed — fix the items below and retry:\n{e}"
+            ) from e
+
+
+# grader 类型 → 内置注册名默认值 (LLM 生成/导入缺 name 时保证可被 runner 解析)
+_TYPE_DEFAULT_NAMES: dict[str, str] = {
+    "code": "code_based",
+    "model": "model_based",
+    "state": "state_check",
+    "tool_calls": "tool_calls",
+    "transcript": "transcript",
+    "artifact": "artifact_check",
+    "human": "human",
+}
+
+
+def make_grader_config(
+    grader_type: str,
+    name: str | None = None,
+    **config: Any,
+) -> GraderConfig:
+    """
+    便捷构造 GraderConfig (数据源/生成器使用)。
+
+    Args:
+        grader_type: grader 类型字符串 (如 "model" / "metric" / "tool_calls")
+        name: grader 名称; 缺省时 metric 用 metric_name (无则 "metric"),
+              其余类型映射到内置注册名 (model → model_based 等)
+        **config: 类型特定配置 (如 metric_name/threshold/rubric)
+
+    Raises:
+        DatasetError: 类型不合法或名称不合法 (含具体字段信息)
+    """
+    from agent_eval.core.types import GraderType
+
+    try:
+        grader_t = GraderType(grader_type)
+    except ValueError:
+        valid = ", ".join(t.value for t in GraderType)
+        raise DatasetError(
+            f"Invalid grader type '{grader_type}' (valid: {valid})"
+        ) from None
+
+    if name is None:
+        if grader_t == GraderType.METRIC:
+            name = config.get("metric_name", "metric")
+        else:
+            name = _TYPE_DEFAULT_NAMES.get(grader_t.value, grader_t.value)
+
+    try:
+        return GraderConfig(type=grader_t, name=name, config=config)
+    except ValidationError as e:
+        raise DatasetError(f"Invalid grader config (name={name!r}):\n{e}") from e