aeval-framework 0.1.0__py3-none-any.whl

agent_eval/storage/memory.py

@@ -0,0 +1,95 @@
+"""
+In-memory storage implementation.
+
+Useful for testing and short-lived runs.
+Data is lost when the process exits.
+
+Usage:
+    storage = MemoryStorage()
+    await storage.save_run(run_result)
+"""
+
+from __future__ import annotations
+
+import copy
+from typing import Any
+
+from agent_eval.core.types import EvalSuite, RunResult
+from agent_eval.dataset.storage import MemoryDatasetStorage
+
+
+class MemoryStorage:
+    """内存存储实现 — 用于测试"""
+
+    def __init__(self):
+        self._runs: dict[str, RunResult] = {}
+        self._suites: dict[str, EvalSuite] = {}
+        self._human_score_requests: list[dict[str, Any]] = []
+        # 数据集存储组合暴露 (与 runs/suites 分表, 见 dataset/storage.py)
+        self.datasets = MemoryDatasetStorage()
+
+    # ── Run 操作 ──
+
+    async def save_run(self, run: RunResult) -> None:
+        """保存运行结果"""
+        self._runs[run.run_id] = copy.deepcopy(run)
+
+    async def get_run(self, run_id: str) -> RunResult | None:
+        """获取运行结果"""
+        run = self._runs.get(run_id)
+        return copy.deepcopy(run) if run else None
+
+    async def list_runs(
+        self, suite_name: str | None = None, limit: int = 50
+    ) -> list[RunResult]:
+        """列出运行历史"""
+        runs = list(self._runs.values())
+        if suite_name:
+            runs = [r for r in runs if r.suite_name == suite_name]
+        runs.sort(key=lambda r: r.started_at, reverse=True)
+        return runs[:limit]
+
+    async def delete_run(self, run_id: str) -> bool:
+        """删除运行结果"""
+        if run_id in self._runs:
+            del self._runs[run_id]
+            return True
+        return False
+
+    # ── Suite 操作 ──
+
+    async def save_suite(self, suite: EvalSuite) -> None:
+        """保存评测套件"""
+        self._suites[suite.name] = copy.deepcopy(suite)
+
+    async def get_suite(self, name: str) -> EvalSuite | None:
+        """获取评测套件"""
+        suite = self._suites.get(name)
+        return copy.deepcopy(suite) if suite else None
+
+    async def list_suites(self) -> list[EvalSuite]:
+        """列出所有评测套件"""
+        return list(self._suites.values())
+
+    async def delete_suite(self, name: str) -> bool:
+        """删除评测套件"""
+        if name in self._suites:
+            del self._suites[name]
+            return True
+        return False
+
+    # ── 人工评分请求 ──
+
+    async def save_human_score_request(self, request: dict[str, Any]) -> None:
+        """保存人工评分请求 (HumanGrader pending 语义)"""
+        self._human_score_requests.append(dict(request))
+
+    async def list_human_score_requests(
+        self, run_id: str | None = None
+    ) -> list[dict[str, Any]]:
+        """列出人工评分请求"""
+        if run_id is None:
+            return [dict(r) for r in self._human_score_requests]
+        return [
+            dict(r) for r in self._human_score_requests if r.get("run_id") == run_id
+        ]

agent_eval/storage/sqlite.py

@@ -0,0 +1,240 @@
+"""
+SQLite storage implementation.
+
+Default storage backend for single-machine use.
+No external services required.
+
+Usage:
+    storage = SqliteStorage(db_path="./aeval.db")
+    await storage.initialize()
+    await storage.save_run(run_result)
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+
+import aiosqlite
+
+from agent_eval.core.types import EvalSuite, RunResult
+from agent_eval.dataset.storage import SqliteDatasetStorage
+
+
+class SqliteStorage:
+    """
+    SQLite 存储实现。
+
+    适合单机开发 / 轻量使用。
+    无需额外服务, 开箱即用。
+    """
+
+    def __init__(self, db_path: str = "./aeval.db"):
+        """
+        Args:
+            db_path: SQLite 数据库文件路径
+        """
+        self.db_path = db_path
+        self._initialized = False
+        # 数据集存储组合暴露 (同一 db 文件, datasets/dataset_items 分表)
+        self.datasets = SqliteDatasetStorage(db_path)
+
+    async def initialize(self) -> None:
+        """初始化数据库表结构"""
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.executescript("""
+                CREATE TABLE IF NOT EXISTS runs (
+                    run_id TEXT PRIMARY KEY,
+                    suite_name TEXT,
+                    status TEXT,
+                    started_at REAL,
+                    completed_at REAL,
+                    data TEXT
+                );
+
+                CREATE TABLE IF NOT EXISTS suites (
+                    name TEXT PRIMARY KEY,
+                    data TEXT,
+                    created_at REAL
+                );
+
+                CREATE TABLE IF NOT EXISTS human_score_requests (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    run_id TEXT,
+                    task_id TEXT,
+                    trial_index INTEGER,
+                    grader_name TEXT,
+                    data TEXT,
+                    created_at REAL
+                );
+
+                CREATE INDEX IF NOT EXISTS idx_runs_suite
+                    ON runs(suite_name);
+                CREATE INDEX IF NOT EXISTS idx_runs_status
+                    ON runs(status);
+                CREATE INDEX IF NOT EXISTS idx_runs_started
+                    ON runs(started_at);
+                CREATE INDEX IF NOT EXISTS idx_human_requests_run
+                    ON human_score_requests(run_id);
+            """)
+        await self.datasets.initialize()
+        self._initialized = True
+
+    def _ensure_initialized(self) -> None:
+        if not self._initialized:
+            raise RuntimeError(
+                "SqliteStorage not initialized. Call await storage.initialize() first."
+            )
+
+    # ── Run 操作 ──
+
+    async def save_run(self, run: RunResult) -> None:
+        """保存运行结果"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(
+                """INSERT OR REPLACE INTO runs
+                   (run_id, suite_name, status, started_at, completed_at, data)
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (
+                    run.run_id,
+                    run.suite_name,
+                    run.status,
+                    run.started_at,
+                    run.completed_at,
+                    json.dumps(run.model_dump(), default=str),
+                ),
+            )
+            await db.commit()
+
+    async def get_run(self, run_id: str) -> RunResult | None:
+        """获取运行结果"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            cursor = await db.execute(
+                "SELECT data FROM runs WHERE run_id = ?", (run_id,)
+            )
+            row = await cursor.fetchone()
+            if row:
+                data = json.loads(row["data"])
+                return RunResult(**data)
+            return None
+
+    async def list_runs(
+        self, suite_name: str | None = None, limit: int = 50
+    ) -> list[RunResult]:
+        """列出运行历史"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            if suite_name:
+                cursor = await db.execute(
+                    "SELECT data FROM runs WHERE suite_name = ? ORDER BY started_at DESC LIMIT ?",
+                    (suite_name, limit),
+                )
+            else:
+                cursor = await db.execute(
+                    "SELECT data FROM runs ORDER BY started_at DESC LIMIT ?",
+                    (limit,),
+                )
+            rows = await cursor.fetchall()
+            return [RunResult(**json.loads(r["data"])) for r in rows]
+
+    async def delete_run(self, run_id: str) -> bool:
+        """删除运行结果"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            cursor = await db.execute(
+                "DELETE FROM runs WHERE run_id = ?", (run_id,)
+            )
+            await db.commit()
+            return cursor.rowcount > 0
+
+    # ── Suite 操作 ──
+
+    async def save_suite(self, suite: EvalSuite) -> None:
+        """保存评测套件"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(
+                """INSERT OR REPLACE INTO suites (name, data, created_at)
+                   VALUES (?, ?, ?)""",
+                (suite.name, json.dumps(suite.model_dump(), default=str), time.time()),
+            )
+            await db.commit()
+
+    async def get_suite(self, name: str) -> EvalSuite | None:
+        """获取评测套件"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            cursor = await db.execute(
+                "SELECT data FROM suites WHERE name = ?", (name,)
+            )
+            row = await cursor.fetchone()
+            if row:
+                data = json.loads(row["data"])
+                return EvalSuite(**data)
+            return None
+
+    async def list_suites(self) -> list[EvalSuite]:
+        """列出所有评测套件"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            cursor = await db.execute("SELECT data FROM suites ORDER BY created_at DESC")
+            rows = await cursor.fetchall()
+            return [EvalSuite(**json.loads(r["data"])) for r in rows]
+
+    async def delete_suite(self, name: str) -> bool:
+        """删除评测套件"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            cursor = await db.execute(
+                "DELETE FROM suites WHERE name = ?", (name,)
+            )
+            await db.commit()
+            return cursor.rowcount > 0
+
+    # ── 人工评分请求 ──
+
+    async def save_human_score_request(self, request: dict[str, Any]) -> None:
+        """保存人工评分请求 (HumanGrader pending 语义)"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(
+                """INSERT INTO human_score_requests
+                   (run_id, task_id, trial_index, grader_name, data, created_at)
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (
+                    request.get("run_id", ""),
+                    request.get("task_id", ""),
+                    request.get("trial_index", 0),
+                    request.get("grader_name", "human"),
+                    json.dumps(request, default=str),
+                    time.time(),
+                ),
+            )
+            await db.commit()
+
+    async def list_human_score_requests(
+        self, run_id: str | None = None
+    ) -> list[dict[str, Any]]:
+        """列出人工评分请求"""
+        self._ensure_initialized()
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            if run_id is not None:
+                cursor = await db.execute(
+                    "SELECT data FROM human_score_requests WHERE run_id = ? "
+                    "ORDER BY id ASC",
+                    (run_id,),
+                )
+            else:
+                cursor = await db.execute(
+                    "SELECT data FROM human_score_requests ORDER BY id ASC"
+                )
+            rows = await cursor.fetchall()
+            return [json.loads(r["data"]) for r in rows]

agent_eval/trace/__init__.py

@@ -0,0 +1,16 @@
+"""
+Trace provider implementations for the Aeval evaluation framework.
+
+Provides a Phoenix-based trace provider out of the box.
+Other backends (Jaeger, Tempo, etc.) can be implemented by users.
+
+Usage:
+    from agent_eval.trace import PhoenixProvider
+
+    provider = PhoenixProvider(endpoint="http://localhost:6006")
+    spans = await provider.get_spans("trace_abc123")
+"""
+
+from agent_eval.trace.phoenix import PhoenixProvider
+
+__all__ = ["PhoenixProvider"]

agent_eval/trace/phoenix.py

@@ -0,0 +1,144 @@
+"""
+Phoenix trace provider implementation.
+
+Fetches trace spans from an Arize Phoenix instance.
+Requires: pip install arize-phoenix
+
+Usage:
+    provider = PhoenixProvider(endpoint="http://localhost:6006")
+    spans = await provider.get_spans("trace_abc123")
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class PhoenixProvider:
+    """
+    Phoenix TraceProvider 实现。
+
+    通过 Phoenix Python SDK 获取 trace span 数据。
+    """
+
+    def __init__(
+        self,
+        endpoint: str = "http://localhost:6006",
+        project: str = "default",
+    ):
+        """
+        Args:
+            endpoint: Phoenix UI endpoint (e.g., http://localhost:6006)
+            project: Phoenix project name
+        """
+        self.endpoint = endpoint
+        self.project = project
+        self._client = None
+
+    def _get_client(self):
+        """Lazy-init Phoenix client"""
+        if self._client is None:
+            try:
+                # phoenix >= 5 移除了顶层 px.Client, 统一走 phoenix.client;
+                # base_url 在旧版叫 endpoint, 新 API 只认 base_url。
+                from phoenix.client import Client as PhoenixClient
+
+                self._client = PhoenixClient(base_url=self.endpoint)
+            except ImportError as e:
+                raise RuntimeError(
+                    "phoenix package not installed. "
+                    "Install with: pip install arize-phoenix"
+                ) from e
+        return self._client
+
+    async def get_spans(self, trace_id: str) -> list[dict[str, Any]]:
+        """
+        获取一个 trace 的所有 span。
+
+        Args:
+            trace_id: OTel trace ID
+
+        Returns:
+            标准化的 span 列表
+        """
+        import asyncio
+
+        # Phoenix SDK 是同步的, 在线程中运行
+        return await asyncio.to_thread(self._get_spans_sync, trace_id)
+
+    def _get_spans_sync(self, trace_id: str) -> list[dict[str, Any]]:
+        """同步获取 spans"""
+        client = self._get_client()
+
+        try:
+            df = client.spans.get_spans_dataframe(project_name=self.project)
+        except Exception:
+            return []
+
+        if df is None or df.empty:
+            return []
+
+        # 过滤指定 trace
+        trace_col = "context.trace_id"
+        if trace_col not in df.columns:
+            return []
+
+        trace_spans = df[df[trace_col] == trace_id]
+        if trace_spans.empty:
+            return []
+
+        # 转换为标准格式
+        return self._normalize_spans(trace_spans.to_dict("records"))
+
+    async def get_trace_ids(
+        self,
+        filters: dict[str, Any] | None = None,
+        limit: int = 100,
+    ) -> list[str]:
+        """
+        查询 trace ID 列表。
+
+        Args:
+            filters: 过滤条件 (暂未实现, 保留接口)
+            limit: 返回数量限制
+
+        Returns:
+            trace ID 列表
+        """
+        import asyncio
+
+        return await asyncio.to_thread(self._get_trace_ids_sync, limit)
+
+    def _get_trace_ids_sync(self, limit: int) -> list[str]:
+        """同步获取 trace IDs"""
+        client = self._get_client()
+
+        try:
+            df = client.spans.get_spans_dataframe(project_name=self.project)
+        except Exception:
+            return []
+
+        if df is None or df.empty:
+            return []
+
+        trace_col = "context.trace_id"
+        if trace_col not in df.columns:
+            return []
+
+        trace_ids = df[trace_col].unique().tolist()
+        return trace_ids[:limit]
+
+    def _normalize_spans(self, raw_spans: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """将 Phoenix DataFrame 格式归一化为标准格式"""
+        normalized = []
+        for span in raw_spans:
+            normalized.append({
+                "name": span.get("name", ""),
+                "attributes": span.get("attributes", {}),
+                "start_time": str(span.get("start_time", "")),
+                "end_time": str(span.get("end_time", "")),
+                "status": span.get("status", {}),
+                "trace_id": span.get("context.trace_id", ""),
+                "span_id": span.get("context.span_id", ""),
+            })
+        return normalized