aeval-framework 0.1.0__tar.gz

aeval_framework-0.1.0/.gitignore

@@ -0,0 +1,23 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+*.db
+*.sqlite3
+
+# Node / Next.js (dashboard)
+node_modules/
+.next/
+out/
+*.tsbuildinfo
+
+# Env & OS
+.env
+.env.*
+.DS_Store
+Thumbs.db

aeval_framework-0.1.0/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.5
+Name: aeval-framework
+Version: 0.1.0
+Summary: Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)
+Project-URL: Homepage, https://github.com/QiYuyyds/Aeval
+Project-URL: Repository, https://github.com/QiYuyyds/Aeval
+Project-URL: Documentation, https://github.com/QiYuyyds/Aeval/tree/main/docs
+Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/releases
+Author: Aeval contributors
+License: MIT
+Keywords: agents,evaluation,llm,otel,testing,tracing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: pydantic>=2.10.0
+Requires-Dist: pyyaml>=6.0.0
+Provides-Extra: api
+Requires-Dist: fastapi>=0.115.0; extra == 'api'
+Requires-Dist: sse-starlette>=2.2.0; extra == 'api'
+Requires-Dist: uvicorn[standard]>=0.32.0; extra == 'api'
+Provides-Extra: cli
+Requires-Dist: rich>=13.0.0; extra == 'cli'
+Requires-Dist: typer>=0.12.0; extra == 'cli'
+Provides-Extra: dev
+Requires-Dist: httpx>=0.28.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
+Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.3.0; extra == 'dev'
+Requires-Dist: ruff>=0.8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# agent-eval
+
+Aeval — open-source agent evaluation framework driven by OTel traces.
+
+See the repository root [README](../../README.md) (English) / [中文说明](../../README.zh-CN.md) and the [docs](../../docs/) for full documentation.

aeval_framework-0.1.0/README.md

@@ -0,0 +1,5 @@
+# agent-eval
+
+Aeval — open-source agent evaluation framework driven by OTel traces.
+
+See the repository root [README](../../README.md) (English) / [中文说明](../../README.zh-CN.md) and the [docs](../../docs/) for full documentation.

aeval_framework-0.1.0/pyproject.toml

@@ -0,0 +1,80 @@
+[project]
+# Distribution name: aeval-framework (PyPI). Python module: agent_eval, CLI: eval-suite.
+# (`agent-eval` on PyPI belongs to an unrelated project — UK AISI agenteval.)
+name = "aeval-framework"
+version = "0.1.0"
+description = "Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "Aeval contributors" }]
+keywords = ["agents", "evaluation", "llm", "otel", "tracing", "testing"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Testing",
+]
+
+dependencies = [
+    "pydantic>=2.10.0",
+    "pyyaml>=6.0.0",
+    "aiosqlite>=0.20.0",
+]
+
+[project.optional-dependencies]
+# REST API / SSE service (create_app / create_standalone_app)
+api = [
+    "fastapi>=0.115.0",
+    "sse-starlette>=2.2.0",
+    "uvicorn[standard]>=0.32.0",
+]
+# eval-suite command line
+cli = [
+    "typer>=0.12.0",
+    "rich>=13.0.0",
+]
+dev = [
+    "pytest>=8.3.0",
+    "pytest-asyncio>=0.24.0",
+    "pytest-cov>=6.0.0",
+    "httpx>=0.28.0",
+    "ruff>=0.8.0",
+]
+
+[project.scripts]
+eval-suite = "agent_eval.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/QiYuyyds/Aeval"
+Repository = "https://github.com/QiYuyyds/Aeval"
+Documentation = "https://github.com/QiYuyyds/Aeval/tree/main/docs"
+Changelog = "https://github.com/QiYuyyds/Aeval/releases"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agent_eval"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+ignore = [
+    "E501",
+    "B008",
+    # (str, Enum) mixins are intentional: StrEnum would change str(member) /
+    # f-string serialization to the bare value and break API/storage payloads.
+    "UP042",
+]

aeval_framework-0.1.0/src/agent_eval/__init__.py

@@ -0,0 +1,14 @@
+"""
+Aeval — Agent Evaluation Framework
+
+A reusable, open-source evaluation framework for AI agents, driven by OTel traces.
+
+Usage:
+    from agent_eval import EvalRunner, EvalSuite
+
+    runner = EvalRunner(agent_runner=my_runner)
+    suite = EvalSuite.from_yaml("suite.yaml")
+    result = await runner.run_suite(suite)
+"""
+
+__version__ = "0.1.0"

aeval_framework-0.1.0/src/agent_eval/api/__init__.py

@@ -0,0 +1,14 @@
+"""
+REST API for the Aeval evaluation framework.
+
+Provides FastAPI routes for managing suites, runs, and viewing results.
+
+Usage:
+    from agent_eval.api import create_app
+
+    app = create_app(runner=my_eval_runner)
+"""
+
+from agent_eval.api.app import create_app
+
+__all__ = ["create_app"]

aeval_framework-0.1.0/src/agent_eval/api/app.py

@@ -0,0 +1,82 @@
+"""
+FastAPI application factory for Aeval.
+
+Creates a FastAPI app with all eval routes mounted.
+Can be used standalone or mounted in an existing app.
+
+Usage:
+    # Standalone
+    app = create_app(runner=my_runner)
+
+    # Mounted in existing FastAPI app
+    from fastapi import FastAPI
+    app = FastAPI()
+    eval_app = create_app(runner=my_runner)
+    app.mount("/api/eval", eval_app)
+"""
+
+from __future__ import annotations
+
+from fastapi import FastAPI
+
+from agent_eval.api.routes import datasets, graders, metrics, runs, suites, tasks
+from agent_eval.core.runner import EvalRunner
+
+# Global runner reference (set by create_app)
+_runner: EvalRunner | None = None
+
+
+def _get_runner() -> EvalRunner | None:
+    """Get the global EvalRunner instance"""
+    return _runner
+
+
+def set_runner(runner: EvalRunner | None) -> None:
+    """Set the global EvalRunner instance.
+
+    Used by the host app to inject the real runner during async startup
+    (e.g. main.py lifespan → eval_integration.config.create_aeval_runner).
+    """
+    global _runner
+    _runner = runner
+
+
+def create_app(runner: EvalRunner | None = None) -> FastAPI:
+    """
+    Create a FastAPI app with Aeval routes.
+
+    Args:
+        runner: EvalRunner instance. If None, routes will return 503.
+
+    Returns:
+        FastAPI application
+    """
+    app = FastAPI(
+        title="Aeval API",
+        version="0.1.0",
+        description="Agent Evaluation Framework API",
+    )
+
+    # Store runner in app state and global reference
+    global _runner
+    _runner = runner
+    app.state.runner = runner
+
+    # Include routers
+    app.include_router(suites.router, prefix="/suites", tags=["suites"])
+    app.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
+    app.include_router(runs.router, prefix="/runs", tags=["runs"])
+    # compare 挂在 /compare (spec §REST API), 不带 /runs 前缀
+    app.include_router(runs.compare_router, tags=["compare"])
+    app.include_router(graders.router, prefix="/graders", tags=["graders"])
+    # 数据集管理 (change ③: 数据集构建闭环 — CRUD/导入/挖掘/生成/质量/to-suite)
+    app.include_router(datasets.router, prefix="/datasets", tags=["datasets"])
+    # 批量评测 (change ④: 对已有输出直接批量打分 — POST /metrics/batch)
+    app.include_router(metrics.router, prefix="/metrics", tags=["metrics"])
+
+    @app.get("/health")
+    async def health():
+        # 动态读取全局 runner — host app 会在 startup 阶段注入真实 runner
+        return {"status": "ok", "runner_configured": _get_runner() is not None}
+
+    return app

aeval_framework-0.1.0/src/agent_eval/api/events.py

@@ -0,0 +1,96 @@
+"""Run 事件总线 — SSE 实时进度的进程内 fan-out (任务 3.1)。
+
+协议 (设计文档 D3 / §17.3):
+    - EvalRunner 进度回调 → per-run 订阅者队列扇出; 事件不落库, 断线恢复
+      完全依赖快照重拉 (GET /runs/{run_id}) + 重订阅
+    - 事件类型: task_start / trial_start / trial_complete / task_complete /
+      run_complete / error; 载荷含 task_id 与 trial_index (适用时), 按
+      (task_id, trial_index) 幂等 — 重复事件由客户端快照自愈兜底
+    - run 生命周期由服务端后台任务持有, 与观察连接解耦
+
+终态保留: run_complete 事件缓存于进程内 (有界, LIFO 淘汰), 供 run 结束后
+订阅的客户端立即收终态并关流; 进程重启后缓存消失, 客户端以快照为准。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import time
+from collections import OrderedDict
+from typing import Any
+
+# 有界队列: 慢订阅者丢最旧事件 (快照可自愈), 不阻塞 run 主流程
+_QUEUE_MAXSIZE = 256
+# 终态事件缓存上限 (防止长进程内存无界增长)
+_TERMINAL_CACHE_MAX = 500
+
+TERMINAL_EVENT_TYPES = {"run_complete"}
+
+
+class RunEventBus:
+    """run_id → 订阅者队列集合的事件扇出枢纽 (单事件循环, 无锁)。"""
+
+    def __init__(self) -> None:
+        self._subscribers: dict[str, set[asyncio.Queue]] = {}
+        self._terminal: OrderedDict[str, dict[str, Any]] = OrderedDict()
+
+    # ── 发布 ─────────────────────────────────────────────────────────────
+
+    def publish(self, run_id: str, event_type: str, data: dict[str, Any] | None = None) -> None:
+        """发布一条 run 事件 (非阻塞)。"""
+        event: dict[str, Any] = {
+            "type": event_type,
+            "run_id": run_id,
+            "timestamp": time.time() * 1000,
+            **(data or {}),
+        }
+        if event_type in TERMINAL_EVENT_TYPES:
+            self._terminal[run_id] = event
+            self._terminal.move_to_end(run_id)
+            while len(self._terminal) > _TERMINAL_CACHE_MAX:
+                self._terminal.popitem(last=False)
+
+        for queue in self._subscribers.get(run_id, set()):
+            _offer(queue, event)
+
+    # ── 订阅 ─────────────────────────────────────────────────────────────
+
+    def subscribe(self, run_id: str) -> asyncio.Queue:
+        """注册订阅者队列; run 已有缓存终态时立即注入终态事件。"""
+        queue: asyncio.Queue = asyncio.Queue(maxsize=_QUEUE_MAXSIZE)
+        self._subscribers.setdefault(run_id, set()).add(queue)
+        terminal = self._terminal.get(run_id)
+        if terminal is not None:
+            _offer(queue, terminal)
+        return queue
+
+    def unsubscribe(self, run_id: str, queue: asyncio.Queue) -> None:
+        """注销订阅者 (SSE 断开时调用, 防泄漏)。"""
+        subscribers = self._subscribers.get(run_id)
+        if subscribers is None:
+            return
+        subscribers.discard(queue)
+        if not subscribers:
+            self._subscribers.pop(run_id, None)
+
+    def subscriber_count(self, run_id: str) -> int:
+        return len(self._subscribers.get(run_id, set()))
+
+    def terminal_event(self, run_id: str) -> dict[str, Any] | None:
+        return self._terminal.get(run_id)
+
+
+def _offer(queue: asyncio.Queue, event: dict[str, Any]) -> None:
+    """put_nowait + 最旧丢弃溢出策略 (与 AChat event_bus 行为一致)。"""
+    try:
+        queue.put_nowait(event)
+    except asyncio.QueueFull:
+        with contextlib.suppress(asyncio.QueueEmpty):
+            queue.get_nowait()
+        with contextlib.suppress(asyncio.QueueFull):  # pragma: no cover - 理论不可达
+            queue.put_nowait(event)
+
+
+# 全局单例 (与 eval API 子应用同生命周期)
+run_event_bus = RunEventBus()