aeval-framework 0.1.0__py3-none-any.whl

aeval_framework-0.1.0.dist-info/METADATA

@@ -0,0 +1,42 @@
+Metadata-Version: 2.5
+Name: aeval-framework
+Version: 0.1.0
+Summary: Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)
+Project-URL: Homepage, https://github.com/QiYuyyds/Aeval
+Project-URL: Repository, https://github.com/QiYuyyds/Aeval
+Project-URL: Documentation, https://github.com/QiYuyyds/Aeval/tree/main/docs
+Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/releases
+Author: Aeval contributors
+License: MIT
+Keywords: agents,evaluation,llm,otel,testing,tracing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: pydantic>=2.10.0
+Requires-Dist: pyyaml>=6.0.0
+Provides-Extra: api
+Requires-Dist: fastapi>=0.115.0; extra == 'api'
+Requires-Dist: sse-starlette>=2.2.0; extra == 'api'
+Requires-Dist: uvicorn[standard]>=0.32.0; extra == 'api'
+Provides-Extra: cli
+Requires-Dist: rich>=13.0.0; extra == 'cli'
+Requires-Dist: typer>=0.12.0; extra == 'cli'
+Provides-Extra: dev
+Requires-Dist: httpx>=0.28.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
+Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.3.0; extra == 'dev'
+Requires-Dist: ruff>=0.8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# agent-eval
+
+Aeval — open-source agent evaluation framework driven by OTel traces.
+
+See the repository root [README](../../README.md) (English) / [中文说明](../../README.zh-CN.md) and the [docs](../../docs/) for full documentation.

aeval_framework-0.1.0.dist-info/RECORD

@@ -0,0 +1,63 @@
+agent_eval/__init__.py,sha256=Bn20xqehr50-Xbz_W-2b_tLA81G2Q6q4xyg7L8iZYMk,347
+agent_eval/cli.py,sha256=fSGZuDwcEjS3rjXjRAmGzDm0Sq90XKD537QQYsjN65I,16526
+agent_eval/api/__init__.py,sha256=HCcNc8esBbtlnZETe2sas6cWe_BEC76WDuMtxxKITNM,290
+agent_eval/api/app.py,sha256=bizUSDaus6AdAmfaMD03WiNsMF6i-2lk0QPTyhoByYg,2591
+agent_eval/api/events.py,sha256=3vfWUTUByGzxjP2jNG_k0pfqb9ZyFGoQfscb35zZey8,4042
+agent_eval/api/standalone.py,sha256=sr9yNNcgeKamP3BNrXwReVnhfCnFB8KylsxPIltx5vM,3112
+agent_eval/api/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+agent_eval/api/routes/datasets.py,sha256=vEWoG-x22XbUBcNUOEiccg2-fJk09xujqQAywBYAXOE,17418
+agent_eval/api/routes/graders.py,sha256=IlpOkPK1W7dz1va4UDhn8Ge92AveeA0afC9yQ9XNvXk,402
+agent_eval/api/routes/metrics.py,sha256=oPdMgS-SuKtpvZWpFZ9xP6OJZo6FxwYeX4h2lfr3o-I,1682
+agent_eval/api/routes/runs.py,sha256=HRB0XvCABYY0KtxKaolACXXE9xkBxU-plHDjYnCM3BQ,18904
+agent_eval/api/routes/suites.py,sha256=wnjx4EgHjQ6IOY8_LE6Vf0jF3DfaCCR5JrtEUMvD1E4,2229
+agent_eval/api/routes/tasks.py,sha256=iPAZJUlmbvrxoPyl2MD6f_Ij-qmXGYsUijGlkAtuaKc,3581
+agent_eval/core/__init__.py,sha256=7wLbb03B1WsvVKJS_JcFLxjEFzra8lUZGhxjKfc6JhY,877
+agent_eval/core/contract.py,sha256=8HtBEz8XDLG3U9dt3858ADwDdAjlEoquZeukoob00ro,8629
+agent_eval/core/metrics.py,sha256=D0Fe3sHhNdNkGW9ROl9TGlDb4qpsSEiUbVt-nRlZftE,5094
+agent_eval/core/runner.py,sha256=n4_pE2csGHAoek1Lng2ZTRC31JmJUpVg-huZSNM0RqI,32991
+agent_eval/core/suite.py,sha256=wVDMo5RaeTwESADgKY0HEh8qDSEfoY0tgNqlwav6mIk,1776
+agent_eval/core/types.py,sha256=4EO6UmeTvpfCcAoGTZPpjdfa0qvpneX5RqATMjtId44,9284
+agent_eval/dataset/__init__.py,sha256=YFWjfanVH7DvCLr2UiPdN905eVq2anupKF8YATClLxE,846
+agent_eval/dataset/models.py,sha256=H_y3w0dpa44SuiOplQwe3wUc0jHi_XTwYnip2G3d0d4,7387
+agent_eval/dataset/quality.py,sha256=gWkju6kOHzJYT384O_Uia1RXzrC4DW13c_xg0J3wZDQ,6739
+agent_eval/dataset/storage.py,sha256=wL3NLL0diL7d-1kHqGuFouemK8dQc4sjpRrxst2PgqU,12769
+agent_eval/dataset/version.py,sha256=jovPkA2IlGFNJZVoQg88CpxhT_IiI-eHBs23el4mlsk,2382
+agent_eval/dataset/sources/__init__.py,sha256=V5qd0_MZv3J4psOFy1ld2LiEgDh9k6RUl-Ih5_zYP9c,1052
+agent_eval/dataset/sources/llm_generator.py,sha256=qH-WIG2ParYmNH-GwJXtbx-13W8ADux4s-Rv1X92JxE,7752
+agent_eval/dataset/sources/manual.py,sha256=D74h2NqAZ7m3ec5W84hKPWQYIA9hveHlb3HqD5y157M,6087
+agent_eval/dataset/sources/regression.py,sha256=uvaYeGqpzv9PRniLZMd4NOA7oa364BqjaZlp4mz5j-M,7387
+agent_eval/dataset/sources/trace_mining.py,sha256=p_ATYS8SsHrfLlZkLpVRwAi63XDKwUaARxd9q6Unuxw,10056
+agent_eval/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+agent_eval/examples/basic_usage.py,sha256=rY4o8eYhTA_zy0WPReQ7BGL1_-dL_DP4X8aI7YSOblw,5410
+agent_eval/examples/mock_runner.py,sha256=XwcNJyfhhgN4WLygNKNLK3WIHvejmnCcvPI8J8_r2GM,6429
+agent_eval/graders/__init__.py,sha256=E8EecyidxG8JwHRUnL594sUEot82T-dgqZH3pWj5noY,3403
+agent_eval/graders/artifact_check.py,sha256=vKVD0AZZTkny9JXDPB9XXMCkqS7WNMJ-AycOoH3lCAk,3607
+agent_eval/graders/code_based.py,sha256=MPIrsdEpyJOusCeM4gwwHiHwN0B3JrLFuEPSqLZHmEg,2984
+agent_eval/graders/human.py,sha256=HEbjmuHWLkScOIbAHZwH3bbck_esCLdyTyrGW_b1NS8,2518
+agent_eval/graders/metric.py,sha256=JE3Oll8bVS371BpPE3BCKBogOzlCm-iJd-RZW_xrrl8,5348
+agent_eval/graders/model_based.py,sha256=PGNNQdanFgGjSO5wYjMvmwAK7L9BropK1DYIaeuAyZA,5477
+agent_eval/graders/state_check.py,sha256=Qb1t-qxMkB9xCQVqsHQqHzGlDRRFZJHoRTDXoI5Oh58,3336
+agent_eval/graders/step_level.py,sha256=Q6p5nurKgyZ_oralaFnNFp9DPJeAshXBC4XIAM3Cwb8,3963
+agent_eval/graders/tool_calls.py,sha256=4XprzqLbB5C3Uqdr2Wgb8wezyNzLH5EWSAPexwrsir0,3053
+agent_eval/graders/transcript.py,sha256=gxY5gMX-IN7OiAtNaCoCcfNIvqAyky0KhgX5DAcaomI,2545
+agent_eval/metrics/__init__.py,sha256=smLtMTeP19q15bJMwIuJbLi2AWcVRbaW9SX3es74GT8,3508
+agent_eval/metrics/answer_relevancy.py,sha256=NMr5T7fw-3xWPAT8WyCZuA7TIhRZWvTq8HsIYw1zrhA,1764
+agent_eval/metrics/base.py,sha256=frv2UQrU6P4vtAKzAU8C5rZKwqIf5Q-ClPD2F9J3tjQ,5080
+agent_eval/metrics/batch_evaluation.py,sha256=T5flsYzP_e7kL9U_IJbv2dib7IUFHI5bYAe06HGrIYc,9551
+agent_eval/metrics/context_precision.py,sha256=GrWKdliY7JmHLN1f3WuTUVzYKtL2tM1PmvOpp-w8an4,2053
+agent_eval/metrics/context_recall.py,sha256=aj9jxubBoJOO_yEeZj4dU1SqSz77e1KHdy-Z8hmHZds,2462
+agent_eval/metrics/faithfulness.py,sha256=0k71cbItS3pzLwh4aRPu3wDfOUybHZJ7PlfIVi_dvns,2594
+agent_eval/metrics/llm_judge.py,sha256=PIDHiFo8ldhW-a0GmhpvII4NqGkPDFkTBrLwlCg5pRI,3202
+agent_eval/metrics/prompt_metric.py,sha256=RY8ISi7dKgbPtl0ovnJIYuUYxUj4fP0msEK5jT_SHuU,5345
+agent_eval/metrics/pytest_plugin.py,sha256=IlT9fVYpvQ0JlhysNucbg0gIbKXCH9fON2swteKUJu4,10901
+agent_eval/metrics/report.py,sha256=7ZIvUGjb82l8DcJ6AC7cPytZX7aZm65aLJWZKe-eheQ,5696
+agent_eval/metrics/synthetic_data.py,sha256=rNQgNF7HpKDrqv9XWFa0nAsDImIOQfsAjYwgV9p--X8,7293
+agent_eval/storage/__init__.py,sha256=1D5QpjD7BI1j5HBfv51m67XwsEr8UFOwm1Hops5D5zg,475
+agent_eval/storage/memory.py,sha256=0bcKS3RoT4Qn9HFpV46RGfKMnzqiMf7Cgybnvo8P_cc,2999
+agent_eval/storage/sqlite.py,sha256=iiVP7vdsq5kI550Hs185TO2ja4g7q0QOtBhu6UBOdXU,8594
+agent_eval/trace/__init__.py,sha256=zaGBaoW2UJ8zbOWJwjf5GsNLEBe1t3DjbrD4WihuaUk,459
+agent_eval/trace/phoenix.py,sha256=XC5VH4bzmXDvlwPbWxelUeBsJxQ5zLfs1yU2mrtQfCg,4208
+aeval_framework-0.1.0.dist-info/METADATA,sha256=gTEq9P4dhAu00Lps2dGoMlPeQQ9lWEcwFwFps2p3rbE,1819
+aeval_framework-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+aeval_framework-0.1.0.dist-info/entry_points.txt,sha256=oKoXCZu-JXxiaSMdl2WaMqkb5VAcvBT3OFXzwwx4TWM,51
+aeval_framework-0.1.0.dist-info/RECORD,,

aeval_framework-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

aeval_framework-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+eval-suite = agent_eval.cli:main

agent_eval/__init__.py

@@ -0,0 +1,14 @@
+"""
+Aeval — Agent Evaluation Framework
+
+A reusable, open-source evaluation framework for AI agents, driven by OTel traces.
+
+Usage:
+    from agent_eval import EvalRunner, EvalSuite
+
+    runner = EvalRunner(agent_runner=my_runner)
+    suite = EvalSuite.from_yaml("suite.yaml")
+    result = await runner.run_suite(suite)
+"""
+
+__version__ = "0.1.0"

agent_eval/api/__init__.py

@@ -0,0 +1,14 @@
+"""
+REST API for the Aeval evaluation framework.
+
+Provides FastAPI routes for managing suites, runs, and viewing results.
+
+Usage:
+    from agent_eval.api import create_app
+
+    app = create_app(runner=my_eval_runner)
+"""
+
+from agent_eval.api.app import create_app
+
+__all__ = ["create_app"]

agent_eval/api/app.py

@@ -0,0 +1,82 @@
+"""
+FastAPI application factory for Aeval.
+
+Creates a FastAPI app with all eval routes mounted.
+Can be used standalone or mounted in an existing app.
+
+Usage:
+    # Standalone
+    app = create_app(runner=my_runner)
+
+    # Mounted in existing FastAPI app
+    from fastapi import FastAPI
+    app = FastAPI()
+    eval_app = create_app(runner=my_runner)
+    app.mount("/api/eval", eval_app)
+"""
+
+from __future__ import annotations
+
+from fastapi import FastAPI
+
+from agent_eval.api.routes import datasets, graders, metrics, runs, suites, tasks
+from agent_eval.core.runner import EvalRunner
+
+# Global runner reference (set by create_app)
+_runner: EvalRunner | None = None
+
+
+def _get_runner() -> EvalRunner | None:
+    """Get the global EvalRunner instance"""
+    return _runner
+
+
+def set_runner(runner: EvalRunner | None) -> None:
+    """Set the global EvalRunner instance.
+
+    Used by the host app to inject the real runner during async startup
+    (e.g. main.py lifespan → eval_integration.config.create_aeval_runner).
+    """
+    global _runner
+    _runner = runner
+
+
+def create_app(runner: EvalRunner | None = None) -> FastAPI:
+    """
+    Create a FastAPI app with Aeval routes.
+
+    Args:
+        runner: EvalRunner instance. If None, routes will return 503.
+
+    Returns:
+        FastAPI application
+    """
+    app = FastAPI(
+        title="Aeval API",
+        version="0.1.0",
+        description="Agent Evaluation Framework API",
+    )
+
+    # Store runner in app state and global reference
+    global _runner
+    _runner = runner
+    app.state.runner = runner
+
+    # Include routers
+    app.include_router(suites.router, prefix="/suites", tags=["suites"])
+    app.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
+    app.include_router(runs.router, prefix="/runs", tags=["runs"])
+    # compare 挂在 /compare (spec §REST API), 不带 /runs 前缀
+    app.include_router(runs.compare_router, tags=["compare"])
+    app.include_router(graders.router, prefix="/graders", tags=["graders"])
+    # 数据集管理 (change ③: 数据集构建闭环 — CRUD/导入/挖掘/生成/质量/to-suite)
+    app.include_router(datasets.router, prefix="/datasets", tags=["datasets"])
+    # 批量评测 (change ④: 对已有输出直接批量打分 — POST /metrics/batch)
+    app.include_router(metrics.router, prefix="/metrics", tags=["metrics"])
+
+    @app.get("/health")
+    async def health():
+        # 动态读取全局 runner — host app 会在 startup 阶段注入真实 runner
+        return {"status": "ok", "runner_configured": _get_runner() is not None}
+
+    return app

agent_eval/api/events.py

@@ -0,0 +1,96 @@
+"""Run 事件总线 — SSE 实时进度的进程内 fan-out (任务 3.1)。
+
+协议 (设计文档 D3 / §17.3):
+    - EvalRunner 进度回调 → per-run 订阅者队列扇出; 事件不落库, 断线恢复
+      完全依赖快照重拉 (GET /runs/{run_id}) + 重订阅
+    - 事件类型: task_start / trial_start / trial_complete / task_complete /
+      run_complete / error; 载荷含 task_id 与 trial_index (适用时), 按
+      (task_id, trial_index) 幂等 — 重复事件由客户端快照自愈兜底
+    - run 生命周期由服务端后台任务持有, 与观察连接解耦
+
+终态保留: run_complete 事件缓存于进程内 (有界, LIFO 淘汰), 供 run 结束后
+订阅的客户端立即收终态并关流; 进程重启后缓存消失, 客户端以快照为准。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import time
+from collections import OrderedDict
+from typing import Any
+
+# 有界队列: 慢订阅者丢最旧事件 (快照可自愈), 不阻塞 run 主流程
+_QUEUE_MAXSIZE = 256
+# 终态事件缓存上限 (防止长进程内存无界增长)
+_TERMINAL_CACHE_MAX = 500
+
+TERMINAL_EVENT_TYPES = {"run_complete"}
+
+
+class RunEventBus:
+    """run_id → 订阅者队列集合的事件扇出枢纽 (单事件循环, 无锁)。"""
+
+    def __init__(self) -> None:
+        self._subscribers: dict[str, set[asyncio.Queue]] = {}
+        self._terminal: OrderedDict[str, dict[str, Any]] = OrderedDict()
+
+    # ── 发布 ─────────────────────────────────────────────────────────────
+
+    def publish(self, run_id: str, event_type: str, data: dict[str, Any] | None = None) -> None:
+        """发布一条 run 事件 (非阻塞)。"""
+        event: dict[str, Any] = {
+            "type": event_type,
+            "run_id": run_id,
+            "timestamp": time.time() * 1000,
+            **(data or {}),
+        }
+        if event_type in TERMINAL_EVENT_TYPES:
+            self._terminal[run_id] = event
+            self._terminal.move_to_end(run_id)
+            while len(self._terminal) > _TERMINAL_CACHE_MAX:
+                self._terminal.popitem(last=False)
+
+        for queue in self._subscribers.get(run_id, set()):
+            _offer(queue, event)
+
+    # ── 订阅 ─────────────────────────────────────────────────────────────
+
+    def subscribe(self, run_id: str) -> asyncio.Queue:
+        """注册订阅者队列; run 已有缓存终态时立即注入终态事件。"""
+        queue: asyncio.Queue = asyncio.Queue(maxsize=_QUEUE_MAXSIZE)
+        self._subscribers.setdefault(run_id, set()).add(queue)
+        terminal = self._terminal.get(run_id)
+        if terminal is not None:
+            _offer(queue, terminal)
+        return queue
+
+    def unsubscribe(self, run_id: str, queue: asyncio.Queue) -> None:
+        """注销订阅者 (SSE 断开时调用, 防泄漏)。"""
+        subscribers = self._subscribers.get(run_id)
+        if subscribers is None:
+            return
+        subscribers.discard(queue)
+        if not subscribers:
+            self._subscribers.pop(run_id, None)
+
+    def subscriber_count(self, run_id: str) -> int:
+        return len(self._subscribers.get(run_id, set()))
+
+    def terminal_event(self, run_id: str) -> dict[str, Any] | None:
+        return self._terminal.get(run_id)
+
+
+def _offer(queue: asyncio.Queue, event: dict[str, Any]) -> None:
+    """put_nowait + 最旧丢弃溢出策略 (与 AChat event_bus 行为一致)。"""
+    try:
+        queue.put_nowait(event)
+    except asyncio.QueueFull:
+        with contextlib.suppress(asyncio.QueueEmpty):
+            queue.get_nowait()
+        with contextlib.suppress(asyncio.QueueFull):  # pragma: no cover - 理论不可达
+            queue.put_nowait(event)
+
+
+# 全局单例 (与 eval API 子应用同生命周期)
+run_event_bus = RunEventBus()