aeval-framework 0.2.0__tar.gz → 0.3.0__tar.gz

{aeval_framework-0.2.0 → aeval_framework-0.3.0}/.gitignore

@@ -9,6 +9,7 @@ build/
 .ruff_cache/
 *.db
 *.sqlite3
+.coverage
 
 # Node / Next.js (dashboard)
 node_modules/
@@ -24,3 +25,15 @@ Thumbs.db
 
 # Editor-local state (Qoder)
 .qoder/
+
+# AI tool state
+.claude/
+.codex/
+.meituan-catpaw/
+
+# Run artifacts & scratch notes
+.live-run-output.txt
+testlist.txt
+
+# Personal (never commit)
+docs/eval-harness-interview-prep.md

{aeval_framework-0.2.0 → aeval_framework-0.3.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.5
 Name: aeval-framework
-Version: 0.2.0
+Version: 0.3.0
 Summary: Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)
 Project-URL: Homepage, https://github.com/QiYuyyds/Aeval
 Project-URL: Repository, https://github.com/QiYuyyds/Aeval
 Project-URL: Documentation, https://github.com/QiYuyyds/Aeval/tree/main/docs
-Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/releases
+Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/blob/main/CHANGELOG.md
 Author: Aeval contributors
 License: MIT
 Keywords: agents,evaluation,llm,otel,testing,tracing

{aeval_framework-0.2.0 → aeval_framework-0.3.0}/pyproject.toml

@@ -2,7 +2,7 @@
 # Distribution name: aeval-framework (PyPI). Python module: agent_eval, CLI: eval-suite.
 # (`agent-eval` on PyPI belongs to an unrelated project — UK AISI agenteval.)
 name = "aeval-framework"
-version = "0.2.0"
+version = "0.3.0"
 description = "Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -52,7 +52,7 @@ eval-suite = "agent_eval.cli:main"
 Homepage = "https://github.com/QiYuyyds/Aeval"
 Repository = "https://github.com/QiYuyyds/Aeval"
 Documentation = "https://github.com/QiYuyyds/Aeval/tree/main/docs"
-Changelog = "https://github.com/QiYuyyds/Aeval/releases"
+Changelog = "https://github.com/QiYuyyds/Aeval/blob/main/CHANGELOG.md"
 
 [build-system]
 requires = ["hatchling"]
@@ -60,6 +60,9 @@ build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/agent_eval"]
+# src/agent_eval/packs/** (含 starter pack 的 suite.yaml/manifest.json) 作为
+# package data 随 wheel 分发 —— `eval-suite run demo` 的零 setup 首跑依赖它;
+# 仓库 packs/starter/ 与之同源, tests/test_packaging.py 钉住两侧一致。
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"

{aeval_framework-0.2.0 → aeval_framework-0.3.0}/src/agent_eval/__init__.py

@@ -11,4 +11,4 @@ Usage:
     result = await runner.run_suite(suite)
 """
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"

{aeval_framework-0.2.0 → aeval_framework-0.3.0}/src/agent_eval/api/app.py

@@ -22,6 +22,7 @@ from typing import Any
 from fastapi import FastAPI
 
 from agent_eval.api.routes import datasets, graders, metrics, runs, suites, tasks
+from agent_eval.core.metrics import MIN_ALIGNED_RATINGS_FOR_AGREEMENT
 from agent_eval.core.runner import EvalRunner
 from agent_eval.core.types import (
     DEFAULT_BOOTSTRAP_ROUNDS,
@@ -82,6 +83,7 @@ def statistics_defaults() -> dict[str, Any]:
         "bootstrap_rounds": DEFAULT_BOOTSTRAP_ROUNDS,
         "min_valid_trials_for_saturation": MIN_VALID_TRIALS_FOR_SATURATION,
         "gate_invalid_ratio_limit": DEFAULT_INVALID_RATIO_LIMIT,
+        "min_aligned_ratings_for_agreement": MIN_ALIGNED_RATINGS_FOR_AGREEMENT,
     }
 
 
@@ -95,6 +97,21 @@ def meta_payload(
     同一大版本内响应结构向后兼容, 但统计口径的数值语义可能变化 —— 口径版本
     必须显式公布, 调用方才能判断两个 run 是否可直接比较。
     """
+    # 扩展点发现与 CLI 同源 (core/discovery.discover_extensions):
+    # 两处给出的自定义判据/环境/模拟器名字集合必须一致, 不允许
+    # 「接口看得见、命令行用不上」的分歧 (spec: cli)。
+    from agent_eval.core.discovery import discover_extensions
+
+    try:
+        discovered = discover_extensions().catalog()
+    except Exception:  # noqa: BLE001 — 元数据异常不阻断 /meta
+        discovered = {}
+    discovered_graders = [e["name"] for e in discovered.get("graders", [])]
+    builtin_graders = [g["name"] for g in get_grader_catalog()]
+    # 发现的自定义判据追加在内置之后 (去重); 与 CLI extensions 清单同一份来源
+    graders = sorted(builtin_graders) + sorted(
+        n for n in discovered_graders if n not in builtin_graders
+    )
     return {
         "name": "Aeval",
         "package": DISTRIBUTION_NAME,
@@ -118,7 +135,12 @@ def meta_payload(
             "capture_is_one_declaration": True,
         },
         "capabilities": {
-            "graders": [g["name"] for g in get_grader_catalog()],
+            "graders": graders,
+            "extensions": discovered,
+            "user_simulation": True,
+            "run_event_injection": True,
+            "judgment_moments": ["at_end", "not_at_end", "any_time", "after_last_event"],
+            "environment_identity_boundary": True,
             "storage": ["memory", "sqlite"],
             "trace_providers": ["phoenix (optional, lazily imported)"],
             "sse": True,
@@ -131,6 +153,11 @@ def meta_payload(
             "cost_axis": True,
             "evidence_provenance_levels": True,
             "deferred_grading": True,
+            # ④ agent 指标目录: 证据感知测量上下文 / 诊断-判分角色轴 / κ/α / 门
+            "evidence_aware_metrics": True,
+            "metric_roles": ["diagnostic", "judging"],
+            "inter_rater_agreement": ["cohen_kappa", "krippendorff_alpha"],
+            "reward_basis_gates": True,
             # 重评分本期只有库层入口 (EvalRunner.regrade_run)。它牵涉同一大版本的
             # 响应结构兼容承诺, 暴露面另立变更 —— 列出来免得调用方以为 /runs 上能调。
             "regrade_over_http": False,

{aeval_framework-0.2.0 → aeval_framework-0.3.0}/src/agent_eval/api/routes/runs.py

@@ -29,6 +29,7 @@ from pydantic import BaseModel, Field
 from sse_starlette.sse import EventSourceResponse
 
 from agent_eval.api.events import run_event_bus
+from agent_eval.core.comparison import runs_comparable
 from agent_eval.core.metrics import classify_trial, trial_invalid_reason
 from agent_eval.core.types import STATISTICS_VERSION, TrialVerdict
 
@@ -185,6 +186,8 @@ async def get_run(run_id: str):
                     "trace_id": t.trace_id,
                     "success": t.success,
                     "score": t.avg_score(),
+                    # trial 总分 (权重/策略/乘性门塌缩后的值; None = 该字段落盘前的历史 run)
+                    "synthesized_score": t.synthesized_score,
                     "duration_ms": t.duration_ms,
                     "error": t.error,
                     "verdict": t.verdict.value,
@@ -217,6 +220,10 @@ async def get_run(run_id: str):
                                 gr.judgment_moment.value if gr.judgment_moment else None
                             ),
                             "subject_only": gr.subject_only,
+                            # 门结果: 因子 / 是否乘入总分 / 未生效原因 (spec: graders)
+                            "gate_factor": gr.gate_factor,
+                            "gate_applied": gr.gate_applied,
+                            "gate_reason": gr.gate_reason,
                         }
                         for gr in t.grader_results
                     ],
@@ -477,6 +484,7 @@ async def submit_human_score(run_id: str, request: HumanScoreRequest):
     # 重算 trial 成功状态与 run 汇总 (含该 task 汇总)
     if task is not None:
         trial.success = runner._compute_trial_success(task, trial.grader_results)
+        trial.synthesized_score = runner._trial_weighted_score(task, trial)
     trial.verdict = classify_trial(trial)
     trial.invalid_reason = trial_invalid_reason(trial)
     if run.statistics_version is None:
@@ -554,13 +562,12 @@ def _build_comparison(run_a, run_b) -> dict[str, Any]:
     summary_a = run_a.summary
     summary_b = run_b.summary
 
+    # 可比性前置与 CLI / 基线门同源 (core.comparison.runs_comparable)
+    comparable, not_comparable_reason = runs_comparable(run_a, run_b)
     version_a = getattr(run_a, "statistics_version", None)
     version_b = getattr(run_b, "statistics_version", None)
     evidence_a = getattr(run_a, "evidence", None)
     evidence_b = getattr(run_b, "evidence", None)
-    same_caliber = version_a is not None and version_a == version_b
-    same_boundary, boundary_reason = _compare_evidence_boundaries(evidence_a, evidence_b)
-    comparable = same_caliber and same_boundary
 
     all_k_values: set[int] = set()
     for summary in (summary_a, summary_b):
@@ -684,42 +691,10 @@ def _build_comparison(run_a, run_b) -> dict[str, Any]:
             "b": evidence_b.model_dump() if evidence_b else None,
         },
         "comparable": comparable,
-        "not_comparable_reason": _not_comparable_reason(
-            comparable, same_caliber, version_a, version_b, boundary_reason
-        ),
+        "not_comparable_reason": not_comparable_reason,
     }
 
 
-def _compare_evidence_boundaries(
-    evidence_a: Any, evidence_b: Any
-) -> tuple[bool, str | None]:
-    """两个 run 的证据边界是否一致 (缺记录即无法判定可比)。"""
-    if evidence_a is None and evidence_b is None:
-        return False, "两个 run 均未记录证据采集边界 (历史 run), 无法判定可比性"
-    if evidence_a is None or evidence_b is None:
-        missing = "a" if evidence_a is None else "b"
-        return False, f"run_{missing} 未记录证据采集边界 (历史 run), 无法判定可比性"
-    return evidence_a.compare_with(evidence_b)
-
-
-def _not_comparable_reason(
-    comparable: bool,
-    same_caliber: bool,
-    version_a: str | None,
-    version_b: str | None,
-    boundary_reason: str | None,
-) -> str | None:
-    if comparable:
-        return None
-    if not same_caliber:
-        return (
-            "两个 run 的统计口径版本不同"
-            if version_a is not None and version_b is not None
-            else "至少一个 run 未记录统计口径版本 (历史 run), 无法判定可比性"
-        )
-    return boundary_reason or "两个 run 的证据采集边界不同"
-
-
 def _delta(a: float | None, b: float | None) -> float | None:
     if a is None or b is None:
         return None

{aeval_framework-0.2.0 → aeval_framework-0.3.0}/src/agent_eval/api/routes/tasks.py

@@ -109,8 +109,9 @@ async def get_task_history(task_id: str):
             "valid_trials": len(valid),
             "invalid_trials": len(buckets[TrialVerdict.INVALID]),
             "pending_trials": len(buckets[TrialVerdict.PENDING]),
+            # 与 run summary 的 avg_score 同一口径: trial 总分 (含门塌缩), 非 grader 简均
             "avg_score": (
-                round(sum(t.avg_score() for t in valid) / len(valid), 4)
+                round(sum(t.total_score() for t in valid) / len(valid), 4)
                 if valid else None
             ),
             "graders": {