aeval-framework 0.1.0__tar.gz → 0.2.0__tar.gz

{aeval_framework-0.1.0 → aeval_framework-0.2.0}/.gitignore

@@ -21,3 +21,6 @@ out/
 .env.*
 .DS_Store
 Thumbs.db
+
+# Editor-local state (Qoder)
+.qoder/

{aeval_framework-0.1.0 → aeval_framework-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: aeval-framework
-Version: 0.1.0
+Version: 0.2.0
 Summary: Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)
 Project-URL: Homepage, https://github.com/QiYuyyds/Aeval
 Project-URL: Repository, https://github.com/QiYuyyds/Aeval

{aeval_framework-0.1.0 → aeval_framework-0.2.0}/pyproject.toml

@@ -2,7 +2,7 @@
 # Distribution name: aeval-framework (PyPI). Python module: agent_eval, CLI: eval-suite.
 # (`agent-eval` on PyPI belongs to an unrelated project — UK AISI agenteval.)
 name = "aeval-framework"
-version = "0.1.0"
+version = "0.2.0"
 description = "Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)"
 readme = "README.md"
 requires-python = ">=3.11"

aeval_framework-0.2.0/src/agent_eval/api/app.py

@@ -0,0 +1,191 @@
+"""
+FastAPI application factory for Aeval.
+
+Creates a FastAPI app with all eval routes mounted.
+Can be used standalone or mounted in an existing app.
+
+Usage:
+    # Standalone
+    app = create_app(runner=my_runner)
+
+    # Mounted in existing FastAPI app
+    from fastapi import FastAPI
+    app = FastAPI()
+    eval_app = create_app(runner=my_runner)
+    app.mount("/api/eval", eval_app)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import FastAPI
+
+from agent_eval.api.routes import datasets, graders, metrics, runs, suites, tasks
+from agent_eval.core.runner import EvalRunner
+from agent_eval.core.types import (
+    DEFAULT_BOOTSTRAP_ROUNDS,
+    DEFAULT_CONFIDENCE_LEVEL,
+    DEFAULT_INVALID_RATIO_LIMIT,
+    MIN_VALID_TRIALS_FOR_SATURATION,
+    STATISTICS_VERSION,
+)
+from agent_eval.graders import get_grader_catalog
+from agent_eval.trace.mapping import (
+    ATTRIBUTE_MAPPING_VERSION,
+    OTEL_GENAI_SPEC_VERSION,
+    VOCABULARY_OTEL_GENAI,
+    VOCABULARY_SPEC_VERSIONS,
+    known_vocabularies,
+)
+
+# Global runner reference (set by create_app)
+_runner: EvalRunner | None = None
+
+
+def _get_runner() -> EvalRunner | None:
+    """Get the global EvalRunner instance"""
+    return _runner
+
+
+def set_runner(runner: EvalRunner | None) -> None:
+    """Set the global EvalRunner instance.
+
+    Used by the host app to inject the real runner during async startup
+    (e.g. main.py lifespan → eval_integration.config.create_aeval_runner).
+    """
+    global _runner
+    _runner = runner
+
+
+DISTRIBUTION_NAME = "aeval-framework"  # PyPI 发行包名 (Python 模块为 agent_eval)
+
+
+def package_version() -> str:
+    """包版本: 优先取已安装元数据, 源码直跑时回退模块常量。"""
+    from importlib.metadata import PackageNotFoundError
+    from importlib.metadata import version as _metadata_version
+
+    try:
+        return _metadata_version(DISTRIBUTION_NAME)
+    except PackageNotFoundError:
+        from agent_eval import __version__
+
+        return __version__
+
+
+def statistics_defaults() -> dict[str, Any]:
+    """统计口径版本与 D7 数值默认值 (经元信息接口公布, 供调用方判断可比性)。"""
+    return {
+        "version": STATISTICS_VERSION,
+        "confidence_level": DEFAULT_CONFIDENCE_LEVEL,
+        "bootstrap_rounds": DEFAULT_BOOTSTRAP_ROUNDS,
+        "min_valid_trials_for_saturation": MIN_VALID_TRIALS_FOR_SATURATION,
+        "gate_invalid_ratio_limit": DEFAULT_INVALID_RATIO_LIMIT,
+    }
+
+
+def meta_payload(
+    api_prefix: str = "",
+    endpoints: list[str] | None = None,
+    version: str | None = None,
+) -> dict[str, Any]:
+    """元信息响应体: 版本 + 统计口径 + 能力清单。
+
+    同一大版本内响应结构向后兼容, 但统计口径的数值语义可能变化 —— 口径版本
+    必须显式公布, 调用方才能判断两个 run 是否可直接比较。
+    """
+    return {
+        "name": "Aeval",
+        "package": DISTRIBUTION_NAME,
+        "version": version or package_version(),
+        "api_prefix": api_prefix,
+        "endpoints": endpoints or ["/suites", "/tasks", "/runs", "/compare", "/graders",
+                                   "/datasets", "/metrics", "/health"],
+        "statistics": statistics_defaults(),
+        "evidence": {
+            "spec_version": OTEL_GENAI_SPEC_VERSION,
+            "mapping_version": ATTRIBUTE_MAPPING_VERSION,
+            # default_mapping(vocabulary=...) 的可填值在这里枚举: 调用方不需要读
+            # 源码猜词汇, 也不需要为了发现一个字符串去 import 内部模块。
+            "vocabularies": {
+                "default": VOCABULARY_OTEL_GENAI,
+                "known": list(known_vocabularies()),
+                "spec_versions": dict(VOCABULARY_SPEC_VERSIONS),
+            },
+            "tool_arguments_captured_by_default": False,
+            "model_content_captured_by_default": False,
+            "capture_is_one_declaration": True,
+        },
+        "capabilities": {
+            "graders": [g["name"] for g in get_grader_catalog()],
+            "storage": ["memory", "sqlite"],
+            "trace_providers": ["phoenix (optional, lazily imported)"],
+            "sse": True,
+            "datasets": True,
+            "metrics": True,
+            "validity_verdicts": True,
+            "confidence_intervals": True,
+            "normalized_trace_observations": True,
+            "termination_reasons": True,
+            "cost_axis": True,
+            "evidence_provenance_levels": True,
+            "deferred_grading": True,
+            # 重评分本期只有库层入口 (EvalRunner.regrade_run)。它牵涉同一大版本的
+            # 响应结构兼容承诺, 暴露面另立变更 —— 列出来免得调用方以为 /runs 上能调。
+            "regrade_over_http": False,
+            "regrade_over_cli": False,
+        },
+        "not_exposed": {
+            "regrade": (
+                "证据归档与重评分已落地 (EvalRunner.regrade_run / verdict_drift), "
+                "但 HTTP 与 CLI 入口本期未提供"
+            )
+        },
+    }
+
+
+def create_app(runner: EvalRunner | None = None) -> FastAPI:
+    """
+    Create a FastAPI app with Aeval routes.
+
+    Args:
+        runner: EvalRunner instance. If None, routes will return 503.
+
+    Returns:
+        FastAPI application
+    """
+    app = FastAPI(
+        title="Aeval API",
+        version="0.1.0",
+        description="Agent Evaluation Framework API",
+    )
+
+    # Store runner in app state and global reference
+    global _runner
+    _runner = runner
+    app.state.runner = runner
+
+    # Include routers
+    app.include_router(suites.router, prefix="/suites", tags=["suites"])
+    app.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
+    app.include_router(runs.router, prefix="/runs", tags=["runs"])
+    # compare 挂在 /compare (spec §REST API), 不带 /runs 前缀
+    app.include_router(runs.compare_router, tags=["compare"])
+    app.include_router(graders.router, prefix="/graders", tags=["graders"])
+    # 数据集管理 (change ③: 数据集构建闭环 — CRUD/导入/挖掘/生成/质量/to-suite)
+    app.include_router(datasets.router, prefix="/datasets", tags=["datasets"])
+    # 批量评测 (change ④: 对已有输出直接批量打分 — POST /metrics/batch)
+    app.include_router(metrics.router, prefix="/metrics", tags=["metrics"])
+
+    @app.get("/health")
+    async def health():
+        # 动态读取全局 runner — host app 会在 startup 阶段注入真实 runner
+        return {"status": "ok", "runner_configured": _get_runner() is not None}
+
+    @app.get("/meta")
+    async def meta():
+        """寄宿形态的元信息接口 (挂载前缀由宿主决定, 例如 /api/eval/meta)。"""
+        return meta_payload()
+
+    return app

{aeval_framework-0.1.0 → aeval_framework-0.2.0}/src/agent_eval/api/routes/runs.py

@@ -29,6 +29,8 @@ from pydantic import BaseModel, Field
 from sse_starlette.sse import EventSourceResponse
 
 from agent_eval.api.events import run_event_bus
+from agent_eval.core.metrics import classify_trial, trial_invalid_reason
+from agent_eval.core.types import STATISTICS_VERSION, TrialVerdict
 
 router = APIRouter()
 
@@ -60,6 +62,9 @@ async def list_runs(suite_name: str | None = None, limit: int = 50):
                 "started_at": r.started_at,
                 "completed_at": r.completed_at,
                 "duration_ms": r.duration_ms,
+                "statistics_version": r.statistics_version,
+                "evidence": r.evidence.model_dump() if r.evidence else None,
+                "regrade": _regrade_entry(r),
                 "task_count": len(r.trials),
                 "summary": r.summary.model_dump() if r.summary else None,
             }
@@ -98,7 +103,12 @@ async def create_run(request: CreateRunRequest):
     from agent_eval.core.types import RunResult
 
     await runner.storage.save_run(
-        RunResult(run_id=run_id, suite_name=suite.name, status="pending")
+        RunResult(
+            run_id=run_id,
+            suite_name=suite.name,
+            status="pending",
+            statistics_version=STATISTICS_VERSION,
+        )
     )
 
     async def _run():
@@ -164,6 +174,10 @@ async def get_run(run_id: str):
         "completed_at": run.completed_at,
         "duration_ms": run.duration_ms,
         "error": run.error,
+        "statistics_version": run.statistics_version,
+        "evidence": run.evidence.model_dump() if run.evidence else None,
+        # 本期只开库层重评分入口: 这里如实标能不能重评与原因, 不提供 HTTP 触发
+        "regrade": _regrade_entry(run),
         "trials": {
             task_id: [
                 {
@@ -173,12 +187,36 @@ async def get_run(run_id: str):
                     "score": t.avg_score(),
                     "duration_ms": t.duration_ms,
                     "error": t.error,
+                    "verdict": t.verdict.value,
+                    "invalid_reason": t.invalid_reason.value if t.invalid_reason else None,
+                    "termination_reason": (
+                        t.termination_reason.value if t.termination_reason else None
+                    ),
+                    "metrics": t.metrics,
+                    "evidence_gaps": [gap.model_dump() for gap in t.evidence_gaps],
+                    "unrecognized_attributes": t.unrecognized_attributes,
+                    # 结论的分量: 它依据的最弱一级证据 + 证据是否已归档
+                    "weakest_evidence": (
+                        t.weakest_evidence.value if t.weakest_evidence else None
+                    ),
+                    "evidence_archived": t.evidence_archived,
                     "grader_results": [
                         {
                             "grader_name": gr.grader_name,
                             "score": gr.score,
                             "passed": gr.passed,
                             "explanation": gr.explanation,
+                            "verdict": gr.verdict.value,
+                            "invalid_reason": (
+                                gr.invalid_reason.value if gr.invalid_reason else None
+                            ),
+                            "evidence_levels": [
+                                level.value for level in gr.evidence_levels
+                            ],
+                            "judgment_moment": (
+                                gr.judgment_moment.value if gr.judgment_moment else None
+                            ),
+                            "subject_only": gr.subject_only,
                         }
                         for gr in t.grader_results
                     ],
@@ -191,6 +229,14 @@ async def get_run(run_id: str):
     }
 
 
+def _regrade_entry(run) -> dict[str, Any]:
+    """能不能重评分 + 为什么不能 (与库层 ``regrade_run`` 共用同一份判断)。"""
+    from agent_eval.core.runner import regrade_state
+
+    possible, reason = regrade_state(run)
+    return {"available": possible, "reason": reason, "exposed_over_http": False}
+
+
 @router.delete("/{run_id}")
 async def delete_run(run_id: str):
     """删除运行"""
@@ -204,6 +250,10 @@ async def delete_run(run_id: str):
     if not deleted:
         raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
 
+    # 派生物一并清除: 内容寻址的评分缓存可能缓存了该 run 采集到的证据内容
+    runner.evict_run_cache(run_id)
+    _background_tasks.pop(run_id, None)
+
     return {"deleted": True}
 
 
@@ -410,11 +460,13 @@ async def submit_human_score(run_id: str, request: HumanScoreRequest):
             "threshold", threshold
         )
 
-    # 更新已存的 GraderResult
+    # 更新已存的 GraderResult (评分回传 → 离开 pending 通道)
     grader_result.score = request.score
     grader_result.passed = request.score >= threshold
     grader_result.explanation = request.explanation or "人工评分"
     grader_result.confidence = 1.0
+    grader_result.verdict = TrialVerdict.VALID
+    grader_result.invalid_reason = None
     grader_result.details = {
         **grader_result.details,
         "status": "scored",
@@ -425,6 +477,10 @@ async def submit_human_score(run_id: str, request: HumanScoreRequest):
     # 重算 trial 成功状态与 run 汇总 (含该 task 汇总)
     if task is not None:
         trial.success = runner._compute_trial_success(task, trial.grader_results)
+    trial.verdict = classify_trial(trial)
+    trial.invalid_reason = trial_invalid_reason(trial)
+    if run.statistics_version is None:
+        run.statistics_version = STATISTICS_VERSION
     run.summary = runner._compute_summary(run, suite) if suite is not None else run.summary
     await runner.storage.save_run(run)
 
@@ -436,6 +492,7 @@ async def submit_human_score(run_id: str, request: HumanScoreRequest):
         "score": grader_result.score,
         "passed": grader_result.passed,
         "trial_success": trial.success,
+        "trial_verdict": trial.verdict.value,
         "summary": run.summary.model_dump() if run.summary else None,
     }
 
@@ -489,85 +546,207 @@ async def compare_runs(request: CompareRequest):
 
 
 def _build_comparison(run_a, run_b) -> dict[str, Any]:
-    """构建两次运行的对比"""
+    """构建两次运行的对比 (口径版本不同则标注不可比; 区间重叠则不判方向)。
+
+    既有字段 (`a`/`b`/`delta`/`regressions`/`improvements`/`tasks`) 的名称与类型
+    保持不变; 分母为 0 时 `a`/`b`/`delta` 为 null 而非 0.0。
+    """
     summary_a = run_a.summary
     summary_b = run_b.summary
 
-    # 全局指标对比
-    all_k_values = set()
-    if summary_a.pass_at_k:
-        all_k_values.update(summary_a.pass_at_k.keys())
-    if summary_b.pass_at_k:
-        all_k_values.update(summary_b.pass_at_k.keys())
-
-    pass_at_k_comparison = {}
-    for k in sorted(all_k_values):
-        a_val = summary_a.pass_at_k.get(k, 0.0)
-        b_val = summary_b.pass_at_k.get(k, 0.0)
-        pass_at_k_comparison[f"pass_at_{k}"] = {
+    version_a = getattr(run_a, "statistics_version", None)
+    version_b = getattr(run_b, "statistics_version", None)
+    evidence_a = getattr(run_a, "evidence", None)
+    evidence_b = getattr(run_b, "evidence", None)
+    same_caliber = version_a is not None and version_a == version_b
+    same_boundary, boundary_reason = _compare_evidence_boundaries(evidence_a, evidence_b)
+    comparable = same_caliber and same_boundary
+
+    all_k_values: set[int] = set()
+    for summary in (summary_a, summary_b):
+        for k in (summary.pass_at_k or {}):
+            all_k_values.add(int(k))
+
+    def _entry(
+        a_val: float | None,
+        b_val: float | None,
+        a_ci: tuple[float | None, float | None] | None,
+        b_ci: tuple[float | None, float | None] | None,
+        extrapolated: bool,
+    ) -> dict[str, Any]:
+        overlapping = _intervals_overlap(a_ci, b_ci)
+        return {
             "a": a_val,
             "b": b_val,
-            "delta": round(b_val - a_val, 4),
+            "delta": _delta(a_val, b_val),
+            "a_ci": _ci_list(a_ci),
+            "b_ci": _ci_list(b_ci),
+            "intervals_overlap": overlapping,
+            "significant": (
+                None if overlapping is None else (comparable and not overlapping)
+            ),
+            "extrapolated": extrapolated,
+            "comparable": comparable,
         }
 
+    def _k_map(summary, field: str) -> dict:
+        raw = getattr(summary, field, None) or {}
+        return {int(k): v for k, v in raw.items()}
+
+    pass_at_k_comparison = {}
     pass_power_k_comparison = {}
     for k in sorted(all_k_values):
-        a_val = summary_a.pass_power_k.get(k, 0.0)
-        b_val = summary_b.pass_power_k.get(k, 0.0)
-        pass_power_k_comparison[f"pass_power_{k}"] = {
-            "a": a_val,
-            "b": b_val,
-            "delta": round(b_val - a_val, 4),
-        }
+        est_a = _k_map(summary_a, "estimates").get(k)
+        est_b = _k_map(summary_b, "estimates").get(k)
+        pass_at_k_comparison[f"pass_at_{k}"] = _entry(
+            est_a.value if est_a else None,
+            est_b.value if est_b else None,
+            _est_ci(est_a),
+            _est_ci(est_b),
+            bool(est_a and est_a.extrapolated) or bool(est_b and est_b.extrapolated),
+        )
+
+        pow_a = _k_map(summary_a, "power_estimates").get(k)
+        pow_b = _k_map(summary_b, "power_estimates").get(k)
+        pass_power_k_comparison[f"pass_power_{k}"] = _entry(
+            pow_a.value if pow_a else None,
+            pow_b.value if pow_b else None,
+            _est_ci(pow_a),
+            _est_ci(pow_b),
+            bool(pow_a and pow_a.extrapolated) or bool(pow_b and pow_b.extrapolated),
+        )
+
+    dist_a = getattr(summary_a, "score_distribution", None)
+    dist_b = getattr(summary_b, "score_distribution", None)
+    avg_entry = _entry(
+        summary_a.avg_score,
+        summary_b.avg_score,
+        _dist_ci(dist_a),
+        _dist_ci(dist_b),
+        False,
+    )
 
-    # 逐 task 对比
+    # 逐 task 对比: 只有区间不重叠才判为退化/提升 (D4)
     task_a_map = {ts.task_id: ts for ts in summary_a.task_summaries}
     task_b_map = {ts.task_id: ts for ts in summary_b.task_summaries}
 
     all_task_ids = set(task_a_map.keys()) | set(task_b_map.keys())
-    regressions = []
-    improvements = []
-    task_comparisons = {}
+    regressions: list[dict[str, Any]] = []
+    improvements: list[dict[str, Any]] = []
+    task_comparisons: dict[str, Any] = {}
 
     for task_id in sorted(all_task_ids):
         ts_a = task_a_map.get(task_id)
         ts_b = task_b_map.get(task_id)
+        if not (ts_a and ts_b):
+            continue
+
+        a_score = ts_a.avg_score
+        b_score = ts_b.avg_score
+        delta = _delta(a_score, b_score)
+        overlapping = _intervals_overlap(
+            _dist_ci(ts_a.score_distribution), _dist_ci(ts_b.score_distribution)
+        )
+        significant = (
+            comparable and overlapping is False and delta is not None and abs(delta) > 0.1
+        )
+        task_comparisons[task_id] = {
+            "a": a_score,
+            "b": b_score,
+            "delta": delta,
+            "a_ci": _ci_list(_dist_ci(ts_a.score_distribution)),
+            "b_ci": _ci_list(_dist_ci(ts_b.score_distribution)),
+            "intervals_overlap": overlapping,
+            "significant": (None if overlapping is None else (
+                comparable and overlapping is False
+            )),
+            "comparable": comparable,
+        }
 
-        if ts_a and ts_b:
-            a_score = ts_a.avg_score
-            b_score = ts_b.avg_score
-            delta = round(b_score - a_score, 4)
-
-            task_comparisons[task_id] = {
-                "a": a_score,
-                "b": b_score,
-                "delta": delta,
-            }
-
-            if delta < -0.1:
-                regressions.append({
-                    "task_id": task_id,
-                    "a": a_score,
-                    "b": b_score,
-                    "delta": delta,
-                })
-            elif delta > 0.1:
-                improvements.append({
-                    "task_id": task_id,
-                    "a": a_score,
-                    "b": b_score,
-                    "delta": delta,
-                })
+        if not significant:
+            continue
+        row = {"task_id": task_id, "a": a_score, "b": b_score, "delta": delta}
+        if delta < 0:
+            regressions.append(row)
+        else:
+            improvements.append(row)
 
     return {
         "pass_at_k": pass_at_k_comparison,
         "pass_power_k": pass_power_k_comparison,
-        "avg_score": {
-            "a": summary_a.avg_score,
-            "b": summary_b.avg_score,
-            "delta": round(summary_b.avg_score - summary_a.avg_score, 4),
-        },
+        "avg_score": avg_entry,
         "regressions": regressions,
         "improvements": improvements,
         "tasks": task_comparisons,
+        "statistics_version": {"a": version_a, "b": version_b},
+        "evidence_boundary": {
+            "a": evidence_a.model_dump() if evidence_a else None,
+            "b": evidence_b.model_dump() if evidence_b else None,
+        },
+        "comparable": comparable,
+        "not_comparable_reason": _not_comparable_reason(
+            comparable, same_caliber, version_a, version_b, boundary_reason
+        ),
     }
+
+
+def _compare_evidence_boundaries(
+    evidence_a: Any, evidence_b: Any
+) -> tuple[bool, str | None]:
+    """两个 run 的证据边界是否一致 (缺记录即无法判定可比)。"""
+    if evidence_a is None and evidence_b is None:
+        return False, "两个 run 均未记录证据采集边界 (历史 run), 无法判定可比性"
+    if evidence_a is None or evidence_b is None:
+        missing = "a" if evidence_a is None else "b"
+        return False, f"run_{missing} 未记录证据采集边界 (历史 run), 无法判定可比性"
+    return evidence_a.compare_with(evidence_b)
+
+
+def _not_comparable_reason(
+    comparable: bool,
+    same_caliber: bool,
+    version_a: str | None,
+    version_b: str | None,
+    boundary_reason: str | None,
+) -> str | None:
+    if comparable:
+        return None
+    if not same_caliber:
+        return (
+            "两个 run 的统计口径版本不同"
+            if version_a is not None and version_b is not None
+            else "至少一个 run 未记录统计口径版本 (历史 run), 无法判定可比性"
+        )
+    return boundary_reason or "两个 run 的证据采集边界不同"
+
+
+def _delta(a: float | None, b: float | None) -> float | None:
+    if a is None or b is None:
+        return None
+    return round(b - a, 4)
+
+
+def _est_ci(est) -> tuple[float | None, float | None] | None:
+    if est is None or est.p_lower_bound is None or est.p_upper_bound is None:
+        return None
+    return (est.p_lower_bound, est.p_upper_bound)
+
+
+def _dist_ci(dist) -> tuple[float | None, float | None] | None:
+    if dist is None or dist.ci_low is None or dist.ci_high is None:
+        return None
+    return (dist.ci_low, dist.ci_high)
+
+
+def _ci_list(ci: tuple[float | None, float | None] | None) -> list[float] | None:
+    return [ci[0], ci[1]] if ci else None
+
+
+def _intervals_overlap(
+    a: tuple[float | None, float | None] | None,
+    b: tuple[float | None, float | None] | None,
+) -> bool | None:
+    """两侧区间是否重叠 (D4)。任一侧无区间 → None (无法判定)。"""
+    if a is None or b is None:
+        return None
+    return not (a[1] < b[0] or b[1] < a[0])

{aeval_framework-0.1.0 → aeval_framework-0.2.0}/src/agent_eval/api/routes/tasks.py

@@ -10,6 +10,9 @@ from __future__ import annotations
 
 from fastapi import APIRouter, HTTPException
 
+from agent_eval.core.metrics import split_trials_by_verdict
+from agent_eval.core.types import TrialVerdict
+
 router = APIRouter()
 
 
@@ -91,18 +94,24 @@ async def get_task_history(task_id: str):
         trials = run.trials.get(task_id)
         if not trials:
             continue
+        buckets = split_trials_by_verdict(trials)
+        valid = [t for _, t in buckets[TrialVerdict.VALID]]
         grader_scores: dict[str, list[float]] = {}
-        for trial in trials:
+        for trial in valid:
             for gr in trial.grader_results:
                 grader_scores.setdefault(gr.grader_name, []).append(gr.score)
         history.append({
             "run_id": run.run_id,
             "suite_name": run.suite_name,
             "started_at": run.started_at,
-            "trials_passed": sum(1 for t in trials if t.success),
+            "trials_passed": sum(1 for t in valid if t.success),
             "trials_total": len(trials),
+            "valid_trials": len(valid),
+            "invalid_trials": len(buckets[TrialVerdict.INVALID]),
+            "pending_trials": len(buckets[TrialVerdict.PENDING]),
             "avg_score": (
-                round(sum(t.avg_score() for t in trials) / len(trials), 4)
+                round(sum(t.avg_score() for t in valid) / len(valid), 4)
+                if valid else None
             ),
             "graders": {
                 name: round(sum(scores) / len(scores), 4)