agentbyte 0.50.0__tar.gz → 0.51.0__tar.gz

{agentbyte-0.50.0 → agentbyte-0.51.0}/CHANGELOG.md

@@ -6,6 +6,18 @@ The format follows Keep a Changelog principles and semantic versioning.
 
 ## [Unreleased]
 
+## [0.51.0] - 2026-10-09
+
+### Added
+
+- Per-dimension scores in the WebUI and reports: `SuiteItemVerdict.dimensions` and
+  `SuiteRunSummary.dimension_means`, `TaskCell.dimensions` and `RunColumn.mean_dimensions` (over the
+  shared tasks). The run page gets a Dimensions table, run comparisons and benchmark reports get
+  per-dimension means with the change against the baseline plus a per-task breakdown, and the run and
+  benchmark Markdown reports list the dimension means. Runs without dimensions are unchanged.
+- Benchmark report page: a "Show per-task" selector switches the task-by-variant matrix between the
+  overall score and any judged dimension, with the change against the baseline variant.
+
 ## [0.50.0] - 2026-10-09
 
 ### Changed (breaking)

{agentbyte-0.50.0 → agentbyte-0.51.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentbyte
-Version: 0.50.0
+Version: 0.51.0
 Summary: A toolkit for designing multiagent systems
 Author-email: MrDataPsycho <mr.data.psycho@gmail.com>
 License-Expression: LicenseRef-Proprietary
@@ -84,7 +84,7 @@ Description-Content-Type: text/markdown
 
 Agentbyte is an observability-first agentic AI framework for building and studying multiagent systems with a learning-first, implementation-oriented workflow.
 
-Current release: **0.50.0**
+Current release: **0.51.0**
 
 ## Building an Agent
 

{agentbyte-0.50.0 → agentbyte-0.51.0}/README.md

@@ -2,7 +2,7 @@
 
 Agentbyte is an observability-first agentic AI framework for building and studying multiagent systems with a learning-first, implementation-oriented workflow.
 
-Current release: **0.50.0**
+Current release: **0.51.0**
 
 ## Building an Agent
 

agentbyte-0.51.0/src/agentbyte/__about__.py

@@ -0,0 +1,2 @@
+__version__ = "0.51.0"
+VERSION = __version__

{agentbyte-0.50.0 → agentbyte-0.51.0}/src/agentbyte/eval/workbench/compare.py

@@ -17,14 +17,13 @@ from collections.abc import Sequence
 from enum import StrEnum
 from typing import Any
 
-from pydantic import BaseModel, ConfigDict
-
+from pydantic import BaseModel, ConfigDict, Field
 from agentbyte.eval.comparison import improvement_pct
 from agentbyte.messages import Usage
 
 from .benchmark import EvalBenchmark, TaskSet, combine_task_sets
 from .suite import EvalSuiteConfigError
-from .verdict import ItemOutcome, SuiteItemVerdict, SuiteRunSummary
+from .verdict import ItemOutcome, SuiteItemVerdict, SuiteRunSummary, mean_dimensions
 
 
 class RegressionFlag(StrEnum):
@@ -44,6 +43,7 @@ class TaskCell(BaseModel):
     run_id: str
     present: bool
     overall: float | None = None
+    dimensions: dict[str, float] = Field(default_factory=dict)
     outcome: ItemOutcome | None = None
     passed: bool | None = None
     delta_vs_baseline: float | None = None
@@ -83,6 +83,7 @@ class RunColumn(BaseModel):
     min_score: float
     is_baseline: bool
     mean_overall: float | None = None
+    mean_dimensions: dict[str, float] = Field(default_factory=dict)
     passed_count: int = 0
     scored_count: int = 0
     improvement_pct: float | None = None
@@ -163,6 +164,19 @@ class ComparisonReport(BaseModel):
             )
         lines.append("")
 
+        dimension_names = list(
+            dict.fromkeys(name for variant in self.variants for name in variant.run.mean_dimensions)
+        )
+        if dimension_names:
+            lines.append("## Dimension means")
+            dimension_header = ["Dimension", *(variant.variant_name for variant in self.variants)]
+            lines.append("| " + " | ".join(dimension_header) + " |")
+            lines.append("| " + " | ".join(["---"] * len(dimension_header)) + " |")
+            for name in dimension_names:
+                means = " | ".join(_fmt(variant.run.mean_dimensions.get(name)) for variant in self.variants)
+                lines.append(f"| {name} | {means} |")
+            lines.append("")
+
         if self.failed_variants:
             lines.append("## Failed variants")
             for failed in self.failed_variants:
@@ -375,6 +389,7 @@ def _cell(
         run_id=run_id,
         present=True,
         overall=verdict.overall,
+        dimensions=dict(verdict.dimensions),
         outcome=verdict.outcome,
         passed=verdict.passed,
         delta_vs_baseline=delta,
@@ -404,6 +419,9 @@ def _column(
         min_score=summary.min_score,
         is_baseline=is_baseline,
         mean_overall=mean,
+        mean_dimensions=mean_dimensions(
+            [v for name in shared_names if (v := run.get(name)) is not None]
+        ),
         passed_count=passed,
         scored_count=scored,
         improvement_pct=None if is_baseline else improvement_pct(baseline_mean, mean),

{agentbyte-0.50.0 → agentbyte-0.51.0}/src/agentbyte/eval/workbench/verdict.py

@@ -6,8 +6,7 @@ import math
 from datetime import datetime
 from enum import StrEnum
 
-from pydantic import BaseModel, ConfigDict
-
+from pydantic import BaseModel, ConfigDict, Field
 from agentbyte.eval.history import (
     CaptureMode,
     EvalItemRecord,
@@ -44,6 +43,7 @@ class SuiteItemVerdict(BaseModel):
     outcome: ItemOutcome
     passed: bool
     overall: float | None = None
+    dimensions: dict[str, float] = Field(default_factory=dict)
     scoring_status: ScoringStatus | None = None
     failure_reason: str | None = None
     execution_status: ExecutionStatus | None = None
@@ -82,6 +82,7 @@ class SuiteRunSummary(BaseModel):
     started_at: datetime | None
     finished_at: datetime | None
     items: list[SuiteItemVerdict]
+    dimension_means: dict[str, float] = Field(default_factory=dict)
 
     @property
     def failed_count(self) -> int:
@@ -120,6 +121,9 @@ class SuiteRunSummary(BaseModel):
             rows.append(("Error", _cell(self.error)))
         lines = [f"# Eval run: {self.suite or self.run_id}", "", "| | |", "|---|---|"]
         lines += [f"| {key} | {value} |" for key, value in rows]
+        if self.dimension_means:
+            lines += ["", "## Dimensions (mean over scored items)", "", "| Dimension | Mean |", "|---|---|"]
+            lines += [f"| {_cell(name)} | {mean:.2f} |" for name, mean in self.dimension_means.items()]
         lines += ["", "## Items", "", "| # | Task | Outcome | Score | Reason |", "|---|---|---|---|---|"]
         for item in self.items:
             score = f"{item.overall:g}" if item.overall is not None else "-"
@@ -148,6 +152,7 @@ def item_verdict(item: EvalItemRecord, min_score: float) -> SuiteItemVerdict:
         outcome=outcome,
         passed=outcome == "passed",
         overall=item.overall if item.scoring_status == "scored" else None,
+        dimensions=dict(item.dimensions) if item.scoring_status == "scored" else {},
         scoring_status=item.scoring_status,
         failure_reason=item.failure_reason,
         execution_status=item.execution_status,
@@ -216,9 +221,20 @@ def summarize(
         started_at=run.started_at,
         finished_at=run.finished_at,
         items=verdicts,
+        dimension_means=mean_dimensions(verdicts),
     )
 
 
+def mean_dimensions(verdicts: list[SuiteItemVerdict]) -> dict[str, float]:
+    """Mean score per dimension over the items that scored it, in first-seen order."""
+    values: dict[str, list[float]] = {}
+    for verdict in verdicts:
+        for name, value in verdict.dimensions.items():
+            if math.isfinite(value):
+                values.setdefault(name, []).append(value)
+    return {name: sum(scores) / len(scores) for name, scores in values.items()}
+
+
 def _total_usage(items: list[EvalItemRecord]) -> Usage | None:
     usages = [item.usage for item in items if item.usage is not None]
     if not usages: