agentbyte 0.30.0__tar.gz → 0.30.1__tar.gz

{agentbyte-0.30.0 → agentbyte-0.30.1}/CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to Agentbyte are documented in this file.
 
 The format follows Keep a Changelog principles and semantic versioning.
 
+## [0.30.1] - 2026-09-30
+
+### Fixed
+
+- Judge score integrity follow-ups (spec 0055 v0.3.0): judges re-check cancellation after each model call, and `EvalRunner`/`PairwiseRunner` record a run as `cancelled` without judging it when the token is cancelled or the trajectory has `execution_status="cancelled"`.
+- Raw-JSON judge responses with duplicate keys are rejected (`duplicate_criterion` for `LLMEvalJudge`, `invalid_response` for `PairwiseJudge`) instead of keeping the last value.
+- When a target raises, `EvalRunner` and `PairwiseRunner` store only the exception type in `trajectory.error`, never the exception message.
+- `ModelEvalTarget` records `execution_status` on failed and cancelled trajectories.
+
+### Changed
+
+- **Breaking:** `criteria=None` selects judge defaults; an explicit empty list raises `JudgeScoringError("invalid_criteria")`. An empty `default_criteria` (`LLMEvalJudge`, `LLMTrajectoryJudge`) or `criteria` (`PairwiseJudge`) raises `ValueError` at construction.
+- **Breaking:** a scored `EvalScore` carrying legacy `fallback` or `judge_failed` metadata fails validation.
+
 ## [0.30.0] - 2026-09-30
 
 ### Added

{agentbyte-0.30.0 → agentbyte-0.30.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentbyte
-Version: 0.30.0
+Version: 0.30.1
 Summary: A toolkit for designing multiagent systems
 Author-email: MrDataPsycho <mr.data.psycho@gmail.com>
 License-Expression: LicenseRef-Proprietary
@@ -86,7 +86,7 @@ Description-Content-Type: text/markdown
 
 Agentbyte is an observability-first agentic AI framework for building and studying multiagent systems with a learning-first, implementation-oriented workflow.
 
-Current release: **0.30.0**
+Current release: **0.30.1**
 
 ## Building an Agent
 

{agentbyte-0.30.0 → agentbyte-0.30.1}/README.md

@@ -2,7 +2,7 @@
 
 Agentbyte is an observability-first agentic AI framework for building and studying multiagent systems with a learning-first, implementation-oriented workflow.
 
-Current release: **0.30.0**
+Current release: **0.30.1**
 
 ## Building an Agent
 

agentbyte-0.30.1/src/agentbyte/__about__.py

@@ -0,0 +1,2 @@
+__version__ = "0.30.1"
+VERSION = __version__

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/judges/llm.py

@@ -15,9 +15,11 @@ from agentbyte.messages import SystemMessage, UserMessage
 from agentbyte.eval.base import BaseEvalJudge, JudgeScoringError
 from agentbyte.eval.judges.validation import (
     mean_score,
+    loads_judge_json,
     normalize_criterion,
+    require_non_empty_defaults,
+    resolve_criteria,
     validate_criterion_scores,
-    validate_requested_criteria,
 )
 from agentbyte.eval.types import AnswerStrategy, EvalScore, EvalTrajectory
 
@@ -60,12 +62,13 @@ class LLMEvalJudge(BaseEvalJudge):
             answer_strategy=answer_strategy,
             source_filter=source_filter,
         )
+        require_non_empty_defaults(default_criteria, "default_criteria")
         self.client = client
-        self.default_criteria = default_criteria or [
-            "accuracy",
-            "completeness",
-            "helpfulness",
-        ]
+        self.default_criteria = (
+            ["accuracy", "completeness", "helpfulness"]
+            if default_criteria is None
+            else list(default_criteria)
+        )
         self.custom_instructions = custom_instructions
         self.use_structured_output = use_structured_output
 
@@ -87,7 +90,7 @@ class LLMEvalJudge(BaseEvalJudge):
                 criteria = raw
             elif isinstance(raw, str):
                 criteria = [raw]
-        eval_criteria = validate_requested_criteria(criteria or self.default_criteria)
+        eval_criteria = resolve_criteria(criteria, self.default_criteria)
         if cancellation_token and cancellation_token.is_cancelled():
             raise asyncio.CancelledError()
 
@@ -131,6 +134,8 @@ class LLMEvalJudge(BaseEvalJudge):
                 )
         except Exception as exc:
             raise JudgeScoringError("provider_error", "Judge model call failed") from exc
+        if cancellation_token and cancellation_token.is_cancelled():
+            raise asyncio.CancelledError()
 
         if self.use_structured_output:
             dimensions, reasoning = self._validate_structured(
@@ -198,7 +203,7 @@ class LLMEvalJudge(BaseEvalJudge):
         criteria: list[str],
     ) -> tuple[dict[str, float], dict[str, str]]:
         try:
-            parsed = _JudgeResponse.model_validate(json.loads(raw_content))
+            parsed = _JudgeResponse.model_validate(loads_judge_json(raw_content))
         except (json.JSONDecodeError, ValidationError, TypeError) as exc:
             raise JudgeScoringError("invalid_response", "Judge returned malformed JSON") from exc
         dimensions, _ = validate_criterion_scores(

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/judges/pairwise.py

@@ -14,6 +14,7 @@ from agentbyte.llm.base import BaseChatCompletionClient
 from agentbyte.messages import SystemMessage, UserMessage
 
 from agentbyte.eval.base import BaseEvalJudge, JudgeScoringError
+from agentbyte.eval.judges.validation import loads_judge_json, require_non_empty_defaults
 from agentbyte.eval.types import AnswerStrategy, EvalScore, EvalTrajectory, PairwiseResult
 
 
@@ -55,8 +56,11 @@ class PairwiseJudge(BaseEvalJudge):
             answer_strategy=answer_strategy,
             source_filter=source_filter,
         )
+        require_non_empty_defaults(criteria, "criteria")
         self.client = client
-        self.criteria = criteria or ["accuracy", "helpfulness", "clarity"]
+        self.criteria = (
+            ["accuracy", "helpfulness", "clarity"] if criteria is None else list(criteria)
+        )
         self.custom_instructions = custom_instructions
 
     async def compare(
@@ -111,9 +115,11 @@ class PairwiseJudge(BaseEvalJudge):
             )
         except Exception as exc:
             raise JudgeScoringError("provider_error", "Pairwise judge model call failed") from exc
+        if cancellation_token and cancellation_token.is_cancelled():
+            raise asyncio.CancelledError()
         try:
             parsed = self._parse_response(result.structured_output, result.message.content)
-        except ValidationError as exc:
+        except (ValidationError, json.JSONDecodeError, TypeError) as exc:
             raise JudgeScoringError("invalid_response", "Malformed pairwise response") from exc
         winner = parsed.winner.strip().lower()
         if winner not in ("a", "b", "tie"):
@@ -148,7 +154,9 @@ class PairwiseJudge(BaseEvalJudge):
             return _PairwiseResponse.model_validate(structured_output.model_dump())
         if structured_output is not None:
             return _PairwiseResponse.model_validate(structured_output)
-        return _PairwiseResponse.model_validate_json(raw_content)
+        return _PairwiseResponse.model_validate(
+            loads_judge_json(raw_content, duplicate_reason="invalid_response")
+        )
 
 
 __all__ = ["PairwiseJudge"]

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/judges/trajectory.py

@@ -14,8 +14,9 @@ from agentbyte.eval.base import BaseEvalJudge, JudgeScoringError
 from agentbyte.eval.judges.validation import (
     mean_score,
     normalize_criterion,
+    require_non_empty_defaults,
+    resolve_criteria,
     validate_criterion_scores,
-    validate_requested_criteria,
 )
 from agentbyte.eval.types import EvalScore, EvalTrajectory
 
@@ -52,12 +53,13 @@ class LLMTrajectoryJudge(BaseEvalJudge):
         super().__init__(name, answer_strategy="last_assistant")
         if max_payload_chars <= 0:
             raise ValueError("max_payload_chars must be positive")
+        require_non_empty_defaults(default_criteria, "default_criteria")
         self.client = client
-        self.default_criteria = default_criteria or [
-            "tool_use_correctness",
-            "coordination",
-            "execution_efficiency",
-        ]
+        self.default_criteria = (
+            ["tool_use_correctness", "coordination", "execution_efficiency"]
+            if default_criteria is None
+            else list(default_criteria)
+        )
         self.custom_instructions = custom_instructions or ""
         self.max_payload_chars = max_payload_chars
 
@@ -76,7 +78,7 @@ class LLMTrajectoryJudge(BaseEvalJudge):
                 if isinstance(raw, str)
                 else None
             )
-        criteria = validate_requested_criteria(criteria or self.default_criteria)
+        criteria = resolve_criteria(criteria, self.default_criteria)
         if cancellation_token and cancellation_token.is_cancelled():
             raise asyncio.CancelledError()
         trace = trajectory.trace
@@ -126,6 +128,8 @@ class LLMTrajectoryJudge(BaseEvalJudge):
             )
         except Exception as exc:
             raise JudgeScoringError("provider_error", "Process judge model call failed") from exc
+        if cancellation_token and cancellation_token.is_cancelled():
+            raise asyncio.CancelledError()
         response = result.structured_output
         if not isinstance(response, ProcessJudgeResponse):
             raise JudgeScoringError(

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/judges/validation.py

@@ -7,6 +7,7 @@ requested criterion exactly once with a finite 0-10 number is rejected with a
 
 from __future__ import annotations
 
+import json
 import math
 from collections.abc import Iterable
 from typing import Any
@@ -36,6 +37,31 @@ def validate_requested_criteria(criteria: list[str]) -> list[str]:
     return list(criteria)
 
 
+def resolve_criteria(criteria: list[str] | None, defaults: list[str]) -> list[str]:
+    """Use ``defaults`` only when criteria is None; an explicit empty list is rejected."""
+    return validate_requested_criteria(defaults if criteria is None else criteria)
+
+
+def require_non_empty_defaults(criteria: list[str] | None, name: str) -> None:
+    """Constructor guard: an explicit empty criteria list never means "use defaults"."""
+    if criteria is not None and not criteria:
+        raise ValueError(f"{name} must not be empty; pass None to use the defaults")
+
+
+def loads_judge_json(text: str, *, duplicate_reason: str = "duplicate_criterion") -> Any:
+    """Decode judge JSON, rejecting duplicate keys that ``json.loads`` would silently drop."""
+
+    def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, value in pairs:
+            if key in result:
+                raise JudgeScoringError(duplicate_reason, f"Duplicate JSON key: {key!r}")
+            result[key] = value
+        return result
+
+    return json.loads(text, object_pairs_hook=reject_duplicates)
+
+
 def validate_score_value(value: Any) -> float:
     """Accept only finite real numbers in [0, 10]; booleans and strings are rejected."""
     if isinstance(value, bool) or not isinstance(value, (int, float)):
@@ -87,6 +113,9 @@ def mean_score(dimensions: dict[str, float]) -> float:
 __all__ = [
     "normalize_criterion",
     "validate_requested_criteria",
+    "resolve_criteria",
+    "require_non_empty_defaults",
+    "loads_judge_json",
     "validate_score_value",
     "validate_criterion_scores",
     "mean_score",

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/pairwise.py

@@ -10,6 +10,7 @@ from agentbyte.messages import Usage
 
 from agentbyte.eval.base import BaseEvalTarget, JudgeScoringError
 from agentbyte.eval.judges.pairwise import PairwiseJudge
+from agentbyte.eval.runner import is_cancelled_run
 from agentbyte.eval.types import EvalTask, EvalTrajectory, PairwiseResult, ScoringStatus
 
 
@@ -100,14 +101,18 @@ class PairwiseRunner:
     ) -> tuple[EvalTrajectory, str | None]:
         """Run one side, returning its trajectory and a failure reason code if it raised."""
         try:
-            return await target.run(task, cancellation_token), None
+            trajectory = await target.run(task, cancellation_token)
         except asyncio.CancelledError:
             if cancellation_token and cancellation_token.is_cancelled():
                 reason, error = "cancelled", "Cancelled"
             else:
                 raise
         except Exception as exc:
-            reason, error = "target_error", str(exc)
+            reason, error = "target_error", type(exc).__name__
+        else:
+            if is_cancelled_run(trajectory, cancellation_token):
+                return trajectory, "cancelled"
+            return trajectory, None
         trajectory = EvalTrajectory(
             task=task, messages=[], success=False, error=error, usage=Usage()
         )

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/runner.py

@@ -113,12 +113,17 @@ class EvalRunner(BaseEvalRunner):
             raise
         except Exception as exc:
             return self._build_unscored(
-                self._build_failed_trajectory(task, error=str(exc)),
+                self._build_failed_trajectory(task, error=type(exc).__name__),
                 status="failed",
                 reason="target_error",
                 error_type=type(exc).__name__,
             )
 
+        # A cancelled run is not an agent failure to be measured (e.g. a 0 from
+        # a reference judge); it carries no numeric evidence.
+        if is_cancelled_run(trajectory, cancellation_token):
+            return self._build_unscored(trajectory, status="cancelled", reason="cancelled")
+
         # resolve: call-level arg > task metadata > judge default
         effective_criteria = criteria
         if effective_criteria is None:
@@ -175,4 +180,13 @@ class EvalRunner(BaseEvalRunner):
         )
 
 
+def is_cancelled_run(
+    trajectory: EvalTrajectory, cancellation_token: CancellationToken | None
+) -> bool:
+    """Whether a returned trajectory was cut short by cancellation rather than failing."""
+    if cancellation_token is not None and cancellation_token.is_cancelled():
+        return True
+    return trajectory.metadata.get("execution_status") == "cancelled"
+
+
 __all__ = ["EvalRunner"]

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/targets/model.py

@@ -36,6 +36,7 @@ class ModelEvalTarget(BaseEvalTarget):
                 task,
                 start_time,
                 error="Execution cancelled before model call",
+                status="cancelled",
             )
 
         messages = []
@@ -69,6 +70,7 @@ class ModelEvalTarget(BaseEvalTarget):
         start_time: float,
         *,
         error: str,
+        status: str = "failed",
     ) -> EvalTrajectory:
         elapsed_ms = int((time.perf_counter() - start_time) * 1000)
         return EvalTrajectory(
@@ -80,6 +82,7 @@ class ModelEvalTarget(BaseEvalTarget):
             metadata={
                 "target_type": "model",
                 "target_name": self.name,
+                "execution_status": status,
                 "execution_time_ms": elapsed_ms,
             },
         )

{agentbyte-0.30.0 → agentbyte-0.30.1}/src/agentbyte/eval/types.py

@@ -113,6 +113,11 @@ class EvalTrajectory(BaseModel):
 ScoringStatus = Literal["scored", "failed", "cancelled"]
 
 
+# Markers written by pre-0055 judges on placeholder scores. Such records are
+# never valid numeric evidence.
+_FALLBACK_FLAGS = ("fallback", "judge_failed")
+
+
 def _is_valid_score_value(value: float) -> bool:
     return math.isfinite(value) and 0.0 <= value <= 10.0
 
@@ -166,6 +171,11 @@ class EvalScore(BaseModel):
                 raise ValueError("scored EvalScore requires finite dimensions in [0, 10]")
             if self.failure_reason is not None:
                 raise ValueError("scored EvalScore cannot carry a failure_reason")
+            if any(self.metadata.get(flag) for flag in _FALLBACK_FLAGS):
+                raise ValueError(
+                    "EvalScore marked as a judge fallback is not a measurement; "
+                    "record it as scoring_status='failed'"
+                )
             return self
         if self.overall is not None or self.dimensions:
             raise ValueError(