agentx-python 0.8.4__py3-none-any.whl → 0.8.5__py3-none-any.whl

agentx/evaluations/client.py

@@ -60,6 +60,13 @@ class AgentXValidationError(AgentXEvaluationsError):
     pass
 
 
+class EvaluationSubmissionError(AgentXEvaluationsError):
+    """A result batch could not be submitted (after one retry). The run is left unfinalized;
+    re-running execute() on the same context resumes past already-submitted cases."""
+
+    pass
+
+
 def _resolve_scorer_id(scorer_id: Optional[str], evaluation_settings_id: Optional[str]) -> Optional[str]:
     """One grader, two spellings: ``scorer_id`` is the post-consolidation name for what the wire
     still calls ``evaluationSettingsId`` (the ids are identical by design). Both kwargs are
@@ -297,10 +304,11 @@ class EvaluationsClient:
         python_version: Optional[str] = None,
         scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
+        split: Optional[str] = None,
     ) -> EvaluationRun:
         """``scorer_id`` names the LLM Judge Scorer grading this run (its id doubles as the
         wire's ``evaluationSettingsId``). ``evaluation_settings_id`` is the pre-consolidation
-        alias and keeps working."""
+        alias and keeps working. ``split`` records the named case subset this run covers."""
         from agentx.version import VERSION
 
         grader_id = _resolve_scorer_id(scorer_id, evaluation_settings_id)
@@ -318,6 +326,8 @@ class EvaluationsClient:
         }
         if grader_id:
             payload["evaluationSettingsId"] = grader_id
+        if split:
+            payload["split"] = split
         data = self._request("POST", "/runs", json=self._with_workspace(payload))
         return EvaluationRun(**data)
 
@@ -436,6 +446,13 @@ class EvaluationsClient:
         data = self._request("GET", f"/runs/{run_id}/missing-results")
         return data if isinstance(data, list) else data.get("missing", [])
 
+    def get_submitted_keys(self, run_id: str) -> List[str]:
+        """Idempotency keys this run has already accepted - what execute() uses to resume a
+        crashed or interrupted run without re-running (and re-paying for) finished cases."""
+        data = self._request("GET", f"/runs/{run_id}/missing-results")
+        keys = data.get("submittedKeys", []) if isinstance(data, dict) else []
+        return [k for k in keys if isinstance(k, str)]
+
     # ------------------------------------------------------------------
     # Self-host analysis fallback
     #

agentx/evaluations/datasets.py

@@ -105,9 +105,15 @@ class DatasetBuilder:
         expected_tools: Optional[List[str]] = None,
         trajectory_match_mode: str = "strict",
         expected_retrieval_context: Optional[Union[str, List[str]]] = None,
+        splits: Optional[List[str]] = None,
     ) -> "DatasetBuilder":
         """Add a case. `judge_guideline` is optional grading guidance specific to this question.
 
+        `splits` tags this case with named subsets (e.g. ``["smoke"]``): a run started with
+        ``client.evaluations.run(dataset_id, subject, split="smoke")`` executes only the tagged
+        cases (original case indexes are preserved, so per-case comparisons still line up with
+        full runs). An untagged case belongs to no split and only runs in full runs.
+
         `expected_tools` declares the tool calls a correct run of this case should make. When a
         result links its trace (return `{"output": ..., "trace_id": span.trace_id}` from the
         agent function), the engine matches the trace's actual tool-call sequence against it and
@@ -149,6 +155,8 @@ class DatasetBuilder:
             main["expectedTrajectory"] = {"tools": expected_tools, "mode": trajectory_match_mode}
         if expected_retrieval_context:
             main["expectedRetrievalContext"] = expected_retrieval_context
+        if splits:
+            main["splits"] = splits
         self._payload["questions"].append(
             {
                 "main_question": main,

agentx/evaluations/models.py

@@ -52,6 +52,9 @@ class TestCase(BaseModel):
     expected_delegations: Optional[List[str]] = Field(default=None, alias="expectedDelegations")
     judge_guideline: Optional[str] = Field(default=None, alias="judgeGuideline")
     smoke_test: Optional[SmokeTestSettings] = Field(default=None, alias="smokeTest")
+    # Named subsets this case belongs to (e.g. ["smoke"], ["full", "regression"]).
+    # ``run(dataset_id, split="smoke")`` runs only cases tagged with that split.
+    splits: Optional[List[str]] = None
 
     class Config:
         populate_by_name = True

agentx/evaluations/runner.py

@@ -4,12 +4,12 @@ import logging
 import os
 import time
 import uuid
-from typing import Any, Callable, Dict, List, Optional, Set, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Union
 
 from agentx.evaluations.adapters.raw import RawCallableAdapter
 from agentx.evaluations.adapters.precomputed import PrecomputedAdapter
 from agentx.evaluations.adapters.http_endpoint import HttpEndpointAdapter
-from agentx.evaluations.client import EvaluationsClient
+from agentx.evaluations.client import EvaluationsClient, EvaluationSubmissionError
 from agentx.evaluations.models import (
     AnalysisStatus,
     Dataset,
@@ -96,11 +96,14 @@ class EvaluationRunContext:
         run: EvaluationRun,
         subject: EvaluationSubject,
         evaluation_settings: Optional[EvaluationSettings] = None,
+        split: Optional[str] = None,
     ):
         self._client = client
         self._dataset = dataset
         self._run = run
         self._subject = subject
+        # Named case subset for this run - only cases tagged with it are built/executed.
+        self._split = split
         # When set, this run was started with an independently chosen grading
         # config (evaluation_settings_id) - its fields take precedence over the
         # dataset's own for anything execution-time reads (see _build_cases).
@@ -118,25 +121,63 @@ class EvaluationRunContext:
     # Step 1: execute
     # ------------------------------------------------------------------
 
-    def execute(self, adapter: AdapterLike) -> "EvaluationRunContext":
+    def execute(
+        self,
+        adapter: AdapterLike,
+        concurrency: int = 1,
+        reuse_outputs_from: Optional[str] = None,
+    ) -> "EvaluationRunContext":
         """Run all cases locally and submit batches to AgentX.
 
         The whole loop runs inside the eval-run scope (tracing/eval_scope.py): any trace the
         agent function creates is stamped source="eval-run" + monitor=False automatically, so
         eval traffic never skews production monitoring and no one has to remember a flag.
+
+        ``concurrency`` > 1 runs the agent callable across a thread pool (results are still
+        submitted in case order, and the eval-run scope is propagated into the workers).
+        ``reuse_outputs_from`` replays a previous run's recorded outputs for cases whose query
+        text is unchanged instead of re-running (and re-paying for) the agent - the judge still
+        re-scores them, which makes iterating on scorers cheap. Changed or new cases run
+        normally.
         """
         from agentx.tracing.eval_scope import enter_eval_run, exit_eval_run
 
         scope_token = enter_eval_run(self._run.run_id)
         try:
-            return self._execute_inner(adapter)
+            return self._execute_inner(adapter, concurrency=concurrency, reuse_outputs_from=reuse_outputs_from)
         finally:
             exit_eval_run(scope_token)
 
-    def _execute_inner(self, adapter: AdapterLike) -> "EvaluationRunContext":
+    def _fetch_reusable_outputs(self, run_id: str) -> Dict[tuple, str]:
+        """(query, run_number, is_smoke_variant) -> output text, from a previous run's rows.
+        Keyed on the query TEXT so a reworded case never silently reuses a stale answer."""
+        try:
+            prior = self._client.get_run(run_id)
+        except Exception as exc:
+            logger.warning("reuse_outputs_from: could not load run %s (%s) - running everything", run_id, exc)
+            return {}
+        reusable: Dict[tuple, str] = {}
+        for row in prior.get("results", []) or []:
+            if row.get("error") or row.get("status") == "failed":
+                continue
+            output = (row.get("output") or {}).get("text")
+            query = (row.get("input") or {}).get("query")
+            if not output or not query:
+                continue
+            key = (query, row.get("runNumber") or 1, bool(row.get("isSmokeTestVariant")))
+            reusable[key] = output
+        return reusable
+
+    def _execute_inner(
+        self,
+        adapter: AdapterLike,
+        concurrency: int = 1,
+        reuse_outputs_from: Optional[str] = None,
+    ) -> "EvaluationRunContext":
         normalized = _wrap_adapter(adapter)
-        cases = _build_cases(self._dataset, self._run, self._evaluation_settings)
+        cases = _build_cases(self._dataset, self._run, self._evaluation_settings, split=self._split)
         max_batch = self._run.limits.max_batch_size
+        reusable = self._fetch_reusable_outputs(reuse_outputs_from) if reuse_outputs_from else {}
 
         # Banner
         sep = "─" * 60
@@ -144,7 +185,7 @@ class EvaluationRunContext:
         framework = self._subject.framework or "custom"
         runtime = self._subject.runtime or "local"
         display = self._subject.display_name or ""
-        n_q = len(self._dataset.questions)
+        n_q = len({c.question_index for c in cases if not c.is_smoke_test_variant})
         n_r = (
             self._evaluation_settings.number_of_requests
             if self._evaluation_settings
@@ -171,6 +212,45 @@ class EvaluationRunContext:
         batch: List[EvaluationResult] = []
         total = len(cases)
 
+        def produce(case: EvaluationCase) -> EvaluationResult:
+            # Cached replay: same query text at the same repetition reuses the recorded output
+            # (the server still re-scores it with THIS run's grading config).
+            cached = reusable.get((case.query, case.run_number, case.is_smoke_test_variant))
+            if cached is not None:
+                return normalize_result(
+                    case, {"output": cached, "metadata": {"reusedFromRun": reuse_outputs_from}}
+                )
+            return normalized(case)
+
+        if concurrency > 1:
+            import concurrent.futures
+            import contextvars
+
+            def in_scope(case: EvaluationCase) -> EvaluationResult:
+                # ContextVars (the eval-run scope) do not cross thread boundaries on their own -
+                # each worker task runs inside a copy of the submitting thread's context so the
+                # agent's traces still get stamped source="eval-run".
+                return contextvars.copy_context().run(produce, case)
+
+            pending = [
+                case
+                for case in cases
+                if _idem_key(self._run.run_id, case.case_id, case.run_number) not in already_done
+            ]
+            executor = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
+            # map() yields in submission order, so batching/submission below stays deterministic.
+            mapped = executor.map(in_scope, pending)
+
+            def ordered() -> "Iterator[EvaluationResult]":
+                try:
+                    yield from mapped
+                finally:
+                    executor.shutdown(wait=True)
+
+            results_iter = ordered()
+        else:
+            results_iter = None  # sequential path below produces inline
+
         for idx, case in enumerate(cases, start=1):
             idem_key = _idem_key(self._run.run_id, case.case_id, case.run_number)
 
@@ -179,7 +259,7 @@ class EvaluationRunContext:
                 _print_progress(idx, total, case, skipped=True)
                 continue
 
-            result = normalized(case)
+            result = next(results_iter) if results_iter is not None else produce(case)
             result.idempotency_key = idem_key
             # Tag the result with the case's model so the server can group it into
             # the Sovereignty & Portability matrix (the callable may also set it).
@@ -207,31 +287,46 @@ class EvaluationRunContext:
         batch_id = str(uuid.uuid4())
         n = len(batch)
         with Spinner(f"Scoring - AI is rating {n} result{'s' if n != 1 else ''}"):
-            try:
-                resp = self._client.append_results(self._run.run_id, batch_id, batch)
-                if resp.live_statistics is not None:
-                    self._live_stats = resp.live_statistics
-                _say(
-                    f"  {green('✓')}  Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}"
-                )
-                logger.info(
-                    "Batch %s: accepted=%d duplicates=%d failed=%d",
-                    batch_id[:8],
-                    resp.accepted,
-                    resp.duplicates,
-                    resp.failed_validation,
-                )
-            except Exception as exc:
-                _say(f"  {red('✗')}  Scoring failed: {dim(str(exc))}")
-                logger.error("Failed to submit batch %s: %s", batch_id[:8], exc)
+            last_exc: Optional[Exception] = None
+            for attempt in (1, 2):
+                try:
+                    resp = self._client.append_results(self._run.run_id, batch_id, batch)
+                    if resp.live_statistics is not None:
+                        self._live_stats = resp.live_statistics
+                    _say(
+                        f"  {green('✓')}  Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}"
+                    )
+                    logger.info(
+                        "Batch %s: accepted=%d duplicates=%d failed=%d",
+                        batch_id[:8],
+                        resp.accepted,
+                        resp.duplicates,
+                        resp.failed_validation,
+                    )
+                    return
+                except Exception as exc:
+                    last_exc = exc
+                    if attempt == 1:
+                        logger.warning("Batch %s submission failed, retrying once: %s", batch_id[:8], exc)
+        # A batch that cannot be submitted must FAIL the run, not print a red line and carry on:
+        # execute() used to finish "successfully" having uploaded nothing. Failing fast also
+        # stops paying for agent calls whose results would hit the same broken engine; a
+        # re-execute() of this context resumes past everything already accepted (idempotency
+        # keys are deterministic and the engine returns the submitted set).
+        _say(f"  {red('✗')}  Scoring failed: {dim(str(last_exc))}")
+        logger.error("Failed to submit batch %s after retry: %s", batch_id[:8], last_exc)
+        raise EvaluationSubmissionError(
+            f"Failed to submit a batch of {n} result(s) to the engine after a retry: {last_exc}. "
+            "The run was left unfinalized; re-running execute() resumes past already-submitted cases."
+        ) from last_exc
 
     def _fetch_submitted_keys(self) -> Set[str]:
+        """Keys already accepted by this run - the engine's /missing-results route returns them
+        so a re-execute() after a crash skips (and never re-pays for) finished cases."""
         try:
-            missing = self._client.get_missing_results(self._run.run_id)
-            # missing-results returns cases NOT yet submitted - we want the inverse
-            # but if the endpoint isn't live yet, just return empty set
-            return set()
+            return set(self._client.get_submitted_keys(self._run.run_id))
         except Exception:
+            # Older engines without the route: no resume, identical to the historical behavior.
             return set()
 
     # ------------------------------------------------------------------
@@ -517,11 +612,16 @@ class EvaluationsRunner:
         subject: Union[Dict[str, Any], EvaluationSubject],
         scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
+        split: Optional[str] = None,
     ) -> EvaluationRunContext:
         """Start a run of ``dataset_id`` against ``subject``. Pass ``scorer_id`` (an LLM Judge
         Scorer's id, e.g. from ``client.monitor.judge_scorers``) to grade with a specific
         scorer instead of the dataset's default. ``evaluation_settings_id`` is the
-        pre-consolidation alias for the same id and keeps working."""
+        pre-consolidation alias for the same id and keeps working.
+
+        ``split`` runs only the cases tagged with that named subset (``add_case(...,
+        splits=["smoke"])``) - the cheap-PR-run vs nightly-full-run workflow. Original case
+        indexes are preserved so per-case comparisons line up with full runs."""
         from agentx.evaluations.client import _resolve_scorer_id
 
         if isinstance(subject, dict):
@@ -532,18 +632,24 @@ class EvaluationsRunner:
         evaluation_settings = (
             self._client.get_evaluation_settings(grader_id) if grader_id else None
         )
-        run = self._client.init_run(dataset_id, subject, scorer_id=grader_id)
+        run = self._client.init_run(dataset_id, subject, scorer_id=grader_id, split=split)
+        case_count = (
+            sum(1 for q in dataset.questions if split in (q.main_question.splits or []))
+            if split
+            else len(dataset.questions)
+        )
         logger.info(
-            "Started evaluation run %s on dataset %s (%d case(s), %d repetition(s))",
+            "Started evaluation run %s on dataset %s (%d case(s)%s, %d repetition(s))",
             run.run_id,
             dataset_id,
-            len(dataset.questions),
+            case_count,
+            f' in split "{split}"' if split else "",
             evaluation_settings.number_of_requests
             if evaluation_settings
             else dataset.number_of_requests,
         )
         return EvaluationRunContext(
-            self._client, dataset, run, subject, evaluation_settings=evaluation_settings
+            self._client, dataset, run, subject, evaluation_settings=evaluation_settings, split=split
         )
 
 
@@ -568,6 +674,7 @@ def _build_cases(
     dataset: Dataset,
     run: EvaluationRun,
     evaluation_settings: Optional[EvaluationSettings] = None,
+    split: Optional[str] = None,
 ) -> List[EvaluationCase]:
     cases: List[EvaluationCase] = []
     # When an independent evaluation_settings was chosen (evaluation_settings_id
@@ -596,6 +703,10 @@ def _build_cases(
     }
     for q_idx, question in enumerate(dataset.questions):
         mq = question.main_question
+        # Split filtering preserves q_idx: a "smoke" run's case 7 is the same case 7 a full run
+        # scores, so per-case comparisons line up across the two.
+        if split and split not in (mq.splits or []):
+            continue
         for run_num in range(1, n_runs + 1):
             for model in models:
                 suffix = f"::{model}" if model else ""

agentx/monitor/client.py

@@ -39,6 +39,35 @@ class AgentXValidationError(AgentXMonitorError):
     pass
 
 
+class CalibrationSummary(dict):
+    """Judge Calibration numbers (dict subclass, so existing key access keeps working).
+    Properties mirror the wire's exact camelCase keys."""
+
+    @property
+    def compared_count(self) -> int:
+        return int(self.get("comparedCount") or 0)
+
+    @property
+    def agreement_rate(self):
+        return self.get("agreementRate")
+
+    @property
+    def false_positive_rate(self):
+        return self.get("falsePositiveRate")
+
+    @property
+    def false_negative_rate(self):
+        return self.get("falseNegativeRate")
+
+    @property
+    def reported_count(self) -> int:
+        return int(self.get("reportedCount") or 0)
+
+    @property
+    def review_label_count(self) -> int:
+        return int(self.get("reviewLabelCount") or 0)
+
+
 class MonitorClient:
     """Low-level HTTP client for the Monitor API (``/monitor``). Accessed via
     ``client.monitor`` on the top-level :class:`agentx.AgentX` instance; most callers
@@ -82,6 +111,11 @@ class MonitorClient:
 
         self.patterns = MonitorPatternClient(self)
         self.signals = MonitorSignalClient(self)
+        from agentx.monitor.review_queue import ReviewQueueClient
+
+        # The human-review queue (list / queue / label / dismiss) - what makes the
+        # label-and-calibrate loop scriptable instead of dashboard-only.
+        self.review_queue = ReviewQueueClient(self)
         from agentx.monitor.scorers import ScorersClient
         # Scorers-catalog administration as code: template enable/disable, code/external scorer
         # CRUD and dry runs - full parity with the dashboard's Scorers page (P1.3).
@@ -230,15 +264,19 @@ class MonitorClient:
         plus deltas vs the prior window and the run-outcome breakdown."""
         return self._request("GET", "/kpis", params={"window": window})
 
-    def calibration(self, window: str = "7d") -> dict:
+    def calibration(self, window: str = "7d") -> "CalibrationSummary":
         """Project-level judge calibration over a window ("24h", "7d", or "30d"): how often
         AgentX's own verdicts agreed with real-world ground truth reported later (ops outcomes
-        via ``client.outcomes`` and end-user downvotes). Returns the dashboard's Judge
-        Calibration numbers: compared count, agreement, falsePositiveRate, falseNegativeRate.
-        Per-evaluator calibration lives on ``client.monitor.online_evaluators.calibration``."""
-        return self._request(
-            "GET", "/agent-monitoring/calibration",
-            base=self._api_root(), params={"window": window},
+        via ``client.outcomes``, end-user downvotes, and human review labels). Returns the
+        dashboard's Judge Calibration numbers with these exact keys: ``comparedCount``,
+        ``agreementRate``, ``falsePositiveRate``, ``falseNegativeRate`` (plus
+        ``reportedCount``/``reviewLabelCount``/``noVerdictCount``). Per-scorer calibration
+        lives on ``client.monitor.judge_scorers.calibration(scorer_id)``."""
+        return CalibrationSummary(
+            self._request(
+                "GET", "/agent-monitoring/calibration",
+                base=self._api_root(), params={"window": window},
+            )
         )
 
     # ------------------------------------------------------------------
@@ -352,10 +390,19 @@ class MonitorClient:
             base=self._api_root(), json={**criteria, "window": window}, timeout=600,
         )
 
-    def publish_online_evaluator_tuning(self, evaluator_id: str, criteria: dict) -> dict:
+    def publish_online_evaluator_tuning(
+        self, evaluator_id: str, criteria: dict, *, validation: Optional[dict] = None, force: bool = False
+    ) -> dict:
+        # The engine gates publish on validation provenance (and refuses a measured regression)
+        # unless forced - see judge_scorers.publish_tuning for the full story.
+        payload = dict(criteria)
+        if validation is not None:
+            payload["validation"] = validation
+        if force:
+            payload["force"] = True
         return self._request(
             "POST", f"/agent-monitoring/online-evaluators/{evaluator_id}/tune/publish",
-            base=self._api_root(), json=criteria, timeout=60,
+            base=self._api_root(), json=payload, timeout=60,
         )
 
     def update_profile(self, agent_id: str, payload: dict) -> MonitorProfile:

agentx/monitor/judge_scorers.py

@@ -280,12 +280,30 @@ class JudgeScorersClient:
             timeout=600,
         )
 
-    def publish_tuning(self, scorer_id: str, criteria: Dict[str, Any]) -> dict:
+    def publish_tuning(
+        self,
+        scorer_id: str,
+        criteria: Dict[str, Any],
+        *,
+        validation: Optional[Dict[str, Any]] = None,
+        force: bool = False,
+    ) -> dict:
         """Write tuned criteria onto the scorer's rubric - it applies everywhere the scorer is
-        used: online scoring, offline dataset runs, and the playground."""
-        return self._request(
-            "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune/publish", json=dict(criteria)
-        )
+        used: online scoring, offline dataset runs, and the playground.
+
+        The engine gates publish on provenance: pass ``validation`` (the dict returned by
+        ``validate_tuning``, or at least its ``verdict``/``netAgreementGain``) so the version
+        history records what the change measurably did; a ``regressed`` verdict is refused.
+        ``force=True`` publishes without (or despite) validation - deliberate escape hatch."""
+        payload = dict(criteria)
+        if validation is not None:
+            payload["validation"] = {
+                "verdict": validation.get("verdict"),
+                "netAgreementGain": validation.get("netAgreementGain"),
+            }
+        if force:
+            payload["force"] = True
+        return self._request("POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune/publish", json=payload)
 
     def ratings(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorRatingPoint]":
         """Bucketed average-rating-over-time for this scorer's live checks - same typed points

agentx/monitor/online_evaluators.py

@@ -157,9 +157,13 @@ class MonitorOnlineEvaluatorClient:
         is {acceptanceCriteria, rejectionCriteria, evaluationCriteria} from tune()."""
         return self._client.validate_online_evaluator_tuning(evaluator_id, criteria, window)
 
-    def publish_tuning(self, evaluator_id: str, criteria: dict) -> dict:
-        """Publish tuned criteria onto the evaluator's config (the human-approval step)."""
-        return self._client.publish_online_evaluator_tuning(evaluator_id, criteria)
+    def publish_tuning(
+        self, evaluator_id: str, criteria: dict, *, validation: Optional[dict] = None, force: bool = False
+    ) -> dict:
+        """Publish tuned criteria onto the evaluator's config (the human-approval step).
+        Pass ``validation`` (the ``validate_tuning`` result) - the engine refuses an unvalidated
+        publish, and a ``regressed`` verdict, unless ``force=True``."""
+        return self._client.publish_online_evaluator_tuning(evaluator_id, criteria, validation=validation, force=force)
 
     def ratings(self, evaluator_id: str, window: str = "7d") -> List[OnlineEvaluatorRatingPoint]:
         """Bucketed average-rating-over-time for this evaluator. ``window`` is one of

agentx/monitor/review_queue.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from agentx.monitor.client import MonitorClient
+
+logger = logging.getLogger(__name__)
+
+
+class ReviewQueueItem(dict):
+    """Wire object for one human-review item (dict subclass so unknown fields round-trip)."""
+
+    @property
+    def id(self) -> str:
+        return self["_id"]
+
+    @property
+    def trace_id(self) -> Optional[str]:
+        return self.get("traceId")
+
+    @property
+    def status(self) -> Optional[str]:
+        return self.get("status")
+
+    @property
+    def label(self) -> Optional[str]:
+        return self.get("label")
+
+    @property
+    def judge_score_at_queue(self) -> Optional[float]:
+        return self.get("judgeScoreAtQueue")
+
+
+class ReviewQueueClient:
+    """Surfaced as ``client.monitor.review_queue``: the human-review queue behind the dashboard's
+    Review tab, scriptable - so the label-and-calibrate loop (sample traces, label them
+    good/bad, optionally re-score) can run end to end from code. Labels feed judge calibration
+    and become judge-tuning evidence.
+
+    The engine refuses duplicates (409, a trace already pending) and a full queue (429, pending
+    cap reached); both surface as raised errors with the engine's reason.
+    """
+
+    def __init__(self, client: "MonitorClient"):
+        self._client = client
+
+    def list(self, status: Optional[str] = None, source: Optional[str] = None, limit: int = 100) -> List[ReviewQueueItem]:
+        """Queue items, newest first. ``status``: "pending" | "labeled" | "skipped" | "all"
+        (server default: pending). ``source``: "manual" | "rule" | "all"."""
+        params: Dict[str, Any] = {"limit": limit}
+        if status is not None:
+            params["status"] = status
+        if source is not None:
+            params["source"] = source
+        data = self._client._request("GET", "/agent-monitoring/review-queue", base=self._client._api_root(), params=params)
+        return [ReviewQueueItem(item) for item in data.get("items", [])]
+
+    def queue(self, trace_id: str, note: Optional[str] = None) -> ReviewQueueItem:
+        """Send a trace to human review (the SDK-side twin of the dashboard's "Send to review")."""
+        payload: Dict[str, Any] = {"traceId": trace_id, "source": "manual"}
+        if note:
+            payload["note"] = note
+        data = self._client._request("POST", "/agent-monitoring/review-queue", base=self._client._api_root(), json=payload)
+        return ReviewQueueItem(data.get("item", data))
+
+    def label(
+        self,
+        item_id: str,
+        label: str,
+        *,
+        corrected_score: Optional[float] = None,
+        note: Optional[str] = None,
+    ) -> ReviewQueueItem:
+        """Record the human verdict on a queued item. ``label`` is "good" or "bad";
+        ``corrected_score`` (0-10) optionally re-scores the judge's own rating for the trace -
+        the pair that calibration consumes."""
+        if label not in ("good", "bad"):
+            raise ValueError('label must be "good" or "bad"')
+        payload: Dict[str, Any] = {"label": label}
+        if corrected_score is not None:
+            payload["correctedScore"] = corrected_score
+        if note is not None:
+            payload["note"] = note
+        data = self._client._request(
+            "PATCH", f"/agent-monitoring/review-queue/{item_id}", base=self._client._api_root(), json=payload
+        )
+        return ReviewQueueItem(data.get("item", data))
+
+    def dismiss(self, item_id: str) -> None:
+        """Remove an item from the queue without a verdict (does not feed calibration)."""
+        self._client._request("DELETE", f"/agent-monitoring/review-queue/{item_id}", base=self._client._api_root())

agentx/testing.py

@@ -46,7 +46,9 @@ def _format_failures(gate: Any) -> str:
     for check in checks:
         get = check.get if isinstance(check, dict) else lambda k, d=None: getattr(check, k, d)
         status = "PASS" if get("passed") else "FAIL"
-        lines.append(f"  [{status}] {get('name', 'check')}: {get('detail', '')}")
+        # The engine names each check under the key "check" ("fail-under" / "no-regression");
+        # "name" is kept as a fallback for any older payload shape.
+        lines.append(f"  [{status}] {get('check') or get('name', 'check')}: {get('detail', '')}")
     average = getattr(gate, "average_rating", None)
     if average is not None:
         lines.append(f"  average rating: {average}")

agentx/version.py

@@ -1 +1 @@
-VERSION = "0.8.4"
+VERSION = "0.8.5"

{agentx_python-0.8.4.dist-info → agentx_python-0.8.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.4
+Version: 0.8.5
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.4.dist-info → agentx_python-0.8.5.dist-info}/RECORD

@@ -7,20 +7,20 @@ agentx/feedback.py,sha256=hMPU3ivUHvhOXOH7gvT4Jy1yztVmoIMOw1MbUNRs5Gk,3643
 agentx/outcomes.py,sha256=yCA6PrO5BrGufEjkzYWTK5wusAOt6w0gSKkH-5x1Fs4,3462
 agentx/projects.py,sha256=JN-nhesELPjEb1P0m9F4-Qw-0eGR68kCtBN5zNhArt0,2540
 agentx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-agentx/testing.py,sha256=VKry0lgqaeMyxy9aPuSV73LTUhRv0WcInVVWOdkz4e0,7194
+agentx/testing.py,sha256=0shZEid_vhJgBegpO_loUq6APYDxQrUgD3jvcRdIDV8,7372
 agentx/traces.py,sha256=sz9gxutlDKNgf0fdCkyzKsGwK5eMAJuIEl3NIwGrsuo,2287
 agentx/util.py,sha256=lt2Kpg4Fj8whrSuAhLEahcPEIYRVpK9RCC5y9ShdTJs,703
-agentx/version.py,sha256=IExNVK5GJVnQo4b-FGsI7pTg9u4gzQTgCzUF5lFpZ_I,18
+agentx/version.py,sha256=EG17K_2R-skew5or95Q31PUAkXCElZYytsIaAs13hLI,18
 agentx/evaluations/__init__.py,sha256=Erv7RGFlRxGTG4rVb2uHhCqLLEX_iAWKqkZKzK6CumE,262
 agentx/evaluations/_term.py,sha256=WFpiNzdgDBeJJ-Gg-6X7TwwxllobuE4OqFUTuDQvS3Y,2529
-agentx/evaluations/client.py,sha256=1MsduXHOSkwS40RvbUpNgGMU4pZvFtChqlU82YqKdOw,30028
-agentx/evaluations/datasets.py,sha256=7OYj5pXpJenViAhPe7Sr_7Sf9KWd7b1OhV-dPL2tso8,14283
+agentx/evaluations/client.py,sha256=MjSBoCXDZPlNHLvlPg4Q5veTIMauU7E7SMtBONqczQk,30876
+agentx/evaluations/datasets.py,sha256=7l-Isy4e7Ud9l37vWSkZSnTh3wnRMdAk0sFKnVTBv1Y,14751
 agentx/evaluations/evaluation_settings.py,sha256=geRydHqKpxwkxBVfMIOqkB6NlStyOZrYwZnMHluDwvA,6276
-agentx/evaluations/models.py,sha256=b8cp0arikLW6tZqzmRs-4DVqgFs9FSR5DU9LleO82VY,31699
+agentx/evaluations/models.py,sha256=4z1efp6mNdIVBtm95Lelbn0zugtfLXXCnrYD3j9KjZ4,31902
 agentx/evaluations/prompts.py,sha256=8xyvMpAl3mD5xuSZozuaohdnCBWe5Q02RldwiPLUO20,3493
 agentx/evaluations/reporting.py,sha256=GtNnL-1eNEQrSqd0yrHi5mGqC9_u0kUAN8Mbjdhw6yY,6389
 agentx/evaluations/results.py,sha256=w7TRUik4eXIqrznXvlF_L9hnObnHvgdRKvihfN1XeAU,4692
-agentx/evaluations/runner.py,sha256=6k-fUYAwf_sbxSWlirQzI7D60ODjSfvGUKeLXnSKD_Q,27947
+agentx/evaluations/runner.py,sha256=qhaUiUaP_D5etjRnsuR71-uQGnwF_8J4LSPQ2lMWTU4,33882
 agentx/evaluations/tool_schemas.py,sha256=ZyrnSOnx9xlSAj4swfDey9ZLS2ly-g21DByTwRAGwNA,2323
 agentx/evaluations/tracing.py,sha256=MSJD9bzfInMi7E70mc8mWUc0fx5DLkOgUzRru2mNQoc,1825
 agentx/evaluations/adapters/__init__.py,sha256=fK8Hx75usbiY03XUSmnLnSrwBXip2CQs24YZcdEgj6w,287
@@ -43,12 +43,13 @@ agentx/integrations/openai.py,sha256=1KLs-hJaeW2emHPwNkmC3zcGksL3zd87uWbcLGRW704
 agentx/integrations/openai_agents.py,sha256=tCa3kPxqux5LnNNDQ6zA05OybQrjkc_LeE08YuNPJEs,12742
 agentx/monitor/__init__.py,sha256=MLiASSvQnCy2jqVNRTgPWFhUSf8lGYRQeEgEVVhE5Tg,574
 agentx/monitor/agents.py,sha256=8Xk4jWmNTvdtjFixHiLmw4jj96uqIXWIT6KW8zXoWrc,1006
-agentx/monitor/client.py,sha256=BXaTDAsKJf_3uwrhZMKn4UqAz5Xr0qap-MH9a-CR7d8,16655
-agentx/monitor/judge_scorers.py,sha256=40Xq60QASLW9z2QT2dSTmZhILGeBTfFGLrog5yeessM,14261
+agentx/monitor/client.py,sha256=PNhtuBRrJvrtgA3gjm4ZN7dpb0BFTi-samNCo15hr2M,18336
+agentx/monitor/judge_scorers.py,sha256=Mu6OOnfjp1SRkZMcOyriO7zPawv-O2H2l2X-0io5WLM,15025
 agentx/monitor/models.py,sha256=yTC3WTdTziHMkdeAXlTro-4YiSbBUHMgf9kNZHGfPDA,8470
-agentx/monitor/online_evaluators.py,sha256=tTXqRgOb6ah39c5nWMEVXLxEUc8Tqu9E9SFdcHy0ExM,8418
+agentx/monitor/online_evaluators.py,sha256=YWnsBmlIbYvj4LCaX4guwx4HS5y3oEzCQfo6ae9VH_w,8693
 agentx/monitor/patterns.py,sha256=ruBqLoA2T2KQuqRbbn1kWkIEkiOal3SvyPl9opGyhzI,4185
 agentx/monitor/profile.py,sha256=uGk-4bXT2vmnEQiY5u6rWhAfzZchHBXL74VD3piVHvU,3102
+agentx/monitor/review_queue.py,sha256=Tr3GDMJLJzyFKEEeB8HPmZBsNzzYVJYElDlfBC4qd68,3766
 agentx/monitor/scorers.py,sha256=huIZBszrfLKI8I7VV6NFpW6piRsj9HI03rXUwWs2tgs,7336
 agentx/monitor/sessions.py,sha256=gePcL284m3UITxQAtHxwrKFl6bRjmRib1QK6JbZeS9s,815
 agentx/monitor/signals.py,sha256=Ld11lW2lbg8NHBUHdgoY6iwCvJDK-eg4ge4qQ9y7nGY,1432
@@ -61,9 +62,9 @@ agentx/tracing/ci_types.py,sha256=zFVcZGvc1qbVDdOlEPQ1i5R6terqJHxCzFF4ZjouxxI,14
 agentx/tracing/eval_scope.py,sha256=ElMbPxpqpQBVuaUnHNlw9RIQu71oyOpd0yR55bDH8eI,2144
 agentx/tracing/ingest_client.py,sha256=5X6RyxtMq1XYgOzsMd3b2f2qpmYpsMnhKm7NB3oup00,17664
 agentx/tracing/tracer.py,sha256=qQojU-T4O8IJKyf9vue0Em66qPBTZKl3FMyr3KJKPnI,53770
-agentx_python-0.8.4.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
-agentx_python-0.8.4.dist-info/METADATA,sha256=g4P1EOltG3-fpNZuo9lmVGA3UhcFuxnYQxj1l-57X44,20992
-agentx_python-0.8.4.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-agentx_python-0.8.4.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
-agentx_python-0.8.4.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
-agentx_python-0.8.4.dist-info/RECORD,,
+agentx_python-0.8.5.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
+agentx_python-0.8.5.dist-info/METADATA,sha256=WHqqeAmz7UcuecsbLSkGeN_8lGDLSQJa1EweRpJrLWc,20992
+agentx_python-0.8.5.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+agentx_python-0.8.5.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
+agentx_python-0.8.5.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
+agentx_python-0.8.5.dist-info/RECORD,,