agentx-python 0.8.15__tar.gz → 0.8.17__tar.gz

{agentx_python-0.8.15/agentx_python.egg-info → agentx_python-0.8.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.15
+Version: 0.8.17
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/evaluations/client.py

@@ -314,10 +314,15 @@ class EvaluationsClient:
         scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
         split: Optional[str] = None,
+        additional_scorer_ids: Optional[List[str]] = None,
+        scorer_group_id: Optional[str] = None,
     ) -> EvaluationRun:
         """``scorer_id`` names the LLM Judge Scorer grading this run (its id doubles as the
         wire's ``evaluationSettingsId``). ``evaluation_settings_id`` is the pre-consolidation
-        alias and keeps working. ``split`` records the named case subset this run covers."""
+        alias and keeps working. ``split`` records the named case subset this run covers.
+        ``additional_scorer_ids`` (self-host): extra judge scorers that each pass their own
+        verdict on every result from the same single agent execution - verdicts land in each
+        result row's ``judgeScorerResults`` and the run's ``scorerBreakdown``."""
         from agentx.version import VERSION
 
         grader_id = _resolve_scorer_id(scorer_id, evaluation_settings_id)
@@ -335,6 +340,12 @@ class EvaluationsClient:
         }
         if grader_id:
             payload["evaluationSettingsId"] = grader_id
+        if additional_scorer_ids:
+            payload["additionalScorerIds"] = additional_scorer_ids
+        # Scorer group grading (self-host): the group's weighted 0-10 aggregate fills the rating
+        # column and member verdicts land per row. Mutually exclusive with scorer_id (group wins).
+        if scorer_group_id:
+            payload["scorerGroupId"] = scorer_group_id
         if split:
             payload["split"] = split
         data = self._request("POST", "/runs", json=self._with_workspace(payload))
@@ -372,6 +383,7 @@ class EvaluationsClient:
         tolerance: Optional[float] = None,
         record: bool = True,
         caller: Optional[str] = "sdk",
+        scorer: Optional[str] = None,
     ) -> Dict[str, Any]:
         # CI gate (self-host): pass/fail a finalized run against an absolute rating floor and/or
         # the dataset's previous completed run. Recorded into gate history by default (the
@@ -389,6 +401,11 @@ class EvaluationsClient:
             params["record"] = "true"
             if caller:
                 params["caller"] = caller
+        # Multi-judge runs (self-host): gate a named additional scorer (id or name, e.g.
+        # scorer="Safety") instead of the primary - failUnder/noRegression then use that
+        # scorer's own per-result verdicts. Unknown names are a hard 400 from the engine.
+        if scorer:
+            params["scorer"] = scorer
         return self._request("GET", f"/runs/{run_id}/gate", params=params)
 
     def analyze_run(

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/evaluations/datasets.py

@@ -48,7 +48,7 @@ class DatasetBuilder:
             "questions": [],
         }
         # LLM-as-judge overrides for this dataset's own grading config. Omit either to keep the
-        # server default (raw prompt template / OpenAI gpt-5.5, see EVALUATIONS.md). judge_model
+        # server default (raw prompt template / gpt-5.6-luna, see EVALUATIONS.md). judge_model
         # must be one of client.evaluations.list_models() (OpenAI or Anthropic).
         if judge_prompt is not None:
             self._payload["judgePrompt"] = judge_prompt

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/evaluations/evaluation_settings.py

@@ -45,7 +45,7 @@ class EvaluationSettingsBuilder:
             "evaluationCriteria": evaluation_criteria,
         }
         # LLM-as-judge overrides. Omit either to keep the server default (raw prompt template /
-        # OpenAI gpt-5.5, see EVALUATIONS.md). judge_model must be one of
+        # gpt-5.6-luna, see EVALUATIONS.md). judge_model must be one of
         # client.evaluations.list_models() (OpenAI or Anthropic).
         if judge_prompt is not None:
             self._payload["judgePrompt"] = judge_prompt

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/evaluations/models.py

@@ -75,6 +75,9 @@ class Dataset(BaseModel):
     rejection_criteria: Optional[str] = Field(default=None, alias="rejectionCriteria")
     evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria")
     questions: List[DatasetQuestion] = Field(default_factory=list)
+    # Custom code scorers attached to this dataset - [{ id, name, code, enabled }]. Retrievable,
+    # so a fetched dataset round-trips them (import_dataset copies them to the new dataset).
+    code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers")
     status: str = "published"
     version_id: Optional[str] = Field(default=None, alias="versionId")
     # Sovereignty & Portability - models selected to compare on this dataset.
@@ -108,8 +111,10 @@ class EvaluationSettings(BaseModel):
     acceptance_criteria: Optional[str] = Field(default=None, alias="acceptanceCriteria")
     rejection_criteria: Optional[str] = Field(default=None, alias="rejectionCriteria")
     evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria")
-    # LLM-as-judge overrides. None means "use the server default" (raw prompt template / OpenAI
-    # gpt-5.5). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...).
+    # Custom code scorers on this scorer's offline profile - [{ id, name, code, enabled }].
+    code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers")
+    # LLM-as-judge overrides. None means "use the server default" (raw prompt template /
+    # gpt-5.6-luna). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...).
     judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt")
     judge_model: Optional[str] = Field(default=None, alias="judgeModel")
     status: str = "published"
@@ -435,6 +440,8 @@ class RunResultRow(BaseModel):
     bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
     rouge_score: Optional[float] = Field(default=None, alias="rougeScore")
     code_scorer_results: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorerResults")
+    # Verdicts from the run's ADDITIONAL judge scorers: [{scorerId, name, rating, justification}].
+    judge_scorer_results: Optional[List[Dict[str, Any]]] = Field(default=None, alias="judgeScorerResults")
     raw: Dict[str, Any] = Field(default_factory=dict)
 
     class Config:

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/evaluations/runner.py

@@ -63,9 +63,6 @@ _ANALYSIS_LEVEL_LABELS = {
     "l4_final_reduce": "writing final report",
 }
 
-_DEFAULT_JUDGE_MODEL = "gpt-5.5"
-
-
 class GateResult:
     """Wire result of the CI gate (GET /runs/:id/gate) with attribute access for the fields a
     CI script actually branches on."""
@@ -77,6 +74,9 @@ class GateResult:
         self.baseline_average: Optional[float] = data.get("baselineAverage")
         self.baseline_run_id: Optional[str] = data.get("baselineRunId")
         self.checks: List[Dict[str, Any]] = data.get("checks", [])
+        # Multi-judge runs: {"id", "name"} of the additional scorer being gated when the gate
+        # ran with scorer=..., None when gating the primary.
+        self.gated_scorer: Optional[Dict[str, Any]] = data.get("gatedScorer")
 
     @property
     def exit_code(self) -> int:
@@ -354,13 +354,16 @@ class EvaluationRunContext:
         no_regression: bool = False,
         tolerance: Optional[float] = None,
         caller: str = "sdk",
+        scorer: Optional[str] = None,
     ) -> "GateResult":
         """CI gate (self-host): pass/fail this finalized run so a CI job can block a merge.
 
         ``fail_under`` fails the gate when the run's average rating is below the floor;
         ``no_regression=True`` fails it when the average dropped more than ``tolerance``
         (default 0.5, judge scores are noisy) below the dataset's previous completed run.
-        At least one check is required. Prints a CI-log-friendly verdict and returns a
+        At least one check is required. On a multi-judge run, ``scorer`` (an additional
+        scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average
+        instead of the primary's - "fail if Safety is low even when the average looks fine". Prints a CI-log-friendly verdict and returns a
         :class:`GateResult` - the caller decides the exit code::
 
             report = client.evaluations.run(...).execute(my_agent).finalize()
@@ -374,6 +377,7 @@ class EvaluationRunContext:
             no_regression=no_regression,
             tolerance=tolerance,
             caller=caller,
+            scorer=scorer,
         )
         result = GateResult(data)
         _say()
@@ -449,16 +453,16 @@ class EvaluationRunContext:
         Args:
             mode: "auto" (default), "sync", or "batch" - how item scoring executes server-side.
             quality_mode: "quality_first" or "balanced" - how many items get a second/third judge.
-            judges: 1-3 model ids, e.g. ``["gpt-5.5", "claude-opus-4-8"]``. Defaults to a single
-                judge, ``["gpt-5.5"]``, rather than the dashboard's 3-judge default - SDK runs are
-                typically lighter-weight, quick-start evaluations.
+            judges: 1-3 model ids from ``client.evaluations.list_models()``, e.g.
+                ``["gpt-5.6-luna", "claude-opus-4-8"]``. Omit to let the engine score with its
+                platform default model (a single judge, rather than the dashboard's 3-judge
+                default - SDK runs are typically lighter-weight, quick-start evaluations).
             poll_interval: seconds between status checks while waiting.
             timeout: give up waiting after this many seconds (the job keeps running server-side;
                 call ``get_report()`` later to check on it).
         """
         if judges is not None and not (1 <= len(judges) <= 3):
             raise ValueError("judges must contain 1-3 model ids")
-        resolved_judges = judges if judges is not None else [_DEFAULT_JUDGE_MODEL]
 
         _say()
         with Spinner("Analyzing - AI is reviewing your results") as spinner:
@@ -467,7 +471,7 @@ class EvaluationRunContext:
                     self._run.run_id,
                     mode=mode,
                     quality_mode=quality_mode,
-                    judges=resolved_judges,
+                    judges=judges,
                 )
                 deadline = time.monotonic() + timeout
                 status = self._client.get_analysis_status(self._run.run_id)
@@ -589,6 +593,7 @@ class EvaluationsRunner:
         tolerance: Optional[float] = None,
         record: bool = True,
         caller: Optional[str] = "sdk",
+        scorer: Optional[str] = None,
     ) -> GateResult:
         """CI-gate any finalized run by id - the standalone form of
         ``EvaluationRunContext.gate()``, for gating a run created elsewhere or
@@ -603,6 +608,7 @@ class EvaluationsRunner:
                 tolerance=tolerance,
                 record=record,
                 caller=caller,
+                scorer=scorer,
             )
         )
 
@@ -613,6 +619,8 @@ class EvaluationsRunner:
         scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
         split: Optional[str] = None,
+        additional_scorer_ids: Optional[List[str]] = None,
+        scorer_group_id: Optional[str] = None,
     ) -> EvaluationRunContext:
         """Start a run of ``dataset_id`` against ``subject``. Pass ``scorer_id`` (an LLM Judge
         Scorer's id, e.g. from ``client.monitor.judge_scorers``) to grade with a specific
@@ -632,7 +640,14 @@ class EvaluationsRunner:
         evaluation_settings = (
             self._client.get_evaluation_settings(grader_id) if grader_id else None
         )
-        run = self._client.init_run(dataset_id, subject, scorer_id=grader_id, split=split)
+        run = self._client.init_run(
+            dataset_id,
+            subject,
+            scorer_id=grader_id,
+            split=split,
+            additional_scorer_ids=additional_scorer_ids,
+            scorer_group_id=scorer_group_id,
+        )
         case_count = (
             sum(1 for q in dataset.questions if split in (q.main_question.splits or []))
             if split

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/monitor/client.py

@@ -147,6 +147,11 @@ class MonitorClient:
         # surface that matches the product; evaluations.settings and online_evaluators below
         # remain as its profile-level views.
         self.judge_scorers = JudgeScorersClient(api_key=api_key, base_url=self._api_root())
+        from agentx.monitor.scorer_groups import ScorerGroupsClient
+
+        # Scorer groups: mixed-kind scorers composed into one 0-10 score (weights + must-pass
+        # gates) - a group grades dataset runs (scorer_group_id) and, when online, live traffic.
+        self.scorer_groups = ScorerGroupsClient(api_key=api_key, base_url=self._api_root())
         from agentx.monitor.improvement_groups import ImprovementGroupsClient
 
         # Auto-improve: confirmed production failures -> improvement report -> code fix (via
@@ -424,6 +429,24 @@ class MonitorClient:
         data = self._request("GET", f"/ingest/sessions/{session_id}/spans", base=self._api_root())
         return data.get("spans", []) if isinstance(data, dict) else data
 
+    def list_session_scores(self, session_id: str) -> List[dict]:
+        """Every session-level verdict on the session, newest first: session-scoped online
+        evaluators (kind ``online-eval:<id>``), session-scoped scorer groups
+        (``scorer-group:<id>``), and legacy coherence rows."""
+        data = self._request(
+            "GET", f"/agent-monitoring/sessions/{session_id}/scores", base=self._api_root()
+        )
+        return data.get("scores", []) if isinstance(data, dict) else data
+
+    def run_session_sweep(self) -> dict:
+        """Run the idle-session sweep once, now - the tick that scores quiet multi-turn
+        sessions with every enabled session-scoped evaluator and scorer group. Production
+        engines run this automatically every minute; the manual trigger exists for demos,
+        tests, and backfills. Returns ``{"judged": n}``."""
+        return self._request(
+            "POST", "/agent-monitoring/session-sweep/run", base=self._api_root(), timeout=300
+        )
+
     # ------------------------------------------------------------------
     # Model portability (self-host): replay a trace's input against other models
     # ------------------------------------------------------------------

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/monitor/judge_scorers.py

@@ -36,6 +36,11 @@ class JudgeScorer(dict):
     def offline(self) -> Dict[str, Any]:
         return self.get("offline", {})
 
+    @property
+    def code_scorers(self) -> List[Dict[str, Any]]:
+        """Custom code scorers on the offline profile - [{ id, name, code, enabled }]."""
+        return list(self.offline.get("codeScorers") or [])
+
     @property
     def online(self) -> Optional[Dict[str, Any]]:
         return self.get("online")

agentx_python-0.8.17/agentx/monitor/scorer_groups.py

@@ -0,0 +1,90 @@
+"""Scorer groups (self-host): scorers of any kind - LLM judges, patterns, custom code/external
+scorers - composed into ONE 0-10 score via per-member weights and optional must-pass gates.
+Members are references: ``{"kind": "judge" | "pattern" | "custom", "refId": ..., "weight": ...,
+"gate": ...}``. Grade a dataset run with a group by passing its id as ``scorer_group_id`` to
+``client.evaluations.run(...)``; give it an ``online`` profile to score sampled live traffic and
+raise Signals below the alert threshold."""
+
+from typing import Any, Dict, List, Optional
+
+import requests
+
+
+class AgentXScorerGroupsError(Exception):
+    pass
+
+
+class ScorerGroup(dict):
+    """Wire object (dict subclass so unknown fields round-trip)."""
+
+    @property
+    def id(self) -> str:
+        return self["_id"]
+
+    @property
+    def name(self) -> str:
+        return self["name"]
+
+    @property
+    def members(self) -> List[Dict[str, Any]]:
+        return list(self.get("members") or [])
+
+    @property
+    def online(self) -> Optional[Dict[str, Any]]:
+        return self.get("online")
+
+
+class ScorerGroupsClient:
+    def __init__(self, api_key: str, base_url: str):
+        self._api_key = api_key
+        self._base = base_url.rstrip("/") + "/agent-monitoring/scorer-groups"
+
+    def _request(self, method: str, url: str, json: Optional[Dict[str, Any]] = None) -> Any:
+        response = requests.request(
+            method,
+            url,
+            headers={"x-api-key": self._api_key, "content-type": "application/json"},
+            json=json,
+            timeout=30,
+        )
+        if response.status_code >= 400:
+            raise AgentXScorerGroupsError(f"HTTP {response.status_code}: {response.text}")
+        return response.json()
+
+    def list(self) -> List[ScorerGroup]:
+        return [ScorerGroup(g) for g in self._request("GET", self._base).get("scorerGroups", [])]
+
+    def get(self, group_id: str) -> ScorerGroup:
+        return ScorerGroup(self._request("GET", f"{self._base}/{group_id}")["scorerGroup"])
+
+    def create(
+        self,
+        name: str,
+        members: List[Dict[str, Any]],
+        description: Optional[str] = None,
+        online: Optional[Dict[str, Any]] = None,
+    ) -> ScorerGroup:
+        """``members``: [{"kind": "judge"|"pattern"|"custom", "refId": ..., "weight": 1, "gate": False}].
+        ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"}.
+        Add ``"scope": "session", "idleSeconds": 120`` to score whole multi-turn sessions once
+        idle, instead of each sampled trace.
+        or None for offline-only."""
+        payload: Dict[str, Any] = {"name": name, "members": members}
+        if description is not None:
+            payload["description"] = description
+        if online is not None:
+            payload["online"] = online
+        return ScorerGroup(self._request("POST", self._base, json=payload)["scorerGroup"])
+
+    def update(self, group_id: str, **fields: Any) -> ScorerGroup:
+        """Sparse update - pass any of name/description/members/online (online=None detaches
+        live scoring)."""
+        return ScorerGroup(self._request("PUT", f"{self._base}/{group_id}", json=fields)["scorerGroup"])
+
+    def delete(self, group_id: str) -> None:
+        self._request("DELETE", f"{self._base}/{group_id}")
+
+    def ratings(self, group_id: str, window: str = "7d") -> Dict[str, Any]:
+        """Live score history for a group - ``{"window", "points": [{ts, averageRating, count}]}``,
+        the same shape online-evaluator ratings use. ``window``: "24h" | "7d" | "30d"."""
+        return self._request("GET", f"{self._base}/{group_id}/ratings?window={window}")

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/monitor/sessions.py

@@ -19,3 +19,15 @@ class MonitorSessionClient:
     def spans(self, session_id: str) -> List[dict]:
         """Every span in the session (roots and children), oldest first."""
         return self._client.list_session_spans(session_id)
+
+    def scores(self, session_id: str) -> List[dict]:
+        """Session-level verdicts, newest first. ``kind`` says who scored: a session-scoped
+        online evaluator (``online-eval:<id>``) or a session-scoped scorer group
+        (``scorer-group:<id>``)."""
+        return self._client.list_session_scores(session_id)
+
+    def run_sweep(self) -> dict:
+        """Trigger the idle-session sweep once (normally automatic, every minute) - scores
+        idle multi-turn sessions with every enabled session-scoped evaluator and scorer
+        group. Returns ``{"judged": n}``."""
+        return self._client.run_session_sweep()

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx/version.py

@@ -1,7 +1,7 @@
-VERSION = "0.8.15"
+VERSION = "0.8.17"
 
 # The AgentX-trace-eval release this SDK version is tested against - what `agentx-trace-eval`
 # installs and converges to (see agentx/cli.py). Bump together with VERSION when releasing, so
 # every published SDK names a known-good engine+dashboard pair. Users can override with
 # AGENTX_TRACE_EVAL_VERSION=<tag|latest>.
-ENGINE_VERSION = "v0.3.11"
+ENGINE_VERSION = "v0.3.16"

{agentx_python-0.8.15 → agentx_python-0.8.17/agentx_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.15
+Version: 0.8.17
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.15 → agentx_python-0.8.17}/agentx_python.egg-info/SOURCES.txt

@@ -55,6 +55,7 @@ agentx/monitor/patterns.py
 agentx/monitor/profile.py
 agentx/monitor/review_queue.py
 agentx/monitor/rules.py
+agentx/monitor/scorer_groups.py
 agentx/monitor/scorers.py
 agentx/monitor/sessions.py
 agentx/monitor/signals.py
@@ -83,6 +84,7 @@ tests/test_framework_detect.py
 tests/test_integration.py
 tests/test_integrations.py
 tests/test_judge_scorers.py
+tests/test_multi_judge.py
 tests/test_pairwise.py
 tests/test_review_queue.py
 tests/test_runner_features.py

{agentx_python-0.8.15 → agentx_python-0.8.17}/tests/test_integrations.py

@@ -599,7 +599,9 @@ def test_crewai_captures_real_per_task_timing_via_event_bus():
     durations are provably unequal — the old "divide latency evenly across
     tasks" approximation would have reported them as identical.
     """
-    crewai = pytest.importorskip("crewai")
+    # exc_type=Exception: on older Pythons crewai can raise TypeError (PEP 604 syntax) at
+    # import time, and a broken optional integration should skip this test, not fail it.
+    crewai = pytest.importorskip("crewai", exc_type=Exception)
     from crewai.events.event_bus import crewai_event_bus
     from crewai.events.types.task_events import TaskCompletedEvent, TaskStartedEvent
     from crewai.tasks.task_output import TaskOutput

{agentx_python-0.8.15 → agentx_python-0.8.17}/tests/test_judge_scorers.py

@@ -268,3 +268,34 @@ def test_validate_and_publish_send_criteria_at_top_level(monkeypatch):
     assert captured["validate"]["window"] == "24h"
     assert "criteria" not in captured["validate"]
     assert captured["publish"]["acceptanceCriteria"] == "a"
+
+
+def test_code_scorers_are_retrievable_from_the_wire_object():
+    """The wire rows may lack ids (SDK-created scorers) - retrieval must hand them back as-is."""
+    from agentx.monitor.judge_scorers import JudgeScorer
+
+    scorer = JudgeScorer(
+        {
+            "_id": "s1",
+            "name": "Blend",
+            "offline": {"codeScorers": [{"name": "Final score", "code": "return 1;", "enabled": True}]},
+        }
+    )
+    assert scorer.code_scorers == [{"name": "Final score", "code": "return 1;", "enabled": True}]
+    # And an offline profile without any stays an empty list, not a KeyError.
+    assert JudgeScorer({"_id": "s2", "name": "Plain", "offline": {}}).code_scorers == []
+
+
+def test_dataset_model_round_trips_code_scorers():
+    """extra="ignore" used to silently drop codeScorers on read - import_dataset lost them."""
+    from agentx.evaluations.models import Dataset
+
+    wire = {
+        "_id": "d1",
+        "name": "Guarded",
+        "questions": [],
+        "codeScorers": [{"id": "cs1", "name": "gate", "code": "return 0;", "enabled": True}],
+    }
+    parsed = Dataset(**wire)
+    assert parsed.code_scorers == wire["codeScorers"]
+    assert parsed.model_dump(by_alias=True)["codeScorers"] == wire["codeScorers"]

agentx_python-0.8.17/tests/test_multi_judge.py

@@ -0,0 +1,76 @@
+"""Wire-level tests for multi-judge dataset runs: additional_scorer_ids on run creation and
+the named-scorer CI gate (gate_run(scorer=...)). The session/HTTP layer is monkeypatched; the
+engine-side behavior is pinned by the engine's multiJudge.integration.test.ts."""
+
+from typing import Any, Dict, List
+
+import pytest
+
+from agentx.evaluations.client import EvaluationsClient
+from agentx.evaluations.models import EvaluationSubject
+
+
+class FakeResponse:
+    def __init__(self, payload: Dict[str, Any], status_code: int = 200):
+        self._payload = payload
+        self.status_code = status_code
+        self.ok = status_code < 400
+        self.text = "x"
+
+    def json(self) -> Dict[str, Any]:
+        return self._payload
+
+
+@pytest.fixture()
+def recorded(monkeypatch):
+    calls: List[Dict[str, Any]] = []
+
+    def fake_request(method, url, timeout=None, **kwargs):
+        calls.append({"method": method, "url": url, **kwargs})
+        return FakeResponse(
+            {
+                "runId": "r1",
+                "datasetId": "ds1",
+                "status": "in_progress",
+                "passed": True,
+                "checks": [],
+                "gatedScorer": None,
+            }
+        )
+
+    client = EvaluationsClient(api_key="k", base_url="http://engine:4700/api/v1")
+    monkeypatch.setattr(client._session, "request", fake_request)
+    return client, calls
+
+
+def test_init_run_sends_additional_scorer_ids_camel_case(recorded):
+    client, calls = recorded
+    client.init_run(
+        "ds1",
+        EvaluationSubject(kind="custom_agent"),
+        scorer_id="primary",
+        additional_scorer_ids=["safety", "tone"],
+    )
+    payload = calls[0]["json"]
+    assert payload["evaluationSettingsId"] == "primary"
+    assert payload["additionalScorerIds"] == ["safety", "tone"]
+
+
+def test_init_run_omits_the_key_when_no_additional_scorers(recorded):
+    client, calls = recorded
+    client.init_run("ds1", EvaluationSubject(kind="custom_agent"), scorer_id="primary")
+    assert "additionalScorerIds" not in calls[0]["json"]
+
+
+def test_gate_run_forwards_the_named_scorer(recorded):
+    client, calls = recorded
+    client.gate_run("r1", fail_under=5, scorer="Safety")
+    params = calls[0]["params"]
+    assert params["failUnder"] == 5
+    assert params["scorer"] == "Safety"
+
+
+def test_gate_run_leaves_scorer_off_for_primary_gates(recorded):
+    client, calls = recorded
+    client.gate_run("r1", fail_under=5)
+    assert "scorer" not in calls[0]["params"]

{agentx_python-0.8.15 → agentx_python-0.8.17}/tests/test_selfhost_analysis_fallback.py

@@ -210,12 +210,12 @@ def test_the_fallback_request_gets_the_long_analysis_timeout():
         {("POST", f"{API_ROOT}/evaluate/analyze/{RUN}"): FakeResponse(200, {"status": "completed"})}
     )
 
-    client.analyze_run(RUN, judges=["gpt-5.5"])
+    client.analyze_run(RUN, judges=["gpt-5.6-luna"])
 
     method, url, kwargs = session.calls[-1]
     assert url == f"{API_ROOT}/evaluate/analyze/{RUN}"
     assert kwargs["timeout"] > 60, "a synchronous judge pass needs more than the 30s default"
-    assert kwargs["json"]["judges"] == [{"model": "gpt-5.5"}]
+    assert kwargs["json"]["judges"] == [{"model": "gpt-5.6-luna"}]
 
 
 # ---------------------------------------------------------------------------