agentx-python 0.8.16__py3-none-any.whl → 0.8.17__py3-none-any.whl

agentx/evaluations/datasets.py

@@ -48,7 +48,7 @@ class DatasetBuilder:
             "questions": [],
         }
         # LLM-as-judge overrides for this dataset's own grading config. Omit either to keep the
-        # server default (raw prompt template / OpenAI gpt-5.5, see EVALUATIONS.md). judge_model
+        # server default (raw prompt template / gpt-5.6-luna, see EVALUATIONS.md). judge_model
         # must be one of client.evaluations.list_models() (OpenAI or Anthropic).
         if judge_prompt is not None:
             self._payload["judgePrompt"] = judge_prompt

agentx/evaluations/evaluation_settings.py

@@ -45,7 +45,7 @@ class EvaluationSettingsBuilder:
             "evaluationCriteria": evaluation_criteria,
         }
         # LLM-as-judge overrides. Omit either to keep the server default (raw prompt template /
-        # OpenAI gpt-5.5, see EVALUATIONS.md). judge_model must be one of
+        # gpt-5.6-luna, see EVALUATIONS.md). judge_model must be one of
         # client.evaluations.list_models() (OpenAI or Anthropic).
         if judge_prompt is not None:
             self._payload["judgePrompt"] = judge_prompt

agentx/evaluations/models.py

@@ -113,8 +113,8 @@ class EvaluationSettings(BaseModel):
     evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria")
     # Custom code scorers on this scorer's offline profile - [{ id, name, code, enabled }].
     code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers")
-    # LLM-as-judge overrides. None means "use the server default" (raw prompt template / OpenAI
-    # gpt-5.5). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...).
+    # LLM-as-judge overrides. None means "use the server default" (raw prompt template /
+    # gpt-5.6-luna). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...).
     judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt")
     judge_model: Optional[str] = Field(default=None, alias="judgeModel")
     status: str = "published"

agentx/evaluations/runner.py

@@ -63,9 +63,6 @@ _ANALYSIS_LEVEL_LABELS = {
     "l4_final_reduce": "writing final report",
 }
 
-_DEFAULT_JUDGE_MODEL = "gpt-5.5"
-
-
 class GateResult:
     """Wire result of the CI gate (GET /runs/:id/gate) with attribute access for the fields a
     CI script actually branches on."""
@@ -456,16 +453,16 @@ class EvaluationRunContext:
         Args:
             mode: "auto" (default), "sync", or "batch" - how item scoring executes server-side.
             quality_mode: "quality_first" or "balanced" - how many items get a second/third judge.
-            judges: 1-3 model ids, e.g. ``["gpt-5.5", "claude-opus-4-8"]``. Defaults to a single
-                judge, ``["gpt-5.5"]``, rather than the dashboard's 3-judge default - SDK runs are
-                typically lighter-weight, quick-start evaluations.
+            judges: 1-3 model ids from ``client.evaluations.list_models()``, e.g.
+                ``["gpt-5.6-luna", "claude-opus-4-8"]``. Omit to let the engine score with its
+                platform default model (a single judge, rather than the dashboard's 3-judge
+                default - SDK runs are typically lighter-weight, quick-start evaluations).
             poll_interval: seconds between status checks while waiting.
             timeout: give up waiting after this many seconds (the job keeps running server-side;
                 call ``get_report()`` later to check on it).
         """
         if judges is not None and not (1 <= len(judges) <= 3):
             raise ValueError("judges must contain 1-3 model ids")
-        resolved_judges = judges if judges is not None else [_DEFAULT_JUDGE_MODEL]
 
         _say()
         with Spinner("Analyzing - AI is reviewing your results") as spinner:
@@ -474,7 +471,7 @@ class EvaluationRunContext:
                     self._run.run_id,
                     mode=mode,
                     quality_mode=quality_mode,
-                    judges=resolved_judges,
+                    judges=judges,
                 )
                 deadline = time.monotonic() + timeout
                 status = self._client.get_analysis_status(self._run.run_id)

agentx/monitor/client.py

@@ -429,6 +429,24 @@ class MonitorClient:
         data = self._request("GET", f"/ingest/sessions/{session_id}/spans", base=self._api_root())
         return data.get("spans", []) if isinstance(data, dict) else data
 
+    def list_session_scores(self, session_id: str) -> List[dict]:
+        """Every session-level verdict on the session, newest first: session-scoped online
+        evaluators (kind ``online-eval:<id>``), session-scoped scorer groups
+        (``scorer-group:<id>``), and legacy coherence rows."""
+        data = self._request(
+            "GET", f"/agent-monitoring/sessions/{session_id}/scores", base=self._api_root()
+        )
+        return data.get("scores", []) if isinstance(data, dict) else data
+
+    def run_session_sweep(self) -> dict:
+        """Run the idle-session sweep once, now - the tick that scores quiet multi-turn
+        sessions with every enabled session-scoped evaluator and scorer group. Production
+        engines run this automatically every minute; the manual trigger exists for demos,
+        tests, and backfills. Returns ``{"judged": n}``."""
+        return self._request(
+            "POST", "/agent-monitoring/session-sweep/run", base=self._api_root(), timeout=300
+        )
+
     # ------------------------------------------------------------------
     # Model portability (self-host): replay a trace's input against other models
     # ------------------------------------------------------------------

agentx/monitor/scorer_groups.py

@@ -65,7 +65,9 @@ class ScorerGroupsClient:
         online: Optional[Dict[str, Any]] = None,
     ) -> ScorerGroup:
         """``members``: [{"kind": "judge"|"pattern"|"custom", "refId": ..., "weight": 1, "gate": False}].
-        ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"}
+        ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"}.
+        Add ``"scope": "session", "idleSeconds": 120`` to score whole multi-turn sessions once
+        idle, instead of each sampled trace.
         or None for offline-only."""
         payload: Dict[str, Any] = {"name": name, "members": members}
         if description is not None:

agentx/monitor/sessions.py

@@ -19,3 +19,15 @@ class MonitorSessionClient:
     def spans(self, session_id: str) -> List[dict]:
         """Every span in the session (roots and children), oldest first."""
         return self._client.list_session_spans(session_id)
+
+    def scores(self, session_id: str) -> List[dict]:
+        """Session-level verdicts, newest first. ``kind`` says who scored: a session-scoped
+        online evaluator (``online-eval:<id>``) or a session-scoped scorer group
+        (``scorer-group:<id>``)."""
+        return self._client.list_session_scores(session_id)
+
+    def run_sweep(self) -> dict:
+        """Trigger the idle-session sweep once (normally automatic, every minute) - scores
+        idle multi-turn sessions with every enabled session-scoped evaluator and scorer
+        group. Returns ``{"judged": n}``."""
+        return self._client.run_session_sweep()

agentx/version.py

@@ -1,7 +1,7 @@
-VERSION = "0.8.16"
+VERSION = "0.8.17"
 
 # The AgentX-trace-eval release this SDK version is tested against - what `agentx-trace-eval`
 # installs and converges to (see agentx/cli.py). Bump together with VERSION when releasing, so
 # every published SDK names a known-good engine+dashboard pair. Users can override with
 # AGENTX_TRACE_EVAL_VERSION=<tag|latest>.
-ENGINE_VERSION = "v0.3.13"
+ENGINE_VERSION = "v0.3.16"

{agentx_python-0.8.16.dist-info → agentx_python-0.8.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.16
+Version: 0.8.17
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.16.dist-info → agentx_python-0.8.17.dist-info}/RECORD

@@ -10,17 +10,17 @@ agentx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 agentx/testing.py,sha256=0shZEid_vhJgBegpO_loUq6APYDxQrUgD3jvcRdIDV8,7372
 agentx/traces.py,sha256=sz9gxutlDKNgf0fdCkyzKsGwK5eMAJuIEl3NIwGrsuo,2287
 agentx/util.py,sha256=lt2Kpg4Fj8whrSuAhLEahcPEIYRVpK9RCC5y9ShdTJs,703
-agentx/version.py,sha256=0obTIpPTOoMyuftj72non9y03CO044fCuNZQlNM6U3k,366
+agentx/version.py,sha256=hFFp4l3xfBrH_LqiA3KPfmmzw-XVqa7R3936Z3truX4,366
 agentx/evaluations/__init__.py,sha256=Erv7RGFlRxGTG4rVb2uHhCqLLEX_iAWKqkZKzK6CumE,262
 agentx/evaluations/_term.py,sha256=WFpiNzdgDBeJJ-Gg-6X7TwwxllobuE4OqFUTuDQvS3Y,2529
 agentx/evaluations/client.py,sha256=f5GqsiuqknE-ENUYocBcXgV6OGMKdzmlsm76takJbwU,33112
-agentx/evaluations/datasets.py,sha256=2--V40JrSJoXxR1s0jI3i5rYT8ys25ouijqQQlj72D8,16053
-agentx/evaluations/evaluation_settings.py,sha256=geRydHqKpxwkxBVfMIOqkB6NlStyOZrYwZnMHluDwvA,6276
-agentx/evaluations/models.py,sha256=YUjo3glVMu9iaL9gwUt8bT2OewJGm_455ZiTwM6kaPo,32938
+agentx/evaluations/datasets.py,sha256=DuX-wu5Kz953s6-605lt8N8-kn3F4WuwaCMq5ISZwbA,16051
+agentx/evaluations/evaluation_settings.py,sha256=PhZtflmEAw33EHuzu4C_Vpm8BBjnIPw60y7VEJ7mU8s,6274
+agentx/evaluations/models.py,sha256=v-t6_HkEQlGNc7ZGfoYXZ7FrFjepuyRODkly-ce6Qjw,32936
 agentx/evaluations/prompts.py,sha256=8xyvMpAl3mD5xuSZozuaohdnCBWe5Q02RldwiPLUO20,3493
 agentx/evaluations/reporting.py,sha256=GtNnL-1eNEQrSqd0yrHi5mGqC9_u0kUAN8Mbjdhw6yY,6389
 agentx/evaluations/results.py,sha256=w7TRUik4eXIqrznXvlF_L9hnObnHvgdRKvihfN1XeAU,4692
-agentx/evaluations/runner.py,sha256=jPGWOHOEhTW6PK-qMK9X8i3EqldwEOh5jDk24lLVFEY,34744
+agentx/evaluations/runner.py,sha256=2lxbYKTE2OVH_DFq5yVdAH6cvW5rf3rL91VlEepWLiQ,34714
 agentx/evaluations/tool_schemas.py,sha256=ZyrnSOnx9xlSAj4swfDey9ZLS2ly-g21DByTwRAGwNA,2323
 agentx/evaluations/tracing.py,sha256=MSJD9bzfInMi7E70mc8mWUc0fx5DLkOgUzRru2mNQoc,1825
 agentx/evaluations/adapters/__init__.py,sha256=fK8Hx75usbiY03XUSmnLnSrwBXip2CQs24YZcdEgj6w,287
@@ -43,7 +43,7 @@ agentx/integrations/openai.py,sha256=1KLs-hJaeW2emHPwNkmC3zcGksL3zd87uWbcLGRW704
 agentx/integrations/openai_agents.py,sha256=tCa3kPxqux5LnNNDQ6zA05OybQrjkc_LeE08YuNPJEs,12742
 agentx/monitor/__init__.py,sha256=MLiASSvQnCy2jqVNRTgPWFhUSf8lGYRQeEgEVVhE5Tg,574
 agentx/monitor/agents.py,sha256=8Xk4jWmNTvdtjFixHiLmw4jj96uqIXWIT6KW8zXoWrc,1006
-agentx/monitor/client.py,sha256=qY3Dulnhsp_Izj7up4EIxYHRxOfbTVgysflTh2YRnPc,22543
+agentx/monitor/client.py,sha256=V-9rPuzTFR4W-iJS73Blippx0tGdSrOQTcFt1oz29jE,23551
 agentx/monitor/improvement_groups.py,sha256=-6-FcJnyiTLqeh9z8D9wyFD1QFqEsdDdDZdDJhC6QXw,3712
 agentx/monitor/judge_scorers.py,sha256=HKCPW_9QJeyGEGwWxr_clep9vn27bwJTKnjN5lYbuSk,15972
 agentx/monitor/models.py,sha256=yTC3WTdTziHMkdeAXlTro-4YiSbBUHMgf9kNZHGfPDA,8470
@@ -52,9 +52,9 @@ agentx/monitor/patterns.py,sha256=ruBqLoA2T2KQuqRbbn1kWkIEkiOal3SvyPl9opGyhzI,41
 agentx/monitor/profile.py,sha256=uGk-4bXT2vmnEQiY5u6rWhAfzZchHBXL74VD3piVHvU,3102
 agentx/monitor/review_queue.py,sha256=Tr3GDMJLJzyFKEEeB8HPmZBsNzzYVJYElDlfBC4qd68,3766
 agentx/monitor/rules.py,sha256=oLQ5RPgMPUKJFTSZzcBEMLzDlICElUNJTyIqxZMAGmQ,3040
-agentx/monitor/scorer_groups.py,sha256=qiQxo6NgNJ50ABKvL1S9OQXBQ-B_YFIiLR59FuXi0hg,3533
+agentx/monitor/scorer_groups.py,sha256=gRFq88B-YkdxftrWwuDLZixrpP0cXzL3ekoZ-lEYFPs,3674
 agentx/monitor/scorers.py,sha256=huIZBszrfLKI8I7VV6NFpW6piRsj9HI03rXUwWs2tgs,7336
-agentx/monitor/sessions.py,sha256=gePcL284m3UITxQAtHxwrKFl6bRjmRib1QK6JbZeS9s,815
+agentx/monitor/sessions.py,sha256=2wTCsSE-Me0ZN_H5DJ58EJ05MQ-LL2m5XKSGIjVmgkM,1444
 agentx/monitor/signals.py,sha256=Ld11lW2lbg8NHBUHdgoY6iwCvJDK-eg4ge4qQ9y7nGY,1432
 agentx/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 agentx/resources/agent.py,sha256=ZKpxDYzJbKgOX4-W86U__b4gko7XlbwB4q6L8Az9uo0,2011
@@ -66,9 +66,9 @@ agentx/tracing/eval_scope.py,sha256=ElMbPxpqpQBVuaUnHNlw9RIQu71oyOpd0yR55bDH8eI,
 agentx/tracing/framework_detect.py,sha256=uV4O7Th-4_2UkdooAyaWCOIA0jwLJbblFQ_ucFDjeJM,2638
 agentx/tracing/ingest_client.py,sha256=spnm6bV3NXGuj2w7HwdKiJdKDJWRUFmSP3jl3rcGqkQ,18202
 agentx/tracing/tracer.py,sha256=TxY6zG6hbM81jpfrqP4imaEvoyy_hnJVUSTnoAVHMG0,56240
-agentx_python-0.8.16.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
-agentx_python-0.8.16.dist-info/METADATA,sha256=LRnDRYperDF64jcVdRcdtBH0Iwr5TtlTv0kLKbnC36o,22117
-agentx_python-0.8.16.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-agentx_python-0.8.16.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
-agentx_python-0.8.16.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
-agentx_python-0.8.16.dist-info/RECORD,,
+agentx_python-0.8.17.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
+agentx_python-0.8.17.dist-info/METADATA,sha256=aqprIQoSBpSb7hcJPKt5C6pAz9R4aaHOx4dK3bWjXj8,22117
+agentx_python-0.8.17.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+agentx_python-0.8.17.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
+agentx_python-0.8.17.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
+agentx_python-0.8.17.dist-info/RECORD,,