agentx-python 0.8.22__tar.gz → 0.8.23__tar.gz

{agentx_python-0.8.22 → agentx_python-0.8.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.22
+Version: 0.8.23
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -255,6 +255,13 @@ extra:
 | LlamaIndex            | `pip install "agentx-python[llamaindex]"`    | `AgentXLlamaIndexHandler` |
 | AutoGen               | `pip install "agentx-python[autogen]"`       | `AgentXAutoGenObserver`   |
 
+> **Warning: pick ONE instrumentation layer per LLM call.** Do not combine
+> `AgentXCallbackHandler` (or any framework integration) with a patched provider client
+> (`patch_openai_client`, `patch_anthropic_client`, `patch_genai_client`) on the same code
+> path. A patched call that runs outside an active span emits its own root trace, so every
+> LLM call the framework already traces gets a duplicate trace - and its cost is counted
+> twice.
+
 Two more platforms are covered by **pull importers** rather than in-process hooks, each with its
 own CLI: `agentx-moveworks` (Moveworks Data API sync, no extra needed) and `agentx-databricks`
 (`pip install "agentx-python[databricks]"`, MLflow/Databricks trace sync).

{agentx_python-0.8.22 → agentx_python-0.8.23}/README.md

@@ -190,6 +190,13 @@ extra:
 | LlamaIndex            | `pip install "agentx-python[llamaindex]"`    | `AgentXLlamaIndexHandler` |
 | AutoGen               | `pip install "agentx-python[autogen]"`       | `AgentXAutoGenObserver`   |
 
+> **Warning: pick ONE instrumentation layer per LLM call.** Do not combine
+> `AgentXCallbackHandler` (or any framework integration) with a patched provider client
+> (`patch_openai_client`, `patch_anthropic_client`, `patch_genai_client`) on the same code
+> path. A patched call that runs outside an active span emits its own root trace, so every
+> LLM call the framework already traces gets a duplicate trace - and its cost is counted
+> twice.
+
 Two more platforms are covered by **pull importers** rather than in-process hooks, each with its
 own CLI: `agentx-moveworks` (Moveworks Data API sync, no extra needed) and `agentx-databricks`
 (`pip install "agentx-python[databricks]"`, MLflow/Databricks trace sync).

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/evaluations/client.py

@@ -221,7 +221,16 @@ class EvaluationsClient:
         the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
         to filter."""
         params = {"provider": provider} if provider else None
-        data = self._request("GET", "/models", params=params)
+        try:
+            data = self._request("GET", "/models", params=params)
+        except AgentXEvaluationsError as exc:
+            if exc.status_code == 404:
+                raise AgentXEvaluationsError(
+                    "list_models is hosted-only; on self-host pass any model id your judge "
+                    "key can reach, or use client.monitor.* portability models",
+                    status_code=404,
+                ) from exc
+            raise
         items = data if isinstance(data, list) else data.get("models", [])
         return [ModelInfo(**m) for m in items]
 
@@ -546,12 +555,20 @@ class EvaluationsClient:
     ) -> bool:
         """Return True if ``exc`` is the 404 that means "this engine is self-host".
 
-        Only a 404 qualifies. Anything else - auth, validation, a 500, a dead connection -
-        is a real failure on a route that does exist, and must propagate rather than be
-        retried against a different endpoint that would mask it.
+        Only a route-level 404 qualifies. Anything else - auth, validation, a 500, a dead
+        connection - is a real failure on a route that does exist, and must propagate rather
+        than be retried against a different endpoint that would mask it.
+
+        A resource 404 does not qualify either: the engine's SDK router answers these routes
+        with bodies naming the missing resource ("Run not found" / "No analysis found for
+        this run"), so latching on one would permanently reroute every later analysis call
+        to the dashboard router because a caller once passed a wrong run id.
         """
         if exc.status_code != 404:
             return False
+        body = str(exc)
+        if "Run not found" in body or "No analysis found for this run" in body:
+            return False
         if self._analysis_on_dashboard_router is None:
             logger.info(
                 "%s is not served from %s; using the dashboard router at %s "

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/evaluations/evaluation_settings.py

@@ -74,8 +74,21 @@ class EvaluationSettingsBuilder:
         # Sandboxed JS scorers run per result alongside the judge - each entry is
         # {"name": ..., "enabled": True, "code": "..."} where the code is a JS function body
         # receiving (input, output, expected, toolCalls) and returning {score, reasoning}.
+        # Normalized the same way DatasetBuilder does: id defaulted, name optional (the
+        # engine defaults it), enabled default True - raw pass-through sent entries the
+        # engine's shape validation rejects.
         if code_scorers:
-            self._payload["codeScorers"] = list(code_scorers)
+            import uuid as _uuid
+
+            self._payload["codeScorers"] = [
+                {
+                    "id": scorer.get("id") or _uuid.uuid4().hex[:12],
+                    "name": scorer.get("name"),
+                    "code": scorer["code"],
+                    "enabled": scorer.get("enabled", True),
+                }
+                for scorer in code_scorers
+            ]
 
     def publish(self) -> EvaluationSettings:
         logger.info("Publishing evaluation settings '%s'", self._payload["name"])

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/evaluations/models.py

@@ -51,6 +51,10 @@ class TestCase(BaseModel):
     expected_knowledge_base: Optional[List[str]] = Field(default=None, alias="expectedKnowledgeBase")
     expected_delegations: Optional[List[str]] = Field(default=None, alias="expectedDelegations")
     judge_guideline: Optional[str] = Field(default=None, alias="judgeGuideline")
+    # Engine-side trajectory match (e.g. {"tools": ["search"], "mode": "in_order"}) and the
+    # expected retrieval context for RAG grading - carried so import_dataset round-trips them.
+    expected_trajectory: Optional[Dict[str, Any]] = Field(default=None, alias="expectedTrajectory")
+    expected_retrieval_context: Optional[Any] = Field(default=None, alias="expectedRetrievalContext")
     smoke_test: Optional[SmokeTestSettings] = Field(default=None, alias="smokeTest")
     # Named subsets this case belongs to (e.g. ["smoke"], ["full", "regression"]).
     # ``run(dataset_id, split="smoke")`` runs only cases tagged with that split.

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/evaluations/reporting.py

@@ -170,7 +170,7 @@ def print_report(report: Report) -> None:
 
     # --- Low-scoring cases ---
     if report.low_scoring_cases:
-        _section("Low-scoring Cases  (rating < 5)")
+        _section("Low-scoring Cases  (rating <= 5)")
         for case in report.low_scoring_cases[:5]:
             q = (case.get("query") or case.get("questionText", ""))[:80]
             rating = case.get("rating", "?")

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/evaluations/results.py

@@ -87,6 +87,16 @@ def normalize_result(
     else:
         output = {"text": str(raw)} if raw is not None else {"text": ""}
 
+    if error is None and (
+        output is None
+        or (set(output) <= {"text"} and not str(output.get("text") or "").strip())
+    ):
+        # An empty output with no error would fail the engine's row validation and silently
+        # vanish from the run - store it as an explicit failed row instead.
+        error = ResultError(type="EmptyOutput", message="Agent returned no output")
+        if output is None:
+            output = {"text": ""}
+
     has_timings = (
         latency_ms is not None or input_tokens is not None or output_tokens is not None
     )

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/evaluations/runner.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import os
+import sys
 import time
 
 import requests
@@ -307,9 +308,23 @@ class EvaluationRunContext:
             # generator happens to be garbage-collected.
             if results_iter is not None:
                 results_iter.close()
-
-        if batch:
-            self._flush_batch(batch)
+            # Flush the trailing partial batch HERE, not after the try: a mid-run exception
+            # (agent crash, Ctrl-C) used to discard up to max_batch - 1 already-paid-for
+            # results still waiting in it.
+            if batch:
+                propagating = sys.exc_info()[1]
+                try:
+                    self._flush_batch(batch)
+                except Exception as flush_exc:
+                    if propagating is None:
+                        raise
+                    # An exception is already propagating out of the loop - a flush failure
+                    # here must not mask it.
+                    logger.error(
+                        "Trailing batch flush failed while handling %r: %s",
+                        propagating,
+                        flush_exc,
+                    )
 
         return self
 
@@ -326,6 +341,15 @@ class EvaluationRunContext:
                     _say(
                         f"  {green('✓')}  Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}"
                     )
+                    if resp.failed_validation > 0:
+                        # The engine accepts the batch but silently drops rows that fail its
+                        # validation (typically empty output and no error) - say so, or those
+                        # cases just vanish from the report.
+                        _say(
+                            f"  {yellow('!')}  {resp.failed_validation} result"
+                            f"{'s' if resp.failed_validation != 1 else ''} failed validation "
+                            "(empty output and no error) and did not get stored"
+                        )
                     logger.info(
                         "Batch %s: accepted=%d duplicates=%d failed=%d",
                         batch_id[:8],
@@ -415,8 +439,11 @@ class EvaluationRunContext:
         ``no_regression=True`` fails it when the average dropped more than ``tolerance``
         (default 0.5, judge scores are noisy) below the dataset's previous completed run.
         At least one check is required. On a multi-judge run, ``scorer`` (an additional
-        scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average
-        instead of the primary's - "fail if Safety is low even when the average looks fine". Prints a CI-log-friendly verdict and returns a
+        judge scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average
+        instead of the primary's - "fail if Safety is low even when the average looks fine".
+        Only judge scorers resolve here: deterministic scorer-group members (pattern/code
+        kinds) have no per-run judge average, so naming one is rejected by the engine.
+        Prints a CI-log-friendly verdict and returns a
         :class:`GateResult` - the caller decides the exit code::
 
             report = client.evaluations.run(...).execute(my_agent).finalize()
@@ -469,6 +496,16 @@ class EvaluationRunContext:
         """Number of submitted results that have received a rating so far."""
         return self._live_stats.rated_count if self._live_stats else 0
 
+    @property
+    def skipped_count(self) -> int:
+        """Number of submitted results the judge could not score."""
+        return self._live_stats.skipped_count if self._live_stats else 0
+
+    @property
+    def failed_count(self) -> int:
+        """Number of submitted results that carried an error."""
+        return self._live_stats.failed_count if self._live_stats else 0
+
     @property
     def average_rating(self) -> Optional[float]:
         """Live average rating across all results scored so far. Populated as
@@ -639,6 +676,36 @@ class EvaluationsRunner:
         script execution)."""
         return self._client.get_analysis_status(run_id)
 
+    # Run-lifecycle calls by id - the standalone forms of what run()/execute()/finalize()/
+    # analyze() drive for you, for scripts operating on a run created elsewhere.
+
+    def init_run(self, dataset_id: str, subject, **kwargs):
+        """Create a run row without executing anything - the standalone form of :meth:`run`.
+        Accepts the same kwargs as ``EvaluationsClient.init_run``."""
+        return self._client.init_run(dataset_id, subject, **kwargs)
+
+    def append_results(self, run_id: str, batch_id: str, results: list):
+        """Submit one batch of results to a run by id (scored synchronously server-side)."""
+        return self._client.append_results(run_id, batch_id, results)
+
+    def finalize_run(self, run_id: str) -> dict:
+        """Mark a run completed by id - the standalone form of
+        ``EvaluationRunContext.finalize()``."""
+        return self._client.finalize_run(run_id)
+
+    def analyze_run(self, run_id: str, **kwargs) -> dict:
+        """Start the LLM analysis of a finalized run by id; poll
+        :meth:`get_analysis_status`, then :meth:`get_report`."""
+        return self._client.analyze_run(run_id, **kwargs)
+
+    def get_report(self, run_id: str):
+        """The analyzed report for a run by id, once analysis has finished."""
+        return self._client.get_report(run_id)
+
+    def get_submitted_keys(self, run_id: str) -> list:
+        """Idempotency keys a run has already accepted - what execute() uses to resume."""
+        return self._client.get_submitted_keys(run_id)
+
     def gate_run(
         self,
         run_id: str,

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/integrations/_traced_call.py

@@ -153,7 +153,11 @@ def finish_llm_call(
         )
         return
 
-    span = tracer.trace(name, metadata=metadata, framework=framework, model=model, session_id=session_id)
+    # A patched provider call outside any active span becomes its own root trace - it is a bare
+    # model call, so stamp it "llm" rather than leaving the kind unset.
+    span = tracer.trace(
+        name, metadata=metadata, framework=framework, model=model, session_id=session_id, span_kind="llm"
+    )
     span.__enter__()
     span._start = start_t
     span.input = input_repr

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/integrations/autogen.py

@@ -107,7 +107,10 @@ class AgentXAutoGenObserver:
             # explicit return/break/continue there would silently swallow any exception
             # propagating from agent_or_team.run() above (see crewai.py's kickoff() for the same
             # hazard spelled out in full).
-            with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
+            # span_kind="agent": the root of a standalone team/agent run is the agent run itself.
+            with self._tracer.trace(
+                self._name, metadata=self._metadata, session_id=self._session_id, span_kind="agent"
+            ) as span:
                 span._start = start_t
                 if error:
                     span.set_error(error)

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/integrations/crewai.py

@@ -93,7 +93,10 @@ class AgentXCrewObserver:
             # `return` here (this whole method body runs inside the try's `finally`) - an
             # explicit return/break/continue in a finally block silently swallows any exception
             # propagating from crew.kickoff() above.
-            with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
+            # span_kind="agent": the root of a standalone crew kickoff is the agent run itself.
+            with self._tracer.trace(
+                self._name, metadata=self._metadata, session_id=self._session_id, span_kind="agent"
+            ) as span:
                 span._start = start
                 if error:
                     span.set_error(error)

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/integrations/langchain.py

@@ -595,6 +595,9 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                     else self._metadata
                 ),
                 session_id=self._session_id,
+                # The root of a standalone chain/agent invocation is the agent run itself,
+                # not one of its llm/tool/retrieval children.
+                span_kind="agent",
             ) as span:
                 # __enter__ just set _start to "now" - overridden to the chain's real start time,
                 # see llamaindex.py's _send_trace for the identical fix and full rationale.
@@ -655,6 +658,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                     else self._metadata
                 ),
                 session_id=self._session_id,
+                span_kind="agent",
             ) as span:
                 span._start = state["start"]
                 span.set_error(str(error))

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/integrations/llamaindex.py

@@ -359,7 +359,12 @@ class AgentXLlamaIndexHandler(BaseCallbackHandler):
         # tool_calls loop reads latency_ms for duration and the timestamps for position, so each
         # tool call lands correctly in the tree panel instead of defaulting to offset 0.
         with self._tracer.trace(
-            self._name, metadata=self._metadata, session_id=self._session_id, framework="llamaindex"
+            self._name,
+            metadata=self._metadata,
+            session_id=self._session_id,
+            framework="llamaindex",
+            # The root of a standalone query/agent invocation is the agent run itself.
+            span_kind="agent",
         ) as span:
             # __enter__ just set _start to "now" - overridden to the query's real start time so
             # __exit__'s latency_ms reflects the actual run, not the few microseconds between this

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/monitor/judge_scorers.py

@@ -116,6 +116,7 @@ class JudgeScorersClient:
         judge_prompt: Optional[str] = None,
         judge_model: Optional[str] = None,
         tool_context: Optional[str] = None,
+        requires_expected: Optional[bool] = None,
         # Offline profile (dataset-run grading)
         number_of_requests: int = 1,
         vector_similarity: bool = False,
@@ -135,13 +136,17 @@ class JudgeScorersClient:
         alert_threshold: Optional[float] = 5,
         severity: str = "medium",
         agent_ids: Optional[List[str]] = None,
-        idle_seconds: int = 120,
+        idle_seconds: Optional[int] = None,
     ) -> "JudgeScorerBuilder":
         """Snake_case builder with ``.publish()``, the unified successor of
         ``client.evaluations.settings.builder(...)`` - same offline fields (plus ``thresholds``,
         ``tool_context``) and, new here, the online profile in the same call. The scorer the
         builder publishes is one entity: its ``.id`` is what ``client.evaluations.run(...,
         scorer_id=...)`` takes, and its live profile is what online scoring keys on."""
+        if idle_seconds is not None and scope != "session":
+            # idleSeconds only applies to session scope - with trace scope the engine ignores
+            # it, so an explicit value here would be silently inert.
+            raise ValueError("idle_seconds requires scope='session'")
         judge: Dict[str, Any] = {}
         for key, value in (
             ("acceptanceCriteria", acceptance_criteria),
@@ -150,6 +155,7 @@ class JudgeScorersClient:
             ("judgePrompt", judge_prompt),
             ("judgeModel", judge_model),
             ("toolContext", tool_context),
+            ("requiresExpected", requires_expected),
         ):
             if value is not None:
                 judge[key] = value
@@ -178,7 +184,7 @@ class JudgeScorersClient:
                 "scope": scope,
                 "alertThreshold": alert_threshold,
                 "severity": severity,
-                "idleSeconds": idle_seconds,
+                "idleSeconds": idle_seconds if idle_seconds is not None else 120,
             }
             if agent_ids:
                 online["scopeMode"] = "selected"
@@ -329,7 +335,9 @@ class JudgeScorersClient:
 
     def ratings(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorRatingPoint]":
         """Bucketed average-rating-over-time for this scorer's live checks - same typed points
-        the legacy online_evaluators client returns, so scripts migrate without shape changes."""
+        the legacy online_evaluators client returns, so scripts migrate without shape changes.
+        ``window`` accepts "24h", "7d", or "30d" only (unlike :meth:`calibration`, which also
+        takes "rubric")."""
         from agentx.monitor.models import OnlineEvaluatorRatingPoint
 
         data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/ratings?window={window}")
@@ -337,7 +345,8 @@ class JudgeScorersClient:
 
     def events(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorEvent]":
         """Individually scored traces behind the ratings series, worst-rated first - typed, same
-        as the legacy online_evaluators client."""
+        as the legacy online_evaluators client. ``window`` accepts "24h", "7d", or "30d" only
+        (unlike :meth:`calibration`, which also takes "rubric")."""
         from agentx.monitor.models import OnlineEvaluatorEvent
 
         data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/events?window={window}")

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/monitor/models.py

@@ -10,13 +10,14 @@ class MonitorPattern(BaseModel):
     ``pattern_ids`` entry in ``tracer.trace(..., monitor=True, pattern_ids=[...])``.
 
     A "failure" pattern (the default) raises a signal to triage; a "proper" pattern logs a
-    healthy tally instead. Only one of ``include_terms``/``regex``/``semantic_prompt`` is
-    meaningful at a time, selected by ``detector_kind``.
+    healthy tally instead.
 
     On self-host the engine stores a pattern as a list of ``conditions`` (each with its own
-    detector kind and match settings) - the flat ``include_terms``/``regex``/
-    ``semantic_prompt`` fields are display-only projections derived from the first condition;
-    ``conditions`` is the truth.
+    detector kind and match settings) and ``conditions`` is the only truth: the wire always
+    carries ``includeTerms``/``excludeTerms`` as ``[]`` and omits ``regex``/``semanticPrompt``
+    entirely (they are legacy display fields kept for wire compatibility), so the flat
+    ``include_terms``/``exclude_terms``/``regex``/``semantic_prompt`` attributes here stay
+    empty/None - read the match settings from ``conditions``.
     """
 
     id: str = Field(alias="_id")
@@ -41,6 +42,7 @@ class MonitorPattern(BaseModel):
     agent_ids: List[str] = Field(default_factory=list, alias="agentIds")
 
     class Config:
+        populate_by_name = True
         extra = "ignore"
 
 

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/tracing/ingest_client.py

@@ -179,7 +179,9 @@ class IngestClient:
                 resp = self._session.post(self._endpoint, json=payload, timeout=10)
             except requests.RequestException as exc:
                 self._warn_delivery(f"{exc.__class__.__name__}: {exc}")
-                logger.debug("agentx ingest sync send error: %s", exc)
+                # WARNING, not debug: the sync caller explicitly asked for a trace_id back, so
+                # a dropped trace here silently becomes trace_id None downstream.
+                logger.warning("agentx sync trace send failed (%s) - trace dropped, no trace_id", exc)
                 return None
             if resp.status_code in (429, 503) and attempt < 2:
                 retry_after = resp.headers.get("Retry-After")
@@ -193,7 +195,13 @@ class IngestClient:
                 continue
             if not resp.ok:
                 self._warn_delivery(f"HTTP {resp.status_code}", status=resp.status_code)
-                logger.debug("agentx ingest sync HTTP %d: %s", resp.status_code, resp.text[:200])
+                # WARNING, not debug: the sync caller explicitly asked for a trace_id back, so
+                # a dropped trace here silently becomes trace_id None downstream.
+                logger.warning(
+                    "agentx sync trace send failed (HTTP %d: %s) - trace dropped, no trace_id",
+                    resp.status_code,
+                    resp.text[:200],
+                )
                 return None
             try:
                 return resp.json().get("trace_id")
@@ -211,11 +219,16 @@ class IngestClient:
             resp = self._session.post(self._endpoint, json=payload, timeout=10)
         except requests.RequestException as exc:
             self._warn_delivery(f"{exc.__class__.__name__}: {exc}")
-            logger.debug("agentx ingest sync send error: %s", exc)
+            # WARNING, not debug - same reasoning as send_trace_sync.
+            logger.warning("agentx sync trace send failed (%s) - trace dropped, no trace_id", exc)
             return None
         if not resp.ok:
             self._warn_delivery(f"HTTP {resp.status_code}", status=resp.status_code)
-            logger.debug("agentx ingest sync HTTP %d: %s", resp.status_code, resp.text[:200])
+            logger.warning(
+                "agentx sync trace send failed (HTTP %d: %s) - trace dropped, no trace_id",
+                resp.status_code,
+                resp.text[:200],
+            )
             return None
         try:
             body = resp.json()

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/tracing/tracer.py

@@ -1283,8 +1283,8 @@ class Tracer:
                             (available as ``span.trace_id`` once that `with` block exits).
             dataset_id:     EvaluationSettings ID to score against.
             question_index: Optional index into the dataset's questions array.
-                            When supplied, that question's ``expectedResults``
-                            is included in the scoring prompt.
+                            Hosted-only: the self-host engine ignores it and
+                            scores the trace's own input/output as-is.
 
         Returns:
             Dict with keys: ``run_id``, ``trace_id``, ``rating``,

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx/version.py

@@ -1,7 +1,7 @@
-VERSION = "0.8.22"
+VERSION = "0.8.23"
 
 # The AgentX-trace-eval release this SDK version is tested against - what `agentx-trace-eval`
 # installs and converges to (see agentx/cli.py). Bump together with VERSION when releasing, so
 # every published SDK names a known-good engine+dashboard pair. Users can override with
 # AGENTX_TRACE_EVAL_VERSION=<tag|latest>.
-ENGINE_VERSION = "v0.3.23"
+ENGINE_VERSION = "v0.3.24"

{agentx_python-0.8.22 → agentx_python-0.8.23}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.22
+Version: 0.8.23
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -255,6 +255,13 @@ extra:
 | LlamaIndex            | `pip install "agentx-python[llamaindex]"`    | `AgentXLlamaIndexHandler` |
 | AutoGen               | `pip install "agentx-python[autogen]"`       | `AgentXAutoGenObserver`   |
 
+> **Warning: pick ONE instrumentation layer per LLM call.** Do not combine
+> `AgentXCallbackHandler` (or any framework integration) with a patched provider client
+> (`patch_openai_client`, `patch_anthropic_client`, `patch_genai_client`) on the same code
+> path. A patched call that runs outside an active span emits its own root trace, so every
+> LLM call the framework already traces gets a duplicate trace - and its cost is counted
+> twice.
+
 Two more platforms are covered by **pull importers** rather than in-process hooks, each with its
 own CLI: `agentx-moveworks` (Moveworks Data API sync, no extra needed) and `agentx-databricks`
 (`pip install "agentx-python[databricks]"`, MLflow/Databricks trace sync).

{agentx_python-0.8.22 → agentx_python-0.8.23}/tests/test_docs_match_sdk.py

@@ -95,6 +95,21 @@ def test_documented_judge_scorer_methods_exist():
     assert not missing, "documented but not on JudgeScorersClient: " + ", ".join(missing)
 
 
+def test_run_lifecycle_calls_documented_on_client_evaluations_exist():
+    """The mintlify API reference shows client.evaluations.<method>(...) for the whole run
+    lifecycle - these delegate to the private EvaluationsClient, and every one must exist on
+    the public runner or the documented snippets raise AttributeError."""
+    for method in (
+        "init_run",
+        "append_results",
+        "finalize_run",
+        "analyze_run",
+        "get_report",
+        "get_submitted_keys",
+    ):
+        assert hasattr(EvaluationsRunner, method), f"client.evaluations.{method} is documented but missing"
+
+
 def test_documented_builder_keywords_are_real_parameters():
     keywords = [
         (doc, keyword)

{agentx_python-0.8.22 → agentx_python-0.8.23}/tests/test_judge_scorers.py

@@ -214,6 +214,39 @@ def test_judge_scorers_builder_matches_legacy_builder_ergonomics(monkeypatch):
     assert payload["online"]["agentIds"] == ["support-agent"]
 
 
+def test_builder_rejects_idle_seconds_without_session_scope():
+    """idleSeconds only applies to session scope - the engine silently ignores it with trace
+    scope, so an explicit idle_seconds without scope="session" is a hard error, not an inert
+    wire field."""
+    client = JudgeScorersClient(api_key="agtx_local_test", base_url="http://localhost:1")
+    with pytest.raises(ValueError, match="idle_seconds requires scope='session'"):
+        client.builder("Support quality", live=True, idle_seconds=300)
+
+
+def test_builder_sends_idle_seconds_and_requires_expected(monkeypatch):
+    """With scope="session", an explicit idle_seconds reaches the wire, and requires_expected
+    lands in the judge section as requiresExpected."""
+    client = JudgeScorersClient(api_key="agtx_local_test", base_url="http://localhost:1")
+    captured = {}
+
+    def fake_request(method, path, **kwargs):
+        captured["payload"] = kwargs.get("json")
+        return {"judgeScorer": {"_id": "s1", "name": "Support quality", "judge": {}, "offline": {}, "online": None}}
+
+    monkeypatch.setattr(client, "_request", fake_request)
+    client.builder(
+        "Support quality",
+        requires_expected=True,
+        live=True,
+        scope="session",
+        idle_seconds=300,
+    ).publish()
+    payload = captured["payload"]
+    assert payload["judge"]["requiresExpected"] is True
+    assert payload["online"]["scope"] == "session"
+    assert payload["online"]["idleSeconds"] == 300
+
+
 def test_from_env_honors_selfhost_base_url_conventions(monkeypatch):
     """from_env silently targeting the hosted default while the shell exports the self-host
     conventions (AGENTX_SELFHOST_BASE_URL / BASE_URL) produced confusing auth errors - it now

{agentx_python-0.8.22 → agentx_python-0.8.23}/tests/test_selfhost_analysis_fallback.py

@@ -271,6 +271,27 @@ def test_failures_that_are_not_404_propagate_untouched(status):
     assert client._analysis_on_dashboard_router is None
 
 
+@pytest.mark.parametrize("body", [{"error": "Run not found"}, {"error": "No analysis found for this run. POST /runs/:runId/analyze first."}])
+def test_resource_404s_do_not_latch_the_dashboard_fallback(body):
+    """A 404 whose body names the missing resource comes from a route that EXISTS - the SDK
+    router answered it. It must propagate as-is and must not permanently reroute every later
+    analysis call to the dashboard router."""
+    client, session = make_client(
+        {
+            ("GET", f"{SDK_ROOT}/runs/{RUN}/analyze-status"): FakeResponse(404, body),
+            # Present, and must not be reached.
+            ("GET", f"{API_ROOT}/evaluate/analyze/{RUN}/status"): FakeResponse(200, STATUS_BODY),
+        }
+    )
+
+    with pytest.raises(AgentXEvaluationsError) as caught:
+        client.get_analysis_status(RUN)
+
+    assert caught.value.status_code == 404
+    assert not [u for u in session.urls() if "/evaluate/" in u], "masked a resource 404"
+    assert client._analysis_on_dashboard_router is None
+
+
 def test_auth_errors_are_not_mistaken_for_a_missing_route():
     client, session = make_client(
         {("GET", f"{SDK_ROOT}/runs/{RUN}/analyze-status"): FakeResponse(401, {"e": "nope"})}