agentx-python 0.6.27__tar.gz → 0.6.28__tar.gz

{agentx_python-0.6.27/agentx_python.egg-info → agentx_python-0.6.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.27
+Version: 0.6.28
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.27 → agentx_python-0.6.28}/agentx/evaluations/client.py

@@ -32,9 +32,23 @@ _RETRYABLE_STATUS = {429, 500, 502, 503, 504}
 _MAX_RETRIES = 3
 _RETRY_BACKOFF = [1.0, 2.0, 4.0]
 
+# The self-host analyze route judges every result before it responds, so the client has to
+# wait out the whole job on one connection. Matches EvaluationRunContext.analyze()'s own
+# default timeout.
+_SELF_HOST_ANALYZE_TIMEOUT = 1800
+
 
 class AgentXEvaluationsError(Exception):
-    pass
+    """An evaluations API call failed.
+
+    ``status_code`` carries the HTTP status when the failure came from a response rather
+    than from the transport, so callers can branch on it instead of matching on the message
+    text. It is ``None`` for connection errors and for retry exhaustion.
+    """
+
+    def __init__(self, message: str, status_code: Optional[int] = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
 
 
 class AgentXAuthError(AgentXEvaluationsError):
@@ -69,6 +83,8 @@ class EvaluationsClient:
         if not _api_base.endswith("/custom-agent-evaluations"):
             _api_base = f"{_api_base}/custom-agent-evaluations"
         self._base_url = _api_base
+        # None until an analysis call tells us which engine this is; see _api_root.
+        self._analysis_on_dashboard_router: Optional[bool] = None
         self._session = requests.Session()
         self._session.headers.update(
             {
@@ -103,10 +119,29 @@ class EvaluationsClient:
         """Same as _with_workspace(), for GET requests that take workspaceId as a query param."""
         return {"workspaceId": self._workspace_id} if self._workspace_id else None
 
-    def _request(self, method: str, path: str, timeout: int = 30, **kwargs) -> Any:
-        url = f"{self._base_url}{path}"
+    def _request(
+        self,
+        method: str,
+        path: str,
+        timeout: int = 30,
+        base: Optional[str] = None,
+        retry: bool = True,
+        **kwargs,
+    ) -> Any:
+        """Call the evaluations API.
+
+        ``base`` overrides the ``/custom-agent-evaluations`` prefix for the handful of
+        routes that live on a different router (see ``_api_root``).
+
+        ``retry=False`` disables the backoff loop entirely. Use it for any request that is
+        both slow and billable: the loop retries on ``requests.RequestException``, which
+        includes read timeouts, so a synchronous endpoint that outlives its timeout would
+        otherwise be re-invoked — and paid for — up to four times.
+        """
+        url = f"{base or self._base_url}{path}"
+        schedule = [0.0] + (_RETRY_BACKOFF if retry else [])
         last_exc: Optional[Exception] = None
-        for attempt, wait in enumerate([0.0] + _RETRY_BACKOFF):
+        for attempt, wait in enumerate(schedule):
             if wait:
                 time.sleep(wait)
             try:
@@ -120,14 +155,22 @@ class EvaluationsClient:
                 raise AgentXAuthError("Invalid or missing API key")
             if resp.status_code == 422:
                 raise AgentXValidationError(resp.text)
-            if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
+            if (
+                resp.status_code in _RETRYABLE_STATUS
+                and retry
+                and attempt < _MAX_RETRIES - 1
+            ):
                 logger.debug(
                     "Retryable status %d (attempt %d)", resp.status_code, attempt + 1
                 )
-                last_exc = AgentXEvaluationsError(f"HTTP {resp.status_code}")
+                last_exc = AgentXEvaluationsError(
+                    f"HTTP {resp.status_code}", status_code=resp.status_code
+                )
                 continue
             if not resp.ok:
-                raise AgentXEvaluationsError(f"HTTP {resp.status_code}: {resp.text}")
+                raise AgentXEvaluationsError(
+                    f"HTTP {resp.status_code}: {resp.text}", status_code=resp.status_code
+                )
             try:
                 return resp.json()
             except Exception:
@@ -303,6 +346,10 @@ class EvaluationsClient:
         # Starts the durable analysis job and returns immediately (e.g. {"jobId": ..., "status":
         # "pending"}); poll get_analysis_status() until it reaches a terminal status, then call
         # get_report(). mode/quality_mode/judges mirror the dashboard's AnalyzeEvaluationRequest.
+        #
+        # On self-host the fallback route runs the analysis synchronously and returns only when
+        # it is done, so the caller's poll loop sees a terminal status on its first check. The
+        # request is therefore given the full analysis timeout and, critically, no retries.
         payload: Dict[str, Any] = {}
         if mode is not None:
             payload["mode"] = mode
@@ -310,23 +357,132 @@ class EvaluationsClient:
             payload["qualityMode"] = quality_mode
         if judges is not None:
             payload["judges"] = [{"model": m} for m in judges]
-        return self._request("POST", f"/runs/{run_id}/analyze", json=payload, timeout=30)
+
+        if not self._analysis_on_dashboard_router:
+            try:
+                return self._request(
+                    "POST", f"/runs/{run_id}/analyze", json=payload, timeout=30
+                )
+            except AgentXEvaluationsError as exc:
+                if not self._note_missing_analysis_route(exc, "analyze"):
+                    raise
+
+        return self._request(
+            "POST",
+            f"/evaluate/analyze/{run_id}",
+            base=self._api_root,
+            json=payload,
+            timeout=_SELF_HOST_ANALYZE_TIMEOUT,
+            retry=False,
+        )
 
     def get_analysis_status(self, run_id: str) -> AnalysisStatus:
-        data = self._request("GET", f"/runs/{run_id}/analyze-status")
+        if not self._analysis_on_dashboard_router:
+            try:
+                return AnalysisStatus(
+                    **self._request("GET", f"/runs/{run_id}/analyze-status")
+                )
+            except AgentXEvaluationsError as exc:
+                if not self._note_missing_analysis_route(exc, "analyze-status"):
+                    raise
+
+        data = self._request(
+            "GET", f"/evaluate/analyze/{run_id}/status", base=self._api_root
+        )
         return AnalysisStatus(**data)
 
     def get_run(self, run_id: str) -> Dict[str, Any]:
         return self._request("GET", f"/runs/{run_id}")
 
     def get_report(self, run_id: str) -> Report:
-        data = self._request("GET", f"/runs/{run_id}/report")
-        return Report(**data)
+        if not self._analysis_on_dashboard_router:
+            try:
+                return Report(**self._request("GET", f"/runs/{run_id}/report"))
+            except AgentXEvaluationsError as exc:
+                if not self._note_missing_analysis_route(exc, "report"):
+                    raise
+
+        return self._report_from_dashboard(run_id)
 
     def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
         data = self._request("GET", f"/runs/{run_id}/missing-results")
         return data if isinstance(data, list) else data.get("missing", [])
 
+    # ------------------------------------------------------------------
+    # Self-host analysis fallback
+    #
+    # The self-host engine (AgentX-trace-eval) mounts two routers: the SDK's
+    # /custom-agent-evaluations, and /evaluate for the dashboard. Its
+    # /custom-agent-evaluations router implements the run lifecycle - runs, results,
+    # finalize, gate - but not /analyze, /analyze-status or /report, which exist only on
+    # /evaluate. Hosted AgentX serves all of them from the SDK's own router.
+    #
+    # So the three analysis calls try the SDK route first and fall back to the dashboard
+    # route on a 404, which means hosted behaviour is byte-for-byte unchanged: it never
+    # 404s, so it never falls back. The outcome is cached on the client so the probe costs
+    # one request per process, not one per call.
+    # ------------------------------------------------------------------
+
+    @property
+    def _api_root(self) -> str:
+        """The API base with the ``/custom-agent-evaluations`` suffix removed."""
+        suffix = "/custom-agent-evaluations"
+        if self._base_url.endswith(suffix):
+            return self._base_url[: -len(suffix)]
+        return self._base_url
+
+    def _note_missing_analysis_route(
+        self, exc: AgentXEvaluationsError, route: str
+    ) -> bool:
+        """Return True if ``exc`` is the 404 that means "this engine is self-host".
+
+        Only a 404 qualifies. Anything else - auth, validation, a 500, a dead connection -
+        is a real failure on a route that does exist, and must propagate rather than be
+        retried against a different endpoint that would mask it.
+        """
+        if exc.status_code != 404:
+            return False
+        if self._analysis_on_dashboard_router is None:
+            logger.info(
+                "%s is not served from %s; using the dashboard router at %s "
+                "(self-host engine)",
+                route,
+                self._base_url,
+                self._api_root,
+            )
+        self._analysis_on_dashboard_router = True
+        return True
+
+    def _report_from_dashboard(self, run_id: str) -> Report:
+        """Assemble a Report from the dashboard's evaluation record.
+
+        Self-host has no /report route; it returns the analysis nested inside the
+        evaluation itself, under ``analysis.analysis`` with its statistics one level up.
+        The field names already match the Report models, so this is a reshape, not a
+        translation.
+        """
+        record = self._request("GET", f"/evaluate/{run_id}", base=self._api_root)
+        envelope = record.get("analysis") or {}
+        body = envelope.get("analysis") or {}
+
+        if not envelope:
+            raise AgentXEvaluationsError(
+                f"Run {run_id} has no analysis to report. Nothing has analyzed it yet, "
+                "or the analysis failed - call analyze_run() first."
+            )
+
+        dataset_id = record.get("datasetId")
+        if isinstance(dataset_id, dict):  # populated reference, not a bare id
+            dataset_id = dataset_id.get("_id") or dataset_id.get("id")
+
+        return Report(
+            runId=run_id,
+            datasetId=dataset_id or "",
+            status=envelope.get("status") or "completed",
+            statistics=envelope.get("statistics"),
+            **body,
+        )
+
 
 # ---------------------------------------------------------------------------
 # Helpers

{agentx_python-0.6.27 → agentx_python-0.6.28}/agentx/evaluations/runner.py

@@ -362,11 +362,18 @@ class EvaluationRunContext:
         try:
             report = self._client.get_report(self._run.run_id)
         except Exception as exc:
+            # Deliberately not status="completed". A placeholder that claims completion is
+            # indistinguishable from a real report of an evaluation that scored nothing -
+            # print_report renders empty statistics and no recommendations either way - so
+            # the one signal that something went wrong used to be a logger.warning that is
+            # invisible unless the caller configured logging. Say it on stdout, and let the
+            # status carry the truth for anything reading the object.
+            print(f"  {red('✗')}  Could not fetch the report: {dim(str(exc))}")
             logger.warning("Could not fetch report: %s", exc)
             report = Report(
                 runId=self._run.run_id,
                 datasetId=self._dataset.id,
-                status="completed",
+                status="unavailable",
             )
 
         self._report = report

agentx_python-0.6.28/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.28"

{agentx_python-0.6.27 → agentx_python-0.6.28/agentx_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.27
+Version: 0.6.28
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.27 → agentx_python-0.6.28}/agentx_python.egg-info/SOURCES.txt

@@ -64,4 +64,5 @@ agentx_python.egg-info/requires.txt
 agentx_python.egg-info/top_level.txt
 tests/test_integration.py
 tests/test_integrations.py
+tests/test_selfhost_analysis_fallback.py
 tests/test_span_tree.py

agentx_python-0.6.28/tests/test_selfhost_analysis_fallback.py

@@ -0,0 +1,251 @@
+"""The analysis calls' self-host fallback.
+
+Self-host (AgentX-trace-eval) serves whole-run analysis from its dashboard router, not from
+the /custom-agent-evaluations router the SDK targets. Older engines lack the SDK routes
+entirely, so the three analysis calls try the SDK route and fall back on a 404.
+
+What these tests pin down is the part that is easy to get wrong: the fallback must be
+invisible to hosted AgentX, must not swallow anything other than a 404, and must not retry a
+billable synchronous request.
+
+No network and no API key - the HTTP session is replaced with a recorder.
+"""
+
+import pytest
+
+from agentx.evaluations.client import (
+    AgentXAuthError,
+    AgentXEvaluationsError,
+    EvaluationsClient,
+)
+
+API_ROOT = "https://example.test/api/v1"
+SDK_ROOT = f"{API_ROOT}/custom-agent-evaluations"
+RUN = "run-123"
+
+
+class FakeResponse:
+    def __init__(self, status_code: int, payload=None):
+        self.status_code = status_code
+        self._payload = payload if payload is not None else {}
+        self.text = str(self._payload)
+
+    @property
+    def ok(self) -> bool:
+        return self.status_code < 400
+
+    def json(self):
+        return self._payload
+
+
+class FakeSession:
+    """Records every call and answers from a {(method, url): [responses]} routing table."""
+
+    def __init__(self, routes):
+        self.routes = routes
+        self.calls = []
+        self.headers = {}
+
+    def request(self, method, url, **kwargs):
+        self.calls.append((method, url, kwargs))
+        key = (method, url)
+        if key not in self.routes:
+            return FakeResponse(404, {"message": "Not found"})
+        answer = self.routes[key]
+        if isinstance(answer, list):
+            return answer.pop(0) if len(answer) > 1 else answer[0]
+        return answer
+
+    def urls(self, method=None):
+        return [u for m, u, _ in self.calls if method is None or m == method]
+
+
+def make_client(routes):
+    client = EvaluationsClient(api_key="k", base_url=API_ROOT)
+    session = FakeSession(routes)
+    client._session = session
+    return client, session
+
+
+STATUS_BODY = {
+    "evaluationId": RUN,
+    "jobId": RUN,
+    "status": "completed",
+    "progress": {"overallPercentage": 100, "currentLevel": None, "levels": {}},
+}
+
+REPORT_BODY = {
+    "runId": RUN,
+    "datasetId": "ds-1",
+    "status": "completed",
+    "summary": "It went fine.",
+    "recommendations": [{"category": "instructions", "priority": "high"}],
+    "statistics": {"numberOfRuns": 4, "averageRating": 7.5, "minRating": 5, "maxRating": 9},
+}
+
+
+# ---------------------------------------------------------------------------
+# Hosted AgentX: the SDK routes answer, so nothing may change
+# ---------------------------------------------------------------------------
+
+
+def test_hosted_uses_the_sdk_routes_and_never_probes_the_dashboard():
+    client, session = make_client(
+        {
+            ("POST", f"{SDK_ROOT}/runs/{RUN}/analyze"): FakeResponse(200, {"status": "pending"}),
+            ("GET", f"{SDK_ROOT}/runs/{RUN}/analyze-status"): FakeResponse(200, STATUS_BODY),
+            ("GET", f"{SDK_ROOT}/runs/{RUN}/report"): FakeResponse(200, REPORT_BODY),
+        }
+    )
+
+    client.analyze_run(RUN)
+    assert client.get_analysis_status(RUN).status == "completed"
+    assert client.get_report(RUN).summary == "It went fine."
+
+    assert all("/custom-agent-evaluations/" in url for url in session.urls())
+    assert client._analysis_on_dashboard_router is None, "hosted must never flip the flag"
+
+
+# ---------------------------------------------------------------------------
+# Self-host: the SDK routes 404, the dashboard routes answer
+# ---------------------------------------------------------------------------
+
+
+def test_analysis_status_falls_back_to_the_dashboard_router_on_404():
+    client, session = make_client(
+        {("GET", f"{API_ROOT}/evaluate/analyze/{RUN}/status"): FakeResponse(200, STATUS_BODY)}
+    )
+
+    assert client.get_analysis_status(RUN).is_terminal is True
+    assert session.urls("GET") == [
+        f"{SDK_ROOT}/runs/{RUN}/analyze-status",
+        f"{API_ROOT}/evaluate/analyze/{RUN}/status",
+    ]
+
+
+def test_the_fallback_is_probed_once_then_remembered():
+    client, session = make_client(
+        {("GET", f"{API_ROOT}/evaluate/analyze/{RUN}/status"): FakeResponse(200, STATUS_BODY)}
+    )
+
+    client.get_analysis_status(RUN)
+    client.get_analysis_status(RUN)
+
+    # One 404 probe, not one per call.
+    assert session.urls("GET").count(f"{SDK_ROOT}/runs/{RUN}/analyze-status") == 1
+    assert client._analysis_on_dashboard_router is True
+
+
+def test_report_is_assembled_from_the_dashboard_evaluation_record():
+    client, _ = make_client(
+        {
+            ("GET", f"{API_ROOT}/evaluate/{RUN}"): FakeResponse(
+                200,
+                {
+                    # datasetId arrives as a populated reference here, not a bare id.
+                    "datasetId": {"_id": "ds-1", "name": "Support"},
+                    "analysis": {
+                        "status": "completed",
+                        "statistics": {"numberOfRuns": 16, "averageRating": 6.21875,
+                                       "minRating": 1, "maxRating": 9.5},
+                        "analysis": {
+                            "summary": "Uneven.",
+                            "recommendations": [
+                                {"category": "instructions", "priority": "high",
+                                 "recommendation": "Cite the doc id.", "reasoning": "None cited."}
+                            ],
+                            # A key the SDK's Report does not model; must not raise.
+                            "overallAssessment": "mixed",
+                        },
+                    },
+                },
+            )
+        }
+    )
+
+    report = client.get_report(RUN)
+
+    assert report.run_id == RUN
+    assert report.dataset_id == "ds-1", "the populated reference must be unwrapped to its id"
+    assert report.status == "completed"
+    assert report.summary == "Uneven."
+    assert report.statistics.average_rating == 6.21875
+    assert report.statistics.min_rating == 1
+    assert len(report.recommendations) == 1
+    assert report.recommendations[0].category == "instructions"
+
+
+def test_report_says_so_when_nothing_has_analyzed_the_run():
+    client, _ = make_client(
+        {("GET", f"{API_ROOT}/evaluate/{RUN}"): FakeResponse(200, {"datasetId": "ds-1"})}
+    )
+
+    with pytest.raises(AgentXEvaluationsError, match="no analysis"):
+        client.get_report(RUN)
+
+
+def test_the_synchronous_fallback_analyze_is_never_retried():
+    """A read timeout on a billable sync endpoint must not re-run the judges."""
+    import requests
+
+    client, session = make_client({})
+
+    def always_times_out(method, url, **kwargs):
+        session.calls.append((method, url, kwargs))
+        if url.endswith("/analyze"):
+            return FakeResponse(404, {"message": "Not found"})
+        raise requests.exceptions.ReadTimeout("too slow")
+
+    session.request = always_times_out
+
+    with pytest.raises(AgentXEvaluationsError):
+        client.analyze_run(RUN)
+
+    dashboard_posts = [u for u in session.urls("POST") if "/evaluate/analyze/" in u]
+    assert len(dashboard_posts) == 1, f"retried a billable request: {dashboard_posts}"
+
+
+def test_the_fallback_request_gets_the_long_analysis_timeout():
+    client, session = make_client(
+        {("POST", f"{API_ROOT}/evaluate/analyze/{RUN}"): FakeResponse(200, {"status": "completed"})}
+    )
+
+    client.analyze_run(RUN, judges=["gpt-5.5"])
+
+    method, url, kwargs = session.calls[-1]
+    assert url == f"{API_ROOT}/evaluate/analyze/{RUN}"
+    assert kwargs["timeout"] > 60, "a synchronous judge pass needs more than the 30s default"
+    assert kwargs["json"]["judges"] == [{"model": "gpt-5.5"}]
+
+
+# ---------------------------------------------------------------------------
+# Only a 404 means "wrong engine"
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("status", [400, 403, 500])
+def test_failures_that_are_not_404_propagate_untouched(status):
+    client, session = make_client(
+        {
+            ("GET", f"{SDK_ROOT}/runs/{RUN}/analyze-status"): FakeResponse(status, {"e": "boom"}),
+            # Present, and must not be reached.
+            ("GET", f"{API_ROOT}/evaluate/analyze/{RUN}/status"): FakeResponse(200, STATUS_BODY),
+        }
+    )
+
+    with pytest.raises(AgentXEvaluationsError) as caught:
+        client.get_analysis_status(RUN)
+
+    assert caught.value.status_code == status
+    assert not [u for u in session.urls() if "/evaluate/" in u], "masked a real failure"
+    assert client._analysis_on_dashboard_router is None
+
+
+def test_auth_errors_are_not_mistaken_for_a_missing_route():
+    client, session = make_client(
+        {("GET", f"{SDK_ROOT}/runs/{RUN}/analyze-status"): FakeResponse(401, {"e": "nope"})}
+    )
+
+    with pytest.raises(AgentXAuthError):
+        client.get_analysis_status(RUN)
+    assert not [u for u in session.urls() if "/evaluate/" in u]

agentx_python-0.6.27/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.27"