agentx-python 0.6.27__tar.gz → 0.6.29__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.6.27/agentx_python.egg-info → agentx_python-0.6.29}/PKG-INFO +1 -1
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/agentx.py +44 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/client.py +272 -11
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/evaluation_settings.py +8 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/prompts.py +19 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/runner.py +30 -1
- agentx_python-0.6.29/agentx/evaluations/tool_schemas.py +48 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/exceptions.py +4 -0
- agentx_python-0.6.29/agentx/monitor/agents.py +28 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/client.py +89 -2
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/online_evaluators.py +21 -0
- agentx_python-0.6.29/agentx/monitor/sessions.py +21 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/ingest_client.py +33 -0
- agentx_python-0.6.29/agentx/version.py +1 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29/agentx_python.egg-info}/PKG-INFO +1 -1
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/SOURCES.txt +4 -0
- agentx_python-0.6.29/tests/test_selfhost_analysis_fallback.py +251 -0
- agentx_python-0.6.27/agentx/version.py +0 -1
- {agentx_python-0.6.27 → agentx_python-0.6.29}/LICENSE +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/README.md +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/__init__.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/cli.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/models.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/redaction.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/feedback.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/__init__.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/_traced_call.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/anthropic.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/autogen.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/crewai.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/databricks.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/google_adk.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/google_genai.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/langchain.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/litellm.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/llamaindex.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/moveworks.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/openai.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/openai_agents.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/__init__.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/models.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/patterns.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/profile.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/signals.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/outcomes.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/py.typed +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/agent.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/conversation.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/workforce.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/tracer.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/util.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/entry_points.txt +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/requires.txt +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/setup.cfg +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/setup.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/tests/test_integration.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/tests/test_integrations.py +0 -0
- {agentx_python-0.6.27 → agentx_python-0.6.29}/tests/test_span_tree.py +0 -0
|
@@ -111,6 +111,50 @@ class AgentX:
|
|
|
111
111
|
f"Failed to list workforces: {response.status_code} - {response.reason}"
|
|
112
112
|
)
|
|
113
113
|
|
|
114
|
+
def ping(self) -> dict:
|
|
115
|
+
"""Verify the client can actually reach AgentX and that the API key is accepted.
|
|
116
|
+
|
|
117
|
+
The constructor is deliberately lazy (no network call - standard SDK behavior, so
|
|
118
|
+
offline construction and tests work), and trace delivery is fire-and-forget, so a
|
|
119
|
+
wrong ``base_url`` or ``api_key`` otherwise surfaces only as a one-time warning in
|
|
120
|
+
logs while traces silently go nowhere. Call this once at startup of a long-running
|
|
121
|
+
service to fail fast instead::
|
|
122
|
+
|
|
123
|
+
client = AgentX.from_env()
|
|
124
|
+
client.ping() # raises immediately on a bad URL or key
|
|
125
|
+
|
|
126
|
+
Raises :class:`agentx.exceptions.AgentXConnectionError` when the URL is unreachable,
|
|
127
|
+
:class:`agentx.exceptions.AgentXAuthError` when the key is rejected, and
|
|
128
|
+
:class:`agentx.exceptions.AgentXAPIError` on any other non-OK response. Returns
|
|
129
|
+
``{"ok": True, "base_url": ...}`` on success.
|
|
130
|
+
"""
|
|
131
|
+
from agentx.exceptions import AgentXAPIError, AgentXAuthError, AgentXConnectionError
|
|
132
|
+
|
|
133
|
+
base = api_base()
|
|
134
|
+
# /monitor/patterns: the cheapest key-authenticated endpoint that exists on both the
|
|
135
|
+
# hosted API and the self-host engine's SDK-facing router.
|
|
136
|
+
url = f"{base}/monitor/patterns"
|
|
137
|
+
try:
|
|
138
|
+
response = requests.get(url, headers=get_headers(self.api_key), timeout=10)
|
|
139
|
+
except requests.RequestException as exc:
|
|
140
|
+
raise AgentXConnectionError(
|
|
141
|
+
f"Cannot reach AgentX at {base} ({exc.__class__.__name__}: {exc}). "
|
|
142
|
+
"Check base_url / AGENTX_API_BASE_URL - for self-host it should look like "
|
|
143
|
+
"http://localhost:4700/api/v1."
|
|
144
|
+
) from exc
|
|
145
|
+
if response.status_code in (401, 403):
|
|
146
|
+
raise AgentXAuthError(
|
|
147
|
+
f"AgentX at {base} rejected the API key (HTTP {response.status_code}). "
|
|
148
|
+
"Check api_key / AGENTX_API_KEY - for self-host, copy the 'Default project "
|
|
149
|
+
"API key' from the engine's startup log."
|
|
150
|
+
)
|
|
151
|
+
if not response.ok:
|
|
152
|
+
raise AgentXAPIError(
|
|
153
|
+
f"AgentX at {base} responded HTTP {response.status_code} to the health probe.",
|
|
154
|
+
status_code=response.status_code,
|
|
155
|
+
)
|
|
156
|
+
return {"ok": True, "base_url": base}
|
|
157
|
+
|
|
114
158
|
def get_profile(self):
|
|
115
159
|
"""Get the current user's profile information."""
|
|
116
160
|
url = f"{api_base()}/access/getProfile"
|
|
@@ -32,9 +32,23 @@ _RETRYABLE_STATUS = {429, 500, 502, 503, 504}
|
|
|
32
32
|
_MAX_RETRIES = 3
|
|
33
33
|
_RETRY_BACKOFF = [1.0, 2.0, 4.0]
|
|
34
34
|
|
|
35
|
+
# The self-host analyze route judges every result before it responds, so the client has to
|
|
36
|
+
# wait out the whole job on one connection. Matches EvaluationRunContext.analyze()'s own
|
|
37
|
+
# default timeout.
|
|
38
|
+
_SELF_HOST_ANALYZE_TIMEOUT = 1800
|
|
39
|
+
|
|
35
40
|
|
|
36
41
|
class AgentXEvaluationsError(Exception):
|
|
37
|
-
|
|
42
|
+
"""An evaluations API call failed.
|
|
43
|
+
|
|
44
|
+
``status_code`` carries the HTTP status when the failure came from a response rather
|
|
45
|
+
than from the transport, so callers can branch on it instead of matching on the message
|
|
46
|
+
text. It is ``None`` for connection errors and for retry exhaustion.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, message: str, status_code: Optional[int] = None) -> None:
|
|
50
|
+
super().__init__(message)
|
|
51
|
+
self.status_code = status_code
|
|
38
52
|
|
|
39
53
|
|
|
40
54
|
class AgentXAuthError(AgentXEvaluationsError):
|
|
@@ -69,6 +83,8 @@ class EvaluationsClient:
|
|
|
69
83
|
if not _api_base.endswith("/custom-agent-evaluations"):
|
|
70
84
|
_api_base = f"{_api_base}/custom-agent-evaluations"
|
|
71
85
|
self._base_url = _api_base
|
|
86
|
+
# None until an analysis call tells us which engine this is; see _api_root.
|
|
87
|
+
self._analysis_on_dashboard_router: Optional[bool] = None
|
|
72
88
|
self._session = requests.Session()
|
|
73
89
|
self._session.headers.update(
|
|
74
90
|
{
|
|
@@ -86,6 +102,8 @@ class EvaluationsClient:
|
|
|
86
102
|
self.datasets = DatasetClient(self)
|
|
87
103
|
self.settings = EvaluationSettingsClient(self)
|
|
88
104
|
self.prompts = PromptClient(self)
|
|
105
|
+
from agentx.evaluations.tool_schemas import ToolSchemaClient
|
|
106
|
+
self.tool_schemas = ToolSchemaClient(self)
|
|
89
107
|
|
|
90
108
|
# ------------------------------------------------------------------
|
|
91
109
|
# Low-level HTTP
|
|
@@ -103,10 +121,29 @@ class EvaluationsClient:
|
|
|
103
121
|
"""Same as _with_workspace(), for GET requests that take workspaceId as a query param."""
|
|
104
122
|
return {"workspaceId": self._workspace_id} if self._workspace_id else None
|
|
105
123
|
|
|
106
|
-
def _request(
|
|
107
|
-
|
|
124
|
+
def _request(
|
|
125
|
+
self,
|
|
126
|
+
method: str,
|
|
127
|
+
path: str,
|
|
128
|
+
timeout: int = 30,
|
|
129
|
+
base: Optional[str] = None,
|
|
130
|
+
retry: bool = True,
|
|
131
|
+
**kwargs,
|
|
132
|
+
) -> Any:
|
|
133
|
+
"""Call the evaluations API.
|
|
134
|
+
|
|
135
|
+
``base`` overrides the ``/custom-agent-evaluations`` prefix for the handful of
|
|
136
|
+
routes that live on a different router (see ``_api_root``).
|
|
137
|
+
|
|
138
|
+
``retry=False`` disables the backoff loop entirely. Use it for any request that is
|
|
139
|
+
both slow and billable: the loop retries on ``requests.RequestException``, which
|
|
140
|
+
includes read timeouts, so a synchronous endpoint that outlives its timeout would
|
|
141
|
+
otherwise be re-invoked, and paid for, up to four times.
|
|
142
|
+
"""
|
|
143
|
+
url = f"{base or self._base_url}{path}"
|
|
144
|
+
schedule = [0.0] + (_RETRY_BACKOFF if retry else [])
|
|
108
145
|
last_exc: Optional[Exception] = None
|
|
109
|
-
for attempt, wait in enumerate(
|
|
146
|
+
for attempt, wait in enumerate(schedule):
|
|
110
147
|
if wait:
|
|
111
148
|
time.sleep(wait)
|
|
112
149
|
try:
|
|
@@ -120,14 +157,22 @@ class EvaluationsClient:
|
|
|
120
157
|
raise AgentXAuthError("Invalid or missing API key")
|
|
121
158
|
if resp.status_code == 422:
|
|
122
159
|
raise AgentXValidationError(resp.text)
|
|
123
|
-
if
|
|
160
|
+
if (
|
|
161
|
+
resp.status_code in _RETRYABLE_STATUS
|
|
162
|
+
and retry
|
|
163
|
+
and attempt < _MAX_RETRIES - 1
|
|
164
|
+
):
|
|
124
165
|
logger.debug(
|
|
125
166
|
"Retryable status %d (attempt %d)", resp.status_code, attempt + 1
|
|
126
167
|
)
|
|
127
|
-
last_exc = AgentXEvaluationsError(
|
|
168
|
+
last_exc = AgentXEvaluationsError(
|
|
169
|
+
f"HTTP {resp.status_code}", status_code=resp.status_code
|
|
170
|
+
)
|
|
128
171
|
continue
|
|
129
172
|
if not resp.ok:
|
|
130
|
-
raise AgentXEvaluationsError(
|
|
173
|
+
raise AgentXEvaluationsError(
|
|
174
|
+
f"HTTP {resp.status_code}: {resp.text}", status_code=resp.status_code
|
|
175
|
+
)
|
|
131
176
|
try:
|
|
132
177
|
return resp.json()
|
|
133
178
|
except Exception:
|
|
@@ -303,6 +348,10 @@ class EvaluationsClient:
|
|
|
303
348
|
# Starts the durable analysis job and returns immediately (e.g. {"jobId": ..., "status":
|
|
304
349
|
# "pending"}); poll get_analysis_status() until it reaches a terminal status, then call
|
|
305
350
|
# get_report(). mode/quality_mode/judges mirror the dashboard's AnalyzeEvaluationRequest.
|
|
351
|
+
#
|
|
352
|
+
# On self-host the fallback route runs the analysis synchronously and returns only when
|
|
353
|
+
# it is done, so the caller's poll loop sees a terminal status on its first check. The
|
|
354
|
+
# request is therefore given the full analysis timeout and, critically, no retries.
|
|
306
355
|
payload: Dict[str, Any] = {}
|
|
307
356
|
if mode is not None:
|
|
308
357
|
payload["mode"] = mode
|
|
@@ -310,23 +359,235 @@ class EvaluationsClient:
|
|
|
310
359
|
payload["qualityMode"] = quality_mode
|
|
311
360
|
if judges is not None:
|
|
312
361
|
payload["judges"] = [{"model": m} for m in judges]
|
|
313
|
-
|
|
362
|
+
|
|
363
|
+
if not self._analysis_on_dashboard_router:
|
|
364
|
+
try:
|
|
365
|
+
return self._request(
|
|
366
|
+
"POST", f"/runs/{run_id}/analyze", json=payload, timeout=30
|
|
367
|
+
)
|
|
368
|
+
except AgentXEvaluationsError as exc:
|
|
369
|
+
if not self._note_missing_analysis_route(exc, "analyze"):
|
|
370
|
+
raise
|
|
371
|
+
|
|
372
|
+
return self._request(
|
|
373
|
+
"POST",
|
|
374
|
+
f"/evaluate/analyze/{run_id}",
|
|
375
|
+
base=self._api_root,
|
|
376
|
+
json=payload,
|
|
377
|
+
timeout=_SELF_HOST_ANALYZE_TIMEOUT,
|
|
378
|
+
retry=False,
|
|
379
|
+
)
|
|
314
380
|
|
|
315
381
|
def get_analysis_status(self, run_id: str) -> AnalysisStatus:
|
|
316
|
-
|
|
382
|
+
if not self._analysis_on_dashboard_router:
|
|
383
|
+
try:
|
|
384
|
+
return AnalysisStatus(
|
|
385
|
+
**self._request("GET", f"/runs/{run_id}/analyze-status")
|
|
386
|
+
)
|
|
387
|
+
except AgentXEvaluationsError as exc:
|
|
388
|
+
if not self._note_missing_analysis_route(exc, "analyze-status"):
|
|
389
|
+
raise
|
|
390
|
+
|
|
391
|
+
data = self._request(
|
|
392
|
+
"GET", f"/evaluate/analyze/{run_id}/status", base=self._api_root
|
|
393
|
+
)
|
|
317
394
|
return AnalysisStatus(**data)
|
|
318
395
|
|
|
319
396
|
def get_run(self, run_id: str) -> Dict[str, Any]:
|
|
320
397
|
return self._request("GET", f"/runs/{run_id}")
|
|
321
398
|
|
|
322
399
|
def get_report(self, run_id: str) -> Report:
|
|
323
|
-
|
|
324
|
-
|
|
400
|
+
if not self._analysis_on_dashboard_router:
|
|
401
|
+
try:
|
|
402
|
+
return Report(**self._request("GET", f"/runs/{run_id}/report"))
|
|
403
|
+
except AgentXEvaluationsError as exc:
|
|
404
|
+
if not self._note_missing_analysis_route(exc, "report"):
|
|
405
|
+
raise
|
|
406
|
+
|
|
407
|
+
return self._report_from_dashboard(run_id)
|
|
325
408
|
|
|
326
409
|
def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
|
|
327
410
|
data = self._request("GET", f"/runs/{run_id}/missing-results")
|
|
328
411
|
return data if isinstance(data, list) else data.get("missing", [])
|
|
329
412
|
|
|
413
|
+
# ------------------------------------------------------------------
|
|
414
|
+
# Self-host analysis fallback
|
|
415
|
+
#
|
|
416
|
+
# The self-host engine (AgentX-trace-eval) mounts two routers: the SDK's
|
|
417
|
+
# /custom-agent-evaluations, and /evaluate for the dashboard. Its
|
|
418
|
+
# /custom-agent-evaluations router implements the run lifecycle - runs, results,
|
|
419
|
+
# finalize, gate - but not /analyze, /analyze-status or /report, which exist only on
|
|
420
|
+
# /evaluate. Hosted AgentX serves all of them from the SDK's own router.
|
|
421
|
+
#
|
|
422
|
+
# So the three analysis calls try the SDK route first and fall back to the dashboard
|
|
423
|
+
# route on a 404, which means hosted behaviour is byte-for-byte unchanged: it never
|
|
424
|
+
# 404s, so it never falls back. The outcome is cached on the client so the probe costs
|
|
425
|
+
# one request per process, not one per call.
|
|
426
|
+
# ------------------------------------------------------------------
|
|
427
|
+
|
|
428
|
+
@property
|
|
429
|
+
def _api_root(self) -> str:
|
|
430
|
+
"""The API base with the ``/custom-agent-evaluations`` suffix removed."""
|
|
431
|
+
suffix = "/custom-agent-evaluations"
|
|
432
|
+
if self._base_url.endswith(suffix):
|
|
433
|
+
return self._base_url[: -len(suffix)]
|
|
434
|
+
return self._base_url
|
|
435
|
+
|
|
436
|
+
def _note_missing_analysis_route(
|
|
437
|
+
self, exc: AgentXEvaluationsError, route: str
|
|
438
|
+
) -> bool:
|
|
439
|
+
"""Return True if ``exc`` is the 404 that means "this engine is self-host".
|
|
440
|
+
|
|
441
|
+
Only a 404 qualifies. Anything else - auth, validation, a 500, a dead connection -
|
|
442
|
+
is a real failure on a route that does exist, and must propagate rather than be
|
|
443
|
+
retried against a different endpoint that would mask it.
|
|
444
|
+
"""
|
|
445
|
+
if exc.status_code != 404:
|
|
446
|
+
return False
|
|
447
|
+
if self._analysis_on_dashboard_router is None:
|
|
448
|
+
logger.info(
|
|
449
|
+
"%s is not served from %s; using the dashboard router at %s "
|
|
450
|
+
"(self-host engine)",
|
|
451
|
+
route,
|
|
452
|
+
self._base_url,
|
|
453
|
+
self._api_root,
|
|
454
|
+
)
|
|
455
|
+
self._analysis_on_dashboard_router = True
|
|
456
|
+
return True
|
|
457
|
+
|
|
458
|
+
def _report_from_dashboard(self, run_id: str) -> Report:
|
|
459
|
+
"""Assemble a Report from the dashboard's evaluation record.
|
|
460
|
+
|
|
461
|
+
Self-host has no /report route; it returns the analysis nested inside the
|
|
462
|
+
evaluation itself, under ``analysis.analysis`` with its statistics one level up.
|
|
463
|
+
The field names already match the Report models, so this is a reshape, not a
|
|
464
|
+
translation.
|
|
465
|
+
"""
|
|
466
|
+
record = self._request("GET", f"/evaluate/{run_id}", base=self._api_root)
|
|
467
|
+
envelope = record.get("analysis") or {}
|
|
468
|
+
body = envelope.get("analysis") or {}
|
|
469
|
+
|
|
470
|
+
if not envelope:
|
|
471
|
+
raise AgentXEvaluationsError(
|
|
472
|
+
f"Run {run_id} has no analysis to report. Nothing has analyzed it yet, "
|
|
473
|
+
"or the analysis failed - call analyze_run() first."
|
|
474
|
+
)
|
|
475
|
+
|
|
476
|
+
dataset_id = record.get("datasetId")
|
|
477
|
+
if isinstance(dataset_id, dict): # populated reference, not a bare id
|
|
478
|
+
dataset_id = dataset_id.get("_id") or dataset_id.get("id")
|
|
479
|
+
|
|
480
|
+
return Report(
|
|
481
|
+
runId=run_id,
|
|
482
|
+
datasetId=dataset_id or "",
|
|
483
|
+
status=envelope.get("status") or "completed",
|
|
484
|
+
statistics=envelope.get("statistics"),
|
|
485
|
+
**body,
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
# ------------------------------------------------------------------
|
|
489
|
+
# Prompt improvement loop (examples -> propose -> publish). These ride the engine's
|
|
490
|
+
# /evaluate dialect via _api_root(), same precedent get_report/_analysis already use for
|
|
491
|
+
# routes that live on the dashboard router (self-host only).
|
|
492
|
+
# ------------------------------------------------------------------
|
|
493
|
+
|
|
494
|
+
def get_prompt_examples(self, prompt_id: str, window: Optional[str] = None) -> dict:
|
|
495
|
+
params = {"window": window} if window else None
|
|
496
|
+
return self._request(
|
|
497
|
+
"GET", f"/evaluate/prompts/{prompt_id}/examples", base=self._api_root, params=params
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
def propose_prompt(self, prompt_id: str) -> dict:
|
|
501
|
+
# One real judge call - no retry (see _request's retry note).
|
|
502
|
+
return self._request(
|
|
503
|
+
"POST", f"/evaluate/prompts/{prompt_id}/propose", base=self._api_root, timeout=180, retry=False
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
def publish_prompt_version(
|
|
507
|
+
self, prompt_id: str, *, text: str, source: str = "proposed",
|
|
508
|
+
reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
|
|
509
|
+
) -> dict:
|
|
510
|
+
payload: dict = {"text": text, "source": source}
|
|
511
|
+
if reasoning is not None:
|
|
512
|
+
payload["reasoning"] = reasoning
|
|
513
|
+
if based_on_version is not None:
|
|
514
|
+
payload["basedOnVersion"] = based_on_version
|
|
515
|
+
return self._request(
|
|
516
|
+
"POST", f"/evaluate/prompts/{prompt_id}/versions", base=self._api_root, json=payload
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
# ------------------------------------------------------------------
|
|
520
|
+
# Tool schema registry (same version-scoped propose/publish loop as prompts)
|
|
521
|
+
# ------------------------------------------------------------------
|
|
522
|
+
|
|
523
|
+
def list_tool_schemas(self) -> List[dict]:
|
|
524
|
+
data = self._request("GET", "/evaluate/tool-schemas", base=self._api_root)
|
|
525
|
+
return data.get("toolSchemas", []) if isinstance(data, dict) else data
|
|
526
|
+
|
|
527
|
+
def create_tool_schema(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
|
|
528
|
+
payload: dict = {"name": name, "definition": definition}
|
|
529
|
+
if description is not None:
|
|
530
|
+
payload["description"] = description
|
|
531
|
+
return self._request("POST", "/evaluate/tool-schemas", base=self._api_root, json=payload)
|
|
532
|
+
|
|
533
|
+
def get_tool_schema_examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
|
|
534
|
+
params = {"window": window} if window else None
|
|
535
|
+
return self._request(
|
|
536
|
+
"GET", f"/evaluate/tool-schemas/{tool_schema_id}/examples", base=self._api_root, params=params
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
def propose_tool_schema(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
|
|
540
|
+
payload = {"window": window} if window else {}
|
|
541
|
+
return self._request(
|
|
542
|
+
"POST", f"/evaluate/tool-schemas/{tool_schema_id}/propose",
|
|
543
|
+
base=self._api_root, json=payload, timeout=180, retry=False,
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
def publish_tool_schema_version(
|
|
547
|
+
self, tool_schema_id: str, *, definition: str, source: str = "proposed",
|
|
548
|
+
reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
|
|
549
|
+
) -> dict:
|
|
550
|
+
payload: dict = {"definition": definition, "source": source}
|
|
551
|
+
if reasoning is not None:
|
|
552
|
+
payload["reasoning"] = reasoning
|
|
553
|
+
if based_on_version is not None:
|
|
554
|
+
payload["basedOnVersion"] = based_on_version
|
|
555
|
+
return self._request(
|
|
556
|
+
"POST", f"/evaluate/tool-schemas/{tool_schema_id}/versions", base=self._api_root, json=payload
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
# ------------------------------------------------------------------
|
|
560
|
+
# CI gate history + conversation simulation (self-host)
|
|
561
|
+
# ------------------------------------------------------------------
|
|
562
|
+
|
|
563
|
+
def list_gates(self) -> List[dict]:
|
|
564
|
+
"""Recorded CI gate verdicts, newest first - the dashboard's CI Gates history."""
|
|
565
|
+
data = self._request("GET", "/evaluate/ci/gates", base=self._api_root)
|
|
566
|
+
return data.get("gates", []) if isinstance(data, dict) else data
|
|
567
|
+
|
|
568
|
+
def simulate_conversation(
|
|
569
|
+
self, *, model: str, system_prompt: str, persona: str, goal: str,
|
|
570
|
+
max_turns: int = 5, tools: Optional[List[dict]] = None, agent_name: Optional[str] = None,
|
|
571
|
+
) -> dict:
|
|
572
|
+
"""Run a persona-driven multi-turn simulation against a prompt (the Playground's
|
|
573
|
+
"Simulate conversation"). Blocking and judge-billed: one LLM call per simulated turn
|
|
574
|
+
plus the closing judgment, so expect it to take tens of seconds."""
|
|
575
|
+
payload: dict = {
|
|
576
|
+
"model": model,
|
|
577
|
+
"messages": [{"role": "system", "content": system_prompt}],
|
|
578
|
+
"persona": persona,
|
|
579
|
+
"goal": goal,
|
|
580
|
+
"maxTurns": max_turns,
|
|
581
|
+
}
|
|
582
|
+
if tools:
|
|
583
|
+
payload["tools"] = tools
|
|
584
|
+
if agent_name:
|
|
585
|
+
payload["agentName"] = agent_name
|
|
586
|
+
return self._request(
|
|
587
|
+
"POST", "/evaluate/playground/simulate", base=self._api_root, json=payload,
|
|
588
|
+
timeout=600, retry=False,
|
|
589
|
+
)
|
|
590
|
+
|
|
330
591
|
|
|
331
592
|
# ---------------------------------------------------------------------------
|
|
332
593
|
# Helpers
|
|
@@ -32,6 +32,7 @@ class EvaluationSettingsBuilder:
|
|
|
32
32
|
rouge_score: bool = False,
|
|
33
33
|
similarity_model: Optional[str] = None,
|
|
34
34
|
sovereignty_models: Optional[List[str]] = None,
|
|
35
|
+
code_scorers: Optional[List[Dict[str, Any]]] = None,
|
|
35
36
|
):
|
|
36
37
|
self._client = client
|
|
37
38
|
self._payload: Dict[str, Any] = {
|
|
@@ -68,6 +69,11 @@ class EvaluationSettingsBuilder:
|
|
|
68
69
|
"enabled": True,
|
|
69
70
|
"models": list(sovereignty_models),
|
|
70
71
|
}
|
|
72
|
+
# Sandboxed JS scorers run per result alongside the judge - each entry is
|
|
73
|
+
# {"name": ..., "enabled": True, "code": "..."} where the code is a JS function body
|
|
74
|
+
# receiving (input, output, expected, toolCalls) and returning {score, reasoning}.
|
|
75
|
+
if code_scorers:
|
|
76
|
+
self._payload["codeScorers"] = list(code_scorers)
|
|
71
77
|
|
|
72
78
|
def publish(self) -> EvaluationSettings:
|
|
73
79
|
logger.info("Publishing evaluation settings '%s'", self._payload["name"])
|
|
@@ -96,6 +102,7 @@ class EvaluationSettingsClient:
|
|
|
96
102
|
rouge_score: bool = False,
|
|
97
103
|
similarity_model: Optional[str] = None,
|
|
98
104
|
sovereignty_models: Optional[List[str]] = None,
|
|
105
|
+
code_scorers: Optional[List[Dict[str, Any]]] = None,
|
|
99
106
|
) -> EvaluationSettingsBuilder:
|
|
100
107
|
return EvaluationSettingsBuilder(
|
|
101
108
|
self._client,
|
|
@@ -113,6 +120,7 @@ class EvaluationSettingsClient:
|
|
|
113
120
|
rouge_score=rouge_score,
|
|
114
121
|
similarity_model=similarity_model,
|
|
115
122
|
sovereignty_models=sovereignty_models,
|
|
123
|
+
code_scorers=code_scorers,
|
|
116
124
|
)
|
|
117
125
|
|
|
118
126
|
def get(self, evaluation_settings_id: str) -> EvaluationSettings:
|
|
@@ -51,5 +51,24 @@ class PromptClient:
|
|
|
51
51
|
second lookup method."""
|
|
52
52
|
return self._client.get_prompt(name, version=version)
|
|
53
53
|
|
|
54
|
+
def examples(self, prompt_id: str, window: Optional[str] = None) -> dict:
|
|
55
|
+
"""The merged evidence (worst eval-run results + low-rated online-evaluator traffic)
|
|
56
|
+
a propose() call will rewrite from - version-scoped to the prompt's current version."""
|
|
57
|
+
return self._client.get_prompt_examples(prompt_id, window=window)
|
|
58
|
+
|
|
59
|
+
def propose(self, prompt_id: str) -> dict:
|
|
60
|
+
"""Ask the judge for a rewrite grounded in examples(). Returns the proposal
|
|
61
|
+
(revisedText/reasoning/sourceBreakdown) without publishing anything."""
|
|
62
|
+
return self._client.propose_prompt(prompt_id)
|
|
63
|
+
|
|
64
|
+
def publish_version(
|
|
65
|
+
self, prompt_id: str, *, text: str, source: str = "proposed",
|
|
66
|
+
reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
|
|
67
|
+
) -> dict:
|
|
68
|
+
"""Publish a new version (the human-approval step of the propose flow)."""
|
|
69
|
+
return self._client.publish_prompt_version(
|
|
70
|
+
prompt_id, text=text, source=source, reasoning=reasoning, based_on_version=based_on_version
|
|
71
|
+
)
|
|
72
|
+
|
|
54
73
|
def list(self) -> List[Prompt]:
|
|
55
74
|
return self._client.list_prompts()
|
|
@@ -271,6 +271,13 @@ class EvaluationRunContext:
|
|
|
271
271
|
# UI reads, computed once in the API.
|
|
272
272
|
# ------------------------------------------------------------------
|
|
273
273
|
|
|
274
|
+
def results(self) -> list:
|
|
275
|
+
"""Per-result rows for this run (rating, justification, code scorer rows, trace ids,
|
|
276
|
+
latency/tokens, similarity metrics) - what the dashboard's run detail table shows,
|
|
277
|
+
fetched fresh from the engine."""
|
|
278
|
+
detail = self._client.get_run(self.run_id)
|
|
279
|
+
return detail.get("results", []) if isinstance(detail, dict) else []
|
|
280
|
+
|
|
274
281
|
@property
|
|
275
282
|
def run_id(self) -> str:
|
|
276
283
|
"""The server-side run id - handy for fetching the run's full results afterwards."""
|
|
@@ -362,11 +369,18 @@ class EvaluationRunContext:
|
|
|
362
369
|
try:
|
|
363
370
|
report = self._client.get_report(self._run.run_id)
|
|
364
371
|
except Exception as exc:
|
|
372
|
+
# Deliberately not status="completed". A placeholder that claims completion is
|
|
373
|
+
# indistinguishable from a real report of an evaluation that scored nothing -
|
|
374
|
+
# print_report renders empty statistics and no recommendations either way - so
|
|
375
|
+
# the one signal that something went wrong used to be a logger.warning that is
|
|
376
|
+
# invisible unless the caller configured logging. Say it on stdout, and let the
|
|
377
|
+
# status carry the truth for anything reading the object.
|
|
378
|
+
print(f" {red('✗')} Could not fetch the report: {dim(str(exc))}")
|
|
365
379
|
logger.warning("Could not fetch report: %s", exc)
|
|
366
380
|
report = Report(
|
|
367
381
|
runId=self._run.run_id,
|
|
368
382
|
datasetId=self._dataset.id,
|
|
369
|
-
status="
|
|
383
|
+
status="unavailable",
|
|
370
384
|
)
|
|
371
385
|
|
|
372
386
|
self._report = report
|
|
@@ -394,6 +408,7 @@ class EvaluationsRunner:
|
|
|
394
408
|
self.datasets = client.datasets
|
|
395
409
|
self.settings = client.settings
|
|
396
410
|
self.prompts = client.prompts
|
|
411
|
+
self.tool_schemas = client.tool_schemas
|
|
397
412
|
|
|
398
413
|
def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
|
|
399
414
|
"""List the LLM models AgentX supports - the same set selectable for
|
|
@@ -402,6 +417,20 @@ class EvaluationsRunner:
|
|
|
402
417
|
against."""
|
|
403
418
|
return self._client.list_models(provider)
|
|
404
419
|
|
|
420
|
+
def list_gates(self) -> list:
|
|
421
|
+
"""Recorded CI gate verdicts, newest first (the dashboard's CI Gates history)."""
|
|
422
|
+
return self._client.list_gates()
|
|
423
|
+
|
|
424
|
+
def simulate_conversation(self, **kwargs) -> dict:
|
|
425
|
+
"""Persona-driven multi-turn simulation against a prompt - see
|
|
426
|
+
EvaluationsClient.simulate_conversation for parameters."""
|
|
427
|
+
return self._client.simulate_conversation(**kwargs)
|
|
428
|
+
|
|
429
|
+
def get_run(self, run_id: str) -> dict:
|
|
430
|
+
"""Run summary + per-result rows by id, without needing the EvaluationRunContext that
|
|
431
|
+
created it (e.g. from a separate process)."""
|
|
432
|
+
return self._client.get_run(run_id)
|
|
433
|
+
|
|
405
434
|
def get_analysis_status(self, run_id: str) -> AnalysisStatus:
|
|
406
435
|
"""Check on an in-progress ``.analyze()`` job by run id, without needing
|
|
407
436
|
the ``EvaluationRunContext`` that started it (e.g. from a separate
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Tool schema registry client, surfaced as ``client.evaluations.tool_schemas``.
|
|
2
|
+
|
|
3
|
+
The tool-definition analog of the prompt registry: register the JSON definition your agent
|
|
4
|
+
actually passes to its LLM, let production failures accumulate as evidence against it, then
|
|
5
|
+
propose -> validate -> publish improved versions. See the dashboard's Improve > Tools & MCPs.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import List, Optional, TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from agentx.evaluations.client import EvaluationsClient
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ToolSchemaClient:
|
|
16
|
+
def __init__(self, client: "EvaluationsClient"):
|
|
17
|
+
self._client = client
|
|
18
|
+
|
|
19
|
+
def list(self) -> List[dict]:
|
|
20
|
+
return self._client.list_tool_schemas()
|
|
21
|
+
|
|
22
|
+
def create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
|
|
23
|
+
return self._client.create_tool_schema(name=name, definition=definition, description=description)
|
|
24
|
+
|
|
25
|
+
def get_or_create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
|
|
26
|
+
"""Idempotent register: returns the existing schema of this name if present."""
|
|
27
|
+
existing = next((t for t in self.list() if t.get("name") == name), None)
|
|
28
|
+
if existing:
|
|
29
|
+
return existing
|
|
30
|
+
return self.create(name=name, definition=definition, description=description)
|
|
31
|
+
|
|
32
|
+
def examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
|
|
33
|
+
"""Failure evidence recorded against this tool (agent-tool-failure signals, low-rated
|
|
34
|
+
eval results that called it) - what propose() rewrites from."""
|
|
35
|
+
return self._client.get_tool_schema_examples(tool_schema_id, window=window)
|
|
36
|
+
|
|
37
|
+
def propose(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
|
|
38
|
+
"""Judge-written definition rewrite grounded in examples(). Nothing is published."""
|
|
39
|
+
return self._client.propose_tool_schema(tool_schema_id, window=window)
|
|
40
|
+
|
|
41
|
+
def publish_version(
|
|
42
|
+
self, tool_schema_id: str, *, definition: str, source: str = "proposed",
|
|
43
|
+
reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
|
|
44
|
+
) -> dict:
|
|
45
|
+
return self._client.publish_tool_schema_version(
|
|
46
|
+
tool_schema_id, definition=definition, source=source,
|
|
47
|
+
reasoning=reasoning, based_on_version=based_on_version,
|
|
48
|
+
)
|
|
@@ -15,6 +15,10 @@ class AgentXAuthError(AgentXError):
|
|
|
15
15
|
"""Invalid or missing API key."""
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
class AgentXConnectionError(AgentXError):
|
|
19
|
+
"""The AgentX API (or self-host engine) could not be reached at the configured base_url."""
|
|
20
|
+
|
|
21
|
+
|
|
18
22
|
class AgentXAPIError(AgentXError):
|
|
19
23
|
"""Unexpected API error."""
|
|
20
24
|
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Agent registry helpers, surfaced as ``client.monitor.agents`` (self-host).
|
|
2
|
+
|
|
3
|
+
Agents in self-host are lightweight name rows - normally auto-created the first time a trace
|
|
4
|
+
arrives under a name. These helpers exist for flows that need the agent id before any traffic
|
|
5
|
+
(e.g. enabling a monitoring profile up front).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import List, Optional, TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from agentx.monitor.client import MonitorClient
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class MonitorAgentClient:
|
|
16
|
+
def __init__(self, client: "MonitorClient"):
|
|
17
|
+
self._client = client
|
|
18
|
+
|
|
19
|
+
def list(self) -> List[dict]:
|
|
20
|
+
return self._client.list_agents()
|
|
21
|
+
|
|
22
|
+
def create(self, name: str) -> dict:
|
|
23
|
+
return self._client.create_agent(name)
|
|
24
|
+
|
|
25
|
+
def ensure(self, name: str) -> dict:
|
|
26
|
+
"""Get-or-create by name - idempotent, safe to re-run."""
|
|
27
|
+
existing = next((a for a in self.list() if a.get("name") == name), None)
|
|
28
|
+
return existing if existing is not None else self.create(name)
|