agentx-python 0.6.27__tar.gz → 0.6.29__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {agentx_python-0.6.27/agentx_python.egg-info → agentx_python-0.6.29}/PKG-INFO +1 -1
  2. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/agentx.py +44 -0
  3. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/client.py +272 -11
  4. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/evaluation_settings.py +8 -0
  5. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/prompts.py +19 -0
  6. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/runner.py +30 -1
  7. agentx_python-0.6.29/agentx/evaluations/tool_schemas.py +48 -0
  8. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/exceptions.py +4 -0
  9. agentx_python-0.6.29/agentx/monitor/agents.py +28 -0
  10. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/client.py +89 -2
  11. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/online_evaluators.py +21 -0
  12. agentx_python-0.6.29/agentx/monitor/sessions.py +21 -0
  13. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/ingest_client.py +33 -0
  14. agentx_python-0.6.29/agentx/version.py +1 -0
  15. {agentx_python-0.6.27 → agentx_python-0.6.29/agentx_python.egg-info}/PKG-INFO +1 -1
  16. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/SOURCES.txt +4 -0
  17. agentx_python-0.6.29/tests/test_selfhost_analysis_fallback.py +251 -0
  18. agentx_python-0.6.27/agentx/version.py +0 -1
  19. {agentx_python-0.6.27 → agentx_python-0.6.29}/LICENSE +0 -0
  20. {agentx_python-0.6.27 → agentx_python-0.6.29}/README.md +0 -0
  21. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/__init__.py +0 -0
  22. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/cli.py +0 -0
  23. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/__init__.py +0 -0
  24. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/_term.py +0 -0
  25. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/__init__.py +0 -0
  26. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/http_endpoint.py +0 -0
  27. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/precomputed.py +0 -0
  28. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/adapters/raw.py +0 -0
  29. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/datasets.py +0 -0
  30. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/models.py +0 -0
  31. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/redaction.py +0 -0
  32. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/reporting.py +0 -0
  33. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/results.py +0 -0
  34. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/evaluations/tracing.py +0 -0
  35. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/feedback.py +0 -0
  36. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/__init__.py +0 -0
  37. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/_traced_call.py +0 -0
  38. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/anthropic.py +0 -0
  39. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/autogen.py +0 -0
  40. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/crewai.py +0 -0
  41. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/databricks.py +0 -0
  42. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/google_adk.py +0 -0
  43. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/google_genai.py +0 -0
  44. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/langchain.py +0 -0
  45. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/litellm.py +0 -0
  46. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/llamaindex.py +0 -0
  47. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/moveworks.py +0 -0
  48. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/openai.py +0 -0
  49. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/integrations/openai_agents.py +0 -0
  50. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/__init__.py +0 -0
  51. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/models.py +0 -0
  52. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/patterns.py +0 -0
  53. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/profile.py +0 -0
  54. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/monitor/signals.py +0 -0
  55. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/outcomes.py +0 -0
  56. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/py.typed +0 -0
  57. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/__init__.py +0 -0
  58. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/agent.py +0 -0
  59. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/conversation.py +0 -0
  60. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/resources/workforce.py +0 -0
  61. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/__init__.py +0 -0
  62. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/ci_types.py +0 -0
  63. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/tracing/tracer.py +0 -0
  64. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx/util.py +0 -0
  65. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/dependency_links.txt +0 -0
  66. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/entry_points.txt +0 -0
  67. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/not-zip-safe +0 -0
  68. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/requires.txt +0 -0
  69. {agentx_python-0.6.27 → agentx_python-0.6.29}/agentx_python.egg-info/top_level.txt +0 -0
  70. {agentx_python-0.6.27 → agentx_python-0.6.29}/setup.cfg +0 -0
  71. {agentx_python-0.6.27 → agentx_python-0.6.29}/setup.py +0 -0
  72. {agentx_python-0.6.27 → agentx_python-0.6.29}/tests/test_integration.py +0 -0
  73. {agentx_python-0.6.27 → agentx_python-0.6.29}/tests/test_integrations.py +0 -0
  74. {agentx_python-0.6.27 → agentx_python-0.6.29}/tests/test_span_tree.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: agentx-python
3
- Version: 0.6.27
3
+ Version: 0.6.29
4
4
  Summary: Official Python SDK for AgentX (https://www.agentx.so/)
5
5
  Home-page: https://github.com/AgentX-ai/AgentX-python
6
6
  Author: Robin Wang and AgentX Team
@@ -111,6 +111,50 @@ class AgentX:
111
111
  f"Failed to list workforces: {response.status_code} - {response.reason}"
112
112
  )
113
113
 
114
+ def ping(self) -> dict:
115
+ """Verify the client can actually reach AgentX and that the API key is accepted.
116
+
117
+ The constructor is deliberately lazy (no network call - standard SDK behavior, so
118
+ offline construction and tests work), and trace delivery is fire-and-forget, so a
119
+ wrong ``base_url`` or ``api_key`` otherwise surfaces only as a one-time warning in
120
+ logs while traces silently go nowhere. Call this once at startup of a long-running
121
+ service to fail fast instead::
122
+
123
+ client = AgentX.from_env()
124
+ client.ping() # raises immediately on a bad URL or key
125
+
126
+ Raises :class:`agentx.exceptions.AgentXConnectionError` when the URL is unreachable,
127
+ :class:`agentx.exceptions.AgentXAuthError` when the key is rejected, and
128
+ :class:`agentx.exceptions.AgentXAPIError` on any other non-OK response. Returns
129
+ ``{"ok": True, "base_url": ...}`` on success.
130
+ """
131
+ from agentx.exceptions import AgentXAPIError, AgentXAuthError, AgentXConnectionError
132
+
133
+ base = api_base()
134
+ # /monitor/patterns: the cheapest key-authenticated endpoint that exists on both the
135
+ # hosted API and the self-host engine's SDK-facing router.
136
+ url = f"{base}/monitor/patterns"
137
+ try:
138
+ response = requests.get(url, headers=get_headers(self.api_key), timeout=10)
139
+ except requests.RequestException as exc:
140
+ raise AgentXConnectionError(
141
+ f"Cannot reach AgentX at {base} ({exc.__class__.__name__}: {exc}). "
142
+ "Check base_url / AGENTX_API_BASE_URL - for self-host it should look like "
143
+ "http://localhost:4700/api/v1."
144
+ ) from exc
145
+ if response.status_code in (401, 403):
146
+ raise AgentXAuthError(
147
+ f"AgentX at {base} rejected the API key (HTTP {response.status_code}). "
148
+ "Check api_key / AGENTX_API_KEY - for self-host, copy the 'Default project "
149
+ "API key' from the engine's startup log."
150
+ )
151
+ if not response.ok:
152
+ raise AgentXAPIError(
153
+ f"AgentX at {base} responded HTTP {response.status_code} to the health probe.",
154
+ status_code=response.status_code,
155
+ )
156
+ return {"ok": True, "base_url": base}
157
+
114
158
  def get_profile(self):
115
159
  """Get the current user's profile information."""
116
160
  url = f"{api_base()}/access/getProfile"
@@ -32,9 +32,23 @@ _RETRYABLE_STATUS = {429, 500, 502, 503, 504}
32
32
  _MAX_RETRIES = 3
33
33
  _RETRY_BACKOFF = [1.0, 2.0, 4.0]
34
34
 
35
+ # The self-host analyze route judges every result before it responds, so the client has to
36
+ # wait out the whole job on one connection. Matches EvaluationRunContext.analyze()'s own
37
+ # default timeout.
38
+ _SELF_HOST_ANALYZE_TIMEOUT = 1800
39
+
35
40
 
36
41
  class AgentXEvaluationsError(Exception):
37
- pass
42
+ """An evaluations API call failed.
43
+
44
+ ``status_code`` carries the HTTP status when the failure came from a response rather
45
+ than from the transport, so callers can branch on it instead of matching on the message
46
+ text. It is ``None`` for connection errors and for retry exhaustion.
47
+ """
48
+
49
+ def __init__(self, message: str, status_code: Optional[int] = None) -> None:
50
+ super().__init__(message)
51
+ self.status_code = status_code
38
52
 
39
53
 
40
54
  class AgentXAuthError(AgentXEvaluationsError):
@@ -69,6 +83,8 @@ class EvaluationsClient:
69
83
  if not _api_base.endswith("/custom-agent-evaluations"):
70
84
  _api_base = f"{_api_base}/custom-agent-evaluations"
71
85
  self._base_url = _api_base
86
+ # None until an analysis call tells us which engine this is; see _api_root.
87
+ self._analysis_on_dashboard_router: Optional[bool] = None
72
88
  self._session = requests.Session()
73
89
  self._session.headers.update(
74
90
  {
@@ -86,6 +102,8 @@ class EvaluationsClient:
86
102
  self.datasets = DatasetClient(self)
87
103
  self.settings = EvaluationSettingsClient(self)
88
104
  self.prompts = PromptClient(self)
105
+ from agentx.evaluations.tool_schemas import ToolSchemaClient
106
+ self.tool_schemas = ToolSchemaClient(self)
89
107
 
90
108
  # ------------------------------------------------------------------
91
109
  # Low-level HTTP
@@ -103,10 +121,29 @@ class EvaluationsClient:
103
121
  """Same as _with_workspace(), for GET requests that take workspaceId as a query param."""
104
122
  return {"workspaceId": self._workspace_id} if self._workspace_id else None
105
123
 
106
- def _request(self, method: str, path: str, timeout: int = 30, **kwargs) -> Any:
107
- url = f"{self._base_url}{path}"
124
+ def _request(
125
+ self,
126
+ method: str,
127
+ path: str,
128
+ timeout: int = 30,
129
+ base: Optional[str] = None,
130
+ retry: bool = True,
131
+ **kwargs,
132
+ ) -> Any:
133
+ """Call the evaluations API.
134
+
135
+ ``base`` overrides the ``/custom-agent-evaluations`` prefix for the handful of
136
+ routes that live on a different router (see ``_api_root``).
137
+
138
+ ``retry=False`` disables the backoff loop entirely. Use it for any request that is
139
+ both slow and billable: the loop retries on ``requests.RequestException``, which
140
+ includes read timeouts, so a synchronous endpoint that outlives its timeout would
141
+ otherwise be re-invoked, and paid for, up to four times.
142
+ """
143
+ url = f"{base or self._base_url}{path}"
144
+ schedule = [0.0] + (_RETRY_BACKOFF if retry else [])
108
145
  last_exc: Optional[Exception] = None
109
- for attempt, wait in enumerate([0.0] + _RETRY_BACKOFF):
146
+ for attempt, wait in enumerate(schedule):
110
147
  if wait:
111
148
  time.sleep(wait)
112
149
  try:
@@ -120,14 +157,22 @@ class EvaluationsClient:
120
157
  raise AgentXAuthError("Invalid or missing API key")
121
158
  if resp.status_code == 422:
122
159
  raise AgentXValidationError(resp.text)
123
- if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
160
+ if (
161
+ resp.status_code in _RETRYABLE_STATUS
162
+ and retry
163
+ and attempt < _MAX_RETRIES - 1
164
+ ):
124
165
  logger.debug(
125
166
  "Retryable status %d (attempt %d)", resp.status_code, attempt + 1
126
167
  )
127
- last_exc = AgentXEvaluationsError(f"HTTP {resp.status_code}")
168
+ last_exc = AgentXEvaluationsError(
169
+ f"HTTP {resp.status_code}", status_code=resp.status_code
170
+ )
128
171
  continue
129
172
  if not resp.ok:
130
- raise AgentXEvaluationsError(f"HTTP {resp.status_code}: {resp.text}")
173
+ raise AgentXEvaluationsError(
174
+ f"HTTP {resp.status_code}: {resp.text}", status_code=resp.status_code
175
+ )
131
176
  try:
132
177
  return resp.json()
133
178
  except Exception:
@@ -303,6 +348,10 @@ class EvaluationsClient:
303
348
  # Starts the durable analysis job and returns immediately (e.g. {"jobId": ..., "status":
304
349
  # "pending"}); poll get_analysis_status() until it reaches a terminal status, then call
305
350
  # get_report(). mode/quality_mode/judges mirror the dashboard's AnalyzeEvaluationRequest.
351
+ #
352
+ # On self-host the fallback route runs the analysis synchronously and returns only when
353
+ # it is done, so the caller's poll loop sees a terminal status on its first check. The
354
+ # request is therefore given the full analysis timeout and, critically, no retries.
306
355
  payload: Dict[str, Any] = {}
307
356
  if mode is not None:
308
357
  payload["mode"] = mode
@@ -310,23 +359,235 @@ class EvaluationsClient:
310
359
  payload["qualityMode"] = quality_mode
311
360
  if judges is not None:
312
361
  payload["judges"] = [{"model": m} for m in judges]
313
- return self._request("POST", f"/runs/{run_id}/analyze", json=payload, timeout=30)
362
+
363
+ if not self._analysis_on_dashboard_router:
364
+ try:
365
+ return self._request(
366
+ "POST", f"/runs/{run_id}/analyze", json=payload, timeout=30
367
+ )
368
+ except AgentXEvaluationsError as exc:
369
+ if not self._note_missing_analysis_route(exc, "analyze"):
370
+ raise
371
+
372
+ return self._request(
373
+ "POST",
374
+ f"/evaluate/analyze/{run_id}",
375
+ base=self._api_root,
376
+ json=payload,
377
+ timeout=_SELF_HOST_ANALYZE_TIMEOUT,
378
+ retry=False,
379
+ )
314
380
 
315
381
  def get_analysis_status(self, run_id: str) -> AnalysisStatus:
316
- data = self._request("GET", f"/runs/{run_id}/analyze-status")
382
+ if not self._analysis_on_dashboard_router:
383
+ try:
384
+ return AnalysisStatus(
385
+ **self._request("GET", f"/runs/{run_id}/analyze-status")
386
+ )
387
+ except AgentXEvaluationsError as exc:
388
+ if not self._note_missing_analysis_route(exc, "analyze-status"):
389
+ raise
390
+
391
+ data = self._request(
392
+ "GET", f"/evaluate/analyze/{run_id}/status", base=self._api_root
393
+ )
317
394
  return AnalysisStatus(**data)
318
395
 
319
396
  def get_run(self, run_id: str) -> Dict[str, Any]:
320
397
  return self._request("GET", f"/runs/{run_id}")
321
398
 
322
399
  def get_report(self, run_id: str) -> Report:
323
- data = self._request("GET", f"/runs/{run_id}/report")
324
- return Report(**data)
400
+ if not self._analysis_on_dashboard_router:
401
+ try:
402
+ return Report(**self._request("GET", f"/runs/{run_id}/report"))
403
+ except AgentXEvaluationsError as exc:
404
+ if not self._note_missing_analysis_route(exc, "report"):
405
+ raise
406
+
407
+ return self._report_from_dashboard(run_id)
325
408
 
326
409
  def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
327
410
  data = self._request("GET", f"/runs/{run_id}/missing-results")
328
411
  return data if isinstance(data, list) else data.get("missing", [])
329
412
 
413
+ # ------------------------------------------------------------------
414
+ # Self-host analysis fallback
415
+ #
416
+ # The self-host engine (AgentX-trace-eval) mounts two routers: the SDK's
417
+ # /custom-agent-evaluations, and /evaluate for the dashboard. Its
418
+ # /custom-agent-evaluations router implements the run lifecycle - runs, results,
419
+ # finalize, gate - but not /analyze, /analyze-status or /report, which exist only on
420
+ # /evaluate. Hosted AgentX serves all of them from the SDK's own router.
421
+ #
422
+ # So the three analysis calls try the SDK route first and fall back to the dashboard
423
+ # route on a 404, which means hosted behaviour is byte-for-byte unchanged: it never
424
+ # 404s, so it never falls back. The outcome is cached on the client so the probe costs
425
+ # one request per process, not one per call.
426
+ # ------------------------------------------------------------------
427
+
428
+ @property
429
+ def _api_root(self) -> str:
430
+ """The API base with the ``/custom-agent-evaluations`` suffix removed."""
431
+ suffix = "/custom-agent-evaluations"
432
+ if self._base_url.endswith(suffix):
433
+ return self._base_url[: -len(suffix)]
434
+ return self._base_url
435
+
436
+ def _note_missing_analysis_route(
437
+ self, exc: AgentXEvaluationsError, route: str
438
+ ) -> bool:
439
+ """Return True if ``exc`` is the 404 that means "this engine is self-host".
440
+
441
+ Only a 404 qualifies. Anything else - auth, validation, a 500, a dead connection -
442
+ is a real failure on a route that does exist, and must propagate rather than be
443
+ retried against a different endpoint that would mask it.
444
+ """
445
+ if exc.status_code != 404:
446
+ return False
447
+ if self._analysis_on_dashboard_router is None:
448
+ logger.info(
449
+ "%s is not served from %s; using the dashboard router at %s "
450
+ "(self-host engine)",
451
+ route,
452
+ self._base_url,
453
+ self._api_root,
454
+ )
455
+ self._analysis_on_dashboard_router = True
456
+ return True
457
+
458
+ def _report_from_dashboard(self, run_id: str) -> Report:
459
+ """Assemble a Report from the dashboard's evaluation record.
460
+
461
+ Self-host has no /report route; it returns the analysis nested inside the
462
+ evaluation itself, under ``analysis.analysis`` with its statistics one level up.
463
+ The field names already match the Report models, so this is a reshape, not a
464
+ translation.
465
+ """
466
+ record = self._request("GET", f"/evaluate/{run_id}", base=self._api_root)
467
+ envelope = record.get("analysis") or {}
468
+ body = envelope.get("analysis") or {}
469
+
470
+ if not envelope:
471
+ raise AgentXEvaluationsError(
472
+ f"Run {run_id} has no analysis to report. Nothing has analyzed it yet, "
473
+ "or the analysis failed - call analyze_run() first."
474
+ )
475
+
476
+ dataset_id = record.get("datasetId")
477
+ if isinstance(dataset_id, dict): # populated reference, not a bare id
478
+ dataset_id = dataset_id.get("_id") or dataset_id.get("id")
479
+
480
+ return Report(
481
+ runId=run_id,
482
+ datasetId=dataset_id or "",
483
+ status=envelope.get("status") or "completed",
484
+ statistics=envelope.get("statistics"),
485
+ **body,
486
+ )
487
+
488
+ # ------------------------------------------------------------------
489
+ # Prompt improvement loop (examples -> propose -> publish). These ride the engine's
490
+ # /evaluate dialect via _api_root(), same precedent get_report/_analysis already use for
491
+ # routes that live on the dashboard router (self-host only).
492
+ # ------------------------------------------------------------------
493
+
494
+ def get_prompt_examples(self, prompt_id: str, window: Optional[str] = None) -> dict:
495
+ params = {"window": window} if window else None
496
+ return self._request(
497
+ "GET", f"/evaluate/prompts/{prompt_id}/examples", base=self._api_root, params=params
498
+ )
499
+
500
+ def propose_prompt(self, prompt_id: str) -> dict:
501
+ # One real judge call - no retry (see _request's retry note).
502
+ return self._request(
503
+ "POST", f"/evaluate/prompts/{prompt_id}/propose", base=self._api_root, timeout=180, retry=False
504
+ )
505
+
506
+ def publish_prompt_version(
507
+ self, prompt_id: str, *, text: str, source: str = "proposed",
508
+ reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
509
+ ) -> dict:
510
+ payload: dict = {"text": text, "source": source}
511
+ if reasoning is not None:
512
+ payload["reasoning"] = reasoning
513
+ if based_on_version is not None:
514
+ payload["basedOnVersion"] = based_on_version
515
+ return self._request(
516
+ "POST", f"/evaluate/prompts/{prompt_id}/versions", base=self._api_root, json=payload
517
+ )
518
+
519
+ # ------------------------------------------------------------------
520
+ # Tool schema registry (same version-scoped propose/publish loop as prompts)
521
+ # ------------------------------------------------------------------
522
+
523
+ def list_tool_schemas(self) -> List[dict]:
524
+ data = self._request("GET", "/evaluate/tool-schemas", base=self._api_root)
525
+ return data.get("toolSchemas", []) if isinstance(data, dict) else data
526
+
527
+ def create_tool_schema(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
528
+ payload: dict = {"name": name, "definition": definition}
529
+ if description is not None:
530
+ payload["description"] = description
531
+ return self._request("POST", "/evaluate/tool-schemas", base=self._api_root, json=payload)
532
+
533
+ def get_tool_schema_examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
534
+ params = {"window": window} if window else None
535
+ return self._request(
536
+ "GET", f"/evaluate/tool-schemas/{tool_schema_id}/examples", base=self._api_root, params=params
537
+ )
538
+
539
+ def propose_tool_schema(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
540
+ payload = {"window": window} if window else {}
541
+ return self._request(
542
+ "POST", f"/evaluate/tool-schemas/{tool_schema_id}/propose",
543
+ base=self._api_root, json=payload, timeout=180, retry=False,
544
+ )
545
+
546
+ def publish_tool_schema_version(
547
+ self, tool_schema_id: str, *, definition: str, source: str = "proposed",
548
+ reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
549
+ ) -> dict:
550
+ payload: dict = {"definition": definition, "source": source}
551
+ if reasoning is not None:
552
+ payload["reasoning"] = reasoning
553
+ if based_on_version is not None:
554
+ payload["basedOnVersion"] = based_on_version
555
+ return self._request(
556
+ "POST", f"/evaluate/tool-schemas/{tool_schema_id}/versions", base=self._api_root, json=payload
557
+ )
558
+
559
+ # ------------------------------------------------------------------
560
+ # CI gate history + conversation simulation (self-host)
561
+ # ------------------------------------------------------------------
562
+
563
+ def list_gates(self) -> List[dict]:
564
+ """Recorded CI gate verdicts, newest first - the dashboard's CI Gates history."""
565
+ data = self._request("GET", "/evaluate/ci/gates", base=self._api_root)
566
+ return data.get("gates", []) if isinstance(data, dict) else data
567
+
568
+ def simulate_conversation(
569
+ self, *, model: str, system_prompt: str, persona: str, goal: str,
570
+ max_turns: int = 5, tools: Optional[List[dict]] = None, agent_name: Optional[str] = None,
571
+ ) -> dict:
572
+ """Run a persona-driven multi-turn simulation against a prompt (the Playground's
573
+ "Simulate conversation"). Blocking and judge-billed: one LLM call per simulated turn
574
+ plus the closing judgment, so expect it to take tens of seconds."""
575
+ payload: dict = {
576
+ "model": model,
577
+ "messages": [{"role": "system", "content": system_prompt}],
578
+ "persona": persona,
579
+ "goal": goal,
580
+ "maxTurns": max_turns,
581
+ }
582
+ if tools:
583
+ payload["tools"] = tools
584
+ if agent_name:
585
+ payload["agentName"] = agent_name
586
+ return self._request(
587
+ "POST", "/evaluate/playground/simulate", base=self._api_root, json=payload,
588
+ timeout=600, retry=False,
589
+ )
590
+
330
591
 
331
592
  # ---------------------------------------------------------------------------
332
593
  # Helpers
@@ -32,6 +32,7 @@ class EvaluationSettingsBuilder:
32
32
  rouge_score: bool = False,
33
33
  similarity_model: Optional[str] = None,
34
34
  sovereignty_models: Optional[List[str]] = None,
35
+ code_scorers: Optional[List[Dict[str, Any]]] = None,
35
36
  ):
36
37
  self._client = client
37
38
  self._payload: Dict[str, Any] = {
@@ -68,6 +69,11 @@ class EvaluationSettingsBuilder:
68
69
  "enabled": True,
69
70
  "models": list(sovereignty_models),
70
71
  }
72
+ # Sandboxed JS scorers run per result alongside the judge - each entry is
73
+ # {"name": ..., "enabled": True, "code": "..."} where the code is a JS function body
74
+ # receiving (input, output, expected, toolCalls) and returning {score, reasoning}.
75
+ if code_scorers:
76
+ self._payload["codeScorers"] = list(code_scorers)
71
77
 
72
78
  def publish(self) -> EvaluationSettings:
73
79
  logger.info("Publishing evaluation settings '%s'", self._payload["name"])
@@ -96,6 +102,7 @@ class EvaluationSettingsClient:
96
102
  rouge_score: bool = False,
97
103
  similarity_model: Optional[str] = None,
98
104
  sovereignty_models: Optional[List[str]] = None,
105
+ code_scorers: Optional[List[Dict[str, Any]]] = None,
99
106
  ) -> EvaluationSettingsBuilder:
100
107
  return EvaluationSettingsBuilder(
101
108
  self._client,
@@ -113,6 +120,7 @@ class EvaluationSettingsClient:
113
120
  rouge_score=rouge_score,
114
121
  similarity_model=similarity_model,
115
122
  sovereignty_models=sovereignty_models,
123
+ code_scorers=code_scorers,
116
124
  )
117
125
 
118
126
  def get(self, evaluation_settings_id: str) -> EvaluationSettings:
@@ -51,5 +51,24 @@ class PromptClient:
51
51
  second lookup method."""
52
52
  return self._client.get_prompt(name, version=version)
53
53
 
54
+ def examples(self, prompt_id: str, window: Optional[str] = None) -> dict:
55
+ """The merged evidence (worst eval-run results + low-rated online-evaluator traffic)
56
+ a propose() call will rewrite from - version-scoped to the prompt's current version."""
57
+ return self._client.get_prompt_examples(prompt_id, window=window)
58
+
59
+ def propose(self, prompt_id: str) -> dict:
60
+ """Ask the judge for a rewrite grounded in examples(). Returns the proposal
61
+ (revisedText/reasoning/sourceBreakdown) without publishing anything."""
62
+ return self._client.propose_prompt(prompt_id)
63
+
64
+ def publish_version(
65
+ self, prompt_id: str, *, text: str, source: str = "proposed",
66
+ reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
67
+ ) -> dict:
68
+ """Publish a new version (the human-approval step of the propose flow)."""
69
+ return self._client.publish_prompt_version(
70
+ prompt_id, text=text, source=source, reasoning=reasoning, based_on_version=based_on_version
71
+ )
72
+
54
73
  def list(self) -> List[Prompt]:
55
74
  return self._client.list_prompts()
@@ -271,6 +271,13 @@ class EvaluationRunContext:
271
271
  # UI reads, computed once in the API.
272
272
  # ------------------------------------------------------------------
273
273
 
274
+ def results(self) -> list:
275
+ """Per-result rows for this run (rating, justification, code scorer rows, trace ids,
276
+ latency/tokens, similarity metrics) - what the dashboard's run detail table shows,
277
+ fetched fresh from the engine."""
278
+ detail = self._client.get_run(self.run_id)
279
+ return detail.get("results", []) if isinstance(detail, dict) else []
280
+
274
281
  @property
275
282
  def run_id(self) -> str:
276
283
  """The server-side run id - handy for fetching the run's full results afterwards."""
@@ -362,11 +369,18 @@ class EvaluationRunContext:
362
369
  try:
363
370
  report = self._client.get_report(self._run.run_id)
364
371
  except Exception as exc:
372
+ # Deliberately not status="completed". A placeholder that claims completion is
373
+ # indistinguishable from a real report of an evaluation that scored nothing -
374
+ # print_report renders empty statistics and no recommendations either way - so
375
+ # the one signal that something went wrong used to be a logger.warning that is
376
+ # invisible unless the caller configured logging. Say it on stdout, and let the
377
+ # status carry the truth for anything reading the object.
378
+ print(f" {red('✗')} Could not fetch the report: {dim(str(exc))}")
365
379
  logger.warning("Could not fetch report: %s", exc)
366
380
  report = Report(
367
381
  runId=self._run.run_id,
368
382
  datasetId=self._dataset.id,
369
- status="completed",
383
+ status="unavailable",
370
384
  )
371
385
 
372
386
  self._report = report
@@ -394,6 +408,7 @@ class EvaluationsRunner:
394
408
  self.datasets = client.datasets
395
409
  self.settings = client.settings
396
410
  self.prompts = client.prompts
411
+ self.tool_schemas = client.tool_schemas
397
412
 
398
413
  def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
399
414
  """List the LLM models AgentX supports - the same set selectable for
@@ -402,6 +417,20 @@ class EvaluationsRunner:
402
417
  against."""
403
418
  return self._client.list_models(provider)
404
419
 
420
+ def list_gates(self) -> list:
421
+ """Recorded CI gate verdicts, newest first (the dashboard's CI Gates history)."""
422
+ return self._client.list_gates()
423
+
424
+ def simulate_conversation(self, **kwargs) -> dict:
425
+ """Persona-driven multi-turn simulation against a prompt - see
426
+ EvaluationsClient.simulate_conversation for parameters."""
427
+ return self._client.simulate_conversation(**kwargs)
428
+
429
+ def get_run(self, run_id: str) -> dict:
430
+ """Run summary + per-result rows by id, without needing the EvaluationRunContext that
431
+ created it (e.g. from a separate process)."""
432
+ return self._client.get_run(run_id)
433
+
405
434
  def get_analysis_status(self, run_id: str) -> AnalysisStatus:
406
435
  """Check on an in-progress ``.analyze()`` job by run id, without needing
407
436
  the ``EvaluationRunContext`` that started it (e.g. from a separate
@@ -0,0 +1,48 @@
1
+ """Tool schema registry client, surfaced as ``client.evaluations.tool_schemas``.
2
+
3
+ The tool-definition analog of the prompt registry: register the JSON definition your agent
4
+ actually passes to its LLM, let production failures accumulate as evidence against it, then
5
+ propose -> validate -> publish improved versions. See the dashboard's Improve > Tools & MCPs.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from typing import List, Optional, TYPE_CHECKING
10
+
11
+ if TYPE_CHECKING:
12
+ from agentx.evaluations.client import EvaluationsClient
13
+
14
+
15
+ class ToolSchemaClient:
16
+ def __init__(self, client: "EvaluationsClient"):
17
+ self._client = client
18
+
19
+ def list(self) -> List[dict]:
20
+ return self._client.list_tool_schemas()
21
+
22
+ def create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
23
+ return self._client.create_tool_schema(name=name, definition=definition, description=description)
24
+
25
+ def get_or_create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
26
+ """Idempotent register: returns the existing schema of this name if present."""
27
+ existing = next((t for t in self.list() if t.get("name") == name), None)
28
+ if existing:
29
+ return existing
30
+ return self.create(name=name, definition=definition, description=description)
31
+
32
+ def examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
33
+ """Failure evidence recorded against this tool (agent-tool-failure signals, low-rated
34
+ eval results that called it) - what propose() rewrites from."""
35
+ return self._client.get_tool_schema_examples(tool_schema_id, window=window)
36
+
37
+ def propose(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
38
+ """Judge-written definition rewrite grounded in examples(). Nothing is published."""
39
+ return self._client.propose_tool_schema(tool_schema_id, window=window)
40
+
41
+ def publish_version(
42
+ self, tool_schema_id: str, *, definition: str, source: str = "proposed",
43
+ reasoning: Optional[str] = None, based_on_version: Optional[int] = None,
44
+ ) -> dict:
45
+ return self._client.publish_tool_schema_version(
46
+ tool_schema_id, definition=definition, source=source,
47
+ reasoning=reasoning, based_on_version=based_on_version,
48
+ )
@@ -15,6 +15,10 @@ class AgentXAuthError(AgentXError):
15
15
  """Invalid or missing API key."""
16
16
 
17
17
 
18
+ class AgentXConnectionError(AgentXError):
19
+ """The AgentX API (or self-host engine) could not be reached at the configured base_url."""
20
+
21
+
18
22
  class AgentXAPIError(AgentXError):
19
23
  """Unexpected API error."""
20
24
 
@@ -0,0 +1,28 @@
1
+ """Agent registry helpers, surfaced as ``client.monitor.agents`` (self-host).
2
+
3
+ Agents in self-host are lightweight name rows - normally auto-created the first time a trace
4
+ arrives under a name. These helpers exist for flows that need the agent id before any traffic
5
+ (e.g. enabling a monitoring profile up front).
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from typing import List, Optional, TYPE_CHECKING
10
+
11
+ if TYPE_CHECKING:
12
+ from agentx.monitor.client import MonitorClient
13
+
14
+
15
+ class MonitorAgentClient:
16
+ def __init__(self, client: "MonitorClient"):
17
+ self._client = client
18
+
19
+ def list(self) -> List[dict]:
20
+ return self._client.list_agents()
21
+
22
+ def create(self, name: str) -> dict:
23
+ return self._client.create_agent(name)
24
+
25
+ def ensure(self, name: str) -> dict:
26
+ """Get-or-create by name - idempotent, safe to re-run."""
27
+ existing = next((a for a in self.list() if a.get("name") == name), None)
28
+ return existing if existing is not None else self.create(name)