agentx-python 0.8.21__tar.gz → 0.8.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. {agentx_python-0.8.21/agentx_python.egg-info → agentx_python-0.8.22}/PKG-INFO +1 -1
  2. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/__init__.py +4 -0
  3. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/agentx.py +35 -5
  4. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/client.py +27 -19
  5. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/models.py +27 -2
  6. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/runner.py +77 -35
  7. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/exceptions.py +19 -1
  8. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/export.py +2 -1
  9. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/feedback.py +2 -1
  10. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/autogen.py +4 -0
  11. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/crewai.py +66 -28
  12. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/google_adk.py +33 -1
  13. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/langchain.py +87 -53
  14. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/llamaindex.py +85 -46
  15. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/openai_agents.py +18 -2
  16. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/client.py +27 -13
  17. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/improvement_groups.py +2 -1
  18. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/judge_scorers.py +11 -2
  19. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/scorer_groups.py +3 -1
  20. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/scorers.py +2 -1
  21. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/outcomes.py +2 -1
  22. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/projects.py +2 -1
  23. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/traces.py +2 -1
  24. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/ingest_client.py +31 -5
  25. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/tracer.py +47 -17
  26. agentx_python-0.8.22/agentx/util.py +29 -0
  27. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/version.py +2 -2
  28. {agentx_python-0.8.21 → agentx_python-0.8.22/agentx_python.egg-info}/PKG-INFO +1 -1
  29. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/SOURCES.txt +3 -1
  30. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_deep_dive_fixes.py +45 -0
  31. agentx_python-0.8.22/tests/test_error_taxonomy.py +86 -0
  32. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_integrations.py +1 -1
  33. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_judge_scorers.py +47 -0
  34. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_runner_features.py +73 -0
  35. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_span_tree.py +232 -3
  36. agentx_python-0.8.22/tests/test_wire_models.py +83 -0
  37. agentx_python-0.8.21/agentx/util.py +0 -20
  38. {agentx_python-0.8.21 → agentx_python-0.8.22}/LICENSE +0 -0
  39. {agentx_python-0.8.21 → agentx_python-0.8.22}/README.md +0 -0
  40. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/cli.py +0 -0
  41. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/__init__.py +0 -0
  42. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/_term.py +0 -0
  43. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/__init__.py +0 -0
  44. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/http_endpoint.py +0 -0
  45. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/precomputed.py +0 -0
  46. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/raw.py +0 -0
  47. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/datasets.py +0 -0
  48. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/evaluation_settings.py +0 -0
  49. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/prompts.py +0 -0
  50. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/reporting.py +0 -0
  51. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/results.py +0 -0
  52. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/tool_schemas.py +0 -0
  53. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/tracing.py +0 -0
  54. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/__init__.py +0 -0
  55. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/_traced_call.py +0 -0
  56. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/anthropic.py +0 -0
  57. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/databricks.py +0 -0
  58. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/google_genai.py +0 -0
  59. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/litellm.py +0 -0
  60. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/moveworks.py +0 -0
  61. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/openai.py +0 -0
  62. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/__init__.py +0 -0
  63. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/agents.py +0 -0
  64. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/models.py +0 -0
  65. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/online_evaluators.py +0 -0
  66. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/patterns.py +0 -0
  67. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/profile.py +0 -0
  68. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/review_queue.py +0 -0
  69. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/rules.py +0 -0
  70. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/sessions.py +0 -0
  71. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/signals.py +0 -0
  72. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/py.typed +0 -0
  73. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/__init__.py +0 -0
  74. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/agent.py +0 -0
  75. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/conversation.py +0 -0
  76. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/workforce.py +0 -0
  77. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/testing.py +0 -0
  78. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/__init__.py +0 -0
  79. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/ci_types.py +0 -0
  80. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/eval_scope.py +0 -0
  81. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/framework_detect.py +0 -0
  82. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/dependency_links.txt +0 -0
  83. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/entry_points.txt +0 -0
  84. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/not-zip-safe +0 -0
  85. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/requires.txt +0 -0
  86. {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/top_level.txt +0 -0
  87. {agentx_python-0.8.21 → agentx_python-0.8.22}/setup.cfg +0 -0
  88. {agentx_python-0.8.21 → agentx_python-0.8.22}/setup.py +0 -0
  89. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_cli_launcher.py +0 -0
  90. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_docs_match_sdk.py +0 -0
  91. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_eval_scope.py +0 -0
  92. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_framework_detect.py +0 -0
  93. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_integration.py +0 -0
  94. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_multi_judge.py +0 -0
  95. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_pairwise.py +0 -0
  96. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_review_queue.py +0 -0
  97. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_selfhost_analysis_fallback.py +0 -0
  98. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_selfhost_compat.py +0 -0
  99. {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_testing.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: agentx-python
3
- Version: 0.8.21
3
+ Version: 0.8.22
4
4
  Summary: Official Python SDK for AgentX (https://www.agentx.so/)
5
5
  Home-page: https://github.com/AgentX-ai/AgentX-python
6
6
  Author: Robin Wang and AgentX Team
@@ -5,6 +5,8 @@ from agentx.version import VERSION
5
5
  from agentx.exceptions import (
6
6
  AgentXError,
7
7
  AgentXAuthError,
8
+ AgentXValidationError,
9
+ AgentXConnectionError,
8
10
  AgentXAPIError,
9
11
  DatasetNotFound,
10
12
  CINotEnabled,
@@ -21,6 +23,8 @@ __all__ = [
21
23
  "AgentX",
22
24
  "AgentXError",
23
25
  "AgentXAuthError",
26
+ "AgentXValidationError",
27
+ "AgentXConnectionError",
24
28
  "AgentXAPIError",
25
29
  "DatasetNotFound",
26
30
  "CINotEnabled",
@@ -3,7 +3,7 @@ import requests
3
3
  import os
4
4
  import logging
5
5
 
6
- from agentx.util import get_headers, api_base
6
+ from agentx.util import get_headers, api_base, normalize_base
7
7
  from agentx.resources.agent import Agent
8
8
  from agentx.resources.workforce import Workforce
9
9
 
@@ -16,16 +16,23 @@ class AgentX:
16
16
  base_url: Optional[str] = None,
17
17
  workspace_id: Optional[str] = None,
18
18
  ):
19
+ # The api_key is NOT written back into os.environ (it used to be): every sub-client
20
+ # below receives it explicitly, and mutating process-global state from a constructor
21
+ # re-pointed unrelated code - the same leak the base_url write below had (deep-dive
22
+ # round 3, bug #1). Static flows that still read the env (AgentX.list_workforces,
23
+ # bare get_headers()) now require the caller to set AGENTX_API_KEY themselves.
19
24
  self.api_key = api_key or os.getenv("AGENTX_API_KEY")
20
- if self.api_key and not os.getenv("AGENTX_API_KEY"):
21
- os.environ["AGENTX_API_KEY"] = self.api_key
22
25
 
23
26
  # base_url overrides AGENTX_API_BASE_URL env var (and the SDK default). It is
24
27
  # deliberately NOT written back into os.environ: the constructor used to do that, which
25
28
  # made the last-constructed client silently re-point every other client in the process
26
29
  # (deep-dive round 3, bug #1). Each sub-client below receives this value explicitly and
27
- # captures it at construction instead.
30
+ # captures it at construction instead. Normalized (trailing slash and the
31
+ # /custom-agent-evaluations suffix stripped) so an evaluations-shaped URL works for
32
+ # every sub-client, not just evaluations.
28
33
  self.base_url = base_url or os.getenv("AGENTX_API_BASE_URL")
34
+ if self.base_url:
35
+ self.base_url = normalize_base(self.base_url)
29
36
 
30
37
  self.workspace_id = workspace_id or os.getenv("AGENTX_WORKSPACE_ID")
31
38
 
@@ -91,6 +98,27 @@ class AgentX:
91
98
  workspace_id=self.workspace_id,
92
99
  )
93
100
  self.tracer = Tracer(_ingest_client)
101
+ self._ingest_client = _ingest_client
102
+
103
+ # ------------------------------------------------------------------
104
+ # Lifecycle
105
+ # ------------------------------------------------------------------
106
+
107
+ def close(self, timeout: float = 5.0) -> bool:
108
+ """Flush queued traces and stop the tracer's background ingest worker.
109
+
110
+ Returns ``True`` when everything drained before ``timeout`` seconds elapsed. Optional -
111
+ an ``atexit`` hook already flushes queued traces on interpreter shutdown - but a
112
+ long-running service that tears clients down mid-process should call it (or use the
113
+ client as a context manager) so worker threads don't accumulate.
114
+ """
115
+ return self._ingest_client.close(timeout)
116
+
117
+ def __enter__(self) -> "AgentX":
118
+ return self
119
+
120
+ def __exit__(self, exc_type, exc_val, tb) -> None:
121
+ self.close()
94
122
 
95
123
  @classmethod
96
124
  def from_env(cls) -> "AgentX":
@@ -129,7 +157,9 @@ class AgentX:
129
157
 
130
158
  @staticmethod
131
159
  def list_workforces() -> List["Workforce"]:
132
- """List all workforces/teams."""
160
+ """List all workforces/teams. Static, so it reads AGENTX_API_KEY from the environment
161
+ directly - the constructor no longer writes ``api_key`` into os.environ, so set the
162
+ env var yourself before calling this."""
133
163
  url = f"{api_base()}/access/teams"
134
164
  response = requests.get(url, headers=get_headers())
135
165
  if response.status_code == 200:
@@ -24,7 +24,12 @@ from agentx.evaluations.models import (
24
24
 
25
25
  logger = logging.getLogger(__name__)
26
26
 
27
- from agentx.util import _DEFAULT_API_BASE as _UTIL_API_BASE
27
+ from agentx.util import _DEFAULT_API_BASE as _UTIL_API_BASE, normalize_base
28
+
29
+ # The canonical error classes (agentx.exceptions) are raised - and re-exported here for
30
+ # compat with code that imported them from this module - so `except agentx.AgentXAuthError`
31
+ # works whichever client raised.
32
+ from agentx.exceptions import AgentXError, AgentXAuthError, AgentXValidationError
28
33
 
29
34
  _DEFAULT_BASE_URL = f"{_UTIL_API_BASE}/custom-agent-evaluations"
30
35
  SDK_NAME = "agentx-python"
@@ -42,7 +47,7 @@ _SELF_HOST_ANALYZE_TIMEOUT = 1800
42
47
  _SELF_HOST_SCORING_TIMEOUT = 900
43
48
 
44
49
 
45
- class AgentXEvaluationsError(Exception):
50
+ class AgentXEvaluationsError(AgentXError):
46
51
  """An evaluations API call failed.
47
52
 
48
53
  ``status_code`` carries the HTTP status when the failure came from a response rather
@@ -55,14 +60,6 @@ class AgentXEvaluationsError(Exception):
55
60
  self.status_code = status_code
56
61
 
57
62
 
58
- class AgentXAuthError(AgentXEvaluationsError):
59
- pass
60
-
61
-
62
- class AgentXValidationError(AgentXEvaluationsError):
63
- pass
64
-
65
-
66
63
  class EvaluationSubmissionError(AgentXEvaluationsError):
67
64
  """A result batch could not be submitted (after one retry). The run is left unfinalized;
68
65
  re-running execute() on the same context resumes past already-submitted cases."""
@@ -96,13 +93,10 @@ class EvaluationsClient:
96
93
  # whatever workspace the API key's user defaults to, not the one the caller intended.
97
94
  self._workspace_id = workspace_id
98
95
  # Priority: constructor arg > env var > SDK default
99
- # Always append /custom-agent-evaluations so users only need to provide /api/v1
100
- _api_base = (
101
- base_url or os.getenv("AGENTX_API_BASE_URL", _UTIL_API_BASE)
102
- ).rstrip("/")
103
- if not _api_base.endswith("/custom-agent-evaluations"):
104
- _api_base = f"{_api_base}/custom-agent-evaluations"
105
- self._base_url = _api_base
96
+ # normalize_base strips a trailing slash and any /custom-agent-evaluations suffix,
97
+ # then the suffix is appended - users only need to provide /api/v1 either way.
98
+ _api_base = normalize_base(base_url or os.getenv("AGENTX_API_BASE_URL", _UTIL_API_BASE))
99
+ self._base_url = f"{_api_base}/custom-agent-evaluations"
106
100
  # None until an analysis call tells us which engine this is; see _api_root.
107
101
  self._analysis_on_dashboard_router: Optional[bool] = None
108
102
  self._session = requests.Session()
@@ -184,13 +178,16 @@ class EvaluationsClient:
184
178
  continue
185
179
 
186
180
  if resp.status_code == 401:
187
- raise AgentXAuthError("Invalid or missing API key")
181
+ raise AgentXAuthError("Invalid or missing API key", status_code=401)
188
182
  if resp.status_code == 422:
189
183
  raise AgentXValidationError(resp.text)
184
+ # Gate on the schedule itself so HTTP-status retries walk the SAME full backoff
185
+ # schedule connection errors do - the old `attempt < _MAX_RETRIES - 1` gate left
186
+ # the schedule's last entry unreachable for HTTP retries (ingest_client precedent).
190
187
  if (
191
188
  resp.status_code in _RETRYABLE_STATUS
192
189
  and retry
193
- and attempt < _MAX_RETRIES - 1
190
+ and attempt < len(schedule) - 1
194
191
  ):
195
192
  logger.debug(
196
193
  "Retryable status %d (attempt %d)", resp.status_code, attempt + 1
@@ -500,6 +497,17 @@ class EvaluationsClient:
500
497
  return self._report_from_dashboard(run_id)
501
498
 
502
499
  def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
500
+ """Deprecated: on self-host the route's response body has no top-level list, so this
501
+ always returns ``[]``. Use :meth:`get_submitted_keys` - the same route's
502
+ ``submittedKeys`` - to find out what a run still needs."""
503
+ import warnings
504
+
505
+ warnings.warn(
506
+ "get_missing_results() always returns [] on self-host - use get_submitted_keys() "
507
+ "to resume a run instead.",
508
+ DeprecationWarning,
509
+ stacklevel=2,
510
+ )
503
511
  data = self._request("GET", f"/runs/{run_id}/missing-results")
504
512
  return data if isinstance(data, list) else data.get("missing", [])
505
513
 
@@ -1,7 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  from typing import Any, Dict, List, Literal, Optional, Union
4
- from pydantic import AliasChoices, BaseModel, Field, model_validator
4
+ from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
5
5
 
6
6
  # ---------------------------------------------------------------------------
7
7
  # Observable trace
@@ -78,6 +78,17 @@ class Dataset(BaseModel):
78
78
  # Custom code scorers attached to this dataset - [{ id, name, code, enabled }]. Retrievable,
79
79
  # so a fetched dataset round-trips them (import_dataset copies them to the new dataset).
80
80
  code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers")
81
+ # Grading config carried on the dataset itself - similarity metric toggles (each a
82
+ # {"enabled": bool, ...} object on the wire), LLM-as-judge overrides, and the raw
83
+ # sovereigntyIndex object. Modeled so a fetched Dataset round-trips them: extra="ignore"
84
+ # used to silently drop all of these on read, and import_dataset lost them on the copy.
85
+ vector_similarity: Optional[Any] = Field(default=None, alias="vectorSimilarity")
86
+ jaccard_similarity: Optional[Any] = Field(default=None, alias="jaccardSimilarity")
87
+ bleu_score: Optional[Any] = Field(default=None, alias="bleuScore")
88
+ rouge_score: Optional[Any] = Field(default=None, alias="rougeScore")
89
+ judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt")
90
+ judge_model: Optional[str] = Field(default=None, alias="judgeModel")
91
+ sovereignty_index: Optional[Dict[str, Any]] = Field(default=None, alias="sovereigntyIndex")
81
92
  status: str = "published"
82
93
  version_id: Optional[str] = Field(default=None, alias="versionId")
83
94
  # Sovereignty & Portability - models selected to compare on this dataset.
@@ -430,7 +441,12 @@ class RunResultRow(BaseModel):
430
441
  question_index: Optional[int] = Field(default=None, alias="questionIndex")
431
442
  run_number: Optional[int] = Field(default=None, alias="runNumber")
432
443
  question_text: Optional[str] = Field(default=None, alias="questionText")
433
- response: Optional[str] = None
444
+ # The engine sends the agent's answer as an `output` OBJECT ({"text": ...}), not a
445
+ # `response` string - accept both spellings and lift the dict's text (see the
446
+ # validator below), so row.response actually populates on self-host.
447
+ response: Optional[str] = Field(
448
+ default=None, validation_alias=AliasChoices("response", "output")
449
+ )
434
450
  trace_id: Optional[str] = Field(default=None, alias="traceId")
435
451
  latency_ms: Optional[float] = Field(default=None, alias="latencyMs")
436
452
  input_tokens: Optional[int] = Field(default=None, alias="inputTokens")
@@ -452,6 +468,15 @@ class RunResultRow(BaseModel):
452
468
  populate_by_name = True
453
469
  extra = "ignore"
454
470
 
471
+ @field_validator("response", mode="before")
472
+ @classmethod
473
+ def _lift_output_text(cls, value: Any) -> Any:
474
+ # The `output` alias delivers the wire's whole output object - keep the declared
475
+ # Optional[str] by lifting its text field.
476
+ if isinstance(value, dict):
477
+ return value.get("text")
478
+ return value
479
+
455
480
  @classmethod
456
481
  def from_wire(cls, wire: Dict[str, Any]) -> "RunResultRow":
457
482
  row = cls.model_validate(wire)
@@ -11,7 +11,11 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Union
11
11
  from agentx.evaluations.adapters.raw import RawCallableAdapter
12
12
  from agentx.evaluations.adapters.precomputed import PrecomputedAdapter
13
13
  from agentx.evaluations.adapters.http_endpoint import HttpEndpointAdapter
14
- from agentx.evaluations.client import EvaluationsClient, EvaluationSubmissionError
14
+ from agentx.evaluations.client import (
15
+ AgentXEvaluationsError,
16
+ EvaluationsClient,
17
+ EvaluationSubmissionError,
18
+ )
15
19
  from agentx.evaluations.models import (
16
20
  AnalysisStatus,
17
21
  Dataset,
@@ -227,6 +231,7 @@ class EvaluationRunContext:
227
231
  if concurrency > 1:
228
232
  import concurrent.futures
229
233
  import contextvars
234
+ from collections import deque
230
235
 
231
236
  def in_scope(case: EvaluationCase) -> EvaluationResult:
232
237
  # ContextVars (the eval-run scope) do not cross thread boundaries on their own -
@@ -240,45 +245,68 @@ class EvaluationRunContext:
240
245
  if _idem_key(self._run.run_id, case.case_id, case.run_number) not in already_done
241
246
  ]
242
247
  executor = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
243
- # map() yields in submission order, so batching/submission below stays deterministic.
244
- mapped = executor.map(in_scope, pending)
245
248
 
246
- def ordered() -> "Iterator[EvaluationResult]":
249
+ def bounded() -> "Iterator[EvaluationResult]":
250
+ # Bounded submit loop instead of executor.map(): map() dispatches EVERY case
251
+ # up front, so a fail-fast flush failure (EvaluationSubmissionError below)
252
+ # still paid for the whole rest of the run in agent calls. Keep at most
253
+ # `concurrency` cases in flight, topping up as results are consumed; yields
254
+ # stay in submission order so batching below is deterministic. On teardown
255
+ # (an exception in the consuming loop closes this generator) whatever is
256
+ # queued but unstarted is cancelled.
257
+ import itertools
258
+
259
+ case_iter = iter(pending)
260
+ in_flight: "deque[concurrent.futures.Future]" = deque()
247
261
  try:
248
- yield from mapped
262
+ for case in itertools.islice(case_iter, concurrency):
263
+ in_flight.append(executor.submit(in_scope, case))
264
+ while in_flight:
265
+ result = in_flight.popleft().result()
266
+ next_case = next(case_iter, None)
267
+ if next_case is not None:
268
+ in_flight.append(executor.submit(in_scope, next_case))
269
+ yield result
249
270
  finally:
250
- executor.shutdown(wait=True)
271
+ executor.shutdown(wait=False, cancel_futures=True)
251
272
 
252
- results_iter = ordered()
273
+ results_iter = bounded()
253
274
  else:
254
275
  results_iter = None # sequential path below produces inline
255
276
 
256
- for idx, case in enumerate(cases, start=1):
257
- idem_key = _idem_key(self._run.run_id, case.case_id, case.run_number)
258
-
259
- if idem_key in already_done:
260
- logger.debug("Skipping already-submitted case: %s", idem_key)
261
- _print_progress(idx, total, case, skipped=True)
262
- continue
263
-
264
- result = next(results_iter) if results_iter is not None else produce(case)
265
- result.idempotency_key = idem_key
266
- # Tag the result with the case's model so the server can group it into
267
- # the Sovereignty & Portability matrix (the callable may also set it).
268
- if case.model:
269
- meta = dict(result.metadata or {})
270
- meta.setdefault("model", case.model)
271
- result.metadata = meta
272
- result = EvaluationResult(
273
- **{**result.model_dump(), "idempotencyKey": idem_key}
274
- )
275
- self._results.append(result)
276
- batch.append(result)
277
- _print_progress(idx, total, case, result=result)
277
+ try:
278
+ for idx, case in enumerate(cases, start=1):
279
+ idem_key = _idem_key(self._run.run_id, case.case_id, case.run_number)
280
+
281
+ if idem_key in already_done:
282
+ logger.debug("Skipping already-submitted case: %s", idem_key)
283
+ _print_progress(idx, total, case, skipped=True)
284
+ continue
285
+
286
+ result = next(results_iter) if results_iter is not None else produce(case)
287
+ result.idempotency_key = idem_key
288
+ # Tag the result with the case's model so the server can group it into
289
+ # the Sovereignty & Portability matrix (the callable may also set it).
290
+ if case.model:
291
+ meta = dict(result.metadata or {})
292
+ meta.setdefault("model", case.model)
293
+ result.metadata = meta
294
+ result = EvaluationResult(
295
+ **{**result.model_dump(), "idempotencyKey": idem_key}
296
+ )
297
+ self._results.append(result)
298
+ batch.append(result)
299
+ _print_progress(idx, total, case, result=result)
278
300
 
279
- if len(batch) >= max_batch:
280
- self._flush_batch(batch)
281
- batch = []
301
+ if len(batch) >= max_batch:
302
+ self._flush_batch(batch)
303
+ batch = []
304
+ finally:
305
+ # Deterministic teardown: a flush failure mid-run must stop the in-flight agent
306
+ # dispatch NOW (bounded()'s finally cancels queued cases), not whenever the
307
+ # generator happens to be garbage-collected.
308
+ if results_iter is not None:
309
+ results_iter.close()
282
310
 
283
311
  if batch:
284
312
  self._flush_batch(batch)
@@ -333,9 +361,21 @@ class EvaluationRunContext:
333
361
  so a re-execute() after a crash skips (and never re-pays for) finished cases."""
334
362
  try:
335
363
  return set(self._client.get_submitted_keys(self._run.run_id))
336
- except Exception:
337
- # Older engines without the route: no resume, identical to the historical behavior.
338
- return set()
364
+ except AgentXEvaluationsError as exc:
365
+ if exc.status_code == 404:
366
+ # Older engines without the route: no resume, identical to the historical
367
+ # behavior. ONLY the 404 qualifies - a transient 502/timeout here used to be
368
+ # swallowed too, and an empty resume set silently re-runs (and re-bills)
369
+ # every already-finished case.
370
+ return set()
371
+ _say(f" {red('✗')} Could not fetch already-submitted keys: {dim(str(exc))}")
372
+ logger.error(
373
+ "Resume-key fetch for run %s failed (%s) - refusing to re-run the whole run "
374
+ "blind; retry execute() once the engine is reachable",
375
+ self._run.run_id,
376
+ exc,
377
+ )
378
+ raise
339
379
 
340
380
  # ------------------------------------------------------------------
341
381
  # Step 2: finalize
@@ -465,6 +505,8 @@ class EvaluationRunContext:
465
505
 
466
506
  Args:
467
507
  mode: "auto" (default), "sync", or "batch" - how item scoring executes server-side.
508
+ Hosted-only: self-host runs the analysis synchronously regardless; see the
509
+ response's mode field for what actually ran.
468
510
  quality_mode: "quality_first" or "balanced" - how many items get a second/third judge.
469
511
  judges: 1-3 model ids from ``client.evaluations.list_models()``, e.g.
470
512
  ``["gpt-5.6-luna", "claude-opus-4-8"]``. Omit to let the engine score with its
@@ -12,7 +12,25 @@ class AgentXError(Exception):
12
12
 
13
13
 
14
14
  class AgentXAuthError(AgentXError):
15
- """Invalid or missing API key."""
15
+ """Invalid or missing API key.
16
+
17
+ Canonical across every sub-client (evaluations, monitor, ...) - ``except
18
+ agentx.AgentXAuthError`` catches an auth failure no matter which client raised it.
19
+ ``status_code`` carries the HTTP status when known (401/403), ``None`` otherwise.
20
+ """
21
+
22
+ def __init__(self, message: str, status_code: int | None = None) -> None:
23
+ super().__init__(message)
24
+ self.status_code = status_code
25
+
26
+
27
+ class AgentXValidationError(AgentXError):
28
+ """The API rejected the request as invalid (HTTP 422). Canonical across every
29
+ sub-client, same as :class:`AgentXAuthError`."""
30
+
31
+ def __init__(self, message: str, status_code: int | None = 422) -> None:
32
+ super().__init__(message)
33
+ self.status_code = status_code
16
34
 
17
35
 
18
36
  class AgentXConnectionError(AgentXError):
@@ -8,11 +8,12 @@ from typing import Any, Dict, Iterator, List, Optional
8
8
  import requests
9
9
 
10
10
  from agentx.util import api_base, get_headers
11
+ from agentx.exceptions import AgentXError
11
12
 
12
13
  logger = logging.getLogger(__name__)
13
14
 
14
15
 
15
- class AgentXExportError(Exception):
16
+ class AgentXExportError(AgentXError):
16
17
  pass
17
18
 
18
19
 
@@ -6,11 +6,12 @@ from typing import Any, Dict, Optional
6
6
  import requests
7
7
 
8
8
  from agentx.util import api_base, get_headers
9
+ from agentx.exceptions import AgentXError
9
10
 
10
11
  logger = logging.getLogger(__name__)
11
12
 
12
13
 
13
- class AgentXFeedbackError(Exception):
14
+ class AgentXFeedbackError(AgentXError):
14
15
  pass
15
16
 
16
17
 
@@ -195,6 +195,10 @@ class AgentXAutoGenObserver:
195
195
  "output": text,
196
196
  "inputTokenSize": input_tokens,
197
197
  "outputTokenSize": out_tokens,
198
+ # A named source is an agent turn in the team's trajectory, not a bare
199
+ # model call - stated so _merge_child_run doesn't stamp it "llm" (the
200
+ # crewai task-step precedent).
201
+ **({"kind": "agent"} if source else {}),
198
202
  })
199
203
 
200
204
  if text:
@@ -18,11 +18,18 @@ Requires: ``pip install "agentx-python[crewai]"``
18
18
  """
19
19
  from __future__ import annotations
20
20
 
21
+ import logging
21
22
  import time
22
23
  from typing import Any, Dict, List, Optional
23
24
 
24
25
  from agentx.tracing.tracer import Tracer, _safe_serialize
25
26
 
27
+ logger = logging.getLogger(__name__)
28
+
29
+ # Warn once per process when CrewAI's event bus can't be imported and task timings fall back
30
+ # to the evenly-divided approximation - fabricated timings shouldn't be silent.
31
+ _warned_no_event_bus = False
32
+
26
33
 
27
34
  class AgentXCrewObserver:
28
35
  """
@@ -53,7 +60,7 @@ class AgentXCrewObserver:
53
60
  present; on older CrewAI versions that predate it, this falls back to
54
61
  evenly dividing the total latency across tasks.
55
62
  """
56
- task_timings, unregister = self._start_task_timing_capture()
63
+ task_timings, unregister = self._start_task_timing_capture(crew)
57
64
 
58
65
  start = time.time()
59
66
  error: Optional[str] = None
@@ -97,7 +104,7 @@ class AgentXCrewObserver:
97
104
  framework="crewai",
98
105
  )
99
106
 
100
- def _start_task_timing_capture(self):
107
+ def _start_task_timing_capture(self, crew: Any = None):
101
108
  """
102
109
  Register temporary, additive listeners on CrewAI's event bus to
103
110
  capture each task's real start/end wall-clock time, keyed by
@@ -105,10 +112,16 @@ class AgentXCrewObserver:
105
112
  ``async_execution=True`` - unlike attributing the most-recently-
106
113
  started task, which would misattribute end times under overlap).
107
114
 
115
+ The event bus is a global singleton, so events from a DIFFERENT crew's
116
+ overlapping kickoff arrive here too - listeners are scoped to ``crew``
117
+ (event/source crew identity when the event carries it, this kickoff's
118
+ task ids otherwise) so each trace only records its own kickoff's tasks.
119
+
108
120
  Returns ``(task_timings, unregister)``. ``task_timings`` stays empty
109
121
  (and ``unregister`` is a no-op) on CrewAI versions that predate the
110
- ``crewai.events`` module - callers should fall back to the
111
- evenly-divided approximation in that case.
122
+ events module - callers should fall back to the evenly-divided
123
+ approximation in that case (warned once per process, since those
124
+ timings are fabricated).
112
125
 
113
126
  Uses ``crewai_event_bus.on()``/``.off()`` directly rather than
114
127
  ``scoped_handlers()`` - the latter temporarily disables *every*
@@ -116,30 +129,60 @@ class AgentXCrewObserver:
116
129
  built-in ones) for the duration of the `with` block, which isn't
117
130
  what we want for a handler meant to run alongside them.
118
131
  """
132
+ global _warned_no_event_bus
119
133
  task_timings: Dict[str, Dict[str, Any]] = {}
120
134
  try:
121
- from crewai.events.event_bus import crewai_event_bus
122
- from crewai.events.types.task_events import (
123
- TaskCompletedEvent,
124
- TaskFailedEvent,
125
- TaskStartedEvent,
126
- )
135
+ # Modern shape first (the crewai.events module), then the older
136
+ # crewai.utilities.events layout that shipped the same bus/events.
137
+ try:
138
+ from crewai.events.event_bus import crewai_event_bus
139
+ from crewai.events.types.task_events import (
140
+ TaskCompletedEvent,
141
+ TaskFailedEvent,
142
+ TaskStartedEvent,
143
+ )
144
+ except ImportError:
145
+ from crewai.utilities.events import crewai_event_bus
146
+ from crewai.utilities.events.task_events import (
147
+ TaskCompletedEvent,
148
+ TaskFailedEvent,
149
+ TaskStartedEvent,
150
+ )
127
151
  except ImportError:
152
+ if not _warned_no_event_bus:
153
+ _warned_no_event_bus = True
154
+ logger.warning(
155
+ "CrewAI's event bus is not importable (tried crewai.events and "
156
+ "crewai.utilities.events) - per-task timings will be approximated by "
157
+ "evenly dividing the kickoff's total latency across tasks"
158
+ )
128
159
  return task_timings, lambda: None
129
160
 
130
- # Double-instrumentation guard (bus-keyed latch, the same idea as the
131
- # other integrations' _agentx_patched flag): the event bus is a global
132
- # singleton, so a notebook re-run or an overlapping kickoff that
133
- # already has AgentX listeners registered would otherwise get a second
134
- # set and duplicate every task span. When already attached, this
135
- # kickoff just falls back to the evenly-divided timing approximation.
136
- if getattr(crewai_event_bus, "_agentx_attached", False):
137
- return task_timings, lambda: None
138
- crewai_event_bus._agentx_attached = True
161
+ # This kickoff's own task ids - the fallback scope filter when an event carries no
162
+ # crew reference to compare against.
163
+ own_task_ids = {
164
+ str(getattr(task, "id", None))
165
+ for task in (getattr(crew, "tasks", None) or [])
166
+ if getattr(task, "id", None) is not None
167
+ }
168
+
169
+ def is_ours(source: Any, event: Any) -> bool:
170
+ """Only record events that belong to THIS kickoff's crew - the bus is global,
171
+ so a concurrent kickoff's task events land on every registered listener."""
172
+ if crew is None:
173
+ return True
174
+ if source is crew:
175
+ return True
176
+ event_crew = getattr(event, "crew", None) or getattr(source, "crew", None)
177
+ if event_crew is not None:
178
+ return event_crew is crew
179
+ if own_task_ids:
180
+ return str(getattr(event, "task_id", None)) in own_task_ids
181
+ return True
139
182
 
140
183
  def on_task_started(source: Any, event: Any) -> None:
141
184
  task_id = getattr(event, "task_id", None)
142
- if task_id is None:
185
+ if task_id is None or not is_ours(source, event):
143
186
  return
144
187
  task_timings[task_id] = {
145
188
  "name": getattr(event, "task_name", None),
@@ -164,14 +207,9 @@ class AgentXCrewObserver:
164
207
  crewai_event_bus.on(TaskFailedEvent)(on_task_failed)
165
208
 
166
209
  def unregister() -> None:
167
- try:
168
- crewai_event_bus.off(TaskStartedEvent, on_task_started)
169
- crewai_event_bus.off(TaskCompletedEvent, on_task_completed)
170
- crewai_event_bus.off(TaskFailedEvent, on_task_failed)
171
- finally:
172
- # Clear the latch even if .off() raises, so a later kickoff
173
- # can re-attach instead of being locked out forever.
174
- crewai_event_bus._agentx_attached = False
210
+ crewai_event_bus.off(TaskStartedEvent, on_task_started)
211
+ crewai_event_bus.off(TaskCompletedEvent, on_task_completed)
212
+ crewai_event_bus.off(TaskFailedEvent, on_task_failed)
175
213
 
176
214
  return task_timings, unregister
177
215