agentx-python 0.8.21__tar.gz → 0.8.22__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.8.21/agentx_python.egg-info → agentx_python-0.8.22}/PKG-INFO +1 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/__init__.py +4 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/agentx.py +35 -5
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/client.py +27 -19
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/models.py +27 -2
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/runner.py +77 -35
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/exceptions.py +19 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/export.py +2 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/feedback.py +2 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/autogen.py +4 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/crewai.py +66 -28
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/google_adk.py +33 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/langchain.py +87 -53
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/llamaindex.py +85 -46
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/openai_agents.py +18 -2
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/client.py +27 -13
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/improvement_groups.py +2 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/judge_scorers.py +11 -2
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/scorer_groups.py +3 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/scorers.py +2 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/outcomes.py +2 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/projects.py +2 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/traces.py +2 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/ingest_client.py +31 -5
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/tracer.py +47 -17
- agentx_python-0.8.22/agentx/util.py +29 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/version.py +2 -2
- {agentx_python-0.8.21 → agentx_python-0.8.22/agentx_python.egg-info}/PKG-INFO +1 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/SOURCES.txt +3 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_deep_dive_fixes.py +45 -0
- agentx_python-0.8.22/tests/test_error_taxonomy.py +86 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_integrations.py +1 -1
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_judge_scorers.py +47 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_runner_features.py +73 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_span_tree.py +232 -3
- agentx_python-0.8.22/tests/test_wire_models.py +83 -0
- agentx_python-0.8.21/agentx/util.py +0 -20
- {agentx_python-0.8.21 → agentx_python-0.8.22}/LICENSE +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/README.md +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/cli.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/evaluation_settings.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/prompts.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/tool_schemas.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/__init__.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/_traced_call.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/anthropic.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/databricks.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/google_genai.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/litellm.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/moveworks.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/integrations/openai.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/__init__.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/agents.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/models.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/online_evaluators.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/patterns.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/profile.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/review_queue.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/rules.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/sessions.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/monitor/signals.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/py.typed +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/agent.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/conversation.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/resources/workforce.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/testing.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/eval_scope.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx/tracing/framework_detect.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/entry_points.txt +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/requires.txt +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/setup.cfg +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/setup.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_cli_launcher.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_docs_match_sdk.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_eval_scope.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_framework_detect.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_integration.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_multi_judge.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_pairwise.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_review_queue.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_selfhost_analysis_fallback.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_selfhost_compat.py +0 -0
- {agentx_python-0.8.21 → agentx_python-0.8.22}/tests/test_testing.py +0 -0
|
@@ -5,6 +5,8 @@ from agentx.version import VERSION
|
|
|
5
5
|
from agentx.exceptions import (
|
|
6
6
|
AgentXError,
|
|
7
7
|
AgentXAuthError,
|
|
8
|
+
AgentXValidationError,
|
|
9
|
+
AgentXConnectionError,
|
|
8
10
|
AgentXAPIError,
|
|
9
11
|
DatasetNotFound,
|
|
10
12
|
CINotEnabled,
|
|
@@ -21,6 +23,8 @@ __all__ = [
|
|
|
21
23
|
"AgentX",
|
|
22
24
|
"AgentXError",
|
|
23
25
|
"AgentXAuthError",
|
|
26
|
+
"AgentXValidationError",
|
|
27
|
+
"AgentXConnectionError",
|
|
24
28
|
"AgentXAPIError",
|
|
25
29
|
"DatasetNotFound",
|
|
26
30
|
"CINotEnabled",
|
|
@@ -3,7 +3,7 @@ import requests
|
|
|
3
3
|
import os
|
|
4
4
|
import logging
|
|
5
5
|
|
|
6
|
-
from agentx.util import get_headers, api_base
|
|
6
|
+
from agentx.util import get_headers, api_base, normalize_base
|
|
7
7
|
from agentx.resources.agent import Agent
|
|
8
8
|
from agentx.resources.workforce import Workforce
|
|
9
9
|
|
|
@@ -16,16 +16,23 @@ class AgentX:
|
|
|
16
16
|
base_url: Optional[str] = None,
|
|
17
17
|
workspace_id: Optional[str] = None,
|
|
18
18
|
):
|
|
19
|
+
# The api_key is NOT written back into os.environ (it used to be): every sub-client
|
|
20
|
+
# below receives it explicitly, and mutating process-global state from a constructor
|
|
21
|
+
# re-pointed unrelated code - the same leak the base_url write below had (deep-dive
|
|
22
|
+
# round 3, bug #1). Static flows that still read the env (AgentX.list_workforces,
|
|
23
|
+
# bare get_headers()) now require the caller to set AGENTX_API_KEY themselves.
|
|
19
24
|
self.api_key = api_key or os.getenv("AGENTX_API_KEY")
|
|
20
|
-
if self.api_key and not os.getenv("AGENTX_API_KEY"):
|
|
21
|
-
os.environ["AGENTX_API_KEY"] = self.api_key
|
|
22
25
|
|
|
23
26
|
# base_url overrides AGENTX_API_BASE_URL env var (and the SDK default). It is
|
|
24
27
|
# deliberately NOT written back into os.environ: the constructor used to do that, which
|
|
25
28
|
# made the last-constructed client silently re-point every other client in the process
|
|
26
29
|
# (deep-dive round 3, bug #1). Each sub-client below receives this value explicitly and
|
|
27
|
-
# captures it at construction instead.
|
|
30
|
+
# captures it at construction instead. Normalized (trailing slash and the
|
|
31
|
+
# /custom-agent-evaluations suffix stripped) so an evaluations-shaped URL works for
|
|
32
|
+
# every sub-client, not just evaluations.
|
|
28
33
|
self.base_url = base_url or os.getenv("AGENTX_API_BASE_URL")
|
|
34
|
+
if self.base_url:
|
|
35
|
+
self.base_url = normalize_base(self.base_url)
|
|
29
36
|
|
|
30
37
|
self.workspace_id = workspace_id or os.getenv("AGENTX_WORKSPACE_ID")
|
|
31
38
|
|
|
@@ -91,6 +98,27 @@ class AgentX:
|
|
|
91
98
|
workspace_id=self.workspace_id,
|
|
92
99
|
)
|
|
93
100
|
self.tracer = Tracer(_ingest_client)
|
|
101
|
+
self._ingest_client = _ingest_client
|
|
102
|
+
|
|
103
|
+
# ------------------------------------------------------------------
|
|
104
|
+
# Lifecycle
|
|
105
|
+
# ------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
def close(self, timeout: float = 5.0) -> bool:
|
|
108
|
+
"""Flush queued traces and stop the tracer's background ingest worker.
|
|
109
|
+
|
|
110
|
+
Returns ``True`` when everything drained before ``timeout`` seconds elapsed. Optional -
|
|
111
|
+
an ``atexit`` hook already flushes queued traces on interpreter shutdown - but a
|
|
112
|
+
long-running service that tears clients down mid-process should call it (or use the
|
|
113
|
+
client as a context manager) so worker threads don't accumulate.
|
|
114
|
+
"""
|
|
115
|
+
return self._ingest_client.close(timeout)
|
|
116
|
+
|
|
117
|
+
def __enter__(self) -> "AgentX":
|
|
118
|
+
return self
|
|
119
|
+
|
|
120
|
+
def __exit__(self, exc_type, exc_val, tb) -> None:
|
|
121
|
+
self.close()
|
|
94
122
|
|
|
95
123
|
@classmethod
|
|
96
124
|
def from_env(cls) -> "AgentX":
|
|
@@ -129,7 +157,9 @@ class AgentX:
|
|
|
129
157
|
|
|
130
158
|
@staticmethod
|
|
131
159
|
def list_workforces() -> List["Workforce"]:
|
|
132
|
-
"""List all workforces/teams.
|
|
160
|
+
"""List all workforces/teams. Static, so it reads AGENTX_API_KEY from the environment
|
|
161
|
+
directly - the constructor no longer writes ``api_key`` into os.environ, so set the
|
|
162
|
+
env var yourself before calling this."""
|
|
133
163
|
url = f"{api_base()}/access/teams"
|
|
134
164
|
response = requests.get(url, headers=get_headers())
|
|
135
165
|
if response.status_code == 200:
|
|
@@ -24,7 +24,12 @@ from agentx.evaluations.models import (
|
|
|
24
24
|
|
|
25
25
|
logger = logging.getLogger(__name__)
|
|
26
26
|
|
|
27
|
-
from agentx.util import _DEFAULT_API_BASE as _UTIL_API_BASE
|
|
27
|
+
from agentx.util import _DEFAULT_API_BASE as _UTIL_API_BASE, normalize_base
|
|
28
|
+
|
|
29
|
+
# The canonical error classes (agentx.exceptions) are raised - and re-exported here for
|
|
30
|
+
# compat with code that imported them from this module - so `except agentx.AgentXAuthError`
|
|
31
|
+
# works whichever client raised.
|
|
32
|
+
from agentx.exceptions import AgentXError, AgentXAuthError, AgentXValidationError
|
|
28
33
|
|
|
29
34
|
_DEFAULT_BASE_URL = f"{_UTIL_API_BASE}/custom-agent-evaluations"
|
|
30
35
|
SDK_NAME = "agentx-python"
|
|
@@ -42,7 +47,7 @@ _SELF_HOST_ANALYZE_TIMEOUT = 1800
|
|
|
42
47
|
_SELF_HOST_SCORING_TIMEOUT = 900
|
|
43
48
|
|
|
44
49
|
|
|
45
|
-
class AgentXEvaluationsError(
|
|
50
|
+
class AgentXEvaluationsError(AgentXError):
|
|
46
51
|
"""An evaluations API call failed.
|
|
47
52
|
|
|
48
53
|
``status_code`` carries the HTTP status when the failure came from a response rather
|
|
@@ -55,14 +60,6 @@ class AgentXEvaluationsError(Exception):
|
|
|
55
60
|
self.status_code = status_code
|
|
56
61
|
|
|
57
62
|
|
|
58
|
-
class AgentXAuthError(AgentXEvaluationsError):
|
|
59
|
-
pass
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
class AgentXValidationError(AgentXEvaluationsError):
|
|
63
|
-
pass
|
|
64
|
-
|
|
65
|
-
|
|
66
63
|
class EvaluationSubmissionError(AgentXEvaluationsError):
|
|
67
64
|
"""A result batch could not be submitted (after one retry). The run is left unfinalized;
|
|
68
65
|
re-running execute() on the same context resumes past already-submitted cases."""
|
|
@@ -96,13 +93,10 @@ class EvaluationsClient:
|
|
|
96
93
|
# whatever workspace the API key's user defaults to, not the one the caller intended.
|
|
97
94
|
self._workspace_id = workspace_id
|
|
98
95
|
# Priority: constructor arg > env var > SDK default
|
|
99
|
-
#
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if not _api_base.endswith("/custom-agent-evaluations"):
|
|
104
|
-
_api_base = f"{_api_base}/custom-agent-evaluations"
|
|
105
|
-
self._base_url = _api_base
|
|
96
|
+
# normalize_base strips a trailing slash and any /custom-agent-evaluations suffix,
|
|
97
|
+
# then the suffix is appended - users only need to provide /api/v1 either way.
|
|
98
|
+
_api_base = normalize_base(base_url or os.getenv("AGENTX_API_BASE_URL", _UTIL_API_BASE))
|
|
99
|
+
self._base_url = f"{_api_base}/custom-agent-evaluations"
|
|
106
100
|
# None until an analysis call tells us which engine this is; see _api_root.
|
|
107
101
|
self._analysis_on_dashboard_router: Optional[bool] = None
|
|
108
102
|
self._session = requests.Session()
|
|
@@ -184,13 +178,16 @@ class EvaluationsClient:
|
|
|
184
178
|
continue
|
|
185
179
|
|
|
186
180
|
if resp.status_code == 401:
|
|
187
|
-
raise AgentXAuthError("Invalid or missing API key")
|
|
181
|
+
raise AgentXAuthError("Invalid or missing API key", status_code=401)
|
|
188
182
|
if resp.status_code == 422:
|
|
189
183
|
raise AgentXValidationError(resp.text)
|
|
184
|
+
# Gate on the schedule itself so HTTP-status retries walk the SAME full backoff
|
|
185
|
+
# schedule connection errors do - the old `attempt < _MAX_RETRIES - 1` gate left
|
|
186
|
+
# the schedule's last entry unreachable for HTTP retries (ingest_client precedent).
|
|
190
187
|
if (
|
|
191
188
|
resp.status_code in _RETRYABLE_STATUS
|
|
192
189
|
and retry
|
|
193
|
-
and attempt <
|
|
190
|
+
and attempt < len(schedule) - 1
|
|
194
191
|
):
|
|
195
192
|
logger.debug(
|
|
196
193
|
"Retryable status %d (attempt %d)", resp.status_code, attempt + 1
|
|
@@ -500,6 +497,17 @@ class EvaluationsClient:
|
|
|
500
497
|
return self._report_from_dashboard(run_id)
|
|
501
498
|
|
|
502
499
|
def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
|
|
500
|
+
"""Deprecated: on self-host the route's response body has no top-level list, so this
|
|
501
|
+
always returns ``[]``. Use :meth:`get_submitted_keys` - the same route's
|
|
502
|
+
``submittedKeys`` - to find out what a run still needs."""
|
|
503
|
+
import warnings
|
|
504
|
+
|
|
505
|
+
warnings.warn(
|
|
506
|
+
"get_missing_results() always returns [] on self-host - use get_submitted_keys() "
|
|
507
|
+
"to resume a run instead.",
|
|
508
|
+
DeprecationWarning,
|
|
509
|
+
stacklevel=2,
|
|
510
|
+
)
|
|
503
511
|
data = self._request("GET", f"/runs/{run_id}/missing-results")
|
|
504
512
|
return data if isinstance(data, list) else data.get("missing", [])
|
|
505
513
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from typing import Any, Dict, List, Literal, Optional, Union
|
|
4
|
-
from pydantic import AliasChoices, BaseModel, Field, model_validator
|
|
4
|
+
from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
|
|
5
5
|
|
|
6
6
|
# ---------------------------------------------------------------------------
|
|
7
7
|
# Observable trace
|
|
@@ -78,6 +78,17 @@ class Dataset(BaseModel):
|
|
|
78
78
|
# Custom code scorers attached to this dataset - [{ id, name, code, enabled }]. Retrievable,
|
|
79
79
|
# so a fetched dataset round-trips them (import_dataset copies them to the new dataset).
|
|
80
80
|
code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers")
|
|
81
|
+
# Grading config carried on the dataset itself - similarity metric toggles (each a
|
|
82
|
+
# {"enabled": bool, ...} object on the wire), LLM-as-judge overrides, and the raw
|
|
83
|
+
# sovereigntyIndex object. Modeled so a fetched Dataset round-trips them: extra="ignore"
|
|
84
|
+
# used to silently drop all of these on read, and import_dataset lost them on the copy.
|
|
85
|
+
vector_similarity: Optional[Any] = Field(default=None, alias="vectorSimilarity")
|
|
86
|
+
jaccard_similarity: Optional[Any] = Field(default=None, alias="jaccardSimilarity")
|
|
87
|
+
bleu_score: Optional[Any] = Field(default=None, alias="bleuScore")
|
|
88
|
+
rouge_score: Optional[Any] = Field(default=None, alias="rougeScore")
|
|
89
|
+
judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt")
|
|
90
|
+
judge_model: Optional[str] = Field(default=None, alias="judgeModel")
|
|
91
|
+
sovereignty_index: Optional[Dict[str, Any]] = Field(default=None, alias="sovereigntyIndex")
|
|
81
92
|
status: str = "published"
|
|
82
93
|
version_id: Optional[str] = Field(default=None, alias="versionId")
|
|
83
94
|
# Sovereignty & Portability - models selected to compare on this dataset.
|
|
@@ -430,7 +441,12 @@ class RunResultRow(BaseModel):
|
|
|
430
441
|
question_index: Optional[int] = Field(default=None, alias="questionIndex")
|
|
431
442
|
run_number: Optional[int] = Field(default=None, alias="runNumber")
|
|
432
443
|
question_text: Optional[str] = Field(default=None, alias="questionText")
|
|
433
|
-
|
|
444
|
+
# The engine sends the agent's answer as an `output` OBJECT ({"text": ...}), not a
|
|
445
|
+
# `response` string - accept both spellings and lift the dict's text (see the
|
|
446
|
+
# validator below), so row.response actually populates on self-host.
|
|
447
|
+
response: Optional[str] = Field(
|
|
448
|
+
default=None, validation_alias=AliasChoices("response", "output")
|
|
449
|
+
)
|
|
434
450
|
trace_id: Optional[str] = Field(default=None, alias="traceId")
|
|
435
451
|
latency_ms: Optional[float] = Field(default=None, alias="latencyMs")
|
|
436
452
|
input_tokens: Optional[int] = Field(default=None, alias="inputTokens")
|
|
@@ -452,6 +468,15 @@ class RunResultRow(BaseModel):
|
|
|
452
468
|
populate_by_name = True
|
|
453
469
|
extra = "ignore"
|
|
454
470
|
|
|
471
|
+
@field_validator("response", mode="before")
|
|
472
|
+
@classmethod
|
|
473
|
+
def _lift_output_text(cls, value: Any) -> Any:
|
|
474
|
+
# The `output` alias delivers the wire's whole output object - keep the declared
|
|
475
|
+
# Optional[str] by lifting its text field.
|
|
476
|
+
if isinstance(value, dict):
|
|
477
|
+
return value.get("text")
|
|
478
|
+
return value
|
|
479
|
+
|
|
455
480
|
@classmethod
|
|
456
481
|
def from_wire(cls, wire: Dict[str, Any]) -> "RunResultRow":
|
|
457
482
|
row = cls.model_validate(wire)
|
|
@@ -11,7 +11,11 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Union
|
|
|
11
11
|
from agentx.evaluations.adapters.raw import RawCallableAdapter
|
|
12
12
|
from agentx.evaluations.adapters.precomputed import PrecomputedAdapter
|
|
13
13
|
from agentx.evaluations.adapters.http_endpoint import HttpEndpointAdapter
|
|
14
|
-
from agentx.evaluations.client import
|
|
14
|
+
from agentx.evaluations.client import (
|
|
15
|
+
AgentXEvaluationsError,
|
|
16
|
+
EvaluationsClient,
|
|
17
|
+
EvaluationSubmissionError,
|
|
18
|
+
)
|
|
15
19
|
from agentx.evaluations.models import (
|
|
16
20
|
AnalysisStatus,
|
|
17
21
|
Dataset,
|
|
@@ -227,6 +231,7 @@ class EvaluationRunContext:
|
|
|
227
231
|
if concurrency > 1:
|
|
228
232
|
import concurrent.futures
|
|
229
233
|
import contextvars
|
|
234
|
+
from collections import deque
|
|
230
235
|
|
|
231
236
|
def in_scope(case: EvaluationCase) -> EvaluationResult:
|
|
232
237
|
# ContextVars (the eval-run scope) do not cross thread boundaries on their own -
|
|
@@ -240,45 +245,68 @@ class EvaluationRunContext:
|
|
|
240
245
|
if _idem_key(self._run.run_id, case.case_id, case.run_number) not in already_done
|
|
241
246
|
]
|
|
242
247
|
executor = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
|
|
243
|
-
# map() yields in submission order, so batching/submission below stays deterministic.
|
|
244
|
-
mapped = executor.map(in_scope, pending)
|
|
245
248
|
|
|
246
|
-
def
|
|
249
|
+
def bounded() -> "Iterator[EvaluationResult]":
|
|
250
|
+
# Bounded submit loop instead of executor.map(): map() dispatches EVERY case
|
|
251
|
+
# up front, so a fail-fast flush failure (EvaluationSubmissionError below)
|
|
252
|
+
# still paid for the whole rest of the run in agent calls. Keep at most
|
|
253
|
+
# `concurrency` cases in flight, topping up as results are consumed; yields
|
|
254
|
+
# stay in submission order so batching below is deterministic. On teardown
|
|
255
|
+
# (an exception in the consuming loop closes this generator) whatever is
|
|
256
|
+
# queued but unstarted is cancelled.
|
|
257
|
+
import itertools
|
|
258
|
+
|
|
259
|
+
case_iter = iter(pending)
|
|
260
|
+
in_flight: "deque[concurrent.futures.Future]" = deque()
|
|
247
261
|
try:
|
|
248
|
-
|
|
262
|
+
for case in itertools.islice(case_iter, concurrency):
|
|
263
|
+
in_flight.append(executor.submit(in_scope, case))
|
|
264
|
+
while in_flight:
|
|
265
|
+
result = in_flight.popleft().result()
|
|
266
|
+
next_case = next(case_iter, None)
|
|
267
|
+
if next_case is not None:
|
|
268
|
+
in_flight.append(executor.submit(in_scope, next_case))
|
|
269
|
+
yield result
|
|
249
270
|
finally:
|
|
250
|
-
executor.shutdown(wait=True)
|
|
271
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
251
272
|
|
|
252
|
-
results_iter =
|
|
273
|
+
results_iter = bounded()
|
|
253
274
|
else:
|
|
254
275
|
results_iter = None # sequential path below produces inline
|
|
255
276
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
277
|
+
try:
|
|
278
|
+
for idx, case in enumerate(cases, start=1):
|
|
279
|
+
idem_key = _idem_key(self._run.run_id, case.case_id, case.run_number)
|
|
280
|
+
|
|
281
|
+
if idem_key in already_done:
|
|
282
|
+
logger.debug("Skipping already-submitted case: %s", idem_key)
|
|
283
|
+
_print_progress(idx, total, case, skipped=True)
|
|
284
|
+
continue
|
|
285
|
+
|
|
286
|
+
result = next(results_iter) if results_iter is not None else produce(case)
|
|
287
|
+
result.idempotency_key = idem_key
|
|
288
|
+
# Tag the result with the case's model so the server can group it into
|
|
289
|
+
# the Sovereignty & Portability matrix (the callable may also set it).
|
|
290
|
+
if case.model:
|
|
291
|
+
meta = dict(result.metadata or {})
|
|
292
|
+
meta.setdefault("model", case.model)
|
|
293
|
+
result.metadata = meta
|
|
294
|
+
result = EvaluationResult(
|
|
295
|
+
**{**result.model_dump(), "idempotencyKey": idem_key}
|
|
296
|
+
)
|
|
297
|
+
self._results.append(result)
|
|
298
|
+
batch.append(result)
|
|
299
|
+
_print_progress(idx, total, case, result=result)
|
|
278
300
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
301
|
+
if len(batch) >= max_batch:
|
|
302
|
+
self._flush_batch(batch)
|
|
303
|
+
batch = []
|
|
304
|
+
finally:
|
|
305
|
+
# Deterministic teardown: a flush failure mid-run must stop the in-flight agent
|
|
306
|
+
# dispatch NOW (bounded()'s finally cancels queued cases), not whenever the
|
|
307
|
+
# generator happens to be garbage-collected.
|
|
308
|
+
if results_iter is not None:
|
|
309
|
+
results_iter.close()
|
|
282
310
|
|
|
283
311
|
if batch:
|
|
284
312
|
self._flush_batch(batch)
|
|
@@ -333,9 +361,21 @@ class EvaluationRunContext:
|
|
|
333
361
|
so a re-execute() after a crash skips (and never re-pays for) finished cases."""
|
|
334
362
|
try:
|
|
335
363
|
return set(self._client.get_submitted_keys(self._run.run_id))
|
|
336
|
-
except
|
|
337
|
-
|
|
338
|
-
|
|
364
|
+
except AgentXEvaluationsError as exc:
|
|
365
|
+
if exc.status_code == 404:
|
|
366
|
+
# Older engines without the route: no resume, identical to the historical
|
|
367
|
+
# behavior. ONLY the 404 qualifies - a transient 502/timeout here used to be
|
|
368
|
+
# swallowed too, and an empty resume set silently re-runs (and re-bills)
|
|
369
|
+
# every already-finished case.
|
|
370
|
+
return set()
|
|
371
|
+
_say(f" {red('✗')} Could not fetch already-submitted keys: {dim(str(exc))}")
|
|
372
|
+
logger.error(
|
|
373
|
+
"Resume-key fetch for run %s failed (%s) - refusing to re-run the whole run "
|
|
374
|
+
"blind; retry execute() once the engine is reachable",
|
|
375
|
+
self._run.run_id,
|
|
376
|
+
exc,
|
|
377
|
+
)
|
|
378
|
+
raise
|
|
339
379
|
|
|
340
380
|
# ------------------------------------------------------------------
|
|
341
381
|
# Step 2: finalize
|
|
@@ -465,6 +505,8 @@ class EvaluationRunContext:
|
|
|
465
505
|
|
|
466
506
|
Args:
|
|
467
507
|
mode: "auto" (default), "sync", or "batch" - how item scoring executes server-side.
|
|
508
|
+
Hosted-only: self-host runs the analysis synchronously regardless; see the
|
|
509
|
+
response's mode field for what actually ran.
|
|
468
510
|
quality_mode: "quality_first" or "balanced" - how many items get a second/third judge.
|
|
469
511
|
judges: 1-3 model ids from ``client.evaluations.list_models()``, e.g.
|
|
470
512
|
``["gpt-5.6-luna", "claude-opus-4-8"]``. Omit to let the engine score with its
|
|
@@ -12,7 +12,25 @@ class AgentXError(Exception):
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class AgentXAuthError(AgentXError):
|
|
15
|
-
"""Invalid or missing API key.
|
|
15
|
+
"""Invalid or missing API key.
|
|
16
|
+
|
|
17
|
+
Canonical across every sub-client (evaluations, monitor, ...) - ``except
|
|
18
|
+
agentx.AgentXAuthError`` catches an auth failure no matter which client raised it.
|
|
19
|
+
``status_code`` carries the HTTP status when known (401/403), ``None`` otherwise.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, message: str, status_code: int | None = None) -> None:
|
|
23
|
+
super().__init__(message)
|
|
24
|
+
self.status_code = status_code
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class AgentXValidationError(AgentXError):
|
|
28
|
+
"""The API rejected the request as invalid (HTTP 422). Canonical across every
|
|
29
|
+
sub-client, same as :class:`AgentXAuthError`."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, message: str, status_code: int | None = 422) -> None:
|
|
32
|
+
super().__init__(message)
|
|
33
|
+
self.status_code = status_code
|
|
16
34
|
|
|
17
35
|
|
|
18
36
|
class AgentXConnectionError(AgentXError):
|
|
@@ -8,11 +8,12 @@ from typing import Any, Dict, Iterator, List, Optional
|
|
|
8
8
|
import requests
|
|
9
9
|
|
|
10
10
|
from agentx.util import api_base, get_headers
|
|
11
|
+
from agentx.exceptions import AgentXError
|
|
11
12
|
|
|
12
13
|
logger = logging.getLogger(__name__)
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
class AgentXExportError(
|
|
16
|
+
class AgentXExportError(AgentXError):
|
|
16
17
|
pass
|
|
17
18
|
|
|
18
19
|
|
|
@@ -6,11 +6,12 @@ from typing import Any, Dict, Optional
|
|
|
6
6
|
import requests
|
|
7
7
|
|
|
8
8
|
from agentx.util import api_base, get_headers
|
|
9
|
+
from agentx.exceptions import AgentXError
|
|
9
10
|
|
|
10
11
|
logger = logging.getLogger(__name__)
|
|
11
12
|
|
|
12
13
|
|
|
13
|
-
class AgentXFeedbackError(
|
|
14
|
+
class AgentXFeedbackError(AgentXError):
|
|
14
15
|
pass
|
|
15
16
|
|
|
16
17
|
|
|
@@ -195,6 +195,10 @@ class AgentXAutoGenObserver:
|
|
|
195
195
|
"output": text,
|
|
196
196
|
"inputTokenSize": input_tokens,
|
|
197
197
|
"outputTokenSize": out_tokens,
|
|
198
|
+
# A named source is an agent turn in the team's trajectory, not a bare
|
|
199
|
+
# model call - stated so _merge_child_run doesn't stamp it "llm" (the
|
|
200
|
+
# crewai task-step precedent).
|
|
201
|
+
**({"kind": "agent"} if source else {}),
|
|
198
202
|
})
|
|
199
203
|
|
|
200
204
|
if text:
|
|
@@ -18,11 +18,18 @@ Requires: ``pip install "agentx-python[crewai]"``
|
|
|
18
18
|
"""
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
+
import logging
|
|
21
22
|
import time
|
|
22
23
|
from typing import Any, Dict, List, Optional
|
|
23
24
|
|
|
24
25
|
from agentx.tracing.tracer import Tracer, _safe_serialize
|
|
25
26
|
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
# Warn once per process when CrewAI's event bus can't be imported and task timings fall back
|
|
30
|
+
# to the evenly-divided approximation - fabricated timings shouldn't be silent.
|
|
31
|
+
_warned_no_event_bus = False
|
|
32
|
+
|
|
26
33
|
|
|
27
34
|
class AgentXCrewObserver:
|
|
28
35
|
"""
|
|
@@ -53,7 +60,7 @@ class AgentXCrewObserver:
|
|
|
53
60
|
present; on older CrewAI versions that predate it, this falls back to
|
|
54
61
|
evenly dividing the total latency across tasks.
|
|
55
62
|
"""
|
|
56
|
-
task_timings, unregister = self._start_task_timing_capture()
|
|
63
|
+
task_timings, unregister = self._start_task_timing_capture(crew)
|
|
57
64
|
|
|
58
65
|
start = time.time()
|
|
59
66
|
error: Optional[str] = None
|
|
@@ -97,7 +104,7 @@ class AgentXCrewObserver:
|
|
|
97
104
|
framework="crewai",
|
|
98
105
|
)
|
|
99
106
|
|
|
100
|
-
def _start_task_timing_capture(self):
|
|
107
|
+
def _start_task_timing_capture(self, crew: Any = None):
|
|
101
108
|
"""
|
|
102
109
|
Register temporary, additive listeners on CrewAI's event bus to
|
|
103
110
|
capture each task's real start/end wall-clock time, keyed by
|
|
@@ -105,10 +112,16 @@ class AgentXCrewObserver:
|
|
|
105
112
|
``async_execution=True`` - unlike attributing the most-recently-
|
|
106
113
|
started task, which would misattribute end times under overlap).
|
|
107
114
|
|
|
115
|
+
The event bus is a global singleton, so events from a DIFFERENT crew's
|
|
116
|
+
overlapping kickoff arrive here too - listeners are scoped to ``crew``
|
|
117
|
+
(event/source crew identity when the event carries it, this kickoff's
|
|
118
|
+
task ids otherwise) so each trace only records its own kickoff's tasks.
|
|
119
|
+
|
|
108
120
|
Returns ``(task_timings, unregister)``. ``task_timings`` stays empty
|
|
109
121
|
(and ``unregister`` is a no-op) on CrewAI versions that predate the
|
|
110
|
-
|
|
111
|
-
|
|
122
|
+
events module - callers should fall back to the evenly-divided
|
|
123
|
+
approximation in that case (warned once per process, since those
|
|
124
|
+
timings are fabricated).
|
|
112
125
|
|
|
113
126
|
Uses ``crewai_event_bus.on()``/``.off()`` directly rather than
|
|
114
127
|
``scoped_handlers()`` - the latter temporarily disables *every*
|
|
@@ -116,30 +129,60 @@ class AgentXCrewObserver:
|
|
|
116
129
|
built-in ones) for the duration of the `with` block, which isn't
|
|
117
130
|
what we want for a handler meant to run alongside them.
|
|
118
131
|
"""
|
|
132
|
+
global _warned_no_event_bus
|
|
119
133
|
task_timings: Dict[str, Dict[str, Any]] = {}
|
|
120
134
|
try:
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
135
|
+
# Modern shape first (the crewai.events module), then the older
|
|
136
|
+
# crewai.utilities.events layout that shipped the same bus/events.
|
|
137
|
+
try:
|
|
138
|
+
from crewai.events.event_bus import crewai_event_bus
|
|
139
|
+
from crewai.events.types.task_events import (
|
|
140
|
+
TaskCompletedEvent,
|
|
141
|
+
TaskFailedEvent,
|
|
142
|
+
TaskStartedEvent,
|
|
143
|
+
)
|
|
144
|
+
except ImportError:
|
|
145
|
+
from crewai.utilities.events import crewai_event_bus
|
|
146
|
+
from crewai.utilities.events.task_events import (
|
|
147
|
+
TaskCompletedEvent,
|
|
148
|
+
TaskFailedEvent,
|
|
149
|
+
TaskStartedEvent,
|
|
150
|
+
)
|
|
127
151
|
except ImportError:
|
|
152
|
+
if not _warned_no_event_bus:
|
|
153
|
+
_warned_no_event_bus = True
|
|
154
|
+
logger.warning(
|
|
155
|
+
"CrewAI's event bus is not importable (tried crewai.events and "
|
|
156
|
+
"crewai.utilities.events) - per-task timings will be approximated by "
|
|
157
|
+
"evenly dividing the kickoff's total latency across tasks"
|
|
158
|
+
)
|
|
128
159
|
return task_timings, lambda: None
|
|
129
160
|
|
|
130
|
-
#
|
|
131
|
-
#
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
161
|
+
# This kickoff's own task ids - the fallback scope filter when an event carries no
|
|
162
|
+
# crew reference to compare against.
|
|
163
|
+
own_task_ids = {
|
|
164
|
+
str(getattr(task, "id", None))
|
|
165
|
+
for task in (getattr(crew, "tasks", None) or [])
|
|
166
|
+
if getattr(task, "id", None) is not None
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
def is_ours(source: Any, event: Any) -> bool:
|
|
170
|
+
"""Only record events that belong to THIS kickoff's crew - the bus is global,
|
|
171
|
+
so a concurrent kickoff's task events land on every registered listener."""
|
|
172
|
+
if crew is None:
|
|
173
|
+
return True
|
|
174
|
+
if source is crew:
|
|
175
|
+
return True
|
|
176
|
+
event_crew = getattr(event, "crew", None) or getattr(source, "crew", None)
|
|
177
|
+
if event_crew is not None:
|
|
178
|
+
return event_crew is crew
|
|
179
|
+
if own_task_ids:
|
|
180
|
+
return str(getattr(event, "task_id", None)) in own_task_ids
|
|
181
|
+
return True
|
|
139
182
|
|
|
140
183
|
def on_task_started(source: Any, event: Any) -> None:
|
|
141
184
|
task_id = getattr(event, "task_id", None)
|
|
142
|
-
if task_id is None:
|
|
185
|
+
if task_id is None or not is_ours(source, event):
|
|
143
186
|
return
|
|
144
187
|
task_timings[task_id] = {
|
|
145
188
|
"name": getattr(event, "task_name", None),
|
|
@@ -164,14 +207,9 @@ class AgentXCrewObserver:
|
|
|
164
207
|
crewai_event_bus.on(TaskFailedEvent)(on_task_failed)
|
|
165
208
|
|
|
166
209
|
def unregister() -> None:
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
crewai_event_bus.off(TaskFailedEvent, on_task_failed)
|
|
171
|
-
finally:
|
|
172
|
-
# Clear the latch even if .off() raises, so a later kickoff
|
|
173
|
-
# can re-attach instead of being locked out forever.
|
|
174
|
-
crewai_event_bus._agentx_attached = False
|
|
210
|
+
crewai_event_bus.off(TaskStartedEvent, on_task_started)
|
|
211
|
+
crewai_event_bus.off(TaskCompletedEvent, on_task_completed)
|
|
212
|
+
crewai_event_bus.off(TaskFailedEvent, on_task_failed)
|
|
175
213
|
|
|
176
214
|
return task_timings, unregister
|
|
177
215
|
|