agentx-python 0.8.24__tar.gz → 0.8.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.8.24 → agentx_python-0.8.26}/PKG-INFO +7 -2
- {agentx_python-0.8.24 → agentx_python-0.8.26}/README.md +4 -1
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/agentx.py +16 -12
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/__init__.py +1 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/crewai.py +2 -2
- agentx_python-0.8.26/agentx/integrations/nvidia_nim.py +70 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/openai.py +5 -1
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/__init__.py +9 -0
- agentx_python-0.8.26/agentx/monitor/_transport.py +48 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/client.py +5 -3
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/improvement_groups.py +16 -4
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/judge_scorers.py +33 -6
- agentx_python-0.8.26/agentx/monitor/patterns.py +250 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/rules.py +15 -2
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/scorer_groups.py +30 -6
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/scorers.py +18 -6
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/resources/agent.py +28 -9
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/resources/conversation.py +31 -10
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/resources/workforce.py +34 -9
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/tracing/tracer.py +6 -4
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/util.py +9 -1
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/version.py +2 -2
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx_python.egg-info/PKG-INFO +7 -2
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx_python.egg-info/SOURCES.txt +5 -1
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx_python.egg-info/requires.txt +3 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/setup.py +2 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_integrations.py +179 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_judge_scorers.py +1 -1
- agentx_python-0.8.26/tests/test_pattern_update_merge.py +200 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_span_tree.py +30 -1
- agentx_python-0.8.26/tests/test_workforce_binding.py +71 -0
- agentx_python-0.8.24/agentx/monitor/patterns.py +0 -158
- {agentx_python-0.8.24 → agentx_python-0.8.26}/LICENSE +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/__init__.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/cli.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/client.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/evaluation_settings.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/models.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/prompts.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/runner.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/tool_schemas.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/exceptions.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/export.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/feedback.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/_traced_call.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/anthropic.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/autogen.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/databricks.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/google_adk.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/google_genai.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/langchain.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/litellm.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/llamaindex.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/moveworks.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/integrations/openai_agents.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/agents.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/models.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/online_evaluators.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/profile.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/review_queue.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/sessions.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/monitor/signals.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/outcomes.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/projects.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/py.typed +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/testing.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/traces.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/tracing/eval_scope.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/tracing/framework_detect.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx/tracing/ingest_client.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx_python.egg-info/entry_points.txt +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/setup.cfg +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_cli_launcher.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_deep_dive_fixes.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_docs_match_sdk.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_error_taxonomy.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_eval_scope.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_framework_detect.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_integration.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_multi_judge.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_pairwise.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_review_queue.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_runner_features.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_selfhost_analysis_fallback.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_selfhost_compat.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_testing.py +0 -0
- {agentx_python-0.8.24 → agentx_python-0.8.26}/tests/test_wire_models.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.26
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -23,6 +23,8 @@ Provides-Extra: openai-agents
|
|
|
23
23
|
Requires-Dist: openai-agents>=0.0.3; extra == "openai-agents"
|
|
24
24
|
Provides-Extra: openai
|
|
25
25
|
Requires-Dist: openai>=1.0.0; extra == "openai"
|
|
26
|
+
Provides-Extra: nvidia-nim
|
|
27
|
+
Requires-Dist: openai>=1.0.0; extra == "nvidia-nim"
|
|
26
28
|
Provides-Extra: anthropic
|
|
27
29
|
Requires-Dist: anthropic>=0.25.0; extra == "anthropic"
|
|
28
30
|
Provides-Extra: google-adk
|
|
@@ -108,7 +110,9 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
108
110
|
pip install --upgrade agentx-python
|
|
109
111
|
```
|
|
110
112
|
|
|
111
|
-
Requires Python 3.9 or newer.
|
|
113
|
+
Requires Python 3.9 or newer for the core SDK. Some integration extras have higher floors set
|
|
114
|
+
by their upstream packages - `[crewai]`, `[autogen]`, and `[databricks]` need Python 3.10+ (`[all]`
|
|
115
|
+
therefore does too); the core tracer and every REST surface stay 3.9-compatible.
|
|
112
116
|
|
|
113
117
|
#### Run the self-host governance suite locally
|
|
114
118
|
|
|
@@ -248,6 +252,7 @@ extra:
|
|
|
248
252
|
| CrewAI | `pip install "agentx-python[crewai]"` | `AgentXCrewObserver` |
|
|
249
253
|
| OpenAI Agents SDK | `pip install "agentx-python[openai-agents]"` | `AgentXTracingProcessor` |
|
|
250
254
|
| OpenAI (raw client) | `pip install "agentx-python[openai]"` | `patch_openai_client` |
|
|
255
|
+
| NVIDIA NIM | `pip install "agentx-python[nvidia-nim]"` | `patch_nim_client` |
|
|
251
256
|
| Anthropic | `pip install "agentx-python[anthropic]"` | `patch_anthropic_client` |
|
|
252
257
|
| Google ADK | `pip install "agentx-python[google-adk]"` | `AgentXADKPlugin` |
|
|
253
258
|
| Google GenAI (Gemini) | `pip install "agentx-python[google-genai]"` | `patch_genai_client` |
|
|
@@ -43,7 +43,9 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
43
43
|
pip install --upgrade agentx-python
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
-
Requires Python 3.9 or newer.
|
|
46
|
+
Requires Python 3.9 or newer for the core SDK. Some integration extras have higher floors set
|
|
47
|
+
by their upstream packages - `[crewai]`, `[autogen]`, and `[databricks]` need Python 3.10+ (`[all]`
|
|
48
|
+
therefore does too); the core tracer and every REST surface stay 3.9-compatible.
|
|
47
49
|
|
|
48
50
|
#### Run the self-host governance suite locally
|
|
49
51
|
|
|
@@ -183,6 +185,7 @@ extra:
|
|
|
183
185
|
| CrewAI | `pip install "agentx-python[crewai]"` | `AgentXCrewObserver` |
|
|
184
186
|
| OpenAI Agents SDK | `pip install "agentx-python[openai-agents]"` | `AgentXTracingProcessor` |
|
|
185
187
|
| OpenAI (raw client) | `pip install "agentx-python[openai]"` | `patch_openai_client` |
|
|
188
|
+
| NVIDIA NIM | `pip install "agentx-python[nvidia-nim]"` | `patch_nim_client` |
|
|
186
189
|
| Anthropic | `pip install "agentx-python[anthropic]"` | `patch_anthropic_client` |
|
|
187
190
|
| Google ADK | `pip install "agentx-python[google-adk]"` | `AgentXADKPlugin` |
|
|
188
191
|
| Google GenAI (Gemini) | `pip install "agentx-python[google-genai]"` | `patch_genai_client` |
|
|
@@ -20,8 +20,8 @@ class AgentX:
|
|
|
20
20
|
# The api_key is NOT written back into os.environ (it used to be): every sub-client
|
|
21
21
|
# below receives it explicitly, and mutating process-global state from a constructor
|
|
22
22
|
# re-pointed unrelated code - the same leak the base_url write below had (deep-dive
|
|
23
|
-
# round 3, bug #1).
|
|
24
|
-
#
|
|
23
|
+
# round 3, bug #1). Flows that still read the env (bare get_headers()) now require
|
|
24
|
+
# the caller to set AGENTX_API_KEY themselves.
|
|
25
25
|
self.api_key = api_key or os.getenv("AGENTX_API_KEY")
|
|
26
26
|
|
|
27
27
|
# base_url overrides AGENTX_API_BASE_URL env var (and the SDK default). It is
|
|
@@ -147,7 +147,7 @@ class AgentX:
|
|
|
147
147
|
response = requests.get(url, headers=get_headers(self.api_key))
|
|
148
148
|
# Check if response was successful
|
|
149
149
|
if response.status_code == 200:
|
|
150
|
-
return Agent(**response.json())
|
|
150
|
+
return Agent(**response.json())._bind(self.api_key, self.base_url)
|
|
151
151
|
else:
|
|
152
152
|
raise AgentXError(
|
|
153
153
|
f"Failed to retrieve agent: {response.reason}. This endpoint is "
|
|
@@ -165,22 +165,26 @@ class AgentX:
|
|
|
165
165
|
response = requests.get(url, headers=get_headers(self.api_key))
|
|
166
166
|
# Check if response was successful
|
|
167
167
|
if response.status_code == 200:
|
|
168
|
-
return [Agent(**agent) for agent in response.json()]
|
|
168
|
+
return [Agent(**agent)._bind(self.api_key, self.base_url) for agent in response.json()]
|
|
169
169
|
else:
|
|
170
170
|
raise AgentXError(
|
|
171
171
|
f"Failed to list agents: {response.reason}. This endpoint is "
|
|
172
172
|
"hosted-platform only - on self-host use client.monitor.agents.list()."
|
|
173
173
|
)
|
|
174
174
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
response
|
|
175
|
+
def list_workforces(self) -> List["Workforce"]:
|
|
176
|
+
"""List all workforces/teams, each bound to this client's credentials - including each
|
|
177
|
+
workforce's ``manager`` and ``agents``, so their calls authenticate the same way.
|
|
178
|
+
|
|
179
|
+
This used to be documented as a static call (``AgentX.list_workforces()``); that form
|
|
180
|
+
was broken (the old staticmethod body referenced ``self`` and raised NameError on any
|
|
181
|
+
non-empty response). Construct a client instead - ``AgentX().list_workforces()`` picks
|
|
182
|
+
up AGENTX_API_KEY / AGENTX_API_BASE_URL from the environment, which is what the static
|
|
183
|
+
form effectively did."""
|
|
184
|
+
url = f"{self.base_url or api_base()}/access/teams"
|
|
185
|
+
response = requests.get(url, headers=get_headers(self.api_key))
|
|
182
186
|
if response.status_code == 200:
|
|
183
|
-
return [Workforce(**workforce) for workforce in response.json()]
|
|
187
|
+
return [Workforce(**workforce)._bind(self.api_key, self.base_url) for workforce in response.json()]
|
|
184
188
|
else:
|
|
185
189
|
raise Exception(
|
|
186
190
|
f"Failed to list workforces: {response.status_code} - {response.reason}"
|
|
@@ -7,4 +7,5 @@
|
|
|
7
7
|
# from agentx.integrations.anthropic import patch_anthropic_client
|
|
8
8
|
# from agentx.integrations.google_adk import AgentXADKPlugin
|
|
9
9
|
# from agentx.integrations.google_genai import patch_genai_client
|
|
10
|
+
# from agentx.integrations.nvidia_nim import patch_nim_client
|
|
10
11
|
# from agentx.integrations.moveworks import MoveworksImporter # Data API pull sync, not in-process
|
|
@@ -144,14 +144,14 @@ class AgentXCrewObserver:
|
|
|
144
144
|
TaskFailedEvent,
|
|
145
145
|
TaskStartedEvent,
|
|
146
146
|
)
|
|
147
|
-
except ImportError
|
|
147
|
+
except Exception: # noqa: BLE001 - crewai import raises TypeError (PEP 604) on py3.9, not just ImportError
|
|
148
148
|
from crewai.utilities.events import crewai_event_bus
|
|
149
149
|
from crewai.utilities.events.task_events import (
|
|
150
150
|
TaskCompletedEvent,
|
|
151
151
|
TaskFailedEvent,
|
|
152
152
|
TaskStartedEvent,
|
|
153
153
|
)
|
|
154
|
-
except ImportError
|
|
154
|
+
except Exception: # noqa: BLE001 - crewai import raises TypeError (PEP 604) on py3.9, not just ImportError
|
|
155
155
|
if not _warned_no_event_bus:
|
|
156
156
|
_warned_no_event_bus = True
|
|
157
157
|
logger.warning(
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""
|
|
2
|
+
NVIDIA NIM integration for AgentX production tracing.
|
|
3
|
+
|
|
4
|
+
NIM (NVIDIA Inference Microservices) serves models behind an OpenAI-compatible
|
|
5
|
+
``/v1/chat/completions`` API, so the client you patch is the ordinary ``openai``
|
|
6
|
+
Python client pointed at a NIM endpoint - a local NIM container
|
|
7
|
+
(``http://localhost:8000/v1``) or NVIDIA's hosted API
|
|
8
|
+
(``https://integrate.api.nvidia.com/v1``). This module reuses the OpenAI patch
|
|
9
|
+
machinery verbatim and differs in exactly one way: traces are stamped
|
|
10
|
+
``framework="nvidia-nim"``, so NIM traffic gets its own row in Monitor's
|
|
11
|
+
Platforms chart and the framework filters instead of blending into "openai".
|
|
12
|
+
|
|
13
|
+
Usage::
|
|
14
|
+
|
|
15
|
+
from agentx.integrations.nvidia_nim import patch_nim_client
|
|
16
|
+
import openai
|
|
17
|
+
|
|
18
|
+
nim = openai.OpenAI(
|
|
19
|
+
base_url="http://localhost:8000/v1", # or https://integrate.api.nvidia.com/v1
|
|
20
|
+
api_key=os.environ.get("NVIDIA_API_KEY", "not-needed-for-local-nim"),
|
|
21
|
+
)
|
|
22
|
+
patch_nim_client(nim, agentx.tracer, name="nim-agent")
|
|
23
|
+
|
|
24
|
+
# All subsequent nim.chat.completions.create() calls are now traced.
|
|
25
|
+
|
|
26
|
+
Works with both ``openai.OpenAI`` and ``openai.AsyncOpenAI`` clients. Token
|
|
27
|
+
usage comes straight off the response's OpenAI-shaped ``usage`` block; NIM
|
|
28
|
+
reports no prompt-cache fields, so cache token counts stay unset.
|
|
29
|
+
|
|
30
|
+
Streaming calls (``stream=True``) are passed through untouched and are not
|
|
31
|
+
currently traced - same posture as ``patch_openai_client``, see its docstring.
|
|
32
|
+
|
|
33
|
+
Requires: ``pip install "agentx-python[nvidia-nim]"`` (installs the ``openai``
|
|
34
|
+
client package; there is no separate NIM SDK dependency).
|
|
35
|
+
"""
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
from typing import Any, Dict, Optional
|
|
39
|
+
|
|
40
|
+
from agentx.tracing.tracer import Tracer
|
|
41
|
+
from agentx.integrations.openai import _patch_chat_completions_create
|
|
42
|
+
|
|
43
|
+
NIM_FRAMEWORK = "nvidia-nim"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def patch_nim_client(
|
|
47
|
+
client: Any,
|
|
48
|
+
tracer: Tracer,
|
|
49
|
+
name: str = "nim-agent",
|
|
50
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
51
|
+
session_id: Optional[str] = None,
|
|
52
|
+
) -> None:
|
|
53
|
+
"""
|
|
54
|
+
Monkey-patch ``client.chat.completions.create`` on an OpenAI-compatible
|
|
55
|
+
client pointed at a NIM endpoint, sending a trace for every non-streaming
|
|
56
|
+
call with ``framework="nvidia-nim"``.
|
|
57
|
+
|
|
58
|
+
The original method is still called and its return value passed through
|
|
59
|
+
unchanged. Sync and async clients both work; ``stream=True`` calls pass
|
|
60
|
+
through untraced. Patching is idempotent - and because it shares the guard
|
|
61
|
+
with ``patch_openai_client``, whichever of the two patched a given client
|
|
62
|
+
first wins (patch each client with the integration that matches where its
|
|
63
|
+
``base_url`` actually points).
|
|
64
|
+
"""
|
|
65
|
+
chat = getattr(client, "chat", None)
|
|
66
|
+
completions = getattr(chat, "completions", None) if chat is not None else None
|
|
67
|
+
if completions is None:
|
|
68
|
+
raise ValueError("Provided client does not have a .chat.completions attribute")
|
|
69
|
+
|
|
70
|
+
_patch_chat_completions_create(completions, tracer, name, metadata, session_id, framework=NIM_FRAMEWORK)
|
|
@@ -113,7 +113,11 @@ def _patch_chat_completions_create(
|
|
|
113
113
|
name: str,
|
|
114
114
|
metadata: Optional[Dict[str, Any]],
|
|
115
115
|
session_id: Optional[str],
|
|
116
|
+
framework: str = "openai",
|
|
116
117
|
) -> None:
|
|
118
|
+
# `framework` exists for OpenAI-compatible endpoints served by other vendors
|
|
119
|
+
# (agentx.integrations.nvidia_nim stamps "nvidia-nim" through here) - the request/response
|
|
120
|
+
# shapes are identical, so they share this machinery instead of duplicating it.
|
|
117
121
|
original = completions_resource.create
|
|
118
122
|
if getattr(original, "_agentx_patched", False):
|
|
119
123
|
return # already patched
|
|
@@ -148,7 +152,7 @@ def _patch_chat_completions_create(
|
|
|
148
152
|
finish_llm_call(
|
|
149
153
|
tracer,
|
|
150
154
|
name=name,
|
|
151
|
-
framework=
|
|
155
|
+
framework=framework,
|
|
152
156
|
metadata=metadata,
|
|
153
157
|
session_id=session_id,
|
|
154
158
|
start_t=start_t,
|
|
@@ -10,6 +10,9 @@ from agentx.monitor.judge_scorers import (
|
|
|
10
10
|
from agentx.monitor.models import MonitorPattern, MonitorProfile, MonitorSignal, SignalOccurrence
|
|
11
11
|
from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
|
|
12
12
|
from agentx.monitor.profile import MonitorProfileClient
|
|
13
|
+
from agentx.monitor.review_queue import ReviewQueueClient, ReviewQueueItem
|
|
14
|
+
from agentx.monitor.rules import MonitorRule, MonitorRulesClient
|
|
15
|
+
from agentx.monitor.scorers import AgentXScorersError, ScorersClient
|
|
13
16
|
from agentx.monitor.scorer_groups import AgentXScorerGroupsError, ScorerGroup, ScorerGroupsClient
|
|
14
17
|
from agentx.monitor.sessions import MonitorSessionClient
|
|
15
18
|
from agentx.monitor.signals import MonitorSignalClient
|
|
@@ -19,6 +22,7 @@ __all__ = [
|
|
|
19
22
|
"AgentXJudgeScorersError",
|
|
20
23
|
"AgentXMonitorError",
|
|
21
24
|
"AgentXScorerGroupsError",
|
|
25
|
+
"AgentXScorersError",
|
|
22
26
|
"ImprovementGroupsClient",
|
|
23
27
|
"JudgeScorer",
|
|
24
28
|
"JudgeScorerBuilder",
|
|
@@ -30,10 +34,15 @@ __all__ = [
|
|
|
30
34
|
"MonitorPatternClient",
|
|
31
35
|
"MonitorProfile",
|
|
32
36
|
"MonitorProfileClient",
|
|
37
|
+
"MonitorRule",
|
|
38
|
+
"MonitorRulesClient",
|
|
33
39
|
"MonitorSessionClient",
|
|
34
40
|
"MonitorSignal",
|
|
35
41
|
"MonitorSignalClient",
|
|
42
|
+
"ReviewQueueClient",
|
|
43
|
+
"ReviewQueueItem",
|
|
36
44
|
"ScorerGroup",
|
|
45
|
+
"ScorersClient",
|
|
37
46
|
"ScorerGroupsClient",
|
|
38
47
|
"SignalOccurrence",
|
|
39
48
|
]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Shared HTTP transport for the monitor sub-clients that own their ``_request`` (scorers,
|
|
2
|
+
judge_scorers, scorer_groups, improvement_groups): one retry schedule mirroring
|
|
3
|
+
``MonitorClient._request``, so ``retry=False`` means the same thing everywhere the client.py
|
|
4
|
+
comment promises it ("retry=False for ANY non-idempotent write")."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
import time
|
|
10
|
+
from typing import Any, Optional
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
# Same schedule as MonitorClient._request (agentx/monitor/client.py).
|
|
17
|
+
_RETRYABLE_STATUS = {429, 500, 502, 503, 504}
|
|
18
|
+
_RETRY_BACKOFF = [1.0, 2.0, 4.0]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def request_with_retries(
|
|
22
|
+
method: str, url: str, *, retry: bool = True, **kwargs: Any
|
|
23
|
+
) -> requests.Response:
|
|
24
|
+
"""``requests.request`` with MonitorClient's transport posture: when ``retry`` is true,
|
|
25
|
+
connection errors and retryable statuses (429/5xx) walk the backoff schedule; the last
|
|
26
|
+
response (whatever its status) is returned for the caller's own error taxonomy.
|
|
27
|
+
|
|
28
|
+
``retry=False`` is single-shot - for non-idempotent writes (creates, deletes) and
|
|
29
|
+
judge-billing POSTs, where a client-side timeout must not fire the same work twice.
|
|
30
|
+
Transport errors keep their ``requests`` exception type (callers guard on
|
|
31
|
+
``requests.Timeout`` for judge-billing endpoints)."""
|
|
32
|
+
schedule = [0.0] + _RETRY_BACKOFF if retry else [0.0]
|
|
33
|
+
last_exc: Optional[Exception] = None
|
|
34
|
+
for attempt, wait in enumerate(schedule):
|
|
35
|
+
if wait:
|
|
36
|
+
time.sleep(wait)
|
|
37
|
+
try:
|
|
38
|
+
resp = requests.request(method, url, **kwargs)
|
|
39
|
+
except requests.RequestException as e:
|
|
40
|
+
last_exc = e
|
|
41
|
+
logger.debug("Request error (attempt %d): %s", attempt + 1, e)
|
|
42
|
+
continue
|
|
43
|
+
if retry and resp.status_code in _RETRYABLE_STATUS and attempt < len(schedule) - 1:
|
|
44
|
+
logger.debug("Retryable status %d (attempt %d)", resp.status_code, attempt + 1)
|
|
45
|
+
continue
|
|
46
|
+
return resp
|
|
47
|
+
assert last_exc is not None # every non-raising path returned above
|
|
48
|
+
raise last_exc
|
|
@@ -205,9 +205,11 @@ class MonitorClient:
|
|
|
205
205
|
def _request(
|
|
206
206
|
self, method: str, path: str, timeout: int = 30, base: Optional[str] = None, retry: bool = True, **kwargs
|
|
207
207
|
) -> Any:
|
|
208
|
-
# retry=False for non-idempotent
|
|
209
|
-
#
|
|
210
|
-
#
|
|
208
|
+
# retry=False for ANY non-idempotent write - duplicating creates, deletes (a lost
|
|
209
|
+
# response + retry turns success into a spurious 404), and judge-spending POSTs
|
|
210
|
+
# (sweep, coherence, portability, tuning: a client-side timeout must not fire the
|
|
211
|
+
# same LLM-billing work twice while the first invocation still runs server-side).
|
|
212
|
+
# The judge list is the example set, not the rule. Same precedent as
|
|
211
213
|
# EvaluationsClient._request / analyze_run.
|
|
212
214
|
url = f"{base or self._base_url}{path}"
|
|
213
215
|
last_exc: Optional[Exception] = None
|
|
@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional
|
|
|
4
4
|
|
|
5
5
|
import requests
|
|
6
6
|
|
|
7
|
+
from agentx.monitor._transport import request_with_retries
|
|
7
8
|
from agentx.util import api_base, get_headers
|
|
8
9
|
from agentx.exceptions import AgentXError, AgentXAuthError, AgentXValidationError
|
|
9
10
|
|
|
@@ -45,7 +46,9 @@ class ImprovementGroupsClient:
|
|
|
45
46
|
self._workspace_id = workspace_id
|
|
46
47
|
self._base_url = (base_url or api_base()).rstrip("/")
|
|
47
48
|
|
|
48
|
-
def _request(
|
|
49
|
+
def _request(
|
|
50
|
+
self, method: str, path: str, json: Any = None, timeout: int = 120, retry: bool = True
|
|
51
|
+
) -> Any:
|
|
49
52
|
params = None
|
|
50
53
|
if self._workspace_id:
|
|
51
54
|
# Mirrors MonitorClient._workspace_params/_with_workspace: GETs (and DELETEs)
|
|
@@ -57,9 +60,12 @@ class ImprovementGroupsClient:
|
|
|
57
60
|
json = {**json, "workspaceId": self._workspace_id}
|
|
58
61
|
else:
|
|
59
62
|
params = {"workspaceId": self._workspace_id}
|
|
60
|
-
|
|
63
|
+
# retry=False for ANY non-idempotent write (member deletes, the report-generating
|
|
64
|
+
# POST) - MonitorClient._request's posture, via the shared monitor transport.
|
|
65
|
+
resp = request_with_retries(
|
|
61
66
|
method,
|
|
62
67
|
f"{self._base_url}/agent-monitoring{path}",
|
|
68
|
+
retry=retry,
|
|
63
69
|
headers={**get_headers(self._api_key), "Content-Type": "application/json"},
|
|
64
70
|
json=json,
|
|
65
71
|
params=params,
|
|
@@ -92,7 +98,9 @@ class ImprovementGroupsClient:
|
|
|
92
98
|
|
|
93
99
|
def remove_member(self, group_id: str, member_id: str) -> None:
|
|
94
100
|
"""Prune a member before spending the group (a confirm that turned out uninteresting)."""
|
|
95
|
-
|
|
101
|
+
# retry=False: a lost response + transport retry would turn a successful delete
|
|
102
|
+
# into a spurious 404.
|
|
103
|
+
self._request("DELETE", f"/improvement-groups/{group_id}/members/{member_id}", retry=False)
|
|
96
104
|
|
|
97
105
|
def generate_report(self, group_id: str, model: Optional[str] = None) -> Dict[str, Any]:
|
|
98
106
|
"""Spend the group: one real LLM call clustering the confirmed failures into issues
|
|
@@ -101,7 +109,11 @@ class ImprovementGroupsClient:
|
|
|
101
109
|
payload: Dict[str, Any] = {}
|
|
102
110
|
if model is not None:
|
|
103
111
|
payload["model"] = model
|
|
104
|
-
|
|
112
|
+
# retry=False: spends the group (real LLM billing) - a client-side timeout must not
|
|
113
|
+
# fire the same generation twice while the first still runs server-side.
|
|
114
|
+
return self._request(
|
|
115
|
+
"POST", f"/improvement-groups/{group_id}/report", json=payload, timeout=300, retry=False
|
|
116
|
+
)["report"]
|
|
105
117
|
|
|
106
118
|
def list_reports(self) -> List[Dict[str, Any]]:
|
|
107
119
|
return self._request("GET", "/improvement-reports").get("improvementReports", [])
|
|
@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional
|
|
|
5
5
|
|
|
6
6
|
import requests
|
|
7
7
|
|
|
8
|
+
from agentx.monitor._transport import request_with_retries
|
|
8
9
|
from agentx.util import api_base, get_headers
|
|
9
10
|
from agentx.exceptions import AgentXError, AgentXAuthError, AgentXValidationError
|
|
10
11
|
|
|
@@ -97,7 +98,9 @@ class JudgeScorersClient:
|
|
|
97
98
|
# Captured once at construction so two clients with different bases can coexist.
|
|
98
99
|
self._base_url = (base_url or api_base()).rstrip("/")
|
|
99
100
|
|
|
100
|
-
def _request(
|
|
101
|
+
def _request(
|
|
102
|
+
self, method: str, path: str, json: Any = None, timeout: int = 60, retry: bool = True
|
|
103
|
+
) -> Any:
|
|
101
104
|
params = None
|
|
102
105
|
if self._workspace_id:
|
|
103
106
|
# Mirrors MonitorClient._workspace_params/_with_workspace: GETs (and DELETEs)
|
|
@@ -109,9 +112,13 @@ class JudgeScorersClient:
|
|
|
109
112
|
json = {**json, "workspaceId": self._workspace_id}
|
|
110
113
|
else:
|
|
111
114
|
params = {"workspaceId": self._workspace_id}
|
|
112
|
-
|
|
115
|
+
# retry=False for ANY non-idempotent write (creates, deletes) and judge-spending POST
|
|
116
|
+
# (tune/validate/publish) - MonitorClient._request's posture, via the shared monitor
|
|
117
|
+
# transport.
|
|
118
|
+
resp = request_with_retries(
|
|
113
119
|
method,
|
|
114
120
|
f"{self._base_url}/agent-monitoring{path}",
|
|
121
|
+
retry=retry,
|
|
115
122
|
headers={**get_headers(self._api_key), "Content-Type": "application/json"},
|
|
116
123
|
json=json,
|
|
117
124
|
params=params,
|
|
@@ -263,7 +270,10 @@ class JudgeScorersClient:
|
|
|
263
270
|
payload["offline"] = offline
|
|
264
271
|
if online is not None:
|
|
265
272
|
payload["online"] = online
|
|
266
|
-
|
|
273
|
+
# Server-side create: a timeout + transport retry would create the scorer twice.
|
|
274
|
+
return JudgeScorer(
|
|
275
|
+
self._request("POST", "/judge-scorers", json=payload, retry=False)["judgeScorer"]
|
|
276
|
+
)
|
|
267
277
|
|
|
268
278
|
def get(self, scorer_id: str) -> JudgeScorer:
|
|
269
279
|
return JudgeScorer(self._request("GET", f"/judge-scorers/{scorer_id}")["judgeScorer"])
|
|
@@ -300,7 +310,9 @@ class JudgeScorersClient:
|
|
|
300
310
|
def delete(self, scorer_id: str) -> None:
|
|
301
311
|
"""Delete the scorer: rubric, version history, and online profile together.
|
|
302
312
|
Irreversible; refused for the built-in Session Baseline Judge."""
|
|
303
|
-
|
|
313
|
+
# retry=False: a lost response + transport retry would turn a successful delete
|
|
314
|
+
# into a spurious 404.
|
|
315
|
+
self._request("DELETE", f"/judge-scorers/{scorer_id}", retry=False)
|
|
304
316
|
|
|
305
317
|
# ------------------------------------------------------------------
|
|
306
318
|
# Online-profile pass-throughs (calibration / tuning / ratings / events)
|
|
@@ -334,8 +346,14 @@ class JudgeScorersClient:
|
|
|
334
346
|
def tune(self, scorer_id: str, window: str = "7d") -> dict:
|
|
335
347
|
"""Propose a rewrite of the rubric from calibration disagreements (LLM call, slow).
|
|
336
348
|
``window`` accepts the same values as :meth:`calibration`, including "rubric"."""
|
|
349
|
+
# retry=False (judge-spending POST, MonitorClient.propose_online_evaluator_tuning's
|
|
350
|
+
# posture): a client-side timeout must not fire the same LLM-billing work twice.
|
|
337
351
|
data = self._request(
|
|
338
|
-
"POST",
|
|
352
|
+
"POST",
|
|
353
|
+
f"/online-evaluators/{self._profile_id(scorer_id)}/tune",
|
|
354
|
+
json={"window": window},
|
|
355
|
+
timeout=300,
|
|
356
|
+
retry=False,
|
|
339
357
|
)
|
|
340
358
|
# The wire wraps the proposal ({"proposal": {...}}); unwrap like the legacy client so
|
|
341
359
|
# proposal["reasoning"] / the criteria fields are directly addressable.
|
|
@@ -344,11 +362,13 @@ class JudgeScorersClient:
|
|
|
344
362
|
def validate_tuning(self, scorer_id: str, criteria: Dict[str, Any], window: str = "7d") -> dict:
|
|
345
363
|
"""Re-judge the disagreement + control cases with candidate criteria (LLM calls, slow)."""
|
|
346
364
|
# The wire takes the criteria fields at the TOP level of the body, not nested.
|
|
365
|
+
# retry=False (judge-spending POST) - same posture as tune() above.
|
|
347
366
|
return self._request(
|
|
348
367
|
"POST",
|
|
349
368
|
f"/online-evaluators/{self._profile_id(scorer_id)}/tune/validate",
|
|
350
369
|
json={**criteria, "window": window},
|
|
351
370
|
timeout=600,
|
|
371
|
+
retry=False,
|
|
352
372
|
)
|
|
353
373
|
|
|
354
374
|
def publish_tuning(
|
|
@@ -382,7 +402,14 @@ class JudgeScorersClient:
|
|
|
382
402
|
payload["validation"] = validation_payload
|
|
383
403
|
if force:
|
|
384
404
|
payload["force"] = True
|
|
385
|
-
|
|
405
|
+
# retry=False: a non-idempotent write (each publish appends a rubric version) -
|
|
406
|
+
# MonitorClient.publish_online_evaluator_tuning's posture.
|
|
407
|
+
return self._request(
|
|
408
|
+
"POST",
|
|
409
|
+
f"/online-evaluators/{self._profile_id(scorer_id)}/tune/publish",
|
|
410
|
+
json=payload,
|
|
411
|
+
retry=False,
|
|
412
|
+
)
|
|
386
413
|
|
|
387
414
|
def ratings(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorRatingPoint]":
|
|
388
415
|
"""Bucketed average-rating-over-time for this scorer's live checks - same typed points
|