agentloop-profiler 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentloop/__init__.py +136 -0
- agentloop/ablation_protocol.py +229 -0
- agentloop/ablations.py +564 -0
- agentloop/aggregate_cli.py +88 -0
- agentloop/aggregate_stats.py +175 -0
- agentloop/aggregates.py +532 -0
- agentloop/audit.py +39 -0
- agentloop/autoinstrument.py +145 -0
- agentloop/batch_experiments.py +295 -0
- agentloop/batching.py +95 -0
- agentloop/budget_types.py +335 -0
- agentloop/budgets.py +539 -0
- agentloop/calibration.py +872 -0
- agentloop/checkpoint_state.py +306 -0
- agentloop/checkpoint_store.py +86 -0
- agentloop/ci.py +202 -0
- agentloop/cli.py +1314 -0
- agentloop/client.py +228 -0
- agentloop/completion.py +345 -0
- agentloop/completion_types.py +199 -0
- agentloop/config.py +50 -0
- agentloop/context_controls.py +443 -0
- agentloop/context_experiments.py +337 -0
- agentloop/context_types.py +216 -0
- agentloop/costs.py +667 -0
- agentloop/cursor_validation.py +48 -0
- agentloop/decorators.py +445 -0
- agentloop/demo.py +300 -0
- agentloop/doctor.py +341 -0
- agentloop/drift.py +361 -0
- agentloop/drift_cli.py +147 -0
- agentloop/drift_types.py +110 -0
- agentloop/drift_windows.py +340 -0
- agentloop/entrypoint.py +201 -0
- agentloop/estimates.py +133 -0
- agentloop/events.py +75 -0
- agentloop/experiment_artifacts.py +80 -0
- agentloop/experiment_observations.py +169 -0
- agentloop/experiment_reports.py +545 -0
- agentloop/experiment_types.py +355 -0
- agentloop/experiments.py +392 -0
- agentloop/exporters.py +117 -0
- agentloop/finding_benchmark_types.py +131 -0
- agentloop/finding_benchmarks.py +424 -0
- agentloop/findings.py +419 -0
- agentloop/graph.py +464 -0
- agentloop/harness.py +825 -0
- agentloop/harness_bench/__init__.py +1 -0
- agentloop/harness_bench/cli.py +77 -0
- agentloop/harness_bench/drivers.py +96 -0
- agentloop/harness_bench/offline.py +507 -0
- agentloop/harness_evidence.py +585 -0
- agentloop/harness_usage.py +91 -0
- agentloop/html_report.py +860 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/harbor/__init__.py +1 -0
- agentloop/integrations/harbor/atif.py +982 -0
- agentloop/integrations/harbor/atif_validation.py +329 -0
- agentloop/integrations/harbor/cli.py +207 -0
- agentloop/integrations/harbor/manifest.py +86 -0
- agentloop/integrations/harbor/trial_evidence.py +196 -0
- agentloop/integrations/harbor/trials.py +1109 -0
- agentloop/integrations/harbor/verifier.py +118 -0
- agentloop/integrations/langgraph.py +150 -0
- agentloop/integrations/langgraph_harness.py +466 -0
- agentloop/integrations/langgraph_recovery.py +366 -0
- agentloop/integrations/omnigent/__init__.py +1 -0
- agentloop/integrations/omnigent/cli.py +78 -0
- agentloop/integrations/omnigent/evidence.py +93 -0
- agentloop/integrations/omnigent/telemetry.py +422 -0
- agentloop/integrations/openai.py +415 -0
- agentloop/integrations/openai_agents.py +291 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/interoperability/__init__.py +5 -0
- agentloop/interoperability/artifacts.py +112 -0
- agentloop/interoperability/cli.py +59 -0
- agentloop/interoperability/cohort_evidence.py +175 -0
- agentloop/interoperability/cohorts.py +681 -0
- agentloop/interoperability/contracts.py +427 -0
- agentloop/interoperability/coordination.py +554 -0
- agentloop/interoperability/evidence.py +215 -0
- agentloop/interoperability/otlp_jsonl.py +775 -0
- agentloop/interoperability/otlp_validation.py +496 -0
- agentloop/interoperability/registry.py +561 -0
- agentloop/interoperability/validation.py +333 -0
- agentloop/intervention_service.py +110 -0
- agentloop/intervention_store.py +118 -0
- agentloop/interventions.py +190 -0
- agentloop/issues.py +90 -0
- agentloop/judgment_adapters.py +125 -0
- agentloop/judgment_benchmark_types.py +98 -0
- agentloop/judgment_benchmarks.py +606 -0
- agentloop/judgment_types.py +241 -0
- agentloop/judgment_views.py +38 -0
- agentloop/judgments.py +342 -0
- agentloop/loop_guards.py +274 -0
- agentloop/loop_types.py +66 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +342 -0
- agentloop/migrations.py +303 -0
- agentloop/model_routing.py +470 -0
- agentloop/onboarding.py +315 -0
- agentloop/operations.py +44 -0
- agentloop/optimizer.py +199 -0
- agentloop/otel.py +715 -0
- agentloop/otel_ids.py +81 -0
- agentloop/otel_semantics.py +134 -0
- agentloop/parallelism.py +141 -0
- agentloop/patches.py +603 -0
- agentloop/plan_export.py +126 -0
- agentloop/policy_promotion.py +325 -0
- agentloop/promotion_evidence.py +304 -0
- agentloop/promotion_types.py +209 -0
- agentloop/quality.py +446 -0
- agentloop/ranking.py +529 -0
- agentloop/recovery_evidence.py +96 -0
- agentloop/replay.py +753 -0
- agentloop/retention.py +375 -0
- agentloop/retention_types.py +113 -0
- agentloop/reuse_experiments.py +247 -0
- agentloop/routing_checks.py +116 -0
- agentloop/routing_types.py +241 -0
- agentloop/rules.py +396 -0
- agentloop/runtime.py +286 -0
- agentloop/savings.py +158 -0
- agentloop/scheduling_plan.py +99 -0
- agentloop/scheduling_types.py +169 -0
- agentloop/schema.py +257 -0
- agentloop/semantic_rules.py +200 -0
- agentloop/semantic_waste.py +345 -0
- agentloop/semantic_waste_types.py +88 -0
- agentloop/server.py +494 -0
- agentloop/store.py +1429 -0
- agentloop/structural_experiments.py +431 -0
- agentloop/structural_types.py +113 -0
- agentloop/structured_quality.py +1020 -0
- agentloop/studies.py +629 -0
- agentloop/study_statistics.py +58 -0
- agentloop/substitution_evidence.py +199 -0
- agentloop/substitution_export.py +190 -0
- agentloop/substitution_reports.py +407 -0
- agentloop/substitution_types.py +284 -0
- agentloop/substitutions.py +267 -0
- agentloop/timing.py +142 -0
- agentloop/tokens.py +188 -0
- agentloop/tool_scheduling.py +410 -0
- agentloop/trace_sources.py +31 -0
- agentloop/tracer.py +494 -0
- agentloop/value.py +306 -0
- agentloop/version.py +3 -0
- agentloop/workflow_types.py +264 -0
- agentloop/workflows.py +284 -0
- agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
- agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
- agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
- agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +5 -0
- dashboard/app.py +922 -0
- dashboard/streamlit_compat.py +63 -0
- dashboard/trace_upload.py +57 -0
- dashboard/value_view.py +105 -0
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import time
|
|
5
|
+
from functools import wraps
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from agentloop.events import format_exception_detail, utc_now_iso
|
|
9
|
+
from agentloop.tokens import PROVIDER, UNAVAILABLE
|
|
10
|
+
from agentloop.tracer import current_event_id, current_trace, record_model_call
|
|
11
|
+
|
|
12
|
+
# Marker set on wrappers so repeated instrumentation of the same callable is a
|
|
13
|
+
# no-op instead of stacking wrappers and double-recording.
|
|
14
|
+
_INSTRUMENTED_FLAG = "__agentloop_instrumented__"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _usage_get(usage: Any, *names: str) -> int:
|
|
18
|
+
for name in names:
|
|
19
|
+
if usage is None:
|
|
20
|
+
continue
|
|
21
|
+
if isinstance(usage, dict) and name in usage:
|
|
22
|
+
return int(usage.get(name) or 0)
|
|
23
|
+
value = getattr(usage, name, None)
|
|
24
|
+
if value is not None:
|
|
25
|
+
return int(value or 0)
|
|
26
|
+
return 0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _find_usage(obj: Any) -> Any:
|
|
30
|
+
"""Locate a usage object on a response or a streaming event.
|
|
31
|
+
|
|
32
|
+
Non-streaming responses and chat chunks expose ``usage`` directly. The locked
|
|
33
|
+
Responses streaming API instead delivers final usage on the terminal
|
|
34
|
+
``ResponseCompletedEvent`` under ``event.response.usage`` (the event itself has
|
|
35
|
+
no ``usage``), so fall back to the nested response shape.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
if obj is None:
|
|
39
|
+
return None
|
|
40
|
+
usage = getattr(obj, "usage", None)
|
|
41
|
+
if usage is None and isinstance(obj, dict):
|
|
42
|
+
usage = obj.get("usage")
|
|
43
|
+
if usage is not None:
|
|
44
|
+
return usage
|
|
45
|
+
response = getattr(obj, "response", None)
|
|
46
|
+
if response is None and isinstance(obj, dict):
|
|
47
|
+
response = obj.get("response")
|
|
48
|
+
if response is None:
|
|
49
|
+
return None
|
|
50
|
+
nested = getattr(response, "usage", None)
|
|
51
|
+
if nested is None and isinstance(response, dict):
|
|
52
|
+
nested = response.get("usage")
|
|
53
|
+
return nested
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _extract_usage(result: Any) -> tuple[int, int, str]:
|
|
57
|
+
"""Return token counts and the provenance that justifies them.
|
|
58
|
+
|
|
59
|
+
A response carrying a usage object gives provider-reported counts. Without
|
|
60
|
+
one there is nothing to count, and AgentLoop records ``unavailable`` rather
|
|
61
|
+
than a zero that would later read as a real measurement of zero tokens.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
usage = _find_usage(result)
|
|
65
|
+
if usage is None:
|
|
66
|
+
return 0, 0, UNAVAILABLE
|
|
67
|
+
input_tokens = _usage_get(usage, "input_tokens", "prompt_tokens")
|
|
68
|
+
output_tokens = _usage_get(usage, "output_tokens", "completion_tokens")
|
|
69
|
+
return input_tokens, output_tokens, PROVIDER
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _metadata_from_call(args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
73
|
+
metadata: dict[str, Any] = {"integration": "openai"}
|
|
74
|
+
if "tools" in kwargs and kwargs["tools"] is not None:
|
|
75
|
+
try:
|
|
76
|
+
metadata["tool_count"] = len(kwargs["tools"])
|
|
77
|
+
except TypeError:
|
|
78
|
+
metadata["tool_count"] = 1
|
|
79
|
+
if "messages" in kwargs and kwargs["messages"] is not None:
|
|
80
|
+
try:
|
|
81
|
+
metadata["message_count"] = len(kwargs["messages"])
|
|
82
|
+
except TypeError:
|
|
83
|
+
metadata["message_count"] = 1
|
|
84
|
+
if args:
|
|
85
|
+
metadata["positional_args"] = len(args)
|
|
86
|
+
return metadata
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _model_from_call(kwargs: dict[str, Any], default_model: str | None = None) -> str | None:
|
|
90
|
+
return (
|
|
91
|
+
str(kwargs.get("model") or default_model)
|
|
92
|
+
if (kwargs.get("model") or default_model)
|
|
93
|
+
else None
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class _CallRecorder:
|
|
98
|
+
"""Collects timing for one instrumented call and records it exactly once.
|
|
99
|
+
|
|
100
|
+
Recording is deferred so streaming calls can be finalized when the stream is
|
|
101
|
+
consumed rather than when the iterator is created. Trace ownership is captured
|
|
102
|
+
at invocation time: the event is recorded into the trace that was active when
|
|
103
|
+
the call was made, regardless of which context consumes the stream later. If no
|
|
104
|
+
trace was active at invocation the call is never recorded, and the application
|
|
105
|
+
call is not disturbed.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
def __init__(
|
|
109
|
+
self,
|
|
110
|
+
name: str,
|
|
111
|
+
args: tuple[Any, ...],
|
|
112
|
+
kwargs: dict[str, Any],
|
|
113
|
+
default_model: str | None,
|
|
114
|
+
) -> None:
|
|
115
|
+
self._name = name
|
|
116
|
+
self._args = args
|
|
117
|
+
self._kwargs = kwargs
|
|
118
|
+
self._default_model = default_model
|
|
119
|
+
self._started_at = utc_now_iso()
|
|
120
|
+
self._start = time.perf_counter()
|
|
121
|
+
self._done = False
|
|
122
|
+
# Capture ownership now, at invocation time, not at finalization time.
|
|
123
|
+
self._trace = current_trace()
|
|
124
|
+
self._parent_id = current_event_id()
|
|
125
|
+
|
|
126
|
+
def finalize(self, result: Any = None, *, status: str = "ok", error: str | None = None) -> None:
|
|
127
|
+
if self._done:
|
|
128
|
+
return
|
|
129
|
+
self._done = True
|
|
130
|
+
if self._trace is None:
|
|
131
|
+
# No trace was active when the call was invoked: never record, and do
|
|
132
|
+
# not turn the caller's successful application call into an error.
|
|
133
|
+
return
|
|
134
|
+
input_tokens, output_tokens, token_provenance = _extract_usage(result)
|
|
135
|
+
record_model_call(
|
|
136
|
+
self._name,
|
|
137
|
+
started_at=self._started_at,
|
|
138
|
+
duration_ms=(time.perf_counter() - self._start) * 1000,
|
|
139
|
+
model=_model_from_call(self._kwargs, self._default_model),
|
|
140
|
+
input_tokens=input_tokens,
|
|
141
|
+
output_tokens=output_tokens,
|
|
142
|
+
token_provenance=token_provenance,
|
|
143
|
+
status=status,
|
|
144
|
+
error=error,
|
|
145
|
+
metadata=_metadata_from_call(self._args, self._kwargs),
|
|
146
|
+
parent_id=self._parent_id,
|
|
147
|
+
trace=self._trace,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def fail(self, exc: BaseException, result: Any = None) -> None:
|
|
151
|
+
"""Record one failed outcome while preserving the caller's exception."""
|
|
152
|
+
|
|
153
|
+
self.finalize(result, status="error", error=format_exception_detail(exc))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _has_usage(obj: Any) -> bool:
|
|
157
|
+
return _find_usage(obj) is not None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _stream_requested(kwargs: dict[str, Any]) -> bool:
|
|
161
|
+
return bool(kwargs.get("stream"))
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _is_sync_stream(result: Any) -> bool:
|
|
165
|
+
if isinstance(result, (dict, list, tuple, set, str, bytes, bytearray)):
|
|
166
|
+
return False
|
|
167
|
+
return (
|
|
168
|
+
hasattr(result, "__iter__") or hasattr(result, "__next__") or hasattr(result, "__enter__")
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _is_async_stream(result: Any) -> bool:
|
|
173
|
+
return (
|
|
174
|
+
hasattr(result, "__aiter__")
|
|
175
|
+
or hasattr(result, "__anext__")
|
|
176
|
+
or hasattr(result, "__aenter__")
|
|
177
|
+
or inspect.isasyncgen(result)
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class _InstrumentedSyncStream:
|
|
182
|
+
"""Proxy over a sync streaming response that finalizes on consumption.
|
|
183
|
+
|
|
184
|
+
Iteration, context-manager use, and ``close()`` all end the stream; whichever
|
|
185
|
+
happens first records the call (once). Every other attribute is delegated to
|
|
186
|
+
the wrapped stream so SDK behavior is preserved.
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
def __init__(self, stream: Any, recorder: _CallRecorder) -> None:
|
|
190
|
+
self._stream = stream
|
|
191
|
+
self._recorder = recorder
|
|
192
|
+
self._iter: Any = None
|
|
193
|
+
self._usage_source: Any = None
|
|
194
|
+
|
|
195
|
+
def __iter__(self) -> _InstrumentedSyncStream:
|
|
196
|
+
try:
|
|
197
|
+
self._iter = iter(self._stream)
|
|
198
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
199
|
+
self._recorder.fail(exc, self._usage_source)
|
|
200
|
+
raise
|
|
201
|
+
return self
|
|
202
|
+
|
|
203
|
+
def __next__(self) -> Any:
|
|
204
|
+
try:
|
|
205
|
+
if self._iter is None:
|
|
206
|
+
self._iter = iter(self._stream)
|
|
207
|
+
chunk = next(self._iter)
|
|
208
|
+
except StopIteration:
|
|
209
|
+
self._recorder.finalize(self._usage_source)
|
|
210
|
+
raise
|
|
211
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
212
|
+
self._recorder.fail(exc, self._usage_source)
|
|
213
|
+
raise
|
|
214
|
+
if _has_usage(chunk):
|
|
215
|
+
self._usage_source = chunk
|
|
216
|
+
return chunk
|
|
217
|
+
|
|
218
|
+
def __enter__(self) -> _InstrumentedSyncStream:
|
|
219
|
+
try:
|
|
220
|
+
if hasattr(self._stream, "__enter__"):
|
|
221
|
+
self._stream.__enter__()
|
|
222
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
223
|
+
self._recorder.fail(exc, self._usage_source)
|
|
224
|
+
raise
|
|
225
|
+
return self
|
|
226
|
+
|
|
227
|
+
def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> Any:
|
|
228
|
+
try:
|
|
229
|
+
suppress = (
|
|
230
|
+
self._stream.__exit__(exc_type, exc, tb)
|
|
231
|
+
if hasattr(self._stream, "__exit__")
|
|
232
|
+
else None
|
|
233
|
+
)
|
|
234
|
+
except BaseException as exit_exc: # noqa: BLE001 - record then propagate unchanged
|
|
235
|
+
self._recorder.fail(exit_exc, self._usage_source)
|
|
236
|
+
raise
|
|
237
|
+
if exc is not None and not suppress:
|
|
238
|
+
self._recorder.fail(exc, self._usage_source)
|
|
239
|
+
else:
|
|
240
|
+
self._recorder.finalize(self._usage_source)
|
|
241
|
+
return suppress
|
|
242
|
+
|
|
243
|
+
def close(self) -> Any:
|
|
244
|
+
try:
|
|
245
|
+
result = self._stream.close() if hasattr(self._stream, "close") else None
|
|
246
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
247
|
+
self._recorder.fail(exc, self._usage_source)
|
|
248
|
+
raise
|
|
249
|
+
self._recorder.finalize(self._usage_source)
|
|
250
|
+
return result
|
|
251
|
+
|
|
252
|
+
def __getattr__(self, item: str) -> Any:
|
|
253
|
+
return getattr(self._stream, item)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
class _InstrumentedAsyncStream:
|
|
257
|
+
"""Async counterpart of :class:`_InstrumentedSyncStream`."""
|
|
258
|
+
|
|
259
|
+
def __init__(self, stream: Any, recorder: _CallRecorder) -> None:
|
|
260
|
+
self._stream = stream
|
|
261
|
+
self._recorder = recorder
|
|
262
|
+
self._iter: Any = None
|
|
263
|
+
self._usage_source: Any = None
|
|
264
|
+
|
|
265
|
+
def __aiter__(self) -> _InstrumentedAsyncStream:
|
|
266
|
+
try:
|
|
267
|
+
self._iter = (
|
|
268
|
+
self._stream.__aiter__() if hasattr(self._stream, "__aiter__") else self._stream
|
|
269
|
+
)
|
|
270
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
271
|
+
self._recorder.fail(exc, self._usage_source)
|
|
272
|
+
raise
|
|
273
|
+
return self
|
|
274
|
+
|
|
275
|
+
async def __anext__(self) -> Any:
|
|
276
|
+
try:
|
|
277
|
+
if self._iter is None:
|
|
278
|
+
self._iter = (
|
|
279
|
+
self._stream.__aiter__() if hasattr(self._stream, "__aiter__") else self._stream
|
|
280
|
+
)
|
|
281
|
+
chunk = await self._iter.__anext__()
|
|
282
|
+
except StopAsyncIteration:
|
|
283
|
+
self._recorder.finalize(self._usage_source)
|
|
284
|
+
raise
|
|
285
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
286
|
+
self._recorder.fail(exc, self._usage_source)
|
|
287
|
+
raise
|
|
288
|
+
if _has_usage(chunk):
|
|
289
|
+
self._usage_source = chunk
|
|
290
|
+
return chunk
|
|
291
|
+
|
|
292
|
+
async def __aenter__(self) -> _InstrumentedAsyncStream:
|
|
293
|
+
try:
|
|
294
|
+
if hasattr(self._stream, "__aenter__"):
|
|
295
|
+
await self._stream.__aenter__()
|
|
296
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
297
|
+
self._recorder.fail(exc, self._usage_source)
|
|
298
|
+
raise
|
|
299
|
+
return self
|
|
300
|
+
|
|
301
|
+
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> Any:
|
|
302
|
+
try:
|
|
303
|
+
suppress = (
|
|
304
|
+
await self._stream.__aexit__(exc_type, exc, tb)
|
|
305
|
+
if hasattr(self._stream, "__aexit__")
|
|
306
|
+
else None
|
|
307
|
+
)
|
|
308
|
+
except BaseException as exit_exc: # noqa: BLE001 - record then propagate unchanged
|
|
309
|
+
self._recorder.fail(exit_exc, self._usage_source)
|
|
310
|
+
raise
|
|
311
|
+
if exc is not None and not suppress:
|
|
312
|
+
self._recorder.fail(exc, self._usage_source)
|
|
313
|
+
else:
|
|
314
|
+
self._recorder.finalize(self._usage_source)
|
|
315
|
+
return suppress
|
|
316
|
+
|
|
317
|
+
async def aclose(self) -> Any:
|
|
318
|
+
try:
|
|
319
|
+
result = await self._stream.aclose() if hasattr(self._stream, "aclose") else None
|
|
320
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
321
|
+
self._recorder.fail(exc, self._usage_source)
|
|
322
|
+
raise
|
|
323
|
+
self._recorder.finalize(self._usage_source)
|
|
324
|
+
return result
|
|
325
|
+
|
|
326
|
+
def __getattr__(self, item: str) -> Any:
|
|
327
|
+
return getattr(self._stream, item)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def instrument_callable(
|
|
331
|
+
fn: Any,
|
|
332
|
+
*,
|
|
333
|
+
name: str = "openai.model_call",
|
|
334
|
+
default_model: str | None = None,
|
|
335
|
+
) -> Any:
|
|
336
|
+
"""Wrap a sync or async OpenAI-like callable and record token usage.
|
|
337
|
+
|
|
338
|
+
The wrapper is dependency-free and works with OpenAI SDK resource methods such
|
|
339
|
+
as `client.responses.create(...)` or `client.chat.completions.create(...)`.
|
|
340
|
+
It records prompt/completion usage when the returned object exposes a
|
|
341
|
+
`.usage` object or a `{"usage": ...}` dict.
|
|
342
|
+
|
|
343
|
+
Wrapping is idempotent: re-instrumenting an already-instrumented callable
|
|
344
|
+
returns it unchanged, so one request records one event. Streaming responses
|
|
345
|
+
(requested with `stream=True`) are finalized when the stream is consumed,
|
|
346
|
+
closed, fails, or is cancelled — not when the iterator is created — so the
|
|
347
|
+
recorded duration covers consumption and final usage is captured when the SDK
|
|
348
|
+
provides it. Calls made without an active trace are not recorded and are not
|
|
349
|
+
turned into errors.
|
|
350
|
+
"""
|
|
351
|
+
|
|
352
|
+
if getattr(fn, _INSTRUMENTED_FLAG, False):
|
|
353
|
+
return fn
|
|
354
|
+
|
|
355
|
+
if inspect.iscoroutinefunction(fn):
|
|
356
|
+
|
|
357
|
+
@wraps(fn)
|
|
358
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
359
|
+
recorder = _CallRecorder(name, args, kwargs, default_model)
|
|
360
|
+
try:
|
|
361
|
+
result = await fn(*args, **kwargs)
|
|
362
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
363
|
+
recorder.fail(exc)
|
|
364
|
+
raise
|
|
365
|
+
if _stream_requested(kwargs) and _is_async_stream(result):
|
|
366
|
+
return _InstrumentedAsyncStream(result, recorder)
|
|
367
|
+
recorder.finalize(result)
|
|
368
|
+
return result
|
|
369
|
+
|
|
370
|
+
setattr(async_wrapper, _INSTRUMENTED_FLAG, True)
|
|
371
|
+
return async_wrapper
|
|
372
|
+
|
|
373
|
+
@wraps(fn)
|
|
374
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
375
|
+
recorder = _CallRecorder(name, args, kwargs, default_model)
|
|
376
|
+
try:
|
|
377
|
+
result = fn(*args, **kwargs)
|
|
378
|
+
except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
|
|
379
|
+
recorder.fail(exc)
|
|
380
|
+
raise
|
|
381
|
+
if _stream_requested(kwargs) and _is_sync_stream(result):
|
|
382
|
+
return _InstrumentedSyncStream(result, recorder)
|
|
383
|
+
recorder.finalize(result)
|
|
384
|
+
return result
|
|
385
|
+
|
|
386
|
+
setattr(wrapper, _INSTRUMENTED_FLAG, True)
|
|
387
|
+
return wrapper
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def instrument_openai_client(client: Any) -> Any:
|
|
391
|
+
"""Patch common OpenAI SDK call sites in-place.
|
|
392
|
+
|
|
393
|
+
Supported call sites when present:
|
|
394
|
+
- client.responses.create
|
|
395
|
+
- client.chat.completions.create
|
|
396
|
+
|
|
397
|
+
The function returns the same client object for ergonomic usage:
|
|
398
|
+
|
|
399
|
+
```python
|
|
400
|
+
client = instrument_openai_client(OpenAI())
|
|
401
|
+
```
|
|
402
|
+
"""
|
|
403
|
+
|
|
404
|
+
responses = getattr(client, "responses", None)
|
|
405
|
+
if responses is not None and callable(getattr(responses, "create", None)):
|
|
406
|
+
responses.create = instrument_callable(responses.create, name="openai.responses.create")
|
|
407
|
+
|
|
408
|
+
chat = getattr(client, "chat", None)
|
|
409
|
+
completions = getattr(chat, "completions", None) if chat is not None else None
|
|
410
|
+
if completions is not None and callable(getattr(completions, "create", None)):
|
|
411
|
+
completions.create = instrument_callable(
|
|
412
|
+
completions.create, name="openai.chat.completions.create"
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
return client
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from agentloop.otel import trace_from_otel
|
|
9
|
+
from agentloop.otel_ids import to_span_id, to_trace_id
|
|
10
|
+
from agentloop.tracer import AgentTrace
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AgentLoopTracingProcessor:
|
|
14
|
+
"""OpenAI Agents SDK-compatible tracing processor.
|
|
15
|
+
|
|
16
|
+
The OpenAI Agents SDK calls `on_trace_start`, `on_trace_end`, `on_span_start`,
|
|
17
|
+
and `on_span_end` on registered processors. This class avoids importing the SDK
|
|
18
|
+
so AgentLoop can expose the integration without adding a hard dependency.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, out_dir: str | Path | None = None, *, retain_exported: bool = True):
|
|
22
|
+
self.out_dir = Path(out_dir) if out_dir is not None else None
|
|
23
|
+
# Retain exported AgentTrace objects on ``exported_traces``. Long-lived
|
|
24
|
+
# processors that never read that list can set ``retain_exported=False``
|
|
25
|
+
# so completed traces are not held in memory.
|
|
26
|
+
self.retain_exported = retain_exported
|
|
27
|
+
self._traces: dict[str, Any] = {}
|
|
28
|
+
# Trace ids started but not yet ended, in start order. Used to attribute
|
|
29
|
+
# spans that arrive without a readable trace id to a single open trace.
|
|
30
|
+
self._active_trace_ids: list[str] = []
|
|
31
|
+
# Spans are grouped by their owning trace id and released when that trace
|
|
32
|
+
# ends, so memory does not grow with the number of completed traces.
|
|
33
|
+
self._spans_by_trace: dict[str, list[Any]] = {}
|
|
34
|
+
self.exported_traces: list[AgentTrace] = []
|
|
35
|
+
|
|
36
|
+
def on_trace_start(self, trace: Any) -> None:
|
|
37
|
+
trace_id = str(_read(trace, "trace_id", "id") or "trace_unknown")
|
|
38
|
+
self._traces[trace_id] = trace
|
|
39
|
+
if trace_id not in self._active_trace_ids:
|
|
40
|
+
self._active_trace_ids.append(trace_id)
|
|
41
|
+
|
|
42
|
+
def on_trace_end(self, trace: Any) -> None:
|
|
43
|
+
trace_id = str(_read(trace, "trace_id", "id") or "trace_unknown")
|
|
44
|
+
if not self._is_known(trace_id):
|
|
45
|
+
# Unknown trace id: already exported (duplicate on_trace_end) or never
|
|
46
|
+
# observed. Nothing to export and nothing to release.
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
agent_trace = self._build_trace(trace, trace_id)
|
|
51
|
+
if self.retain_exported:
|
|
52
|
+
self.exported_traces.append(agent_trace)
|
|
53
|
+
if self.out_dir is not None:
|
|
54
|
+
self.out_dir.mkdir(parents=True, exist_ok=True)
|
|
55
|
+
agent_trace.export_json(self.out_dir / f"{agent_trace.run_id}.json")
|
|
56
|
+
finally:
|
|
57
|
+
# Release state for the completed trace even if build/export fails, so a
|
|
58
|
+
# failed export cannot re-introduce the completed-trace memory leak.
|
|
59
|
+
self._traces.pop(trace_id, None)
|
|
60
|
+
self._spans_by_trace.pop(trace_id, None)
|
|
61
|
+
if trace_id in self._active_trace_ids:
|
|
62
|
+
self._active_trace_ids.remove(trace_id)
|
|
63
|
+
|
|
64
|
+
def on_span_start(self, span: Any) -> None:
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
def on_span_end(self, span: Any) -> None:
|
|
68
|
+
trace_id = _span_trace_id(span)
|
|
69
|
+
if trace_id is None:
|
|
70
|
+
# No readable trace id: attribute the span to the most recently started
|
|
71
|
+
# open trace. If none is open, drop it rather than copy it into an
|
|
72
|
+
# unrelated trace that ends later.
|
|
73
|
+
trace_id = self._current_active_trace_id()
|
|
74
|
+
if trace_id is None:
|
|
75
|
+
return
|
|
76
|
+
self._spans_by_trace.setdefault(trace_id, []).append(span)
|
|
77
|
+
|
|
78
|
+
def shutdown(self) -> None:
|
|
79
|
+
# Release all retained state; further callbacks start from empty.
|
|
80
|
+
self._traces.clear()
|
|
81
|
+
self._active_trace_ids.clear()
|
|
82
|
+
self._spans_by_trace.clear()
|
|
83
|
+
|
|
84
|
+
def force_flush(self) -> None:
|
|
85
|
+
# Spans are exported when their trace ends; nothing is buffered to flush.
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
def _current_active_trace_id(self) -> str | None:
|
|
89
|
+
return self._active_trace_ids[-1] if self._active_trace_ids else None
|
|
90
|
+
|
|
91
|
+
def _is_known(self, trace_id: str) -> bool:
|
|
92
|
+
return (
|
|
93
|
+
trace_id in self._traces
|
|
94
|
+
or trace_id in self._spans_by_trace
|
|
95
|
+
or trace_id in self._active_trace_ids
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
def _build_trace(self, trace: Any, trace_id: str | None = None) -> AgentTrace:
|
|
99
|
+
if trace_id is None:
|
|
100
|
+
trace_id = str(_read(trace, "trace_id", "id") or "trace_unknown")
|
|
101
|
+
name = str(_read(trace, "name", "workflow_name") or "openai_agents_trace")
|
|
102
|
+
spans = [_span_to_otel(span, trace_id) for span in self._spans_by_trace.get(trace_id, [])]
|
|
103
|
+
return trace_from_otel({"spans": spans}, name=name)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _span_to_otel(span: Any, trace_id: str) -> dict[str, Any]:
|
|
107
|
+
span_data = _read(span, "span_data") or _read(span, "data") or {}
|
|
108
|
+
span_type = _read(span_data, "type") or span_data.__class__.__name__.lower()
|
|
109
|
+
name = str(_read(span_data, "name") or _read(span, "name") or span_type or "openai_agents_span")
|
|
110
|
+
model = _read(span_data, "model")
|
|
111
|
+
usage = _read(span_data, "usage") or {}
|
|
112
|
+
attrs = [
|
|
113
|
+
_attribute("gen_ai.operation.name", _operation_from_span_type(str(span_type))),
|
|
114
|
+
_attribute("agentloop.event_type", _event_type_from_span_type(str(span_type))),
|
|
115
|
+
_attribute("agentloop.name", name),
|
|
116
|
+
]
|
|
117
|
+
if model:
|
|
118
|
+
attrs.append(_attribute("gen_ai.request.model", str(model)))
|
|
119
|
+
input_tokens = (
|
|
120
|
+
_read(usage, "input_tokens", "prompt_tokens")
|
|
121
|
+
or _read(span_data, "input_tokens", "prompt_tokens")
|
|
122
|
+
or 0
|
|
123
|
+
)
|
|
124
|
+
output_tokens = (
|
|
125
|
+
_read(usage, "output_tokens", "completion_tokens")
|
|
126
|
+
or _read(span_data, "output_tokens", "completion_tokens")
|
|
127
|
+
or 0
|
|
128
|
+
)
|
|
129
|
+
attrs.append(_attribute("gen_ai.usage.input_tokens", int(input_tokens or 0)))
|
|
130
|
+
attrs.append(_attribute("gen_ai.usage.output_tokens", int(output_tokens or 0)))
|
|
131
|
+
|
|
132
|
+
native_span_id = str(_read(span, "span_id", "id") or "0")
|
|
133
|
+
# Preserve the original native ids for diagnosability (see agentloop.otel_ids).
|
|
134
|
+
attrs.append(_attribute("agentloop.native_span_id", native_span_id))
|
|
135
|
+
attrs.append(_attribute("agentloop.native_trace_id", str(trace_id)))
|
|
136
|
+
# The SDK span id is this event's native identity. Emitting it under the same
|
|
137
|
+
# attribute the core exporter uses keeps event/parent ids in one id space, so
|
|
138
|
+
# the importer links parents by native id consistently (see agentloop.otel).
|
|
139
|
+
attrs.append(_attribute("agentloop.native_event_id", native_span_id))
|
|
140
|
+
error = _read(span, "error")
|
|
141
|
+
status = {"code": "STATUS_CODE_OK"}
|
|
142
|
+
if error is not None:
|
|
143
|
+
status = {
|
|
144
|
+
"code": "STATUS_CODE_ERROR",
|
|
145
|
+
"message": _error_message(error),
|
|
146
|
+
}
|
|
147
|
+
error_data = _read(error, "data")
|
|
148
|
+
if error_data is not None:
|
|
149
|
+
attrs.append(
|
|
150
|
+
_attribute("openai_agents.error.data", _normalized_error_data_json(error_data))
|
|
151
|
+
)
|
|
152
|
+
out = {
|
|
153
|
+
"traceId": to_trace_id(trace_id),
|
|
154
|
+
"spanId": to_span_id(native_span_id),
|
|
155
|
+
"name": name,
|
|
156
|
+
"startTimeUnixNano": str(_time_ns(_read(span, "started_at", "start_time"))),
|
|
157
|
+
"endTimeUnixNano": str(_time_ns(_read(span, "ended_at", "end_time"))),
|
|
158
|
+
"attributes": attrs,
|
|
159
|
+
"status": status,
|
|
160
|
+
}
|
|
161
|
+
parent_id = _read(span, "parent_id", "parent_span_id")
|
|
162
|
+
if parent_id:
|
|
163
|
+
attrs.append(_attribute("agentloop.native_parent_id", str(parent_id)))
|
|
164
|
+
out["parentSpanId"] = to_span_id(str(parent_id))
|
|
165
|
+
return out
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _read(obj: Any, *names: str) -> Any:
|
|
169
|
+
for name in names:
|
|
170
|
+
if isinstance(obj, dict) and name in obj:
|
|
171
|
+
return obj[name]
|
|
172
|
+
if hasattr(obj, name):
|
|
173
|
+
return getattr(obj, name)
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _span_trace_id(span: Any) -> str | None:
|
|
178
|
+
value = _read(span, "trace_id")
|
|
179
|
+
return str(value) if value else None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _error_message(error: Any) -> str:
|
|
183
|
+
message = _read(error, "message")
|
|
184
|
+
if message is None and isinstance(error, str):
|
|
185
|
+
message = error
|
|
186
|
+
if message is not None:
|
|
187
|
+
normalized = str(message).strip()
|
|
188
|
+
if normalized:
|
|
189
|
+
return normalized
|
|
190
|
+
|
|
191
|
+
data = _read(error, "data")
|
|
192
|
+
error_type = _read(data, "type", "error_type", "name", "code")
|
|
193
|
+
if error_type is not None:
|
|
194
|
+
normalized_type = str(error_type).strip()
|
|
195
|
+
if normalized_type:
|
|
196
|
+
return f"OpenAI Agents span failed ({normalized_type})"
|
|
197
|
+
return "OpenAI Agents span failed"
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _normalized_error_data_json(data: Any) -> str:
|
|
201
|
+
normalized = _normalize_error_data(data, seen=set())
|
|
202
|
+
return json.dumps(
|
|
203
|
+
normalized,
|
|
204
|
+
ensure_ascii=False,
|
|
205
|
+
sort_keys=True,
|
|
206
|
+
separators=(",", ":"),
|
|
207
|
+
allow_nan=False,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _normalize_error_data(value: Any, *, seen: set[int]) -> Any:
|
|
212
|
+
if value is None or isinstance(value, str | bool | int):
|
|
213
|
+
return value
|
|
214
|
+
if isinstance(value, float):
|
|
215
|
+
return value if math.isfinite(value) else "<non-finite float>"
|
|
216
|
+
|
|
217
|
+
if isinstance(value, dict):
|
|
218
|
+
identity = id(value)
|
|
219
|
+
if identity in seen:
|
|
220
|
+
return "<recursive dict>"
|
|
221
|
+
seen.add(identity)
|
|
222
|
+
try:
|
|
223
|
+
normalized: dict[str, Any] = {}
|
|
224
|
+
for key, item in value.items():
|
|
225
|
+
normalized_key = key if isinstance(key, str) else f"<{type(key).__name__} key>"
|
|
226
|
+
normalized[normalized_key] = _normalize_error_data(item, seen=seen)
|
|
227
|
+
return normalized
|
|
228
|
+
finally:
|
|
229
|
+
seen.remove(identity)
|
|
230
|
+
|
|
231
|
+
if isinstance(value, list | tuple):
|
|
232
|
+
identity = id(value)
|
|
233
|
+
if identity in seen:
|
|
234
|
+
return f"<recursive {type(value).__name__}>"
|
|
235
|
+
seen.add(identity)
|
|
236
|
+
try:
|
|
237
|
+
return [_normalize_error_data(item, seen=seen) for item in value]
|
|
238
|
+
finally:
|
|
239
|
+
seen.remove(identity)
|
|
240
|
+
|
|
241
|
+
return f"<{type(value).__name__}>"
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _operation_from_span_type(span_type: str) -> str:
|
|
245
|
+
lower = span_type.lower()
|
|
246
|
+
if "generation" in lower or "response" in lower or "llm" in lower:
|
|
247
|
+
return "chat"
|
|
248
|
+
if "function" in lower or "tool" in lower:
|
|
249
|
+
return "execute_tool"
|
|
250
|
+
return "invoke_agent"
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _event_type_from_span_type(span_type: str) -> str:
|
|
254
|
+
operation = _operation_from_span_type(span_type)
|
|
255
|
+
return "model_call" if operation == "chat" else "tool_call"
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _attribute(key: str, value: Any) -> dict[str, Any]:
|
|
259
|
+
if isinstance(value, bool):
|
|
260
|
+
encoded = {"boolValue": value}
|
|
261
|
+
elif isinstance(value, int):
|
|
262
|
+
encoded = {"intValue": value}
|
|
263
|
+
elif isinstance(value, float):
|
|
264
|
+
encoded = {"doubleValue": value}
|
|
265
|
+
else:
|
|
266
|
+
encoded = {"stringValue": str(value)}
|
|
267
|
+
return {"key": key, "value": encoded}
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _time_ns(value: Any) -> int:
|
|
271
|
+
if value is None:
|
|
272
|
+
return 0
|
|
273
|
+
if isinstance(value, int):
|
|
274
|
+
return value
|
|
275
|
+
if isinstance(value, float):
|
|
276
|
+
return int(value * 1_000_000_000)
|
|
277
|
+
try:
|
|
278
|
+
from datetime import datetime
|
|
279
|
+
|
|
280
|
+
return int(
|
|
281
|
+
datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() * 1_000_000_000
|
|
282
|
+
)
|
|
283
|
+
except ValueError:
|
|
284
|
+
return 0
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def export_trace_json(trace: AgentTrace, path: str | Path) -> Path:
|
|
288
|
+
out = Path(path)
|
|
289
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
290
|
+
out.write_text(json.dumps(trace.to_dict(), indent=2), encoding="utf-8")
|
|
291
|
+
return out
|