agentloop-profiler 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. agentloop/__init__.py +136 -0
  2. agentloop/ablation_protocol.py +229 -0
  3. agentloop/ablations.py +564 -0
  4. agentloop/aggregate_cli.py +88 -0
  5. agentloop/aggregate_stats.py +175 -0
  6. agentloop/aggregates.py +532 -0
  7. agentloop/audit.py +39 -0
  8. agentloop/autoinstrument.py +145 -0
  9. agentloop/batch_experiments.py +295 -0
  10. agentloop/batching.py +95 -0
  11. agentloop/budget_types.py +335 -0
  12. agentloop/budgets.py +539 -0
  13. agentloop/calibration.py +872 -0
  14. agentloop/checkpoint_state.py +306 -0
  15. agentloop/checkpoint_store.py +86 -0
  16. agentloop/ci.py +202 -0
  17. agentloop/cli.py +1314 -0
  18. agentloop/client.py +228 -0
  19. agentloop/completion.py +345 -0
  20. agentloop/completion_types.py +199 -0
  21. agentloop/config.py +50 -0
  22. agentloop/context_controls.py +443 -0
  23. agentloop/context_experiments.py +337 -0
  24. agentloop/context_types.py +216 -0
  25. agentloop/costs.py +667 -0
  26. agentloop/cursor_validation.py +48 -0
  27. agentloop/decorators.py +445 -0
  28. agentloop/demo.py +300 -0
  29. agentloop/doctor.py +341 -0
  30. agentloop/drift.py +361 -0
  31. agentloop/drift_cli.py +147 -0
  32. agentloop/drift_types.py +110 -0
  33. agentloop/drift_windows.py +340 -0
  34. agentloop/entrypoint.py +201 -0
  35. agentloop/estimates.py +133 -0
  36. agentloop/events.py +75 -0
  37. agentloop/experiment_artifacts.py +80 -0
  38. agentloop/experiment_observations.py +169 -0
  39. agentloop/experiment_reports.py +545 -0
  40. agentloop/experiment_types.py +355 -0
  41. agentloop/experiments.py +392 -0
  42. agentloop/exporters.py +117 -0
  43. agentloop/finding_benchmark_types.py +131 -0
  44. agentloop/finding_benchmarks.py +424 -0
  45. agentloop/findings.py +419 -0
  46. agentloop/graph.py +464 -0
  47. agentloop/harness.py +825 -0
  48. agentloop/harness_bench/__init__.py +1 -0
  49. agentloop/harness_bench/cli.py +77 -0
  50. agentloop/harness_bench/drivers.py +96 -0
  51. agentloop/harness_bench/offline.py +507 -0
  52. agentloop/harness_evidence.py +585 -0
  53. agentloop/harness_usage.py +91 -0
  54. agentloop/html_report.py +860 -0
  55. agentloop/integrations/__init__.py +1 -0
  56. agentloop/integrations/crewai.py +144 -0
  57. agentloop/integrations/harbor/__init__.py +1 -0
  58. agentloop/integrations/harbor/atif.py +982 -0
  59. agentloop/integrations/harbor/atif_validation.py +329 -0
  60. agentloop/integrations/harbor/cli.py +207 -0
  61. agentloop/integrations/harbor/manifest.py +86 -0
  62. agentloop/integrations/harbor/trial_evidence.py +196 -0
  63. agentloop/integrations/harbor/trials.py +1109 -0
  64. agentloop/integrations/harbor/verifier.py +118 -0
  65. agentloop/integrations/langgraph.py +150 -0
  66. agentloop/integrations/langgraph_harness.py +466 -0
  67. agentloop/integrations/langgraph_recovery.py +366 -0
  68. agentloop/integrations/omnigent/__init__.py +1 -0
  69. agentloop/integrations/omnigent/cli.py +78 -0
  70. agentloop/integrations/omnigent/evidence.py +93 -0
  71. agentloop/integrations/omnigent/telemetry.py +422 -0
  72. agentloop/integrations/openai.py +415 -0
  73. agentloop/integrations/openai_agents.py +291 -0
  74. agentloop/integrations/sdk.py +82 -0
  75. agentloop/integrations/vercel_ai.py +83 -0
  76. agentloop/interoperability/__init__.py +5 -0
  77. agentloop/interoperability/artifacts.py +112 -0
  78. agentloop/interoperability/cli.py +59 -0
  79. agentloop/interoperability/cohort_evidence.py +175 -0
  80. agentloop/interoperability/cohorts.py +681 -0
  81. agentloop/interoperability/contracts.py +427 -0
  82. agentloop/interoperability/coordination.py +554 -0
  83. agentloop/interoperability/evidence.py +215 -0
  84. agentloop/interoperability/otlp_jsonl.py +775 -0
  85. agentloop/interoperability/otlp_validation.py +496 -0
  86. agentloop/interoperability/registry.py +561 -0
  87. agentloop/interoperability/validation.py +333 -0
  88. agentloop/intervention_service.py +110 -0
  89. agentloop/intervention_store.py +118 -0
  90. agentloop/interventions.py +190 -0
  91. agentloop/issues.py +90 -0
  92. agentloop/judgment_adapters.py +125 -0
  93. agentloop/judgment_benchmark_types.py +98 -0
  94. agentloop/judgment_benchmarks.py +606 -0
  95. agentloop/judgment_types.py +241 -0
  96. agentloop/judgment_views.py +38 -0
  97. agentloop/judgments.py +342 -0
  98. agentloop/loop_guards.py +274 -0
  99. agentloop/loop_types.py +66 -0
  100. agentloop/markdown.py +102 -0
  101. agentloop/metrics.py +342 -0
  102. agentloop/migrations.py +303 -0
  103. agentloop/model_routing.py +470 -0
  104. agentloop/onboarding.py +315 -0
  105. agentloop/operations.py +44 -0
  106. agentloop/optimizer.py +199 -0
  107. agentloop/otel.py +715 -0
  108. agentloop/otel_ids.py +81 -0
  109. agentloop/otel_semantics.py +134 -0
  110. agentloop/parallelism.py +141 -0
  111. agentloop/patches.py +603 -0
  112. agentloop/plan_export.py +126 -0
  113. agentloop/policy_promotion.py +325 -0
  114. agentloop/promotion_evidence.py +304 -0
  115. agentloop/promotion_types.py +209 -0
  116. agentloop/quality.py +446 -0
  117. agentloop/ranking.py +529 -0
  118. agentloop/recovery_evidence.py +96 -0
  119. agentloop/replay.py +753 -0
  120. agentloop/retention.py +375 -0
  121. agentloop/retention_types.py +113 -0
  122. agentloop/reuse_experiments.py +247 -0
  123. agentloop/routing_checks.py +116 -0
  124. agentloop/routing_types.py +241 -0
  125. agentloop/rules.py +396 -0
  126. agentloop/runtime.py +286 -0
  127. agentloop/savings.py +158 -0
  128. agentloop/scheduling_plan.py +99 -0
  129. agentloop/scheduling_types.py +169 -0
  130. agentloop/schema.py +257 -0
  131. agentloop/semantic_rules.py +200 -0
  132. agentloop/semantic_waste.py +345 -0
  133. agentloop/semantic_waste_types.py +88 -0
  134. agentloop/server.py +494 -0
  135. agentloop/store.py +1429 -0
  136. agentloop/structural_experiments.py +431 -0
  137. agentloop/structural_types.py +113 -0
  138. agentloop/structured_quality.py +1020 -0
  139. agentloop/studies.py +629 -0
  140. agentloop/study_statistics.py +58 -0
  141. agentloop/substitution_evidence.py +199 -0
  142. agentloop/substitution_export.py +190 -0
  143. agentloop/substitution_reports.py +407 -0
  144. agentloop/substitution_types.py +284 -0
  145. agentloop/substitutions.py +267 -0
  146. agentloop/timing.py +142 -0
  147. agentloop/tokens.py +188 -0
  148. agentloop/tool_scheduling.py +410 -0
  149. agentloop/trace_sources.py +31 -0
  150. agentloop/tracer.py +494 -0
  151. agentloop/value.py +306 -0
  152. agentloop/version.py +3 -0
  153. agentloop/workflow_types.py +264 -0
  154. agentloop/workflows.py +284 -0
  155. agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
  156. agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
  157. agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
  158. agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
  159. agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
  160. agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
  161. agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
  162. dashboard/__init__.py +5 -0
  163. dashboard/app.py +922 -0
  164. dashboard/streamlit_compat.py +63 -0
  165. dashboard/trace_upload.py +57 -0
  166. dashboard/value_view.py +105 -0
@@ -0,0 +1,415 @@
1
+ from __future__ import annotations
2
+
3
+ import inspect
4
+ import time
5
+ from functools import wraps
6
+ from typing import Any
7
+
8
+ from agentloop.events import format_exception_detail, utc_now_iso
9
+ from agentloop.tokens import PROVIDER, UNAVAILABLE
10
+ from agentloop.tracer import current_event_id, current_trace, record_model_call
11
+
12
+ # Marker set on wrappers so repeated instrumentation of the same callable is a
13
+ # no-op instead of stacking wrappers and double-recording.
14
+ _INSTRUMENTED_FLAG = "__agentloop_instrumented__"
15
+
16
+
17
+ def _usage_get(usage: Any, *names: str) -> int:
18
+ for name in names:
19
+ if usage is None:
20
+ continue
21
+ if isinstance(usage, dict) and name in usage:
22
+ return int(usage.get(name) or 0)
23
+ value = getattr(usage, name, None)
24
+ if value is not None:
25
+ return int(value or 0)
26
+ return 0
27
+
28
+
29
+ def _find_usage(obj: Any) -> Any:
30
+ """Locate a usage object on a response or a streaming event.
31
+
32
+ Non-streaming responses and chat chunks expose ``usage`` directly. The locked
33
+ Responses streaming API instead delivers final usage on the terminal
34
+ ``ResponseCompletedEvent`` under ``event.response.usage`` (the event itself has
35
+ no ``usage``), so fall back to the nested response shape.
36
+ """
37
+
38
+ if obj is None:
39
+ return None
40
+ usage = getattr(obj, "usage", None)
41
+ if usage is None and isinstance(obj, dict):
42
+ usage = obj.get("usage")
43
+ if usage is not None:
44
+ return usage
45
+ response = getattr(obj, "response", None)
46
+ if response is None and isinstance(obj, dict):
47
+ response = obj.get("response")
48
+ if response is None:
49
+ return None
50
+ nested = getattr(response, "usage", None)
51
+ if nested is None and isinstance(response, dict):
52
+ nested = response.get("usage")
53
+ return nested
54
+
55
+
56
+ def _extract_usage(result: Any) -> tuple[int, int, str]:
57
+ """Return token counts and the provenance that justifies them.
58
+
59
+ A response carrying a usage object gives provider-reported counts. Without
60
+ one there is nothing to count, and AgentLoop records ``unavailable`` rather
61
+ than a zero that would later read as a real measurement of zero tokens.
62
+ """
63
+
64
+ usage = _find_usage(result)
65
+ if usage is None:
66
+ return 0, 0, UNAVAILABLE
67
+ input_tokens = _usage_get(usage, "input_tokens", "prompt_tokens")
68
+ output_tokens = _usage_get(usage, "output_tokens", "completion_tokens")
69
+ return input_tokens, output_tokens, PROVIDER
70
+
71
+
72
+ def _metadata_from_call(args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]:
73
+ metadata: dict[str, Any] = {"integration": "openai"}
74
+ if "tools" in kwargs and kwargs["tools"] is not None:
75
+ try:
76
+ metadata["tool_count"] = len(kwargs["tools"])
77
+ except TypeError:
78
+ metadata["tool_count"] = 1
79
+ if "messages" in kwargs and kwargs["messages"] is not None:
80
+ try:
81
+ metadata["message_count"] = len(kwargs["messages"])
82
+ except TypeError:
83
+ metadata["message_count"] = 1
84
+ if args:
85
+ metadata["positional_args"] = len(args)
86
+ return metadata
87
+
88
+
89
+ def _model_from_call(kwargs: dict[str, Any], default_model: str | None = None) -> str | None:
90
+ return (
91
+ str(kwargs.get("model") or default_model)
92
+ if (kwargs.get("model") or default_model)
93
+ else None
94
+ )
95
+
96
+
97
+ class _CallRecorder:
98
+ """Collects timing for one instrumented call and records it exactly once.
99
+
100
+ Recording is deferred so streaming calls can be finalized when the stream is
101
+ consumed rather than when the iterator is created. Trace ownership is captured
102
+ at invocation time: the event is recorded into the trace that was active when
103
+ the call was made, regardless of which context consumes the stream later. If no
104
+ trace was active at invocation the call is never recorded, and the application
105
+ call is not disturbed.
106
+ """
107
+
108
+ def __init__(
109
+ self,
110
+ name: str,
111
+ args: tuple[Any, ...],
112
+ kwargs: dict[str, Any],
113
+ default_model: str | None,
114
+ ) -> None:
115
+ self._name = name
116
+ self._args = args
117
+ self._kwargs = kwargs
118
+ self._default_model = default_model
119
+ self._started_at = utc_now_iso()
120
+ self._start = time.perf_counter()
121
+ self._done = False
122
+ # Capture ownership now, at invocation time, not at finalization time.
123
+ self._trace = current_trace()
124
+ self._parent_id = current_event_id()
125
+
126
+ def finalize(self, result: Any = None, *, status: str = "ok", error: str | None = None) -> None:
127
+ if self._done:
128
+ return
129
+ self._done = True
130
+ if self._trace is None:
131
+ # No trace was active when the call was invoked: never record, and do
132
+ # not turn the caller's successful application call into an error.
133
+ return
134
+ input_tokens, output_tokens, token_provenance = _extract_usage(result)
135
+ record_model_call(
136
+ self._name,
137
+ started_at=self._started_at,
138
+ duration_ms=(time.perf_counter() - self._start) * 1000,
139
+ model=_model_from_call(self._kwargs, self._default_model),
140
+ input_tokens=input_tokens,
141
+ output_tokens=output_tokens,
142
+ token_provenance=token_provenance,
143
+ status=status,
144
+ error=error,
145
+ metadata=_metadata_from_call(self._args, self._kwargs),
146
+ parent_id=self._parent_id,
147
+ trace=self._trace,
148
+ )
149
+
150
+ def fail(self, exc: BaseException, result: Any = None) -> None:
151
+ """Record one failed outcome while preserving the caller's exception."""
152
+
153
+ self.finalize(result, status="error", error=format_exception_detail(exc))
154
+
155
+
156
+ def _has_usage(obj: Any) -> bool:
157
+ return _find_usage(obj) is not None
158
+
159
+
160
+ def _stream_requested(kwargs: dict[str, Any]) -> bool:
161
+ return bool(kwargs.get("stream"))
162
+
163
+
164
+ def _is_sync_stream(result: Any) -> bool:
165
+ if isinstance(result, (dict, list, tuple, set, str, bytes, bytearray)):
166
+ return False
167
+ return (
168
+ hasattr(result, "__iter__") or hasattr(result, "__next__") or hasattr(result, "__enter__")
169
+ )
170
+
171
+
172
+ def _is_async_stream(result: Any) -> bool:
173
+ return (
174
+ hasattr(result, "__aiter__")
175
+ or hasattr(result, "__anext__")
176
+ or hasattr(result, "__aenter__")
177
+ or inspect.isasyncgen(result)
178
+ )
179
+
180
+
181
+ class _InstrumentedSyncStream:
182
+ """Proxy over a sync streaming response that finalizes on consumption.
183
+
184
+ Iteration, context-manager use, and ``close()`` all end the stream; whichever
185
+ happens first records the call (once). Every other attribute is delegated to
186
+ the wrapped stream so SDK behavior is preserved.
187
+ """
188
+
189
+ def __init__(self, stream: Any, recorder: _CallRecorder) -> None:
190
+ self._stream = stream
191
+ self._recorder = recorder
192
+ self._iter: Any = None
193
+ self._usage_source: Any = None
194
+
195
+ def __iter__(self) -> _InstrumentedSyncStream:
196
+ try:
197
+ self._iter = iter(self._stream)
198
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
199
+ self._recorder.fail(exc, self._usage_source)
200
+ raise
201
+ return self
202
+
203
+ def __next__(self) -> Any:
204
+ try:
205
+ if self._iter is None:
206
+ self._iter = iter(self._stream)
207
+ chunk = next(self._iter)
208
+ except StopIteration:
209
+ self._recorder.finalize(self._usage_source)
210
+ raise
211
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
212
+ self._recorder.fail(exc, self._usage_source)
213
+ raise
214
+ if _has_usage(chunk):
215
+ self._usage_source = chunk
216
+ return chunk
217
+
218
+ def __enter__(self) -> _InstrumentedSyncStream:
219
+ try:
220
+ if hasattr(self._stream, "__enter__"):
221
+ self._stream.__enter__()
222
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
223
+ self._recorder.fail(exc, self._usage_source)
224
+ raise
225
+ return self
226
+
227
+ def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> Any:
228
+ try:
229
+ suppress = (
230
+ self._stream.__exit__(exc_type, exc, tb)
231
+ if hasattr(self._stream, "__exit__")
232
+ else None
233
+ )
234
+ except BaseException as exit_exc: # noqa: BLE001 - record then propagate unchanged
235
+ self._recorder.fail(exit_exc, self._usage_source)
236
+ raise
237
+ if exc is not None and not suppress:
238
+ self._recorder.fail(exc, self._usage_source)
239
+ else:
240
+ self._recorder.finalize(self._usage_source)
241
+ return suppress
242
+
243
+ def close(self) -> Any:
244
+ try:
245
+ result = self._stream.close() if hasattr(self._stream, "close") else None
246
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
247
+ self._recorder.fail(exc, self._usage_source)
248
+ raise
249
+ self._recorder.finalize(self._usage_source)
250
+ return result
251
+
252
+ def __getattr__(self, item: str) -> Any:
253
+ return getattr(self._stream, item)
254
+
255
+
256
+ class _InstrumentedAsyncStream:
257
+ """Async counterpart of :class:`_InstrumentedSyncStream`."""
258
+
259
+ def __init__(self, stream: Any, recorder: _CallRecorder) -> None:
260
+ self._stream = stream
261
+ self._recorder = recorder
262
+ self._iter: Any = None
263
+ self._usage_source: Any = None
264
+
265
+ def __aiter__(self) -> _InstrumentedAsyncStream:
266
+ try:
267
+ self._iter = (
268
+ self._stream.__aiter__() if hasattr(self._stream, "__aiter__") else self._stream
269
+ )
270
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
271
+ self._recorder.fail(exc, self._usage_source)
272
+ raise
273
+ return self
274
+
275
+ async def __anext__(self) -> Any:
276
+ try:
277
+ if self._iter is None:
278
+ self._iter = (
279
+ self._stream.__aiter__() if hasattr(self._stream, "__aiter__") else self._stream
280
+ )
281
+ chunk = await self._iter.__anext__()
282
+ except StopAsyncIteration:
283
+ self._recorder.finalize(self._usage_source)
284
+ raise
285
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
286
+ self._recorder.fail(exc, self._usage_source)
287
+ raise
288
+ if _has_usage(chunk):
289
+ self._usage_source = chunk
290
+ return chunk
291
+
292
+ async def __aenter__(self) -> _InstrumentedAsyncStream:
293
+ try:
294
+ if hasattr(self._stream, "__aenter__"):
295
+ await self._stream.__aenter__()
296
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
297
+ self._recorder.fail(exc, self._usage_source)
298
+ raise
299
+ return self
300
+
301
+ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> Any:
302
+ try:
303
+ suppress = (
304
+ await self._stream.__aexit__(exc_type, exc, tb)
305
+ if hasattr(self._stream, "__aexit__")
306
+ else None
307
+ )
308
+ except BaseException as exit_exc: # noqa: BLE001 - record then propagate unchanged
309
+ self._recorder.fail(exit_exc, self._usage_source)
310
+ raise
311
+ if exc is not None and not suppress:
312
+ self._recorder.fail(exc, self._usage_source)
313
+ else:
314
+ self._recorder.finalize(self._usage_source)
315
+ return suppress
316
+
317
+ async def aclose(self) -> Any:
318
+ try:
319
+ result = await self._stream.aclose() if hasattr(self._stream, "aclose") else None
320
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
321
+ self._recorder.fail(exc, self._usage_source)
322
+ raise
323
+ self._recorder.finalize(self._usage_source)
324
+ return result
325
+
326
+ def __getattr__(self, item: str) -> Any:
327
+ return getattr(self._stream, item)
328
+
329
+
330
+ def instrument_callable(
331
+ fn: Any,
332
+ *,
333
+ name: str = "openai.model_call",
334
+ default_model: str | None = None,
335
+ ) -> Any:
336
+ """Wrap a sync or async OpenAI-like callable and record token usage.
337
+
338
+ The wrapper is dependency-free and works with OpenAI SDK resource methods such
339
+ as `client.responses.create(...)` or `client.chat.completions.create(...)`.
340
+ It records prompt/completion usage when the returned object exposes a
341
+ `.usage` object or a `{"usage": ...}` dict.
342
+
343
+ Wrapping is idempotent: re-instrumenting an already-instrumented callable
344
+ returns it unchanged, so one request records one event. Streaming responses
345
+ (requested with `stream=True`) are finalized when the stream is consumed,
346
+ closed, fails, or is cancelled — not when the iterator is created — so the
347
+ recorded duration covers consumption and final usage is captured when the SDK
348
+ provides it. Calls made without an active trace are not recorded and are not
349
+ turned into errors.
350
+ """
351
+
352
+ if getattr(fn, _INSTRUMENTED_FLAG, False):
353
+ return fn
354
+
355
+ if inspect.iscoroutinefunction(fn):
356
+
357
+ @wraps(fn)
358
+ async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
359
+ recorder = _CallRecorder(name, args, kwargs, default_model)
360
+ try:
361
+ result = await fn(*args, **kwargs)
362
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
363
+ recorder.fail(exc)
364
+ raise
365
+ if _stream_requested(kwargs) and _is_async_stream(result):
366
+ return _InstrumentedAsyncStream(result, recorder)
367
+ recorder.finalize(result)
368
+ return result
369
+
370
+ setattr(async_wrapper, _INSTRUMENTED_FLAG, True)
371
+ return async_wrapper
372
+
373
+ @wraps(fn)
374
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
375
+ recorder = _CallRecorder(name, args, kwargs, default_model)
376
+ try:
377
+ result = fn(*args, **kwargs)
378
+ except BaseException as exc: # noqa: BLE001 - record then propagate unchanged
379
+ recorder.fail(exc)
380
+ raise
381
+ if _stream_requested(kwargs) and _is_sync_stream(result):
382
+ return _InstrumentedSyncStream(result, recorder)
383
+ recorder.finalize(result)
384
+ return result
385
+
386
+ setattr(wrapper, _INSTRUMENTED_FLAG, True)
387
+ return wrapper
388
+
389
+
390
+ def instrument_openai_client(client: Any) -> Any:
391
+ """Patch common OpenAI SDK call sites in-place.
392
+
393
+ Supported call sites when present:
394
+ - client.responses.create
395
+ - client.chat.completions.create
396
+
397
+ The function returns the same client object for ergonomic usage:
398
+
399
+ ```python
400
+ client = instrument_openai_client(OpenAI())
401
+ ```
402
+ """
403
+
404
+ responses = getattr(client, "responses", None)
405
+ if responses is not None and callable(getattr(responses, "create", None)):
406
+ responses.create = instrument_callable(responses.create, name="openai.responses.create")
407
+
408
+ chat = getattr(client, "chat", None)
409
+ completions = getattr(chat, "completions", None) if chat is not None else None
410
+ if completions is not None and callable(getattr(completions, "create", None)):
411
+ completions.create = instrument_callable(
412
+ completions.create, name="openai.chat.completions.create"
413
+ )
414
+
415
+ return client
@@ -0,0 +1,291 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import math
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from agentloop.otel import trace_from_otel
9
+ from agentloop.otel_ids import to_span_id, to_trace_id
10
+ from agentloop.tracer import AgentTrace
11
+
12
+
13
+ class AgentLoopTracingProcessor:
14
+ """OpenAI Agents SDK-compatible tracing processor.
15
+
16
+ The OpenAI Agents SDK calls `on_trace_start`, `on_trace_end`, `on_span_start`,
17
+ and `on_span_end` on registered processors. This class avoids importing the SDK
18
+ so AgentLoop can expose the integration without adding a hard dependency.
19
+ """
20
+
21
+ def __init__(self, out_dir: str | Path | None = None, *, retain_exported: bool = True):
22
+ self.out_dir = Path(out_dir) if out_dir is not None else None
23
+ # Retain exported AgentTrace objects on ``exported_traces``. Long-lived
24
+ # processors that never read that list can set ``retain_exported=False``
25
+ # so completed traces are not held in memory.
26
+ self.retain_exported = retain_exported
27
+ self._traces: dict[str, Any] = {}
28
+ # Trace ids started but not yet ended, in start order. Used to attribute
29
+ # spans that arrive without a readable trace id to a single open trace.
30
+ self._active_trace_ids: list[str] = []
31
+ # Spans are grouped by their owning trace id and released when that trace
32
+ # ends, so memory does not grow with the number of completed traces.
33
+ self._spans_by_trace: dict[str, list[Any]] = {}
34
+ self.exported_traces: list[AgentTrace] = []
35
+
36
+ def on_trace_start(self, trace: Any) -> None:
37
+ trace_id = str(_read(trace, "trace_id", "id") or "trace_unknown")
38
+ self._traces[trace_id] = trace
39
+ if trace_id not in self._active_trace_ids:
40
+ self._active_trace_ids.append(trace_id)
41
+
42
+ def on_trace_end(self, trace: Any) -> None:
43
+ trace_id = str(_read(trace, "trace_id", "id") or "trace_unknown")
44
+ if not self._is_known(trace_id):
45
+ # Unknown trace id: already exported (duplicate on_trace_end) or never
46
+ # observed. Nothing to export and nothing to release.
47
+ return
48
+
49
+ try:
50
+ agent_trace = self._build_trace(trace, trace_id)
51
+ if self.retain_exported:
52
+ self.exported_traces.append(agent_trace)
53
+ if self.out_dir is not None:
54
+ self.out_dir.mkdir(parents=True, exist_ok=True)
55
+ agent_trace.export_json(self.out_dir / f"{agent_trace.run_id}.json")
56
+ finally:
57
+ # Release state for the completed trace even if build/export fails, so a
58
+ # failed export cannot re-introduce the completed-trace memory leak.
59
+ self._traces.pop(trace_id, None)
60
+ self._spans_by_trace.pop(trace_id, None)
61
+ if trace_id in self._active_trace_ids:
62
+ self._active_trace_ids.remove(trace_id)
63
+
64
+ def on_span_start(self, span: Any) -> None:
65
+ return None
66
+
67
+ def on_span_end(self, span: Any) -> None:
68
+ trace_id = _span_trace_id(span)
69
+ if trace_id is None:
70
+ # No readable trace id: attribute the span to the most recently started
71
+ # open trace. If none is open, drop it rather than copy it into an
72
+ # unrelated trace that ends later.
73
+ trace_id = self._current_active_trace_id()
74
+ if trace_id is None:
75
+ return
76
+ self._spans_by_trace.setdefault(trace_id, []).append(span)
77
+
78
+ def shutdown(self) -> None:
79
+ # Release all retained state; further callbacks start from empty.
80
+ self._traces.clear()
81
+ self._active_trace_ids.clear()
82
+ self._spans_by_trace.clear()
83
+
84
+ def force_flush(self) -> None:
85
+ # Spans are exported when their trace ends; nothing is buffered to flush.
86
+ return None
87
+
88
+ def _current_active_trace_id(self) -> str | None:
89
+ return self._active_trace_ids[-1] if self._active_trace_ids else None
90
+
91
+ def _is_known(self, trace_id: str) -> bool:
92
+ return (
93
+ trace_id in self._traces
94
+ or trace_id in self._spans_by_trace
95
+ or trace_id in self._active_trace_ids
96
+ )
97
+
98
+ def _build_trace(self, trace: Any, trace_id: str | None = None) -> AgentTrace:
99
+ if trace_id is None:
100
+ trace_id = str(_read(trace, "trace_id", "id") or "trace_unknown")
101
+ name = str(_read(trace, "name", "workflow_name") or "openai_agents_trace")
102
+ spans = [_span_to_otel(span, trace_id) for span in self._spans_by_trace.get(trace_id, [])]
103
+ return trace_from_otel({"spans": spans}, name=name)
104
+
105
+
106
+ def _span_to_otel(span: Any, trace_id: str) -> dict[str, Any]:
107
+ span_data = _read(span, "span_data") or _read(span, "data") or {}
108
+ span_type = _read(span_data, "type") or span_data.__class__.__name__.lower()
109
+ name = str(_read(span_data, "name") or _read(span, "name") or span_type or "openai_agents_span")
110
+ model = _read(span_data, "model")
111
+ usage = _read(span_data, "usage") or {}
112
+ attrs = [
113
+ _attribute("gen_ai.operation.name", _operation_from_span_type(str(span_type))),
114
+ _attribute("agentloop.event_type", _event_type_from_span_type(str(span_type))),
115
+ _attribute("agentloop.name", name),
116
+ ]
117
+ if model:
118
+ attrs.append(_attribute("gen_ai.request.model", str(model)))
119
+ input_tokens = (
120
+ _read(usage, "input_tokens", "prompt_tokens")
121
+ or _read(span_data, "input_tokens", "prompt_tokens")
122
+ or 0
123
+ )
124
+ output_tokens = (
125
+ _read(usage, "output_tokens", "completion_tokens")
126
+ or _read(span_data, "output_tokens", "completion_tokens")
127
+ or 0
128
+ )
129
+ attrs.append(_attribute("gen_ai.usage.input_tokens", int(input_tokens or 0)))
130
+ attrs.append(_attribute("gen_ai.usage.output_tokens", int(output_tokens or 0)))
131
+
132
+ native_span_id = str(_read(span, "span_id", "id") or "0")
133
+ # Preserve the original native ids for diagnosability (see agentloop.otel_ids).
134
+ attrs.append(_attribute("agentloop.native_span_id", native_span_id))
135
+ attrs.append(_attribute("agentloop.native_trace_id", str(trace_id)))
136
+ # The SDK span id is this event's native identity. Emitting it under the same
137
+ # attribute the core exporter uses keeps event/parent ids in one id space, so
138
+ # the importer links parents by native id consistently (see agentloop.otel).
139
+ attrs.append(_attribute("agentloop.native_event_id", native_span_id))
140
+ error = _read(span, "error")
141
+ status = {"code": "STATUS_CODE_OK"}
142
+ if error is not None:
143
+ status = {
144
+ "code": "STATUS_CODE_ERROR",
145
+ "message": _error_message(error),
146
+ }
147
+ error_data = _read(error, "data")
148
+ if error_data is not None:
149
+ attrs.append(
150
+ _attribute("openai_agents.error.data", _normalized_error_data_json(error_data))
151
+ )
152
+ out = {
153
+ "traceId": to_trace_id(trace_id),
154
+ "spanId": to_span_id(native_span_id),
155
+ "name": name,
156
+ "startTimeUnixNano": str(_time_ns(_read(span, "started_at", "start_time"))),
157
+ "endTimeUnixNano": str(_time_ns(_read(span, "ended_at", "end_time"))),
158
+ "attributes": attrs,
159
+ "status": status,
160
+ }
161
+ parent_id = _read(span, "parent_id", "parent_span_id")
162
+ if parent_id:
163
+ attrs.append(_attribute("agentloop.native_parent_id", str(parent_id)))
164
+ out["parentSpanId"] = to_span_id(str(parent_id))
165
+ return out
166
+
167
+
168
+ def _read(obj: Any, *names: str) -> Any:
169
+ for name in names:
170
+ if isinstance(obj, dict) and name in obj:
171
+ return obj[name]
172
+ if hasattr(obj, name):
173
+ return getattr(obj, name)
174
+ return None
175
+
176
+
177
+ def _span_trace_id(span: Any) -> str | None:
178
+ value = _read(span, "trace_id")
179
+ return str(value) if value else None
180
+
181
+
182
+ def _error_message(error: Any) -> str:
183
+ message = _read(error, "message")
184
+ if message is None and isinstance(error, str):
185
+ message = error
186
+ if message is not None:
187
+ normalized = str(message).strip()
188
+ if normalized:
189
+ return normalized
190
+
191
+ data = _read(error, "data")
192
+ error_type = _read(data, "type", "error_type", "name", "code")
193
+ if error_type is not None:
194
+ normalized_type = str(error_type).strip()
195
+ if normalized_type:
196
+ return f"OpenAI Agents span failed ({normalized_type})"
197
+ return "OpenAI Agents span failed"
198
+
199
+
200
+ def _normalized_error_data_json(data: Any) -> str:
201
+ normalized = _normalize_error_data(data, seen=set())
202
+ return json.dumps(
203
+ normalized,
204
+ ensure_ascii=False,
205
+ sort_keys=True,
206
+ separators=(",", ":"),
207
+ allow_nan=False,
208
+ )
209
+
210
+
211
+ def _normalize_error_data(value: Any, *, seen: set[int]) -> Any:
212
+ if value is None or isinstance(value, str | bool | int):
213
+ return value
214
+ if isinstance(value, float):
215
+ return value if math.isfinite(value) else "<non-finite float>"
216
+
217
+ if isinstance(value, dict):
218
+ identity = id(value)
219
+ if identity in seen:
220
+ return "<recursive dict>"
221
+ seen.add(identity)
222
+ try:
223
+ normalized: dict[str, Any] = {}
224
+ for key, item in value.items():
225
+ normalized_key = key if isinstance(key, str) else f"<{type(key).__name__} key>"
226
+ normalized[normalized_key] = _normalize_error_data(item, seen=seen)
227
+ return normalized
228
+ finally:
229
+ seen.remove(identity)
230
+
231
+ if isinstance(value, list | tuple):
232
+ identity = id(value)
233
+ if identity in seen:
234
+ return f"<recursive {type(value).__name__}>"
235
+ seen.add(identity)
236
+ try:
237
+ return [_normalize_error_data(item, seen=seen) for item in value]
238
+ finally:
239
+ seen.remove(identity)
240
+
241
+ return f"<{type(value).__name__}>"
242
+
243
+
244
+ def _operation_from_span_type(span_type: str) -> str:
245
+ lower = span_type.lower()
246
+ if "generation" in lower or "response" in lower or "llm" in lower:
247
+ return "chat"
248
+ if "function" in lower or "tool" in lower:
249
+ return "execute_tool"
250
+ return "invoke_agent"
251
+
252
+
253
+ def _event_type_from_span_type(span_type: str) -> str:
254
+ operation = _operation_from_span_type(span_type)
255
+ return "model_call" if operation == "chat" else "tool_call"
256
+
257
+
258
+ def _attribute(key: str, value: Any) -> dict[str, Any]:
259
+ if isinstance(value, bool):
260
+ encoded = {"boolValue": value}
261
+ elif isinstance(value, int):
262
+ encoded = {"intValue": value}
263
+ elif isinstance(value, float):
264
+ encoded = {"doubleValue": value}
265
+ else:
266
+ encoded = {"stringValue": str(value)}
267
+ return {"key": key, "value": encoded}
268
+
269
+
270
+ def _time_ns(value: Any) -> int:
271
+ if value is None:
272
+ return 0
273
+ if isinstance(value, int):
274
+ return value
275
+ if isinstance(value, float):
276
+ return int(value * 1_000_000_000)
277
+ try:
278
+ from datetime import datetime
279
+
280
+ return int(
281
+ datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() * 1_000_000_000
282
+ )
283
+ except ValueError:
284
+ return 0
285
+
286
+
287
+ def export_trace_json(trace: AgentTrace, path: str | Path) -> Path:
288
+ out = Path(path)
289
+ out.parent.mkdir(parents=True, exist_ok=True)
290
+ out.write_text(json.dumps(trace.to_dict(), indent=2), encoding="utf-8")
291
+ return out