ioa-observe-sdk 1.0.25__py3-none-any.whl → 1.0.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ioa_observe/sdk/connectors/slim.py +16 -17
- ioa_observe/sdk/decorators/base.py +12 -12
- ioa_observe/sdk/instrumentations/a2a.py +142 -4
- ioa_observe/sdk/instrumentations/mcp.py +5 -2
- ioa_observe/sdk/instrumentations/nats.py +9 -4
- ioa_observe/sdk/instrumentations/slim.py +35 -8
- ioa_observe/sdk/metrics/metrics.py +0 -22
- ioa_observe/sdk/tracing/tracing.py +97 -34
- {ioa_observe_sdk-1.0.25.dist-info → ioa_observe_sdk-1.0.27.dist-info}/METADATA +1 -1
- {ioa_observe_sdk-1.0.25.dist-info → ioa_observe_sdk-1.0.27.dist-info}/RECORD +13 -13
- {ioa_observe_sdk-1.0.25.dist-info → ioa_observe_sdk-1.0.27.dist-info}/WHEEL +1 -1
- {ioa_observe_sdk-1.0.25.dist-info → ioa_observe_sdk-1.0.27.dist-info}/licenses/LICENSE.md +0 -0
- {ioa_observe_sdk-1.0.25.dist-info → ioa_observe_sdk-1.0.27.dist-info}/top_level.txt +0 -0
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
|
|
4
4
|
import inspect
|
|
5
5
|
import string
|
|
6
|
-
import time
|
|
7
6
|
from functools import wraps
|
|
8
7
|
from json import JSONEncoder
|
|
9
8
|
from typing import Optional
|
|
@@ -82,23 +81,23 @@ def process_slim_msg(name: Optional[str] = None):
|
|
|
82
81
|
span, ctx, ctx_token = _setup_span(entity_name)
|
|
83
82
|
_handle_span_input(span, args, kwargs, cls=JSONEncoder)
|
|
84
83
|
|
|
85
|
-
start_time = time.time()
|
|
84
|
+
# start_time = time.time()
|
|
86
85
|
|
|
87
86
|
try:
|
|
88
87
|
async for item in _ahandle_generator(
|
|
89
88
|
span, ctx_token, fn(*args, **kwargs)
|
|
90
89
|
):
|
|
91
90
|
# Measure throughput and processing time per item
|
|
92
|
-
item_process_time = time.time() - start_time
|
|
93
|
-
TracerWrapper().processing_time.record(
|
|
94
|
-
|
|
95
|
-
)
|
|
91
|
+
# item_process_time = time.time() - start_time
|
|
92
|
+
# TracerWrapper().processing_time.record(
|
|
93
|
+
# item_process_time, {"agent": entity_name}
|
|
94
|
+
# )
|
|
96
95
|
TracerWrapper().throughput_counter.add(
|
|
97
96
|
1, {"agent": entity_name}
|
|
98
97
|
)
|
|
99
98
|
|
|
100
99
|
# Reset timer for next item
|
|
101
|
-
start_time = time.time()
|
|
100
|
+
# start_time = time.time()
|
|
102
101
|
# Count each yielded item as a published message
|
|
103
102
|
TracerWrapper().messages_received_counter.add(
|
|
104
103
|
1, {"agent": entity_name}
|
|
@@ -122,16 +121,16 @@ def process_slim_msg(name: Optional[str] = None):
|
|
|
122
121
|
|
|
123
122
|
# span, ctx, ctx_token = _setup_span(entity_name)
|
|
124
123
|
# _handle_span_input(span, args, kwargs, cls=JSONEncoder)
|
|
125
|
-
start_time = time.time()
|
|
124
|
+
# start_time = time.time()
|
|
126
125
|
|
|
127
126
|
try:
|
|
128
127
|
res = await fn(*args, **kwargs)
|
|
129
128
|
|
|
130
129
|
# Measure processing time
|
|
131
|
-
process_time = time.time() - start_time
|
|
132
|
-
TracerWrapper().processing_time.record(
|
|
133
|
-
|
|
134
|
-
)
|
|
130
|
+
# process_time = time.time() - start_time
|
|
131
|
+
# TracerWrapper().processing_time.record(
|
|
132
|
+
# process_time, {"agent": entity_name}
|
|
133
|
+
# )
|
|
135
134
|
TracerWrapper().throughput_counter.add(
|
|
136
135
|
1, {"agent": entity_name}
|
|
137
136
|
)
|
|
@@ -172,16 +171,16 @@ def process_slim_msg(name: Optional[str] = None):
|
|
|
172
171
|
# span, ctx, ctx_token = _setup_span(entity_name)
|
|
173
172
|
# _handle_span_input(span, args, kwargs, cls=JSONEncoder)
|
|
174
173
|
|
|
175
|
-
start_time = time.time()
|
|
174
|
+
# start_time = time.time()
|
|
176
175
|
|
|
177
176
|
try:
|
|
178
177
|
res = fn(*args, **kwargs)
|
|
179
178
|
|
|
180
179
|
# Measure processing time
|
|
181
|
-
process_time = time.time() - start_time
|
|
182
|
-
TracerWrapper().processing_time.record(
|
|
183
|
-
|
|
184
|
-
)
|
|
180
|
+
# process_time = time.time() - start_time
|
|
181
|
+
# TracerWrapper().processing_time.record(
|
|
182
|
+
# process_time, {"agent": entity_name}
|
|
183
|
+
# )
|
|
185
184
|
TracerWrapper().throughput_counter.add(1, {"agent": entity_name})
|
|
186
185
|
# span will be ended in the generator
|
|
187
186
|
# if isinstance(res, types.GeneratorType):
|
|
@@ -312,18 +312,18 @@ def _cleanup_span(span, ctx_token):
|
|
|
312
312
|
"""End the span process and detach the context token"""
|
|
313
313
|
|
|
314
314
|
# Calculate agent chain completion time before ending span
|
|
315
|
-
span_kind = span.attributes.get(OBSERVE_SPAN_KIND)
|
|
316
|
-
if span_kind == ObserveSpanKindValues.AGENT.value:
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
315
|
+
# span_kind = span.attributes.get(OBSERVE_SPAN_KIND)
|
|
316
|
+
# if span_kind == ObserveSpanKindValues.AGENT.value:
|
|
317
|
+
# start_time = span.attributes.get("agent_chain_start_time")
|
|
318
|
+
# if start_time is not None:
|
|
319
|
+
# import time
|
|
320
|
+
#
|
|
321
|
+
# # completion_time = time.time() - start_time
|
|
322
|
+
#
|
|
323
|
+
# # Emit the metric
|
|
324
|
+
# # TracerWrapper().agent_chain_completion_time_histogram.record(
|
|
325
|
+
# # completion_time, attributes=span.attributes
|
|
326
|
+
# # )
|
|
327
327
|
span.end()
|
|
328
328
|
context_api.detach(ctx_token)
|
|
329
329
|
|
|
@@ -5,6 +5,7 @@ from typing import Collection
|
|
|
5
5
|
import functools
|
|
6
6
|
import threading
|
|
7
7
|
|
|
8
|
+
from opentelemetry.context import get_value
|
|
8
9
|
from opentelemetry import baggage
|
|
9
10
|
from opentelemetry.baggage.propagation import W3CBaggagePropagator
|
|
10
11
|
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
|
|
@@ -47,8 +48,10 @@ class A2AInstrumentor(BaseInstrumentor):
|
|
|
47
48
|
session_id = None
|
|
48
49
|
if traceparent:
|
|
49
50
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
50
|
-
if session_id:
|
|
51
|
-
|
|
51
|
+
if not session_id:
|
|
52
|
+
session_id = get_value("session.id")
|
|
53
|
+
if session_id:
|
|
54
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
52
55
|
|
|
53
56
|
# Ensure metadata dict exists
|
|
54
57
|
try:
|
|
@@ -101,8 +104,10 @@ class A2AInstrumentor(BaseInstrumentor):
|
|
|
101
104
|
session_id = None
|
|
102
105
|
if traceparent:
|
|
103
106
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
104
|
-
if session_id:
|
|
105
|
-
|
|
107
|
+
if not session_id:
|
|
108
|
+
session_id = get_value("session.id")
|
|
109
|
+
if session_id:
|
|
110
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
106
111
|
|
|
107
112
|
# Ensure metadata dict exists
|
|
108
113
|
try:
|
|
@@ -199,6 +204,122 @@ class A2AInstrumentor(BaseInstrumentor):
|
|
|
199
204
|
|
|
200
205
|
# original_server_on_message_send = DefaultRequestHandler.on_message_send
|
|
201
206
|
|
|
207
|
+
# Instrumentation for slima2a
|
|
208
|
+
if importlib.util.find_spec("slima2a"):
|
|
209
|
+
from slima2a.client_transport import SRPCTransport
|
|
210
|
+
|
|
211
|
+
# send_message
|
|
212
|
+
original_srpc_transport_send_message = SRPCTransport.send_message
|
|
213
|
+
|
|
214
|
+
@functools.wraps(original_srpc_transport_send_message)
|
|
215
|
+
async def instrumented_srpc_transport_send_message(
|
|
216
|
+
self, request, *args, **kwargs
|
|
217
|
+
):
|
|
218
|
+
# Put context into A2A message metadata instead of HTTP headers
|
|
219
|
+
with _global_tracer.start_as_current_span("slima2a.send_message"):
|
|
220
|
+
traceparent = get_current_traceparent()
|
|
221
|
+
session_id = None
|
|
222
|
+
if traceparent:
|
|
223
|
+
session_id = kv_store.get(f"execution.{traceparent}")
|
|
224
|
+
if not session_id:
|
|
225
|
+
session_id = get_value("session.id")
|
|
226
|
+
if session_id:
|
|
227
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
228
|
+
|
|
229
|
+
# Ensure metadata dict exists
|
|
230
|
+
try:
|
|
231
|
+
md = getattr(request.params, "metadata", None)
|
|
232
|
+
except AttributeError:
|
|
233
|
+
md = None
|
|
234
|
+
metadata = md if isinstance(md, dict) else {}
|
|
235
|
+
|
|
236
|
+
observe_meta = dict(metadata.get("observe", {}))
|
|
237
|
+
|
|
238
|
+
# Inject W3C trace context + baggage into observe_meta
|
|
239
|
+
TraceContextTextMapPropagator().inject(carrier=observe_meta)
|
|
240
|
+
W3CBaggagePropagator().inject(carrier=observe_meta)
|
|
241
|
+
|
|
242
|
+
if traceparent:
|
|
243
|
+
observe_meta["traceparent"] = traceparent
|
|
244
|
+
if session_id:
|
|
245
|
+
observe_meta["session_id"] = session_id
|
|
246
|
+
baggage.set_baggage(f"execution.{traceparent}", session_id)
|
|
247
|
+
|
|
248
|
+
metadata["observe"] = observe_meta
|
|
249
|
+
|
|
250
|
+
# Write back metadata (pydantic models are mutable by default in v2)
|
|
251
|
+
try:
|
|
252
|
+
request.metadata = metadata
|
|
253
|
+
except Exception:
|
|
254
|
+
# Fallback
|
|
255
|
+
request = request.model_copy(update={"metadata": metadata})
|
|
256
|
+
|
|
257
|
+
# Call through without transport-specific kwargs
|
|
258
|
+
return await original_srpc_transport_send_message(
|
|
259
|
+
self, request, *args, **kwargs
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
SRPCTransport.send_message = instrumented_srpc_transport_send_message
|
|
263
|
+
|
|
264
|
+
# send_message_streaming
|
|
265
|
+
original_srpc_transport_send_message_streaming = (
|
|
266
|
+
SRPCTransport.send_message_streaming
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
@functools.wraps(original_srpc_transport_send_message_streaming)
|
|
270
|
+
async def instrumented_srpc_transport_send_message_streaming(
|
|
271
|
+
self, request, *args, **kwargs
|
|
272
|
+
):
|
|
273
|
+
# Put context into A2A message metadata instead of HTTP headers
|
|
274
|
+
with _global_tracer.start_as_current_span(
|
|
275
|
+
"slima2a.send_message_streaming"
|
|
276
|
+
):
|
|
277
|
+
traceparent = get_current_traceparent()
|
|
278
|
+
session_id = None
|
|
279
|
+
if traceparent:
|
|
280
|
+
session_id = kv_store.get(f"execution.{traceparent}")
|
|
281
|
+
if not session_id:
|
|
282
|
+
session_id = get_value("session.id")
|
|
283
|
+
if session_id:
|
|
284
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
285
|
+
|
|
286
|
+
# Ensure metadata dict exists
|
|
287
|
+
try:
|
|
288
|
+
md = getattr(request.params, "metadata", None)
|
|
289
|
+
except AttributeError:
|
|
290
|
+
md = None
|
|
291
|
+
metadata = md if isinstance(md, dict) else {}
|
|
292
|
+
|
|
293
|
+
observe_meta = dict(metadata.get("observe", {}))
|
|
294
|
+
|
|
295
|
+
# Inject W3C trace context + baggage into observe_meta
|
|
296
|
+
TraceContextTextMapPropagator().inject(carrier=observe_meta)
|
|
297
|
+
W3CBaggagePropagator().inject(carrier=observe_meta)
|
|
298
|
+
|
|
299
|
+
if traceparent:
|
|
300
|
+
observe_meta["traceparent"] = traceparent
|
|
301
|
+
if session_id:
|
|
302
|
+
observe_meta["session_id"] = session_id
|
|
303
|
+
baggage.set_baggage(f"execution.{traceparent}", session_id)
|
|
304
|
+
|
|
305
|
+
metadata["observe"] = observe_meta
|
|
306
|
+
|
|
307
|
+
# Write back metadata (pydantic models are mutable by default in v2)
|
|
308
|
+
try:
|
|
309
|
+
request.metadata = metadata
|
|
310
|
+
except Exception:
|
|
311
|
+
# Fallback
|
|
312
|
+
request = request.model_copy(update={"metadata": metadata})
|
|
313
|
+
|
|
314
|
+
# Call through without transport-specific kwargs
|
|
315
|
+
return await original_srpc_transport_send_message_streaming(
|
|
316
|
+
self, request, *args, **kwargs
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
SRPCTransport.send_message_streaming = (
|
|
320
|
+
instrumented_srpc_transport_send_message_streaming
|
|
321
|
+
)
|
|
322
|
+
|
|
202
323
|
def _uninstrument(self, **kwargs):
|
|
203
324
|
import importlib
|
|
204
325
|
|
|
@@ -222,3 +343,20 @@ class A2AInstrumentor(BaseInstrumentor):
|
|
|
222
343
|
DefaultRequestHandler.on_message_send = (
|
|
223
344
|
DefaultRequestHandler.on_message_send.__wrapped__
|
|
224
345
|
)
|
|
346
|
+
|
|
347
|
+
# handle slima2a
|
|
348
|
+
if importlib.util.find_spec("slima2a"):
|
|
349
|
+
from slima2a.client_transport import SRPCTransport
|
|
350
|
+
|
|
351
|
+
# Uninstrument `send_message`
|
|
352
|
+
if hasattr(SRPCTransport, "send_message") and hasattr(
|
|
353
|
+
SRPCTransport.send_message, "__wrapped__"
|
|
354
|
+
):
|
|
355
|
+
SRPCTransport.send_message = SRPCTransport.send_message.__wrapped__
|
|
356
|
+
|
|
357
|
+
if hasattr(SRPCTransport, "send_message_streaming") and hasattr(
|
|
358
|
+
SRPCTransport.send_message_streaming, "__wrapped__"
|
|
359
|
+
):
|
|
360
|
+
SRPCTransport.send_message_streaming = (
|
|
361
|
+
SRPCTransport.send_message.__wrapped__
|
|
362
|
+
)
|
|
@@ -10,6 +10,7 @@ import traceback
|
|
|
10
10
|
import re
|
|
11
11
|
from http import HTTPStatus
|
|
12
12
|
|
|
13
|
+
from opentelemetry.context import get_value
|
|
13
14
|
from opentelemetry import context, propagate
|
|
14
15
|
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
|
|
15
16
|
from opentelemetry.instrumentation.utils import unwrap
|
|
@@ -212,8 +213,10 @@ class McpInstrumentor(BaseInstrumentor):
|
|
|
212
213
|
session_id = None
|
|
213
214
|
if traceparent:
|
|
214
215
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
215
|
-
if session_id:
|
|
216
|
-
|
|
216
|
+
if not session_id:
|
|
217
|
+
session_id = get_value("session.id")
|
|
218
|
+
if session_id:
|
|
219
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
217
220
|
|
|
218
221
|
meta = meta or {}
|
|
219
222
|
if isinstance(meta, dict):
|
|
@@ -7,6 +7,7 @@ import json
|
|
|
7
7
|
import base64
|
|
8
8
|
import threading
|
|
9
9
|
|
|
10
|
+
from opentelemetry.context import get_value
|
|
10
11
|
from opentelemetry import baggage, context
|
|
11
12
|
from opentelemetry.baggage.propagation import W3CBaggagePropagator
|
|
12
13
|
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
|
|
@@ -53,8 +54,10 @@ class NATSInstrumentor(BaseInstrumentor):
|
|
|
53
54
|
if traceparent:
|
|
54
55
|
with _kv_lock:
|
|
55
56
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
56
|
-
if session_id:
|
|
57
|
-
|
|
57
|
+
if not session_id:
|
|
58
|
+
session_id = get_value("session.id")
|
|
59
|
+
if session_id:
|
|
60
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
58
61
|
|
|
59
62
|
headers = {
|
|
60
63
|
"session_id": session_id if session_id else None,
|
|
@@ -104,8 +107,10 @@ class NATSInstrumentor(BaseInstrumentor):
|
|
|
104
107
|
if traceparent:
|
|
105
108
|
with _kv_lock:
|
|
106
109
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
107
|
-
if session_id:
|
|
108
|
-
|
|
110
|
+
if not session_id:
|
|
111
|
+
session_id = get_value("session.id")
|
|
112
|
+
if session_id:
|
|
113
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
109
114
|
|
|
110
115
|
headers = {
|
|
111
116
|
"session_id": session_id if session_id else None,
|
|
@@ -7,6 +7,7 @@ import json
|
|
|
7
7
|
import base64
|
|
8
8
|
import threading
|
|
9
9
|
|
|
10
|
+
from opentelemetry.context import get_value
|
|
10
11
|
from opentelemetry import baggage, context
|
|
11
12
|
from opentelemetry.baggage.propagation import W3CBaggagePropagator
|
|
12
13
|
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
|
|
@@ -15,6 +16,7 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapProp
|
|
|
15
16
|
from ioa_observe.sdk import TracerWrapper
|
|
16
17
|
from ioa_observe.sdk.client import kv_store
|
|
17
18
|
from ioa_observe.sdk.tracing import set_session_id, get_current_traceparent
|
|
19
|
+
from ioa_observe.sdk.tracing.context_manager import get_tracer
|
|
18
20
|
|
|
19
21
|
_instruments = ("slim-bindings >= 0.4.0",)
|
|
20
22
|
_global_tracer = None
|
|
@@ -79,8 +81,10 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
79
81
|
if traceparent:
|
|
80
82
|
with _kv_lock:
|
|
81
83
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
82
|
-
if session_id:
|
|
83
|
-
|
|
84
|
+
if not session_id:
|
|
85
|
+
session_id = get_value("session.id")
|
|
86
|
+
if session_id:
|
|
87
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
84
88
|
|
|
85
89
|
headers = {
|
|
86
90
|
"session_id": session_id if session_id else None,
|
|
@@ -138,8 +142,10 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
138
142
|
if traceparent:
|
|
139
143
|
with _kv_lock:
|
|
140
144
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
141
|
-
if session_id:
|
|
142
|
-
|
|
145
|
+
if not session_id:
|
|
146
|
+
session_id = get_value("session.id")
|
|
147
|
+
if session_id:
|
|
148
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
143
149
|
|
|
144
150
|
headers = {
|
|
145
151
|
"session_id": session_id if session_id else None,
|
|
@@ -201,8 +207,10 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
201
207
|
if traceparent:
|
|
202
208
|
with _kv_lock:
|
|
203
209
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
204
|
-
if session_id:
|
|
205
|
-
|
|
210
|
+
if not session_id:
|
|
211
|
+
session_id = get_value("session.id")
|
|
212
|
+
if session_id:
|
|
213
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
206
214
|
|
|
207
215
|
headers = {
|
|
208
216
|
"session_id": session_id if session_id else None,
|
|
@@ -523,7 +531,15 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
523
531
|
if timeout is not None:
|
|
524
532
|
kwargs["timeout"] = timeout
|
|
525
533
|
|
|
526
|
-
|
|
534
|
+
if _global_tracer:
|
|
535
|
+
with _global_tracer.start_as_current_span(
|
|
536
|
+
"session.get_message"
|
|
537
|
+
) as span:
|
|
538
|
+
if hasattr(self, "id"):
|
|
539
|
+
span.set_attribute("slim.session.id", str(self.id))
|
|
540
|
+
result = await original_get_message(self, **kwargs)
|
|
541
|
+
else:
|
|
542
|
+
result = await original_get_message(self, **kwargs)
|
|
527
543
|
|
|
528
544
|
# Handle different return types from get_message
|
|
529
545
|
if result is None:
|
|
@@ -670,6 +686,8 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
670
686
|
if traceparent:
|
|
671
687
|
with _kv_lock:
|
|
672
688
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
689
|
+
if not session_id:
|
|
690
|
+
session_id = get_value("session.id")
|
|
673
691
|
if session_id:
|
|
674
692
|
kv_store.set(f"execution.{traceparent}", session_id)
|
|
675
693
|
|
|
@@ -722,6 +740,10 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
722
740
|
if traceparent:
|
|
723
741
|
with _kv_lock:
|
|
724
742
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
743
|
+
if not session_id:
|
|
744
|
+
session_id = get_value("session.id")
|
|
745
|
+
if session_id:
|
|
746
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
725
747
|
|
|
726
748
|
if traceparent or session_id:
|
|
727
749
|
headers = {
|
|
@@ -778,7 +800,8 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
778
800
|
@functools.wraps(original_method)
|
|
779
801
|
async def instrumented_session_method(self, *args, **kwargs):
|
|
780
802
|
if _global_tracer:
|
|
781
|
-
|
|
803
|
+
tracer = get_tracer()
|
|
804
|
+
with tracer.start_as_current_span(f"session.{method_name}"):
|
|
782
805
|
traceparent = get_current_traceparent()
|
|
783
806
|
|
|
784
807
|
# Handle message wrapping for publish methods
|
|
@@ -834,6 +857,10 @@ class SLIMInstrumentor(BaseInstrumentor):
|
|
|
834
857
|
if traceparent:
|
|
835
858
|
with _kv_lock:
|
|
836
859
|
session_id = kv_store.get(f"execution.{traceparent}")
|
|
860
|
+
if not session_id:
|
|
861
|
+
session_id = get_value("session.id")
|
|
862
|
+
if session_id:
|
|
863
|
+
kv_store.set(f"execution.{traceparent}", session_id)
|
|
837
864
|
|
|
838
865
|
if traceparent or session_id:
|
|
839
866
|
headers = {
|
|
@@ -194,26 +194,4 @@ def metric_views() -> Sequence[View]:
|
|
|
194
194
|
]
|
|
195
195
|
),
|
|
196
196
|
),
|
|
197
|
-
# response latency in ms
|
|
198
|
-
View(
|
|
199
|
-
instrument_name="gen_ai.ioa.llm.response_latency",
|
|
200
|
-
aggregation=ExplicitBucketHistogramAggregation(
|
|
201
|
-
[
|
|
202
|
-
0.01,
|
|
203
|
-
0.02,
|
|
204
|
-
0.04,
|
|
205
|
-
0.08,
|
|
206
|
-
0.16,
|
|
207
|
-
0.32,
|
|
208
|
-
0.64,
|
|
209
|
-
1.28,
|
|
210
|
-
2.56,
|
|
211
|
-
5.12,
|
|
212
|
-
10.24,
|
|
213
|
-
20.48,
|
|
214
|
-
40.96,
|
|
215
|
-
81.92,
|
|
216
|
-
]
|
|
217
|
-
),
|
|
218
|
-
),
|
|
219
197
|
]
|
|
@@ -7,6 +7,7 @@ import json
|
|
|
7
7
|
import logging
|
|
8
8
|
import os
|
|
9
9
|
import re
|
|
10
|
+
import threading
|
|
10
11
|
import time
|
|
11
12
|
import uuid
|
|
12
13
|
|
|
@@ -64,6 +65,9 @@ from typing import Callable, Dict, Optional, Set
|
|
|
64
65
|
TRACER_NAME = "ioa.observe.tracer"
|
|
65
66
|
APP_NAME = ""
|
|
66
67
|
|
|
68
|
+
SESSION_IDLE_TIMEOUT_SECONDS = 300 # e.g. 5 minutes
|
|
69
|
+
SESSION_WATCHER_INTERVAL_SECONDS = 30
|
|
70
|
+
|
|
67
71
|
|
|
68
72
|
def determine_reliability_score(span):
|
|
69
73
|
if "observe.entity.output" in span.attributes:
|
|
@@ -132,6 +136,9 @@ class TracerWrapper(object):
|
|
|
132
136
|
if not TracerWrapper.endpoint:
|
|
133
137
|
return obj
|
|
134
138
|
|
|
139
|
+
# session activity tracking
|
|
140
|
+
obj._session_last_activity: dict[str, float] = {}
|
|
141
|
+
obj._session_lock = threading.Lock()
|
|
135
142
|
obj.__image_uploader = image_uploader
|
|
136
143
|
# {(agent_name): [success_count, total_count]}
|
|
137
144
|
obj._agent_execution_counts = {}
|
|
@@ -205,11 +212,11 @@ class TracerWrapper(object):
|
|
|
205
212
|
description="Counts agent failures by agent and reason",
|
|
206
213
|
unit="1",
|
|
207
214
|
)
|
|
208
|
-
obj.response_latency_histogram = meter.create_histogram(
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
)
|
|
215
|
+
# obj.response_latency_histogram = meter.create_histogram(
|
|
216
|
+
# name="response_latency",
|
|
217
|
+
# description="Records the latency of responses",
|
|
218
|
+
# unit="ms",
|
|
219
|
+
# )
|
|
213
220
|
obj.messages_received_counter = meter.create_counter(
|
|
214
221
|
"slim.messages.received",
|
|
215
222
|
description="Number of SLIM messages received per agent",
|
|
@@ -226,10 +233,10 @@ class TracerWrapper(object):
|
|
|
226
233
|
"slim.messages.processed",
|
|
227
234
|
description="Number of SLIM messages processed",
|
|
228
235
|
)
|
|
229
|
-
obj.processing_time = meter.create_histogram(
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
)
|
|
236
|
+
# obj.processing_time = meter.create_histogram(
|
|
237
|
+
# "slim.message.processing_time",
|
|
238
|
+
# description="Time taken to process SLIM messages",
|
|
239
|
+
# )
|
|
233
240
|
obj.throughput_counter = meter.create_counter(
|
|
234
241
|
"slim.message.throughput",
|
|
235
242
|
description="Message throughput for SLIM operations",
|
|
@@ -237,11 +244,11 @@ class TracerWrapper(object):
|
|
|
237
244
|
obj.error_counter = meter.create_counter(
|
|
238
245
|
"slim.errors", description="Number of SLIM message errors or drops"
|
|
239
246
|
)
|
|
240
|
-
obj.agent_chain_completion_time_histogram = meter.create_histogram(
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
)
|
|
247
|
+
# obj.agent_chain_completion_time_histogram = meter.create_histogram(
|
|
248
|
+
# name="gen_ai.client.ioa.agent.end_to_end_chain_completion_time",
|
|
249
|
+
# description="Records the end-to-end chain completion time for a single agent",
|
|
250
|
+
# unit="s",
|
|
251
|
+
# )
|
|
245
252
|
obj.agent_execution_success_rate = meter.create_observable_gauge(
|
|
246
253
|
name="gen_ai.client.ioa.agent.execution_success_rate",
|
|
247
254
|
description="Success rate of agent executions",
|
|
@@ -270,12 +277,70 @@ class TracerWrapper(object):
|
|
|
270
277
|
|
|
271
278
|
obj.__content_allow_list = ContentAllowList()
|
|
272
279
|
|
|
280
|
+
# start background watcher for session end
|
|
281
|
+
obj._start_session_watcher()
|
|
282
|
+
|
|
273
283
|
# Force flushes for debug environments (e.g. local development)
|
|
274
284
|
atexit.register(obj.exit_handler)
|
|
275
285
|
|
|
276
286
|
return cls.instance
|
|
277
287
|
|
|
288
|
+
def _start_session_watcher(self) -> None:
|
|
289
|
+
t = threading.Thread(
|
|
290
|
+
target=self._session_watcher_loop,
|
|
291
|
+
name="ioa-session-watcher",
|
|
292
|
+
daemon=True,
|
|
293
|
+
)
|
|
294
|
+
t.start()
|
|
295
|
+
|
|
296
|
+
def _session_watcher_loop(self) -> None:
|
|
297
|
+
while True:
|
|
298
|
+
time.sleep(SESSION_WATCHER_INTERVAL_SECONDS)
|
|
299
|
+
now = time.time()
|
|
300
|
+
expired: dict[str, float] = {}
|
|
301
|
+
|
|
302
|
+
# Find idle sessions and remove them from _session_last_activity
|
|
303
|
+
with self._session_lock:
|
|
304
|
+
for session_id, last_ts in list(self._session_last_activity.items()):
|
|
305
|
+
if now - last_ts > SESSION_IDLE_TIMEOUT_SECONDS:
|
|
306
|
+
expired[session_id] = last_ts
|
|
307
|
+
del self._session_last_activity[session_id]
|
|
308
|
+
|
|
309
|
+
if not expired:
|
|
310
|
+
continue
|
|
311
|
+
|
|
312
|
+
tracer = self.get_tracer()
|
|
313
|
+
|
|
314
|
+
# Iterate over a snapshot and do *not* modify `expired` in the loop
|
|
315
|
+
for session_id, _last_ts in list(expired.items()):
|
|
316
|
+
print("ending session", session_id)
|
|
317
|
+
with tracer.start_as_current_span("session.end") as span:
|
|
318
|
+
span.set_attribute("session.id", session_id)
|
|
319
|
+
workflow_name = get_value("workflow_name")
|
|
320
|
+
if workflow_name:
|
|
321
|
+
span.set_attribute(OBSERVE_WORKFLOW_NAME, workflow_name)
|
|
322
|
+
span.set_attribute("session.ended_at", _last_ts)
|
|
323
|
+
|
|
324
|
+
# ensure end spans are exported reasonably fast
|
|
325
|
+
self.flush()
|
|
326
|
+
|
|
278
327
|
def exit_handler(self):
|
|
328
|
+
# emit end spans for any sessions that never went idle
|
|
329
|
+
now = time.time()
|
|
330
|
+
tracer = self.get_tracer()
|
|
331
|
+
|
|
332
|
+
with self._session_lock:
|
|
333
|
+
remaining_ids = list(self._session_last_activity.keys())
|
|
334
|
+
self._session_last_activity.clear()
|
|
335
|
+
|
|
336
|
+
for session_id in remaining_ids:
|
|
337
|
+
with tracer.start_as_current_span("session.end") as span:
|
|
338
|
+
span.set_attribute("session.id", session_id)
|
|
339
|
+
workflow_name = get_value("workflow_name")
|
|
340
|
+
if workflow_name:
|
|
341
|
+
span.set_attribute(OBSERVE_WORKFLOW_NAME, workflow_name)
|
|
342
|
+
span.set_attribute("session.ended_at", now)
|
|
343
|
+
|
|
279
344
|
self.flush()
|
|
280
345
|
|
|
281
346
|
def _span_processor_on_start(self, span, parent_context):
|
|
@@ -362,8 +427,14 @@ class TracerWrapper(object):
|
|
|
362
427
|
# Mark this span as processed
|
|
363
428
|
self._processed_spans.add(span_id)
|
|
364
429
|
|
|
430
|
+
# update last activity per session
|
|
431
|
+
session_id = span.attributes.get("session.id")
|
|
432
|
+
if session_id:
|
|
433
|
+
with self._session_lock:
|
|
434
|
+
self._session_last_activity[session_id] = time.time()
|
|
435
|
+
|
|
365
436
|
determine_reliability_score(span)
|
|
366
|
-
start_time = span.attributes.get("ioa_start_time")
|
|
437
|
+
# start_time = span.attributes.get("ioa_start_time")
|
|
367
438
|
|
|
368
439
|
# Apply transformations if enabled
|
|
369
440
|
apply_transform = get_value("apply_transform")
|
|
@@ -382,10 +453,6 @@ class TracerWrapper(object):
|
|
|
382
453
|
except Exception as e:
|
|
383
454
|
logging.error(f"Error applying span transformation: {e}")
|
|
384
455
|
|
|
385
|
-
if start_time is not None:
|
|
386
|
-
latency = (time.time() - start_time) * 1000
|
|
387
|
-
self.response_latency_histogram.record(latency, attributes=span.attributes)
|
|
388
|
-
|
|
389
456
|
# Call original on_end method if it exists
|
|
390
457
|
if (
|
|
391
458
|
hasattr(self, "_TracerWrapper__spans_processor_original_on_end")
|
|
@@ -684,28 +751,24 @@ def set_session_id(session_id: str, traceparent: str = None) -> None:
|
|
|
684
751
|
|
|
685
752
|
# Check if we have an active span first
|
|
686
753
|
current_span = trace.get_current_span()
|
|
754
|
+
extracted_traceparent = None
|
|
687
755
|
|
|
688
756
|
if current_span.is_recording():
|
|
689
757
|
# We have an active span, use its context
|
|
690
758
|
carrier = {}
|
|
691
759
|
TraceContextTextMapPropagator().inject(carrier)
|
|
692
760
|
extracted_traceparent = carrier.get("traceparent")
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
#
|
|
761
|
+
# Store execution ID with traceparent as key
|
|
762
|
+
if extracted_traceparent:
|
|
763
|
+
kv_key = f"execution.{extracted_traceparent}"
|
|
764
|
+
if kv_store.get(kv_key) is None:
|
|
765
|
+
kv_store.set(kv_key, session_id)
|
|
766
|
+
# If there is no active span, we do nothing now and when we need the traceparent
|
|
767
|
+
# e.g. when propagating context, we update the kv_store with the session_id then
|
|
768
|
+
|
|
769
|
+
# Finally, attach to context session.id and current_traceparent if present
|
|
770
|
+
attach(set_value("session.id", session_id))
|
|
702
771
|
if extracted_traceparent:
|
|
703
|
-
kv_key = f"execution.{extracted_traceparent}"
|
|
704
|
-
if kv_store.get(kv_key) is None:
|
|
705
|
-
kv_store.set(kv_key, session_id)
|
|
706
|
-
|
|
707
|
-
# Also store in OpenTelemetry context
|
|
708
|
-
attach(set_value("session.id", session_id))
|
|
709
772
|
attach(set_value("current_traceparent", extracted_traceparent))
|
|
710
773
|
|
|
711
774
|
|
|
@@ -8,21 +8,21 @@ ioa_observe/sdk/client/client.py,sha256=6TVOo_E1ulE3WO_CYG7oPgeucs-qegOA09uTO3yQ
|
|
|
8
8
|
ioa_observe/sdk/client/http.py,sha256=LdLYSQPFIhKN5BTB-N78jLO7ITl7jGjA0-qpewEIvO4,1724
|
|
9
9
|
ioa_observe/sdk/config/__init__.py,sha256=8aVNaw0yRNLFPxlf97iOZLlJVcV81ivSDnudH2m1OIo,572
|
|
10
10
|
ioa_observe/sdk/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
-
ioa_observe/sdk/connectors/slim.py,sha256=
|
|
11
|
+
ioa_observe/sdk/connectors/slim.py,sha256=A_ojiJ_zm-GWuVkZPlwhY7nCt3w2kAohKgnkGO_S634,8518
|
|
12
12
|
ioa_observe/sdk/decorators/__init__.py,sha256=qCpJAv98eLKs3I5EMXJVTV0s49Nc6QDSOHNh5rW5vLg,4268
|
|
13
|
-
ioa_observe/sdk/decorators/base.py,sha256=
|
|
13
|
+
ioa_observe/sdk/decorators/base.py,sha256=gezoLMLOo-CWsxzsXbQMsj5HxmchUEmlhTspbqzOrPc,32732
|
|
14
14
|
ioa_observe/sdk/decorators/helpers.py,sha256=I9HXMBivkZpGDtPe9Ad_UU35p_m_wEPate4r_fU0oOA,2705
|
|
15
15
|
ioa_observe/sdk/decorators/util.py,sha256=IebvH9gwZN1en3LblYJUh4bAV2STl6xmp8WpZzBDH2g,30068
|
|
16
16
|
ioa_observe/sdk/instrumentations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
-
ioa_observe/sdk/instrumentations/a2a.py,sha256=
|
|
18
|
-
ioa_observe/sdk/instrumentations/mcp.py,sha256=
|
|
19
|
-
ioa_observe/sdk/instrumentations/nats.py,sha256=
|
|
20
|
-
ioa_observe/sdk/instrumentations/slim.py,sha256=
|
|
17
|
+
ioa_observe/sdk/instrumentations/a2a.py,sha256=LHvvQUluPtmvbRNuoqf-EkWlZJa_NicerTpM3t1UZws,15299
|
|
18
|
+
ioa_observe/sdk/instrumentations/mcp.py,sha256=UG82fWitFT-K-ecnkTKZa-uFTDAs-j2HymvhUwCBRLY,18721
|
|
19
|
+
ioa_observe/sdk/instrumentations/nats.py,sha256=MEYCFqHyRK5HtkjrbGVV5qIWplZ1ZjJjFCB8vtQl6qE,14736
|
|
20
|
+
ioa_observe/sdk/instrumentations/slim.py,sha256=c_Q1cb6pdjZLOw1MbclUf1eRFrc_sUEe9pQJvAyoD1Y,45712
|
|
21
21
|
ioa_observe/sdk/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
22
|
ioa_observe/sdk/logging/logging.py,sha256=HZxW9s8Due7jgiNkdI38cIjv5rC9D-Flta3RQMOnpow,2891
|
|
23
23
|
ioa_observe/sdk/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
24
|
ioa_observe/sdk/metrics/agent.py,sha256=I4ygssTqHm_BPI4IB5rsId5EmRNvW9Te1gRbBp9aSJ0,731
|
|
25
|
-
ioa_observe/sdk/metrics/metrics.py,sha256=
|
|
25
|
+
ioa_observe/sdk/metrics/metrics.py,sha256=J1aRMZ9B6bbTE7Yv5ehs4bczZOvXm1KByk0lpRk56ik,5942
|
|
26
26
|
ioa_observe/sdk/metrics/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
27
|
ioa_observe/sdk/metrics/agents/agent_connections.py,sha256=r-g49czOQLnyIy3bMyJW15NsG3Pgeve9sxcgbpKbSXo,4314
|
|
28
28
|
ioa_observe/sdk/metrics/agents/availability.py,sha256=761o2NiA9SomUWIZrzKjttsz5Fthf72nsgDNfDzUk8s,5028
|
|
@@ -35,15 +35,15 @@ ioa_observe/sdk/tracing/content_allow_list.py,sha256=1fAkpIwUQ7vDwCTkIVrqeltWQtr
|
|
|
35
35
|
ioa_observe/sdk/tracing/context_manager.py,sha256=O0JEXYa9h8anhW78R8KKBuqS0j4by1E1KXxNIMPnLr8,400
|
|
36
36
|
ioa_observe/sdk/tracing/context_utils.py,sha256=-sYS9vPLI87davV9ubneq5xqbV583CC_c0SmOQS1TAs,2933
|
|
37
37
|
ioa_observe/sdk/tracing/manual.py,sha256=KS6WN-zw9vAACzXYmnMoJm9d1fenYMfvzeK1GrGDPDE,1937
|
|
38
|
-
ioa_observe/sdk/tracing/tracing.py,sha256=
|
|
38
|
+
ioa_observe/sdk/tracing/tracing.py,sha256=7eJAEqaAEBZ2gjIXrdb7SyUidA8P10GnpBLzr8InLKI,50007
|
|
39
39
|
ioa_observe/sdk/tracing/transform_span.py,sha256=XTApi_gJxum7ynvhtcoCfDyK8VVOj91Q1DT6hAeLHA8,8419
|
|
40
40
|
ioa_observe/sdk/utils/__init__.py,sha256=UPn182U-UblF_XwXaFpx8F-TmQTbm1LYf9y89uSp5Hw,704
|
|
41
41
|
ioa_observe/sdk/utils/const.py,sha256=d67dUTAH9UpWvUV9GLBUqn1Sc2knJ55dy-e6YoLrvSo,1318
|
|
42
42
|
ioa_observe/sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQvzJk_fZH0OOsRAM1iZQXsR8,2112
|
|
43
43
|
ioa_observe/sdk/utils/json_encoder.py,sha256=g4NQ0tTqgWssY6I1D7r4zo0G6PiUo61jhofTAw5-jno,639
|
|
44
44
|
ioa_observe/sdk/utils/package_check.py,sha256=1d1MjxhwoEZIx9dumirT2pRsEWgn-m-SI4npDeEalew,576
|
|
45
|
-
ioa_observe_sdk-1.0.
|
|
46
|
-
ioa_observe_sdk-1.0.
|
|
47
|
-
ioa_observe_sdk-1.0.
|
|
48
|
-
ioa_observe_sdk-1.0.
|
|
49
|
-
ioa_observe_sdk-1.0.
|
|
45
|
+
ioa_observe_sdk-1.0.27.dist-info/licenses/LICENSE.md,sha256=55VjUfgjWOS4vv3Cf55gfq-RxjPgRIO2vlgYPUuC5lA,11362
|
|
46
|
+
ioa_observe_sdk-1.0.27.dist-info/METADATA,sha256=OrhfkqzR0Qj2qwSGXtSmL9rYjS1OMk-IYmTU5wHMHdE,7997
|
|
47
|
+
ioa_observe_sdk-1.0.27.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
48
|
+
ioa_observe_sdk-1.0.27.dist-info/top_level.txt,sha256=Yt-6Y1olZEDqCs2REeqI30WjYx0pLGQSVqzYmDd67N8,12
|
|
49
|
+
ioa_observe_sdk-1.0.27.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|