monocle-apptrace 0.6.0__py3-none-any.whl → 0.6.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of monocle-apptrace might be problematic. Click here for more details.
- monocle_apptrace/instrumentation/common/constants.py +8 -0
- monocle_apptrace/instrumentation/common/span_handler.py +73 -23
- monocle_apptrace/instrumentation/common/utils.py +63 -6
- monocle_apptrace/instrumentation/common/wrapper.py +111 -42
- monocle_apptrace/instrumentation/common/wrapper_method.py +4 -2
- monocle_apptrace/instrumentation/metamodel/a2a/methods.py +1 -1
- monocle_apptrace/instrumentation/metamodel/adk/_helper.py +2 -1
- monocle_apptrace/instrumentation/metamodel/agents/_helper.py +3 -3
- monocle_apptrace/instrumentation/metamodel/agents/entities/inference.py +2 -0
- monocle_apptrace/instrumentation/metamodel/aiohttp/_helper.py +1 -1
- monocle_apptrace/instrumentation/metamodel/anthropic/entities/inference.py +1 -4
- monocle_apptrace/instrumentation/metamodel/azfunc/_helper.py +1 -1
- monocle_apptrace/instrumentation/metamodel/botocore/_helper.py +5 -0
- monocle_apptrace/instrumentation/metamodel/botocore/entities/inference.py +4 -0
- monocle_apptrace/instrumentation/metamodel/fastapi/_helper.py +4 -4
- monocle_apptrace/instrumentation/metamodel/fastapi/methods.py +4 -4
- monocle_apptrace/instrumentation/metamodel/flask/_helper.py +3 -3
- monocle_apptrace/instrumentation/metamodel/hugging_face/_helper.py +1 -1
- monocle_apptrace/instrumentation/metamodel/hugging_face/entities/inference.py +1 -4
- monocle_apptrace/instrumentation/metamodel/lambdafunc/_helper.py +1 -1
- monocle_apptrace/instrumentation/metamodel/langchain/entities/inference.py +1 -4
- monocle_apptrace/instrumentation/metamodel/langgraph/_helper.py +30 -6
- monocle_apptrace/instrumentation/metamodel/langgraph/entities/inference.py +1 -1
- monocle_apptrace/instrumentation/metamodel/langgraph/langgraph_processor.py +88 -19
- monocle_apptrace/instrumentation/metamodel/langgraph/methods.py +22 -6
- monocle_apptrace/instrumentation/metamodel/llamaindex/_helper.py +30 -10
- monocle_apptrace/instrumentation/metamodel/llamaindex/entities/agent.py +4 -3
- monocle_apptrace/instrumentation/metamodel/llamaindex/llamaindex_processor.py +15 -7
- monocle_apptrace/instrumentation/metamodel/llamaindex/methods.py +1 -8
- monocle_apptrace/instrumentation/metamodel/mcp/_helper.py +1 -1
- monocle_apptrace/instrumentation/metamodel/mistral/_helper.py +1 -1
- monocle_apptrace/instrumentation/metamodel/mistral/entities/inference.py +1 -4
- monocle_apptrace/instrumentation/metamodel/mistral/methods.py +0 -8
- monocle_apptrace/instrumentation/metamodel/openai/_helper.py +47 -7
- monocle_apptrace/instrumentation/metamodel/openai/entities/inference.py +20 -4
- monocle_apptrace/instrumentation/metamodel/openai/methods.py +1 -1
- monocle_apptrace/instrumentation/metamodel/strands/_helper.py +44 -0
- monocle_apptrace/instrumentation/metamodel/strands/entities/agent.py +179 -0
- monocle_apptrace/instrumentation/metamodel/strands/entities/tool.py +62 -0
- monocle_apptrace/instrumentation/metamodel/strands/methods.py +20 -0
- {monocle_apptrace-0.6.0.dist-info → monocle_apptrace-0.6.6.dist-info}/METADATA +15 -4
- {monocle_apptrace-0.6.0.dist-info → monocle_apptrace-0.6.6.dist-info}/RECORD +45 -41
- {monocle_apptrace-0.6.0.dist-info → monocle_apptrace-0.6.6.dist-info}/WHEEL +0 -0
- {monocle_apptrace-0.6.0.dist-info → monocle_apptrace-0.6.6.dist-info}/entry_points.txt +0 -0
- {monocle_apptrace-0.6.0.dist-info → monocle_apptrace-0.6.6.dist-info}/licenses/LICENSE +0 -0
|
@@ -92,6 +92,9 @@ CHILD_ERROR_CODE = "child.error.code"
|
|
|
92
92
|
|
|
93
93
|
AGENT_PREFIX_KEY = "monocle.agent.prefix"
|
|
94
94
|
|
|
95
|
+
MONOCLE_SKIP_EXECUTIONS = "monocle.skip_executions"
|
|
96
|
+
SKIPPED_EXECUTION = "monocle.skipped_execution"
|
|
97
|
+
|
|
95
98
|
# agentic sub types
|
|
96
99
|
INFERENCE_AGENT_DELEGATION = "delegation"
|
|
97
100
|
INFERENCE_TOOL_CALL = "tool_call"
|
|
@@ -178,3 +181,8 @@ MAP_ATTRIBUTES_TO_SPAN_SUBTYPE = {
|
|
|
178
181
|
|
|
179
182
|
|
|
180
183
|
}
|
|
184
|
+
|
|
185
|
+
PROVIDER_BASE_URLS = {
|
|
186
|
+
"deepseek": "https://api.deepseek.com",
|
|
187
|
+
# Add more providers here later
|
|
188
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
from contextlib import contextmanager
|
|
4
|
+
from typing import Union
|
|
4
5
|
from opentelemetry.context import get_value, set_value, attach, detach
|
|
5
6
|
from opentelemetry.sdk.trace import Span
|
|
6
7
|
from opentelemetry.trace.status import Status, StatusCode
|
|
@@ -10,8 +11,8 @@ from monocle_apptrace.instrumentation.common.constants import (
|
|
|
10
11
|
service_type_map,
|
|
11
12
|
MONOCLE_SDK_VERSION, MONOCLE_SDK_LANGUAGE, MONOCLE_DETECTED_SPAN_ERROR
|
|
12
13
|
)
|
|
13
|
-
from monocle_apptrace.instrumentation.common.utils import set_attribute, get_scopes, MonocleSpanException, get_monocle_version
|
|
14
|
-
from monocle_apptrace.instrumentation.common.constants import WORKFLOW_TYPE_KEY, WORKFLOW_TYPE_GENERIC, CHILD_ERROR_CODE
|
|
14
|
+
from monocle_apptrace.instrumentation.common.utils import set_attribute, get_scopes, MonocleSpanException, get_monocle_version, replace_placeholders
|
|
15
|
+
from monocle_apptrace.instrumentation.common.constants import WORKFLOW_TYPE_KEY, WORKFLOW_TYPE_GENERIC, CHILD_ERROR_CODE, MONOCLE_SKIP_EXECUTIONS, SKIPPED_EXECUTION
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
17
18
|
|
|
@@ -49,7 +50,7 @@ class SpanHandler:
|
|
|
49
50
|
pass
|
|
50
51
|
|
|
51
52
|
def pre_tracing(self, to_wrap, wrapped, instance, args, kwargs):
|
|
52
|
-
|
|
53
|
+
return None, None
|
|
53
54
|
|
|
54
55
|
def post_tracing(self, to_wrap, wrapped, instance, args, kwargs, return_value, token=None):
|
|
55
56
|
pass
|
|
@@ -67,8 +68,6 @@ class SpanHandler:
|
|
|
67
68
|
span.set_attribute("span.type", span_type)
|
|
68
69
|
else:
|
|
69
70
|
logger.warning("type of span not found or incorrect written in entity json")
|
|
70
|
-
if "subtype" in output_processor:
|
|
71
|
-
span.set_attribute("span.subtype", output_processor["subtype"])
|
|
72
71
|
return span_type
|
|
73
72
|
|
|
74
73
|
def pre_task_processing(self, to_wrap, wrapped, instance, args,kwargs, span):
|
|
@@ -103,28 +102,32 @@ class SpanHandler:
|
|
|
103
102
|
def post_task_processing(self, to_wrap, wrapped, instance, args, kwargs, result, ex, span:Span, parent_span:Span):
|
|
104
103
|
pass
|
|
105
104
|
|
|
106
|
-
def should_skip(self, processor, instance, args, kwargs) -> bool:
|
|
105
|
+
def should_skip(self, processor, instance, span, parent_span, args, kwargs) -> bool:
|
|
107
106
|
should_skip = False
|
|
108
107
|
accessor = processor.get('should_skip')
|
|
109
108
|
if accessor:
|
|
110
|
-
arguments = {"instance":instance, "args":args, "kwargs":kwargs}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
109
|
+
arguments = {"instance":instance, "span":span, "parent_span":parent_span, "args":args, "kwargs":kwargs}
|
|
110
|
+
try:
|
|
111
|
+
should_skip = accessor(arguments)
|
|
112
|
+
if not isinstance(should_skip, bool):
|
|
113
|
+
logger.warning("Warning: 'should_skip' accessor did not return a boolean value")
|
|
114
|
+
return False
|
|
115
|
+
except Exception as e:
|
|
116
|
+
logger.warning("Warning: Error occurred in 'should_skip' accessor: %s", str(e))
|
|
115
117
|
return should_skip
|
|
116
118
|
|
|
117
|
-
def hydrate_span(self, to_wrap, wrapped, instance, args, kwargs, result, span, parent_span = None,
|
|
119
|
+
def hydrate_span(self, to_wrap, wrapped, instance, args, kwargs, result, span, parent_span = None,
|
|
120
|
+
ex:Exception = None, is_post_exec:bool= False) -> bool:
|
|
118
121
|
try:
|
|
119
|
-
detected_error_in_attribute = self.hydrate_attributes(to_wrap, wrapped, instance, args, kwargs, result, span, parent_span)
|
|
120
|
-
detected_error_in_event = self.hydrate_events(to_wrap, wrapped, instance, args, kwargs, result, span, parent_span, ex)
|
|
122
|
+
detected_error_in_attribute = self.hydrate_attributes(to_wrap, wrapped, instance, args, kwargs, result, span, parent_span, is_post_exec)
|
|
123
|
+
detected_error_in_event = self.hydrate_events(to_wrap, wrapped, instance, args, kwargs, result, span, parent_span, ex, is_post_exec)
|
|
121
124
|
if detected_error_in_attribute or detected_error_in_event:
|
|
122
125
|
span.set_attribute(MONOCLE_DETECTED_SPAN_ERROR, True)
|
|
123
126
|
finally:
|
|
124
|
-
if span.status.status_code == StatusCode.UNSET and ex is None:
|
|
127
|
+
if is_post_exec and span.status.status_code == StatusCode.UNSET and ex is None:
|
|
125
128
|
span.set_status(StatusCode.OK)
|
|
126
129
|
|
|
127
|
-
def hydrate_attributes(self, to_wrap, wrapped, instance, args, kwargs, result, span:Span, parent_span:Span) -> bool:
|
|
130
|
+
def hydrate_attributes(self, to_wrap, wrapped, instance, args, kwargs, result, span:Span, parent_span:Span, is_post_exec:bool) -> bool:
|
|
128
131
|
detected_error:bool = False
|
|
129
132
|
span_index = 0
|
|
130
133
|
if SpanHandler.is_root_span(span):
|
|
@@ -133,7 +136,6 @@ class SpanHandler:
|
|
|
133
136
|
output_processor=to_wrap['output_processor']
|
|
134
137
|
self.set_span_type(to_wrap, wrapped, instance, output_processor, span, args, kwargs)
|
|
135
138
|
skip_processors:list[str] = self.skip_processor(to_wrap, wrapped, instance, span, args, kwargs) or []
|
|
136
|
-
|
|
137
139
|
if 'attributes' in output_processor and 'attributes' not in skip_processors:
|
|
138
140
|
arguments = {"instance":instance, "args":args, "kwargs":kwargs, "result":result, "parent_span":parent_span, "span":span}
|
|
139
141
|
for processors in output_processor["attributes"]:
|
|
@@ -144,9 +146,10 @@ class SpanHandler:
|
|
|
144
146
|
if attribute and accessor:
|
|
145
147
|
attribute_name = f"entity.{span_index+1}.{attribute}"
|
|
146
148
|
try:
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
149
|
+
if (not is_post_exec and processor.get('phase', '') != 'post_execution') or (is_post_exec and processor.get('phase', '') == 'post_execution'):
|
|
150
|
+
processor_result = accessor(arguments)
|
|
151
|
+
if processor_result and isinstance(processor_result, (str, list)):
|
|
152
|
+
span.set_attribute(attribute_name, processor_result)
|
|
150
153
|
except MonocleSpanException as e:
|
|
151
154
|
span.set_status(StatusCode.ERROR, e.message)
|
|
152
155
|
detected_error = True
|
|
@@ -165,13 +168,27 @@ class SpanHandler:
|
|
|
165
168
|
span.set_attribute("entity.count", span_index)
|
|
166
169
|
return detected_error
|
|
167
170
|
|
|
168
|
-
def hydrate_events(self, to_wrap, wrapped, instance, args, kwargs, ret_result, span: Span, parent_span=None, ex:Exception=None
|
|
171
|
+
def hydrate_events(self, to_wrap, wrapped, instance, args, kwargs, ret_result, span: Span, parent_span=None, ex:Exception=None,
|
|
172
|
+
is_post_exec: bool = False) -> bool:
|
|
169
173
|
detected_error:bool = False
|
|
170
174
|
if 'output_processor' in to_wrap and to_wrap["output_processor"] is not None:
|
|
171
175
|
output_processor=to_wrap['output_processor']
|
|
172
|
-
|
|
173
|
-
|
|
176
|
+
if is_post_exec:
|
|
177
|
+
skip_events:list[str] = ['events.data.input']
|
|
178
|
+
else:
|
|
179
|
+
skip_events:list[str] = ['events.data.output', 'events.metadata']
|
|
180
|
+
skip_processors:list[str] = list(set(self.skip_processor(to_wrap, wrapped, instance, span, args, kwargs) or []).union(set(skip_events)))
|
|
174
181
|
arguments = {"instance": instance, "args": args, "kwargs": kwargs, "result": ret_result, "exception":ex, "parent_span":parent_span, "span": span}
|
|
182
|
+
subtype = output_processor.get('subtype')
|
|
183
|
+
if subtype:
|
|
184
|
+
if callable(subtype):
|
|
185
|
+
try:
|
|
186
|
+
subtype_result = subtype(arguments)
|
|
187
|
+
span.set_attribute("span.subtype", subtype_result)
|
|
188
|
+
except Exception as e:
|
|
189
|
+
logger.debug(f"Error processing subtype: {e}")
|
|
190
|
+
else:
|
|
191
|
+
span.set_attribute("span.subtype", subtype)
|
|
175
192
|
# Process events if they are defined in the output_processor.
|
|
176
193
|
# In case of inference.modelapi skip the event processing unless the span has an exception
|
|
177
194
|
if 'events' in output_processor and ('events' not in skip_processors or ex is not None):
|
|
@@ -200,6 +217,13 @@ class SpanHandler:
|
|
|
200
217
|
event_attributes[attribute_key] = result
|
|
201
218
|
else:
|
|
202
219
|
event_attributes.update(result)
|
|
220
|
+
if not is_post_exec and event_name == "data.input" and attribute_key == "input" and result:
|
|
221
|
+
# append memory to input if available
|
|
222
|
+
# update the result with new input
|
|
223
|
+
pass
|
|
224
|
+
elif not detected_error and is_post_exec and event_name == "data.output" and attribute_key == "response" and result:
|
|
225
|
+
# capture memory
|
|
226
|
+
pass
|
|
203
227
|
except Exception as e:
|
|
204
228
|
logger.debug(f"Error evaluating accessor for attribute '{attribute_key}': {e}")
|
|
205
229
|
matching_timestamp = getattr(ret_result, "timestamps", {}).get(event_name, None)
|
|
@@ -299,6 +323,32 @@ class SpanHandler:
|
|
|
299
323
|
finally:
|
|
300
324
|
SpanHandler.detach_workflow_type(token)
|
|
301
325
|
|
|
326
|
+
@staticmethod
|
|
327
|
+
def get_iput_entity_type(span: Span) -> str:
|
|
328
|
+
for event in span.events:
|
|
329
|
+
if event.name == "data.input":
|
|
330
|
+
return event.attributes.get("entity.type", "")
|
|
331
|
+
|
|
332
|
+
@staticmethod
|
|
333
|
+
def skip_execution(span:Span) -> tuple[bool, Union[dict, list, str, None]]:
|
|
334
|
+
skip_execs = get_value(MONOCLE_SKIP_EXECUTIONS)
|
|
335
|
+
if skip_execs is not None:
|
|
336
|
+
skip_exec_entity = skip_execs.get(span.attributes.get("entity.1.name", ""),{})
|
|
337
|
+
if ((span.attributes.get("span.type") is not None and skip_exec_entity.get("span.type", "") == span.attributes.get("span.type")) and
|
|
338
|
+
(span.attributes.get("entity.1.type") is not None and skip_exec_entity.get("entity.type", "") == span.attributes.get("entity.1.type"))):
|
|
339
|
+
span.set_attribute(SKIPPED_EXECUTION, True)
|
|
340
|
+
if skip_exec_entity.get("raise_error", False):
|
|
341
|
+
raise MonocleSpanException(skip_exec_entity.get("error_message", ""))
|
|
342
|
+
response = skip_exec_entity.get("response", None)
|
|
343
|
+
response = replace_placeholders(response, span)
|
|
344
|
+
return True, response
|
|
345
|
+
return False, None
|
|
346
|
+
|
|
347
|
+
@staticmethod
|
|
348
|
+
def replace_placeholders_in_response(response: Union[dict, list, str], span:Span) -> Union[dict, list, str]:
|
|
349
|
+
if span.attributes.get(SKIPPED_EXECUTION, False):
|
|
350
|
+
return replace_placeholders(response, span)
|
|
351
|
+
return response
|
|
302
352
|
|
|
303
353
|
class NonFrameworkSpanHandler(SpanHandler):
|
|
304
354
|
# If the language framework is being executed, then skip generating direct openAI attributes and events
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import ast
|
|
1
2
|
import logging, json
|
|
2
3
|
import os
|
|
3
4
|
import traceback
|
|
4
|
-
from typing import Callable, Generic, Optional, TypeVar, Mapping
|
|
5
|
+
from typing import Callable, Generic, Optional, TypeVar, Mapping, Union
|
|
5
6
|
|
|
6
7
|
from opentelemetry.context import attach, detach, get_current, get_value, set_value, Context
|
|
7
8
|
from opentelemetry.trace import NonRecordingSpan, Span
|
|
@@ -217,13 +218,16 @@ def remove_scopes(token:object) -> None:
|
|
|
217
218
|
if token is not None:
|
|
218
219
|
detach(token)
|
|
219
220
|
|
|
220
|
-
def get_scopes() -> dict[str, object]:
|
|
221
|
+
def get_scopes(scope_name: Optional[str] = None) -> dict[str, object]:
|
|
221
222
|
monocle_scopes:dict[str, object] = {}
|
|
222
223
|
for key, val in baggage.get_all().items():
|
|
223
|
-
if key.startswith(MONOCLE_SCOPE_NAME_PREFIX):
|
|
224
|
+
if key.startswith(MONOCLE_SCOPE_NAME_PREFIX) and (scope_name is None or key == f"{MONOCLE_SCOPE_NAME_PREFIX}{scope_name}"):
|
|
224
225
|
monocle_scopes[key[len(MONOCLE_SCOPE_NAME_PREFIX):]] = val
|
|
225
226
|
return monocle_scopes
|
|
226
227
|
|
|
228
|
+
def is_scope_set(scepe_name: str) -> bool:
|
|
229
|
+
return len(get_scopes(scepe_name)) > 0
|
|
230
|
+
|
|
227
231
|
def get_baggage_for_scopes():
|
|
228
232
|
baggage_context:Context = None
|
|
229
233
|
for scope_key, scope_value in get_scopes():
|
|
@@ -357,7 +361,7 @@ class Option(Generic[T]):
|
|
|
357
361
|
|
|
358
362
|
def and_then(self, func: Callable[[T], 'Option[U]']) -> 'Option[U]':
|
|
359
363
|
if self.is_some():
|
|
360
|
-
return func(self.value)
|
|
364
|
+
return Option(func(self.value))
|
|
361
365
|
return Option(None)
|
|
362
366
|
|
|
363
367
|
# Example usage
|
|
@@ -409,7 +413,7 @@ def get_error_message(arguments):
|
|
|
409
413
|
|
|
410
414
|
|
|
411
415
|
def get_status_code(arguments):
|
|
412
|
-
if arguments["exception"] is not None:
|
|
416
|
+
if "exception" in arguments and arguments["exception"] is not None:
|
|
413
417
|
return get_exception_status_code(arguments)
|
|
414
418
|
elif hasattr(arguments["result"], "status"):
|
|
415
419
|
return arguments["result"].status
|
|
@@ -467,4 +471,57 @@ def get_current_monocle_span(context: Optional[Context] = None) -> Span:
|
|
|
467
471
|
span = get_value(_MONOCLE_SPAN_KEY, context=context)
|
|
468
472
|
if span is None or not isinstance(span, Span):
|
|
469
473
|
return INVALID_SPAN
|
|
470
|
-
return span
|
|
474
|
+
return span
|
|
475
|
+
|
|
476
|
+
def get_input_event_from_span(events: list[dict], search_key:str) -> Optional[Mapping]:
|
|
477
|
+
"""Extract the 'data.input' event from the span if it exists.
|
|
478
|
+
|
|
479
|
+
Args:
|
|
480
|
+
span: The Span to extract the event from.
|
|
481
|
+
|
|
482
|
+
Returns:
|
|
483
|
+
The 'data.input' event if it exists, None otherwise.
|
|
484
|
+
"""
|
|
485
|
+
input_request = None
|
|
486
|
+
for event in events:
|
|
487
|
+
if event.name == "data.input":
|
|
488
|
+
try:
|
|
489
|
+
#load the input attribute as dictionary from string
|
|
490
|
+
try:
|
|
491
|
+
input_dict = json.loads(event.attributes.get("input", "{}"))
|
|
492
|
+
except Exception as e:
|
|
493
|
+
if isinstance(e, json.JSONDecodeError):
|
|
494
|
+
input_dict = ast.literal_eval(event.attributes.get("input", {}))
|
|
495
|
+
else:
|
|
496
|
+
raise
|
|
497
|
+
if search_key in input_dict:
|
|
498
|
+
input_request = input_dict[search_key]
|
|
499
|
+
except Exception as e:
|
|
500
|
+
logger.debug(f"Error parsing input event attribute: {e}")
|
|
501
|
+
break
|
|
502
|
+
return input_request
|
|
503
|
+
|
|
504
|
+
def replace_placeholders(obj: Union[dict, list, str], span: Span) -> Union[dict, list, str]:
|
|
505
|
+
"""Replace placeholders in strings with span context values."""
|
|
506
|
+
if isinstance(obj, dict):
|
|
507
|
+
return {k: replace_placeholders(v, span) for k, v in obj.items()}
|
|
508
|
+
elif isinstance(obj, list):
|
|
509
|
+
return [replace_placeholders(item, span) for item in obj]
|
|
510
|
+
elif isinstance(obj, str):
|
|
511
|
+
startIndex = 0
|
|
512
|
+
while True:
|
|
513
|
+
start = obj.find("{{", startIndex)
|
|
514
|
+
end = obj.find("}}", start + 2)
|
|
515
|
+
if start == -1 or end == -1:
|
|
516
|
+
break
|
|
517
|
+
key = obj[start + 2:end].strip()
|
|
518
|
+
value = get_input_event_from_span(span.events, key)
|
|
519
|
+
if value is not None:
|
|
520
|
+
obj = obj[:start] + str(value) + obj[end + 2:]
|
|
521
|
+
startIndex = start + len(str(value))
|
|
522
|
+
if startIndex >= len(obj):
|
|
523
|
+
break
|
|
524
|
+
return obj
|
|
525
|
+
else:
|
|
526
|
+
return obj
|
|
527
|
+
|
|
@@ -1,28 +1,35 @@
|
|
|
1
1
|
# pylint: disable=protected-access
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
2
4
|
from contextlib import contextmanager
|
|
3
5
|
import os
|
|
4
|
-
from typing import AsyncGenerator, Iterator
|
|
6
|
+
from typing import AsyncGenerator, Iterator
|
|
5
7
|
import logging
|
|
6
8
|
from opentelemetry.trace import Tracer
|
|
7
|
-
from opentelemetry.trace.propagation import
|
|
8
|
-
from opentelemetry.trace import propagation
|
|
9
|
+
from opentelemetry.trace.propagation import set_span_in_context, get_current_span
|
|
9
10
|
from opentelemetry.context import set_value, attach, detach, get_value
|
|
10
|
-
from opentelemetry.context import create_key, get_value, set_value
|
|
11
|
-
from opentelemetry.context.context import Context
|
|
12
11
|
from opentelemetry.trace.span import INVALID_SPAN, Span
|
|
13
12
|
from opentelemetry.trace.status import StatusCode
|
|
14
13
|
|
|
14
|
+
from monocle_apptrace.instrumentation.common.constants import (
|
|
15
|
+
ADD_NEW_WORKFLOW,
|
|
16
|
+
AGENTIC_SPANS,
|
|
17
|
+
WORKFLOW_TYPE_KEY,
|
|
18
|
+
)
|
|
19
|
+
from monocle_apptrace.instrumentation.common.scope_wrapper import monocle_trace_scope
|
|
15
20
|
from monocle_apptrace.instrumentation.common.span_handler import SpanHandler
|
|
16
21
|
from monocle_apptrace.instrumentation.common.utils import (
|
|
22
|
+
get_current_monocle_span,
|
|
23
|
+
remove_scope,
|
|
24
|
+
set_monocle_span_in_context,
|
|
25
|
+
set_scope,
|
|
17
26
|
set_scopes,
|
|
18
27
|
with_tracer_wrapper,
|
|
19
28
|
set_scope,
|
|
20
29
|
remove_scope,
|
|
21
30
|
get_current_monocle_span,
|
|
22
|
-
set_monocle_span_in_context
|
|
31
|
+
set_monocle_span_in_context,
|
|
23
32
|
)
|
|
24
|
-
from monocle_apptrace.instrumentation.common.constants import WORKFLOW_TYPE_KEY, ADD_NEW_WORKFLOW, AGENTIC_SPANS
|
|
25
|
-
from monocle_apptrace.instrumentation.common.scope_wrapper import monocle_trace_scope
|
|
26
33
|
|
|
27
34
|
logger = logging.getLogger(__name__)
|
|
28
35
|
ISOLATE_MONOCLE_SPANS = os.getenv("MONOCLE_ISOLATE_SPANS", "true").lower() == "true"
|
|
@@ -53,7 +60,9 @@ def post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, return_
|
|
|
53
60
|
try:
|
|
54
61
|
if parent_span == INVALID_SPAN:
|
|
55
62
|
parent_span = None
|
|
56
|
-
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, return_value, span, parent_span, ex
|
|
63
|
+
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, return_value, span, parent_span, ex,
|
|
64
|
+
is_post_exec=True)
|
|
65
|
+
return_value = SpanHandler.replace_placeholders_in_response(return_value, span)
|
|
57
66
|
except Exception as e:
|
|
58
67
|
logger.info(f"Warning: Error occurred in hydrate_span: {e}")
|
|
59
68
|
|
|
@@ -61,6 +70,7 @@ def post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, return_
|
|
|
61
70
|
handler.post_task_processing(to_wrap, wrapped, instance, args, kwargs, return_value, ex, span, parent_span)
|
|
62
71
|
except Exception as e:
|
|
63
72
|
logger.info(f"Warning: Error occurred in post_task_processing: {e}")
|
|
73
|
+
return return_value
|
|
64
74
|
|
|
65
75
|
def get_span_name(to_wrap, instance):
|
|
66
76
|
if to_wrap.get("span_name"):
|
|
@@ -87,19 +97,32 @@ def monocle_wrapper_span_processor(tracer: Tracer, handler: SpanHandler, to_wrap
|
|
|
87
97
|
span.end()
|
|
88
98
|
else:
|
|
89
99
|
ex:Exception = None
|
|
90
|
-
to_wrap = get_wrapper_with_next_processor(to_wrap, handler, instance, args, kwargs)
|
|
100
|
+
to_wrap = get_wrapper_with_next_processor(to_wrap, handler, instance, span, parent_span, args, kwargs)
|
|
91
101
|
if has_more_processors(to_wrap):
|
|
92
102
|
try:
|
|
93
|
-
|
|
103
|
+
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, None, span, parent_span, ex,
|
|
104
|
+
is_post_exec=False)
|
|
105
|
+
except Exception as e:
|
|
106
|
+
logger.info(f"Warning: Error occurred in hydrate_span pre_process_span: {e}")
|
|
107
|
+
try:
|
|
108
|
+
with monocle_trace_scope(get_builtin_scope_names(to_wrap)):
|
|
109
|
+
return_value, span_status = monocle_wrapper_span_processor(tracer, handler, to_wrap, wrapped, instance, source_path, False, args, kwargs)
|
|
94
110
|
except Exception as e:
|
|
95
111
|
ex = e
|
|
96
112
|
raise
|
|
97
113
|
finally:
|
|
98
|
-
|
|
114
|
+
return_value = post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, return_value, span, parent_span ,ex)
|
|
99
115
|
else:
|
|
100
116
|
try:
|
|
101
|
-
|
|
102
|
-
|
|
117
|
+
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, None, span, parent_span, ex,
|
|
118
|
+
is_post_exec=False)
|
|
119
|
+
except Exception as e:
|
|
120
|
+
logger.info(f"Warning: Error occurred in hydrate_span pre_process_span: {e}")
|
|
121
|
+
try:
|
|
122
|
+
skip_execution, return_value = SpanHandler.skip_execution(span)
|
|
123
|
+
if not skip_execution:
|
|
124
|
+
with SpanHandler.workflow_type(to_wrap, span):
|
|
125
|
+
return_value = wrapped(*args, **kwargs)
|
|
103
126
|
except Exception as e:
|
|
104
127
|
ex = e
|
|
105
128
|
raise
|
|
@@ -108,10 +131,11 @@ def monocle_wrapper_span_processor(tracer: Tracer, handler: SpanHandler, to_wrap
|
|
|
108
131
|
post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, ret_val, span, parent_span ,ex)
|
|
109
132
|
if not auto_close_span:
|
|
110
133
|
span.end()
|
|
134
|
+
return ret_val
|
|
111
135
|
if ex is None and not auto_close_span and to_wrap.get("output_processor") and to_wrap.get("output_processor").get("response_processor"):
|
|
112
136
|
to_wrap.get("output_processor").get("response_processor")(to_wrap, return_value, post_process_span_internal)
|
|
113
137
|
else:
|
|
114
|
-
post_process_span_internal(return_value)
|
|
138
|
+
return_value = post_process_span_internal(return_value)
|
|
115
139
|
span_status = span.status
|
|
116
140
|
return return_value, span_status
|
|
117
141
|
|
|
@@ -121,7 +145,9 @@ def monocle_wrapper(tracer: Tracer, handler: SpanHandler, to_wrap, wrapped, inst
|
|
|
121
145
|
token = None
|
|
122
146
|
try:
|
|
123
147
|
try:
|
|
124
|
-
pre_trace_token = handler.pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
148
|
+
pre_trace_token, alternate_to_wrapp = handler.pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
149
|
+
if alternate_to_wrapp is not None:
|
|
150
|
+
to_wrap = alternate_to_wrapp
|
|
125
151
|
except Exception as e:
|
|
126
152
|
logger.info(f"Warning: Error occurred in pre_tracing: {e}")
|
|
127
153
|
if to_wrap.get('skip_span', False) or handler.skip_span(to_wrap, wrapped, instance, args, kwargs):
|
|
@@ -160,31 +186,46 @@ async def amonocle_wrapper_span_processor(tracer: Tracer, handler: SpanHandler,
|
|
|
160
186
|
span.end()
|
|
161
187
|
else:
|
|
162
188
|
ex:Exception = None
|
|
163
|
-
to_wrap = get_wrapper_with_next_processor(to_wrap, handler, instance, args, kwargs)
|
|
189
|
+
to_wrap = get_wrapper_with_next_processor(to_wrap, handler, instance, span, parent_span,args, kwargs)
|
|
164
190
|
if has_more_processors(to_wrap):
|
|
165
191
|
try:
|
|
166
|
-
|
|
192
|
+
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, None, span, parent_span, ex,
|
|
193
|
+
is_post_exec=False)
|
|
194
|
+
except Exception as e:
|
|
195
|
+
logger.info(f"Warning: Error occurred in hydrate_span pre_process_span: {e}")
|
|
196
|
+
|
|
197
|
+
try:
|
|
198
|
+
with monocle_trace_scope(get_builtin_scope_names(to_wrap)):
|
|
199
|
+
return_value, span_status = await amonocle_wrapper_span_processor(tracer, handler, to_wrap, wrapped, instance, source_path, False, args, kwargs)
|
|
167
200
|
except Exception as e:
|
|
168
201
|
ex = e
|
|
169
202
|
raise
|
|
170
203
|
finally:
|
|
171
|
-
|
|
204
|
+
return_value = post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, return_value, span, parent_span ,ex)
|
|
172
205
|
else:
|
|
173
206
|
try:
|
|
174
|
-
|
|
175
|
-
|
|
207
|
+
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, None, span, parent_span, ex,
|
|
208
|
+
is_post_exec=False)
|
|
209
|
+
except Exception as e:
|
|
210
|
+
logger.info(f"Warning: Error occurred in hydrate_span pre_process_span: {e}")
|
|
211
|
+
try:
|
|
212
|
+
skip_execution, return_value = SpanHandler.skip_execution(span)
|
|
213
|
+
if not skip_execution:
|
|
214
|
+
with SpanHandler.workflow_type(to_wrap, span):
|
|
215
|
+
return_value = await wrapped(*args, **kwargs)
|
|
176
216
|
except Exception as e:
|
|
177
217
|
ex = e
|
|
178
218
|
raise
|
|
179
219
|
finally:
|
|
180
220
|
def post_process_span_internal(ret_val):
|
|
181
|
-
post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, ret_val, span, parent_span, ex)
|
|
221
|
+
ret_val = post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, ret_val, span, parent_span, ex)
|
|
182
222
|
if not auto_close_span:
|
|
183
223
|
span.end()
|
|
224
|
+
return ret_val
|
|
184
225
|
if ex is None and not auto_close_span and to_wrap.get("output_processor") and to_wrap.get("output_processor").get("response_processor"):
|
|
185
226
|
to_wrap.get("output_processor").get("response_processor")(to_wrap, return_value, post_process_span_internal)
|
|
186
227
|
else:
|
|
187
|
-
post_process_span_internal(return_value)
|
|
228
|
+
return_value = post_process_span_internal(return_value)
|
|
188
229
|
span_status = span.status
|
|
189
230
|
return return_value, span_status
|
|
190
231
|
|
|
@@ -208,35 +249,50 @@ async def amonocle_iter_wrapper_span_processor(tracer: Tracer, handler: SpanHand
|
|
|
208
249
|
span.end()
|
|
209
250
|
else:
|
|
210
251
|
ex:Exception = None
|
|
211
|
-
to_wrap = get_wrapper_with_next_processor(to_wrap, handler, instance, args, kwargs)
|
|
252
|
+
to_wrap = get_wrapper_with_next_processor(to_wrap, handler, span, parent_span, instance, args, kwargs)
|
|
212
253
|
if has_more_processors(to_wrap):
|
|
213
254
|
try:
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
255
|
+
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, None, span, parent_span, ex,
|
|
256
|
+
is_post_exec=False)
|
|
257
|
+
except Exception as e:
|
|
258
|
+
logger.info(f"Warning: Error occurred in hydrate_span pre_process_span: {e}")
|
|
259
|
+
try:
|
|
260
|
+
with monocle_trace_scope(get_builtin_scope_names(to_wrap)):
|
|
261
|
+
async for item in amonocle_iter_wrapper_span_processor(tracer, handler, to_wrap, wrapped, instance, source_path, False, args, kwargs):
|
|
262
|
+
last_item = item
|
|
263
|
+
yield item
|
|
217
264
|
except Exception as e:
|
|
218
265
|
ex = e
|
|
219
266
|
raise
|
|
220
267
|
finally:
|
|
221
|
-
post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, last_item, span, parent_span, ex)
|
|
268
|
+
last_item = post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, last_item, span, parent_span, ex)
|
|
222
269
|
else:
|
|
223
270
|
try:
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
271
|
+
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, None, span, parent_span, ex,
|
|
272
|
+
is_post_exec=False)
|
|
273
|
+
except Exception as e:
|
|
274
|
+
logger.info(f"Warning: Error occurred in hydrate_span pre_process_span: {e}")
|
|
275
|
+
try:
|
|
276
|
+
skip_execution, last_item = SpanHandler.skip_execution(span)
|
|
277
|
+
if not skip_execution:
|
|
278
|
+
with SpanHandler.workflow_type(to_wrap, span):
|
|
279
|
+
async for item in wrapped(*args, **kwargs):
|
|
280
|
+
last_item = item
|
|
281
|
+
yield item
|
|
282
|
+
else:
|
|
283
|
+
yield last_item
|
|
228
284
|
except Exception as e:
|
|
229
285
|
ex = e
|
|
230
286
|
raise
|
|
231
287
|
finally:
|
|
232
288
|
def post_process_span_internal(ret_val):
|
|
233
|
-
post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, ret_val, span, parent_span, ex)
|
|
289
|
+
ret_val = post_process_span(handler, to_wrap, wrapped, instance, args, kwargs, ret_val, span, parent_span, ex)
|
|
234
290
|
if not auto_close_span:
|
|
235
291
|
span.end()
|
|
236
292
|
if ex is None and not auto_close_span and to_wrap.get("output_processor") and to_wrap.get("output_processor").get("response_processor"):
|
|
237
293
|
to_wrap.get("output_processor").get("response_processor")(to_wrap, None, post_process_span_internal)
|
|
238
294
|
else:
|
|
239
|
-
post_process_span_internal(last_item)
|
|
295
|
+
last_item = post_process_span_internal(last_item)
|
|
240
296
|
return
|
|
241
297
|
|
|
242
298
|
async def amonocle_wrapper(tracer: Tracer, handler: SpanHandler, to_wrap, wrapped, instance, source_path, args, kwargs):
|
|
@@ -245,7 +301,9 @@ async def amonocle_wrapper(tracer: Tracer, handler: SpanHandler, to_wrap, wrappe
|
|
|
245
301
|
pre_trace_token = None
|
|
246
302
|
try:
|
|
247
303
|
try:
|
|
248
|
-
pre_trace_token = handler.pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
304
|
+
pre_trace_token, alternate_to_wrapp = handler.pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
305
|
+
if alternate_to_wrapp is not None:
|
|
306
|
+
to_wrap = alternate_to_wrapp
|
|
249
307
|
except Exception as e:
|
|
250
308
|
logger.info(f"Warning: Error occurred in pre_tracing: {e}")
|
|
251
309
|
if to_wrap.get('skip_span', False) or handler.skip_span(to_wrap, wrapped, instance, args, kwargs):
|
|
@@ -271,7 +329,9 @@ async def amonocle_iter_wrapper(tracer: Tracer, handler: SpanHandler, to_wrap, w
|
|
|
271
329
|
pre_trace_token = None
|
|
272
330
|
try:
|
|
273
331
|
try:
|
|
274
|
-
pre_trace_token = handler.pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
332
|
+
pre_trace_token, alternate_to_wrapp = handler.pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
333
|
+
if alternate_to_wrapp is not None:
|
|
334
|
+
to_wrap = alternate_to_wrapp
|
|
275
335
|
except Exception as e:
|
|
276
336
|
logger.info(f"Warning: Error occurred in pre_tracing: {e}")
|
|
277
337
|
if to_wrap.get('skip_span', False) or handler.skip_span(to_wrap, wrapped, instance, args, kwargs):
|
|
@@ -373,13 +433,14 @@ def evaluate_scope_values(args, kwargs, to_wrap, scope_values):
|
|
|
373
433
|
@contextmanager
|
|
374
434
|
def start_as_monocle_span(tracer: Tracer, name: str, auto_close_span: bool) -> Iterator["Span"]:
|
|
375
435
|
""" Wrapper to OTEL start_as_current_span to isolate monocle and non monocle spans.
|
|
376
|
-
This
|
|
436
|
+
This essentially links monocle and non-monocle spans separately which is default behavior.
|
|
377
437
|
It can be optionally overridden by setting the environment variable MONOCLE_ISOLATE_SPANS to false.
|
|
378
438
|
"""
|
|
379
439
|
if not ISOLATE_MONOCLE_SPANS:
|
|
380
440
|
# If not isolating, use the default start_as_current_span
|
|
381
441
|
yield tracer.start_as_current_span(name, end_on_exit=auto_close_span)
|
|
382
442
|
return
|
|
443
|
+
|
|
383
444
|
original_span = get_current_span()
|
|
384
445
|
monocle_span_token = attach(set_span_in_context(get_current_monocle_span()))
|
|
385
446
|
with tracer.start_as_current_span(name, end_on_exit=auto_close_span) as span:
|
|
@@ -391,18 +452,26 @@ def start_as_monocle_span(tracer: Tracer, name: str, auto_close_span: bool) -> I
|
|
|
391
452
|
detach(monocle_span_token)
|
|
392
453
|
|
|
393
454
|
def get_builtin_scope_names(to_wrap) -> str:
|
|
394
|
-
output_processor =
|
|
395
|
-
|
|
455
|
+
output_processor = None
|
|
456
|
+
if "output_processor" in to_wrap:
|
|
457
|
+
output_processor = to_wrap.get("output_processor", None)
|
|
458
|
+
if "output_processor_list" in to_wrap:
|
|
459
|
+
for processor in to_wrap["output_processor_list"]:
|
|
460
|
+
if processor.get("type", None) in AGENTIC_SPANS:
|
|
461
|
+
output_processor = processor
|
|
462
|
+
break
|
|
463
|
+
|
|
464
|
+
span_type = output_processor.get("type", None) if output_processor and isinstance(output_processor, dict) else None
|
|
396
465
|
if span_type and span_type in AGENTIC_SPANS:
|
|
397
466
|
return span_type
|
|
398
467
|
return None
|
|
399
468
|
|
|
400
|
-
def get_wrapper_with_next_processor(to_wrap, handler, instance, args, kwargs):
|
|
469
|
+
def get_wrapper_with_next_processor(to_wrap, handler, instance, span, parent_span, args, kwargs):
|
|
401
470
|
if has_more_processors(to_wrap):
|
|
402
471
|
next_output_processor_list = to_wrap.get('output_processor_list',[]).copy()
|
|
403
472
|
while len(next_output_processor_list) > 0:
|
|
404
473
|
next_output_processor = next_output_processor_list.pop(0)
|
|
405
|
-
if handler.should_skip(next_output_processor, instance, args, kwargs):
|
|
474
|
+
if handler.should_skip(next_output_processor, instance, span, parent_span, args, kwargs):
|
|
406
475
|
next_output_processor = None
|
|
407
476
|
else:
|
|
408
477
|
break
|
|
@@ -412,4 +481,4 @@ def get_wrapper_with_next_processor(to_wrap, handler, instance, args, kwargs):
|
|
|
412
481
|
return to_wrap
|
|
413
482
|
|
|
414
483
|
def has_more_processors(to_wrap) -> bool:
|
|
415
|
-
return len(to_wrap.get('output_processor_list', [])) > 0
|
|
484
|
+
return len(to_wrap.get('output_processor_list', [])) > 0
|
|
@@ -39,6 +39,7 @@ from monocle_apptrace.instrumentation.metamodel.a2a.methods import A2A_CLIENT_ME
|
|
|
39
39
|
from monocle_apptrace.instrumentation.metamodel.litellm.methods import LITELLM_METHODS
|
|
40
40
|
from monocle_apptrace.instrumentation.metamodel.adk.methods import ADK_METHODS
|
|
41
41
|
from monocle_apptrace.instrumentation.metamodel.mistral.methods import MISTRAL_METHODS
|
|
42
|
+
from monocle_apptrace.instrumentation.metamodel.strands.methods import STRAND_METHODS
|
|
42
43
|
|
|
43
44
|
class WrapperMethod:
|
|
44
45
|
def __init__(
|
|
@@ -111,7 +112,8 @@ DEFAULT_METHODS_LIST = (
|
|
|
111
112
|
LITELLM_METHODS +
|
|
112
113
|
ADK_METHODS +
|
|
113
114
|
MISTRAL_METHODS +
|
|
114
|
-
HUGGING_FACE_METHODS
|
|
115
|
+
HUGGING_FACE_METHODS +
|
|
116
|
+
STRAND_METHODS
|
|
115
117
|
)
|
|
116
118
|
|
|
117
119
|
MONOCLE_SPAN_HANDLERS: Dict[str, SpanHandler] = {
|
|
@@ -133,5 +135,5 @@ MONOCLE_SPAN_HANDLERS: Dict[str, SpanHandler] = {
|
|
|
133
135
|
"llamaindex_tool_handler": LlamaIndexToolHandler(),
|
|
134
136
|
"llamaindex_agent_handler": LlamaIndexAgentHandler(),
|
|
135
137
|
"llamaindex_single_agent_tool_handler": LlamaIndexSingleAgenttToolHandlerWrapper(),
|
|
136
|
-
"lambda_func_handler": lambdaSpanHandler()
|
|
138
|
+
"lambda_func_handler": lambdaSpanHandler()
|
|
137
139
|
}
|