monocle-apptrace 0.3.1b1__py3-none-any.whl → 0.4.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of monocle-apptrace might be problematic. Click here for more details.
- monocle_apptrace/exporters/aws/s3_exporter.py +3 -1
- monocle_apptrace/exporters/azure/blob_exporter.py +2 -2
- monocle_apptrace/exporters/base_exporter.py +10 -4
- monocle_apptrace/exporters/file_exporter.py +19 -4
- monocle_apptrace/exporters/monocle_exporters.py +8 -5
- monocle_apptrace/exporters/okahu/okahu_exporter.py +5 -2
- monocle_apptrace/instrumentation/common/__init__.py +1 -1
- monocle_apptrace/instrumentation/common/constants.py +8 -1
- monocle_apptrace/instrumentation/common/instrumentor.py +44 -22
- monocle_apptrace/instrumentation/common/span_handler.py +67 -41
- monocle_apptrace/instrumentation/common/tracing.md +68 -0
- monocle_apptrace/instrumentation/common/utils.py +86 -63
- monocle_apptrace/instrumentation/common/wrapper.py +185 -46
- monocle_apptrace/instrumentation/common/wrapper_method.py +12 -6
- monocle_apptrace/instrumentation/metamodel/aiohttp/__init__.py +0 -0
- monocle_apptrace/instrumentation/metamodel/aiohttp/_helper.py +66 -0
- monocle_apptrace/instrumentation/metamodel/aiohttp/entities/http.py +51 -0
- monocle_apptrace/instrumentation/metamodel/aiohttp/methods.py +13 -0
- monocle_apptrace/instrumentation/metamodel/anthropic/methods.py +4 -2
- monocle_apptrace/instrumentation/metamodel/flask/_helper.py +50 -3
- monocle_apptrace/instrumentation/metamodel/flask/entities/http.py +48 -0
- monocle_apptrace/instrumentation/metamodel/flask/methods.py +10 -1
- monocle_apptrace/instrumentation/metamodel/haystack/_helper.py +17 -4
- monocle_apptrace/instrumentation/metamodel/haystack/entities/inference.py +4 -1
- monocle_apptrace/instrumentation/metamodel/haystack/methods.py +8 -4
- monocle_apptrace/instrumentation/metamodel/langchain/_helper.py +12 -4
- monocle_apptrace/instrumentation/metamodel/langchain/methods.py +6 -14
- monocle_apptrace/instrumentation/metamodel/llamaindex/_helper.py +13 -9
- monocle_apptrace/instrumentation/metamodel/llamaindex/methods.py +16 -15
- monocle_apptrace/instrumentation/metamodel/openai/entities/inference.py +174 -26
- monocle_apptrace/instrumentation/metamodel/openai/methods.py +0 -2
- monocle_apptrace/instrumentation/metamodel/requests/_helper.py +31 -0
- monocle_apptrace/instrumentation/metamodel/requests/entities/http.py +51 -0
- monocle_apptrace/instrumentation/metamodel/requests/methods.py +2 -1
- monocle_apptrace/instrumentation/metamodel/teamsai/_helper.py +19 -1
- monocle_apptrace/instrumentation/metamodel/teamsai/entities/inference/actionplanner_output_processor.py +1 -1
- monocle_apptrace/instrumentation/metamodel/teamsai/entities/inference/teamsai_output_processor.py +24 -18
- monocle_apptrace/instrumentation/metamodel/teamsai/methods.py +42 -8
- {monocle_apptrace-0.3.1b1.dist-info → monocle_apptrace-0.4.0b1.dist-info}/METADATA +1 -1
- {monocle_apptrace-0.3.1b1.dist-info → monocle_apptrace-0.4.0b1.dist-info}/RECORD +43 -36
- {monocle_apptrace-0.3.1b1.dist-info → monocle_apptrace-0.4.0b1.dist-info}/WHEEL +0 -0
- {monocle_apptrace-0.3.1b1.dist-info → monocle_apptrace-0.4.0b1.dist-info}/licenses/LICENSE +0 -0
- {monocle_apptrace-0.3.1b1.dist-info → monocle_apptrace-0.4.0b1.dist-info}/licenses/NOTICE +0 -0
|
@@ -129,6 +129,8 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
129
129
|
# Serialize spans to JSON or any other format you prefer
|
|
130
130
|
valid_json_list = []
|
|
131
131
|
for span in spans:
|
|
132
|
+
if self.skip_export(span):
|
|
133
|
+
continue
|
|
132
134
|
try:
|
|
133
135
|
valid_json_list.append(span.to_json(indent=0).replace("\n", ""))
|
|
134
136
|
except json.JSONDecodeError as e:
|
|
@@ -169,7 +171,7 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
169
171
|
Key=file_name,
|
|
170
172
|
Body=span_data_batch
|
|
171
173
|
)
|
|
172
|
-
logger.
|
|
174
|
+
logger.debug(f"Span batch uploaded to AWS S3 as {file_name}.")
|
|
173
175
|
|
|
174
176
|
async def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
175
177
|
await self.__export_spans() # Export any remaining spans in the queue
|
|
@@ -82,8 +82,8 @@ class AzureBlobSpanExporter(SpanExporterBase):
|
|
|
82
82
|
# With Monocle,OpenTelemetry is always loaded. If the Azure trace package is installed, then it triggers the blob trace generation on every blob operation.
|
|
83
83
|
# Thus, the Monocle span write ends up generating a blob span which again comes back to the exporter .. and would result in an infinite loop.
|
|
84
84
|
# To avoid this, we check if the span has the Monocle SDK version attribute and skip it if it doesn't. That way the blob span genearted by Azure library are not exported.
|
|
85
|
-
if
|
|
86
|
-
continue
|
|
85
|
+
if self.skip_export(span):
|
|
86
|
+
continue
|
|
87
87
|
self.export_queue.append(span)
|
|
88
88
|
if len(self.export_queue) >= self.max_batch_size:
|
|
89
89
|
await self.__export_spans()
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import time
|
|
1
|
+
import time, os
|
|
2
2
|
import random
|
|
3
3
|
import logging
|
|
4
4
|
from abc import ABC, abstractmethod
|
|
5
5
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
6
|
-
from opentelemetry.sdk.trace.export import
|
|
6
|
+
from opentelemetry.sdk.trace.export import SpanExportResult
|
|
7
|
+
from monocle_apptrace.instrumentation.common.constants import MONOCLE_SDK_VERSION
|
|
7
8
|
from typing import Sequence
|
|
8
|
-
import asyncio
|
|
9
9
|
|
|
10
10
|
logger = logging.getLogger(__name__)
|
|
11
11
|
|
|
12
12
|
class SpanExporterBase(ABC):
|
|
13
|
-
def __init__(self):
|
|
13
|
+
def __init__(self, export_monocle_only: bool = True):
|
|
14
14
|
self.backoff_factor = 2
|
|
15
15
|
self.max_retries = 10
|
|
16
16
|
self.export_queue = []
|
|
17
17
|
self.last_export_time = time.time()
|
|
18
|
+
self.export_monocle_only = export_monocle_only or os.environ.get("MONOCLE_EXPORTS_ONLY", True)
|
|
18
19
|
|
|
19
20
|
@abstractmethod
|
|
20
21
|
async def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
@@ -27,6 +28,11 @@ class SpanExporterBase(ABC):
|
|
|
27
28
|
def shutdown(self) -> None:
|
|
28
29
|
pass
|
|
29
30
|
|
|
31
|
+
def skip_export(self, span:ReadableSpan) -> bool:
|
|
32
|
+
if self.export_monocle_only and (not span.attributes.get(MONOCLE_SDK_VERSION)):
|
|
33
|
+
return True
|
|
34
|
+
return False
|
|
35
|
+
|
|
30
36
|
@staticmethod
|
|
31
37
|
def retry_with_backoff(retries=3, backoff_in_seconds=1, max_backoff_in_seconds=32, exceptions=(Exception,)):
|
|
32
38
|
def decorator(func):
|
|
@@ -7,12 +7,13 @@ from typing import Optional, Callable, Sequence
|
|
|
7
7
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
8
8
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
9
9
|
from opentelemetry.sdk.resources import SERVICE_NAME
|
|
10
|
+
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
10
11
|
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
11
12
|
|
|
12
13
|
DEFAULT_FILE_PREFIX:str = "monocle_trace_"
|
|
13
14
|
DEFAULT_TIME_FORMAT:str = "%Y-%m-%d_%H.%M.%S"
|
|
14
15
|
|
|
15
|
-
class FileSpanExporter(
|
|
16
|
+
class FileSpanExporter(SpanExporterBase):
|
|
16
17
|
current_trace_id: int = None
|
|
17
18
|
current_file_path: str = None
|
|
18
19
|
|
|
@@ -28,6 +29,7 @@ class FileSpanExporter(SpanExporter):
|
|
|
28
29
|
+ linesep,
|
|
29
30
|
task_processor: Optional[ExportTaskProcessor] = None
|
|
30
31
|
):
|
|
32
|
+
super().__init__()
|
|
31
33
|
self.out_handle:TextIOWrapper = None
|
|
32
34
|
self.formatter = formatter
|
|
33
35
|
self.service_name = service_name
|
|
@@ -39,21 +41,32 @@ class FileSpanExporter(SpanExporter):
|
|
|
39
41
|
self.task_processor.start()
|
|
40
42
|
|
|
41
43
|
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
44
|
+
is_root_span = any(not span.parent for span in spans)
|
|
42
45
|
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
43
46
|
# Check if any span is a root span (no parent)
|
|
44
|
-
is_root_span = any(not span.parent for span in spans)
|
|
45
47
|
self.task_processor.queue_task(self._process_spans, spans, is_root_span)
|
|
46
48
|
return SpanExportResult.SUCCESS
|
|
47
49
|
else:
|
|
48
|
-
return self._process_spans(spans)
|
|
50
|
+
return self._process_spans(spans, is_root_span=is_root_span)
|
|
49
51
|
|
|
50
52
|
def _process_spans(self, spans: Sequence[ReadableSpan], is_root_span: bool = False) -> SpanExportResult:
|
|
53
|
+
first_span:bool = True
|
|
51
54
|
for span in spans:
|
|
55
|
+
if self.skip_export(span):
|
|
56
|
+
continue
|
|
52
57
|
if span.context.trace_id != self.current_trace_id:
|
|
53
58
|
self.rotate_file(span.resource.attributes[SERVICE_NAME],
|
|
54
59
|
span.context.trace_id)
|
|
60
|
+
first_span = True
|
|
61
|
+
if first_span:
|
|
62
|
+
first_span = False
|
|
63
|
+
else:
|
|
64
|
+
self.out_handle.write(",")
|
|
55
65
|
self.out_handle.write(self.formatter(span))
|
|
56
|
-
|
|
66
|
+
if is_root_span:
|
|
67
|
+
self.reset_handle()
|
|
68
|
+
else:
|
|
69
|
+
self.out_handle.flush()
|
|
57
70
|
return SpanExportResult.SUCCESS
|
|
58
71
|
|
|
59
72
|
def rotate_file(self, trace_name:str, trace_id:int) -> None:
|
|
@@ -62,6 +75,7 @@ class FileSpanExporter(SpanExporter):
|
|
|
62
75
|
self.file_prefix + trace_name + "_" + hex(trace_id) + "_"
|
|
63
76
|
+ datetime.now().strftime(self.time_format) + ".json")
|
|
64
77
|
self.out_handle = open(self.current_file_path, "w", encoding='UTF-8')
|
|
78
|
+
self.out_handle.write("[")
|
|
65
79
|
self.current_trace_id = trace_id
|
|
66
80
|
|
|
67
81
|
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
@@ -70,6 +84,7 @@ class FileSpanExporter(SpanExporter):
|
|
|
70
84
|
|
|
71
85
|
def reset_handle(self) -> None:
|
|
72
86
|
if self.out_handle is not None:
|
|
87
|
+
self.out_handle.write("]")
|
|
73
88
|
self.out_handle.close()
|
|
74
89
|
self.out_handle = None
|
|
75
90
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from typing import Dict, Any, List
|
|
2
2
|
import os
|
|
3
|
-
import logging
|
|
3
|
+
import logging, warnings
|
|
4
4
|
from importlib import import_module
|
|
5
5
|
from opentelemetry.sdk.trace.export import SpanExporter, ConsoleSpanExporter
|
|
6
6
|
from monocle_apptrace.exporters.exporter_processor import LambdaExportTaskProcessor, is_aws_lambda_environment
|
|
@@ -18,9 +18,12 @@ monocle_exporters: Dict[str, Any] = {
|
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
def get_monocle_exporter() -> List[SpanExporter]:
|
|
21
|
+
def get_monocle_exporter(exporters_list:str=None) -> List[SpanExporter]:
|
|
22
22
|
# Retrieve the MONOCLE_EXPORTER environment variable and split it into a list
|
|
23
|
-
|
|
23
|
+
if exporters_list:
|
|
24
|
+
exporter_names = exporters_list.split(",")
|
|
25
|
+
else:
|
|
26
|
+
exporter_names = os.environ.get("MONOCLE_EXPORTER", "file").split(",")
|
|
24
27
|
exporters = []
|
|
25
28
|
|
|
26
29
|
# Create task processor for AWS Lambda environment
|
|
@@ -31,7 +34,7 @@ def get_monocle_exporter() -> List[SpanExporter]:
|
|
|
31
34
|
try:
|
|
32
35
|
exporter_class_path = monocle_exporters[exporter_name]
|
|
33
36
|
except KeyError:
|
|
34
|
-
|
|
37
|
+
warnings.warn(f"Unsupported Monocle span exporter '{exporter_name}', skipping.")
|
|
35
38
|
continue
|
|
36
39
|
try:
|
|
37
40
|
exporter_module = import_module(exporter_class_path["module"])
|
|
@@ -42,7 +45,7 @@ def get_monocle_exporter() -> List[SpanExporter]:
|
|
|
42
45
|
else:
|
|
43
46
|
exporters.append(exporter_class())
|
|
44
47
|
except Exception as ex:
|
|
45
|
-
|
|
48
|
+
warnings.warn(
|
|
46
49
|
f"Unable to initialize Monocle span exporter '{exporter_name}', error: {ex}. Using ConsoleSpanExporter as a fallback.")
|
|
47
50
|
exporters.append(ConsoleSpanExporter())
|
|
48
51
|
continue
|
|
@@ -6,7 +6,7 @@ import requests
|
|
|
6
6
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
7
7
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult, ConsoleSpanExporter
|
|
8
8
|
from requests.exceptions import ReadTimeout
|
|
9
|
-
|
|
9
|
+
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
10
10
|
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
11
11
|
|
|
12
12
|
REQUESTS_SUCCESS_STATUS_CODES = (200, 202)
|
|
@@ -15,7 +15,7 @@ OKAHU_PROD_INGEST_ENDPOINT = "https://ingest.okahu.co/api/v1/trace/ingest"
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
class OkahuSpanExporter(
|
|
18
|
+
class OkahuSpanExporter(SpanExporterBase):
|
|
19
19
|
def __init__(
|
|
20
20
|
self,
|
|
21
21
|
endpoint: Optional[str] = None,
|
|
@@ -24,6 +24,7 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
24
24
|
task_processor: ExportTaskProcessor = None
|
|
25
25
|
):
|
|
26
26
|
"""Okahu exporter."""
|
|
27
|
+
super().__init__()
|
|
27
28
|
okahu_endpoint: str = os.environ.get("OKAHU_INGESTION_ENDPOINT", OKAHU_PROD_INGEST_ENDPOINT)
|
|
28
29
|
self.endpoint = endpoint or okahu_endpoint
|
|
29
30
|
api_key: str = os.environ.get("OKAHU_API_KEY")
|
|
@@ -58,6 +59,8 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
58
59
|
|
|
59
60
|
# append the batch object with all the spans object
|
|
60
61
|
for span in spans:
|
|
62
|
+
if self.skip_export(span):
|
|
63
|
+
continue
|
|
61
64
|
# create a object from serialized span
|
|
62
65
|
obj = json.loads(span.to_json())
|
|
63
66
|
if obj["parent_id"] is None:
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
from .instrumentor import setup_monocle_telemetry, start_trace, stop_trace, start_scope, stop_scope, http_route_handler, monocle_trace_scope, monocle_trace_scope_method, monocle_trace
|
|
2
|
-
from .utils import MonocleSpanException
|
|
2
|
+
from .utils import MonocleSpanException
|
|
@@ -51,6 +51,9 @@ llm_type_map = {
|
|
|
51
51
|
"openaigenerator": "openai",
|
|
52
52
|
"bedrockruntime":"aws_bedrock",
|
|
53
53
|
"sagemakerruntime":"aws_sagemaker",
|
|
54
|
+
"anthropic": "anthropic",
|
|
55
|
+
"chatanthropic":"anthropic",
|
|
56
|
+
"anthropicchatgenerator":"anthropic",
|
|
54
57
|
}
|
|
55
58
|
|
|
56
59
|
MONOCLE_INSTRUMENTOR = "monocle_apptrace"
|
|
@@ -70,5 +73,9 @@ SCOPE_METHOD_FILE = 'monocle_scopes.json'
|
|
|
70
73
|
SCOPE_CONFIG_PATH = 'MONOCLE_SCOPE_CONFIG_PATH'
|
|
71
74
|
TRACE_PROPOGATION_URLS = "MONOCLE_TRACE_PROPAGATATION_URLS"
|
|
72
75
|
WORKFLOW_TYPE_KEY = "monocle.workflow_type"
|
|
76
|
+
ADD_NEW_WORKFLOW = "monocle.add_new_workflow"
|
|
73
77
|
WORKFLOW_TYPE_GENERIC = "workflow.generic"
|
|
74
|
-
MONOCLE_SDK_VERSION = "monocle_apptrace.version"
|
|
78
|
+
MONOCLE_SDK_VERSION = "monocle_apptrace.version"
|
|
79
|
+
MONOCLE_SDK_LANGUAGE = "monocle_apptrace.language"
|
|
80
|
+
MONOCLE_DETECTED_SPAN_ERROR = "monocle_apptrace.detected_span_error"
|
|
81
|
+
HTTP_SUCCESS_CODES = ('200', '201', '202', '204', '205', '206')
|
|
@@ -3,6 +3,7 @@ import inspect
|
|
|
3
3
|
from typing import Collection, Dict, List, Union
|
|
4
4
|
import random
|
|
5
5
|
import uuid
|
|
6
|
+
import inspect
|
|
6
7
|
from opentelemetry import trace
|
|
7
8
|
from contextlib import contextmanager
|
|
8
9
|
from opentelemetry.context import attach, get_value, set_value, get_current, detach
|
|
@@ -13,6 +14,7 @@ from opentelemetry.sdk.trace import TracerProvider, Span, id_generator
|
|
|
13
14
|
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
|
|
14
15
|
from opentelemetry.sdk.trace import Span, TracerProvider
|
|
15
16
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanProcessor
|
|
17
|
+
from opentelemetry.sdk.trace.export import SpanExporter
|
|
16
18
|
from opentelemetry.trace import get_tracer
|
|
17
19
|
from wrapt import wrap_function_wrapper
|
|
18
20
|
from opentelemetry.trace.propagation import set_span_in_context, _SPAN_KEY
|
|
@@ -23,11 +25,11 @@ from monocle_apptrace.instrumentation.common.wrapper_method import (
|
|
|
23
25
|
WrapperMethod,
|
|
24
26
|
MONOCLE_SPAN_HANDLERS
|
|
25
27
|
)
|
|
26
|
-
from monocle_apptrace.instrumentation.common.wrapper import scope_wrapper, ascope_wrapper,
|
|
28
|
+
from monocle_apptrace.instrumentation.common.wrapper import scope_wrapper, ascope_wrapper, monocle_wrapper, amonocle_wrapper
|
|
27
29
|
from monocle_apptrace.instrumentation.common.utils import (
|
|
28
|
-
set_scope, remove_scope, http_route_handler, load_scopes,
|
|
30
|
+
set_scope, remove_scope, http_route_handler, load_scopes, http_async_route_handler
|
|
29
31
|
)
|
|
30
|
-
from monocle_apptrace.instrumentation.common.constants import MONOCLE_INSTRUMENTOR,
|
|
32
|
+
from monocle_apptrace.instrumentation.common.constants import MONOCLE_INSTRUMENTOR, WORKFLOW_TYPE_GENERIC
|
|
31
33
|
from functools import wraps
|
|
32
34
|
logger = logging.getLogger(__name__)
|
|
33
35
|
|
|
@@ -39,19 +41,22 @@ monocle_tracer_provider: TracerProvider = None
|
|
|
39
41
|
|
|
40
42
|
class MonocleInstrumentor(BaseInstrumentor):
|
|
41
43
|
workflow_name: str = ""
|
|
42
|
-
user_wrapper_methods: list[Union[dict,WrapperMethod]] = []
|
|
44
|
+
user_wrapper_methods: list[Union[dict,WrapperMethod]] = [],
|
|
45
|
+
exporters: list[SpanExporter] = [],
|
|
43
46
|
instrumented_method_list: list[object] = []
|
|
44
|
-
handlers:Dict[str,SpanHandler] =
|
|
47
|
+
handlers:Dict[str,SpanHandler] = None # dict of handlers
|
|
45
48
|
union_with_default_methods: bool = False
|
|
46
49
|
|
|
47
50
|
def __init__(
|
|
48
51
|
self,
|
|
49
52
|
handlers,
|
|
50
53
|
user_wrapper_methods: list[Union[dict,WrapperMethod]] = None,
|
|
54
|
+
exporters: list[SpanExporter] = None,
|
|
51
55
|
union_with_default_methods: bool = True
|
|
52
56
|
) -> None:
|
|
53
57
|
self.user_wrapper_methods = user_wrapper_methods or []
|
|
54
58
|
self.handlers = handlers
|
|
59
|
+
self.exporters = exporters
|
|
55
60
|
if self.handlers is not None:
|
|
56
61
|
for key, val in MONOCLE_SPAN_HANDLERS.items():
|
|
57
62
|
if key not in self.handlers:
|
|
@@ -63,13 +68,20 @@ class MonocleInstrumentor(BaseInstrumentor):
|
|
|
63
68
|
|
|
64
69
|
def get_instrumentor(self, tracer):
|
|
65
70
|
def instrumented_endpoint_invoke(to_wrap,wrapped, span_name, instance,fn):
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
boto_method_to_wrap, fn, instance, args, kwargs)
|
|
71
|
+
if inspect.iscoroutinefunction(fn):
|
|
72
|
+
@wraps(fn)
|
|
73
|
+
async def with_instrumentation(*args, **kwargs):
|
|
74
|
+
boto_method_to_wrap = to_wrap.copy()
|
|
75
|
+
boto_method_to_wrap['skip_span'] = False
|
|
76
|
+
return await amonocle_wrapper(tracer, NonFrameworkSpanHandler(),
|
|
77
|
+
boto_method_to_wrap, fn, instance, "", args, kwargs)
|
|
78
|
+
else:
|
|
79
|
+
@wraps(fn)
|
|
80
|
+
def with_instrumentation(*args, **kwargs):
|
|
81
|
+
boto_method_to_wrap = to_wrap.copy()
|
|
82
|
+
boto_method_to_wrap['skip_span'] = False
|
|
83
|
+
return monocle_wrapper(tracer, NonFrameworkSpanHandler(),
|
|
84
|
+
boto_method_to_wrap, fn, instance, "", args, kwargs)
|
|
73
85
|
return with_instrumentation
|
|
74
86
|
return instrumented_endpoint_invoke
|
|
75
87
|
|
|
@@ -155,7 +167,8 @@ def setup_monocle_telemetry(
|
|
|
155
167
|
span_processors: List[SpanProcessor] = None,
|
|
156
168
|
span_handlers: Dict[str,SpanHandler] = None,
|
|
157
169
|
wrapper_methods: List[Union[dict,WrapperMethod]] = None,
|
|
158
|
-
union_with_default_methods: bool = True
|
|
170
|
+
union_with_default_methods: bool = True,
|
|
171
|
+
monocle_exporters_list:str = None) -> None:
|
|
159
172
|
"""
|
|
160
173
|
Set up Monocle telemetry for the application.
|
|
161
174
|
|
|
@@ -165,7 +178,7 @@ def setup_monocle_telemetry(
|
|
|
165
178
|
The name of the workflow to be used as the service name in telemetry.
|
|
166
179
|
span_processors : List[SpanProcessor], optional
|
|
167
180
|
Custom span processors to use instead of the default ones. If None,
|
|
168
|
-
BatchSpanProcessors with Monocle exporters will be used.
|
|
181
|
+
BatchSpanProcessors with Monocle exporters will be used. This can't be combined with `monocle_exporters_list`.
|
|
169
182
|
span_handlers : Dict[str, SpanHandler], optional
|
|
170
183
|
Dictionary of span handlers to be used by the instrumentor, mapping handler names to handler objects.
|
|
171
184
|
wrapper_methods : List[Union[dict, WrapperMethod]], optional
|
|
@@ -173,11 +186,16 @@ def setup_monocle_telemetry(
|
|
|
173
186
|
union_with_default_methods : bool, default=True
|
|
174
187
|
If True, combine the provided wrapper_methods with the default methods.
|
|
175
188
|
If False, only use the provided wrapper_methods.
|
|
189
|
+
monocle_exporters_list : str, optional
|
|
190
|
+
Comma-separated list of exporters to use. This will override the env setting MONOCLE_EXPORTERS.
|
|
191
|
+
Supported exporters are: s3, blob, okahu, file, memory, console. This can't be combined with `span_processors`.
|
|
176
192
|
"""
|
|
177
193
|
resource = Resource(attributes={
|
|
178
194
|
SERVICE_NAME: workflow_name
|
|
179
195
|
})
|
|
180
|
-
|
|
196
|
+
if span_processors and monocle_exporters_list:
|
|
197
|
+
raise ValueError("span_processors and monocle_exporters_list can't be used together")
|
|
198
|
+
exporters:List[SpanExporter] = get_monocle_exporter(monocle_exporters_list)
|
|
181
199
|
span_processors = span_processors or [BatchSpanProcessor(exporter) for exporter in exporters]
|
|
182
200
|
set_tracer_provider(TracerProvider(resource=resource))
|
|
183
201
|
attach(set_value("workflow_name", workflow_name))
|
|
@@ -192,7 +210,7 @@ def setup_monocle_telemetry(
|
|
|
192
210
|
get_tracer_provider().add_span_processor(processor)
|
|
193
211
|
if is_proxy_provider:
|
|
194
212
|
trace.set_tracer_provider(get_tracer_provider())
|
|
195
|
-
instrumentor = MonocleInstrumentor(user_wrapper_methods=wrapper_methods or [],
|
|
213
|
+
instrumentor = MonocleInstrumentor(user_wrapper_methods=wrapper_methods or [], exporters=exporters,
|
|
196
214
|
handlers=span_handlers, union_with_default_methods = union_with_default_methods)
|
|
197
215
|
# instrumentor.app_name = workflow_name
|
|
198
216
|
if not instrumentor.is_instrumented_by_opentelemetry:
|
|
@@ -228,10 +246,10 @@ def start_trace():
|
|
|
228
246
|
updated_span_context = set_span_in_context(span=span)
|
|
229
247
|
SpanHandler.set_default_monocle_attributes(span)
|
|
230
248
|
SpanHandler.set_workflow_properties(span)
|
|
231
|
-
token =
|
|
249
|
+
token = attach(updated_span_context)
|
|
232
250
|
return token
|
|
233
|
-
except:
|
|
234
|
-
logger.warning("Failed to start trace")
|
|
251
|
+
except Exception as e:
|
|
252
|
+
logger.warning("Failed to start trace {e}")
|
|
235
253
|
return None
|
|
236
254
|
|
|
237
255
|
def stop_trace(token) -> None:
|
|
@@ -250,7 +268,7 @@ def stop_trace(token) -> None:
|
|
|
250
268
|
if parent_span is not None:
|
|
251
269
|
parent_span.end()
|
|
252
270
|
if token is not None:
|
|
253
|
-
|
|
271
|
+
detach(token)
|
|
254
272
|
except:
|
|
255
273
|
logger.warning("Failed to stop trace")
|
|
256
274
|
|
|
@@ -317,8 +335,12 @@ def monocle_trace_scope_method(scope_name: str, scope_value:str=None):
|
|
|
317
335
|
if inspect.iscoroutinefunction(func):
|
|
318
336
|
@wraps(func)
|
|
319
337
|
async def wrapper(*args, **kwargs):
|
|
320
|
-
|
|
321
|
-
|
|
338
|
+
token = start_scope(scope_name, scope_value)
|
|
339
|
+
try:
|
|
340
|
+
result = await func(*args, **kwargs)
|
|
341
|
+
return result
|
|
342
|
+
finally:
|
|
343
|
+
stop_scope(token)
|
|
322
344
|
return wrapper
|
|
323
345
|
else:
|
|
324
346
|
@wraps(func)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
|
-
from
|
|
3
|
+
from contextlib import contextmanager
|
|
4
4
|
from opentelemetry.context import get_value, set_value, attach, detach
|
|
5
5
|
from opentelemetry.sdk.trace import Span
|
|
6
6
|
from opentelemetry.trace.status import Status, StatusCode
|
|
@@ -8,9 +8,9 @@ from monocle_apptrace.instrumentation.common.constants import (
|
|
|
8
8
|
QUERY,
|
|
9
9
|
service_name_map,
|
|
10
10
|
service_type_map,
|
|
11
|
-
MONOCLE_SDK_VERSION
|
|
11
|
+
MONOCLE_SDK_VERSION, MONOCLE_SDK_LANGUAGE, MONOCLE_DETECTED_SPAN_ERROR
|
|
12
12
|
)
|
|
13
|
-
from monocle_apptrace.instrumentation.common.utils import set_attribute, get_scopes, MonocleSpanException
|
|
13
|
+
from monocle_apptrace.instrumentation.common.utils import set_attribute, get_scopes, MonocleSpanException, get_monocle_version
|
|
14
14
|
from monocle_apptrace.instrumentation.common.constants import WORKFLOW_TYPE_KEY, WORKFLOW_TYPE_GENERIC
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
@@ -18,7 +18,8 @@ logger = logging.getLogger(__name__)
|
|
|
18
18
|
WORKFLOW_TYPE_MAP = {
|
|
19
19
|
"llama_index": "workflow.llamaindex",
|
|
20
20
|
"langchain": "workflow.langchain",
|
|
21
|
-
"haystack": "workflow.haystack"
|
|
21
|
+
"haystack": "workflow.haystack",
|
|
22
|
+
"teams.ai": "workflow.teams_ai",
|
|
22
23
|
}
|
|
23
24
|
|
|
24
25
|
class SpanHandler:
|
|
@@ -39,9 +40,9 @@ class SpanHandler:
|
|
|
39
40
|
pass
|
|
40
41
|
|
|
41
42
|
def skip_span(self, to_wrap, wrapped, instance, args, kwargs) -> bool:
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
return False
|
|
44
|
+
|
|
45
|
+
def skip_processor(self, to_wrap, wrapped, instance, args, kwargs) -> bool:
|
|
45
46
|
return False
|
|
46
47
|
|
|
47
48
|
def pre_task_processing(self, to_wrap, wrapped, instance, args,kwargs, span):
|
|
@@ -49,13 +50,11 @@ class SpanHandler:
|
|
|
49
50
|
set_attribute(QUERY, args[0]['prompt_builder']['question'])
|
|
50
51
|
|
|
51
52
|
@staticmethod
|
|
52
|
-
def set_default_monocle_attributes(span: Span):
|
|
53
|
+
def set_default_monocle_attributes(span: Span, source_path = "" ):
|
|
53
54
|
""" Set default monocle attributes for all spans """
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
except Exception as e:
|
|
58
|
-
logger.warning("Exception finding monocle-apptrace version.")
|
|
55
|
+
span.set_attribute(MONOCLE_SDK_VERSION, get_monocle_version())
|
|
56
|
+
span.set_attribute(MONOCLE_SDK_LANGUAGE, "python")
|
|
57
|
+
span.set_attribute("span_source", source_path)
|
|
59
58
|
for scope_key, scope_value in get_scopes().items():
|
|
60
59
|
span.set_attribute(f"scope.{scope_key}", scope_value)
|
|
61
60
|
|
|
@@ -64,25 +63,33 @@ class SpanHandler:
|
|
|
64
63
|
""" Set attributes of workflow if this is a root span"""
|
|
65
64
|
SpanHandler.set_workflow_attributes(to_wrap, span)
|
|
66
65
|
SpanHandler.set_app_hosting_identifier_attribute(span)
|
|
67
|
-
span.set_status(StatusCode.OK)
|
|
68
66
|
|
|
67
|
+
@staticmethod
|
|
68
|
+
def set_non_workflow_properties(span: Span, to_wrap = None):
|
|
69
|
+
workflow_name = SpanHandler.get_workflow_name(span=span)
|
|
70
|
+
if workflow_name:
|
|
71
|
+
span.set_attribute("workflow.name", workflow_name)
|
|
69
72
|
|
|
70
73
|
def post_task_processing(self, to_wrap, wrapped, instance, args, kwargs, result, span:Span):
|
|
71
74
|
if span.status.status_code == StatusCode.UNSET:
|
|
72
75
|
span.set_status(StatusCode.OK)
|
|
73
76
|
|
|
74
|
-
def hydrate_span(self, to_wrap, wrapped, instance, args, kwargs, result, span):
|
|
75
|
-
self.hydrate_attributes(to_wrap, wrapped, instance, args, kwargs, result, span)
|
|
76
|
-
self.hydrate_events(to_wrap, wrapped, instance, args, kwargs, result, span)
|
|
77
|
+
def hydrate_span(self, to_wrap, wrapped, instance, args, kwargs, result, span) -> bool:
|
|
78
|
+
detected_error_in_attribute = self.hydrate_attributes(to_wrap, wrapped, instance, args, kwargs, result, span)
|
|
79
|
+
detected_error_in_event = self.hydrate_events(to_wrap, wrapped, instance, args, kwargs, result, span)
|
|
80
|
+
if detected_error_in_attribute or detected_error_in_event:
|
|
81
|
+
span.set_attribute(MONOCLE_DETECTED_SPAN_ERROR, True)
|
|
77
82
|
|
|
78
|
-
def hydrate_attributes(self, to_wrap, wrapped, instance, args, kwargs, result, span):
|
|
83
|
+
def hydrate_attributes(self, to_wrap, wrapped, instance, args, kwargs, result, span:Span) -> bool:
|
|
84
|
+
detected_error:bool = False
|
|
79
85
|
span_index = 0
|
|
80
86
|
if SpanHandler.is_root_span(span):
|
|
81
87
|
span_index = 2 # root span will have workflow and hosting entities pre-populated
|
|
82
|
-
if
|
|
88
|
+
if not self.skip_processor(to_wrap, wrapped, instance, args, kwargs) and (
|
|
89
|
+
'output_processor' in to_wrap and to_wrap["output_processor"] is not None):
|
|
83
90
|
output_processor=to_wrap['output_processor']
|
|
84
91
|
if 'type' in output_processor:
|
|
85
|
-
|
|
92
|
+
span.set_attribute("span.type", output_processor['type'])
|
|
86
93
|
else:
|
|
87
94
|
logger.warning("type of span not found or incorrect written in entity json")
|
|
88
95
|
if 'attributes' in output_processor:
|
|
@@ -100,6 +107,7 @@ class SpanHandler:
|
|
|
100
107
|
span.set_attribute(attribute_name, result)
|
|
101
108
|
except MonocleSpanException as e:
|
|
102
109
|
span.set_status(StatusCode.ERROR, e.message)
|
|
110
|
+
detected_error = True
|
|
103
111
|
except Exception as e:
|
|
104
112
|
logger.debug(f"Error processing accessor: {e}")
|
|
105
113
|
else:
|
|
@@ -107,6 +115,8 @@ class SpanHandler:
|
|
|
107
115
|
span_index += 1
|
|
108
116
|
else:
|
|
109
117
|
logger.debug("attributes not found or incorrect written in entity json")
|
|
118
|
+
else:
|
|
119
|
+
span.set_attribute("span.type", "generic")
|
|
110
120
|
|
|
111
121
|
# set scopes as attributes by calling get_scopes()
|
|
112
122
|
# scopes is a Mapping[str:object], iterate directly with .items()
|
|
@@ -115,12 +125,14 @@ class SpanHandler:
|
|
|
115
125
|
|
|
116
126
|
if span_index > 0:
|
|
117
127
|
span.set_attribute("entity.count", span_index)
|
|
128
|
+
return detected_error
|
|
118
129
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
if
|
|
130
|
+
def hydrate_events(self, to_wrap, wrapped, instance, args, kwargs, ret_result, span) -> bool:
|
|
131
|
+
detected_error:bool = False
|
|
132
|
+
if not self.skip_processor(to_wrap, wrapped, instance, args, kwargs) and (
|
|
133
|
+
'output_processor' in to_wrap and to_wrap["output_processor"] is not None):
|
|
122
134
|
output_processor=to_wrap['output_processor']
|
|
123
|
-
arguments = {"instance": instance, "args": args, "kwargs": kwargs, "result":
|
|
135
|
+
arguments = {"instance": instance, "args": args, "kwargs": kwargs, "result": ret_result}
|
|
124
136
|
if 'events' in output_processor:
|
|
125
137
|
events = output_processor['events']
|
|
126
138
|
for event in events:
|
|
@@ -132,21 +144,32 @@ class SpanHandler:
|
|
|
132
144
|
accessor = attribute.get("accessor")
|
|
133
145
|
if accessor:
|
|
134
146
|
try:
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
147
|
+
result = accessor(arguments)
|
|
148
|
+
if result and isinstance(result, dict):
|
|
149
|
+
result = dict((key, value) for key, value in result.items() if value is not None)
|
|
150
|
+
if result and isinstance(result, (str, list, dict)):
|
|
151
|
+
if attribute_key is not None:
|
|
152
|
+
event_attributes[attribute_key] = result
|
|
153
|
+
else:
|
|
154
|
+
event_attributes.update(result)
|
|
139
155
|
except MonocleSpanException as e:
|
|
140
156
|
span.set_status(StatusCode.ERROR, e.message)
|
|
157
|
+
detected_error = True
|
|
141
158
|
except Exception as e:
|
|
142
159
|
logger.debug(f"Error evaluating accessor for attribute '{attribute_key}': {e}")
|
|
143
|
-
|
|
160
|
+
matching_timestamp = getattr(ret_result, "timestamps", {}).get(event_name, None)
|
|
161
|
+
if isinstance(matching_timestamp, int):
|
|
162
|
+
span.add_event(name=event_name, attributes=event_attributes, timestamp=matching_timestamp)
|
|
163
|
+
else:
|
|
164
|
+
span.add_event(name=event_name, attributes=event_attributes)
|
|
165
|
+
return detected_error
|
|
144
166
|
|
|
145
167
|
@staticmethod
|
|
146
168
|
def set_workflow_attributes(to_wrap, span: Span):
|
|
147
169
|
span_index = 1
|
|
148
170
|
workflow_name = SpanHandler.get_workflow_name(span=span)
|
|
149
171
|
if workflow_name:
|
|
172
|
+
span.update_name("workflow")
|
|
150
173
|
span.set_attribute("span.type", "workflow")
|
|
151
174
|
span.set_attribute(f"entity.{span_index}.name", workflow_name)
|
|
152
175
|
workflow_type = SpanHandler.get_workflow_type(to_wrap)
|
|
@@ -186,26 +209,19 @@ class SpanHandler:
|
|
|
186
209
|
@staticmethod
|
|
187
210
|
def is_root_span(curr_span: Span) -> bool:
|
|
188
211
|
try:
|
|
189
|
-
if curr_span is not None and hasattr(curr_span, "parent"):
|
|
212
|
+
if curr_span is not None and hasattr(curr_span, "parent") or curr_span.context.trace_state:
|
|
190
213
|
return curr_span.parent is None
|
|
191
214
|
except Exception as e:
|
|
192
215
|
logger.warning(f"Error finding root span: {e}")
|
|
193
216
|
|
|
194
|
-
def is_non_workflow_root_span(self, curr_span: Span, to_wrap) -> bool:
|
|
195
|
-
return SpanHandler.is_root_span(curr_span) and to_wrap.get("span_type") != "workflow"
|
|
196
|
-
|
|
197
|
-
def is_workflow_span_active(self):
|
|
198
|
-
return get_value(WORKFLOW_TYPE_KEY) is not None
|
|
199
|
-
|
|
200
217
|
@staticmethod
|
|
201
218
|
def attach_workflow_type(to_wrap=None, context=None):
|
|
202
219
|
token = None
|
|
203
220
|
if to_wrap:
|
|
204
|
-
|
|
221
|
+
workflow_type = SpanHandler.get_workflow_type(to_wrap)
|
|
222
|
+
if workflow_type != WORKFLOW_TYPE_GENERIC:
|
|
205
223
|
token = attach(set_value(WORKFLOW_TYPE_KEY,
|
|
206
224
|
SpanHandler.get_workflow_type(to_wrap), context))
|
|
207
|
-
else:
|
|
208
|
-
token = attach(set_value(WORKFLOW_TYPE_KEY, WORKFLOW_TYPE_GENERIC, context))
|
|
209
225
|
return token
|
|
210
226
|
|
|
211
227
|
@staticmethod
|
|
@@ -213,8 +229,18 @@ class SpanHandler:
|
|
|
213
229
|
if token:
|
|
214
230
|
return detach(token)
|
|
215
231
|
|
|
232
|
+
@staticmethod
|
|
233
|
+
@contextmanager
|
|
234
|
+
def workflow_type(to_wrap=None):
|
|
235
|
+
token = SpanHandler.attach_workflow_type(to_wrap)
|
|
236
|
+
try:
|
|
237
|
+
yield
|
|
238
|
+
finally:
|
|
239
|
+
SpanHandler.detach_workflow_type(token)
|
|
240
|
+
|
|
241
|
+
|
|
216
242
|
class NonFrameworkSpanHandler(SpanHandler):
|
|
217
243
|
|
|
218
|
-
# If the language framework is being executed, then skip generating direct openAI
|
|
219
|
-
def
|
|
244
|
+
# If the language framework is being executed, then skip generating direct openAI attributes and events
|
|
245
|
+
def skip_processor(self, to_wrap, wrapped, instance, args, kwargs) -> bool:
|
|
220
246
|
return get_value(WORKFLOW_TYPE_KEY) in WORKFLOW_TYPE_MAP.values()
|