monocle-apptrace 0.3.0b5__py3-none-any.whl → 0.3.0b7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of monocle-apptrace might be problematic. Click here for more details.
- monocle_apptrace/__init__.py +1 -0
- monocle_apptrace/exporters/aws/s3_exporter.py +20 -6
- monocle_apptrace/exporters/aws/s3_exporter_opendal.py +22 -11
- monocle_apptrace/exporters/azure/blob_exporter.py +29 -8
- monocle_apptrace/exporters/azure/blob_exporter_opendal.py +23 -8
- monocle_apptrace/exporters/exporter_processor.py +128 -3
- monocle_apptrace/exporters/file_exporter.py +16 -0
- monocle_apptrace/exporters/monocle_exporters.py +10 -1
- monocle_apptrace/exporters/okahu/okahu_exporter.py +8 -6
- monocle_apptrace/instrumentation/__init__.py +1 -0
- monocle_apptrace/instrumentation/common/__init__.py +2 -0
- monocle_apptrace/instrumentation/common/constants.py +17 -0
- monocle_apptrace/instrumentation/common/instrumentor.py +136 -53
- monocle_apptrace/instrumentation/common/span_handler.py +92 -41
- monocle_apptrace/instrumentation/common/utils.py +84 -6
- monocle_apptrace/instrumentation/common/wrapper.py +43 -45
- monocle_apptrace/instrumentation/common/wrapper_method.py +8 -4
- monocle_apptrace/instrumentation/metamodel/botocore/handlers/botocore_span_handler.py +2 -1
- monocle_apptrace/instrumentation/metamodel/haystack/entities/inference.py +1 -1
- monocle_apptrace/instrumentation/metamodel/haystack/methods.py +2 -1
- monocle_apptrace/instrumentation/metamodel/langchain/entities/inference.py +3 -2
- monocle_apptrace/instrumentation/metamodel/langchain/methods.py +12 -6
- monocle_apptrace/instrumentation/metamodel/llamaindex/entities/inference.py +5 -3
- monocle_apptrace/instrumentation/metamodel/llamaindex/methods.py +6 -3
- monocle_apptrace/instrumentation/metamodel/openai/_helper.py +31 -7
- monocle_apptrace/instrumentation/metamodel/openai/entities/inference.py +1 -1
- monocle_apptrace/instrumentation/metamodel/openai/entities/retrieval.py +20 -1
- monocle_apptrace/instrumentation/metamodel/openai/methods.py +21 -1
- monocle_apptrace/instrumentation/metamodel/requests/__init__.py +3 -1
- {monocle_apptrace-0.3.0b5.dist-info → monocle_apptrace-0.3.0b7.dist-info}/METADATA +1 -1
- {monocle_apptrace-0.3.0b5.dist-info → monocle_apptrace-0.3.0b7.dist-info}/RECORD +34 -34
- {monocle_apptrace-0.3.0b5.dist-info → monocle_apptrace-0.3.0b7.dist-info}/WHEEL +0 -0
- {monocle_apptrace-0.3.0b5.dist-info → monocle_apptrace-0.3.0b7.dist-info}/licenses/LICENSE +0 -0
- {monocle_apptrace-0.3.0b5.dist-info → monocle_apptrace-0.3.0b7.dist-info}/licenses/NOTICE +0 -0
monocle_apptrace/__init__.py
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .instrumentation import *
|
|
@@ -16,12 +16,13 @@ from botocore.exceptions import (
|
|
|
16
16
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
17
17
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
18
18
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
19
|
-
from
|
|
19
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
20
|
+
from typing import Sequence, Optional
|
|
20
21
|
import json
|
|
21
22
|
logger = logging.getLogger(__name__)
|
|
22
23
|
|
|
23
24
|
class S3SpanExporter(SpanExporterBase):
|
|
24
|
-
def __init__(self, bucket_name=None, region_name=None):
|
|
25
|
+
def __init__(self, bucket_name=None, region_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
25
26
|
super().__init__()
|
|
26
27
|
# Use environment variables if credentials are not provided
|
|
27
28
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
@@ -47,6 +48,9 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
47
48
|
self.time_format = DEFAULT_TIME_FORMAT
|
|
48
49
|
self.export_queue = []
|
|
49
50
|
self.last_export_time = time.time()
|
|
51
|
+
self.task_processor = task_processor
|
|
52
|
+
if self.task_processor is not None:
|
|
53
|
+
self.task_processor.start()
|
|
50
54
|
|
|
51
55
|
# Check if bucket exists or create it
|
|
52
56
|
if not self.__bucket_exists(self.bucket_name):
|
|
@@ -92,6 +96,7 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
92
96
|
"""Synchronous export method that internally handles async logic."""
|
|
93
97
|
try:
|
|
94
98
|
# Run the asynchronous export logic in an event loop
|
|
99
|
+
logger.info(f"Exporting {len(spans)} spans to S3.")
|
|
95
100
|
asyncio.run(self.__export_async(spans))
|
|
96
101
|
return SpanExportResult.SUCCESS
|
|
97
102
|
except Exception as e:
|
|
@@ -100,6 +105,7 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
100
105
|
|
|
101
106
|
async def __export_async(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
102
107
|
try:
|
|
108
|
+
logger.info(f"__export_async {len(spans)} spans to S3.")
|
|
103
109
|
# Add spans to the export queue
|
|
104
110
|
for span in spans:
|
|
105
111
|
self.export_queue.append(span)
|
|
@@ -142,10 +148,16 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
142
148
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
143
149
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
144
150
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
151
|
+
# to calculate is_root_span loop over each span in batch_to_export and check if parent id is none or null
|
|
152
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
153
|
+
logger.info(f"Exporting {len(batch_to_export)} spans to S3 is_root_span : {is_root_span}.")
|
|
154
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
155
|
+
self.task_processor.queue_task(self.__upload_to_s3, serialized_data, is_root_span)
|
|
156
|
+
else:
|
|
157
|
+
try:
|
|
158
|
+
self.__upload_to_s3(serialized_data)
|
|
159
|
+
except Exception as e:
|
|
160
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
149
161
|
|
|
150
162
|
@SpanExporterBase.retry_with_backoff(exceptions=(EndpointConnectionError, ConnectionClosedError, ReadTimeoutError, ConnectTimeoutError))
|
|
151
163
|
def __upload_to_s3(self, span_data_batch: str):
|
|
@@ -164,4 +176,6 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
164
176
|
return True
|
|
165
177
|
|
|
166
178
|
def shutdown(self) -> None:
|
|
179
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
180
|
+
self.task_processor.stop()
|
|
167
181
|
logger.info("S3SpanExporter has been shut down.")
|
|
@@ -3,19 +3,19 @@ import time
|
|
|
3
3
|
import datetime
|
|
4
4
|
import logging
|
|
5
5
|
import asyncio
|
|
6
|
-
from typing import Sequence
|
|
6
|
+
from typing import Sequence, Optional
|
|
7
7
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
8
8
|
from opentelemetry.sdk.trace.export import SpanExportResult
|
|
9
9
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
10
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
10
11
|
from opendal import Operator
|
|
11
12
|
from opendal.exceptions import PermissionDenied, ConfigInvalid, Unexpected
|
|
12
13
|
|
|
13
|
-
|
|
14
14
|
import json
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
17
17
|
class OpenDALS3Exporter(SpanExporterBase):
|
|
18
|
-
def __init__(self, bucket_name=None, region_name=None):
|
|
18
|
+
def __init__(self, bucket_name=None, region_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
19
19
|
super().__init__()
|
|
20
20
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
21
21
|
DEFAULT_TIME_FORMAT = "%Y-%m-%d__%H.%M.%S"
|
|
@@ -36,7 +36,10 @@ class OpenDALS3Exporter(SpanExporterBase):
|
|
|
36
36
|
access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
|
|
37
37
|
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
|
|
38
38
|
)
|
|
39
|
-
|
|
39
|
+
|
|
40
|
+
self.task_processor = task_processor
|
|
41
|
+
if self.task_processor is not None:
|
|
42
|
+
self.task_processor.start()
|
|
40
43
|
|
|
41
44
|
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
42
45
|
"""Synchronous export method that internally handles async logic."""
|
|
@@ -88,20 +91,26 @@ class OpenDALS3Exporter(SpanExporterBase):
|
|
|
88
91
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
89
92
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
90
93
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
94
|
+
|
|
95
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
96
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
97
|
+
|
|
98
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
99
|
+
self.task_processor.queue_task(self.__upload_to_s3, serialized_data, is_root_span)
|
|
100
|
+
else:
|
|
101
|
+
try:
|
|
102
|
+
self.__upload_to_s3(serialized_data, is_root_span)
|
|
103
|
+
except Exception as e:
|
|
104
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
95
105
|
|
|
96
106
|
@SpanExporterBase.retry_with_backoff(exceptions=(Unexpected))
|
|
97
|
-
def __upload_to_s3(self, span_data_batch: str):
|
|
98
|
-
|
|
107
|
+
def __upload_to_s3(self, span_data_batch: str, is_root_span: bool = False):
|
|
99
108
|
current_time = datetime.datetime.now().strftime(self.time_format)
|
|
100
109
|
file_name = f"{self.file_prefix}{current_time}.ndjson"
|
|
101
110
|
try:
|
|
102
111
|
# Attempt to write the span data batch to S3
|
|
103
112
|
self.op.write(file_name, span_data_batch.encode("utf-8"))
|
|
104
|
-
logger.info(f"Span batch uploaded to S3 as {file_name}.")
|
|
113
|
+
logger.info(f"Span batch uploaded to S3 as {file_name}. Is root span: {is_root_span}")
|
|
105
114
|
|
|
106
115
|
except PermissionDenied as e:
|
|
107
116
|
# S3 bucket is forbidden.
|
|
@@ -123,4 +132,6 @@ class OpenDALS3Exporter(SpanExporterBase):
|
|
|
123
132
|
return True
|
|
124
133
|
|
|
125
134
|
def shutdown(self) -> None:
|
|
135
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
136
|
+
self.task_processor.stop()
|
|
126
137
|
logger.info("S3SpanExporter has been shut down.")
|
|
@@ -8,13 +8,15 @@ from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
|
|
|
8
8
|
from azure.core.exceptions import ResourceNotFoundError, ClientAuthenticationError, ServiceRequestError
|
|
9
9
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
10
10
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
11
|
-
from typing import Sequence
|
|
11
|
+
from typing import Sequence, Optional
|
|
12
12
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
13
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
13
14
|
import json
|
|
15
|
+
from monocle_apptrace.instrumentation.common.constants import MONOCLE_SDK_VERSION
|
|
14
16
|
logger = logging.getLogger(__name__)
|
|
15
17
|
|
|
16
18
|
class AzureBlobSpanExporter(SpanExporterBase):
|
|
17
|
-
def __init__(self, connection_string=None, container_name=None):
|
|
19
|
+
def __init__(self, connection_string=None, container_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
18
20
|
super().__init__()
|
|
19
21
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
20
22
|
DEFAULT_TIME_FORMAT = "%Y-%m-%d_%H.%M.%S"
|
|
@@ -43,6 +45,10 @@ class AzureBlobSpanExporter(SpanExporterBase):
|
|
|
43
45
|
logger.error(f"Error creating container {container_name}: {e}")
|
|
44
46
|
raise e
|
|
45
47
|
|
|
48
|
+
self.task_processor = task_processor
|
|
49
|
+
if self.task_processor is not None:
|
|
50
|
+
self.task_processor.start()
|
|
51
|
+
|
|
46
52
|
def __container_exists(self, container_name):
|
|
47
53
|
try:
|
|
48
54
|
container_client = self.blob_service_client.get_container_client(container_name)
|
|
@@ -72,6 +78,12 @@ class AzureBlobSpanExporter(SpanExporterBase):
|
|
|
72
78
|
"""The actual async export logic is run here."""
|
|
73
79
|
# Add spans to the export queue
|
|
74
80
|
for span in spans:
|
|
81
|
+
# Azure blob library has a check to generate it's own span if OpenTelemetry is loaded and Azure trace package is installed (just pip install azure-trace-opentelemetry)
|
|
82
|
+
# With Monocle,OpenTelemetry is always loaded. If the Azure trace package is installed, then it triggers the blob trace generation on every blob operation.
|
|
83
|
+
# Thus, the Monocle span write ends up generating a blob span which again comes back to the exporter .. and would result in an infinite loop.
|
|
84
|
+
# To avoid this, we check if the span has the Monocle SDK version attribute and skip it if it doesn't. That way the blob span genearted by Azure library are not exported.
|
|
85
|
+
if not span.attributes.get(MONOCLE_SDK_VERSION):
|
|
86
|
+
continue # TODO: All exporters to use same base class and check it there
|
|
75
87
|
self.export_queue.append(span)
|
|
76
88
|
if len(self.export_queue) >= self.max_batch_size:
|
|
77
89
|
await self.__export_spans()
|
|
@@ -104,22 +116,31 @@ class AzureBlobSpanExporter(SpanExporterBase):
|
|
|
104
116
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
105
117
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
106
118
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
119
|
+
|
|
120
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
121
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
122
|
+
|
|
123
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
124
|
+
self.task_processor.queue_task(self.__upload_to_blob, serialized_data, is_root_span)
|
|
125
|
+
else:
|
|
126
|
+
try:
|
|
127
|
+
self.__upload_to_blob(serialized_data, is_root_span)
|
|
128
|
+
except Exception as e:
|
|
129
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
111
130
|
|
|
112
131
|
@SpanExporterBase.retry_with_backoff(exceptions=(ResourceNotFoundError, ClientAuthenticationError, ServiceRequestError))
|
|
113
|
-
def __upload_to_blob(self, span_data_batch: str):
|
|
132
|
+
def __upload_to_blob(self, span_data_batch: str, is_root_span: bool = False):
|
|
114
133
|
current_time = datetime.datetime.now().strftime(self.time_format)
|
|
115
134
|
file_name = f"{self.file_prefix}{current_time}.ndjson"
|
|
116
135
|
blob_client = self.blob_service_client.get_blob_client(container=self.container_name, blob=file_name)
|
|
117
136
|
blob_client.upload_blob(span_data_batch, overwrite=True)
|
|
118
|
-
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}.")
|
|
137
|
+
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}. Is root span: {is_root_span}")
|
|
119
138
|
|
|
120
139
|
async def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
121
140
|
await self.__export_spans()
|
|
122
141
|
return True
|
|
123
142
|
|
|
124
143
|
def shutdown(self) -> None:
|
|
144
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
145
|
+
self.task_processor.stop()
|
|
125
146
|
logger.info("AzureBlobSpanExporter has been shut down.")
|
|
@@ -5,16 +5,17 @@ import logging
|
|
|
5
5
|
import asyncio
|
|
6
6
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
7
7
|
from opentelemetry.sdk.trace.export import SpanExportResult
|
|
8
|
-
from typing import Sequence
|
|
8
|
+
from typing import Sequence, Optional
|
|
9
9
|
from opendal import Operator
|
|
10
10
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
11
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
11
12
|
from opendal.exceptions import Unexpected, PermissionDenied, NotFound
|
|
12
13
|
import json
|
|
13
14
|
|
|
14
15
|
logger = logging.getLogger(__name__)
|
|
15
16
|
|
|
16
17
|
class OpenDALAzureExporter(SpanExporterBase):
|
|
17
|
-
def __init__(self, connection_string=None, container_name=None):
|
|
18
|
+
def __init__(self, connection_string=None, container_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
18
19
|
super().__init__()
|
|
19
20
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
20
21
|
DEFAULT_TIME_FORMAT = "%Y-%m-%d_%H.%M.%S"
|
|
@@ -25,6 +26,8 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
25
26
|
# Default values
|
|
26
27
|
self.file_prefix = DEFAULT_FILE_PREFIX
|
|
27
28
|
self.time_format = DEFAULT_TIME_FORMAT
|
|
29
|
+
self.export_queue = [] # Add this line to initialize export_queue
|
|
30
|
+
self.last_export_time = time.time() # Add this line to initialize last_export_time
|
|
28
31
|
|
|
29
32
|
# Validate input
|
|
30
33
|
if not connection_string:
|
|
@@ -51,6 +54,9 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
51
54
|
except Exception as e:
|
|
52
55
|
raise RuntimeError(f"Failed to initialize OpenDAL operator: {e}")
|
|
53
56
|
|
|
57
|
+
self.task_processor = task_processor
|
|
58
|
+
if self.task_processor is not None:
|
|
59
|
+
self.task_processor.start()
|
|
54
60
|
|
|
55
61
|
def parse_connection_string(self,connection_string):
|
|
56
62
|
connection_params = dict(item.split('=', 1) for item in connection_string.split(';') if '=' in item)
|
|
@@ -112,19 +118,26 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
112
118
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
113
119
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
114
120
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
121
|
+
|
|
122
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
123
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
124
|
+
|
|
125
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
126
|
+
self.task_processor.queue_task(self.__upload_to_opendal, serialized_data, is_root_span)
|
|
127
|
+
else:
|
|
128
|
+
try:
|
|
129
|
+
self.__upload_to_opendal(serialized_data, is_root_span)
|
|
130
|
+
except Exception as e:
|
|
131
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
119
132
|
|
|
120
133
|
@SpanExporterBase.retry_with_backoff(exceptions=(Unexpected,))
|
|
121
|
-
def __upload_to_opendal(self, span_data_batch: str):
|
|
134
|
+
def __upload_to_opendal(self, span_data_batch: str, is_root_span: bool = False):
|
|
122
135
|
current_time = datetime.datetime.now().strftime(self.time_format)
|
|
123
136
|
file_name = f"{self.file_prefix}{current_time}.ndjson"
|
|
124
137
|
|
|
125
138
|
try:
|
|
126
139
|
self.operator.write(file_name, span_data_batch.encode('utf-8'))
|
|
127
|
-
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}.")
|
|
140
|
+
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}. Is root span: {is_root_span}")
|
|
128
141
|
except PermissionDenied as e:
|
|
129
142
|
# Azure Container is forbidden.
|
|
130
143
|
logger.error(f"Access to container {self.container_name} is forbidden (403).")
|
|
@@ -144,4 +157,6 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
144
157
|
return True
|
|
145
158
|
|
|
146
159
|
def shutdown(self) -> None:
|
|
160
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
161
|
+
self.task_processor.stop()
|
|
147
162
|
logger.info("OpenDALAzureExporter has been shut down.")
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
import logging
|
|
3
|
+
import os
|
|
4
|
+
import queue
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
3
7
|
from typing import Callable
|
|
8
|
+
import requests
|
|
9
|
+
from monocle_apptrace.instrumentation.common.constants import AWS_LAMBDA_ENV_NAME
|
|
4
10
|
|
|
5
11
|
logger = logging.getLogger(__name__)
|
|
12
|
+
LAMBDA_EXTENSION_NAME = "AsyncProcessorMonocle"
|
|
6
13
|
|
|
7
14
|
class ExportTaskProcessor(ABC):
|
|
8
|
-
|
|
15
|
+
|
|
9
16
|
@abstractmethod
|
|
10
17
|
def start(self):
|
|
11
18
|
return
|
|
@@ -15,5 +22,123 @@ class ExportTaskProcessor(ABC):
|
|
|
15
22
|
return
|
|
16
23
|
|
|
17
24
|
@abstractmethod
|
|
18
|
-
def queue_task(self, async_task: Callable[[Callable, any], any] = None, args: any = None):
|
|
19
|
-
return
|
|
25
|
+
def queue_task(self, async_task: Callable[[Callable, any], any] = None, args: any = None, is_root_span: bool = False):
|
|
26
|
+
return
|
|
27
|
+
|
|
28
|
+
class LambdaExportTaskProcessor(ExportTaskProcessor):
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
span_check_interval_seconds: int = 1,
|
|
33
|
+
max_time_allowed_seconds: int = 30):
|
|
34
|
+
# An internal queue used by the handler to notify the extension that it can
|
|
35
|
+
# start processing the async task.
|
|
36
|
+
self.async_tasks_queue = queue.Queue()
|
|
37
|
+
self.span_check_interval = span_check_interval_seconds
|
|
38
|
+
self.max_time_allowed = max_time_allowed_seconds
|
|
39
|
+
|
|
40
|
+
def start(self):
|
|
41
|
+
try:
|
|
42
|
+
self._start_async_processor()
|
|
43
|
+
except Exception as e:
|
|
44
|
+
logger.error(f"LambdaExportTaskProcessor| Failed to start. {e}")
|
|
45
|
+
|
|
46
|
+
def stop(self):
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
def queue_task(self, async_task=None, args=None, is_root_span=False):
|
|
50
|
+
self.async_tasks_queue.put((async_task, args, is_root_span))
|
|
51
|
+
|
|
52
|
+
def set_sagemaker_model(self, endpoint_name: str, span: dict[str, dict[str, str]]):
|
|
53
|
+
try:
|
|
54
|
+
try:
|
|
55
|
+
import boto3
|
|
56
|
+
except ImportError:
|
|
57
|
+
logger.error("LambdaExportTaskProcessor| Failed to import boto3")
|
|
58
|
+
return
|
|
59
|
+
|
|
60
|
+
client = boto3.client('sagemaker')
|
|
61
|
+
response = client.describe_endpoint(
|
|
62
|
+
EndpointName=endpoint_name
|
|
63
|
+
)
|
|
64
|
+
endpoint_config_name = response["EndpointConfigName"]
|
|
65
|
+
endpoint_config_response = client.describe_endpoint_config(
|
|
66
|
+
EndpointConfigName=endpoint_config_name
|
|
67
|
+
)
|
|
68
|
+
model_name = endpoint_config_response["ProductionVariants"][0]["ModelName"]
|
|
69
|
+
model_name_response = client.describe_model(ModelName = model_name)
|
|
70
|
+
model_name_id = ""
|
|
71
|
+
try:
|
|
72
|
+
model_name_id = model_name_response["PrimaryContainer"]["Environment"]["HF_MODEL_ID"]
|
|
73
|
+
except:
|
|
74
|
+
pass
|
|
75
|
+
span["attributes"]["model_name"] = model_name_id
|
|
76
|
+
except Exception as e:
|
|
77
|
+
logger.error(f"LambdaExportTaskProcessor| Failed to get sagemaker model. {e}")
|
|
78
|
+
|
|
79
|
+
def update_spans(self, export_args):
|
|
80
|
+
try:
|
|
81
|
+
if 'batch' in export_args:
|
|
82
|
+
for span in export_args["batch"]:
|
|
83
|
+
try:
|
|
84
|
+
if len(span["attributes"]["sagemaker_endpoint_name"]) > 0 :
|
|
85
|
+
self.set_sagemaker_model(endpoint_name=span["attributes"]["sagemaker_endpoint_name"], span=span)
|
|
86
|
+
except:
|
|
87
|
+
pass
|
|
88
|
+
except Exception as e:
|
|
89
|
+
logger.error(f"LambdaExportTaskProcessor| Failed to update spans. {e}")
|
|
90
|
+
|
|
91
|
+
def _start_async_processor(self):
|
|
92
|
+
# Register internal extension
|
|
93
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Registering with Lambda service...")
|
|
94
|
+
response = requests.post(
|
|
95
|
+
url=f"http://{os.environ['AWS_LAMBDA_RUNTIME_API']}/2020-01-01/extension/register",
|
|
96
|
+
json={'events': ['INVOKE']},
|
|
97
|
+
headers={'Lambda-Extension-Name': LAMBDA_EXTENSION_NAME}
|
|
98
|
+
)
|
|
99
|
+
ext_id = response.headers['Lambda-Extension-Identifier']
|
|
100
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Registered with ID: {ext_id}")
|
|
101
|
+
|
|
102
|
+
def process_tasks():
|
|
103
|
+
while True:
|
|
104
|
+
# Call /next to get notified when there is a new invocation and let
|
|
105
|
+
# Lambda know that we are done processing the previous task.
|
|
106
|
+
|
|
107
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Waiting for invocation...")
|
|
108
|
+
response = requests.get(
|
|
109
|
+
url=f"http://{os.environ['AWS_LAMBDA_RUNTIME_API']}/2020-01-01/extension/event/next",
|
|
110
|
+
headers={'Lambda-Extension-Identifier': ext_id},
|
|
111
|
+
timeout=None
|
|
112
|
+
)
|
|
113
|
+
root_span_found = False
|
|
114
|
+
# all values in seconds
|
|
115
|
+
total_time_elapsed = 0
|
|
116
|
+
while root_span_found is False and total_time_elapsed < self.max_time_allowed:
|
|
117
|
+
logger.debug(response.json())
|
|
118
|
+
# Get next task from internal queue
|
|
119
|
+
logger.info(f"[{LAMBDA_EXTENSION_NAME}] Async thread running, waiting for task from handler")
|
|
120
|
+
while self.async_tasks_queue.empty() is False :
|
|
121
|
+
logger.info(f"[{LAMBDA_EXTENSION_NAME}] Processing task from handler")
|
|
122
|
+
async_task, arg, is_root_span = self.async_tasks_queue.get()
|
|
123
|
+
root_span_found = is_root_span
|
|
124
|
+
# self.update_spans(export_args=arg)
|
|
125
|
+
|
|
126
|
+
if async_task is None:
|
|
127
|
+
# No task to run this invocation
|
|
128
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Received null task. Ignoring.")
|
|
129
|
+
else:
|
|
130
|
+
# Invoke task
|
|
131
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Received async task from handler. Starting task.")
|
|
132
|
+
async_task(arg)
|
|
133
|
+
total_time_elapsed+=self.span_check_interval
|
|
134
|
+
logger.info(f"[{LAMBDA_EXTENSION_NAME}] Waiting for root span. total_time_elapsed: {total_time_elapsed}, root_span_found: {root_span_found}.")
|
|
135
|
+
time.sleep(self.span_check_interval)
|
|
136
|
+
|
|
137
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Finished processing task. total_time_elapsed: {total_time_elapsed}, root_span_found: {root_span_found}.")
|
|
138
|
+
|
|
139
|
+
# Start processing extension events in a separate thread
|
|
140
|
+
threading.Thread(target=process_tasks, daemon=True, name=LAMBDA_EXTENSION_NAME).start()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def is_aws_lambda_environment():
|
|
144
|
+
return AWS_LAMBDA_ENV_NAME in os.environ
|
|
@@ -7,6 +7,7 @@ from typing import Optional, Callable, Sequence
|
|
|
7
7
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
8
8
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
9
9
|
from opentelemetry.sdk.resources import SERVICE_NAME
|
|
10
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
10
11
|
|
|
11
12
|
DEFAULT_FILE_PREFIX:str = "monocle_trace_"
|
|
12
13
|
DEFAULT_TIME_FORMAT:str = "%Y-%m-%d_%H.%M.%S"
|
|
@@ -25,6 +26,7 @@ class FileSpanExporter(SpanExporter):
|
|
|
25
26
|
[ReadableSpan], str
|
|
26
27
|
] = lambda span: span.to_json()
|
|
27
28
|
+ linesep,
|
|
29
|
+
task_processor: Optional[ExportTaskProcessor] = None
|
|
28
30
|
):
|
|
29
31
|
self.out_handle:TextIOWrapper = None
|
|
30
32
|
self.formatter = formatter
|
|
@@ -32,8 +34,20 @@ class FileSpanExporter(SpanExporter):
|
|
|
32
34
|
self.output_path = out_path
|
|
33
35
|
self.file_prefix = file_prefix
|
|
34
36
|
self.time_format = time_format
|
|
37
|
+
self.task_processor = task_processor
|
|
38
|
+
if self.task_processor is not None:
|
|
39
|
+
self.task_processor.start()
|
|
35
40
|
|
|
36
41
|
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
42
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
43
|
+
# Check if any span is a root span (no parent)
|
|
44
|
+
is_root_span = any(not span.parent for span in spans)
|
|
45
|
+
self.task_processor.queue_task(self._process_spans, spans, is_root_span)
|
|
46
|
+
return SpanExportResult.SUCCESS
|
|
47
|
+
else:
|
|
48
|
+
return self._process_spans(spans)
|
|
49
|
+
|
|
50
|
+
def _process_spans(self, spans: Sequence[ReadableSpan], is_root_span: bool = False) -> SpanExportResult:
|
|
37
51
|
for span in spans:
|
|
38
52
|
if span.context.trace_id != self.current_trace_id:
|
|
39
53
|
self.rotate_file(span.resource.attributes[SERVICE_NAME],
|
|
@@ -60,4 +74,6 @@ class FileSpanExporter(SpanExporter):
|
|
|
60
74
|
self.out_handle = None
|
|
61
75
|
|
|
62
76
|
def shutdown(self) -> None:
|
|
77
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
78
|
+
self.task_processor.stop()
|
|
63
79
|
self.reset_handle()
|
|
@@ -3,7 +3,9 @@ import os
|
|
|
3
3
|
import logging
|
|
4
4
|
from importlib import import_module
|
|
5
5
|
from opentelemetry.sdk.trace.export import SpanExporter, ConsoleSpanExporter
|
|
6
|
+
from monocle_apptrace.exporters.exporter_processor import LambdaExportTaskProcessor, is_aws_lambda_environment
|
|
6
7
|
from monocle_apptrace.exporters.file_exporter import FileSpanExporter
|
|
8
|
+
|
|
7
9
|
logger = logging.getLogger(__name__)
|
|
8
10
|
|
|
9
11
|
monocle_exporters: Dict[str, Any] = {
|
|
@@ -20,6 +22,9 @@ def get_monocle_exporter() -> List[SpanExporter]:
|
|
|
20
22
|
# Retrieve the MONOCLE_EXPORTER environment variable and split it into a list
|
|
21
23
|
exporter_names = os.environ.get("MONOCLE_EXPORTER", "file").split(",")
|
|
22
24
|
exporters = []
|
|
25
|
+
|
|
26
|
+
# Create task processor for AWS Lambda environment
|
|
27
|
+
task_processor = LambdaExportTaskProcessor() if is_aws_lambda_environment() else None
|
|
23
28
|
|
|
24
29
|
for exporter_name in exporter_names:
|
|
25
30
|
exporter_name = exporter_name.strip()
|
|
@@ -31,7 +36,11 @@ def get_monocle_exporter() -> List[SpanExporter]:
|
|
|
31
36
|
try:
|
|
32
37
|
exporter_module = import_module(exporter_class_path["module"])
|
|
33
38
|
exporter_class = getattr(exporter_module, exporter_class_path["class"])
|
|
34
|
-
exporters
|
|
39
|
+
# Pass task_processor to all exporters when in AWS Lambda environment
|
|
40
|
+
if task_processor is not None and exporter_module.__name__.startswith("monocle_apptrace"):
|
|
41
|
+
exporters.append(exporter_class(task_processor=task_processor))
|
|
42
|
+
else:
|
|
43
|
+
exporters.append(exporter_class())
|
|
35
44
|
except Exception as ex:
|
|
36
45
|
logger.debug(
|
|
37
46
|
f"Unable to initialize Monocle span exporter '{exporter_name}', error: {ex}. Using ConsoleSpanExporter as a fallback.")
|
|
@@ -48,7 +48,7 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
48
48
|
|
|
49
49
|
if self._closed:
|
|
50
50
|
logger.warning("Exporter already shutdown, ignoring batch")
|
|
51
|
-
return SpanExportResult.
|
|
51
|
+
return SpanExportResult.FAILURE
|
|
52
52
|
if len(spans) == 0:
|
|
53
53
|
return
|
|
54
54
|
|
|
@@ -69,7 +69,10 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
69
69
|
obj["context"]["span_id"] = remove_0x_from_start(obj["context"]["span_id"])
|
|
70
70
|
span_list["batch"].append(obj)
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
73
|
+
is_root_span = any(not span.parent for span in spans)
|
|
74
|
+
|
|
75
|
+
def send_spans_to_okahu(span_list_local=None, is_root=False):
|
|
73
76
|
try:
|
|
74
77
|
result = self.session.post(
|
|
75
78
|
url=self.endpoint,
|
|
@@ -83,18 +86,17 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
83
86
|
result.text,
|
|
84
87
|
)
|
|
85
88
|
return SpanExportResult.FAILURE
|
|
86
|
-
logger.
|
|
89
|
+
logger.debug("spans successfully exported to okahu. Is root span: %s", is_root)
|
|
87
90
|
return SpanExportResult.SUCCESS
|
|
88
91
|
except ReadTimeout as e:
|
|
89
92
|
logger.warning("Trace export timed out: %s", str(e))
|
|
90
93
|
return SpanExportResult.FAILURE
|
|
91
94
|
|
|
92
95
|
# if async task function is present, then push the request to asnc task
|
|
93
|
-
|
|
94
96
|
if self.task_processor is not None and callable(self.task_processor.queue_task):
|
|
95
|
-
self.task_processor.queue_task(send_spans_to_okahu, span_list)
|
|
97
|
+
self.task_processor.queue_task(send_spans_to_okahu, span_list, is_root_span)
|
|
96
98
|
return SpanExportResult.SUCCESS
|
|
97
|
-
return send_spans_to_okahu(span_list)
|
|
99
|
+
return send_spans_to_okahu(span_list, is_root_span)
|
|
98
100
|
|
|
99
101
|
def shutdown(self) -> None:
|
|
100
102
|
if self._closed:
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .common import *
|
|
@@ -37,6 +37,19 @@ service_name_map = {
|
|
|
37
37
|
GITHUB_CODESPACE_SERVICE_NAME: GITHUB_CODESPACE_IDENTIFIER_ENV_NAME
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
|
|
41
|
+
llm_type_map = {
|
|
42
|
+
"sagemakerendpoint": "aws_sagemaker",
|
|
43
|
+
"azureopenai": "azure_openai",
|
|
44
|
+
"openai": "openai",
|
|
45
|
+
"chatopenai": "openai",
|
|
46
|
+
"azurechatopenai": "azure_openai",
|
|
47
|
+
"bedrock": "aws_bedrock",
|
|
48
|
+
"sagemakerllm": "aws_sagemaker",
|
|
49
|
+
"chatbedrock": "aws_bedrock",
|
|
50
|
+
"openaigenerator": "openai",
|
|
51
|
+
}
|
|
52
|
+
|
|
40
53
|
MONOCLE_INSTRUMENTOR = "monocle_apptrace"
|
|
41
54
|
WORKFLOW_TYPE_KEY = "workflow_type"
|
|
42
55
|
DATA_INPUT_KEY = "data.input"
|
|
@@ -52,3 +65,7 @@ MONOCLE_SCOPE_NAME_PREFIX = "monocle.scope."
|
|
|
52
65
|
SCOPE_METHOD_LIST = 'MONOCLE_SCOPE_METHODS'
|
|
53
66
|
SCOPE_METHOD_FILE = 'monocle_scopes.json'
|
|
54
67
|
SCOPE_CONFIG_PATH = 'MONOCLE_SCOPE_CONFIG_PATH'
|
|
68
|
+
TRACE_PROPOGATION_URLS = "MONOCLE_TRACE_PROPAGATATION_URLS"
|
|
69
|
+
WORKFLOW_TYPE_KEY = "monocle.workflow_type"
|
|
70
|
+
WORKFLOW_TYPE_GENERIC = "workflow.generic"
|
|
71
|
+
MONOCLE_SDK_VERSION = "monocle_apptrace.version"
|