monocle-apptrace 0.3.0b6__py3-none-any.whl → 0.3.0b7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of monocle-apptrace might be problematic. Click here for more details.
- monocle_apptrace/__init__.py +1 -0
- monocle_apptrace/exporters/aws/s3_exporter.py +20 -6
- monocle_apptrace/exporters/aws/s3_exporter_opendal.py +22 -11
- monocle_apptrace/exporters/azure/blob_exporter.py +22 -8
- monocle_apptrace/exporters/azure/blob_exporter_opendal.py +23 -8
- monocle_apptrace/exporters/exporter_processor.py +128 -3
- monocle_apptrace/exporters/file_exporter.py +16 -0
- monocle_apptrace/exporters/monocle_exporters.py +10 -1
- monocle_apptrace/exporters/okahu/okahu_exporter.py +8 -6
- monocle_apptrace/instrumentation/__init__.py +1 -0
- monocle_apptrace/instrumentation/common/__init__.py +2 -0
- monocle_apptrace/instrumentation/common/instrumentor.py +86 -12
- monocle_apptrace/instrumentation/common/span_handler.py +11 -4
- monocle_apptrace/instrumentation/common/utils.py +46 -17
- monocle_apptrace/instrumentation/common/wrapper.py +6 -4
- monocle_apptrace/instrumentation/metamodel/botocore/handlers/botocore_span_handler.py +2 -1
- {monocle_apptrace-0.3.0b6.dist-info → monocle_apptrace-0.3.0b7.dist-info}/METADATA +1 -1
- {monocle_apptrace-0.3.0b6.dist-info → monocle_apptrace-0.3.0b7.dist-info}/RECORD +21 -21
- {monocle_apptrace-0.3.0b6.dist-info → monocle_apptrace-0.3.0b7.dist-info}/WHEEL +0 -0
- {monocle_apptrace-0.3.0b6.dist-info → monocle_apptrace-0.3.0b7.dist-info}/licenses/LICENSE +0 -0
- {monocle_apptrace-0.3.0b6.dist-info → monocle_apptrace-0.3.0b7.dist-info}/licenses/NOTICE +0 -0
monocle_apptrace/__init__.py
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .instrumentation import *
|
|
@@ -16,12 +16,13 @@ from botocore.exceptions import (
|
|
|
16
16
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
17
17
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
18
18
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
19
|
-
from
|
|
19
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
20
|
+
from typing import Sequence, Optional
|
|
20
21
|
import json
|
|
21
22
|
logger = logging.getLogger(__name__)
|
|
22
23
|
|
|
23
24
|
class S3SpanExporter(SpanExporterBase):
|
|
24
|
-
def __init__(self, bucket_name=None, region_name=None):
|
|
25
|
+
def __init__(self, bucket_name=None, region_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
25
26
|
super().__init__()
|
|
26
27
|
# Use environment variables if credentials are not provided
|
|
27
28
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
@@ -47,6 +48,9 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
47
48
|
self.time_format = DEFAULT_TIME_FORMAT
|
|
48
49
|
self.export_queue = []
|
|
49
50
|
self.last_export_time = time.time()
|
|
51
|
+
self.task_processor = task_processor
|
|
52
|
+
if self.task_processor is not None:
|
|
53
|
+
self.task_processor.start()
|
|
50
54
|
|
|
51
55
|
# Check if bucket exists or create it
|
|
52
56
|
if not self.__bucket_exists(self.bucket_name):
|
|
@@ -92,6 +96,7 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
92
96
|
"""Synchronous export method that internally handles async logic."""
|
|
93
97
|
try:
|
|
94
98
|
# Run the asynchronous export logic in an event loop
|
|
99
|
+
logger.info(f"Exporting {len(spans)} spans to S3.")
|
|
95
100
|
asyncio.run(self.__export_async(spans))
|
|
96
101
|
return SpanExportResult.SUCCESS
|
|
97
102
|
except Exception as e:
|
|
@@ -100,6 +105,7 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
100
105
|
|
|
101
106
|
async def __export_async(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
102
107
|
try:
|
|
108
|
+
logger.info(f"__export_async {len(spans)} spans to S3.")
|
|
103
109
|
# Add spans to the export queue
|
|
104
110
|
for span in spans:
|
|
105
111
|
self.export_queue.append(span)
|
|
@@ -142,10 +148,16 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
142
148
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
143
149
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
144
150
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
151
|
+
# to calculate is_root_span loop over each span in batch_to_export and check if parent id is none or null
|
|
152
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
153
|
+
logger.info(f"Exporting {len(batch_to_export)} spans to S3 is_root_span : {is_root_span}.")
|
|
154
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
155
|
+
self.task_processor.queue_task(self.__upload_to_s3, serialized_data, is_root_span)
|
|
156
|
+
else:
|
|
157
|
+
try:
|
|
158
|
+
self.__upload_to_s3(serialized_data)
|
|
159
|
+
except Exception as e:
|
|
160
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
149
161
|
|
|
150
162
|
@SpanExporterBase.retry_with_backoff(exceptions=(EndpointConnectionError, ConnectionClosedError, ReadTimeoutError, ConnectTimeoutError))
|
|
151
163
|
def __upload_to_s3(self, span_data_batch: str):
|
|
@@ -164,4 +176,6 @@ class S3SpanExporter(SpanExporterBase):
|
|
|
164
176
|
return True
|
|
165
177
|
|
|
166
178
|
def shutdown(self) -> None:
|
|
179
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
180
|
+
self.task_processor.stop()
|
|
167
181
|
logger.info("S3SpanExporter has been shut down.")
|
|
@@ -3,19 +3,19 @@ import time
|
|
|
3
3
|
import datetime
|
|
4
4
|
import logging
|
|
5
5
|
import asyncio
|
|
6
|
-
from typing import Sequence
|
|
6
|
+
from typing import Sequence, Optional
|
|
7
7
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
8
8
|
from opentelemetry.sdk.trace.export import SpanExportResult
|
|
9
9
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
10
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
10
11
|
from opendal import Operator
|
|
11
12
|
from opendal.exceptions import PermissionDenied, ConfigInvalid, Unexpected
|
|
12
13
|
|
|
13
|
-
|
|
14
14
|
import json
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
17
17
|
class OpenDALS3Exporter(SpanExporterBase):
|
|
18
|
-
def __init__(self, bucket_name=None, region_name=None):
|
|
18
|
+
def __init__(self, bucket_name=None, region_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
19
19
|
super().__init__()
|
|
20
20
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
21
21
|
DEFAULT_TIME_FORMAT = "%Y-%m-%d__%H.%M.%S"
|
|
@@ -36,7 +36,10 @@ class OpenDALS3Exporter(SpanExporterBase):
|
|
|
36
36
|
access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
|
|
37
37
|
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
|
|
38
38
|
)
|
|
39
|
-
|
|
39
|
+
|
|
40
|
+
self.task_processor = task_processor
|
|
41
|
+
if self.task_processor is not None:
|
|
42
|
+
self.task_processor.start()
|
|
40
43
|
|
|
41
44
|
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
42
45
|
"""Synchronous export method that internally handles async logic."""
|
|
@@ -88,20 +91,26 @@ class OpenDALS3Exporter(SpanExporterBase):
|
|
|
88
91
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
89
92
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
90
93
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
94
|
+
|
|
95
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
96
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
97
|
+
|
|
98
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
99
|
+
self.task_processor.queue_task(self.__upload_to_s3, serialized_data, is_root_span)
|
|
100
|
+
else:
|
|
101
|
+
try:
|
|
102
|
+
self.__upload_to_s3(serialized_data, is_root_span)
|
|
103
|
+
except Exception as e:
|
|
104
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
95
105
|
|
|
96
106
|
@SpanExporterBase.retry_with_backoff(exceptions=(Unexpected))
|
|
97
|
-
def __upload_to_s3(self, span_data_batch: str):
|
|
98
|
-
|
|
107
|
+
def __upload_to_s3(self, span_data_batch: str, is_root_span: bool = False):
|
|
99
108
|
current_time = datetime.datetime.now().strftime(self.time_format)
|
|
100
109
|
file_name = f"{self.file_prefix}{current_time}.ndjson"
|
|
101
110
|
try:
|
|
102
111
|
# Attempt to write the span data batch to S3
|
|
103
112
|
self.op.write(file_name, span_data_batch.encode("utf-8"))
|
|
104
|
-
logger.info(f"Span batch uploaded to S3 as {file_name}.")
|
|
113
|
+
logger.info(f"Span batch uploaded to S3 as {file_name}. Is root span: {is_root_span}")
|
|
105
114
|
|
|
106
115
|
except PermissionDenied as e:
|
|
107
116
|
# S3 bucket is forbidden.
|
|
@@ -123,4 +132,6 @@ class OpenDALS3Exporter(SpanExporterBase):
|
|
|
123
132
|
return True
|
|
124
133
|
|
|
125
134
|
def shutdown(self) -> None:
|
|
135
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
136
|
+
self.task_processor.stop()
|
|
126
137
|
logger.info("S3SpanExporter has been shut down.")
|
|
@@ -8,14 +8,15 @@ from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
|
|
|
8
8
|
from azure.core.exceptions import ResourceNotFoundError, ClientAuthenticationError, ServiceRequestError
|
|
9
9
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
10
10
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
11
|
-
from typing import Sequence
|
|
11
|
+
from typing import Sequence, Optional
|
|
12
12
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
13
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
13
14
|
import json
|
|
14
15
|
from monocle_apptrace.instrumentation.common.constants import MONOCLE_SDK_VERSION
|
|
15
16
|
logger = logging.getLogger(__name__)
|
|
16
17
|
|
|
17
18
|
class AzureBlobSpanExporter(SpanExporterBase):
|
|
18
|
-
def __init__(self, connection_string=None, container_name=None):
|
|
19
|
+
def __init__(self, connection_string=None, container_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
19
20
|
super().__init__()
|
|
20
21
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
21
22
|
DEFAULT_TIME_FORMAT = "%Y-%m-%d_%H.%M.%S"
|
|
@@ -44,6 +45,10 @@ class AzureBlobSpanExporter(SpanExporterBase):
|
|
|
44
45
|
logger.error(f"Error creating container {container_name}: {e}")
|
|
45
46
|
raise e
|
|
46
47
|
|
|
48
|
+
self.task_processor = task_processor
|
|
49
|
+
if self.task_processor is not None:
|
|
50
|
+
self.task_processor.start()
|
|
51
|
+
|
|
47
52
|
def __container_exists(self, container_name):
|
|
48
53
|
try:
|
|
49
54
|
container_client = self.blob_service_client.get_container_client(container_name)
|
|
@@ -111,22 +116,31 @@ class AzureBlobSpanExporter(SpanExporterBase):
|
|
|
111
116
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
112
117
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
113
118
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
119
|
+
|
|
120
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
121
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
122
|
+
|
|
123
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
124
|
+
self.task_processor.queue_task(self.__upload_to_blob, serialized_data, is_root_span)
|
|
125
|
+
else:
|
|
126
|
+
try:
|
|
127
|
+
self.__upload_to_blob(serialized_data, is_root_span)
|
|
128
|
+
except Exception as e:
|
|
129
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
118
130
|
|
|
119
131
|
@SpanExporterBase.retry_with_backoff(exceptions=(ResourceNotFoundError, ClientAuthenticationError, ServiceRequestError))
|
|
120
|
-
def __upload_to_blob(self, span_data_batch: str):
|
|
132
|
+
def __upload_to_blob(self, span_data_batch: str, is_root_span: bool = False):
|
|
121
133
|
current_time = datetime.datetime.now().strftime(self.time_format)
|
|
122
134
|
file_name = f"{self.file_prefix}{current_time}.ndjson"
|
|
123
135
|
blob_client = self.blob_service_client.get_blob_client(container=self.container_name, blob=file_name)
|
|
124
136
|
blob_client.upload_blob(span_data_batch, overwrite=True)
|
|
125
|
-
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}.")
|
|
137
|
+
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}. Is root span: {is_root_span}")
|
|
126
138
|
|
|
127
139
|
async def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
128
140
|
await self.__export_spans()
|
|
129
141
|
return True
|
|
130
142
|
|
|
131
143
|
def shutdown(self) -> None:
|
|
144
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
145
|
+
self.task_processor.stop()
|
|
132
146
|
logger.info("AzureBlobSpanExporter has been shut down.")
|
|
@@ -5,16 +5,17 @@ import logging
|
|
|
5
5
|
import asyncio
|
|
6
6
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
7
7
|
from opentelemetry.sdk.trace.export import SpanExportResult
|
|
8
|
-
from typing import Sequence
|
|
8
|
+
from typing import Sequence, Optional
|
|
9
9
|
from opendal import Operator
|
|
10
10
|
from monocle_apptrace.exporters.base_exporter import SpanExporterBase
|
|
11
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
11
12
|
from opendal.exceptions import Unexpected, PermissionDenied, NotFound
|
|
12
13
|
import json
|
|
13
14
|
|
|
14
15
|
logger = logging.getLogger(__name__)
|
|
15
16
|
|
|
16
17
|
class OpenDALAzureExporter(SpanExporterBase):
|
|
17
|
-
def __init__(self, connection_string=None, container_name=None):
|
|
18
|
+
def __init__(self, connection_string=None, container_name=None, task_processor: Optional[ExportTaskProcessor] = None):
|
|
18
19
|
super().__init__()
|
|
19
20
|
DEFAULT_FILE_PREFIX = "monocle_trace_"
|
|
20
21
|
DEFAULT_TIME_FORMAT = "%Y-%m-%d_%H.%M.%S"
|
|
@@ -25,6 +26,8 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
25
26
|
# Default values
|
|
26
27
|
self.file_prefix = DEFAULT_FILE_PREFIX
|
|
27
28
|
self.time_format = DEFAULT_TIME_FORMAT
|
|
29
|
+
self.export_queue = [] # Add this line to initialize export_queue
|
|
30
|
+
self.last_export_time = time.time() # Add this line to initialize last_export_time
|
|
28
31
|
|
|
29
32
|
# Validate input
|
|
30
33
|
if not connection_string:
|
|
@@ -51,6 +54,9 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
51
54
|
except Exception as e:
|
|
52
55
|
raise RuntimeError(f"Failed to initialize OpenDAL operator: {e}")
|
|
53
56
|
|
|
57
|
+
self.task_processor = task_processor
|
|
58
|
+
if self.task_processor is not None:
|
|
59
|
+
self.task_processor.start()
|
|
54
60
|
|
|
55
61
|
def parse_connection_string(self,connection_string):
|
|
56
62
|
connection_params = dict(item.split('=', 1) for item in connection_string.split(';') if '=' in item)
|
|
@@ -112,19 +118,26 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
112
118
|
batch_to_export = self.export_queue[:self.max_batch_size]
|
|
113
119
|
serialized_data = self.__serialize_spans(batch_to_export)
|
|
114
120
|
self.export_queue = self.export_queue[self.max_batch_size:]
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
121
|
+
|
|
122
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
123
|
+
is_root_span = any(not span.parent for span in batch_to_export)
|
|
124
|
+
|
|
125
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
126
|
+
self.task_processor.queue_task(self.__upload_to_opendal, serialized_data, is_root_span)
|
|
127
|
+
else:
|
|
128
|
+
try:
|
|
129
|
+
self.__upload_to_opendal(serialized_data, is_root_span)
|
|
130
|
+
except Exception as e:
|
|
131
|
+
logger.error(f"Failed to upload span batch: {e}")
|
|
119
132
|
|
|
120
133
|
@SpanExporterBase.retry_with_backoff(exceptions=(Unexpected,))
|
|
121
|
-
def __upload_to_opendal(self, span_data_batch: str):
|
|
134
|
+
def __upload_to_opendal(self, span_data_batch: str, is_root_span: bool = False):
|
|
122
135
|
current_time = datetime.datetime.now().strftime(self.time_format)
|
|
123
136
|
file_name = f"{self.file_prefix}{current_time}.ndjson"
|
|
124
137
|
|
|
125
138
|
try:
|
|
126
139
|
self.operator.write(file_name, span_data_batch.encode('utf-8'))
|
|
127
|
-
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}.")
|
|
140
|
+
logger.info(f"Span batch uploaded to Azure Blob Storage as {file_name}. Is root span: {is_root_span}")
|
|
128
141
|
except PermissionDenied as e:
|
|
129
142
|
# Azure Container is forbidden.
|
|
130
143
|
logger.error(f"Access to container {self.container_name} is forbidden (403).")
|
|
@@ -144,4 +157,6 @@ class OpenDALAzureExporter(SpanExporterBase):
|
|
|
144
157
|
return True
|
|
145
158
|
|
|
146
159
|
def shutdown(self) -> None:
|
|
160
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
161
|
+
self.task_processor.stop()
|
|
147
162
|
logger.info("OpenDALAzureExporter has been shut down.")
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
import logging
|
|
3
|
+
import os
|
|
4
|
+
import queue
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
3
7
|
from typing import Callable
|
|
8
|
+
import requests
|
|
9
|
+
from monocle_apptrace.instrumentation.common.constants import AWS_LAMBDA_ENV_NAME
|
|
4
10
|
|
|
5
11
|
logger = logging.getLogger(__name__)
|
|
12
|
+
LAMBDA_EXTENSION_NAME = "AsyncProcessorMonocle"
|
|
6
13
|
|
|
7
14
|
class ExportTaskProcessor(ABC):
|
|
8
|
-
|
|
15
|
+
|
|
9
16
|
@abstractmethod
|
|
10
17
|
def start(self):
|
|
11
18
|
return
|
|
@@ -15,5 +22,123 @@ class ExportTaskProcessor(ABC):
|
|
|
15
22
|
return
|
|
16
23
|
|
|
17
24
|
@abstractmethod
|
|
18
|
-
def queue_task(self, async_task: Callable[[Callable, any], any] = None, args: any = None):
|
|
19
|
-
return
|
|
25
|
+
def queue_task(self, async_task: Callable[[Callable, any], any] = None, args: any = None, is_root_span: bool = False):
|
|
26
|
+
return
|
|
27
|
+
|
|
28
|
+
class LambdaExportTaskProcessor(ExportTaskProcessor):
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
span_check_interval_seconds: int = 1,
|
|
33
|
+
max_time_allowed_seconds: int = 30):
|
|
34
|
+
# An internal queue used by the handler to notify the extension that it can
|
|
35
|
+
# start processing the async task.
|
|
36
|
+
self.async_tasks_queue = queue.Queue()
|
|
37
|
+
self.span_check_interval = span_check_interval_seconds
|
|
38
|
+
self.max_time_allowed = max_time_allowed_seconds
|
|
39
|
+
|
|
40
|
+
def start(self):
|
|
41
|
+
try:
|
|
42
|
+
self._start_async_processor()
|
|
43
|
+
except Exception as e:
|
|
44
|
+
logger.error(f"LambdaExportTaskProcessor| Failed to start. {e}")
|
|
45
|
+
|
|
46
|
+
def stop(self):
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
def queue_task(self, async_task=None, args=None, is_root_span=False):
|
|
50
|
+
self.async_tasks_queue.put((async_task, args, is_root_span))
|
|
51
|
+
|
|
52
|
+
def set_sagemaker_model(self, endpoint_name: str, span: dict[str, dict[str, str]]):
|
|
53
|
+
try:
|
|
54
|
+
try:
|
|
55
|
+
import boto3
|
|
56
|
+
except ImportError:
|
|
57
|
+
logger.error("LambdaExportTaskProcessor| Failed to import boto3")
|
|
58
|
+
return
|
|
59
|
+
|
|
60
|
+
client = boto3.client('sagemaker')
|
|
61
|
+
response = client.describe_endpoint(
|
|
62
|
+
EndpointName=endpoint_name
|
|
63
|
+
)
|
|
64
|
+
endpoint_config_name = response["EndpointConfigName"]
|
|
65
|
+
endpoint_config_response = client.describe_endpoint_config(
|
|
66
|
+
EndpointConfigName=endpoint_config_name
|
|
67
|
+
)
|
|
68
|
+
model_name = endpoint_config_response["ProductionVariants"][0]["ModelName"]
|
|
69
|
+
model_name_response = client.describe_model(ModelName = model_name)
|
|
70
|
+
model_name_id = ""
|
|
71
|
+
try:
|
|
72
|
+
model_name_id = model_name_response["PrimaryContainer"]["Environment"]["HF_MODEL_ID"]
|
|
73
|
+
except:
|
|
74
|
+
pass
|
|
75
|
+
span["attributes"]["model_name"] = model_name_id
|
|
76
|
+
except Exception as e:
|
|
77
|
+
logger.error(f"LambdaExportTaskProcessor| Failed to get sagemaker model. {e}")
|
|
78
|
+
|
|
79
|
+
def update_spans(self, export_args):
|
|
80
|
+
try:
|
|
81
|
+
if 'batch' in export_args:
|
|
82
|
+
for span in export_args["batch"]:
|
|
83
|
+
try:
|
|
84
|
+
if len(span["attributes"]["sagemaker_endpoint_name"]) > 0 :
|
|
85
|
+
self.set_sagemaker_model(endpoint_name=span["attributes"]["sagemaker_endpoint_name"], span=span)
|
|
86
|
+
except:
|
|
87
|
+
pass
|
|
88
|
+
except Exception as e:
|
|
89
|
+
logger.error(f"LambdaExportTaskProcessor| Failed to update spans. {e}")
|
|
90
|
+
|
|
91
|
+
def _start_async_processor(self):
|
|
92
|
+
# Register internal extension
|
|
93
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Registering with Lambda service...")
|
|
94
|
+
response = requests.post(
|
|
95
|
+
url=f"http://{os.environ['AWS_LAMBDA_RUNTIME_API']}/2020-01-01/extension/register",
|
|
96
|
+
json={'events': ['INVOKE']},
|
|
97
|
+
headers={'Lambda-Extension-Name': LAMBDA_EXTENSION_NAME}
|
|
98
|
+
)
|
|
99
|
+
ext_id = response.headers['Lambda-Extension-Identifier']
|
|
100
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Registered with ID: {ext_id}")
|
|
101
|
+
|
|
102
|
+
def process_tasks():
|
|
103
|
+
while True:
|
|
104
|
+
# Call /next to get notified when there is a new invocation and let
|
|
105
|
+
# Lambda know that we are done processing the previous task.
|
|
106
|
+
|
|
107
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Waiting for invocation...")
|
|
108
|
+
response = requests.get(
|
|
109
|
+
url=f"http://{os.environ['AWS_LAMBDA_RUNTIME_API']}/2020-01-01/extension/event/next",
|
|
110
|
+
headers={'Lambda-Extension-Identifier': ext_id},
|
|
111
|
+
timeout=None
|
|
112
|
+
)
|
|
113
|
+
root_span_found = False
|
|
114
|
+
# all values in seconds
|
|
115
|
+
total_time_elapsed = 0
|
|
116
|
+
while root_span_found is False and total_time_elapsed < self.max_time_allowed:
|
|
117
|
+
logger.debug(response.json())
|
|
118
|
+
# Get next task from internal queue
|
|
119
|
+
logger.info(f"[{LAMBDA_EXTENSION_NAME}] Async thread running, waiting for task from handler")
|
|
120
|
+
while self.async_tasks_queue.empty() is False :
|
|
121
|
+
logger.info(f"[{LAMBDA_EXTENSION_NAME}] Processing task from handler")
|
|
122
|
+
async_task, arg, is_root_span = self.async_tasks_queue.get()
|
|
123
|
+
root_span_found = is_root_span
|
|
124
|
+
# self.update_spans(export_args=arg)
|
|
125
|
+
|
|
126
|
+
if async_task is None:
|
|
127
|
+
# No task to run this invocation
|
|
128
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Received null task. Ignoring.")
|
|
129
|
+
else:
|
|
130
|
+
# Invoke task
|
|
131
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Received async task from handler. Starting task.")
|
|
132
|
+
async_task(arg)
|
|
133
|
+
total_time_elapsed+=self.span_check_interval
|
|
134
|
+
logger.info(f"[{LAMBDA_EXTENSION_NAME}] Waiting for root span. total_time_elapsed: {total_time_elapsed}, root_span_found: {root_span_found}.")
|
|
135
|
+
time.sleep(self.span_check_interval)
|
|
136
|
+
|
|
137
|
+
logger.debug(f"[{LAMBDA_EXTENSION_NAME}] Finished processing task. total_time_elapsed: {total_time_elapsed}, root_span_found: {root_span_found}.")
|
|
138
|
+
|
|
139
|
+
# Start processing extension events in a separate thread
|
|
140
|
+
threading.Thread(target=process_tasks, daemon=True, name=LAMBDA_EXTENSION_NAME).start()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def is_aws_lambda_environment():
|
|
144
|
+
return AWS_LAMBDA_ENV_NAME in os.environ
|
|
@@ -7,6 +7,7 @@ from typing import Optional, Callable, Sequence
|
|
|
7
7
|
from opentelemetry.sdk.trace import ReadableSpan
|
|
8
8
|
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
9
9
|
from opentelemetry.sdk.resources import SERVICE_NAME
|
|
10
|
+
from monocle_apptrace.exporters.exporter_processor import ExportTaskProcessor
|
|
10
11
|
|
|
11
12
|
DEFAULT_FILE_PREFIX:str = "monocle_trace_"
|
|
12
13
|
DEFAULT_TIME_FORMAT:str = "%Y-%m-%d_%H.%M.%S"
|
|
@@ -25,6 +26,7 @@ class FileSpanExporter(SpanExporter):
|
|
|
25
26
|
[ReadableSpan], str
|
|
26
27
|
] = lambda span: span.to_json()
|
|
27
28
|
+ linesep,
|
|
29
|
+
task_processor: Optional[ExportTaskProcessor] = None
|
|
28
30
|
):
|
|
29
31
|
self.out_handle:TextIOWrapper = None
|
|
30
32
|
self.formatter = formatter
|
|
@@ -32,8 +34,20 @@ class FileSpanExporter(SpanExporter):
|
|
|
32
34
|
self.output_path = out_path
|
|
33
35
|
self.file_prefix = file_prefix
|
|
34
36
|
self.time_format = time_format
|
|
37
|
+
self.task_processor = task_processor
|
|
38
|
+
if self.task_processor is not None:
|
|
39
|
+
self.task_processor.start()
|
|
35
40
|
|
|
36
41
|
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
42
|
+
if self.task_processor is not None and callable(getattr(self.task_processor, 'queue_task', None)):
|
|
43
|
+
# Check if any span is a root span (no parent)
|
|
44
|
+
is_root_span = any(not span.parent for span in spans)
|
|
45
|
+
self.task_processor.queue_task(self._process_spans, spans, is_root_span)
|
|
46
|
+
return SpanExportResult.SUCCESS
|
|
47
|
+
else:
|
|
48
|
+
return self._process_spans(spans)
|
|
49
|
+
|
|
50
|
+
def _process_spans(self, spans: Sequence[ReadableSpan], is_root_span: bool = False) -> SpanExportResult:
|
|
37
51
|
for span in spans:
|
|
38
52
|
if span.context.trace_id != self.current_trace_id:
|
|
39
53
|
self.rotate_file(span.resource.attributes[SERVICE_NAME],
|
|
@@ -60,4 +74,6 @@ class FileSpanExporter(SpanExporter):
|
|
|
60
74
|
self.out_handle = None
|
|
61
75
|
|
|
62
76
|
def shutdown(self) -> None:
|
|
77
|
+
if hasattr(self, 'task_processor') and self.task_processor is not None:
|
|
78
|
+
self.task_processor.stop()
|
|
63
79
|
self.reset_handle()
|
|
@@ -3,7 +3,9 @@ import os
|
|
|
3
3
|
import logging
|
|
4
4
|
from importlib import import_module
|
|
5
5
|
from opentelemetry.sdk.trace.export import SpanExporter, ConsoleSpanExporter
|
|
6
|
+
from monocle_apptrace.exporters.exporter_processor import LambdaExportTaskProcessor, is_aws_lambda_environment
|
|
6
7
|
from monocle_apptrace.exporters.file_exporter import FileSpanExporter
|
|
8
|
+
|
|
7
9
|
logger = logging.getLogger(__name__)
|
|
8
10
|
|
|
9
11
|
monocle_exporters: Dict[str, Any] = {
|
|
@@ -20,6 +22,9 @@ def get_monocle_exporter() -> List[SpanExporter]:
|
|
|
20
22
|
# Retrieve the MONOCLE_EXPORTER environment variable and split it into a list
|
|
21
23
|
exporter_names = os.environ.get("MONOCLE_EXPORTER", "file").split(",")
|
|
22
24
|
exporters = []
|
|
25
|
+
|
|
26
|
+
# Create task processor for AWS Lambda environment
|
|
27
|
+
task_processor = LambdaExportTaskProcessor() if is_aws_lambda_environment() else None
|
|
23
28
|
|
|
24
29
|
for exporter_name in exporter_names:
|
|
25
30
|
exporter_name = exporter_name.strip()
|
|
@@ -31,7 +36,11 @@ def get_monocle_exporter() -> List[SpanExporter]:
|
|
|
31
36
|
try:
|
|
32
37
|
exporter_module = import_module(exporter_class_path["module"])
|
|
33
38
|
exporter_class = getattr(exporter_module, exporter_class_path["class"])
|
|
34
|
-
exporters
|
|
39
|
+
# Pass task_processor to all exporters when in AWS Lambda environment
|
|
40
|
+
if task_processor is not None and exporter_module.__name__.startswith("monocle_apptrace"):
|
|
41
|
+
exporters.append(exporter_class(task_processor=task_processor))
|
|
42
|
+
else:
|
|
43
|
+
exporters.append(exporter_class())
|
|
35
44
|
except Exception as ex:
|
|
36
45
|
logger.debug(
|
|
37
46
|
f"Unable to initialize Monocle span exporter '{exporter_name}', error: {ex}. Using ConsoleSpanExporter as a fallback.")
|
|
@@ -48,7 +48,7 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
48
48
|
|
|
49
49
|
if self._closed:
|
|
50
50
|
logger.warning("Exporter already shutdown, ignoring batch")
|
|
51
|
-
return SpanExportResult.
|
|
51
|
+
return SpanExportResult.FAILURE
|
|
52
52
|
if len(spans) == 0:
|
|
53
53
|
return
|
|
54
54
|
|
|
@@ -69,7 +69,10 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
69
69
|
obj["context"]["span_id"] = remove_0x_from_start(obj["context"]["span_id"])
|
|
70
70
|
span_list["batch"].append(obj)
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
# Calculate is_root_span by checking if any span has no parent
|
|
73
|
+
is_root_span = any(not span.parent for span in spans)
|
|
74
|
+
|
|
75
|
+
def send_spans_to_okahu(span_list_local=None, is_root=False):
|
|
73
76
|
try:
|
|
74
77
|
result = self.session.post(
|
|
75
78
|
url=self.endpoint,
|
|
@@ -83,18 +86,17 @@ class OkahuSpanExporter(SpanExporter):
|
|
|
83
86
|
result.text,
|
|
84
87
|
)
|
|
85
88
|
return SpanExportResult.FAILURE
|
|
86
|
-
logger.debug("spans successfully exported to okahu")
|
|
89
|
+
logger.debug("spans successfully exported to okahu. Is root span: %s", is_root)
|
|
87
90
|
return SpanExportResult.SUCCESS
|
|
88
91
|
except ReadTimeout as e:
|
|
89
92
|
logger.warning("Trace export timed out: %s", str(e))
|
|
90
93
|
return SpanExportResult.FAILURE
|
|
91
94
|
|
|
92
95
|
# if async task function is present, then push the request to asnc task
|
|
93
|
-
|
|
94
96
|
if self.task_processor is not None and callable(self.task_processor.queue_task):
|
|
95
|
-
self.task_processor.queue_task(send_spans_to_okahu, span_list)
|
|
97
|
+
self.task_processor.queue_task(send_spans_to_okahu, span_list, is_root_span)
|
|
96
98
|
return SpanExportResult.SUCCESS
|
|
97
|
-
return send_spans_to_okahu(span_list)
|
|
99
|
+
return send_spans_to_okahu(span_list, is_root_span)
|
|
98
100
|
|
|
99
101
|
def shutdown(self) -> None:
|
|
100
102
|
if self._closed:
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .common import *
|
|
@@ -17,13 +17,13 @@ from opentelemetry.trace import get_tracer
|
|
|
17
17
|
from wrapt import wrap_function_wrapper
|
|
18
18
|
from opentelemetry.trace.propagation import set_span_in_context, _SPAN_KEY
|
|
19
19
|
from monocle_apptrace.exporters.monocle_exporters import get_monocle_exporter
|
|
20
|
-
from monocle_apptrace.instrumentation.common.span_handler import SpanHandler
|
|
20
|
+
from monocle_apptrace.instrumentation.common.span_handler import SpanHandler, NonFrameworkSpanHandler
|
|
21
21
|
from monocle_apptrace.instrumentation.common.wrapper_method import (
|
|
22
22
|
DEFAULT_METHODS_LIST,
|
|
23
23
|
WrapperMethod,
|
|
24
24
|
MONOCLE_SPAN_HANDLERS
|
|
25
25
|
)
|
|
26
|
-
from monocle_apptrace.instrumentation.common.wrapper import scope_wrapper, ascope_wrapper
|
|
26
|
+
from monocle_apptrace.instrumentation.common.wrapper import scope_wrapper, ascope_wrapper, wrapper_processor
|
|
27
27
|
from monocle_apptrace.instrumentation.common.utils import (
|
|
28
28
|
set_scope, remove_scope, http_route_handler, load_scopes, async_wrapper, http_async_route_handler
|
|
29
29
|
)
|
|
@@ -65,13 +65,11 @@ class MonocleInstrumentor(BaseInstrumentor):
|
|
|
65
65
|
def instrumented_endpoint_invoke(to_wrap,wrapped, span_name, instance,fn):
|
|
66
66
|
@wraps(fn)
|
|
67
67
|
def with_instrumentation(*args, **kwargs):
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
return response
|
|
74
|
-
|
|
68
|
+
async_task = inspect.iscoroutinefunction(fn)
|
|
69
|
+
boto_method_to_wrap = to_wrap.copy()
|
|
70
|
+
boto_method_to_wrap['skip_span'] = False
|
|
71
|
+
return wrapper_processor(async_task, tracer, NonFrameworkSpanHandler(),
|
|
72
|
+
boto_method_to_wrap, fn, instance, args, kwargs)
|
|
75
73
|
return with_instrumentation
|
|
76
74
|
return instrumented_endpoint_invoke
|
|
77
75
|
|
|
@@ -158,6 +156,24 @@ def setup_monocle_telemetry(
|
|
|
158
156
|
span_handlers: Dict[str,SpanHandler] = None,
|
|
159
157
|
wrapper_methods: List[Union[dict,WrapperMethod]] = None,
|
|
160
158
|
union_with_default_methods: bool = True) -> None:
|
|
159
|
+
"""
|
|
160
|
+
Set up Monocle telemetry for the application.
|
|
161
|
+
|
|
162
|
+
Parameters
|
|
163
|
+
----------
|
|
164
|
+
workflow_name : str
|
|
165
|
+
The name of the workflow to be used as the service name in telemetry.
|
|
166
|
+
span_processors : List[SpanProcessor], optional
|
|
167
|
+
Custom span processors to use instead of the default ones. If None,
|
|
168
|
+
BatchSpanProcessors with Monocle exporters will be used.
|
|
169
|
+
span_handlers : Dict[str, SpanHandler], optional
|
|
170
|
+
Dictionary of span handlers to be used by the instrumentor, mapping handler names to handler objects.
|
|
171
|
+
wrapper_methods : List[Union[dict, WrapperMethod]], optional
|
|
172
|
+
Custom wrapper methods for instrumentation. If None, default methods will be used.
|
|
173
|
+
union_with_default_methods : bool, default=True
|
|
174
|
+
If True, combine the provided wrapper_methods with the default methods.
|
|
175
|
+
If False, only use the provided wrapper_methods.
|
|
176
|
+
"""
|
|
161
177
|
resource = Resource(attributes={
|
|
162
178
|
SERVICE_NAME: workflow_name
|
|
163
179
|
})
|
|
@@ -196,6 +212,16 @@ def set_context_properties(properties: dict) -> None:
|
|
|
196
212
|
attach(set_value(SESSION_PROPERTIES_KEY, properties))
|
|
197
213
|
|
|
198
214
|
def start_trace():
|
|
215
|
+
"""
|
|
216
|
+
Starts a new trace. All the spans created after this call will be part of the same trace.
|
|
217
|
+
Returns:
|
|
218
|
+
Token: A token representing the attached context for the workflow span.
|
|
219
|
+
This token is to be used later to stop the current trace.
|
|
220
|
+
Returns None if tracing fails.
|
|
221
|
+
|
|
222
|
+
Raises:
|
|
223
|
+
Exception: The function catches all exceptions internally and logs a warning.
|
|
224
|
+
"""
|
|
199
225
|
try:
|
|
200
226
|
tracer = get_tracer(instrumenting_module_name= MONOCLE_INSTRUMENTOR, tracer_provider= get_tracer_provider())
|
|
201
227
|
span = tracer.start_span(name = "workflow")
|
|
@@ -209,6 +235,14 @@ def start_trace():
|
|
|
209
235
|
return None
|
|
210
236
|
|
|
211
237
|
def stop_trace(token) -> None:
|
|
238
|
+
"""
|
|
239
|
+
Stop the active trace and detach workflow type if token is provided. All the spans created after this will not be part of the trace.
|
|
240
|
+
Args:
|
|
241
|
+
token: The token that was returned when the trace was started. Used to detach
|
|
242
|
+
workflow type. Can be None in which case only the span is ended.
|
|
243
|
+
Returns:
|
|
244
|
+
None
|
|
245
|
+
"""
|
|
212
246
|
try:
|
|
213
247
|
_parent_span_context = get_current()
|
|
214
248
|
if _parent_span_context is not None:
|
|
@@ -229,32 +263,67 @@ def is_valid_trace_id_uuid(traceId: str) -> bool:
|
|
|
229
263
|
return False
|
|
230
264
|
|
|
231
265
|
def start_scope(scope_name: str, scope_value:str = None) -> object:
|
|
266
|
+
"""
|
|
267
|
+
Start a new scope with the given name and and optional value. If no value is provided, a random UUID will be generated.
|
|
268
|
+
All the spans, across traces created after this call will have the scope attached until the scope is stopped.
|
|
269
|
+
Args:
|
|
270
|
+
scope_name: The name of the scope.
|
|
271
|
+
scope_value: Optional value of the scope. If None, a random UUID will be generated.
|
|
272
|
+
Returns:
|
|
273
|
+
Token: A token representing the attached context for the scope. This token is to be used later to stop the current scope.
|
|
274
|
+
"""
|
|
232
275
|
return set_scope(scope_name, scope_value)
|
|
233
276
|
|
|
234
277
|
def stop_scope(token:object) -> None:
|
|
278
|
+
"""
|
|
279
|
+
Stop the active scope. All the spans created after this will not have the scope attached.
|
|
280
|
+
Args:
|
|
281
|
+
token: The token that was returned when the scope was started.
|
|
282
|
+
Returns:
|
|
283
|
+
None
|
|
284
|
+
"""
|
|
235
285
|
remove_scope(token)
|
|
236
286
|
return
|
|
237
287
|
|
|
288
|
+
@contextmanager
|
|
289
|
+
def monocle_trace():
|
|
290
|
+
"""
|
|
291
|
+
Context manager to start and stop a scope. All the spans, across traces created within the encapsulated code will have same trace ID
|
|
292
|
+
"""
|
|
293
|
+
token = start_trace()
|
|
294
|
+
try:
|
|
295
|
+
yield
|
|
296
|
+
finally:
|
|
297
|
+
stop_trace(token)
|
|
298
|
+
|
|
238
299
|
@contextmanager
|
|
239
300
|
def monocle_trace_scope(scope_name: str, scope_value:str = None):
|
|
301
|
+
"""
|
|
302
|
+
Context manager to start and stop a scope. All the spans, across traces created within the encapsulated code will have the scope attached.
|
|
303
|
+
Args:
|
|
304
|
+
scope_name: The name of the scope.
|
|
305
|
+
scope_value: Optional value of the scope. If None, a random UUID will be generated."""
|
|
240
306
|
token = start_scope(scope_name, scope_value)
|
|
241
307
|
try:
|
|
242
308
|
yield
|
|
243
309
|
finally:
|
|
244
310
|
stop_scope(token)
|
|
245
311
|
|
|
246
|
-
def monocle_trace_scope_method(scope_name: str):
|
|
312
|
+
def monocle_trace_scope_method(scope_name: str, scope_value:str=None):
|
|
313
|
+
"""
|
|
314
|
+
Decorator to start and stop a scope for a method. All the spans, across traces created in the method will have the scope attached.
|
|
315
|
+
"""
|
|
247
316
|
def decorator(func):
|
|
248
317
|
if inspect.iscoroutinefunction(func):
|
|
249
318
|
@wraps(func)
|
|
250
319
|
async def wrapper(*args, **kwargs):
|
|
251
|
-
result = async_wrapper(func, scope_name, None, *args, **kwargs)
|
|
320
|
+
result = async_wrapper(func, scope_name, scope_value, None, *args, **kwargs)
|
|
252
321
|
return result
|
|
253
322
|
return wrapper
|
|
254
323
|
else:
|
|
255
324
|
@wraps(func)
|
|
256
325
|
def wrapper(*args, **kwargs):
|
|
257
|
-
token = start_scope(scope_name)
|
|
326
|
+
token = start_scope(scope_name, scope_value)
|
|
258
327
|
try:
|
|
259
328
|
result = func(*args, **kwargs)
|
|
260
329
|
return result
|
|
@@ -264,6 +333,10 @@ def monocle_trace_scope_method(scope_name: str):
|
|
|
264
333
|
return decorator
|
|
265
334
|
|
|
266
335
|
def monocle_trace_http_route(func):
|
|
336
|
+
"""
|
|
337
|
+
Decorator to start and stop a continue traces and scope for a http route. It will also initiate new scopes from the http headers if configured in ``monocle_scopes.json``
|
|
338
|
+
All the spans, across traces created in the route will have the scope attached.
|
|
339
|
+
"""
|
|
267
340
|
if inspect.iscoroutinefunction(func):
|
|
268
341
|
@wraps(func)
|
|
269
342
|
async def wrapper(*args, **kwargs):
|
|
@@ -286,3 +359,4 @@ class FixedIdGenerator(id_generator.IdGenerator):
|
|
|
286
359
|
|
|
287
360
|
def generate_trace_id(self) -> int:
|
|
288
361
|
return self.trace_id
|
|
362
|
+
|
|
@@ -3,14 +3,14 @@ import os
|
|
|
3
3
|
from importlib.metadata import version
|
|
4
4
|
from opentelemetry.context import get_value, set_value, attach, detach
|
|
5
5
|
from opentelemetry.sdk.trace import Span
|
|
6
|
-
|
|
6
|
+
from opentelemetry.trace.status import Status, StatusCode
|
|
7
7
|
from monocle_apptrace.instrumentation.common.constants import (
|
|
8
8
|
QUERY,
|
|
9
9
|
service_name_map,
|
|
10
10
|
service_type_map,
|
|
11
11
|
MONOCLE_SDK_VERSION
|
|
12
12
|
)
|
|
13
|
-
from monocle_apptrace.instrumentation.common.utils import set_attribute, get_scopes
|
|
13
|
+
from monocle_apptrace.instrumentation.common.utils import set_attribute, get_scopes, MonocleSpanException
|
|
14
14
|
from monocle_apptrace.instrumentation.common.constants import WORKFLOW_TYPE_KEY, WORKFLOW_TYPE_GENERIC
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
@@ -64,9 +64,12 @@ class SpanHandler:
|
|
|
64
64
|
""" Set attributes of workflow if this is a root span"""
|
|
65
65
|
SpanHandler.set_workflow_attributes(to_wrap, span)
|
|
66
66
|
SpanHandler.set_app_hosting_identifier_attribute(span)
|
|
67
|
+
span.set_status(StatusCode.OK)
|
|
67
68
|
|
|
68
|
-
|
|
69
|
-
|
|
69
|
+
|
|
70
|
+
def post_task_processing(self, to_wrap, wrapped, instance, args, kwargs, result, span:Span):
|
|
71
|
+
if span.status.status_code == StatusCode.UNSET:
|
|
72
|
+
span.set_status(StatusCode.OK)
|
|
70
73
|
|
|
71
74
|
def hydrate_span(self, to_wrap, wrapped, instance, args, kwargs, result, span):
|
|
72
75
|
self.hydrate_attributes(to_wrap, wrapped, instance, args, kwargs, result, span)
|
|
@@ -95,6 +98,8 @@ class SpanHandler:
|
|
|
95
98
|
result = accessor(arguments)
|
|
96
99
|
if result and isinstance(result, (str, list)):
|
|
97
100
|
span.set_attribute(attribute_name, result)
|
|
101
|
+
except MonocleSpanException as e:
|
|
102
|
+
span.set_status(StatusCode.ERROR, e.message)
|
|
98
103
|
except Exception as e:
|
|
99
104
|
logger.debug(f"Error processing accessor: {e}")
|
|
100
105
|
else:
|
|
@@ -131,6 +136,8 @@ class SpanHandler:
|
|
|
131
136
|
event_attributes[attribute_key] = accessor(arguments)
|
|
132
137
|
else:
|
|
133
138
|
event_attributes.update(accessor(arguments))
|
|
139
|
+
except MonocleSpanException as e:
|
|
140
|
+
span.set_status(StatusCode.ERROR, e.message)
|
|
134
141
|
except Exception as e:
|
|
135
142
|
logger.debug(f"Error evaluating accessor for attribute '{attribute_key}': {e}")
|
|
136
143
|
span.add_event(name=event_name, attributes=event_attributes)
|
|
@@ -21,6 +21,21 @@ embedding_model_context = {}
|
|
|
21
21
|
scope_id_generator = id_generator.RandomIdGenerator()
|
|
22
22
|
http_scopes:dict[str:str] = {}
|
|
23
23
|
|
|
24
|
+
class MonocleSpanException(Exception):
|
|
25
|
+
def __init__(self, err_message:str):
|
|
26
|
+
"""
|
|
27
|
+
Monocle exeption to indicate error in span processing.
|
|
28
|
+
Parameters:
|
|
29
|
+
- err_message (str): Error message.
|
|
30
|
+
- status (str): Status code
|
|
31
|
+
"""
|
|
32
|
+
super().__init__(err_message)
|
|
33
|
+
self.message = err_message
|
|
34
|
+
|
|
35
|
+
def __str__(self):
|
|
36
|
+
"""String representation of the exception."""
|
|
37
|
+
return f"[Monocle Span Error: {self.message} {self.status}"
|
|
38
|
+
|
|
24
39
|
def set_tracer_provider(tracer_provider: TracerProvider):
|
|
25
40
|
global monocle_tracer_provider
|
|
26
41
|
monocle_tracer_provider = tracer_provider
|
|
@@ -252,35 +267,49 @@ async def http_async_route_handler(func, *args, **kwargs):
|
|
|
252
267
|
headers = kwargs['req'].headers
|
|
253
268
|
else:
|
|
254
269
|
headers = None
|
|
255
|
-
return async_wrapper(func, None, headers, *args, **kwargs)
|
|
270
|
+
return async_wrapper(func, None, None, headers, *args, **kwargs)
|
|
256
271
|
|
|
257
|
-
def run_async_with_scope(method,
|
|
272
|
+
def run_async_with_scope(method, current_context, exceptions, *args, **kwargs):
|
|
258
273
|
token = None
|
|
259
|
-
if scope_name:
|
|
260
|
-
token = set_scope(scope_name)
|
|
261
|
-
elif headers:
|
|
262
|
-
token = extract_http_headers(headers)
|
|
263
274
|
try:
|
|
275
|
+
if current_context:
|
|
276
|
+
token = attach(current_context)
|
|
264
277
|
return asyncio.run(method(*args, **kwargs))
|
|
278
|
+
except Exception as e:
|
|
279
|
+
exceptions['exception'] = e
|
|
280
|
+
raise e
|
|
265
281
|
finally:
|
|
266
282
|
if token:
|
|
267
|
-
|
|
283
|
+
detach(token)
|
|
268
284
|
|
|
269
|
-
def async_wrapper(method, scope_name=None, headers=None, *args, **kwargs):
|
|
285
|
+
def async_wrapper(method, scope_name=None, scope_value=None, headers=None, *args, **kwargs):
|
|
270
286
|
try:
|
|
271
287
|
run_loop = asyncio.get_running_loop()
|
|
272
288
|
except RuntimeError:
|
|
273
289
|
run_loop = None
|
|
274
290
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
291
|
+
token = None
|
|
292
|
+
exceptions = {}
|
|
293
|
+
if scope_name:
|
|
294
|
+
token = set_scope(scope_name, scope_value)
|
|
295
|
+
elif headers:
|
|
296
|
+
token = extract_http_headers(headers)
|
|
297
|
+
current_context = get_current()
|
|
298
|
+
try:
|
|
299
|
+
if run_loop and run_loop.is_running():
|
|
300
|
+
results = []
|
|
301
|
+
thread = threading.Thread(target=lambda: results.append(run_async_with_scope(method, current_context, exceptions, *args, **kwargs)))
|
|
302
|
+
thread.start()
|
|
303
|
+
thread.join()
|
|
304
|
+
if 'exception' in exceptions:
|
|
305
|
+
raise exceptions['exception']
|
|
306
|
+
return_value = results[0] if len(results) > 0 else None
|
|
307
|
+
return return_value
|
|
308
|
+
else:
|
|
309
|
+
return run_async_with_scope(method, None, exceptions, *args, **kwargs)
|
|
310
|
+
finally:
|
|
311
|
+
if token:
|
|
312
|
+
remove_scope(token)
|
|
284
313
|
|
|
285
314
|
class Option(Generic[T]):
|
|
286
315
|
def __init__(self, value: Optional[T]):
|
|
@@ -31,10 +31,11 @@ def wrapper_processor(async_task: bool, tracer: Tracer, handler: SpanHandler, to
|
|
|
31
31
|
try:
|
|
32
32
|
handler.pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
33
33
|
skip_scan:bool = to_wrap.get('skip_span') or handler.skip_span(to_wrap, wrapped, instance, args, kwargs)
|
|
34
|
-
|
|
34
|
+
if not to_wrap.get('skip_span'):
|
|
35
|
+
token = SpanHandler.attach_workflow_type(to_wrap=to_wrap)
|
|
35
36
|
if skip_scan:
|
|
36
37
|
if async_task:
|
|
37
|
-
return_value = async_wrapper(wrapped, None, None, *args, **kwargs)
|
|
38
|
+
return_value = async_wrapper(wrapped, None, None, None, *args, **kwargs)
|
|
38
39
|
else:
|
|
39
40
|
return_value = wrapped(*args, **kwargs)
|
|
40
41
|
else:
|
|
@@ -58,7 +59,7 @@ def span_processor(name: str, async_task: bool, tracer: Tracer, handler: SpanHan
|
|
|
58
59
|
else:
|
|
59
60
|
handler.pre_task_processing(to_wrap, wrapped, instance, args, kwargs, span)
|
|
60
61
|
if async_task:
|
|
61
|
-
return_value = async_wrapper(wrapped, None, None, *args, **kwargs)
|
|
62
|
+
return_value = async_wrapper(wrapped, None, None, None, *args, **kwargs)
|
|
62
63
|
else:
|
|
63
64
|
return_value = wrapped(*args, **kwargs)
|
|
64
65
|
handler.hydrate_span(to_wrap, wrapped, instance, args, kwargs, return_value, span)
|
|
@@ -86,5 +87,6 @@ def scope_wrapper(tracer: Tracer, handler: SpanHandler, to_wrap, wrapped, instan
|
|
|
86
87
|
@with_tracer_wrapper
|
|
87
88
|
async def ascope_wrapper(tracer: Tracer, handler: SpanHandler, to_wrap, wrapped, instance, args, kwargs):
|
|
88
89
|
scope_name = to_wrap.get('scope_name', None)
|
|
89
|
-
|
|
90
|
+
scope_value = to_wrap.get('scope_value', None)
|
|
91
|
+
return_value = async_wrapper(wrapped, scope_name, scope_value, None, *args, **kwargs)
|
|
90
92
|
return return_value
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from opentelemetry.context import get_value, set_value, attach, detach
|
|
1
2
|
from monocle_apptrace.instrumentation.common.span_handler import SpanHandler
|
|
2
3
|
|
|
3
4
|
class BotoCoreSpanHandler(SpanHandler):
|
|
@@ -22,4 +23,4 @@ class BotoCoreSpanHandler(SpanHandler):
|
|
|
22
23
|
def post_tracing(self, to_wrap, wrapped, instance, args, kwargs, return_value):
|
|
23
24
|
self._botocore_processor(to_wrap=to_wrap, wrapped=wrapped, instance=instance, return_value=return_value, args=args,
|
|
24
25
|
kwargs=kwargs)
|
|
25
|
-
return super().pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
26
|
+
return super().pre_tracing(to_wrap, wrapped, instance, args, kwargs)
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
monocle_apptrace/README.md,sha256=T5NFC01bF8VR0oVnAX_n0bhsEtttwqfTxDNAe5Y_ivE,3765
|
|
2
|
-
monocle_apptrace/__init__.py,sha256=
|
|
2
|
+
monocle_apptrace/__init__.py,sha256=XtoX7gHUSZgkY1nry8IFny8RslPhutZQUuEkqIrBzFQ,30
|
|
3
3
|
monocle_apptrace/__main__.py,sha256=wBwV0fpwIuj9XSorPRP1MpkHHkZPM9Tg-lIFj1nokkU,609
|
|
4
4
|
monocle_apptrace/exporters/base_exporter.py,sha256=Gov_QKp5fonVZ-YdNM2ynoPot7GCaSNmKbCHIP3bDlE,1680
|
|
5
|
-
monocle_apptrace/exporters/exporter_processor.py,sha256
|
|
6
|
-
monocle_apptrace/exporters/file_exporter.py,sha256=
|
|
7
|
-
monocle_apptrace/exporters/monocle_exporters.py,sha256=
|
|
8
|
-
monocle_apptrace/exporters/aws/s3_exporter.py,sha256=
|
|
9
|
-
monocle_apptrace/exporters/aws/s3_exporter_opendal.py,sha256=
|
|
10
|
-
monocle_apptrace/exporters/azure/blob_exporter.py,sha256=
|
|
11
|
-
monocle_apptrace/exporters/azure/blob_exporter_opendal.py,sha256=
|
|
12
|
-
monocle_apptrace/exporters/okahu/okahu_exporter.py,sha256=
|
|
13
|
-
monocle_apptrace/instrumentation/__init__.py,sha256=
|
|
14
|
-
monocle_apptrace/instrumentation/common/__init__.py,sha256=
|
|
5
|
+
monocle_apptrace/exporters/exporter_processor.py,sha256=-spCIJ_UfJ0fax_jE-ii3ODQBwtnHZgYIGVNd91Q718,6298
|
|
6
|
+
monocle_apptrace/exporters/file_exporter.py,sha256=BSEYUb9Z_dascR9i_FL_HxnxnxjyxtR_5teoSjIpZQc,3198
|
|
7
|
+
monocle_apptrace/exporters/monocle_exporters.py,sha256=TKULSQDZLIrf76NMhxYfsnG3vV11B1l2liI1wEWGaLo,2759
|
|
8
|
+
monocle_apptrace/exporters/aws/s3_exporter.py,sha256=fvUUuukFM6hIliGqP61WXlVMFbxlIQtMgT3iwjUYDTA,8187
|
|
9
|
+
monocle_apptrace/exporters/aws/s3_exporter_opendal.py,sha256=0aEUxdMgJaDUwqjw0DqlCMr8kjl01KgwUt3_RRCVFds,5917
|
|
10
|
+
monocle_apptrace/exporters/azure/blob_exporter.py,sha256=75G8rcISQ0sZCECN2G67-DGFkJGGu2clNyrcoxEm9H8,7371
|
|
11
|
+
monocle_apptrace/exporters/azure/blob_exporter_opendal.py,sha256=wQUtciyFMD28tpWTiP0-kBjUuxy4LuQSo04aMuHwtb4,7140
|
|
12
|
+
monocle_apptrace/exporters/okahu/okahu_exporter.py,sha256=qj7paDHbWbYudH18xanUuxmhOHHlYEVj1kpzK7f2OTY,4601
|
|
13
|
+
monocle_apptrace/instrumentation/__init__.py,sha256=oa412OuokRm9Vf3XlCJLqpZjz9ZcuxAKxnEBvOK7u2M,21
|
|
14
|
+
monocle_apptrace/instrumentation/common/__init__.py,sha256=_YD94HPvDvHcrkt9Ll11BaHNzJ4W56GUJ7GPjp_diyA,223
|
|
15
15
|
monocle_apptrace/instrumentation/common/constants.py,sha256=6H5oLxGUD0Gd4bvEGq-fKm-W-ULWQ0bMOQs4puz5--I,2676
|
|
16
|
-
monocle_apptrace/instrumentation/common/instrumentor.py,sha256=
|
|
17
|
-
monocle_apptrace/instrumentation/common/span_handler.py,sha256=
|
|
18
|
-
monocle_apptrace/instrumentation/common/utils.py,sha256=
|
|
19
|
-
monocle_apptrace/instrumentation/common/wrapper.py,sha256=
|
|
16
|
+
monocle_apptrace/instrumentation/common/instrumentor.py,sha256=v-ZriWJdHlSOWkwprlwDaxm6kOVKyqehZ3m_kbECm0k,15087
|
|
17
|
+
monocle_apptrace/instrumentation/common/span_handler.py,sha256=WHvLc3TSqsrv62qJ_qclC57QT0bFoTCJ4hc-qe3SOYg,10229
|
|
18
|
+
monocle_apptrace/instrumentation/common/utils.py,sha256=iGxvC8V-2uLbrhFG9u9NKOyHkbd1moIkg6ukujDT88Y,12023
|
|
19
|
+
monocle_apptrace/instrumentation/common/wrapper.py,sha256=FNam-sz5gbTxa0Ym6-xyVhCA5HVAEObKDdQFubasIpU,4474
|
|
20
20
|
monocle_apptrace/instrumentation/common/wrapper_method.py,sha256=7k_rHOzbFRfeW40CMfa78wwyPVfSgcXiyDsgezjDcaA,3188
|
|
21
21
|
monocle_apptrace/instrumentation/metamodel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
22
|
monocle_apptrace/instrumentation/metamodel/botocore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -24,7 +24,7 @@ monocle_apptrace/instrumentation/metamodel/botocore/_helper.py,sha256=JIYtaN57OX
|
|
|
24
24
|
monocle_apptrace/instrumentation/metamodel/botocore/methods.py,sha256=LzmjbZjDWy7Uozc0chNjWG6PZhLngh_KJe5L6rw5rqI,452
|
|
25
25
|
monocle_apptrace/instrumentation/metamodel/botocore/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
26
|
monocle_apptrace/instrumentation/metamodel/botocore/entities/inference.py,sha256=JfTRmrxgU6e-b3dBbunWt5ObY_Ry_ZBYJBwKJB5UlJ8,2255
|
|
27
|
-
monocle_apptrace/instrumentation/metamodel/botocore/handlers/botocore_span_handler.py,sha256=
|
|
27
|
+
monocle_apptrace/instrumentation/metamodel/botocore/handlers/botocore_span_handler.py,sha256=Vfbx4g7P3_9iXXCySuqc2FOU_CTP-OZy7PHc7D2qOls,1419
|
|
28
28
|
monocle_apptrace/instrumentation/metamodel/flask/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
29
29
|
monocle_apptrace/instrumentation/metamodel/flask/_helper.py,sha256=AcQ5F6_IDmu9PXaeKKeiGIyq2I2YzA7wu1cvLzR-uyU,1175
|
|
30
30
|
monocle_apptrace/instrumentation/metamodel/flask/methods.py,sha256=QkWHX4wKQf_GiJBHmiS9_JD2CiKMTCWMcig2dxAiKgU,340
|
|
@@ -61,8 +61,8 @@ monocle_apptrace/instrumentation/metamodel/openai/entities/retrieval.py,sha256=L
|
|
|
61
61
|
monocle_apptrace/instrumentation/metamodel/requests/__init__.py,sha256=mg04UgoPzzcH-cPOahYUqN9T-TolZyOZipnBwDg5TP8,250
|
|
62
62
|
monocle_apptrace/instrumentation/metamodel/requests/_helper.py,sha256=lKU7py-M0eweHA_LWatwdyWbSGSlQNhScGZ43Xko7us,1115
|
|
63
63
|
monocle_apptrace/instrumentation/metamodel/requests/methods.py,sha256=OJtosy_07xy01o5Qv-53--aCLQLkr82NZtyi2t6ZDEM,326
|
|
64
|
-
monocle_apptrace-0.3.
|
|
65
|
-
monocle_apptrace-0.3.
|
|
66
|
-
monocle_apptrace-0.3.
|
|
67
|
-
monocle_apptrace-0.3.
|
|
68
|
-
monocle_apptrace-0.3.
|
|
64
|
+
monocle_apptrace-0.3.0b7.dist-info/METADATA,sha256=sKSON_WUMObjmugvQx9wCLmoy_jj6yp5RwQgVHtQ6qE,6314
|
|
65
|
+
monocle_apptrace-0.3.0b7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
66
|
+
monocle_apptrace-0.3.0b7.dist-info/licenses/LICENSE,sha256=ay9trLiP5I7ZsFXo6AqtkLYdRqe5S9r-DrPOvsNlZrg,9136
|
|
67
|
+
monocle_apptrace-0.3.0b7.dist-info/licenses/NOTICE,sha256=9jn4xtwM_uUetJMx5WqGnhrR7MIhpoRlpokjSTlyt8c,112
|
|
68
|
+
monocle_apptrace-0.3.0b7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|