dt-extensions-sdk 1.1.22__py3-none-any.whl → 1.1.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dt_extensions_sdk-1.1.22.dist-info → dt_extensions_sdk-1.1.24.dist-info}/METADATA +2 -2
- dt_extensions_sdk-1.1.24.dist-info/RECORD +33 -0
- {dt_extensions_sdk-1.1.22.dist-info → dt_extensions_sdk-1.1.24.dist-info}/WHEEL +1 -1
- {dt_extensions_sdk-1.1.22.dist-info → dt_extensions_sdk-1.1.24.dist-info}/licenses/LICENSE.txt +9 -9
- dynatrace_extension/__about__.py +5 -5
- dynatrace_extension/__init__.py +27 -27
- dynatrace_extension/cli/__init__.py +5 -5
- dynatrace_extension/cli/create/__init__.py +1 -1
- dynatrace_extension/cli/create/create.py +76 -76
- dynatrace_extension/cli/create/extension_template/.gitignore.template +160 -160
- dynatrace_extension/cli/create/extension_template/README.md.template +33 -33
- dynatrace_extension/cli/create/extension_template/activation.json.template +15 -15
- dynatrace_extension/cli/create/extension_template/extension/activationSchema.json.template +118 -118
- dynatrace_extension/cli/create/extension_template/extension/extension.yaml.template +17 -17
- dynatrace_extension/cli/create/extension_template/extension_name/__main__.py.template +43 -43
- dynatrace_extension/cli/create/extension_template/setup.py.template +28 -28
- dynatrace_extension/cli/main.py +428 -428
- dynatrace_extension/cli/schema.py +129 -129
- dynatrace_extension/sdk/__init__.py +3 -3
- dynatrace_extension/sdk/activation.py +43 -43
- dynatrace_extension/sdk/callback.py +134 -134
- dynatrace_extension/sdk/communication.py +482 -482
- dynatrace_extension/sdk/event.py +19 -19
- dynatrace_extension/sdk/extension.py +1045 -1045
- dynatrace_extension/sdk/helper.py +191 -191
- dynatrace_extension/sdk/metric.py +118 -118
- dynatrace_extension/sdk/runtime.py +67 -67
- dynatrace_extension/sdk/vendor/mureq/LICENSE +13 -13
- dynatrace_extension/sdk/vendor/mureq/mureq.py +447 -447
- dt_extensions_sdk-1.1.22.dist-info/RECORD +0 -33
- {dt_extensions_sdk-1.1.22.dist-info → dt_extensions_sdk-1.1.24.dist-info}/entry_points.txt +0 -0
@@ -1,1045 +1,1045 @@
|
|
1
|
-
# SPDX-FileCopyrightText: 2023-present Dynatrace LLC
|
2
|
-
#
|
3
|
-
# SPDX-License-Identifier: MIT
|
4
|
-
|
5
|
-
import logging
|
6
|
-
import sched
|
7
|
-
import signal
|
8
|
-
import sys
|
9
|
-
import threading
|
10
|
-
import time
|
11
|
-
from argparse import ArgumentParser
|
12
|
-
from collections import deque
|
13
|
-
from concurrent.futures import ThreadPoolExecutor
|
14
|
-
from datetime import datetime, timedelta, timezone
|
15
|
-
from enum import Enum
|
16
|
-
from itertools import chain
|
17
|
-
from threading import Lock, RLock, active_count
|
18
|
-
from typing import Any, Callable, ClassVar, Dict, List, NamedTuple, Optional, Union
|
19
|
-
|
20
|
-
from ..__about__ import __version__
|
21
|
-
from .activation import ActivationConfig, ActivationType
|
22
|
-
from .callback import WrappedCallback
|
23
|
-
from .communication import CommunicationClient, DebugClient, HttpClient, Status, StatusValue
|
24
|
-
from .event import Severity
|
25
|
-
from .metric import Metric, MetricType, SfmMetric, SummaryStat
|
26
|
-
from .runtime import RuntimeProperties
|
27
|
-
|
28
|
-
HEARTBEAT_INTERVAL = timedelta(seconds=30)
|
29
|
-
METRIC_SENDING_INTERVAL = timedelta(seconds=30)
|
30
|
-
SFM_METRIC_SENDING_INTERVAL = timedelta(seconds=60)
|
31
|
-
TIME_DIFF_INTERVAL = timedelta(seconds=60)
|
32
|
-
|
33
|
-
CALLBACKS_THREAD_POOL_SIZE = 100
|
34
|
-
INTERNAL_THREAD_POOL_SIZE = 20
|
35
|
-
|
36
|
-
RFC_3339_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
37
|
-
DATASOURCE_TYPE = "python"
|
38
|
-
|
39
|
-
logging.raiseExceptions = False
|
40
|
-
formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s (%(threadName)s): %(message)s")
|
41
|
-
error_handler = logging.StreamHandler()
|
42
|
-
error_handler.addFilter(lambda record: record.levelno >= logging.ERROR)
|
43
|
-
error_handler.setFormatter(formatter)
|
44
|
-
std_handler = logging.StreamHandler(sys.stdout)
|
45
|
-
std_handler.addFilter(lambda record: record.levelno < logging.ERROR)
|
46
|
-
std_handler.setFormatter(formatter)
|
47
|
-
extension_logger = logging.getLogger(__name__)
|
48
|
-
extension_logger.setLevel(logging.INFO)
|
49
|
-
extension_logger.addHandler(error_handler)
|
50
|
-
extension_logger.addHandler(std_handler)
|
51
|
-
|
52
|
-
api_logger = logging.getLogger("api")
|
53
|
-
api_logger.setLevel(logging.INFO)
|
54
|
-
api_logger.addHandler(error_handler)
|
55
|
-
api_logger.addHandler(std_handler)
|
56
|
-
|
57
|
-
DT_EVENT_SCHEMA = {
|
58
|
-
"eventType": str,
|
59
|
-
"title": str,
|
60
|
-
"startTime": int,
|
61
|
-
"endTime": int,
|
62
|
-
"timeout": int,
|
63
|
-
"entitySelector": str,
|
64
|
-
"properties": dict,
|
65
|
-
}
|
66
|
-
|
67
|
-
|
68
|
-
class AggregationMode(Enum):
|
69
|
-
ALL = "include_all"
|
70
|
-
NONE = "include_none"
|
71
|
-
LIST = "include_list"
|
72
|
-
|
73
|
-
|
74
|
-
class DtEventType(str, Enum):
|
75
|
-
"""Event type.
|
76
|
-
|
77
|
-
Note:
|
78
|
-
Official API v2 documentation:
|
79
|
-
|
80
|
-
https://docs.dynatrace.com/docs/dynatrace-api/environment-api/events-v2/post-event
|
81
|
-
"""
|
82
|
-
|
83
|
-
AVAILABILITY_EVENT = "AVAILABILITY_EVENT"
|
84
|
-
CUSTOM_INFO = "CUSTOM_INFO"
|
85
|
-
CUSTOM_ALERT = "CUSTOM_ALERT"
|
86
|
-
CUSTOM_ANNOTATION = "CUSTOM_ANNOTATION"
|
87
|
-
CUSTOM_CONFIGURATION = "CUSTOM_CONFIGURATION"
|
88
|
-
CUSTOM_DEPLOYMENT = "CUSTOM_DEPLOYMENT"
|
89
|
-
ERROR_EVENT = "ERROR_EVENT"
|
90
|
-
MARKED_FOR_TERMINATION = "MARKED_FOR_TERMINATION"
|
91
|
-
PERFORMANCE_EVENT = "PERFORMANCE_EVENT"
|
92
|
-
RESOURCE_CONTENTION_EVENT = "RESOURCE_CONTENTION_EVENT"
|
93
|
-
|
94
|
-
|
95
|
-
class CountMetricRegistrationEntry(NamedTuple):
|
96
|
-
metric_key: str
|
97
|
-
aggregation_mode: AggregationMode
|
98
|
-
dimensions_list: list[str]
|
99
|
-
|
100
|
-
@staticmethod
|
101
|
-
def make_list(metric_key: str, dimensions_list: List[str]):
|
102
|
-
"""Build an entry that uses defined list of dimensions for aggregation.
|
103
|
-
|
104
|
-
Args:
|
105
|
-
metric_key: Metric key in string.
|
106
|
-
dimensions_list: List of dimensions.
|
107
|
-
"""
|
108
|
-
return CountMetricRegistrationEntry(metric_key, AggregationMode.LIST, dimensions_list)
|
109
|
-
|
110
|
-
@staticmethod
|
111
|
-
def make_all(metric_key: str):
|
112
|
-
"""Build an entry that uses all mint dimensions for aggregation.
|
113
|
-
|
114
|
-
Args:
|
115
|
-
metric_key: Metric key in string.
|
116
|
-
"""
|
117
|
-
return CountMetricRegistrationEntry(metric_key, AggregationMode.ALL, [])
|
118
|
-
|
119
|
-
@staticmethod
|
120
|
-
def make_none(metric_key: str):
|
121
|
-
"""Build an entry that uses none of mint dimensions for aggregation.
|
122
|
-
|
123
|
-
Args:
|
124
|
-
metric_key: Metric key in string.
|
125
|
-
"""
|
126
|
-
return CountMetricRegistrationEntry(metric_key, AggregationMode.NONE, [])
|
127
|
-
|
128
|
-
def registration_items_dict(self):
|
129
|
-
result = {"aggregation_mode": self.aggregation_mode.value}
|
130
|
-
if self.aggregation_mode == AggregationMode.LIST:
|
131
|
-
result["dimensions_list"] = self.dimensions_list
|
132
|
-
return result
|
133
|
-
else:
|
134
|
-
return result
|
135
|
-
|
136
|
-
|
137
|
-
def _add_sfm_metric(metric: Metric, sfm_metrics: Optional[List[Metric]] = None):
|
138
|
-
if sfm_metrics is None:
|
139
|
-
sfm_metrics = []
|
140
|
-
metric.validate()
|
141
|
-
sfm_metrics.append(metric)
|
142
|
-
|
143
|
-
|
144
|
-
class Extension:
|
145
|
-
"""Base class for Python extensions.
|
146
|
-
|
147
|
-
Attributes:
|
148
|
-
logger: Embedded logger object for the extension.
|
149
|
-
"""
|
150
|
-
|
151
|
-
_instance: ClassVar = None
|
152
|
-
schedule_decorators: ClassVar = []
|
153
|
-
|
154
|
-
def __new__(cls):
|
155
|
-
if Extension._instance is None:
|
156
|
-
Extension._instance = super(__class__, cls).__new__(cls)
|
157
|
-
return Extension._instance
|
158
|
-
|
159
|
-
def __init__(self) -> None:
|
160
|
-
# do not initialize already created singleton
|
161
|
-
if hasattr(self, "logger"):
|
162
|
-
return
|
163
|
-
|
164
|
-
self.logger = extension_logger
|
165
|
-
|
166
|
-
self.extension_config: str = ""
|
167
|
-
self._feature_sets: dict[str, list[str]] = {}
|
168
|
-
|
169
|
-
# Useful metadata, populated once the extension is started
|
170
|
-
self.extension_name = "" # Needs to be set by the developer if they so decide
|
171
|
-
self.extension_version = ""
|
172
|
-
self.monitoring_config_name = ""
|
173
|
-
self._task_id = "development_task_id"
|
174
|
-
self._monitoring_config_id = "development_config_id"
|
175
|
-
|
176
|
-
# The user can override default EEC enrichment for logs
|
177
|
-
self.log_event_enrichment = True
|
178
|
-
|
179
|
-
# The Communication client
|
180
|
-
self._client: CommunicationClient = None # type: ignore
|
181
|
-
|
182
|
-
# Set to true when --fastcheck is passed as a parameter
|
183
|
-
self._is_fastcheck: bool = True
|
184
|
-
|
185
|
-
# If this is true, we are running locally during development
|
186
|
-
self._running_in_sim: bool = False
|
187
|
-
|
188
|
-
# Response from EEC to /alive/ requests
|
189
|
-
self._runtime_properties: RuntimeProperties = RuntimeProperties({})
|
190
|
-
|
191
|
-
# The time difference between the local machine and the cluster time, used to sync callbacks with cluster
|
192
|
-
self._cluster_time_diff: int = 0
|
193
|
-
|
194
|
-
# Optional callback to be invoked during the fastcheck
|
195
|
-
self._fast_check_callback: Optional[Callable[[ActivationConfig, str], Status]] = None
|
196
|
-
|
197
|
-
# List of all scheduled callbacks we must run
|
198
|
-
self._scheduled_callbacks: List[WrappedCallback] = []
|
199
|
-
self._scheduled_callbacks_before_run: List[WrappedCallback] = []
|
200
|
-
|
201
|
-
# Internal callbacks results, used to report statuses
|
202
|
-
self._internal_callbacks_results: Dict[str, Status] = {}
|
203
|
-
self._internal_callbacks_results_lock: Lock = Lock()
|
204
|
-
|
205
|
-
# Running callbacks, used to get the callback info when reporting metrics
|
206
|
-
self._running_callbacks: Dict[int, WrappedCallback] = {}
|
207
|
-
self._running_callbacks_lock: Lock = Lock()
|
208
|
-
|
209
|
-
self._scheduler = sched.scheduler(time.time, time.sleep)
|
210
|
-
|
211
|
-
# Executors for the callbacks and internal methods
|
212
|
-
self._callbacks_executor = ThreadPoolExecutor(max_workers=CALLBACKS_THREAD_POOL_SIZE)
|
213
|
-
self._internal_executor = ThreadPoolExecutor(max_workers=INTERNAL_THREAD_POOL_SIZE)
|
214
|
-
|
215
|
-
# Extension metrics
|
216
|
-
self._metrics_lock = RLock()
|
217
|
-
self._metrics: List[str] = []
|
218
|
-
|
219
|
-
# Self monitoring metrics
|
220
|
-
self._sfm_metrics_lock = Lock()
|
221
|
-
self._callbackSfmReport: Dict[str, WrappedCallback] = {}
|
222
|
-
|
223
|
-
# Count metric delta signals
|
224
|
-
self._delta_signal_buffer: set[str] = set()
|
225
|
-
self._registered_count_metrics: set[str] = set()
|
226
|
-
|
227
|
-
# Self tech rule
|
228
|
-
self._techrule = ""
|
229
|
-
|
230
|
-
# Error message from caught exception in self.initialize()
|
231
|
-
self._initialization_error: str = ""
|
232
|
-
|
233
|
-
self._parse_args()
|
234
|
-
|
235
|
-
for function, interval, args, activation_type in Extension.schedule_decorators:
|
236
|
-
params = (self,)
|
237
|
-
if args is not None:
|
238
|
-
params = params + args
|
239
|
-
self.schedule(function, interval, params, activation_type)
|
240
|
-
|
241
|
-
api_logger.info("-----------------------------------------------------")
|
242
|
-
api_logger.info(f"Starting {self.__class__} {self.extension_name}, version: {self.get_version()}")
|
243
|
-
|
244
|
-
@property
|
245
|
-
def is_helper(self) -> bool:
|
246
|
-
"""Internal property used by the EEC."""
|
247
|
-
|
248
|
-
return False
|
249
|
-
|
250
|
-
@property
|
251
|
-
def task_id(self) -> str:
|
252
|
-
"""Internal property used by the EEC."""
|
253
|
-
|
254
|
-
return self._task_id
|
255
|
-
|
256
|
-
@property
|
257
|
-
def monitoring_config_id(self) -> str:
|
258
|
-
"""Internal property used by the EEC.
|
259
|
-
|
260
|
-
Represents a unique identifier of the monitoring configuration.
|
261
|
-
that is assigned to this particular extension instance.
|
262
|
-
"""
|
263
|
-
|
264
|
-
return self._monitoring_config_id
|
265
|
-
|
266
|
-
def run(self):
|
267
|
-
"""Launch the extension instance.
|
268
|
-
|
269
|
-
Calling this method starts the main loop of the extension.
|
270
|
-
|
271
|
-
This method must be invoked once to start the extension,
|
272
|
-
|
273
|
-
if `--fastcheck` is set, the extension will run in fastcheck mode,
|
274
|
-
otherwise the main loop is started, which periodically runs:
|
275
|
-
|
276
|
-
* The scheduled callbacks
|
277
|
-
* The heartbeat method
|
278
|
-
* The metrics publisher method
|
279
|
-
"""
|
280
|
-
|
281
|
-
self._setup_signal_handlers()
|
282
|
-
if self._is_fastcheck:
|
283
|
-
return self._run_fastcheck()
|
284
|
-
self._start_extension_loop()
|
285
|
-
|
286
|
-
def _setup_signal_handlers(self):
|
287
|
-
if sys.platform == "win32":
|
288
|
-
signal.signal(signal.SIGBREAK, self._shutdown_signal_handler)
|
289
|
-
signal.signal(signal.SIGINT, self._shutdown_signal_handler)
|
290
|
-
|
291
|
-
def _shutdown_signal_handler(self, sig, frame): # noqa: ARG002
|
292
|
-
api_logger.info(f"{signal.Signals(sig).name} captured. Flushing metrics and exiting...")
|
293
|
-
self.on_shutdown()
|
294
|
-
self._send_metrics()
|
295
|
-
self._send_sfm_metrics()
|
296
|
-
sys.exit(0)
|
297
|
-
|
298
|
-
def on_shutdown(self):
|
299
|
-
"""Callback method to be invoked when the extension is shutting down.
|
300
|
-
|
301
|
-
Called when extension exits after it has received shutdown signal from EEC
|
302
|
-
This is executed before metrics are flushed to EEC
|
303
|
-
"""
|
304
|
-
pass
|
305
|
-
|
306
|
-
def _schedule_callback(self, callback: WrappedCallback):
|
307
|
-
if callback.activation_type is not None and callback.activation_type != self.activation_config.type:
|
308
|
-
api_logger.info(
|
309
|
-
f"Skipping {callback} with activation type {callback.activation_type} because it is not {self.activation_config.type}"
|
310
|
-
)
|
311
|
-
return
|
312
|
-
|
313
|
-
api_logger.debug(f"Scheduling callback {callback}")
|
314
|
-
|
315
|
-
# These properties are updated after the extension starts
|
316
|
-
callback.cluster_time_diff = self._cluster_time_diff
|
317
|
-
callback.running_in_sim = self._running_in_sim
|
318
|
-
self._scheduled_callbacks.append(callback)
|
319
|
-
self._scheduler.enter(callback.initial_wait_time(), 1, self._callback_iteration, (callback,))
|
320
|
-
|
321
|
-
def schedule(
|
322
|
-
self,
|
323
|
-
callback: Callable,
|
324
|
-
interval: Union[timedelta, int],
|
325
|
-
args: Optional[tuple] = None,
|
326
|
-
activation_type: Optional[ActivationType] = None,
|
327
|
-
) -> None:
|
328
|
-
"""Schedule a method to be executed periodically.
|
329
|
-
|
330
|
-
The callback method will be periodically invoked in a separate thread.
|
331
|
-
The callback method is always immediately scheduled for execution.
|
332
|
-
|
333
|
-
Args:
|
334
|
-
callback: The callback method to be invoked
|
335
|
-
interval: The time interval between invocations, can be a timedelta object,
|
336
|
-
or an int representing the number of seconds
|
337
|
-
args: Arguments to the callback, if any
|
338
|
-
activation_type: Optional activation type when this callback should run,
|
339
|
-
can be 'ActivationType.LOCAL' or 'ActivationType.REMOTE'
|
340
|
-
"""
|
341
|
-
|
342
|
-
if isinstance(interval, int):
|
343
|
-
interval = timedelta(seconds=interval)
|
344
|
-
|
345
|
-
callback = WrappedCallback(interval, callback, api_logger, args, activation_type=activation_type)
|
346
|
-
if self._is_fastcheck:
|
347
|
-
self._scheduled_callbacks_before_run.append(callback)
|
348
|
-
else:
|
349
|
-
self._schedule_callback(callback)
|
350
|
-
|
351
|
-
def query(self):
|
352
|
-
"""Callback to be executed every minute by default.
|
353
|
-
|
354
|
-
Optional method that can be implemented by subclasses.
|
355
|
-
The query method is always scheduled to run every minute.
|
356
|
-
"""
|
357
|
-
pass
|
358
|
-
|
359
|
-
def initialize(self):
|
360
|
-
"""Callback to be executed when the extension starts.
|
361
|
-
|
362
|
-
Called once after the extension starts and the processes arguments are parsed.
|
363
|
-
Sometimes there are tasks the user needs to do that must happen before runtime,
|
364
|
-
but after the activation config has been received, example: Setting the schedule frequency
|
365
|
-
based on the user input on the monitoring configuration, this can be done on this method
|
366
|
-
"""
|
367
|
-
pass
|
368
|
-
|
369
|
-
def fastcheck(self) -> Status:
|
370
|
-
"""Callback executed when extension is launched.
|
371
|
-
|
372
|
-
Called if the extension is run in the `fastcheck` mode. Only invoked for remote
|
373
|
-
extensions.
|
374
|
-
This method is not called if fastcheck callback was already registered with
|
375
|
-
Extension.register_fastcheck().
|
376
|
-
|
377
|
-
Returns:
|
378
|
-
Status with optional message whether the fastcheck succeed or failed.
|
379
|
-
"""
|
380
|
-
return Status(StatusValue.OK)
|
381
|
-
|
382
|
-
def register_fastcheck(self, fast_check_callback: Callable[[ActivationConfig, str], Status]):
|
383
|
-
"""Registers fastcheck callback that is executed in the `fastcheck` mode.
|
384
|
-
|
385
|
-
Extension.fastcheck() is not called if fastcheck callback is registered with this method
|
386
|
-
|
387
|
-
Args:
|
388
|
-
fast_check_callback: callable called with ActivationConfig and
|
389
|
-
extension_config arguments. Must return the Status with optional message
|
390
|
-
whether the fastcheck succeed or failed.
|
391
|
-
"""
|
392
|
-
if self._fast_check_callback:
|
393
|
-
api_logger.error("More than one function assigned to fastcheck, last registered one was kept.")
|
394
|
-
|
395
|
-
self._fast_check_callback = fast_check_callback
|
396
|
-
|
397
|
-
def _register_count_metrics(self, *count_metric_entries: CountMetricRegistrationEntry) -> None:
|
398
|
-
"""Send a count metric registration request to EEC.
|
399
|
-
|
400
|
-
Args:
|
401
|
-
count_metric_entries: CountMetricRegistrationEntry objects for each count metric to register
|
402
|
-
"""
|
403
|
-
json_pattern = {
|
404
|
-
metric_entry.metric_key: metric_entry.registration_items_dict() for metric_entry in count_metric_entries
|
405
|
-
}
|
406
|
-
self._client.register_count_metrics(json_pattern)
|
407
|
-
|
408
|
-
def _send_count_delta_signal(self, metric_keys: set[str], force: bool = True) -> None:
|
409
|
-
"""Send calculate-delta signal to EEC monotonic converter.
|
410
|
-
|
411
|
-
Args:
|
412
|
-
metric_keys: List with metrics for which we want to calculate deltas
|
413
|
-
force: If true, it forces the metrics from cache to be pushed into EEC and then delta signal request is
|
414
|
-
sent. Otherwise, it puts delta signal request in cache and request is sent after nearest (in time) sending
|
415
|
-
metrics to EEC event
|
416
|
-
"""
|
417
|
-
|
418
|
-
with self._metrics_lock:
|
419
|
-
if not force:
|
420
|
-
for key in metric_keys:
|
421
|
-
self._delta_signal_buffer.add(key)
|
422
|
-
return
|
423
|
-
|
424
|
-
self._send_metrics()
|
425
|
-
self._client.send_count_delta_signal(metric_keys)
|
426
|
-
self._delta_signal_buffer = {
|
427
|
-
metric_key for metric_key in self._delta_signal_buffer if metric_key not in metric_keys
|
428
|
-
}
|
429
|
-
|
430
|
-
def report_metric(
|
431
|
-
self,
|
432
|
-
key: str,
|
433
|
-
value: Union[float, str, int, SummaryStat],
|
434
|
-
dimensions: Optional[Dict[str, str]] = None,
|
435
|
-
techrule: Optional[str] = None,
|
436
|
-
timestamp: Optional[datetime] = None,
|
437
|
-
metric_type: MetricType = MetricType.GAUGE,
|
438
|
-
) -> None:
|
439
|
-
"""Report a metric.
|
440
|
-
|
441
|
-
Metric is sent to EEC using an HTTP request and MINT protocol. EEC then
|
442
|
-
sends the metrics to the tenant.
|
443
|
-
|
444
|
-
By default, it reports a gauge metric.
|
445
|
-
|
446
|
-
Args:
|
447
|
-
key: The metric key, must follow the MINT specification
|
448
|
-
value: The metric value, can be a simple value or a SummaryStat
|
449
|
-
dimensions: A dictionary of dimensions
|
450
|
-
techrule: The technology rule string set by self.techrule setter.
|
451
|
-
timestamp: The timestamp of the metric, defaults to the current time
|
452
|
-
metric_type: The type of the metric, defaults to MetricType.GAUGE
|
453
|
-
"""
|
454
|
-
|
455
|
-
if techrule:
|
456
|
-
if not dimensions:
|
457
|
-
dimensions = {}
|
458
|
-
if "dt.techrule.id" not in dimensions:
|
459
|
-
dimensions["dt.techrule.id"] = techrule
|
460
|
-
|
461
|
-
if metric_type == MetricType.COUNT and timestamp is None:
|
462
|
-
# We must report a timestamp for count metrics
|
463
|
-
timestamp = datetime.now()
|
464
|
-
|
465
|
-
metric = Metric(key=key, value=value, dimensions=dimensions, metric_type=metric_type, timestamp=timestamp)
|
466
|
-
self._add_metric(metric)
|
467
|
-
|
468
|
-
def report_mint_lines(self, lines: List[str]) -> None:
|
469
|
-
"""Report mint lines using the MINT protocol
|
470
|
-
|
471
|
-
Examples:
|
472
|
-
Metric lines must comply with the MINT format.
|
473
|
-
|
474
|
-
>>> self.report_mint_lines(["my_metric 1", "my_other_metric 2"])
|
475
|
-
|
476
|
-
Args:
|
477
|
-
lines: A list of mint lines
|
478
|
-
"""
|
479
|
-
self._add_mint_lines(lines)
|
480
|
-
|
481
|
-
def report_event(
|
482
|
-
self,
|
483
|
-
title: str,
|
484
|
-
description: str,
|
485
|
-
properties: Optional[dict] = None,
|
486
|
-
timestamp: Optional[datetime] = None,
|
487
|
-
severity: Union[Severity, str] = Severity.INFO,
|
488
|
-
) -> None:
|
489
|
-
"""Report an event using log ingest.
|
490
|
-
|
491
|
-
Args:
|
492
|
-
title: The title of the event
|
493
|
-
description: The description of the event
|
494
|
-
properties: A dictionary of extra event properties
|
495
|
-
timestamp: The timestamp of the event, defaults to the current time
|
496
|
-
severity: The severity of the event, defaults to Severity.INFO
|
497
|
-
"""
|
498
|
-
if timestamp is None:
|
499
|
-
timestamp = datetime.now(tz=timezone.utc)
|
500
|
-
|
501
|
-
if properties is None:
|
502
|
-
properties = {}
|
503
|
-
|
504
|
-
event = {
|
505
|
-
"content": f"{title}\n{description}",
|
506
|
-
"title": title,
|
507
|
-
"description": description,
|
508
|
-
"timestamp": timestamp.strftime(RFC_3339_FORMAT),
|
509
|
-
"severity": severity.value if isinstance(severity, Severity) else severity,
|
510
|
-
**self._metadata,
|
511
|
-
**properties,
|
512
|
-
}
|
513
|
-
self._send_events(event)
|
514
|
-
|
515
|
-
def report_dt_event(
|
516
|
-
self,
|
517
|
-
event_type: DtEventType,
|
518
|
-
title: str,
|
519
|
-
start_time: Optional[int] = None,
|
520
|
-
end_time: Optional[int] = None,
|
521
|
-
timeout: Optional[int] = None,
|
522
|
-
entity_selector: Optional[str] = None,
|
523
|
-
properties: Optional[dict[str, str]] = None,
|
524
|
-
) -> None:
|
525
|
-
"""
|
526
|
-
Reports an event using the v2 event ingest API.
|
527
|
-
|
528
|
-
Unlike ``report_event``, this directly raises an event or even a problem
|
529
|
-
based on the specified ``event_type``.
|
530
|
-
|
531
|
-
Note:
|
532
|
-
For reference see: https://www.dynatrace.com/support/help/dynatrace-api/environment-api/events-v2/post-event
|
533
|
-
|
534
|
-
Args:
|
535
|
-
event_type: The event type chosen from type Enum (required)
|
536
|
-
title: The title of the event (required)
|
537
|
-
start_time: The start time of event in UTC ms, if not set, current timestamp (optional)
|
538
|
-
end_time: The end time of event in UTC ms, if not set, current timestamp + timeout (optional)
|
539
|
-
timeout: The timeout of event in minutes, if not set, 15 (optional)
|
540
|
-
entity_selector: The entity selector, if not set, the event is associated with environment entity (optional)
|
541
|
-
properties: A map of event properties (optional)
|
542
|
-
"""
|
543
|
-
event: Dict[str, Any] = {"eventType": event_type, "title": title}
|
544
|
-
if start_time:
|
545
|
-
event["startTime"] = start_time
|
546
|
-
if end_time:
|
547
|
-
event["endTime"] = end_time
|
548
|
-
if timeout:
|
549
|
-
event["timeout"] = timeout
|
550
|
-
if entity_selector:
|
551
|
-
event["entitySelector"] = entity_selector
|
552
|
-
if properties:
|
553
|
-
event["properties"] = properties
|
554
|
-
|
555
|
-
self._send_dt_event(event)
|
556
|
-
|
557
|
-
def report_dt_event_dict(self, event: dict):
|
558
|
-
"""Report an event using event ingest API with provided dictionary.
|
559
|
-
|
560
|
-
Note:
|
561
|
-
For reference see: https://www.dynatrace.com/support/help/dynatrace-api/environment-api/events-v2/post-event
|
562
|
-
|
563
|
-
Format of the event dictionary::
|
564
|
-
|
565
|
-
{
|
566
|
-
"type": "object",
|
567
|
-
"required": ["eventType", "title"],
|
568
|
-
"properties": {
|
569
|
-
"eventType": {
|
570
|
-
"type": "string",
|
571
|
-
"enum": [
|
572
|
-
"CUSTOM_INFO",
|
573
|
-
"CUSTOM_ANNOTATION",
|
574
|
-
"CUSTOM_CONFIGURATION",
|
575
|
-
"CUSTOM_DEPLOYMENT",
|
576
|
-
"MARKED_FOR_TERMINATION",
|
577
|
-
"ERROR_EVENT",
|
578
|
-
"AVAILABILITY_EVENT",
|
579
|
-
"PERFORMANCE_EVENT",
|
580
|
-
"RESOURCE_CONTENTION_EVENT",
|
581
|
-
"CUSTOM_ALERT"
|
582
|
-
]
|
583
|
-
},
|
584
|
-
"title": {
|
585
|
-
"type": "string",
|
586
|
-
"minLength": 1
|
587
|
-
},
|
588
|
-
"startTime": {"type": "integer"},
|
589
|
-
"endTime": {"type": "integer"},
|
590
|
-
"timeout": {"type": "integer"},
|
591
|
-
"entitySelector": {"type": "string"},
|
592
|
-
"properties": {
|
593
|
-
"type": "object",
|
594
|
-
"patternProperties": {
|
595
|
-
"^.*$": {"type": "string"}
|
596
|
-
}
|
597
|
-
}
|
598
|
-
}
|
599
|
-
}
|
600
|
-
"""
|
601
|
-
|
602
|
-
if "eventType" not in event or "title" not in event:
|
603
|
-
raise ValueError('"eventType" not present' if "eventType" not in event else '"title" not present in event')
|
604
|
-
for key, value in event.items():
|
605
|
-
if DT_EVENT_SCHEMA[key] is None:
|
606
|
-
msg = f'invalid member: "{key}"'
|
607
|
-
raise ValueError(msg)
|
608
|
-
if key == "eventType" and value not in list(DtEventType):
|
609
|
-
msg = f"Event type must be a DtEventType enum value, got: {value}"
|
610
|
-
raise ValueError(msg)
|
611
|
-
if key == "properties":
|
612
|
-
for prop_key, prop_val in event[key].items():
|
613
|
-
if not isinstance(prop_key, str) or not isinstance(prop_val, str):
|
614
|
-
msg = f'invalid "properties" member: {prop_key}: {prop_val}, required: "str": str'
|
615
|
-
raise ValueError(msg)
|
616
|
-
self._send_dt_event(event)
|
617
|
-
|
618
|
-
def report_log_event(self, log_event: dict):
|
619
|
-
"""Report a custom log event using log ingest.
|
620
|
-
|
621
|
-
Note:
|
622
|
-
See reference: https://www.dynatrace.com/support/help/shortlink/log-monitoring-log-data-ingestion
|
623
|
-
|
624
|
-
Args:
|
625
|
-
log_event: The log event dictionary.
|
626
|
-
"""
|
627
|
-
self._send_events(log_event)
|
628
|
-
|
629
|
-
def report_log_events(self, log_events: List[dict]):
|
630
|
-
"""Report a list of custom log events using log ingest.
|
631
|
-
|
632
|
-
Args:
|
633
|
-
log_events: The list of log events
|
634
|
-
"""
|
635
|
-
self._send_events(log_events)
|
636
|
-
|
637
|
-
def report_log_lines(self, log_lines: List[Union[str, bytes]]):
|
638
|
-
"""Report a list of log lines using log ingest
|
639
|
-
|
640
|
-
Args:
|
641
|
-
log_lines: The list of log lines
|
642
|
-
"""
|
643
|
-
events = [{"content": line} for line in log_lines]
|
644
|
-
self._send_events(events)
|
645
|
-
|
646
|
-
@property
|
647
|
-
def enabled_feature_sets(self) -> dict[str, list[str]]:
|
648
|
-
"""Map of enabled feautre sets and corresponding metrics.
|
649
|
-
|
650
|
-
Returns:
|
651
|
-
Dictionary containing enabled feature sets with corresponding
|
652
|
-
metrics defined in ``extension.yaml``.
|
653
|
-
"""
|
654
|
-
return {
|
655
|
-
feature_set_name: metric_keys
|
656
|
-
for feature_set_name, metric_keys in self._feature_sets.items()
|
657
|
-
if feature_set_name in self.activation_config.feature_sets or feature_set_name == "default"
|
658
|
-
}
|
659
|
-
|
660
|
-
@property
|
661
|
-
def enabled_feature_sets_names(self) -> list[str]:
|
662
|
-
"""Names of enabled feature sets.
|
663
|
-
|
664
|
-
Returns:
|
665
|
-
List containing names of enabled feature sets.
|
666
|
-
"""
|
667
|
-
return list(self.enabled_feature_sets.keys())
|
668
|
-
|
669
|
-
@property
|
670
|
-
def enabled_feature_sets_metrics(self) -> list[str]:
|
671
|
-
"""Enabled metrics.
|
672
|
-
|
673
|
-
Returns:
|
674
|
-
List of all metric keys from enabled feature sets
|
675
|
-
"""
|
676
|
-
return list(chain(*self.enabled_feature_sets.values()))
|
677
|
-
|
678
|
-
def _parse_args(self):
|
679
|
-
parser = ArgumentParser(description="Python extension parameters")
|
680
|
-
|
681
|
-
# Production parameters, these are passed by the EEC
|
682
|
-
parser.add_argument("--dsid", required=False, default=None)
|
683
|
-
parser.add_argument("--url", required=False)
|
684
|
-
parser.add_argument("--idtoken", required=False)
|
685
|
-
parser.add_argument(
|
686
|
-
"--loglevel",
|
687
|
-
help="Set extension log level. Info is default.",
|
688
|
-
type=str,
|
689
|
-
choices=["debug", "info"],
|
690
|
-
default="info",
|
691
|
-
)
|
692
|
-
parser.add_argument("--fastcheck", action="store_true", default=False)
|
693
|
-
parser.add_argument("--monitoring_config_id", required=False, default=None)
|
694
|
-
parser.add_argument("--local-ingest", action="store_true", default=False)
|
695
|
-
parser.add_argument("--local-ingest-port", required=False, default=14499)
|
696
|
-
|
697
|
-
# Debug parameters, these are used when running the extension locally
|
698
|
-
parser.add_argument("--extensionconfig", required=False, default=None)
|
699
|
-
parser.add_argument("--activationconfig", required=False, default="activation.json")
|
700
|
-
parser.add_argument("--no-print-metrics", required=False, action="store_true")
|
701
|
-
|
702
|
-
args, unknown = parser.parse_known_args()
|
703
|
-
self._is_fastcheck = args.fastcheck
|
704
|
-
if args.dsid is None:
|
705
|
-
# DEV mode
|
706
|
-
self._running_in_sim = True
|
707
|
-
print_metrics = not args.no_print_metrics
|
708
|
-
self._client = DebugClient(
|
709
|
-
activation_config_path=args.activationconfig,
|
710
|
-
extension_config_path=args.extensionconfig,
|
711
|
-
logger=api_logger,
|
712
|
-
local_ingest=args.local_ingest,
|
713
|
-
local_ingest_port=args.local_ingest_port,
|
714
|
-
print_metrics=print_metrics
|
715
|
-
)
|
716
|
-
RuntimeProperties.set_default_log_level(args.loglevel)
|
717
|
-
else:
|
718
|
-
# EEC mode
|
719
|
-
self._client = HttpClient(args.url, args.dsid, args.idtoken, api_logger)
|
720
|
-
self._task_id = args.dsid
|
721
|
-
self._monitoring_config_id = args.monitoring_config_id
|
722
|
-
api_logger.info(f"DSID = {self.task_id}, monitoring config id = {self._monitoring_config_id}")
|
723
|
-
|
724
|
-
self.activation_config = ActivationConfig(self._client.get_activation_config())
|
725
|
-
self.extension_config = self._client.get_extension_config()
|
726
|
-
self._feature_sets = self._client.get_feature_sets()
|
727
|
-
|
728
|
-
self.monitoring_config_name = self.activation_config.description
|
729
|
-
self.extension_version = self.activation_config.version
|
730
|
-
|
731
|
-
if not self._is_fastcheck:
|
732
|
-
try:
|
733
|
-
self.initialize()
|
734
|
-
if not self.is_helper:
|
735
|
-
self.schedule(self.query, timedelta(minutes=1))
|
736
|
-
except Exception as e:
|
737
|
-
msg = f"Error running self.initialize {self}: {e!r}"
|
738
|
-
api_logger.exception(msg)
|
739
|
-
self._client.send_status(Status(StatusValue.GENERIC_ERROR, msg))
|
740
|
-
self._initialization_error = msg
|
741
|
-
raise e
|
742
|
-
|
743
|
-
@property
|
744
|
-
def _metadata(self) -> dict:
|
745
|
-
return {
|
746
|
-
"dt.extension.config.id": self._runtime_properties.extconfig,
|
747
|
-
"dt.extension.ds": DATASOURCE_TYPE,
|
748
|
-
"dt.extension.version": self.extension_version,
|
749
|
-
"dt.extension.name": self.extension_name,
|
750
|
-
"monitoring.configuration": self.monitoring_config_name,
|
751
|
-
}
|
752
|
-
|
753
|
-
def _run_fastcheck(self):
|
754
|
-
api_logger.info(f"Running fastcheck for monitoring configuration '{self.monitoring_config_name}'")
|
755
|
-
try:
|
756
|
-
if self._fast_check_callback:
|
757
|
-
status = self._fast_check_callback(self.activation_config, self.extension_config)
|
758
|
-
api_logger.info(f"Sending fastcheck status: {status}")
|
759
|
-
self._client.send_status(status)
|
760
|
-
return
|
761
|
-
|
762
|
-
status = self.fastcheck()
|
763
|
-
api_logger.info(f"Sending fastcheck status: {status}")
|
764
|
-
self._client.send_status(status)
|
765
|
-
except Exception as e:
|
766
|
-
status = Status(StatusValue.GENERIC_ERROR, f"Python datasource fastcheck error: {e!r}")
|
767
|
-
api_logger.error(f"Error running fastcheck {self}: {e!r}")
|
768
|
-
self._client.send_status(status)
|
769
|
-
raise
|
770
|
-
|
771
|
-
def _run_callback(self, callback: WrappedCallback):
|
772
|
-
if not callback.running:
|
773
|
-
# Add the callback to the list of running callbacks
|
774
|
-
with self._running_callbacks_lock:
|
775
|
-
current_thread_id = threading.get_ident()
|
776
|
-
self._running_callbacks[current_thread_id] = callback
|
777
|
-
|
778
|
-
callback()
|
779
|
-
|
780
|
-
with self._sfm_metrics_lock:
|
781
|
-
self._callbackSfmReport[callback.name()] = callback
|
782
|
-
# Remove the callback from the list of running callbacks
|
783
|
-
with self._running_callbacks_lock:
|
784
|
-
self._running_callbacks.pop(current_thread_id, None)
|
785
|
-
|
786
|
-
def _callback_iteration(self, callback: WrappedCallback):
|
787
|
-
self._callbacks_executor.submit(self._run_callback, callback)
|
788
|
-
self._scheduler.enter(callback.interval.total_seconds(), 1, self._callback_iteration, (callback,))
|
789
|
-
|
790
|
-
def _start_extension_loop(self):
|
791
|
-
api_logger.debug(f"Starting main loop for monitoring configuration: '{self.monitoring_config_name}'")
|
792
|
-
|
793
|
-
# These were scheduled before the extension started, schedule them now
|
794
|
-
for callback in self._scheduled_callbacks_before_run:
|
795
|
-
self._schedule_callback(callback)
|
796
|
-
self._heartbeat_iteration()
|
797
|
-
self._metrics_iteration()
|
798
|
-
self._sfm_metrics_iteration()
|
799
|
-
self._timediff_iteration()
|
800
|
-
self._scheduler.run()
|
801
|
-
|
802
|
-
def _timediff_iteration(self):
|
803
|
-
self._internal_executor.submit(self._update_cluster_time_diff)
|
804
|
-
self._scheduler.enter(TIME_DIFF_INTERVAL.total_seconds(), 1, self._timediff_iteration)
|
805
|
-
|
806
|
-
def _heartbeat_iteration(self):
|
807
|
-
self._internal_executor.submit(self._heartbeat)
|
808
|
-
self._scheduler.enter(HEARTBEAT_INTERVAL.total_seconds(), 1, self._heartbeat_iteration)
|
809
|
-
|
810
|
-
def _metrics_iteration(self):
|
811
|
-
self._internal_executor.submit(self._send_metrics)
|
812
|
-
self._scheduler.enter(METRIC_SENDING_INTERVAL.total_seconds(), 1, self._metrics_iteration)
|
813
|
-
|
814
|
-
def _sfm_metrics_iteration(self):
|
815
|
-
self._internal_executor.submit(self._send_sfm_metrics)
|
816
|
-
self._scheduler.enter(SFM_METRIC_SENDING_INTERVAL.total_seconds(), 1, self._sfm_metrics_iteration)
|
817
|
-
|
818
|
-
def _send_metrics(self):
|
819
|
-
with self._metrics_lock:
|
820
|
-
with self._internal_callbacks_results_lock:
|
821
|
-
if self._metrics:
|
822
|
-
number_of_metrics = len(self._metrics)
|
823
|
-
responses = self._client.send_metrics(self._metrics)
|
824
|
-
|
825
|
-
self._internal_callbacks_results[self._send_metrics.__name__] = Status(StatusValue.OK)
|
826
|
-
lines_invalid = sum(response.lines_invalid for response in responses)
|
827
|
-
if lines_invalid > 0:
|
828
|
-
message = f"{lines_invalid} invalid metric lines found"
|
829
|
-
self._internal_callbacks_results[self._send_metrics.__name__] = Status(
|
830
|
-
StatusValue.GENERIC_ERROR, message
|
831
|
-
)
|
832
|
-
|
833
|
-
api_logger.info(f"Sent {number_of_metrics} metric lines to EEC: {responses}")
|
834
|
-
self._metrics = []
|
835
|
-
|
836
|
-
def _prepare_sfm_metrics(self) -> List[str]:
|
837
|
-
"""Prepare self monitoring metrics.
|
838
|
-
|
839
|
-
Builds the list of mint metric lines to send as self monitoring metrics.
|
840
|
-
"""
|
841
|
-
|
842
|
-
sfm_metrics: List[Metric] = []
|
843
|
-
sfm_dimensions = {"dt.extension.config.id": self.monitoring_config_id}
|
844
|
-
_add_sfm_metric(
|
845
|
-
SfmMetric("threads", active_count(), sfm_dimensions, client_facing=True, metric_type=MetricType.DELTA),
|
846
|
-
sfm_metrics,
|
847
|
-
)
|
848
|
-
|
849
|
-
for name, callback in self._callbackSfmReport.items():
|
850
|
-
sfm_dimensions = {"callback": name, "dt.extension.config.id": self.monitoring_config_id}
|
851
|
-
_add_sfm_metric(
|
852
|
-
SfmMetric(
|
853
|
-
"execution.time",
|
854
|
-
f"{callback.duration_interval_total:.4f}",
|
855
|
-
sfm_dimensions,
|
856
|
-
client_facing=True,
|
857
|
-
metric_type=MetricType.GAUGE,
|
858
|
-
),
|
859
|
-
sfm_metrics,
|
860
|
-
)
|
861
|
-
_add_sfm_metric(
|
862
|
-
SfmMetric(
|
863
|
-
"execution.total.count",
|
864
|
-
callback.executions_total,
|
865
|
-
sfm_dimensions,
|
866
|
-
client_facing=True,
|
867
|
-
metric_type=MetricType.DELTA,
|
868
|
-
),
|
869
|
-
sfm_metrics,
|
870
|
-
)
|
871
|
-
_add_sfm_metric(
|
872
|
-
SfmMetric(
|
873
|
-
"execution.count",
|
874
|
-
callback.executions_per_interval,
|
875
|
-
sfm_dimensions,
|
876
|
-
client_facing=True,
|
877
|
-
metric_type=MetricType.DELTA,
|
878
|
-
),
|
879
|
-
sfm_metrics,
|
880
|
-
)
|
881
|
-
_add_sfm_metric(
|
882
|
-
SfmMetric(
|
883
|
-
"execution.ok.count",
|
884
|
-
callback.ok_count,
|
885
|
-
sfm_dimensions,
|
886
|
-
client_facing=True,
|
887
|
-
metric_type=MetricType.DELTA,
|
888
|
-
),
|
889
|
-
sfm_metrics,
|
890
|
-
)
|
891
|
-
_add_sfm_metric(
|
892
|
-
SfmMetric(
|
893
|
-
"execution.timeout.count",
|
894
|
-
callback.timeouts_count,
|
895
|
-
sfm_dimensions,
|
896
|
-
client_facing=True,
|
897
|
-
metric_type=MetricType.DELTA,
|
898
|
-
),
|
899
|
-
sfm_metrics,
|
900
|
-
)
|
901
|
-
_add_sfm_metric(
|
902
|
-
SfmMetric(
|
903
|
-
"execution.exception.count",
|
904
|
-
callback.exception_count,
|
905
|
-
sfm_dimensions,
|
906
|
-
client_facing=True,
|
907
|
-
metric_type=MetricType.DELTA,
|
908
|
-
),
|
909
|
-
sfm_metrics,
|
910
|
-
)
|
911
|
-
callback.clear_sfm_metrics()
|
912
|
-
return [metric.to_mint_line() for metric in sfm_metrics]
|
913
|
-
|
914
|
-
def _send_sfm_metrics(self):
|
915
|
-
with self._sfm_metrics_lock:
|
916
|
-
lines = self._prepare_sfm_metrics()
|
917
|
-
# Flushes the cache of metrics, maybe we should only flush if they were successfully sent
|
918
|
-
self._callbackSfmReport.clear()
|
919
|
-
response = self._client.send_sfm_metrics(lines)
|
920
|
-
|
921
|
-
with self._internal_callbacks_results_lock:
|
922
|
-
self._internal_callbacks_results[self._send_sfm_metrics.__name__] = Status(StatusValue.OK)
|
923
|
-
if response.lines_invalid > 0:
|
924
|
-
message = f"{response.lines_invalid} invalid metric lines found"
|
925
|
-
self._internal_callbacks_results[self._send_sfm_metrics.__name__] = Status(
|
926
|
-
StatusValue.GENERIC_ERROR, message
|
927
|
-
)
|
928
|
-
|
929
|
-
def _build_current_status(self):
|
930
|
-
overall_status = Status(StatusValue.OK)
|
931
|
-
|
932
|
-
if self._initialization_error:
|
933
|
-
overall_status.status = StatusValue.GENERIC_ERROR
|
934
|
-
overall_status.message = self._initialization_error
|
935
|
-
return overall_status
|
936
|
-
|
937
|
-
internal_callback_error = False
|
938
|
-
messages = []
|
939
|
-
with self._internal_callbacks_results_lock:
|
940
|
-
for callback, result in self._internal_callbacks_results.items():
|
941
|
-
if result.is_error():
|
942
|
-
internal_callback_error = True
|
943
|
-
overall_status.status = result.status
|
944
|
-
messages.append(f"{callback}: {result.message}")
|
945
|
-
if internal_callback_error:
|
946
|
-
overall_status.message = "\n".join(messages)
|
947
|
-
return overall_status
|
948
|
-
|
949
|
-
for callback in self._scheduled_callbacks:
|
950
|
-
overall_status.timestamp = int(callback.get_adjusted_metric_timestamp().timestamp() * 1000)
|
951
|
-
if callback.status.is_error():
|
952
|
-
overall_status.status = callback.status.status
|
953
|
-
messages.append(f"{callback}: {callback.status.message}")
|
954
|
-
|
955
|
-
overall_status.message = "\n".join(messages)
|
956
|
-
return overall_status
|
957
|
-
|
958
|
-
def _update_cluster_time_diff(self):
|
959
|
-
self._cluster_time_diff = self._client.get_cluster_time_diff()
|
960
|
-
for callback in self._scheduled_callbacks:
|
961
|
-
callback.cluster_time_diff = self._cluster_time_diff
|
962
|
-
|
963
|
-
def _heartbeat(self):
|
964
|
-
response = bytes("not set", "utf-8")
|
965
|
-
try:
|
966
|
-
overall_status = self._build_current_status()
|
967
|
-
response = self._client.send_status(overall_status)
|
968
|
-
self._runtime_properties = RuntimeProperties(response)
|
969
|
-
except Exception as e:
|
970
|
-
api_logger.error(f"Heartbeat failed because {e}, response {response}", exc_info=True)
|
971
|
-
|
972
|
-
def __del__(self):
|
973
|
-
self._callbacks_executor.shutdown()
|
974
|
-
self._internal_executor.shutdown()
|
975
|
-
|
976
|
-
def _add_metric(self, metric: Metric):
|
977
|
-
metric.validate()
|
978
|
-
|
979
|
-
with self._running_callbacks_lock:
|
980
|
-
current_thread_id = threading.get_ident()
|
981
|
-
current_callback = self._running_callbacks.get(current_thread_id)
|
982
|
-
|
983
|
-
if current_callback is not None and metric.timestamp is None:
|
984
|
-
# Adjust the metric timestamp according to the callback start time
|
985
|
-
# If the user manually set a metric timestamp, don't adjust it
|
986
|
-
metric.timestamp = current_callback.get_adjusted_metric_timestamp()
|
987
|
-
elif current_callback is None and metric.timestamp is None:
|
988
|
-
api_logger.debug(
|
989
|
-
f"Metric {metric} was added by unknown thread {current_thread_id}, cannot adjust the timestamp"
|
990
|
-
)
|
991
|
-
|
992
|
-
with self._metrics_lock:
|
993
|
-
self._metrics.append(metric.to_mint_line())
|
994
|
-
|
995
|
-
def _add_mint_lines(self, lines: List[str]):
|
996
|
-
with self._metrics_lock:
|
997
|
-
self._metrics.extend(lines)
|
998
|
-
|
999
|
-
def _send_events_internal(self, events: Union[dict, List[dict]]):
|
1000
|
-
try:
|
1001
|
-
responses = self._client.send_events(events, self.log_event_enrichment)
|
1002
|
-
|
1003
|
-
for response in responses:
|
1004
|
-
with self._internal_callbacks_results_lock:
|
1005
|
-
self._internal_callbacks_results[self._send_events.__name__] = Status(StatusValue.OK)
|
1006
|
-
if not response or "error" not in response or "message" not in response["error"]:
|
1007
|
-
return
|
1008
|
-
self._internal_callbacks_results[self._send_events.__name__] = Status(
|
1009
|
-
StatusValue.GENERIC_ERROR, response["error"]["message"]
|
1010
|
-
)
|
1011
|
-
except Exception as e:
|
1012
|
-
api_logger.error(f"Error sending events: {e!r}", exc_info=True)
|
1013
|
-
with self._internal_callbacks_results_lock:
|
1014
|
-
self._internal_callbacks_results[self._send_events.__name__] = Status(StatusValue.GENERIC_ERROR, str(e))
|
1015
|
-
|
1016
|
-
def _send_events(self, events: Union[dict, List[dict]]):
|
1017
|
-
self._internal_executor.submit(self._send_events_internal, events)
|
1018
|
-
|
1019
|
-
def _send_dt_event(self, event: dict[str, str | int | dict[str, str]]):
|
1020
|
-
self._client.send_dt_event(event)
|
1021
|
-
|
1022
|
-
def get_version(self) -> str:
|
1023
|
-
"""Return the extension version."""
|
1024
|
-
return self.activation_config.version
|
1025
|
-
|
1026
|
-
@property
|
1027
|
-
def techrule(self) -> str:
|
1028
|
-
"""Internal property used by the EEC."""
|
1029
|
-
|
1030
|
-
return self._techrule
|
1031
|
-
|
1032
|
-
@techrule.setter
|
1033
|
-
def techrule(self, value):
|
1034
|
-
self._techrule = value
|
1035
|
-
|
1036
|
-
def get_activation_config(self) -> ActivationConfig:
|
1037
|
-
"""Retrieve the activation config.
|
1038
|
-
|
1039
|
-
Represents activation configuration assigned to this particular
|
1040
|
-
extension instance.
|
1041
|
-
|
1042
|
-
Returns:
|
1043
|
-
ActivationConfig object.
|
1044
|
-
"""
|
1045
|
-
return self.activation_config
|
1
|
+
# SPDX-FileCopyrightText: 2023-present Dynatrace LLC
|
2
|
+
#
|
3
|
+
# SPDX-License-Identifier: MIT
|
4
|
+
|
5
|
+
import logging
|
6
|
+
import sched
|
7
|
+
import signal
|
8
|
+
import sys
|
9
|
+
import threading
|
10
|
+
import time
|
11
|
+
from argparse import ArgumentParser
|
12
|
+
from collections import deque
|
13
|
+
from concurrent.futures import ThreadPoolExecutor
|
14
|
+
from datetime import datetime, timedelta, timezone
|
15
|
+
from enum import Enum
|
16
|
+
from itertools import chain
|
17
|
+
from threading import Lock, RLock, active_count
|
18
|
+
from typing import Any, Callable, ClassVar, Dict, List, NamedTuple, Optional, Union
|
19
|
+
|
20
|
+
from ..__about__ import __version__
|
21
|
+
from .activation import ActivationConfig, ActivationType
|
22
|
+
from .callback import WrappedCallback
|
23
|
+
from .communication import CommunicationClient, DebugClient, HttpClient, Status, StatusValue
|
24
|
+
from .event import Severity
|
25
|
+
from .metric import Metric, MetricType, SfmMetric, SummaryStat
|
26
|
+
from .runtime import RuntimeProperties
|
27
|
+
|
28
|
+
HEARTBEAT_INTERVAL = timedelta(seconds=30)
|
29
|
+
METRIC_SENDING_INTERVAL = timedelta(seconds=30)
|
30
|
+
SFM_METRIC_SENDING_INTERVAL = timedelta(seconds=60)
|
31
|
+
TIME_DIFF_INTERVAL = timedelta(seconds=60)
|
32
|
+
|
33
|
+
CALLBACKS_THREAD_POOL_SIZE = 100
|
34
|
+
INTERNAL_THREAD_POOL_SIZE = 20
|
35
|
+
|
36
|
+
RFC_3339_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
37
|
+
DATASOURCE_TYPE = "python"
|
38
|
+
|
39
|
+
logging.raiseExceptions = False
|
40
|
+
formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s (%(threadName)s): %(message)s")
|
41
|
+
error_handler = logging.StreamHandler()
|
42
|
+
error_handler.addFilter(lambda record: record.levelno >= logging.ERROR)
|
43
|
+
error_handler.setFormatter(formatter)
|
44
|
+
std_handler = logging.StreamHandler(sys.stdout)
|
45
|
+
std_handler.addFilter(lambda record: record.levelno < logging.ERROR)
|
46
|
+
std_handler.setFormatter(formatter)
|
47
|
+
extension_logger = logging.getLogger(__name__)
|
48
|
+
extension_logger.setLevel(logging.INFO)
|
49
|
+
extension_logger.addHandler(error_handler)
|
50
|
+
extension_logger.addHandler(std_handler)
|
51
|
+
|
52
|
+
api_logger = logging.getLogger("api")
|
53
|
+
api_logger.setLevel(logging.INFO)
|
54
|
+
api_logger.addHandler(error_handler)
|
55
|
+
api_logger.addHandler(std_handler)
|
56
|
+
|
57
|
+
DT_EVENT_SCHEMA = {
|
58
|
+
"eventType": str,
|
59
|
+
"title": str,
|
60
|
+
"startTime": int,
|
61
|
+
"endTime": int,
|
62
|
+
"timeout": int,
|
63
|
+
"entitySelector": str,
|
64
|
+
"properties": dict,
|
65
|
+
}
|
66
|
+
|
67
|
+
|
68
|
+
class AggregationMode(Enum):
|
69
|
+
ALL = "include_all"
|
70
|
+
NONE = "include_none"
|
71
|
+
LIST = "include_list"
|
72
|
+
|
73
|
+
|
74
|
+
class DtEventType(str, Enum):
|
75
|
+
"""Event type.
|
76
|
+
|
77
|
+
Note:
|
78
|
+
Official API v2 documentation:
|
79
|
+
|
80
|
+
https://docs.dynatrace.com/docs/dynatrace-api/environment-api/events-v2/post-event
|
81
|
+
"""
|
82
|
+
|
83
|
+
AVAILABILITY_EVENT = "AVAILABILITY_EVENT"
|
84
|
+
CUSTOM_INFO = "CUSTOM_INFO"
|
85
|
+
CUSTOM_ALERT = "CUSTOM_ALERT"
|
86
|
+
CUSTOM_ANNOTATION = "CUSTOM_ANNOTATION"
|
87
|
+
CUSTOM_CONFIGURATION = "CUSTOM_CONFIGURATION"
|
88
|
+
CUSTOM_DEPLOYMENT = "CUSTOM_DEPLOYMENT"
|
89
|
+
ERROR_EVENT = "ERROR_EVENT"
|
90
|
+
MARKED_FOR_TERMINATION = "MARKED_FOR_TERMINATION"
|
91
|
+
PERFORMANCE_EVENT = "PERFORMANCE_EVENT"
|
92
|
+
RESOURCE_CONTENTION_EVENT = "RESOURCE_CONTENTION_EVENT"
|
93
|
+
|
94
|
+
|
95
|
+
class CountMetricRegistrationEntry(NamedTuple):
|
96
|
+
metric_key: str
|
97
|
+
aggregation_mode: AggregationMode
|
98
|
+
dimensions_list: list[str]
|
99
|
+
|
100
|
+
@staticmethod
|
101
|
+
def make_list(metric_key: str, dimensions_list: List[str]):
|
102
|
+
"""Build an entry that uses defined list of dimensions for aggregation.
|
103
|
+
|
104
|
+
Args:
|
105
|
+
metric_key: Metric key in string.
|
106
|
+
dimensions_list: List of dimensions.
|
107
|
+
"""
|
108
|
+
return CountMetricRegistrationEntry(metric_key, AggregationMode.LIST, dimensions_list)
|
109
|
+
|
110
|
+
@staticmethod
|
111
|
+
def make_all(metric_key: str):
|
112
|
+
"""Build an entry that uses all mint dimensions for aggregation.
|
113
|
+
|
114
|
+
Args:
|
115
|
+
metric_key: Metric key in string.
|
116
|
+
"""
|
117
|
+
return CountMetricRegistrationEntry(metric_key, AggregationMode.ALL, [])
|
118
|
+
|
119
|
+
@staticmethod
|
120
|
+
def make_none(metric_key: str):
|
121
|
+
"""Build an entry that uses none of mint dimensions for aggregation.
|
122
|
+
|
123
|
+
Args:
|
124
|
+
metric_key: Metric key in string.
|
125
|
+
"""
|
126
|
+
return CountMetricRegistrationEntry(metric_key, AggregationMode.NONE, [])
|
127
|
+
|
128
|
+
def registration_items_dict(self):
|
129
|
+
result = {"aggregation_mode": self.aggregation_mode.value}
|
130
|
+
if self.aggregation_mode == AggregationMode.LIST:
|
131
|
+
result["dimensions_list"] = self.dimensions_list
|
132
|
+
return result
|
133
|
+
else:
|
134
|
+
return result
|
135
|
+
|
136
|
+
|
137
|
+
def _add_sfm_metric(metric: Metric, sfm_metrics: Optional[List[Metric]] = None):
|
138
|
+
if sfm_metrics is None:
|
139
|
+
sfm_metrics = []
|
140
|
+
metric.validate()
|
141
|
+
sfm_metrics.append(metric)
|
142
|
+
|
143
|
+
|
144
|
+
class Extension:
|
145
|
+
"""Base class for Python extensions.
|
146
|
+
|
147
|
+
Attributes:
|
148
|
+
logger: Embedded logger object for the extension.
|
149
|
+
"""
|
150
|
+
|
151
|
+
_instance: ClassVar = None
|
152
|
+
schedule_decorators: ClassVar = []
|
153
|
+
|
154
|
+
def __new__(cls):
|
155
|
+
if Extension._instance is None:
|
156
|
+
Extension._instance = super(__class__, cls).__new__(cls)
|
157
|
+
return Extension._instance
|
158
|
+
|
159
|
+
def __init__(self) -> None:
|
160
|
+
# do not initialize already created singleton
|
161
|
+
if hasattr(self, "logger"):
|
162
|
+
return
|
163
|
+
|
164
|
+
self.logger = extension_logger
|
165
|
+
|
166
|
+
self.extension_config: str = ""
|
167
|
+
self._feature_sets: dict[str, list[str]] = {}
|
168
|
+
|
169
|
+
# Useful metadata, populated once the extension is started
|
170
|
+
self.extension_name = "" # Needs to be set by the developer if they so decide
|
171
|
+
self.extension_version = ""
|
172
|
+
self.monitoring_config_name = ""
|
173
|
+
self._task_id = "development_task_id"
|
174
|
+
self._monitoring_config_id = "development_config_id"
|
175
|
+
|
176
|
+
# The user can override default EEC enrichment for logs
|
177
|
+
self.log_event_enrichment = True
|
178
|
+
|
179
|
+
# The Communication client
|
180
|
+
self._client: CommunicationClient = None # type: ignore
|
181
|
+
|
182
|
+
# Set to true when --fastcheck is passed as a parameter
|
183
|
+
self._is_fastcheck: bool = True
|
184
|
+
|
185
|
+
# If this is true, we are running locally during development
|
186
|
+
self._running_in_sim: bool = False
|
187
|
+
|
188
|
+
# Response from EEC to /alive/ requests
|
189
|
+
self._runtime_properties: RuntimeProperties = RuntimeProperties({})
|
190
|
+
|
191
|
+
# The time difference between the local machine and the cluster time, used to sync callbacks with cluster
|
192
|
+
self._cluster_time_diff: int = 0
|
193
|
+
|
194
|
+
# Optional callback to be invoked during the fastcheck
|
195
|
+
self._fast_check_callback: Optional[Callable[[ActivationConfig, str], Status]] = None
|
196
|
+
|
197
|
+
# List of all scheduled callbacks we must run
|
198
|
+
self._scheduled_callbacks: List[WrappedCallback] = []
|
199
|
+
self._scheduled_callbacks_before_run: List[WrappedCallback] = []
|
200
|
+
|
201
|
+
# Internal callbacks results, used to report statuses
|
202
|
+
self._internal_callbacks_results: Dict[str, Status] = {}
|
203
|
+
self._internal_callbacks_results_lock: Lock = Lock()
|
204
|
+
|
205
|
+
# Running callbacks, used to get the callback info when reporting metrics
|
206
|
+
self._running_callbacks: Dict[int, WrappedCallback] = {}
|
207
|
+
self._running_callbacks_lock: Lock = Lock()
|
208
|
+
|
209
|
+
self._scheduler = sched.scheduler(time.time, time.sleep)
|
210
|
+
|
211
|
+
# Executors for the callbacks and internal methods
|
212
|
+
self._callbacks_executor = ThreadPoolExecutor(max_workers=CALLBACKS_THREAD_POOL_SIZE)
|
213
|
+
self._internal_executor = ThreadPoolExecutor(max_workers=INTERNAL_THREAD_POOL_SIZE)
|
214
|
+
|
215
|
+
# Extension metrics
|
216
|
+
self._metrics_lock = RLock()
|
217
|
+
self._metrics: List[str] = []
|
218
|
+
|
219
|
+
# Self monitoring metrics
|
220
|
+
self._sfm_metrics_lock = Lock()
|
221
|
+
self._callbackSfmReport: Dict[str, WrappedCallback] = {}
|
222
|
+
|
223
|
+
# Count metric delta signals
|
224
|
+
self._delta_signal_buffer: set[str] = set()
|
225
|
+
self._registered_count_metrics: set[str] = set()
|
226
|
+
|
227
|
+
# Self tech rule
|
228
|
+
self._techrule = ""
|
229
|
+
|
230
|
+
# Error message from caught exception in self.initialize()
|
231
|
+
self._initialization_error: str = ""
|
232
|
+
|
233
|
+
self._parse_args()
|
234
|
+
|
235
|
+
for function, interval, args, activation_type in Extension.schedule_decorators:
|
236
|
+
params = (self,)
|
237
|
+
if args is not None:
|
238
|
+
params = params + args
|
239
|
+
self.schedule(function, interval, params, activation_type)
|
240
|
+
|
241
|
+
api_logger.info("-----------------------------------------------------")
|
242
|
+
api_logger.info(f"Starting {self.__class__} {self.extension_name}, version: {self.get_version()}")
|
243
|
+
|
244
|
+
@property
|
245
|
+
def is_helper(self) -> bool:
|
246
|
+
"""Internal property used by the EEC."""
|
247
|
+
|
248
|
+
return False
|
249
|
+
|
250
|
+
@property
|
251
|
+
def task_id(self) -> str:
|
252
|
+
"""Internal property used by the EEC."""
|
253
|
+
|
254
|
+
return self._task_id
|
255
|
+
|
256
|
+
@property
|
257
|
+
def monitoring_config_id(self) -> str:
|
258
|
+
"""Internal property used by the EEC.
|
259
|
+
|
260
|
+
Represents a unique identifier of the monitoring configuration.
|
261
|
+
that is assigned to this particular extension instance.
|
262
|
+
"""
|
263
|
+
|
264
|
+
return self._monitoring_config_id
|
265
|
+
|
266
|
+
def run(self):
|
267
|
+
"""Launch the extension instance.
|
268
|
+
|
269
|
+
Calling this method starts the main loop of the extension.
|
270
|
+
|
271
|
+
This method must be invoked once to start the extension,
|
272
|
+
|
273
|
+
if `--fastcheck` is set, the extension will run in fastcheck mode,
|
274
|
+
otherwise the main loop is started, which periodically runs:
|
275
|
+
|
276
|
+
* The scheduled callbacks
|
277
|
+
* The heartbeat method
|
278
|
+
* The metrics publisher method
|
279
|
+
"""
|
280
|
+
|
281
|
+
self._setup_signal_handlers()
|
282
|
+
if self._is_fastcheck:
|
283
|
+
return self._run_fastcheck()
|
284
|
+
self._start_extension_loop()
|
285
|
+
|
286
|
+
def _setup_signal_handlers(self):
|
287
|
+
if sys.platform == "win32":
|
288
|
+
signal.signal(signal.SIGBREAK, self._shutdown_signal_handler)
|
289
|
+
signal.signal(signal.SIGINT, self._shutdown_signal_handler)
|
290
|
+
|
291
|
+
def _shutdown_signal_handler(self, sig, frame): # noqa: ARG002
|
292
|
+
api_logger.info(f"{signal.Signals(sig).name} captured. Flushing metrics and exiting...")
|
293
|
+
self.on_shutdown()
|
294
|
+
self._send_metrics()
|
295
|
+
self._send_sfm_metrics()
|
296
|
+
sys.exit(0)
|
297
|
+
|
298
|
+
def on_shutdown(self):
|
299
|
+
"""Callback method to be invoked when the extension is shutting down.
|
300
|
+
|
301
|
+
Called when extension exits after it has received shutdown signal from EEC
|
302
|
+
This is executed before metrics are flushed to EEC
|
303
|
+
"""
|
304
|
+
pass
|
305
|
+
|
306
|
+
def _schedule_callback(self, callback: WrappedCallback):
|
307
|
+
if callback.activation_type is not None and callback.activation_type != self.activation_config.type:
|
308
|
+
api_logger.info(
|
309
|
+
f"Skipping {callback} with activation type {callback.activation_type} because it is not {self.activation_config.type}"
|
310
|
+
)
|
311
|
+
return
|
312
|
+
|
313
|
+
api_logger.debug(f"Scheduling callback {callback}")
|
314
|
+
|
315
|
+
# These properties are updated after the extension starts
|
316
|
+
callback.cluster_time_diff = self._cluster_time_diff
|
317
|
+
callback.running_in_sim = self._running_in_sim
|
318
|
+
self._scheduled_callbacks.append(callback)
|
319
|
+
self._scheduler.enter(callback.initial_wait_time(), 1, self._callback_iteration, (callback,))
|
320
|
+
|
321
|
+
def schedule(
|
322
|
+
self,
|
323
|
+
callback: Callable,
|
324
|
+
interval: Union[timedelta, int],
|
325
|
+
args: Optional[tuple] = None,
|
326
|
+
activation_type: Optional[ActivationType] = None,
|
327
|
+
) -> None:
|
328
|
+
"""Schedule a method to be executed periodically.
|
329
|
+
|
330
|
+
The callback method will be periodically invoked in a separate thread.
|
331
|
+
The callback method is always immediately scheduled for execution.
|
332
|
+
|
333
|
+
Args:
|
334
|
+
callback: The callback method to be invoked
|
335
|
+
interval: The time interval between invocations, can be a timedelta object,
|
336
|
+
or an int representing the number of seconds
|
337
|
+
args: Arguments to the callback, if any
|
338
|
+
activation_type: Optional activation type when this callback should run,
|
339
|
+
can be 'ActivationType.LOCAL' or 'ActivationType.REMOTE'
|
340
|
+
"""
|
341
|
+
|
342
|
+
if isinstance(interval, int):
|
343
|
+
interval = timedelta(seconds=interval)
|
344
|
+
|
345
|
+
callback = WrappedCallback(interval, callback, api_logger, args, activation_type=activation_type)
|
346
|
+
if self._is_fastcheck:
|
347
|
+
self._scheduled_callbacks_before_run.append(callback)
|
348
|
+
else:
|
349
|
+
self._schedule_callback(callback)
|
350
|
+
|
351
|
+
def query(self):
|
352
|
+
"""Callback to be executed every minute by default.
|
353
|
+
|
354
|
+
Optional method that can be implemented by subclasses.
|
355
|
+
The query method is always scheduled to run every minute.
|
356
|
+
"""
|
357
|
+
pass
|
358
|
+
|
359
|
+
def initialize(self):
|
360
|
+
"""Callback to be executed when the extension starts.
|
361
|
+
|
362
|
+
Called once after the extension starts and the processes arguments are parsed.
|
363
|
+
Sometimes there are tasks the user needs to do that must happen before runtime,
|
364
|
+
but after the activation config has been received, example: Setting the schedule frequency
|
365
|
+
based on the user input on the monitoring configuration, this can be done on this method
|
366
|
+
"""
|
367
|
+
pass
|
368
|
+
|
369
|
+
def fastcheck(self) -> Status:
|
370
|
+
"""Callback executed when extension is launched.
|
371
|
+
|
372
|
+
Called if the extension is run in the `fastcheck` mode. Only invoked for remote
|
373
|
+
extensions.
|
374
|
+
This method is not called if fastcheck callback was already registered with
|
375
|
+
Extension.register_fastcheck().
|
376
|
+
|
377
|
+
Returns:
|
378
|
+
Status with optional message whether the fastcheck succeed or failed.
|
379
|
+
"""
|
380
|
+
return Status(StatusValue.OK)
|
381
|
+
|
382
|
+
def register_fastcheck(self, fast_check_callback: Callable[[ActivationConfig, str], Status]):
|
383
|
+
"""Registers fastcheck callback that is executed in the `fastcheck` mode.
|
384
|
+
|
385
|
+
Extension.fastcheck() is not called if fastcheck callback is registered with this method
|
386
|
+
|
387
|
+
Args:
|
388
|
+
fast_check_callback: callable called with ActivationConfig and
|
389
|
+
extension_config arguments. Must return the Status with optional message
|
390
|
+
whether the fastcheck succeed or failed.
|
391
|
+
"""
|
392
|
+
if self._fast_check_callback:
|
393
|
+
api_logger.error("More than one function assigned to fastcheck, last registered one was kept.")
|
394
|
+
|
395
|
+
self._fast_check_callback = fast_check_callback
|
396
|
+
|
397
|
+
def _register_count_metrics(self, *count_metric_entries: CountMetricRegistrationEntry) -> None:
|
398
|
+
"""Send a count metric registration request to EEC.
|
399
|
+
|
400
|
+
Args:
|
401
|
+
count_metric_entries: CountMetricRegistrationEntry objects for each count metric to register
|
402
|
+
"""
|
403
|
+
json_pattern = {
|
404
|
+
metric_entry.metric_key: metric_entry.registration_items_dict() for metric_entry in count_metric_entries
|
405
|
+
}
|
406
|
+
self._client.register_count_metrics(json_pattern)
|
407
|
+
|
408
|
+
def _send_count_delta_signal(self, metric_keys: set[str], force: bool = True) -> None:
|
409
|
+
"""Send calculate-delta signal to EEC monotonic converter.
|
410
|
+
|
411
|
+
Args:
|
412
|
+
metric_keys: List with metrics for which we want to calculate deltas
|
413
|
+
force: If true, it forces the metrics from cache to be pushed into EEC and then delta signal request is
|
414
|
+
sent. Otherwise, it puts delta signal request in cache and request is sent after nearest (in time) sending
|
415
|
+
metrics to EEC event
|
416
|
+
"""
|
417
|
+
|
418
|
+
with self._metrics_lock:
|
419
|
+
if not force:
|
420
|
+
for key in metric_keys:
|
421
|
+
self._delta_signal_buffer.add(key)
|
422
|
+
return
|
423
|
+
|
424
|
+
self._send_metrics()
|
425
|
+
self._client.send_count_delta_signal(metric_keys)
|
426
|
+
self._delta_signal_buffer = {
|
427
|
+
metric_key for metric_key in self._delta_signal_buffer if metric_key not in metric_keys
|
428
|
+
}
|
429
|
+
|
430
|
+
def report_metric(
|
431
|
+
self,
|
432
|
+
key: str,
|
433
|
+
value: Union[float, str, int, SummaryStat],
|
434
|
+
dimensions: Optional[Dict[str, str]] = None,
|
435
|
+
techrule: Optional[str] = None,
|
436
|
+
timestamp: Optional[datetime] = None,
|
437
|
+
metric_type: MetricType = MetricType.GAUGE,
|
438
|
+
) -> None:
|
439
|
+
"""Report a metric.
|
440
|
+
|
441
|
+
Metric is sent to EEC using an HTTP request and MINT protocol. EEC then
|
442
|
+
sends the metrics to the tenant.
|
443
|
+
|
444
|
+
By default, it reports a gauge metric.
|
445
|
+
|
446
|
+
Args:
|
447
|
+
key: The metric key, must follow the MINT specification
|
448
|
+
value: The metric value, can be a simple value or a SummaryStat
|
449
|
+
dimensions: A dictionary of dimensions
|
450
|
+
techrule: The technology rule string set by self.techrule setter.
|
451
|
+
timestamp: The timestamp of the metric, defaults to the current time
|
452
|
+
metric_type: The type of the metric, defaults to MetricType.GAUGE
|
453
|
+
"""
|
454
|
+
|
455
|
+
if techrule:
|
456
|
+
if not dimensions:
|
457
|
+
dimensions = {}
|
458
|
+
if "dt.techrule.id" not in dimensions:
|
459
|
+
dimensions["dt.techrule.id"] = techrule
|
460
|
+
|
461
|
+
if metric_type == MetricType.COUNT and timestamp is None:
|
462
|
+
# We must report a timestamp for count metrics
|
463
|
+
timestamp = datetime.now()
|
464
|
+
|
465
|
+
metric = Metric(key=key, value=value, dimensions=dimensions, metric_type=metric_type, timestamp=timestamp)
|
466
|
+
self._add_metric(metric)
|
467
|
+
|
468
|
+
def report_mint_lines(self, lines: List[str]) -> None:
|
469
|
+
"""Report mint lines using the MINT protocol
|
470
|
+
|
471
|
+
Examples:
|
472
|
+
Metric lines must comply with the MINT format.
|
473
|
+
|
474
|
+
>>> self.report_mint_lines(["my_metric 1", "my_other_metric 2"])
|
475
|
+
|
476
|
+
Args:
|
477
|
+
lines: A list of mint lines
|
478
|
+
"""
|
479
|
+
self._add_mint_lines(lines)
|
480
|
+
|
481
|
+
def report_event(
|
482
|
+
self,
|
483
|
+
title: str,
|
484
|
+
description: str,
|
485
|
+
properties: Optional[dict] = None,
|
486
|
+
timestamp: Optional[datetime] = None,
|
487
|
+
severity: Union[Severity, str] = Severity.INFO,
|
488
|
+
) -> None:
|
489
|
+
"""Report an event using log ingest.
|
490
|
+
|
491
|
+
Args:
|
492
|
+
title: The title of the event
|
493
|
+
description: The description of the event
|
494
|
+
properties: A dictionary of extra event properties
|
495
|
+
timestamp: The timestamp of the event, defaults to the current time
|
496
|
+
severity: The severity of the event, defaults to Severity.INFO
|
497
|
+
"""
|
498
|
+
if timestamp is None:
|
499
|
+
timestamp = datetime.now(tz=timezone.utc)
|
500
|
+
|
501
|
+
if properties is None:
|
502
|
+
properties = {}
|
503
|
+
|
504
|
+
event = {
|
505
|
+
"content": f"{title}\n{description}",
|
506
|
+
"title": title,
|
507
|
+
"description": description,
|
508
|
+
"timestamp": timestamp.strftime(RFC_3339_FORMAT),
|
509
|
+
"severity": severity.value if isinstance(severity, Severity) else severity,
|
510
|
+
**self._metadata,
|
511
|
+
**properties,
|
512
|
+
}
|
513
|
+
self._send_events(event)
|
514
|
+
|
515
|
+
def report_dt_event(
|
516
|
+
self,
|
517
|
+
event_type: DtEventType,
|
518
|
+
title: str,
|
519
|
+
start_time: Optional[int] = None,
|
520
|
+
end_time: Optional[int] = None,
|
521
|
+
timeout: Optional[int] = None,
|
522
|
+
entity_selector: Optional[str] = None,
|
523
|
+
properties: Optional[dict[str, str]] = None,
|
524
|
+
) -> None:
|
525
|
+
"""
|
526
|
+
Reports an event using the v2 event ingest API.
|
527
|
+
|
528
|
+
Unlike ``report_event``, this directly raises an event or even a problem
|
529
|
+
based on the specified ``event_type``.
|
530
|
+
|
531
|
+
Note:
|
532
|
+
For reference see: https://www.dynatrace.com/support/help/dynatrace-api/environment-api/events-v2/post-event
|
533
|
+
|
534
|
+
Args:
|
535
|
+
event_type: The event type chosen from type Enum (required)
|
536
|
+
title: The title of the event (required)
|
537
|
+
start_time: The start time of event in UTC ms, if not set, current timestamp (optional)
|
538
|
+
end_time: The end time of event in UTC ms, if not set, current timestamp + timeout (optional)
|
539
|
+
timeout: The timeout of event in minutes, if not set, 15 (optional)
|
540
|
+
entity_selector: The entity selector, if not set, the event is associated with environment entity (optional)
|
541
|
+
properties: A map of event properties (optional)
|
542
|
+
"""
|
543
|
+
event: Dict[str, Any] = {"eventType": event_type, "title": title}
|
544
|
+
if start_time:
|
545
|
+
event["startTime"] = start_time
|
546
|
+
if end_time:
|
547
|
+
event["endTime"] = end_time
|
548
|
+
if timeout:
|
549
|
+
event["timeout"] = timeout
|
550
|
+
if entity_selector:
|
551
|
+
event["entitySelector"] = entity_selector
|
552
|
+
if properties:
|
553
|
+
event["properties"] = properties
|
554
|
+
|
555
|
+
self._send_dt_event(event)
|
556
|
+
|
557
|
+
def report_dt_event_dict(self, event: dict):
|
558
|
+
"""Report an event using event ingest API with provided dictionary.
|
559
|
+
|
560
|
+
Note:
|
561
|
+
For reference see: https://www.dynatrace.com/support/help/dynatrace-api/environment-api/events-v2/post-event
|
562
|
+
|
563
|
+
Format of the event dictionary::
|
564
|
+
|
565
|
+
{
|
566
|
+
"type": "object",
|
567
|
+
"required": ["eventType", "title"],
|
568
|
+
"properties": {
|
569
|
+
"eventType": {
|
570
|
+
"type": "string",
|
571
|
+
"enum": [
|
572
|
+
"CUSTOM_INFO",
|
573
|
+
"CUSTOM_ANNOTATION",
|
574
|
+
"CUSTOM_CONFIGURATION",
|
575
|
+
"CUSTOM_DEPLOYMENT",
|
576
|
+
"MARKED_FOR_TERMINATION",
|
577
|
+
"ERROR_EVENT",
|
578
|
+
"AVAILABILITY_EVENT",
|
579
|
+
"PERFORMANCE_EVENT",
|
580
|
+
"RESOURCE_CONTENTION_EVENT",
|
581
|
+
"CUSTOM_ALERT"
|
582
|
+
]
|
583
|
+
},
|
584
|
+
"title": {
|
585
|
+
"type": "string",
|
586
|
+
"minLength": 1
|
587
|
+
},
|
588
|
+
"startTime": {"type": "integer"},
|
589
|
+
"endTime": {"type": "integer"},
|
590
|
+
"timeout": {"type": "integer"},
|
591
|
+
"entitySelector": {"type": "string"},
|
592
|
+
"properties": {
|
593
|
+
"type": "object",
|
594
|
+
"patternProperties": {
|
595
|
+
"^.*$": {"type": "string"}
|
596
|
+
}
|
597
|
+
}
|
598
|
+
}
|
599
|
+
}
|
600
|
+
"""
|
601
|
+
|
602
|
+
if "eventType" not in event or "title" not in event:
|
603
|
+
raise ValueError('"eventType" not present' if "eventType" not in event else '"title" not present in event')
|
604
|
+
for key, value in event.items():
|
605
|
+
if DT_EVENT_SCHEMA[key] is None:
|
606
|
+
msg = f'invalid member: "{key}"'
|
607
|
+
raise ValueError(msg)
|
608
|
+
if key == "eventType" and value not in list(DtEventType):
|
609
|
+
msg = f"Event type must be a DtEventType enum value, got: {value}"
|
610
|
+
raise ValueError(msg)
|
611
|
+
if key == "properties":
|
612
|
+
for prop_key, prop_val in event[key].items():
|
613
|
+
if not isinstance(prop_key, str) or not isinstance(prop_val, str):
|
614
|
+
msg = f'invalid "properties" member: {prop_key}: {prop_val}, required: "str": str'
|
615
|
+
raise ValueError(msg)
|
616
|
+
self._send_dt_event(event)
|
617
|
+
|
618
|
+
def report_log_event(self, log_event: dict):
|
619
|
+
"""Report a custom log event using log ingest.
|
620
|
+
|
621
|
+
Note:
|
622
|
+
See reference: https://www.dynatrace.com/support/help/shortlink/log-monitoring-log-data-ingestion
|
623
|
+
|
624
|
+
Args:
|
625
|
+
log_event: The log event dictionary.
|
626
|
+
"""
|
627
|
+
self._send_events(log_event)
|
628
|
+
|
629
|
+
def report_log_events(self, log_events: List[dict]):
|
630
|
+
"""Report a list of custom log events using log ingest.
|
631
|
+
|
632
|
+
Args:
|
633
|
+
log_events: The list of log events
|
634
|
+
"""
|
635
|
+
self._send_events(log_events)
|
636
|
+
|
637
|
+
def report_log_lines(self, log_lines: List[Union[str, bytes]]):
|
638
|
+
"""Report a list of log lines using log ingest
|
639
|
+
|
640
|
+
Args:
|
641
|
+
log_lines: The list of log lines
|
642
|
+
"""
|
643
|
+
events = [{"content": line} for line in log_lines]
|
644
|
+
self._send_events(events)
|
645
|
+
|
646
|
+
@property
|
647
|
+
def enabled_feature_sets(self) -> dict[str, list[str]]:
|
648
|
+
"""Map of enabled feautre sets and corresponding metrics.
|
649
|
+
|
650
|
+
Returns:
|
651
|
+
Dictionary containing enabled feature sets with corresponding
|
652
|
+
metrics defined in ``extension.yaml``.
|
653
|
+
"""
|
654
|
+
return {
|
655
|
+
feature_set_name: metric_keys
|
656
|
+
for feature_set_name, metric_keys in self._feature_sets.items()
|
657
|
+
if feature_set_name in self.activation_config.feature_sets or feature_set_name == "default"
|
658
|
+
}
|
659
|
+
|
660
|
+
@property
|
661
|
+
def enabled_feature_sets_names(self) -> list[str]:
|
662
|
+
"""Names of enabled feature sets.
|
663
|
+
|
664
|
+
Returns:
|
665
|
+
List containing names of enabled feature sets.
|
666
|
+
"""
|
667
|
+
return list(self.enabled_feature_sets.keys())
|
668
|
+
|
669
|
+
@property
|
670
|
+
def enabled_feature_sets_metrics(self) -> list[str]:
|
671
|
+
"""Enabled metrics.
|
672
|
+
|
673
|
+
Returns:
|
674
|
+
List of all metric keys from enabled feature sets
|
675
|
+
"""
|
676
|
+
return list(chain(*self.enabled_feature_sets.values()))
|
677
|
+
|
678
|
+
def _parse_args(self):
|
679
|
+
parser = ArgumentParser(description="Python extension parameters")
|
680
|
+
|
681
|
+
# Production parameters, these are passed by the EEC
|
682
|
+
parser.add_argument("--dsid", required=False, default=None)
|
683
|
+
parser.add_argument("--url", required=False)
|
684
|
+
parser.add_argument("--idtoken", required=False)
|
685
|
+
parser.add_argument(
|
686
|
+
"--loglevel",
|
687
|
+
help="Set extension log level. Info is default.",
|
688
|
+
type=str,
|
689
|
+
choices=["debug", "info"],
|
690
|
+
default="info",
|
691
|
+
)
|
692
|
+
parser.add_argument("--fastcheck", action="store_true", default=False)
|
693
|
+
parser.add_argument("--monitoring_config_id", required=False, default=None)
|
694
|
+
parser.add_argument("--local-ingest", action="store_true", default=False)
|
695
|
+
parser.add_argument("--local-ingest-port", required=False, default=14499)
|
696
|
+
|
697
|
+
# Debug parameters, these are used when running the extension locally
|
698
|
+
parser.add_argument("--extensionconfig", required=False, default=None)
|
699
|
+
parser.add_argument("--activationconfig", required=False, default="activation.json")
|
700
|
+
parser.add_argument("--no-print-metrics", required=False, action="store_true")
|
701
|
+
|
702
|
+
args, unknown = parser.parse_known_args()
|
703
|
+
self._is_fastcheck = args.fastcheck
|
704
|
+
if args.dsid is None:
|
705
|
+
# DEV mode
|
706
|
+
self._running_in_sim = True
|
707
|
+
print_metrics = not args.no_print_metrics
|
708
|
+
self._client = DebugClient(
|
709
|
+
activation_config_path=args.activationconfig,
|
710
|
+
extension_config_path=args.extensionconfig,
|
711
|
+
logger=api_logger,
|
712
|
+
local_ingest=args.local_ingest,
|
713
|
+
local_ingest_port=args.local_ingest_port,
|
714
|
+
print_metrics=print_metrics
|
715
|
+
)
|
716
|
+
RuntimeProperties.set_default_log_level(args.loglevel)
|
717
|
+
else:
|
718
|
+
# EEC mode
|
719
|
+
self._client = HttpClient(args.url, args.dsid, args.idtoken, api_logger)
|
720
|
+
self._task_id = args.dsid
|
721
|
+
self._monitoring_config_id = args.monitoring_config_id
|
722
|
+
api_logger.info(f"DSID = {self.task_id}, monitoring config id = {self._monitoring_config_id}")
|
723
|
+
|
724
|
+
self.activation_config = ActivationConfig(self._client.get_activation_config())
|
725
|
+
self.extension_config = self._client.get_extension_config()
|
726
|
+
self._feature_sets = self._client.get_feature_sets()
|
727
|
+
|
728
|
+
self.monitoring_config_name = self.activation_config.description
|
729
|
+
self.extension_version = self.activation_config.version
|
730
|
+
|
731
|
+
if not self._is_fastcheck:
|
732
|
+
try:
|
733
|
+
self.initialize()
|
734
|
+
if not self.is_helper:
|
735
|
+
self.schedule(self.query, timedelta(minutes=1))
|
736
|
+
except Exception as e:
|
737
|
+
msg = f"Error running self.initialize {self}: {e!r}"
|
738
|
+
api_logger.exception(msg)
|
739
|
+
self._client.send_status(Status(StatusValue.GENERIC_ERROR, msg))
|
740
|
+
self._initialization_error = msg
|
741
|
+
raise e
|
742
|
+
|
743
|
+
@property
|
744
|
+
def _metadata(self) -> dict:
|
745
|
+
return {
|
746
|
+
"dt.extension.config.id": self._runtime_properties.extconfig,
|
747
|
+
"dt.extension.ds": DATASOURCE_TYPE,
|
748
|
+
"dt.extension.version": self.extension_version,
|
749
|
+
"dt.extension.name": self.extension_name,
|
750
|
+
"monitoring.configuration": self.monitoring_config_name,
|
751
|
+
}
|
752
|
+
|
753
|
+
def _run_fastcheck(self):
|
754
|
+
api_logger.info(f"Running fastcheck for monitoring configuration '{self.monitoring_config_name}'")
|
755
|
+
try:
|
756
|
+
if self._fast_check_callback:
|
757
|
+
status = self._fast_check_callback(self.activation_config, self.extension_config)
|
758
|
+
api_logger.info(f"Sending fastcheck status: {status}")
|
759
|
+
self._client.send_status(status)
|
760
|
+
return
|
761
|
+
|
762
|
+
status = self.fastcheck()
|
763
|
+
api_logger.info(f"Sending fastcheck status: {status}")
|
764
|
+
self._client.send_status(status)
|
765
|
+
except Exception as e:
|
766
|
+
status = Status(StatusValue.GENERIC_ERROR, f"Python datasource fastcheck error: {e!r}")
|
767
|
+
api_logger.error(f"Error running fastcheck {self}: {e!r}")
|
768
|
+
self._client.send_status(status)
|
769
|
+
raise
|
770
|
+
|
771
|
+
def _run_callback(self, callback: WrappedCallback):
|
772
|
+
if not callback.running:
|
773
|
+
# Add the callback to the list of running callbacks
|
774
|
+
with self._running_callbacks_lock:
|
775
|
+
current_thread_id = threading.get_ident()
|
776
|
+
self._running_callbacks[current_thread_id] = callback
|
777
|
+
|
778
|
+
callback()
|
779
|
+
|
780
|
+
with self._sfm_metrics_lock:
|
781
|
+
self._callbackSfmReport[callback.name()] = callback
|
782
|
+
# Remove the callback from the list of running callbacks
|
783
|
+
with self._running_callbacks_lock:
|
784
|
+
self._running_callbacks.pop(current_thread_id, None)
|
785
|
+
|
786
|
+
def _callback_iteration(self, callback: WrappedCallback):
|
787
|
+
self._callbacks_executor.submit(self._run_callback, callback)
|
788
|
+
self._scheduler.enter(callback.interval.total_seconds(), 1, self._callback_iteration, (callback,))
|
789
|
+
|
790
|
+
def _start_extension_loop(self):
|
791
|
+
api_logger.debug(f"Starting main loop for monitoring configuration: '{self.monitoring_config_name}'")
|
792
|
+
|
793
|
+
# These were scheduled before the extension started, schedule them now
|
794
|
+
for callback in self._scheduled_callbacks_before_run:
|
795
|
+
self._schedule_callback(callback)
|
796
|
+
self._heartbeat_iteration()
|
797
|
+
self._metrics_iteration()
|
798
|
+
self._sfm_metrics_iteration()
|
799
|
+
self._timediff_iteration()
|
800
|
+
self._scheduler.run()
|
801
|
+
|
802
|
+
def _timediff_iteration(self):
|
803
|
+
self._internal_executor.submit(self._update_cluster_time_diff)
|
804
|
+
self._scheduler.enter(TIME_DIFF_INTERVAL.total_seconds(), 1, self._timediff_iteration)
|
805
|
+
|
806
|
+
def _heartbeat_iteration(self):
|
807
|
+
self._internal_executor.submit(self._heartbeat)
|
808
|
+
self._scheduler.enter(HEARTBEAT_INTERVAL.total_seconds(), 1, self._heartbeat_iteration)
|
809
|
+
|
810
|
+
def _metrics_iteration(self):
|
811
|
+
self._internal_executor.submit(self._send_metrics)
|
812
|
+
self._scheduler.enter(METRIC_SENDING_INTERVAL.total_seconds(), 1, self._metrics_iteration)
|
813
|
+
|
814
|
+
def _sfm_metrics_iteration(self):
|
815
|
+
self._internal_executor.submit(self._send_sfm_metrics)
|
816
|
+
self._scheduler.enter(SFM_METRIC_SENDING_INTERVAL.total_seconds(), 1, self._sfm_metrics_iteration)
|
817
|
+
|
818
|
+
def _send_metrics(self):
|
819
|
+
with self._metrics_lock:
|
820
|
+
with self._internal_callbacks_results_lock:
|
821
|
+
if self._metrics:
|
822
|
+
number_of_metrics = len(self._metrics)
|
823
|
+
responses = self._client.send_metrics(self._metrics)
|
824
|
+
|
825
|
+
self._internal_callbacks_results[self._send_metrics.__name__] = Status(StatusValue.OK)
|
826
|
+
lines_invalid = sum(response.lines_invalid for response in responses)
|
827
|
+
if lines_invalid > 0:
|
828
|
+
message = f"{lines_invalid} invalid metric lines found"
|
829
|
+
self._internal_callbacks_results[self._send_metrics.__name__] = Status(
|
830
|
+
StatusValue.GENERIC_ERROR, message
|
831
|
+
)
|
832
|
+
|
833
|
+
api_logger.info(f"Sent {number_of_metrics} metric lines to EEC: {responses}")
|
834
|
+
self._metrics = []
|
835
|
+
|
836
|
+
def _prepare_sfm_metrics(self) -> List[str]:
|
837
|
+
"""Prepare self monitoring metrics.
|
838
|
+
|
839
|
+
Builds the list of mint metric lines to send as self monitoring metrics.
|
840
|
+
"""
|
841
|
+
|
842
|
+
sfm_metrics: List[Metric] = []
|
843
|
+
sfm_dimensions = {"dt.extension.config.id": self.monitoring_config_id}
|
844
|
+
_add_sfm_metric(
|
845
|
+
SfmMetric("threads", active_count(), sfm_dimensions, client_facing=True, metric_type=MetricType.DELTA),
|
846
|
+
sfm_metrics,
|
847
|
+
)
|
848
|
+
|
849
|
+
for name, callback in self._callbackSfmReport.items():
|
850
|
+
sfm_dimensions = {"callback": name, "dt.extension.config.id": self.monitoring_config_id}
|
851
|
+
_add_sfm_metric(
|
852
|
+
SfmMetric(
|
853
|
+
"execution.time",
|
854
|
+
f"{callback.duration_interval_total:.4f}",
|
855
|
+
sfm_dimensions,
|
856
|
+
client_facing=True,
|
857
|
+
metric_type=MetricType.GAUGE,
|
858
|
+
),
|
859
|
+
sfm_metrics,
|
860
|
+
)
|
861
|
+
_add_sfm_metric(
|
862
|
+
SfmMetric(
|
863
|
+
"execution.total.count",
|
864
|
+
callback.executions_total,
|
865
|
+
sfm_dimensions,
|
866
|
+
client_facing=True,
|
867
|
+
metric_type=MetricType.DELTA,
|
868
|
+
),
|
869
|
+
sfm_metrics,
|
870
|
+
)
|
871
|
+
_add_sfm_metric(
|
872
|
+
SfmMetric(
|
873
|
+
"execution.count",
|
874
|
+
callback.executions_per_interval,
|
875
|
+
sfm_dimensions,
|
876
|
+
client_facing=True,
|
877
|
+
metric_type=MetricType.DELTA,
|
878
|
+
),
|
879
|
+
sfm_metrics,
|
880
|
+
)
|
881
|
+
_add_sfm_metric(
|
882
|
+
SfmMetric(
|
883
|
+
"execution.ok.count",
|
884
|
+
callback.ok_count,
|
885
|
+
sfm_dimensions,
|
886
|
+
client_facing=True,
|
887
|
+
metric_type=MetricType.DELTA,
|
888
|
+
),
|
889
|
+
sfm_metrics,
|
890
|
+
)
|
891
|
+
_add_sfm_metric(
|
892
|
+
SfmMetric(
|
893
|
+
"execution.timeout.count",
|
894
|
+
callback.timeouts_count,
|
895
|
+
sfm_dimensions,
|
896
|
+
client_facing=True,
|
897
|
+
metric_type=MetricType.DELTA,
|
898
|
+
),
|
899
|
+
sfm_metrics,
|
900
|
+
)
|
901
|
+
_add_sfm_metric(
|
902
|
+
SfmMetric(
|
903
|
+
"execution.exception.count",
|
904
|
+
callback.exception_count,
|
905
|
+
sfm_dimensions,
|
906
|
+
client_facing=True,
|
907
|
+
metric_type=MetricType.DELTA,
|
908
|
+
),
|
909
|
+
sfm_metrics,
|
910
|
+
)
|
911
|
+
callback.clear_sfm_metrics()
|
912
|
+
return [metric.to_mint_line() for metric in sfm_metrics]
|
913
|
+
|
914
|
+
def _send_sfm_metrics(self):
|
915
|
+
with self._sfm_metrics_lock:
|
916
|
+
lines = self._prepare_sfm_metrics()
|
917
|
+
# Flushes the cache of metrics, maybe we should only flush if they were successfully sent
|
918
|
+
self._callbackSfmReport.clear()
|
919
|
+
response = self._client.send_sfm_metrics(lines)
|
920
|
+
|
921
|
+
with self._internal_callbacks_results_lock:
|
922
|
+
self._internal_callbacks_results[self._send_sfm_metrics.__name__] = Status(StatusValue.OK)
|
923
|
+
if response.lines_invalid > 0:
|
924
|
+
message = f"{response.lines_invalid} invalid metric lines found"
|
925
|
+
self._internal_callbacks_results[self._send_sfm_metrics.__name__] = Status(
|
926
|
+
StatusValue.GENERIC_ERROR, message
|
927
|
+
)
|
928
|
+
|
929
|
+
def _build_current_status(self):
|
930
|
+
overall_status = Status(StatusValue.OK)
|
931
|
+
|
932
|
+
if self._initialization_error:
|
933
|
+
overall_status.status = StatusValue.GENERIC_ERROR
|
934
|
+
overall_status.message = self._initialization_error
|
935
|
+
return overall_status
|
936
|
+
|
937
|
+
internal_callback_error = False
|
938
|
+
messages = []
|
939
|
+
with self._internal_callbacks_results_lock:
|
940
|
+
for callback, result in self._internal_callbacks_results.items():
|
941
|
+
if result.is_error():
|
942
|
+
internal_callback_error = True
|
943
|
+
overall_status.status = result.status
|
944
|
+
messages.append(f"{callback}: {result.message}")
|
945
|
+
if internal_callback_error:
|
946
|
+
overall_status.message = "\n".join(messages)
|
947
|
+
return overall_status
|
948
|
+
|
949
|
+
for callback in self._scheduled_callbacks:
|
950
|
+
overall_status.timestamp = int(callback.get_adjusted_metric_timestamp().timestamp() * 1000)
|
951
|
+
if callback.status.is_error():
|
952
|
+
overall_status.status = callback.status.status
|
953
|
+
messages.append(f"{callback}: {callback.status.message}")
|
954
|
+
|
955
|
+
overall_status.message = "\n".join(messages)
|
956
|
+
return overall_status
|
957
|
+
|
958
|
+
def _update_cluster_time_diff(self):
|
959
|
+
self._cluster_time_diff = self._client.get_cluster_time_diff()
|
960
|
+
for callback in self._scheduled_callbacks:
|
961
|
+
callback.cluster_time_diff = self._cluster_time_diff
|
962
|
+
|
963
|
+
def _heartbeat(self):
|
964
|
+
response = bytes("not set", "utf-8")
|
965
|
+
try:
|
966
|
+
overall_status = self._build_current_status()
|
967
|
+
response = self._client.send_status(overall_status)
|
968
|
+
self._runtime_properties = RuntimeProperties(response)
|
969
|
+
except Exception as e:
|
970
|
+
api_logger.error(f"Heartbeat failed because {e}, response {response}", exc_info=True)
|
971
|
+
|
972
|
+
def __del__(self):
|
973
|
+
self._callbacks_executor.shutdown()
|
974
|
+
self._internal_executor.shutdown()
|
975
|
+
|
976
|
+
def _add_metric(self, metric: Metric):
|
977
|
+
metric.validate()
|
978
|
+
|
979
|
+
with self._running_callbacks_lock:
|
980
|
+
current_thread_id = threading.get_ident()
|
981
|
+
current_callback = self._running_callbacks.get(current_thread_id)
|
982
|
+
|
983
|
+
if current_callback is not None and metric.timestamp is None:
|
984
|
+
# Adjust the metric timestamp according to the callback start time
|
985
|
+
# If the user manually set a metric timestamp, don't adjust it
|
986
|
+
metric.timestamp = current_callback.get_adjusted_metric_timestamp()
|
987
|
+
elif current_callback is None and metric.timestamp is None:
|
988
|
+
api_logger.debug(
|
989
|
+
f"Metric {metric} was added by unknown thread {current_thread_id}, cannot adjust the timestamp"
|
990
|
+
)
|
991
|
+
|
992
|
+
with self._metrics_lock:
|
993
|
+
self._metrics.append(metric.to_mint_line())
|
994
|
+
|
995
|
+
def _add_mint_lines(self, lines: List[str]):
|
996
|
+
with self._metrics_lock:
|
997
|
+
self._metrics.extend(lines)
|
998
|
+
|
999
|
+
def _send_events_internal(self, events: Union[dict, List[dict]]):
|
1000
|
+
try:
|
1001
|
+
responses = self._client.send_events(events, self.log_event_enrichment)
|
1002
|
+
|
1003
|
+
for response in responses:
|
1004
|
+
with self._internal_callbacks_results_lock:
|
1005
|
+
self._internal_callbacks_results[self._send_events.__name__] = Status(StatusValue.OK)
|
1006
|
+
if not response or "error" not in response or "message" not in response["error"]:
|
1007
|
+
return
|
1008
|
+
self._internal_callbacks_results[self._send_events.__name__] = Status(
|
1009
|
+
StatusValue.GENERIC_ERROR, response["error"]["message"]
|
1010
|
+
)
|
1011
|
+
except Exception as e:
|
1012
|
+
api_logger.error(f"Error sending events: {e!r}", exc_info=True)
|
1013
|
+
with self._internal_callbacks_results_lock:
|
1014
|
+
self._internal_callbacks_results[self._send_events.__name__] = Status(StatusValue.GENERIC_ERROR, str(e))
|
1015
|
+
|
1016
|
+
def _send_events(self, events: Union[dict, List[dict]]):
|
1017
|
+
self._internal_executor.submit(self._send_events_internal, events)
|
1018
|
+
|
1019
|
+
def _send_dt_event(self, event: dict[str, str | int | dict[str, str]]):
|
1020
|
+
self._client.send_dt_event(event)
|
1021
|
+
|
1022
|
+
def get_version(self) -> str:
|
1023
|
+
"""Return the extension version."""
|
1024
|
+
return self.activation_config.version
|
1025
|
+
|
1026
|
+
@property
|
1027
|
+
def techrule(self) -> str:
|
1028
|
+
"""Internal property used by the EEC."""
|
1029
|
+
|
1030
|
+
return self._techrule
|
1031
|
+
|
1032
|
+
@techrule.setter
|
1033
|
+
def techrule(self, value):
|
1034
|
+
self._techrule = value
|
1035
|
+
|
1036
|
+
def get_activation_config(self) -> ActivationConfig:
|
1037
|
+
"""Retrieve the activation config.
|
1038
|
+
|
1039
|
+
Represents activation configuration assigned to this particular
|
1040
|
+
extension instance.
|
1041
|
+
|
1042
|
+
Returns:
|
1043
|
+
ActivationConfig object.
|
1044
|
+
"""
|
1045
|
+
return self.activation_config
|