detectkit 0.2.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. detectkit/__init__.py +17 -0
  2. detectkit/alerting/__init__.py +13 -0
  3. detectkit/alerting/channels/__init__.py +21 -0
  4. detectkit/alerting/channels/base.py +193 -0
  5. detectkit/alerting/channels/email.py +146 -0
  6. detectkit/alerting/channels/factory.py +193 -0
  7. detectkit/alerting/channels/mattermost.py +53 -0
  8. detectkit/alerting/channels/slack.py +55 -0
  9. detectkit/alerting/channels/telegram.py +110 -0
  10. detectkit/alerting/channels/webhook.py +139 -0
  11. detectkit/alerting/orchestrator.py +369 -0
  12. detectkit/cli/__init__.py +1 -0
  13. detectkit/cli/commands/__init__.py +1 -0
  14. detectkit/cli/commands/init.py +282 -0
  15. detectkit/cli/commands/run.py +486 -0
  16. detectkit/cli/commands/test_alert.py +184 -0
  17. detectkit/cli/main.py +186 -0
  18. detectkit/config/__init__.py +30 -0
  19. detectkit/config/metric_config.py +499 -0
  20. detectkit/config/profile.py +285 -0
  21. detectkit/config/project_config.py +164 -0
  22. detectkit/config/validator.py +124 -0
  23. detectkit/core/__init__.py +6 -0
  24. detectkit/core/interval.py +132 -0
  25. detectkit/core/models.py +106 -0
  26. detectkit/database/__init__.py +27 -0
  27. detectkit/database/clickhouse_manager.py +393 -0
  28. detectkit/database/internal_tables.py +724 -0
  29. detectkit/database/manager.py +324 -0
  30. detectkit/database/tables.py +138 -0
  31. detectkit/detectors/__init__.py +6 -0
  32. detectkit/detectors/base.py +441 -0
  33. detectkit/detectors/factory.py +138 -0
  34. detectkit/detectors/statistical/__init__.py +8 -0
  35. detectkit/detectors/statistical/iqr.py +508 -0
  36. detectkit/detectors/statistical/mad.py +478 -0
  37. detectkit/detectors/statistical/manual_bounds.py +206 -0
  38. detectkit/detectors/statistical/zscore.py +491 -0
  39. detectkit/loaders/__init__.py +6 -0
  40. detectkit/loaders/metric_loader.py +470 -0
  41. detectkit/loaders/query_template.py +164 -0
  42. detectkit/orchestration/__init__.py +9 -0
  43. detectkit/orchestration/task_manager.py +746 -0
  44. detectkit/utils/__init__.py +17 -0
  45. detectkit/utils/stats.py +196 -0
  46. detectkit-0.2.4.dist-info/METADATA +237 -0
  47. detectkit-0.2.4.dist-info/RECORD +51 -0
  48. detectkit-0.2.4.dist-info/WHEEL +5 -0
  49. detectkit-0.2.4.dist-info/entry_points.txt +2 -0
  50. detectkit-0.2.4.dist-info/licenses/LICENSE +21 -0
  51. detectkit-0.2.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,746 @@
1
+ """
2
+ Task manager for metric processing pipeline.
3
+
4
+ Orchestrates the complete workflow:
5
+ 1. Load data from database
6
+ 2. Run anomaly detection
7
+ 3. Send alerts
8
+ """
9
+
10
+ from datetime import datetime, timezone, timedelta
11
+ from enum import Enum
12
+ from typing import Dict, List, Optional
13
+ import json
14
+
15
+ import click
16
+ import numpy as np
17
+
18
+ from detectkit.alerting.channels.base import AlertData, BaseAlertChannel
19
+ from detectkit.alerting.channels.factory import AlertChannelFactory
20
+ from detectkit.alerting.orchestrator import (
21
+ AlertConditions,
22
+ AlertOrchestrator,
23
+ DetectionRecord,
24
+ )
25
+ from detectkit.config.metric_config import MetricConfig
26
+ from detectkit.core.interval import Interval
27
+ from detectkit.database.internal_tables import InternalTablesManager
28
+ from detectkit.detectors.base import BaseDetector
29
+ from detectkit.detectors.factory import DetectorFactory
30
+ from detectkit.loaders.metric_loader import MetricLoader
31
+
32
+
33
+ class PipelineStep(str, Enum):
34
+ """Pipeline execution steps."""
35
+
36
+ LOAD = "load"
37
+ DETECT = "detect"
38
+ ALERT = "alert"
39
+
40
+
41
+ class TaskStatus(str, Enum):
42
+ """Task execution status."""
43
+
44
+ RUNNING = "running"
45
+ SUCCESS = "success"
46
+ FAILED = "failed"
47
+
48
+
49
+ class TaskManager:
50
+ """
51
+ Manages metric processing pipeline.
52
+
53
+ Responsibilities:
54
+ - Execute pipeline steps (load, detect, alert)
55
+ - Task locking to prevent concurrent runs
56
+ - Idempotency (resume from last processed timestamp)
57
+ - Error handling and status tracking
58
+
59
+ Example:
60
+ >>> config = MetricConfig.from_yaml("metrics/cpu_usage.yml")
61
+ >>> manager = TaskManager(
62
+ ... internal_manager=internal_tables,
63
+ ... db_manager=clickhouse,
64
+ ... )
65
+ >>> manager.run_metric(
66
+ ... config,
67
+ ... steps=[PipelineStep.LOAD, PipelineStep.DETECT, PipelineStep.ALERT]
68
+ ... )
69
+ """
70
+
71
+ def __init__(
72
+ self,
73
+ internal_manager: InternalTablesManager,
74
+ db_manager, # BaseDatabaseManager
75
+ profiles_config=None, # ProfilesConfig (optional for backward compatibility)
76
+ ):
77
+ """
78
+ Initialize task manager.
79
+
80
+ Args:
81
+ internal_manager: Manager for internal detectk tables
82
+ db_manager: Database manager for metric data
83
+ profiles_config: Profiles configuration (for alert channels)
84
+ """
85
+ self.internal = internal_manager
86
+ self.db_manager = db_manager
87
+ self.profiles_config = profiles_config
88
+
89
+ def run_metric(
90
+ self,
91
+ config: MetricConfig,
92
+ steps: Optional[List[PipelineStep]] = None,
93
+ from_date: Optional[datetime] = None,
94
+ to_date: Optional[datetime] = None,
95
+ full_refresh: bool = False,
96
+ force: bool = False,
97
+ ) -> Dict[str, any]:
98
+ """
99
+ Run metric processing pipeline.
100
+
101
+ Args:
102
+ config: Metric configuration
103
+ steps: Pipeline steps to execute (default: all steps)
104
+ from_date: Start date for data loading (optional)
105
+ to_date: End date for data loading (optional)
106
+ full_refresh: Delete all existing data and reload from scratch
107
+ force: Ignore task locks
108
+
109
+ Returns:
110
+ Dict with execution results:
111
+ {
112
+ "status": "success" | "failed",
113
+ "steps_completed": ["load", "detect"],
114
+ "datapoints_loaded": 1000,
115
+ "anomalies_detected": 5,
116
+ "alerts_sent": 2,
117
+ "error": None | "error message"
118
+ }
119
+
120
+ Example:
121
+ >>> result = manager.run_metric(
122
+ ... config,
123
+ ... steps=[PipelineStep.LOAD, PipelineStep.DETECT],
124
+ ... from_date=datetime(2024, 1, 1),
125
+ ... )
126
+ >>> print(result["status"])
127
+ success
128
+ """
129
+ steps = steps or [PipelineStep.LOAD, PipelineStep.DETECT, PipelineStep.ALERT]
130
+ metric_name = config.name
131
+
132
+ result = {
133
+ "status": TaskStatus.SUCCESS,
134
+ "steps_completed": [],
135
+ "datapoints_loaded": 0,
136
+ "anomalies_detected": 0,
137
+ "alerts_sent": 0,
138
+ "error": None,
139
+ }
140
+
141
+ try:
142
+ # Step 0: Acquire lock
143
+ if not force:
144
+ # Default timeout: 1 hour (can be overridden via ProjectConfig in future)
145
+ timeout_seconds = 3600
146
+ lock_acquired = self.internal.acquire_lock(
147
+ metric_name=metric_name,
148
+ detector_id="pipeline", # General pipeline lock
149
+ process_type="pipeline", # Full pipeline execution
150
+ timeout_seconds=timeout_seconds,
151
+ )
152
+ if not lock_acquired:
153
+ raise RuntimeError(
154
+ f"Failed to acquire lock for metric '{metric_name}'. "
155
+ f"Another task is running. Use --force to override."
156
+ )
157
+
158
+ try:
159
+ # Step 1: Load data
160
+ if PipelineStep.LOAD in steps:
161
+ load_result = self._run_load_step(
162
+ config, from_date, to_date, full_refresh
163
+ )
164
+ result["datapoints_loaded"] = load_result["points_loaded"]
165
+ result["steps_completed"].append(PipelineStep.LOAD)
166
+
167
+ # Step 2: Detect anomalies
168
+ if PipelineStep.DETECT in steps:
169
+ click.echo()
170
+ click.echo(click.style(" ┌─ DETECT", fg="cyan", bold=True))
171
+ detect_result = self._run_detect_step(config, from_date, to_date, full_refresh)
172
+ result["anomalies_detected"] = detect_result["anomalies_count"]
173
+ result["steps_completed"].append(PipelineStep.DETECT)
174
+
175
+ # Step 3: Send alerts
176
+ if PipelineStep.ALERT in steps:
177
+ click.echo()
178
+ click.echo(click.style(" ┌─ ALERT", fg="cyan", bold=True))
179
+ alert_result = self._run_alert_step(config)
180
+ result["alerts_sent"] = alert_result["alerts_sent"]
181
+ result["steps_completed"].append(PipelineStep.ALERT)
182
+
183
+ finally:
184
+ # Always release lock
185
+ if not force:
186
+ status = "completed" if result["status"] == TaskStatus.SUCCESS else "failed"
187
+ error_msg = result.get("error")
188
+ self.internal.release_lock(
189
+ metric_name=metric_name,
190
+ detector_id="pipeline",
191
+ process_type="pipeline",
192
+ status=status,
193
+ error_message=error_msg,
194
+ )
195
+
196
+ except Exception as e:
197
+ result["status"] = TaskStatus.FAILED
198
+ result["error"] = str(e)
199
+
200
+ return result
201
+
202
+ def _run_load_step(
203
+ self,
204
+ config: MetricConfig,
205
+ from_date: Optional[datetime],
206
+ to_date: Optional[datetime],
207
+ full_refresh: bool,
208
+ ) -> Dict[str, int]:
209
+ """
210
+ Execute data loading step with batching.
211
+
212
+ Args:
213
+ config: Metric configuration
214
+ from_date: Start date (optional)
215
+ to_date: End date (optional)
216
+ full_refresh: Delete existing data
217
+
218
+ Returns:
219
+ Dict with {"points_loaded": N}
220
+ """
221
+ loader = MetricLoader(
222
+ config=config,
223
+ db_manager=self.db_manager,
224
+ internal_manager=self.internal,
225
+ )
226
+
227
+ # Determine date range
228
+ if full_refresh:
229
+ click.echo(" │ Deleting existing datapoints...")
230
+ # Delete existing data for this metric
231
+ self.internal.delete_datapoints(
232
+ metric_name=config.name,
233
+ from_timestamp=from_date,
234
+ to_timestamp=to_date,
235
+ )
236
+
237
+ # Determine actual from_date and to_date
238
+ actual_from = from_date
239
+ actual_to = to_date
240
+
241
+ if actual_from is None:
242
+ # Get last saved timestamp
243
+ last_ts = self.internal.get_last_datapoint_timestamp(config.name)
244
+ if last_ts:
245
+ # Start from next interval after last timestamp
246
+ interval = config.get_interval()
247
+ actual_from = last_ts + timedelta(seconds=interval.seconds)
248
+ click.echo(f" │ Resuming from last saved: {last_ts.strftime('%Y-%m-%d %H:%M:%S')}")
249
+ else:
250
+ # No data yet - use loading_start_time from config
251
+ if config.loading_start_time:
252
+ actual_from = datetime.strptime(
253
+ config.loading_start_time, "%Y-%m-%d %H:%M:%S"
254
+ ).replace(tzinfo=timezone.utc)
255
+ click.echo(f" │ Starting fresh from: {config.loading_start_time}")
256
+ else:
257
+ raise ValueError(
258
+ "No existing data and no loading_start_time configured. "
259
+ "Please specify from_date or set loading_start_time in config."
260
+ )
261
+
262
+ if actual_to is None:
263
+ actual_to = datetime.now(timezone.utc)
264
+
265
+ # Calculate total points and batch size
266
+ interval = config.get_interval()
267
+ total_seconds = (actual_to - actual_from).total_seconds()
268
+ total_points = int(total_seconds / interval.seconds)
269
+ batch_size = config.loading_batch_size
270
+
271
+ click.echo(f" │ Loading from {actual_from.strftime('%Y-%m-%d %H:%M:%S')} to {actual_to.strftime('%Y-%m-%d %H:%M:%S')}")
272
+ click.echo(f" │ Total points: ~{total_points:,} | Batch size: {batch_size:,}")
273
+
274
+ # If total points <= batch_size, load in one go
275
+ if total_points <= batch_size:
276
+ click.echo(" │ Loading in single batch...")
277
+ rows_inserted = loader.load_and_save(from_date=actual_from, to_date=actual_to)
278
+ click.echo(click.style(f" └─ Loaded {rows_inserted:,} datapoints", fg="green"))
279
+ return {"points_loaded": rows_inserted}
280
+
281
+ # Load in batches
282
+ total_loaded = 0
283
+ current_from = actual_from
284
+ num_batches = int(total_points / batch_size) + 1
285
+ batch_num = 0
286
+
287
+ click.echo(f" │ Loading in {num_batches} batches...")
288
+
289
+ while current_from < actual_to:
290
+ batch_num += 1
291
+ # Calculate batch end time
292
+ batch_seconds = batch_size * interval.seconds
293
+ batch_to = current_from + timedelta(seconds=batch_seconds)
294
+ if batch_to > actual_to:
295
+ batch_to = actual_to
296
+
297
+ # Load batch
298
+ rows = loader.load_and_save(from_date=current_from, to_date=batch_to)
299
+ total_loaded += rows
300
+
301
+ click.echo(f" │ Batch {batch_num}/{num_batches}: +{rows:,} points (total: {total_loaded:,})")
302
+
303
+ # Move to next batch
304
+ current_from = batch_to
305
+
306
+ click.echo(click.style(f" └─ Loaded {total_loaded:,} datapoints", fg="green"))
307
+ return {"points_loaded": total_loaded}
308
+
309
+ def _run_detect_step(
310
+ self,
311
+ config: MetricConfig,
312
+ from_date: Optional[datetime],
313
+ to_date: Optional[datetime],
314
+ full_refresh: bool = False,
315
+ ) -> Dict[str, int]:
316
+ """
317
+ Execute anomaly detection step with batching and idempotency.
318
+
319
+ Args:
320
+ config: Metric configuration
321
+ from_date: Start date for detection (optional)
322
+ to_date: End date for detection (optional)
323
+ full_refresh: Delete existing detections before running
324
+
325
+ Returns:
326
+ Dict with {"anomalies_count": N}
327
+ """
328
+ anomalies_count = 0
329
+
330
+ # Skip if no detectors configured
331
+ if not config.detectors:
332
+ click.echo(" │ No detectors configured, skipping detection")
333
+ return {"anomalies_count": 0}
334
+
335
+ # Get interval
336
+ interval = config.get_interval()
337
+ click.echo(f" │ Running {len(config.detectors)} detector(s)...")
338
+
339
+ # Determine to_date if not specified
340
+ actual_to = to_date or datetime.now(timezone.utc)
341
+ # Normalize to naive datetime (remove timezone info)
342
+ if actual_to and actual_to.tzinfo is not None:
343
+ actual_to = actual_to.replace(tzinfo=None)
344
+
345
+ # Normalize from_date to naive
346
+ normalized_from_date = from_date
347
+ if normalized_from_date and normalized_from_date.tzinfo is not None:
348
+ normalized_from_date = normalized_from_date.replace(tzinfo=None)
349
+
350
+ # Run each detector
351
+ for idx, detector_config in enumerate(config.detectors, 1):
352
+ click.echo(f" │")
353
+ click.echo(f" │ [{idx}/{len(config.detectors)}] Detector: {detector_config.type}")
354
+
355
+ # Create detector to get detector_id
356
+ # Combine algorithm params with execution params (seasonality_components)
357
+ detector_params = detector_config.get_algorithm_params()
358
+
359
+ # Add seasonality_components if configured
360
+ seasonality_components = detector_config.get_seasonality_components()
361
+ if seasonality_components is not None:
362
+ detector_params["seasonality_components"] = seasonality_components
363
+
364
+ detector_dict = {
365
+ "type": detector_config.type,
366
+ "params": detector_params
367
+ }
368
+ detector = DetectorFactory.create_from_config(detector_dict)
369
+ detector_id = detector.get_detector_id()
370
+
371
+ # Delete existing detections if full_refresh
372
+ if full_refresh:
373
+ click.echo(" │ Deleting existing detections...")
374
+ self.internal.delete_detections(
375
+ metric_name=config.name,
376
+ detector_id=detector_id,
377
+ from_timestamp=normalized_from_date,
378
+ to_timestamp=actual_to,
379
+ )
380
+
381
+ # IDEMPOTENCY: Get last detected timestamp
382
+ last_detection_ts = self.internal.get_last_detection_timestamp(
383
+ metric_name=config.name,
384
+ detector_id=detector_id
385
+ )
386
+ # Normalize last_detection_ts to naive if needed
387
+ if last_detection_ts and last_detection_ts.tzinfo is not None:
388
+ last_detection_ts = last_detection_ts.replace(tzinfo=None)
389
+
390
+ # Determine actual from_date
391
+ actual_from = normalized_from_date
392
+ if not full_refresh and last_detection_ts:
393
+ # Resume from last detected point + 1 interval
394
+ resume_from = last_detection_ts + timedelta(seconds=interval.seconds)
395
+ if actual_from:
396
+ actual_from = max(actual_from, resume_from)
397
+ else:
398
+ actual_from = resume_from
399
+
400
+ # Apply start_time filter if configured
401
+ start_time_str = detector_config.get_start_time()
402
+ if start_time_str:
403
+ start_time = datetime.fromisoformat(start_time_str.replace('Z', '+00:00'))
404
+ # Always normalize to naive datetime
405
+ start_time = start_time.replace(tzinfo=None)
406
+ if actual_from:
407
+ actual_from = max(actual_from, start_time)
408
+ else:
409
+ actual_from = start_time
410
+
411
+ # Ensure actual_from is naive (for comparison with actual_to)
412
+ if actual_from and actual_from.tzinfo is not None:
413
+ actual_from = actual_from.replace(tzinfo=None)
414
+
415
+ # Skip if nothing to detect
416
+ if not actual_from or actual_from >= actual_to:
417
+ click.echo(" │ Nothing to detect (already up to date)")
418
+ continue
419
+
420
+ # Get batch_size and context_size
421
+ batch_size = detector_config.get_batch_size() or 1000 # Default batch size
422
+ context_size = detector.get_context_size() # Historical points needed
423
+
424
+ # Calculate total points to detect
425
+ total_seconds = (actual_to - actual_from).total_seconds()
426
+ total_points = int(total_seconds / interval.seconds)
427
+
428
+ click.echo(f" │ Detecting from {actual_from.strftime('%Y-%m-%d %H:%M:%S')} to {actual_to.strftime('%Y-%m-%d %H:%M:%S')}")
429
+ click.echo(f" │ Total points: ~{total_points:,} | Batch size: {batch_size:,}")
430
+
431
+ # BATCHING: Process in batches
432
+ current_from = actual_from
433
+ detector_anomalies = 0
434
+ num_batches = int(total_points / batch_size) + 1 if total_points > batch_size else 1
435
+ batch_num = 0
436
+
437
+ while current_from < actual_to:
438
+ batch_num += 1
439
+ # Calculate batch end
440
+ batch_seconds = batch_size * interval.seconds
441
+ batch_to = current_from + timedelta(seconds=batch_seconds)
442
+ if batch_to > actual_to:
443
+ batch_to = actual_to
444
+
445
+ # Calculate context start (need historical data for context)
446
+ # context_size includes window_size + any extra needed for preprocessing
447
+ context_seconds = context_size * interval.seconds
448
+ context_from = current_from - timedelta(seconds=context_seconds)
449
+
450
+ # Load data with context (historical window)
451
+ data = self.internal.load_datapoints(
452
+ metric_name=config.name,
453
+ from_timestamp=context_from,
454
+ to_timestamp=batch_to,
455
+ )
456
+
457
+ if not data or len(data["timestamp"]) == 0:
458
+ # No data, move to next batch
459
+ current_from = batch_to
460
+ continue
461
+
462
+ # Run detection on data
463
+ # Detector will handle window sliding internally
464
+ results = detector.detect(data)
465
+
466
+ # Filter results to only current batch (not historical window)
467
+ current_from_np = np.datetime64(current_from, 'ms')
468
+ batch_to_np = np.datetime64(batch_to, 'ms')
469
+ batch_results = [
470
+ r for r in results
471
+ if current_from_np <= np.datetime64(r.timestamp, 'ms') < batch_to_np
472
+ ]
473
+
474
+ # Save results to _dtk_detections
475
+ if batch_results and len(batch_results) > 0:
476
+ # Convert List[DetectionResult] to dict with numpy arrays
477
+ detection_data = {
478
+ "timestamp": np.array([r.timestamp for r in batch_results], dtype="datetime64[ms]"),
479
+ "is_anomaly": np.array([r.is_anomaly for r in batch_results], dtype=bool),
480
+ "confidence_lower": np.array([r.confidence_lower for r in batch_results], dtype=np.float64),
481
+ "confidence_upper": np.array([r.confidence_upper for r in batch_results], dtype=np.float64),
482
+ "value": np.array([r.value for r in batch_results], dtype=np.float64),
483
+ "processed_value": np.array([r.processed_value for r in batch_results], dtype=np.float64),
484
+ "detection_metadata": np.array([
485
+ json.dumps(r.detection_metadata) if r.detection_metadata else "{}"
486
+ for r in batch_results
487
+ ], dtype=object),
488
+ }
489
+
490
+ self.internal.save_detections(
491
+ metric_name=config.name,
492
+ detector_id=detector_id,
493
+ detector_name=detector.__class__.__name__,
494
+ data=detection_data,
495
+ detector_params=detector.get_detector_params(),
496
+ )
497
+
498
+ # Count anomalies
499
+ batch_anomalies = sum(1 for r in batch_results if r.is_anomaly)
500
+ detector_anomalies += batch_anomalies
501
+ anomalies_count += batch_anomalies
502
+
503
+ if num_batches > 1:
504
+ click.echo(f" │ Batch {batch_num}/{num_batches}: {len(batch_results):,} points, {batch_anomalies} anomalies")
505
+
506
+ # Move to next batch
507
+ current_from = batch_to
508
+
509
+ click.echo(click.style(f" │ └─ Detected {detector_anomalies:,} anomalies", fg="yellow" if detector_anomalies > 0 else "green"))
510
+
511
+ click.echo(click.style(f" └─ Total anomalies: {anomalies_count:,}", fg="yellow" if anomalies_count > 0 else "green"))
512
+ return {"anomalies_count": anomalies_count}
513
+
514
+ def _run_alert_step(self, config: MetricConfig) -> Dict[str, int]:
515
+ """
516
+ Execute alerting step.
517
+
518
+ Args:
519
+ config: Metric configuration
520
+
521
+ Returns:
522
+ Dict with {"alerts_sent": N}
523
+ """
524
+ alerts_sent = 0
525
+
526
+ # Check if alerting is configured
527
+ if not config.alerting or not config.alerting.enabled:
528
+ click.echo(" │ Alerting not enabled")
529
+ return {"alerts_sent": 0}
530
+
531
+ if not config.alerting.channels:
532
+ click.echo(" │ No alert channels configured")
533
+ return {"alerts_sent": 0}
534
+
535
+ click.echo(f" │ Checking alert conditions...")
536
+
537
+ # Get alerting config
538
+ alerting_config = config.alerting
539
+
540
+ # Create alert orchestrator
541
+ interval = config.get_interval()
542
+ orchestrator = AlertOrchestrator(
543
+ metric_name=config.name,
544
+ interval=interval,
545
+ conditions=AlertConditions(
546
+ min_detectors=1, # At least one detector must flag anomaly
547
+ direction="any", # Any direction (up or down)
548
+ consecutive_anomalies=alerting_config.consecutive_anomalies,
549
+ ),
550
+ timezone_display="UTC",
551
+ )
552
+
553
+ # Get last complete point
554
+ last_point = orchestrator.get_last_complete_point()
555
+
556
+ # Load recent detections for consecutive anomaly checking
557
+ # We need N recent points where N = consecutive_anomalies
558
+ recent_detections = self._load_recent_detections(
559
+ metric_name=config.name,
560
+ last_point=last_point,
561
+ num_points=alerting_config.consecutive_anomalies,
562
+ )
563
+
564
+ if not recent_detections:
565
+ click.echo(" │ No recent detections found")
566
+ return {"alerts_sent": 0}
567
+
568
+ # Check if alert should be sent
569
+ should_alert, alert_data = orchestrator.should_alert(recent_detections)
570
+
571
+ if should_alert:
572
+ click.echo(click.style(f" │ ⚠ Alert triggered! Sending to {len(alerting_config.channels)} channel(s)...", fg="yellow", bold=True))
573
+
574
+ # Create alert channels from config
575
+ channels = self._create_alert_channels(alerting_config.channels)
576
+
577
+ if channels:
578
+ # Send alerts
579
+ results = orchestrator.send_alerts(alert_data, channels)
580
+ alerts_sent = sum(1 for success in results.values() if success)
581
+
582
+ for channel_name, success in results.items():
583
+ status = click.style("✓", fg="green") if success else click.style("✗", fg="red")
584
+ click.echo(f" │ {status} {channel_name}")
585
+
586
+ click.echo(click.style(f" └─ Sent {alerts_sent}/{len(channels)} alerts", fg="green" if alerts_sent > 0 else "yellow"))
587
+ else:
588
+ click.echo(click.style(" └─ No valid alert channels available", fg="yellow"))
589
+ else:
590
+ click.echo(" └─ No alert needed (conditions not met)")
591
+
592
+ return {"alerts_sent": alerts_sent}
593
+
594
+ def _load_recent_detections(
595
+ self,
596
+ metric_name: str,
597
+ last_point: datetime,
598
+ num_points: int,
599
+ ) -> List[DetectionRecord]:
600
+ """
601
+ Load recent detection records for consecutive anomaly checking.
602
+
603
+ Args:
604
+ metric_name: Metric name
605
+ last_point: Last complete timestamp
606
+ num_points: Number of recent points to load
607
+
608
+ Returns:
609
+ List of DetectionRecord objects
610
+ """
611
+ # Use internal tables manager method (database-agnostic)
612
+ results = self.internal.get_recent_detections(
613
+ metric_name=metric_name,
614
+ last_point=last_point,
615
+ num_points=num_points,
616
+ )
617
+
618
+ if not results:
619
+ return []
620
+
621
+ # Convert to DetectionRecord objects
622
+ records = []
623
+ for row in results:
624
+ # Check if any detector flagged this point as anomaly
625
+ is_anomaly = any(row["is_anomaly_flags"])
626
+
627
+ # Get detector data for anomalous detections
628
+ anomaly_indices = [
629
+ i
630
+ for i, flag in enumerate(row["is_anomaly_flags"])
631
+ if flag
632
+ ]
633
+
634
+ # Determine direction and severity for the most severe detector
635
+ direction = "none"
636
+ severity = 0.0
637
+ confidence_lower = None
638
+ confidence_upper = None
639
+ detector_name = "unknown"
640
+ detector_id = "unknown"
641
+ detector_params = "{}"
642
+
643
+ if is_anomaly and anomaly_indices:
644
+ # Get data from first anomalous detector
645
+ first_idx = anomaly_indices[0]
646
+ detector_name = row["detector_names"][first_idx]
647
+ detector_id = row["detector_ids"][first_idx]
648
+ detector_params = row["detector_params_list"][first_idx]
649
+ confidence_lower = row["confidence_lowers"][first_idx]
650
+ confidence_upper = row["confidence_uppers"][first_idx]
651
+
652
+ # Determine direction
653
+ value = row["value"]
654
+ if value < confidence_lower:
655
+ direction = "down"
656
+ severity = (confidence_lower - value) / max(abs(confidence_lower), 1e-10)
657
+ elif value > confidence_upper:
658
+ direction = "up"
659
+ severity = (value - confidence_upper) / max(abs(confidence_upper), 1e-10)
660
+
661
+ records.append(
662
+ DetectionRecord(
663
+ timestamp=row["timestamp"],
664
+ detector_name=detector_name,
665
+ detector_id=detector_id,
666
+ detector_params=detector_params,
667
+ value=row["value"],
668
+ is_anomaly=is_anomaly,
669
+ confidence_lower=confidence_lower,
670
+ confidence_upper=confidence_upper,
671
+ direction=direction,
672
+ severity=severity,
673
+ detection_metadata={},
674
+ )
675
+ )
676
+
677
+ # Reverse to get chronological order
678
+ return list(reversed(records))
679
+
680
+ def _create_alert_channels(
681
+ self, channel_names: List[str]
682
+ ) -> List[BaseAlertChannel]:
683
+ """
684
+ Create alert channel instances from channel names.
685
+
686
+ Args:
687
+ channel_names: List of channel names to create
688
+
689
+ Returns:
690
+ List of alert channel instances
691
+ """
692
+ if not self.profiles_config:
693
+ # No profiles config available, return empty list
694
+ return []
695
+
696
+ channels = []
697
+ for channel_name in channel_names:
698
+ try:
699
+ # Get channel config from profiles
700
+ channel_config = self.profiles_config.get_alert_channel_config(
701
+ channel_name
702
+ )
703
+
704
+ # Create channel instance using factory
705
+ channel = AlertChannelFactory.create_from_config(channel_config)
706
+ channels.append(channel)
707
+
708
+ except Exception as e:
709
+ # Log error but continue with other channels
710
+ print(f"Warning: Failed to create channel '{channel_name}': {e}")
711
+ continue
712
+
713
+ return channels
714
+
715
+ def get_metric_status(self, metric_name: str) -> Optional[Dict]:
716
+ """
717
+ Get current status of a metric.
718
+
719
+ Args:
720
+ metric_name: Name of the metric
721
+
722
+ Returns:
723
+ Dict with status information or None if not found
724
+
725
+ Example:
726
+ >>> status = manager.get_metric_status("cpu_usage")
727
+ >>> print(status["last_run"])
728
+ 2024-01-01 12:00:00
729
+ """
730
+ # Check if locked
731
+ lock_info = self.internal.check_lock(metric_name)
732
+
733
+ # Get last datapoint timestamp
734
+ last_timestamp = self.internal.get_last_datapoint_timestamp(metric_name)
735
+
736
+ return {
737
+ "metric_name": metric_name,
738
+ "is_locked": lock_info is not None,
739
+ "locked_by": lock_info.get("locked_by") if lock_info else None,
740
+ "locked_at": lock_info.get("locked_at") if lock_info else None,
741
+ "last_datapoint": last_timestamp,
742
+ }
743
+
744
+ def __repr__(self) -> str:
745
+ """String representation."""
746
+ return f"TaskManager(db={self.db_manager.__class__.__name__})"