truthound-dashboard 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- truthound_dashboard/api/alerts.py +258 -0
- truthound_dashboard/api/anomaly.py +1302 -0
- truthound_dashboard/api/cross_alerts.py +352 -0
- truthound_dashboard/api/deps.py +143 -0
- truthound_dashboard/api/drift_monitor.py +540 -0
- truthound_dashboard/api/lineage.py +1151 -0
- truthound_dashboard/api/maintenance.py +363 -0
- truthound_dashboard/api/middleware.py +373 -1
- truthound_dashboard/api/model_monitoring.py +805 -0
- truthound_dashboard/api/notifications_advanced.py +2452 -0
- truthound_dashboard/api/plugins.py +2096 -0
- truthound_dashboard/api/profile.py +211 -14
- truthound_dashboard/api/reports.py +853 -0
- truthound_dashboard/api/router.py +147 -0
- truthound_dashboard/api/rule_suggestions.py +310 -0
- truthound_dashboard/api/schema_evolution.py +231 -0
- truthound_dashboard/api/sources.py +47 -3
- truthound_dashboard/api/triggers.py +190 -0
- truthound_dashboard/api/validations.py +13 -0
- truthound_dashboard/api/validators.py +333 -4
- truthound_dashboard/api/versioning.py +309 -0
- truthound_dashboard/api/websocket.py +301 -0
- truthound_dashboard/core/__init__.py +27 -0
- truthound_dashboard/core/anomaly.py +1395 -0
- truthound_dashboard/core/anomaly_explainer.py +633 -0
- truthound_dashboard/core/cache.py +206 -0
- truthound_dashboard/core/cached_services.py +422 -0
- truthound_dashboard/core/charts.py +352 -0
- truthound_dashboard/core/connections.py +1069 -42
- truthound_dashboard/core/cross_alerts.py +837 -0
- truthound_dashboard/core/drift_monitor.py +1477 -0
- truthound_dashboard/core/drift_sampling.py +669 -0
- truthound_dashboard/core/i18n/__init__.py +42 -0
- truthound_dashboard/core/i18n/detector.py +173 -0
- truthound_dashboard/core/i18n/messages.py +564 -0
- truthound_dashboard/core/lineage.py +971 -0
- truthound_dashboard/core/maintenance.py +443 -5
- truthound_dashboard/core/model_monitoring.py +1043 -0
- truthound_dashboard/core/notifications/channels.py +1020 -1
- truthound_dashboard/core/notifications/deduplication/__init__.py +143 -0
- truthound_dashboard/core/notifications/deduplication/policies.py +274 -0
- truthound_dashboard/core/notifications/deduplication/service.py +400 -0
- truthound_dashboard/core/notifications/deduplication/stores.py +2365 -0
- truthound_dashboard/core/notifications/deduplication/strategies.py +422 -0
- truthound_dashboard/core/notifications/dispatcher.py +43 -0
- truthound_dashboard/core/notifications/escalation/__init__.py +149 -0
- truthound_dashboard/core/notifications/escalation/backends.py +1384 -0
- truthound_dashboard/core/notifications/escalation/engine.py +429 -0
- truthound_dashboard/core/notifications/escalation/models.py +336 -0
- truthound_dashboard/core/notifications/escalation/scheduler.py +1187 -0
- truthound_dashboard/core/notifications/escalation/state_machine.py +330 -0
- truthound_dashboard/core/notifications/escalation/stores.py +2896 -0
- truthound_dashboard/core/notifications/events.py +49 -0
- truthound_dashboard/core/notifications/metrics/__init__.py +115 -0
- truthound_dashboard/core/notifications/metrics/base.py +528 -0
- truthound_dashboard/core/notifications/metrics/collectors.py +583 -0
- truthound_dashboard/core/notifications/routing/__init__.py +169 -0
- truthound_dashboard/core/notifications/routing/combinators.py +184 -0
- truthound_dashboard/core/notifications/routing/config.py +375 -0
- truthound_dashboard/core/notifications/routing/config_parser.py +867 -0
- truthound_dashboard/core/notifications/routing/engine.py +382 -0
- truthound_dashboard/core/notifications/routing/expression_engine.py +1269 -0
- truthound_dashboard/core/notifications/routing/jinja2_engine.py +774 -0
- truthound_dashboard/core/notifications/routing/rules.py +625 -0
- truthound_dashboard/core/notifications/routing/validator.py +678 -0
- truthound_dashboard/core/notifications/service.py +2 -0
- truthound_dashboard/core/notifications/stats_aggregator.py +850 -0
- truthound_dashboard/core/notifications/throttling/__init__.py +83 -0
- truthound_dashboard/core/notifications/throttling/builder.py +311 -0
- truthound_dashboard/core/notifications/throttling/stores.py +1859 -0
- truthound_dashboard/core/notifications/throttling/throttlers.py +633 -0
- truthound_dashboard/core/openlineage.py +1028 -0
- truthound_dashboard/core/plugins/__init__.py +39 -0
- truthound_dashboard/core/plugins/docs/__init__.py +39 -0
- truthound_dashboard/core/plugins/docs/extractor.py +703 -0
- truthound_dashboard/core/plugins/docs/renderers.py +804 -0
- truthound_dashboard/core/plugins/hooks/__init__.py +63 -0
- truthound_dashboard/core/plugins/hooks/decorators.py +367 -0
- truthound_dashboard/core/plugins/hooks/manager.py +403 -0
- truthound_dashboard/core/plugins/hooks/protocols.py +265 -0
- truthound_dashboard/core/plugins/lifecycle/__init__.py +41 -0
- truthound_dashboard/core/plugins/lifecycle/hot_reload.py +584 -0
- truthound_dashboard/core/plugins/lifecycle/machine.py +419 -0
- truthound_dashboard/core/plugins/lifecycle/states.py +266 -0
- truthound_dashboard/core/plugins/loader.py +504 -0
- truthound_dashboard/core/plugins/registry.py +810 -0
- truthound_dashboard/core/plugins/reporter_executor.py +588 -0
- truthound_dashboard/core/plugins/sandbox/__init__.py +59 -0
- truthound_dashboard/core/plugins/sandbox/code_validator.py +243 -0
- truthound_dashboard/core/plugins/sandbox/engines.py +770 -0
- truthound_dashboard/core/plugins/sandbox/protocols.py +194 -0
- truthound_dashboard/core/plugins/sandbox.py +617 -0
- truthound_dashboard/core/plugins/security/__init__.py +68 -0
- truthound_dashboard/core/plugins/security/analyzer.py +535 -0
- truthound_dashboard/core/plugins/security/policies.py +311 -0
- truthound_dashboard/core/plugins/security/protocols.py +296 -0
- truthound_dashboard/core/plugins/security/signing.py +842 -0
- truthound_dashboard/core/plugins/security.py +446 -0
- truthound_dashboard/core/plugins/validator_executor.py +401 -0
- truthound_dashboard/core/plugins/versioning/__init__.py +51 -0
- truthound_dashboard/core/plugins/versioning/constraints.py +377 -0
- truthound_dashboard/core/plugins/versioning/dependencies.py +541 -0
- truthound_dashboard/core/plugins/versioning/semver.py +266 -0
- truthound_dashboard/core/profile_comparison.py +601 -0
- truthound_dashboard/core/report_history.py +570 -0
- truthound_dashboard/core/reporters/__init__.py +57 -0
- truthound_dashboard/core/reporters/base.py +296 -0
- truthound_dashboard/core/reporters/csv_reporter.py +155 -0
- truthound_dashboard/core/reporters/html_reporter.py +598 -0
- truthound_dashboard/core/reporters/i18n/__init__.py +65 -0
- truthound_dashboard/core/reporters/i18n/base.py +494 -0
- truthound_dashboard/core/reporters/i18n/catalogs.py +930 -0
- truthound_dashboard/core/reporters/json_reporter.py +160 -0
- truthound_dashboard/core/reporters/junit_reporter.py +233 -0
- truthound_dashboard/core/reporters/markdown_reporter.py +207 -0
- truthound_dashboard/core/reporters/pdf_reporter.py +209 -0
- truthound_dashboard/core/reporters/registry.py +272 -0
- truthound_dashboard/core/rule_generator.py +2088 -0
- truthound_dashboard/core/scheduler.py +822 -12
- truthound_dashboard/core/schema_evolution.py +858 -0
- truthound_dashboard/core/services.py +152 -9
- truthound_dashboard/core/statistics.py +718 -0
- truthound_dashboard/core/streaming_anomaly.py +883 -0
- truthound_dashboard/core/triggers/__init__.py +45 -0
- truthound_dashboard/core/triggers/base.py +226 -0
- truthound_dashboard/core/triggers/evaluators.py +609 -0
- truthound_dashboard/core/triggers/factory.py +363 -0
- truthound_dashboard/core/unified_alerts.py +870 -0
- truthound_dashboard/core/validation_limits.py +509 -0
- truthound_dashboard/core/versioning.py +709 -0
- truthound_dashboard/core/websocket/__init__.py +59 -0
- truthound_dashboard/core/websocket/manager.py +512 -0
- truthound_dashboard/core/websocket/messages.py +130 -0
- truthound_dashboard/db/__init__.py +30 -0
- truthound_dashboard/db/models.py +3375 -3
- truthound_dashboard/main.py +22 -0
- truthound_dashboard/schemas/__init__.py +396 -1
- truthound_dashboard/schemas/anomaly.py +1258 -0
- truthound_dashboard/schemas/base.py +4 -0
- truthound_dashboard/schemas/cross_alerts.py +334 -0
- truthound_dashboard/schemas/drift_monitor.py +890 -0
- truthound_dashboard/schemas/lineage.py +428 -0
- truthound_dashboard/schemas/maintenance.py +154 -0
- truthound_dashboard/schemas/model_monitoring.py +374 -0
- truthound_dashboard/schemas/notifications_advanced.py +1363 -0
- truthound_dashboard/schemas/openlineage.py +704 -0
- truthound_dashboard/schemas/plugins.py +1293 -0
- truthound_dashboard/schemas/profile.py +420 -34
- truthound_dashboard/schemas/profile_comparison.py +242 -0
- truthound_dashboard/schemas/reports.py +285 -0
- truthound_dashboard/schemas/rule_suggestion.py +434 -0
- truthound_dashboard/schemas/schema_evolution.py +164 -0
- truthound_dashboard/schemas/source.py +117 -2
- truthound_dashboard/schemas/triggers.py +511 -0
- truthound_dashboard/schemas/unified_alerts.py +223 -0
- truthound_dashboard/schemas/validation.py +25 -1
- truthound_dashboard/schemas/validators/__init__.py +11 -0
- truthound_dashboard/schemas/validators/base.py +151 -0
- truthound_dashboard/schemas/versioning.py +152 -0
- truthound_dashboard/static/index.html +2 -2
- {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/METADATA +142 -18
- truthound_dashboard-1.4.0.dist-info/RECORD +239 -0
- truthound_dashboard/static/assets/index-BCA8H1hO.js +0 -574
- truthound_dashboard/static/assets/index-BNsSQ2fN.css +0 -1
- truthound_dashboard/static/assets/unmerged_dictionaries-CsJWCRx9.js +0 -1
- truthound_dashboard-1.3.0.dist-info/RECORD +0 -110
- {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/WHEEL +0 -0
- {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/entry_points.txt +0 -0
- {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/licenses/LICENSE +0 -0
truthound_dashboard/db/models.py
CHANGED
|
@@ -109,6 +109,43 @@ class ActivityAction(str, Enum):
|
|
|
109
109
|
UNMAPPED = "unmapped"
|
|
110
110
|
|
|
111
111
|
|
|
112
|
+
# =============================================================================
|
|
113
|
+
# Schema Evolution Enums
|
|
114
|
+
# =============================================================================
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class SchemaChangeType(str, Enum):
|
|
118
|
+
"""Type of schema change detected."""
|
|
119
|
+
|
|
120
|
+
COLUMN_ADDED = "column_added"
|
|
121
|
+
COLUMN_REMOVED = "column_removed"
|
|
122
|
+
TYPE_CHANGED = "type_changed"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class SchemaChangeSeverity(str, Enum):
|
|
126
|
+
"""Severity level of schema change."""
|
|
127
|
+
|
|
128
|
+
BREAKING = "breaking"
|
|
129
|
+
NON_BREAKING = "non_breaking"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# =============================================================================
|
|
133
|
+
# Schedule Trigger Enums
|
|
134
|
+
# =============================================================================
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class TriggerType(str, Enum):
|
|
138
|
+
"""Type of schedule trigger."""
|
|
139
|
+
|
|
140
|
+
CRON = "cron"
|
|
141
|
+
INTERVAL = "interval"
|
|
142
|
+
DATA_CHANGE = "data_change"
|
|
143
|
+
COMPOSITE = "composite"
|
|
144
|
+
EVENT = "event"
|
|
145
|
+
MANUAL = "manual"
|
|
146
|
+
WEBHOOK = "webhook" # External webhook triggers
|
|
147
|
+
|
|
148
|
+
|
|
112
149
|
class Source(Base, UUIDMixin, TimestampMixin):
|
|
113
150
|
"""Data source model.
|
|
114
151
|
|
|
@@ -178,6 +215,13 @@ class Source(Base, UUIDMixin, TimestampMixin):
|
|
|
178
215
|
back_populates="source",
|
|
179
216
|
lazy="selectin",
|
|
180
217
|
)
|
|
218
|
+
# Generated reports for this source
|
|
219
|
+
generated_reports: Mapped[list["GeneratedReport"]] = relationship(
|
|
220
|
+
"GeneratedReport",
|
|
221
|
+
back_populates="source",
|
|
222
|
+
lazy="selectin",
|
|
223
|
+
order_by="desc(GeneratedReport.created_at)",
|
|
224
|
+
)
|
|
181
225
|
|
|
182
226
|
@property
|
|
183
227
|
def source_path(self) -> str | None:
|
|
@@ -399,6 +443,12 @@ class Validation(Base, UUIDMixin):
|
|
|
399
443
|
|
|
400
444
|
# Relationships
|
|
401
445
|
source: Mapped[Source] = relationship("Source", back_populates="validations")
|
|
446
|
+
generated_reports: Mapped[list["GeneratedReport"]] = relationship(
|
|
447
|
+
"GeneratedReport",
|
|
448
|
+
back_populates="validation",
|
|
449
|
+
lazy="selectin",
|
|
450
|
+
order_by="desc(GeneratedReport.created_at)",
|
|
451
|
+
)
|
|
402
452
|
|
|
403
453
|
@property
|
|
404
454
|
def issues(self) -> list[dict[str, Any]]:
|
|
@@ -484,17 +534,22 @@ class Profile(Base, UUIDMixin, TimestampMixin):
|
|
|
484
534
|
class Schedule(Base, UUIDMixin, TimestampMixin):
|
|
485
535
|
"""Validation schedule model.
|
|
486
536
|
|
|
487
|
-
Manages scheduled validation runs
|
|
537
|
+
Manages scheduled validation runs with flexible trigger types.
|
|
538
|
+
Supports cron, interval, data change detection, and composite triggers.
|
|
488
539
|
|
|
489
540
|
Attributes:
|
|
490
541
|
id: Unique identifier (UUID).
|
|
491
542
|
name: Human-readable schedule name.
|
|
492
543
|
source_id: Reference to Source to validate.
|
|
493
|
-
|
|
544
|
+
trigger_type: Type of trigger (cron, interval, data_change, composite).
|
|
545
|
+
trigger_config: JSON configuration specific to trigger type.
|
|
546
|
+
cron_expression: Legacy cron expression (for backward compatibility).
|
|
494
547
|
is_active: Whether schedule is active.
|
|
495
548
|
notify_on_failure: Send notification on validation failure.
|
|
496
549
|
last_run_at: Timestamp of last execution.
|
|
497
550
|
next_run_at: Timestamp of next scheduled run.
|
|
551
|
+
trigger_count: Total number of times this schedule has triggered.
|
|
552
|
+
last_trigger_result: Result of last trigger evaluation.
|
|
498
553
|
config: Additional configuration (validators, schema_path, etc.).
|
|
499
554
|
"""
|
|
500
555
|
|
|
@@ -507,13 +562,33 @@ class Schedule(Base, UUIDMixin, TimestampMixin):
|
|
|
507
562
|
nullable=False,
|
|
508
563
|
index=True,
|
|
509
564
|
)
|
|
510
|
-
|
|
565
|
+
# New flexible trigger system
|
|
566
|
+
trigger_type: Mapped[str] = mapped_column(
|
|
567
|
+
String(50),
|
|
568
|
+
nullable=False,
|
|
569
|
+
default=TriggerType.CRON.value,
|
|
570
|
+
index=True,
|
|
571
|
+
)
|
|
572
|
+
trigger_config: Mapped[dict[str, Any] | None] = mapped_column(
|
|
573
|
+
JSON,
|
|
574
|
+
nullable=True,
|
|
575
|
+
comment="Trigger-specific configuration (threshold, metrics, etc.)",
|
|
576
|
+
)
|
|
577
|
+
# Legacy field for backward compatibility
|
|
578
|
+
cron_expression: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
|
511
579
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
512
580
|
notify_on_failure: Mapped[bool] = mapped_column(
|
|
513
581
|
Boolean, default=True, nullable=False
|
|
514
582
|
)
|
|
515
583
|
last_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
516
584
|
next_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
585
|
+
# Trigger state tracking
|
|
586
|
+
trigger_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
587
|
+
last_trigger_result: Mapped[dict[str, Any] | None] = mapped_column(
|
|
588
|
+
JSON,
|
|
589
|
+
nullable=True,
|
|
590
|
+
comment="Result of last trigger evaluation",
|
|
591
|
+
)
|
|
517
592
|
config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
518
593
|
|
|
519
594
|
# Relationships
|
|
@@ -531,6 +606,78 @@ class Schedule(Base, UUIDMixin, TimestampMixin):
|
|
|
531
606
|
"""Mark schedule as run and update next run time."""
|
|
532
607
|
self.last_run_at = datetime.utcnow()
|
|
533
608
|
self.next_run_at = next_run
|
|
609
|
+
self.trigger_count += 1
|
|
610
|
+
|
|
611
|
+
def update_trigger_result(self, result: dict[str, Any]) -> None:
|
|
612
|
+
"""Update the last trigger evaluation result."""
|
|
613
|
+
self.last_trigger_result = result
|
|
614
|
+
|
|
615
|
+
@property
|
|
616
|
+
def effective_trigger_type(self) -> TriggerType:
|
|
617
|
+
"""Get the effective trigger type.
|
|
618
|
+
|
|
619
|
+
Falls back to CRON if trigger_type is not set (legacy schedules).
|
|
620
|
+
"""
|
|
621
|
+
try:
|
|
622
|
+
return TriggerType(self.trigger_type)
|
|
623
|
+
except ValueError:
|
|
624
|
+
return TriggerType.CRON
|
|
625
|
+
|
|
626
|
+
@property
|
|
627
|
+
def effective_cron_expression(self) -> str | None:
|
|
628
|
+
"""Get the effective cron expression.
|
|
629
|
+
|
|
630
|
+
For CRON triggers, returns the expression from trigger_config or legacy field.
|
|
631
|
+
"""
|
|
632
|
+
if self.trigger_type == TriggerType.CRON.value:
|
|
633
|
+
if self.trigger_config and "expression" in self.trigger_config:
|
|
634
|
+
return self.trigger_config["expression"]
|
|
635
|
+
return self.cron_expression
|
|
636
|
+
return None
|
|
637
|
+
|
|
638
|
+
def get_trigger_summary(self) -> str:
|
|
639
|
+
"""Get a human-readable summary of the trigger configuration."""
|
|
640
|
+
trigger_type = self.effective_trigger_type
|
|
641
|
+
|
|
642
|
+
if trigger_type == TriggerType.CRON:
|
|
643
|
+
expr = self.effective_cron_expression
|
|
644
|
+
return f"Cron: {expr}" if expr else "Cron: (not configured)"
|
|
645
|
+
|
|
646
|
+
elif trigger_type == TriggerType.INTERVAL:
|
|
647
|
+
if self.trigger_config:
|
|
648
|
+
parts = []
|
|
649
|
+
if self.trigger_config.get("days"):
|
|
650
|
+
parts.append(f"{self.trigger_config['days']}d")
|
|
651
|
+
if self.trigger_config.get("hours"):
|
|
652
|
+
parts.append(f"{self.trigger_config['hours']}h")
|
|
653
|
+
if self.trigger_config.get("minutes"):
|
|
654
|
+
parts.append(f"{self.trigger_config['minutes']}m")
|
|
655
|
+
if self.trigger_config.get("seconds"):
|
|
656
|
+
parts.append(f"{self.trigger_config['seconds']}s")
|
|
657
|
+
return f"Every {' '.join(parts)}" if parts else "Interval: (not configured)"
|
|
658
|
+
return "Interval: (not configured)"
|
|
659
|
+
|
|
660
|
+
elif trigger_type == TriggerType.DATA_CHANGE:
|
|
661
|
+
threshold = self.trigger_config.get("change_threshold", 0.05) if self.trigger_config else 0.05
|
|
662
|
+
return f"Data change >= {threshold * 100:.0f}%"
|
|
663
|
+
|
|
664
|
+
elif trigger_type == TriggerType.COMPOSITE:
|
|
665
|
+
if self.trigger_config:
|
|
666
|
+
operator = self.trigger_config.get("operator", "and").upper()
|
|
667
|
+
count = len(self.trigger_config.get("triggers", []))
|
|
668
|
+
return f"Composite: {count} triggers ({operator})"
|
|
669
|
+
return "Composite: (not configured)"
|
|
670
|
+
|
|
671
|
+
elif trigger_type == TriggerType.EVENT:
|
|
672
|
+
if self.trigger_config:
|
|
673
|
+
events = self.trigger_config.get("event_types", [])
|
|
674
|
+
return f"Events: {', '.join(events[:2])}{'...' if len(events) > 2 else ''}"
|
|
675
|
+
return "Event: (not configured)"
|
|
676
|
+
|
|
677
|
+
elif trigger_type == TriggerType.MANUAL:
|
|
678
|
+
return "Manual trigger only"
|
|
679
|
+
|
|
680
|
+
return f"{trigger_type.value}: (unknown)"
|
|
534
681
|
|
|
535
682
|
|
|
536
683
|
class DriftComparison(Base, UUIDMixin, TimestampMixin):
|
|
@@ -1672,3 +1819,3228 @@ class Activity(Base, UUIDMixin):
|
|
|
1672
1819
|
def resource_key(self) -> str:
|
|
1673
1820
|
"""Get unique resource key."""
|
|
1674
1821
|
return f"{self.resource_type}:{self.resource_id}"
|
|
1822
|
+
|
|
1823
|
+
|
|
1824
|
+
# =============================================================================
|
|
1825
|
+
# Schema Evolution Models
|
|
1826
|
+
# =============================================================================
|
|
1827
|
+
|
|
1828
|
+
|
|
1829
|
+
class SchemaVersion(Base, UUIDMixin, TimestampMixin):
|
|
1830
|
+
"""Schema version snapshot for evolution tracking.
|
|
1831
|
+
|
|
1832
|
+
Stores a snapshot of schema structure at a point in time
|
|
1833
|
+
to enable schema change detection and history tracking.
|
|
1834
|
+
|
|
1835
|
+
Attributes:
|
|
1836
|
+
id: Unique identifier (UUID).
|
|
1837
|
+
source_id: Reference to parent Source.
|
|
1838
|
+
schema_id: Reference to the Schema record.
|
|
1839
|
+
version_number: Sequential version number for this source.
|
|
1840
|
+
schema_hash: SHA256 hash of normalized schema structure.
|
|
1841
|
+
column_snapshot: JSON snapshot of column definitions.
|
|
1842
|
+
"""
|
|
1843
|
+
|
|
1844
|
+
__tablename__ = "schema_versions"
|
|
1845
|
+
|
|
1846
|
+
__table_args__ = (
|
|
1847
|
+
Index("idx_schema_versions_source", "source_id", "version_number"),
|
|
1848
|
+
Index("idx_schema_versions_hash", "schema_hash"),
|
|
1849
|
+
)
|
|
1850
|
+
|
|
1851
|
+
source_id: Mapped[str] = mapped_column(
|
|
1852
|
+
String(36),
|
|
1853
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
1854
|
+
nullable=False,
|
|
1855
|
+
index=True,
|
|
1856
|
+
)
|
|
1857
|
+
schema_id: Mapped[str] = mapped_column(
|
|
1858
|
+
String(36),
|
|
1859
|
+
ForeignKey("schemas.id", ondelete="CASCADE"),
|
|
1860
|
+
nullable=False,
|
|
1861
|
+
)
|
|
1862
|
+
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
1863
|
+
schema_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
1864
|
+
column_snapshot: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
|
1865
|
+
|
|
1866
|
+
# Relationships
|
|
1867
|
+
source: Mapped[Source] = relationship("Source", lazy="selectin")
|
|
1868
|
+
schema: Mapped[Schema] = relationship("Schema", lazy="selectin")
|
|
1869
|
+
|
|
1870
|
+
@property
|
|
1871
|
+
def column_names(self) -> list[str]:
|
|
1872
|
+
"""Get list of column names from snapshot."""
|
|
1873
|
+
return list(self.column_snapshot.keys())
|
|
1874
|
+
|
|
1875
|
+
@property
|
|
1876
|
+
def column_count(self) -> int:
|
|
1877
|
+
"""Get number of columns in this version."""
|
|
1878
|
+
return len(self.column_snapshot)
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
class SchemaChange(Base, UUIDMixin):
|
|
1882
|
+
"""Individual schema change record.
|
|
1883
|
+
|
|
1884
|
+
Records a single change detected between two schema versions.
|
|
1885
|
+
|
|
1886
|
+
Attributes:
|
|
1887
|
+
id: Unique identifier (UUID).
|
|
1888
|
+
source_id: Reference to parent Source.
|
|
1889
|
+
from_version_id: Reference to previous SchemaVersion (null for first version).
|
|
1890
|
+
to_version_id: Reference to new SchemaVersion.
|
|
1891
|
+
change_type: Type of change (column_added, column_removed, type_changed).
|
|
1892
|
+
column_name: Name of the affected column.
|
|
1893
|
+
old_value: Previous value (for type changes).
|
|
1894
|
+
new_value: New value (for type changes or additions).
|
|
1895
|
+
severity: Severity of the change (breaking/non_breaking).
|
|
1896
|
+
created_at: When the change was detected.
|
|
1897
|
+
"""
|
|
1898
|
+
|
|
1899
|
+
__tablename__ = "schema_changes"
|
|
1900
|
+
|
|
1901
|
+
__table_args__ = (
|
|
1902
|
+
Index("idx_schema_changes_source", "source_id", "created_at"),
|
|
1903
|
+
Index("idx_schema_changes_type", "change_type"),
|
|
1904
|
+
)
|
|
1905
|
+
|
|
1906
|
+
source_id: Mapped[str] = mapped_column(
|
|
1907
|
+
String(36),
|
|
1908
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
1909
|
+
nullable=False,
|
|
1910
|
+
index=True,
|
|
1911
|
+
)
|
|
1912
|
+
from_version_id: Mapped[str | None] = mapped_column(
|
|
1913
|
+
String(36),
|
|
1914
|
+
ForeignKey("schema_versions.id", ondelete="SET NULL"),
|
|
1915
|
+
nullable=True,
|
|
1916
|
+
)
|
|
1917
|
+
to_version_id: Mapped[str] = mapped_column(
|
|
1918
|
+
String(36),
|
|
1919
|
+
ForeignKey("schema_versions.id", ondelete="CASCADE"),
|
|
1920
|
+
nullable=False,
|
|
1921
|
+
)
|
|
1922
|
+
change_type: Mapped[str] = mapped_column(
|
|
1923
|
+
String(30),
|
|
1924
|
+
nullable=False,
|
|
1925
|
+
index=True,
|
|
1926
|
+
)
|
|
1927
|
+
column_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
1928
|
+
old_value: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
1929
|
+
new_value: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
1930
|
+
severity: Mapped[str] = mapped_column(
|
|
1931
|
+
String(20),
|
|
1932
|
+
nullable=False,
|
|
1933
|
+
default="non_breaking",
|
|
1934
|
+
)
|
|
1935
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
1936
|
+
DateTime,
|
|
1937
|
+
default=datetime.utcnow,
|
|
1938
|
+
nullable=False,
|
|
1939
|
+
)
|
|
1940
|
+
|
|
1941
|
+
# Relationships
|
|
1942
|
+
source: Mapped[Source] = relationship("Source", lazy="selectin")
|
|
1943
|
+
from_version: Mapped[SchemaVersion | None] = relationship(
|
|
1944
|
+
"SchemaVersion",
|
|
1945
|
+
foreign_keys=[from_version_id],
|
|
1946
|
+
lazy="selectin",
|
|
1947
|
+
)
|
|
1948
|
+
to_version: Mapped[SchemaVersion] = relationship(
|
|
1949
|
+
"SchemaVersion",
|
|
1950
|
+
foreign_keys=[to_version_id],
|
|
1951
|
+
lazy="selectin",
|
|
1952
|
+
)
|
|
1953
|
+
|
|
1954
|
+
@property
|
|
1955
|
+
def is_breaking(self) -> bool:
|
|
1956
|
+
"""Check if this is a breaking change."""
|
|
1957
|
+
return self.severity == "breaking"
|
|
1958
|
+
|
|
1959
|
+
@property
|
|
1960
|
+
def description(self) -> str:
|
|
1961
|
+
"""Generate human-readable description of the change."""
|
|
1962
|
+
if self.change_type == "column_added":
|
|
1963
|
+
return f"Column '{self.column_name}' added with type {self.new_value}"
|
|
1964
|
+
elif self.change_type == "column_removed":
|
|
1965
|
+
return f"Column '{self.column_name}' removed (was {self.old_value})"
|
|
1966
|
+
elif self.change_type == "type_changed":
|
|
1967
|
+
return f"Column '{self.column_name}' type changed from {self.old_value} to {self.new_value}"
|
|
1968
|
+
return f"Unknown change on '{self.column_name}'"
|
|
1969
|
+
|
|
1970
|
+
|
|
1971
|
+
# =============================================================================
|
|
1972
|
+
# Phase 14: Advanced Notification Models
|
|
1973
|
+
# =============================================================================
|
|
1974
|
+
|
|
1975
|
+
|
|
1976
|
+
class DeduplicationStrategyEnum(str, Enum):
|
|
1977
|
+
"""Deduplication window strategies."""
|
|
1978
|
+
|
|
1979
|
+
SLIDING = "sliding"
|
|
1980
|
+
TUMBLING = "tumbling"
|
|
1981
|
+
SESSION = "session"
|
|
1982
|
+
ADAPTIVE = "adaptive"
|
|
1983
|
+
|
|
1984
|
+
|
|
1985
|
+
class DeduplicationPolicyEnum(str, Enum):
|
|
1986
|
+
"""Deduplication policies."""
|
|
1987
|
+
|
|
1988
|
+
NONE = "none"
|
|
1989
|
+
BASIC = "basic"
|
|
1990
|
+
SEVERITY = "severity"
|
|
1991
|
+
ISSUE_BASED = "issue_based"
|
|
1992
|
+
STRICT = "strict"
|
|
1993
|
+
CUSTOM = "custom"
|
|
1994
|
+
|
|
1995
|
+
|
|
1996
|
+
class EscalationStateEnum(str, Enum):
|
|
1997
|
+
"""Escalation incident states."""
|
|
1998
|
+
|
|
1999
|
+
PENDING = "pending"
|
|
2000
|
+
TRIGGERED = "triggered"
|
|
2001
|
+
ACKNOWLEDGED = "acknowledged"
|
|
2002
|
+
ESCALATED = "escalated"
|
|
2003
|
+
RESOLVED = "resolved"
|
|
2004
|
+
|
|
2005
|
+
|
|
2006
|
+
class TargetTypeEnum(str, Enum):
|
|
2007
|
+
"""Escalation target types."""
|
|
2008
|
+
|
|
2009
|
+
USER = "user"
|
|
2010
|
+
GROUP = "group"
|
|
2011
|
+
ONCALL = "oncall"
|
|
2012
|
+
CHANNEL = "channel"
|
|
2013
|
+
|
|
2014
|
+
|
|
2015
|
+
class RoutingRuleModel(Base, UUIDMixin, TimestampMixin):
|
|
2016
|
+
"""Advanced routing rule model.
|
|
2017
|
+
|
|
2018
|
+
Stores rule-based routing configuration for directing
|
|
2019
|
+
notifications to appropriate channels based on conditions.
|
|
2020
|
+
|
|
2021
|
+
Attributes:
|
|
2022
|
+
id: Unique identifier (UUID).
|
|
2023
|
+
name: Human-readable rule name.
|
|
2024
|
+
rule_config: JSON configuration defining the rule logic.
|
|
2025
|
+
actions: List of channel IDs to notify when rule matches.
|
|
2026
|
+
priority: Priority for rule evaluation (higher = evaluated first).
|
|
2027
|
+
is_active: Whether rule is active.
|
|
2028
|
+
stop_on_match: Stop processing after this rule matches.
|
|
2029
|
+
metadata: Additional metadata.
|
|
2030
|
+
"""
|
|
2031
|
+
|
|
2032
|
+
__tablename__ = "routing_rules"
|
|
2033
|
+
|
|
2034
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2035
|
+
rule_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
|
2036
|
+
actions: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list)
|
|
2037
|
+
priority: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
2038
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
2039
|
+
stop_on_match: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
2040
|
+
routing_metadata: Mapped[dict[str, Any] | None] = mapped_column(
|
|
2041
|
+
"metadata", JSON, nullable=True
|
|
2042
|
+
)
|
|
2043
|
+
|
|
2044
|
+
@property
|
|
2045
|
+
def rule_type(self) -> str:
|
|
2046
|
+
"""Get the rule type from config."""
|
|
2047
|
+
return self.rule_config.get("type", "unknown")
|
|
2048
|
+
|
|
2049
|
+
|
|
2050
|
+
class DeduplicationConfig(Base, UUIDMixin, TimestampMixin):
|
|
2051
|
+
"""Deduplication configuration model.
|
|
2052
|
+
|
|
2053
|
+
Stores configuration for notification deduplication
|
|
2054
|
+
to prevent duplicate notifications within time windows.
|
|
2055
|
+
|
|
2056
|
+
Attributes:
|
|
2057
|
+
id: Unique identifier (UUID).
|
|
2058
|
+
name: Configuration name.
|
|
2059
|
+
strategy: Window strategy (sliding, tumbling, session, adaptive).
|
|
2060
|
+
policy: Deduplication policy (basic, severity, issue_based, etc.).
|
|
2061
|
+
window_seconds: Duration of deduplication window.
|
|
2062
|
+
is_active: Whether config is active.
|
|
2063
|
+
"""
|
|
2064
|
+
|
|
2065
|
+
__tablename__ = "deduplication_configs"
|
|
2066
|
+
|
|
2067
|
+
__table_args__ = (
|
|
2068
|
+
Index("idx_dedup_config_is_active", "is_active"),
|
|
2069
|
+
Index("idx_dedup_config_strategy", "strategy"),
|
|
2070
|
+
Index("idx_dedup_config_policy", "policy"),
|
|
2071
|
+
Index("idx_dedup_config_created_at", "created_at"),
|
|
2072
|
+
)
|
|
2073
|
+
|
|
2074
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2075
|
+
strategy: Mapped[str] = mapped_column(
|
|
2076
|
+
String(20),
|
|
2077
|
+
nullable=False,
|
|
2078
|
+
default=DeduplicationStrategyEnum.SLIDING.value,
|
|
2079
|
+
)
|
|
2080
|
+
policy: Mapped[str] = mapped_column(
|
|
2081
|
+
String(20),
|
|
2082
|
+
nullable=False,
|
|
2083
|
+
default=DeduplicationPolicyEnum.BASIC.value,
|
|
2084
|
+
)
|
|
2085
|
+
window_seconds: Mapped[int] = mapped_column(Integer, default=300, nullable=False)
|
|
2086
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
2087
|
+
|
|
2088
|
+
|
|
2089
|
+
class ThrottlingConfig(Base, UUIDMixin, TimestampMixin):
|
|
2090
|
+
"""Throttling configuration model.
|
|
2091
|
+
|
|
2092
|
+
Stores rate limiting configuration for notifications
|
|
2093
|
+
to prevent overwhelming notification channels.
|
|
2094
|
+
|
|
2095
|
+
Attributes:
|
|
2096
|
+
id: Unique identifier (UUID).
|
|
2097
|
+
name: Configuration name.
|
|
2098
|
+
per_minute: Max notifications per minute.
|
|
2099
|
+
per_hour: Max notifications per hour.
|
|
2100
|
+
per_day: Max notifications per day.
|
|
2101
|
+
burst_allowance: Factor to allow temporary bursts.
|
|
2102
|
+
channel_id: Optional channel ID for per-channel throttling.
|
|
2103
|
+
is_active: Whether config is active.
|
|
2104
|
+
"""
|
|
2105
|
+
|
|
2106
|
+
__tablename__ = "throttling_configs"
|
|
2107
|
+
|
|
2108
|
+
__table_args__ = (
|
|
2109
|
+
Index("idx_throttle_config_is_active", "is_active"),
|
|
2110
|
+
Index("idx_throttle_config_created_at", "created_at"),
|
|
2111
|
+
)
|
|
2112
|
+
|
|
2113
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2114
|
+
per_minute: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
2115
|
+
per_hour: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
2116
|
+
per_day: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
2117
|
+
burst_allowance: Mapped[float] = mapped_column(Float, default=1.5, nullable=False)
|
|
2118
|
+
channel_id: Mapped[str | None] = mapped_column(
|
|
2119
|
+
String(36),
|
|
2120
|
+
ForeignKey("notification_channels.id", ondelete="SET NULL"),
|
|
2121
|
+
nullable=True,
|
|
2122
|
+
index=True,
|
|
2123
|
+
)
|
|
2124
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
2125
|
+
|
|
2126
|
+
# Relationships
|
|
2127
|
+
channel: Mapped[NotificationChannel | None] = relationship(
|
|
2128
|
+
"NotificationChannel",
|
|
2129
|
+
lazy="selectin",
|
|
2130
|
+
)
|
|
2131
|
+
|
|
2132
|
+
|
|
2133
|
+
class EscalationPolicyModel(Base, UUIDMixin, TimestampMixin):
|
|
2134
|
+
"""Escalation policy model.
|
|
2135
|
+
|
|
2136
|
+
Stores multi-level escalation policy configuration
|
|
2137
|
+
for handling unacknowledged alerts.
|
|
2138
|
+
|
|
2139
|
+
Attributes:
|
|
2140
|
+
id: Unique identifier (UUID).
|
|
2141
|
+
name: Policy name.
|
|
2142
|
+
description: Policy description.
|
|
2143
|
+
levels: JSON array of escalation levels.
|
|
2144
|
+
auto_resolve_on_success: Whether to auto-resolve on validation success.
|
|
2145
|
+
max_escalations: Maximum number of escalation attempts.
|
|
2146
|
+
is_active: Whether policy is active.
|
|
2147
|
+
"""
|
|
2148
|
+
|
|
2149
|
+
__tablename__ = "escalation_policies"
|
|
2150
|
+
|
|
2151
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2152
|
+
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
2153
|
+
levels: Mapped[list[dict[str, Any]]] = mapped_column(JSON, nullable=False)
|
|
2154
|
+
auto_resolve_on_success: Mapped[bool] = mapped_column(
|
|
2155
|
+
Boolean, default=True, nullable=False
|
|
2156
|
+
)
|
|
2157
|
+
max_escalations: Mapped[int] = mapped_column(Integer, default=3, nullable=False)
|
|
2158
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
2159
|
+
|
|
2160
|
+
# Relationships
|
|
2161
|
+
incidents: Mapped[list[EscalationIncidentModel]] = relationship(
|
|
2162
|
+
"EscalationIncidentModel",
|
|
2163
|
+
back_populates="policy",
|
|
2164
|
+
cascade="all, delete-orphan",
|
|
2165
|
+
lazy="selectin",
|
|
2166
|
+
)
|
|
2167
|
+
|
|
2168
|
+
@property
|
|
2169
|
+
def level_count(self) -> int:
|
|
2170
|
+
"""Get number of escalation levels."""
|
|
2171
|
+
return len(self.levels) if self.levels else 0
|
|
2172
|
+
|
|
2173
|
+
|
|
2174
|
+
class EscalationIncidentModel(Base, UUIDMixin):
|
|
2175
|
+
"""Escalation incident model.
|
|
2176
|
+
|
|
2177
|
+
Tracks the lifecycle of an escalation from trigger to resolution.
|
|
2178
|
+
|
|
2179
|
+
Attributes:
|
|
2180
|
+
id: Unique identifier (UUID).
|
|
2181
|
+
policy_id: Reference to escalation policy.
|
|
2182
|
+
incident_ref: External reference (e.g., validation ID).
|
|
2183
|
+
state: Current incident state.
|
|
2184
|
+
current_level: Current escalation level.
|
|
2185
|
+
escalation_count: Number of escalation attempts.
|
|
2186
|
+
context: JSON context data.
|
|
2187
|
+
acknowledged_by: Who acknowledged the incident.
|
|
2188
|
+
acknowledged_at: When acknowledged.
|
|
2189
|
+
resolved_by: Who resolved the incident.
|
|
2190
|
+
resolved_at: When resolved.
|
|
2191
|
+
events: JSON array of state transition events.
|
|
2192
|
+
next_escalation_at: When next escalation will occur.
|
|
2193
|
+
"""
|
|
2194
|
+
|
|
2195
|
+
__tablename__ = "escalation_incidents"
|
|
2196
|
+
|
|
2197
|
+
__table_args__ = (
|
|
2198
|
+
Index("idx_escalation_incidents_policy", "policy_id"),
|
|
2199
|
+
Index("idx_escalation_incidents_ref", "incident_ref"),
|
|
2200
|
+
Index("idx_escalation_incidents_state", "state"),
|
|
2201
|
+
Index("idx_escalation_incidents_created_at", "created_at"),
|
|
2202
|
+
Index("idx_escalation_incidents_state_created", "state", "created_at"),
|
|
2203
|
+
)
|
|
2204
|
+
|
|
2205
|
+
policy_id: Mapped[str] = mapped_column(
|
|
2206
|
+
String(36),
|
|
2207
|
+
ForeignKey("escalation_policies.id", ondelete="CASCADE"),
|
|
2208
|
+
nullable=False,
|
|
2209
|
+
index=True,
|
|
2210
|
+
)
|
|
2211
|
+
incident_ref: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2212
|
+
state: Mapped[str] = mapped_column(
|
|
2213
|
+
String(20),
|
|
2214
|
+
nullable=False,
|
|
2215
|
+
default=EscalationStateEnum.PENDING.value,
|
|
2216
|
+
index=True,
|
|
2217
|
+
)
|
|
2218
|
+
current_level: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
|
2219
|
+
escalation_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
2220
|
+
context: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
2221
|
+
acknowledged_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
2222
|
+
acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
2223
|
+
resolved_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
2224
|
+
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
2225
|
+
events: Mapped[list[dict[str, Any]]] = mapped_column(JSON, nullable=False, default=list)
|
|
2226
|
+
next_escalation_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
2227
|
+
|
|
2228
|
+
# Timestamps
|
|
2229
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
2230
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
2231
|
+
)
|
|
2232
|
+
updated_at: Mapped[datetime] = mapped_column(
|
|
2233
|
+
DateTime,
|
|
2234
|
+
default=datetime.utcnow,
|
|
2235
|
+
onupdate=datetime.utcnow,
|
|
2236
|
+
nullable=False,
|
|
2237
|
+
)
|
|
2238
|
+
|
|
2239
|
+
# Relationships
|
|
2240
|
+
policy: Mapped[EscalationPolicyModel] = relationship(
|
|
2241
|
+
"EscalationPolicyModel",
|
|
2242
|
+
back_populates="incidents",
|
|
2243
|
+
)
|
|
2244
|
+
|
|
2245
|
+
@property
|
|
2246
|
+
def is_active(self) -> bool:
|
|
2247
|
+
"""Check if incident is active (not resolved)."""
|
|
2248
|
+
return self.state != EscalationStateEnum.RESOLVED.value
|
|
2249
|
+
|
|
2250
|
+
@property
|
|
2251
|
+
def is_acknowledged(self) -> bool:
|
|
2252
|
+
"""Check if incident has been acknowledged."""
|
|
2253
|
+
return self.state == EscalationStateEnum.ACKNOWLEDGED.value
|
|
2254
|
+
|
|
2255
|
+
@property
|
|
2256
|
+
def is_resolved(self) -> bool:
|
|
2257
|
+
"""Check if incident is resolved."""
|
|
2258
|
+
return self.state == EscalationStateEnum.RESOLVED.value
|
|
2259
|
+
|
|
2260
|
+
def can_escalate(self, max_escalations: int) -> bool:
|
|
2261
|
+
"""Check if incident can be escalated further.
|
|
2262
|
+
|
|
2263
|
+
Args:
|
|
2264
|
+
max_escalations: Maximum allowed escalations from policy.
|
|
2265
|
+
|
|
2266
|
+
Returns:
|
|
2267
|
+
True if escalation is allowed, False otherwise.
|
|
2268
|
+
"""
|
|
2269
|
+
# Cannot escalate resolved or acknowledged incidents
|
|
2270
|
+
if self.state in (
|
|
2271
|
+
EscalationStateEnum.RESOLVED.value,
|
|
2272
|
+
EscalationStateEnum.ACKNOWLEDGED.value,
|
|
2273
|
+
):
|
|
2274
|
+
return False
|
|
2275
|
+
|
|
2276
|
+
# Check escalation count limit
|
|
2277
|
+
if self.escalation_count >= max_escalations:
|
|
2278
|
+
return False
|
|
2279
|
+
|
|
2280
|
+
return True
|
|
2281
|
+
|
|
2282
|
+
def escalate(
|
|
2283
|
+
self,
|
|
2284
|
+
next_level: int,
|
|
2285
|
+
next_escalation_at: datetime | None,
|
|
2286
|
+
max_escalations: int,
|
|
2287
|
+
) -> bool:
|
|
2288
|
+
"""Attempt to escalate the incident.
|
|
2289
|
+
|
|
2290
|
+
Thread-safe escalation with validation. Returns False if
|
|
2291
|
+
escalation is not allowed.
|
|
2292
|
+
|
|
2293
|
+
Args:
|
|
2294
|
+
next_level: The new escalation level.
|
|
2295
|
+
next_escalation_at: When the next escalation should occur.
|
|
2296
|
+
max_escalations: Maximum allowed escalations.
|
|
2297
|
+
|
|
2298
|
+
Returns:
|
|
2299
|
+
True if escalation succeeded, False if not allowed.
|
|
2300
|
+
"""
|
|
2301
|
+
if not self.can_escalate(max_escalations):
|
|
2302
|
+
return False
|
|
2303
|
+
|
|
2304
|
+
old_state = self.state
|
|
2305
|
+
self.state = EscalationStateEnum.ESCALATED.value
|
|
2306
|
+
self.current_level = next_level
|
|
2307
|
+
self.escalation_count += 1
|
|
2308
|
+
self.next_escalation_at = next_escalation_at
|
|
2309
|
+
self.updated_at = datetime.utcnow()
|
|
2310
|
+
|
|
2311
|
+
self.add_event(
|
|
2312
|
+
from_state=old_state,
|
|
2313
|
+
to_state=EscalationStateEnum.ESCALATED.value,
|
|
2314
|
+
actor="system",
|
|
2315
|
+
message=f"Escalated to level {next_level}",
|
|
2316
|
+
)
|
|
2317
|
+
|
|
2318
|
+
return True
|
|
2319
|
+
|
|
2320
|
+
def add_event(
|
|
2321
|
+
self,
|
|
2322
|
+
from_state: str | None,
|
|
2323
|
+
to_state: str,
|
|
2324
|
+
actor: str | None = None,
|
|
2325
|
+
message: str = "",
|
|
2326
|
+
) -> None:
|
|
2327
|
+
"""Add a state transition event."""
|
|
2328
|
+
event = {
|
|
2329
|
+
"from_state": from_state,
|
|
2330
|
+
"to_state": to_state,
|
|
2331
|
+
"actor": actor,
|
|
2332
|
+
"message": message,
|
|
2333
|
+
"timestamp": datetime.utcnow().isoformat(),
|
|
2334
|
+
}
|
|
2335
|
+
if self.events is None:
|
|
2336
|
+
self.events = []
|
|
2337
|
+
self.events.append(event)
|
|
2338
|
+
|
|
2339
|
+
|
|
2340
|
+
# =============================================================================
|
|
2341
|
+
# Scheduler Job Model (Persistent Job Storage)
|
|
2342
|
+
# =============================================================================
|
|
2343
|
+
|
|
2344
|
+
|
|
2345
|
+
class SchedulerJobState(str, Enum):
|
|
2346
|
+
"""State of a scheduled job."""
|
|
2347
|
+
|
|
2348
|
+
PENDING = "pending"
|
|
2349
|
+
RUNNING = "running"
|
|
2350
|
+
COMPLETED = "completed"
|
|
2351
|
+
FAILED = "failed"
|
|
2352
|
+
MISFIRED = "misfired"
|
|
2353
|
+
PAUSED = "paused"
|
|
2354
|
+
|
|
2355
|
+
|
|
2356
|
+
class SchedulerJob(Base, UUIDMixin):
|
|
2357
|
+
"""Persistent scheduler job model.
|
|
2358
|
+
|
|
2359
|
+
Stores scheduled jobs for APScheduler with SQLAlchemy backend,
|
|
2360
|
+
enabling job persistence across restarts.
|
|
2361
|
+
|
|
2362
|
+
Features:
|
|
2363
|
+
- Survives process restarts
|
|
2364
|
+
- Supports exponential backoff retries
|
|
2365
|
+
- Tracks execution history
|
|
2366
|
+
- Enables job recovery on startup
|
|
2367
|
+
|
|
2368
|
+
Attributes:
|
|
2369
|
+
id: Unique job identifier (UUID).
|
|
2370
|
+
name: Human-readable job name.
|
|
2371
|
+
func_ref: Reference to the function to execute (module:function).
|
|
2372
|
+
trigger_type: Type of trigger (interval, cron, date).
|
|
2373
|
+
trigger_args: Arguments for the trigger configuration.
|
|
2374
|
+
args: Positional arguments for the function.
|
|
2375
|
+
kwargs: Keyword arguments for the function.
|
|
2376
|
+
next_run_time: Next scheduled execution time.
|
|
2377
|
+
state: Current job state.
|
|
2378
|
+
retry_count: Number of retry attempts.
|
|
2379
|
+
last_run_time: Last execution time.
|
|
2380
|
+
last_error: Last error message.
|
|
2381
|
+
job_metadata: Additional job metadata.
|
|
2382
|
+
created_at: When the job was created.
|
|
2383
|
+
updated_at: Last update timestamp.
|
|
2384
|
+
"""
|
|
2385
|
+
|
|
2386
|
+
__tablename__ = "scheduler_jobs"
|
|
2387
|
+
|
|
2388
|
+
__table_args__ = (
|
|
2389
|
+
Index("idx_scheduler_jobs_state", "state"),
|
|
2390
|
+
Index("idx_scheduler_jobs_next_run", "next_run_time"),
|
|
2391
|
+
Index("idx_scheduler_jobs_state_next_run", "state", "next_run_time"),
|
|
2392
|
+
)
|
|
2393
|
+
|
|
2394
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2395
|
+
func_ref: Mapped[str] = mapped_column(String(512), nullable=False)
|
|
2396
|
+
trigger_type: Mapped[str] = mapped_column(
|
|
2397
|
+
String(20),
|
|
2398
|
+
nullable=False,
|
|
2399
|
+
default="interval",
|
|
2400
|
+
)
|
|
2401
|
+
trigger_args: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
2402
|
+
args: Mapped[list[Any] | None] = mapped_column(JSON, nullable=True)
|
|
2403
|
+
kwargs: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
2404
|
+
next_run_time: Mapped[datetime | None] = mapped_column(
|
|
2405
|
+
DateTime,
|
|
2406
|
+
nullable=True,
|
|
2407
|
+
index=True,
|
|
2408
|
+
)
|
|
2409
|
+
state: Mapped[str] = mapped_column(
|
|
2410
|
+
String(20),
|
|
2411
|
+
nullable=False,
|
|
2412
|
+
default=SchedulerJobState.PENDING.value,
|
|
2413
|
+
index=True,
|
|
2414
|
+
)
|
|
2415
|
+
retry_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
2416
|
+
last_run_time: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
2417
|
+
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
2418
|
+
job_metadata: Mapped[dict[str, Any] | None] = mapped_column(
|
|
2419
|
+
"metadata",
|
|
2420
|
+
JSON,
|
|
2421
|
+
nullable=True,
|
|
2422
|
+
)
|
|
2423
|
+
|
|
2424
|
+
# Timestamps
|
|
2425
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
2426
|
+
DateTime,
|
|
2427
|
+
default=datetime.utcnow,
|
|
2428
|
+
nullable=False,
|
|
2429
|
+
)
|
|
2430
|
+
updated_at: Mapped[datetime] = mapped_column(
|
|
2431
|
+
DateTime,
|
|
2432
|
+
default=datetime.utcnow,
|
|
2433
|
+
onupdate=datetime.utcnow,
|
|
2434
|
+
nullable=False,
|
|
2435
|
+
)
|
|
2436
|
+
|
|
2437
|
+
@property
|
|
2438
|
+
def is_due(self) -> bool:
|
|
2439
|
+
"""Check if job is due for execution."""
|
|
2440
|
+
if not self.next_run_time:
|
|
2441
|
+
return False
|
|
2442
|
+
return (
|
|
2443
|
+
self.state in (SchedulerJobState.PENDING.value, SchedulerJobState.MISFIRED.value)
|
|
2444
|
+
and datetime.utcnow() >= self.next_run_time
|
|
2445
|
+
)
|
|
2446
|
+
|
|
2447
|
+
@property
|
|
2448
|
+
def is_running(self) -> bool:
|
|
2449
|
+
"""Check if job is currently running."""
|
|
2450
|
+
return self.state == SchedulerJobState.RUNNING.value
|
|
2451
|
+
|
|
2452
|
+
@property
|
|
2453
|
+
def is_completed(self) -> bool:
|
|
2454
|
+
"""Check if job has completed successfully."""
|
|
2455
|
+
return self.state == SchedulerJobState.COMPLETED.value
|
|
2456
|
+
|
|
2457
|
+
@property
|
|
2458
|
+
def is_failed(self) -> bool:
|
|
2459
|
+
"""Check if job has failed permanently."""
|
|
2460
|
+
return self.state == SchedulerJobState.FAILED.value
|
|
2461
|
+
|
|
2462
|
+
def mark_running(self) -> None:
|
|
2463
|
+
"""Mark job as running."""
|
|
2464
|
+
self.state = SchedulerJobState.RUNNING.value
|
|
2465
|
+
self.last_run_time = datetime.utcnow()
|
|
2466
|
+
self.updated_at = datetime.utcnow()
|
|
2467
|
+
|
|
2468
|
+
def mark_completed(self, next_run_time: datetime | None = None) -> None:
|
|
2469
|
+
"""Mark job as completed.
|
|
2470
|
+
|
|
2471
|
+
Args:
|
|
2472
|
+
next_run_time: Next scheduled run time (for recurring jobs).
|
|
2473
|
+
"""
|
|
2474
|
+
if next_run_time:
|
|
2475
|
+
self.state = SchedulerJobState.PENDING.value
|
|
2476
|
+
self.next_run_time = next_run_time
|
|
2477
|
+
else:
|
|
2478
|
+
self.state = SchedulerJobState.COMPLETED.value
|
|
2479
|
+
self.retry_count = 0
|
|
2480
|
+
self.last_error = None
|
|
2481
|
+
self.updated_at = datetime.utcnow()
|
|
2482
|
+
|
|
2483
|
+
def mark_failed(self, error: str, can_retry: bool = True) -> None:
|
|
2484
|
+
"""Mark job as failed.
|
|
2485
|
+
|
|
2486
|
+
Args:
|
|
2487
|
+
error: Error message.
|
|
2488
|
+
can_retry: Whether job can be retried.
|
|
2489
|
+
"""
|
|
2490
|
+
self.last_error = error
|
|
2491
|
+
self.updated_at = datetime.utcnow()
|
|
2492
|
+
|
|
2493
|
+
if can_retry:
|
|
2494
|
+
self.state = SchedulerJobState.PENDING.value
|
|
2495
|
+
self.retry_count += 1
|
|
2496
|
+
else:
|
|
2497
|
+
self.state = SchedulerJobState.FAILED.value
|
|
2498
|
+
|
|
2499
|
+
def mark_misfired(self) -> None:
|
|
2500
|
+
"""Mark job as misfired."""
|
|
2501
|
+
self.state = SchedulerJobState.MISFIRED.value
|
|
2502
|
+
self.updated_at = datetime.utcnow()
|
|
2503
|
+
if self.job_metadata is None:
|
|
2504
|
+
self.job_metadata = {}
|
|
2505
|
+
self.job_metadata["misfire_count"] = self.job_metadata.get("misfire_count", 0) + 1
|
|
2506
|
+
self.job_metadata["last_misfire_at"] = datetime.utcnow().isoformat()
|
|
2507
|
+
|
|
2508
|
+
|
|
2509
|
+
# =============================================================================
|
|
2510
|
+
# Phase 10: Data Lineage Models
|
|
2511
|
+
# =============================================================================
|
|
2512
|
+
|
|
2513
|
+
|
|
2514
|
+
class LineageNodeType(str, Enum):
|
|
2515
|
+
"""Type of lineage node."""
|
|
2516
|
+
|
|
2517
|
+
SOURCE = "source"
|
|
2518
|
+
TRANSFORM = "transform"
|
|
2519
|
+
SINK = "sink"
|
|
2520
|
+
|
|
2521
|
+
|
|
2522
|
+
class LineageEdgeType(str, Enum):
|
|
2523
|
+
"""Type of lineage edge."""
|
|
2524
|
+
|
|
2525
|
+
DERIVES_FROM = "derives_from"
|
|
2526
|
+
TRANSFORMS_TO = "transforms_to"
|
|
2527
|
+
JOINS_WITH = "joins_with"
|
|
2528
|
+
FILTERS_FROM = "filters_from"
|
|
2529
|
+
|
|
2530
|
+
|
|
2531
|
+
class LineageNode(Base, UUIDMixin, TimestampMixin):
|
|
2532
|
+
"""Data lineage node model.
|
|
2533
|
+
|
|
2534
|
+
Represents a node in the data lineage graph (source, transformation, or sink).
|
|
2535
|
+
|
|
2536
|
+
Attributes:
|
|
2537
|
+
id: Unique identifier (UUID).
|
|
2538
|
+
name: Human-readable node name.
|
|
2539
|
+
node_type: Type of node (source, transform, sink).
|
|
2540
|
+
source_id: Optional reference to a data source.
|
|
2541
|
+
metadata_json: Additional metadata for the node.
|
|
2542
|
+
position_x: X coordinate for graph visualization.
|
|
2543
|
+
position_y: Y coordinate for graph visualization.
|
|
2544
|
+
"""
|
|
2545
|
+
|
|
2546
|
+
__tablename__ = "lineage_nodes"
|
|
2547
|
+
|
|
2548
|
+
__table_args__ = (
|
|
2549
|
+
Index("idx_lineage_nodes_type", "node_type"),
|
|
2550
|
+
Index("idx_lineage_nodes_source", "source_id"),
|
|
2551
|
+
)
|
|
2552
|
+
|
|
2553
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2554
|
+
node_type: Mapped[str] = mapped_column(
|
|
2555
|
+
String(20),
|
|
2556
|
+
nullable=False,
|
|
2557
|
+
default=LineageNodeType.SOURCE.value,
|
|
2558
|
+
index=True,
|
|
2559
|
+
)
|
|
2560
|
+
source_id: Mapped[str | None] = mapped_column(
|
|
2561
|
+
String(36),
|
|
2562
|
+
ForeignKey("sources.id", ondelete="SET NULL"),
|
|
2563
|
+
nullable=True,
|
|
2564
|
+
index=True,
|
|
2565
|
+
)
|
|
2566
|
+
metadata_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
2567
|
+
position_x: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
2568
|
+
position_y: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
2569
|
+
|
|
2570
|
+
# Relationships
|
|
2571
|
+
source: Mapped[Source | None] = relationship(
|
|
2572
|
+
"Source",
|
|
2573
|
+
lazy="selectin",
|
|
2574
|
+
)
|
|
2575
|
+
outgoing_edges: Mapped[list[LineageEdge]] = relationship(
|
|
2576
|
+
"LineageEdge",
|
|
2577
|
+
foreign_keys="LineageEdge.source_node_id",
|
|
2578
|
+
back_populates="source_node",
|
|
2579
|
+
cascade="all, delete-orphan",
|
|
2580
|
+
lazy="selectin",
|
|
2581
|
+
)
|
|
2582
|
+
incoming_edges: Mapped[list[LineageEdge]] = relationship(
|
|
2583
|
+
"LineageEdge",
|
|
2584
|
+
foreign_keys="LineageEdge.target_node_id",
|
|
2585
|
+
back_populates="target_node",
|
|
2586
|
+
cascade="all, delete-orphan",
|
|
2587
|
+
lazy="selectin",
|
|
2588
|
+
)
|
|
2589
|
+
|
|
2590
|
+
@property
|
|
2591
|
+
def upstream_count(self) -> int:
|
|
2592
|
+
"""Get number of upstream (incoming) connections."""
|
|
2593
|
+
return len(self.incoming_edges)
|
|
2594
|
+
|
|
2595
|
+
@property
|
|
2596
|
+
def downstream_count(self) -> int:
|
|
2597
|
+
"""Get number of downstream (outgoing) connections."""
|
|
2598
|
+
return len(self.outgoing_edges)
|
|
2599
|
+
|
|
2600
|
+
@property
|
|
2601
|
+
def is_source_type(self) -> bool:
|
|
2602
|
+
"""Check if this is a source node."""
|
|
2603
|
+
return self.node_type == LineageNodeType.SOURCE.value
|
|
2604
|
+
|
|
2605
|
+
@property
|
|
2606
|
+
def is_transform_type(self) -> bool:
|
|
2607
|
+
"""Check if this is a transform node."""
|
|
2608
|
+
return self.node_type == LineageNodeType.TRANSFORM.value
|
|
2609
|
+
|
|
2610
|
+
@property
|
|
2611
|
+
def is_sink_type(self) -> bool:
|
|
2612
|
+
"""Check if this is a sink node."""
|
|
2613
|
+
return self.node_type == LineageNodeType.SINK.value
|
|
2614
|
+
|
|
2615
|
+
|
|
2616
|
+
class LineageEdge(Base, UUIDMixin):
|
|
2617
|
+
"""Data lineage edge model.
|
|
2618
|
+
|
|
2619
|
+
Represents a connection (data flow) between two lineage nodes.
|
|
2620
|
+
|
|
2621
|
+
Attributes:
|
|
2622
|
+
id: Unique identifier (UUID).
|
|
2623
|
+
source_node_id: ID of the source node (origin of data flow).
|
|
2624
|
+
target_node_id: ID of the target node (destination of data flow).
|
|
2625
|
+
edge_type: Type of relationship.
|
|
2626
|
+
metadata_json: Additional metadata for the edge.
|
|
2627
|
+
created_at: When the edge was created.
|
|
2628
|
+
"""
|
|
2629
|
+
|
|
2630
|
+
__tablename__ = "lineage_edges"
|
|
2631
|
+
|
|
2632
|
+
__table_args__ = (
|
|
2633
|
+
Index("idx_lineage_edges_source", "source_node_id"),
|
|
2634
|
+
Index("idx_lineage_edges_target", "target_node_id"),
|
|
2635
|
+
Index(
|
|
2636
|
+
"idx_lineage_edges_unique",
|
|
2637
|
+
"source_node_id",
|
|
2638
|
+
"target_node_id",
|
|
2639
|
+
"edge_type",
|
|
2640
|
+
unique=True,
|
|
2641
|
+
),
|
|
2642
|
+
)
|
|
2643
|
+
|
|
2644
|
+
source_node_id: Mapped[str] = mapped_column(
|
|
2645
|
+
String(36),
|
|
2646
|
+
ForeignKey("lineage_nodes.id", ondelete="CASCADE"),
|
|
2647
|
+
nullable=False,
|
|
2648
|
+
index=True,
|
|
2649
|
+
)
|
|
2650
|
+
target_node_id: Mapped[str] = mapped_column(
|
|
2651
|
+
String(36),
|
|
2652
|
+
ForeignKey("lineage_nodes.id", ondelete="CASCADE"),
|
|
2653
|
+
nullable=False,
|
|
2654
|
+
index=True,
|
|
2655
|
+
)
|
|
2656
|
+
edge_type: Mapped[str] = mapped_column(
|
|
2657
|
+
String(30),
|
|
2658
|
+
nullable=False,
|
|
2659
|
+
default=LineageEdgeType.DERIVES_FROM.value,
|
|
2660
|
+
index=True,
|
|
2661
|
+
)
|
|
2662
|
+
metadata_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
2663
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
2664
|
+
DateTime,
|
|
2665
|
+
default=datetime.utcnow,
|
|
2666
|
+
nullable=False,
|
|
2667
|
+
)
|
|
2668
|
+
|
|
2669
|
+
# Relationships
|
|
2670
|
+
source_node: Mapped[LineageNode] = relationship(
|
|
2671
|
+
"LineageNode",
|
|
2672
|
+
foreign_keys=[source_node_id],
|
|
2673
|
+
back_populates="outgoing_edges",
|
|
2674
|
+
)
|
|
2675
|
+
target_node: Mapped[LineageNode] = relationship(
|
|
2676
|
+
"LineageNode",
|
|
2677
|
+
foreign_keys=[target_node_id],
|
|
2678
|
+
back_populates="incoming_edges",
|
|
2679
|
+
)
|
|
2680
|
+
|
|
2681
|
+
|
|
2682
|
+
# =============================================================================
|
|
2683
|
+
# OpenLineage Webhook Models
|
|
2684
|
+
# =============================================================================
|
|
2685
|
+
|
|
2686
|
+
|
|
2687
|
+
class OpenLineageEventType(str, Enum):
|
|
2688
|
+
"""Types of OpenLineage events that can be emitted."""
|
|
2689
|
+
|
|
2690
|
+
JOB = "job"
|
|
2691
|
+
DATASET = "dataset"
|
|
2692
|
+
ALL = "all"
|
|
2693
|
+
|
|
2694
|
+
|
|
2695
|
+
class OpenLineageWebhook(Base, UUIDMixin, TimestampMixin):
|
|
2696
|
+
"""OpenLineage webhook configuration model.
|
|
2697
|
+
|
|
2698
|
+
Stores configuration for emitting OpenLineage events to external endpoints.
|
|
2699
|
+
|
|
2700
|
+
Attributes:
|
|
2701
|
+
id: Unique identifier (UUID).
|
|
2702
|
+
name: Human-readable name for the webhook.
|
|
2703
|
+
url: Target URL for the webhook.
|
|
2704
|
+
is_active: Whether the webhook is enabled.
|
|
2705
|
+
headers_json: Custom headers as JSON (excluding auth).
|
|
2706
|
+
api_key: Optional API key for authentication.
|
|
2707
|
+
event_types: Types of events to emit (job, dataset, all).
|
|
2708
|
+
batch_size: Number of events per batch.
|
|
2709
|
+
timeout_seconds: Request timeout.
|
|
2710
|
+
last_sent_at: Timestamp of last successful emission.
|
|
2711
|
+
success_count: Total successful emissions.
|
|
2712
|
+
failure_count: Total failed emissions.
|
|
2713
|
+
last_error: Last error message if any.
|
|
2714
|
+
"""
|
|
2715
|
+
|
|
2716
|
+
__tablename__ = "openlineage_webhooks"
|
|
2717
|
+
|
|
2718
|
+
__table_args__ = (
|
|
2719
|
+
Index("idx_openlineage_webhooks_active", "is_active"),
|
|
2720
|
+
)
|
|
2721
|
+
|
|
2722
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
2723
|
+
url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
2724
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
2725
|
+
headers_json: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True)
|
|
2726
|
+
api_key: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
2727
|
+
event_types: Mapped[str] = mapped_column(
|
|
2728
|
+
String(20),
|
|
2729
|
+
nullable=False,
|
|
2730
|
+
default=OpenLineageEventType.ALL.value,
|
|
2731
|
+
)
|
|
2732
|
+
batch_size: Mapped[int] = mapped_column(Integer, nullable=False, default=100)
|
|
2733
|
+
timeout_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
|
|
2734
|
+
|
|
2735
|
+
# Statistics
|
|
2736
|
+
last_sent_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
2737
|
+
success_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
2738
|
+
failure_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
2739
|
+
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
2740
|
+
|
|
2741
|
+
@property
|
|
2742
|
+
def headers(self) -> dict[str, str]:
|
|
2743
|
+
"""Get headers dictionary."""
|
|
2744
|
+
return self.headers_json or {}
|
|
2745
|
+
|
|
2746
|
+
@property
|
|
2747
|
+
def total_emissions(self) -> int:
|
|
2748
|
+
"""Get total emission attempts."""
|
|
2749
|
+
return self.success_count + self.failure_count
|
|
2750
|
+
|
|
2751
|
+
@property
|
|
2752
|
+
def success_rate(self) -> float:
|
|
2753
|
+
"""Get success rate as percentage."""
|
|
2754
|
+
if self.total_emissions == 0:
|
|
2755
|
+
return 100.0
|
|
2756
|
+
return (self.success_count / self.total_emissions) * 100
|
|
2757
|
+
|
|
2758
|
+
def record_success(self) -> None:
|
|
2759
|
+
"""Record a successful emission."""
|
|
2760
|
+
self.success_count += 1
|
|
2761
|
+
self.last_sent_at = datetime.utcnow()
|
|
2762
|
+
self.last_error = None
|
|
2763
|
+
|
|
2764
|
+
def record_failure(self, error: str) -> None:
|
|
2765
|
+
"""Record a failed emission."""
|
|
2766
|
+
self.failure_count += 1
|
|
2767
|
+
self.last_error = error
|
|
2768
|
+
|
|
2769
|
+
def activate(self) -> None:
|
|
2770
|
+
"""Enable the webhook."""
|
|
2771
|
+
self.is_active = True
|
|
2772
|
+
|
|
2773
|
+
def deactivate(self) -> None:
|
|
2774
|
+
"""Disable the webhook."""
|
|
2775
|
+
self.is_active = False
|
|
2776
|
+
|
|
2777
|
+
|
|
2778
|
+
# =============================================================================
|
|
2779
|
+
# Phase 10: Anomaly Detection Models
|
|
2780
|
+
# =============================================================================
|
|
2781
|
+
|
|
2782
|
+
|
|
2783
|
+
class AnomalyAlgorithm(str, Enum):
|
|
2784
|
+
"""Supported anomaly detection algorithms."""
|
|
2785
|
+
|
|
2786
|
+
ISOLATION_FOREST = "isolation_forest"
|
|
2787
|
+
LOF = "lof"
|
|
2788
|
+
ONE_CLASS_SVM = "one_class_svm"
|
|
2789
|
+
DBSCAN = "dbscan"
|
|
2790
|
+
STATISTICAL = "statistical"
|
|
2791
|
+
AUTOENCODER = "autoencoder"
|
|
2792
|
+
|
|
2793
|
+
|
|
2794
|
+
class AnomalyDetectionStatus(str, Enum):
|
|
2795
|
+
"""Status of an anomaly detection run."""
|
|
2796
|
+
|
|
2797
|
+
PENDING = "pending"
|
|
2798
|
+
RUNNING = "running"
|
|
2799
|
+
SUCCESS = "success"
|
|
2800
|
+
ERROR = "error"
|
|
2801
|
+
|
|
2802
|
+
|
|
2803
|
+
class AnomalyDetection(Base, UUIDMixin):
|
|
2804
|
+
"""Anomaly detection result model.
|
|
2805
|
+
|
|
2806
|
+
Stores results from ML-based anomaly detection runs.
|
|
2807
|
+
|
|
2808
|
+
Attributes:
|
|
2809
|
+
id: Unique identifier (UUID).
|
|
2810
|
+
source_id: Reference to parent Source.
|
|
2811
|
+
status: Current status (pending, running, success, error).
|
|
2812
|
+
algorithm: Detection algorithm used.
|
|
2813
|
+
config: Algorithm-specific configuration.
|
|
2814
|
+
total_rows: Total rows analyzed.
|
|
2815
|
+
anomaly_count: Number of anomalies found.
|
|
2816
|
+
anomaly_rate: Rate of anomalies (0-1).
|
|
2817
|
+
columns_analyzed: List of columns that were analyzed.
|
|
2818
|
+
result_json: Full detection result as JSON.
|
|
2819
|
+
duration_ms: Execution time in milliseconds.
|
|
2820
|
+
error_message: Error message if failed.
|
|
2821
|
+
"""
|
|
2822
|
+
|
|
2823
|
+
__tablename__ = "anomaly_detections"
|
|
2824
|
+
|
|
2825
|
+
__table_args__ = (
|
|
2826
|
+
Index("idx_anomaly_detections_source_created", "source_id", "created_at"),
|
|
2827
|
+
Index("idx_anomaly_detections_status", "status"),
|
|
2828
|
+
Index("idx_anomaly_detections_algorithm", "algorithm"),
|
|
2829
|
+
)
|
|
2830
|
+
|
|
2831
|
+
source_id: Mapped[str] = mapped_column(
|
|
2832
|
+
String(36),
|
|
2833
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
2834
|
+
nullable=False,
|
|
2835
|
+
index=True,
|
|
2836
|
+
)
|
|
2837
|
+
|
|
2838
|
+
# Status tracking
|
|
2839
|
+
status: Mapped[str] = mapped_column(
|
|
2840
|
+
String(20),
|
|
2841
|
+
nullable=False,
|
|
2842
|
+
default=AnomalyDetectionStatus.PENDING.value,
|
|
2843
|
+
index=True,
|
|
2844
|
+
)
|
|
2845
|
+
|
|
2846
|
+
# Algorithm configuration
|
|
2847
|
+
algorithm: Mapped[str] = mapped_column(
|
|
2848
|
+
String(30),
|
|
2849
|
+
nullable=False,
|
|
2850
|
+
default=AnomalyAlgorithm.ISOLATION_FOREST.value,
|
|
2851
|
+
index=True,
|
|
2852
|
+
)
|
|
2853
|
+
config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
2854
|
+
|
|
2855
|
+
# Results summary
|
|
2856
|
+
total_rows: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
2857
|
+
anomaly_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
2858
|
+
anomaly_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
2859
|
+
columns_analyzed: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
2860
|
+
|
|
2861
|
+
# Full result and timing
|
|
2862
|
+
result_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
2863
|
+
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
2864
|
+
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
2865
|
+
|
|
2866
|
+
# Timestamps
|
|
2867
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
2868
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
2869
|
+
)
|
|
2870
|
+
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
2871
|
+
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
2872
|
+
|
|
2873
|
+
# Relationships
|
|
2874
|
+
source: Mapped[Source] = relationship(
|
|
2875
|
+
"Source",
|
|
2876
|
+
backref="anomaly_detections",
|
|
2877
|
+
)
|
|
2878
|
+
|
|
2879
|
+
@property
|
|
2880
|
+
def is_complete(self) -> bool:
|
|
2881
|
+
"""Check if detection has completed (success or error)."""
|
|
2882
|
+
return self.status in (
|
|
2883
|
+
AnomalyDetectionStatus.SUCCESS.value,
|
|
2884
|
+
AnomalyDetectionStatus.ERROR.value,
|
|
2885
|
+
)
|
|
2886
|
+
|
|
2887
|
+
@property
|
|
2888
|
+
def anomalies(self) -> list[dict[str, Any]]:
|
|
2889
|
+
"""Get list of anomaly records from result JSON."""
|
|
2890
|
+
if self.result_json and "anomalies" in self.result_json:
|
|
2891
|
+
return self.result_json["anomalies"]
|
|
2892
|
+
return []
|
|
2893
|
+
|
|
2894
|
+
@property
|
|
2895
|
+
def column_summaries(self) -> list[dict[str, Any]]:
|
|
2896
|
+
"""Get per-column anomaly summaries from result JSON."""
|
|
2897
|
+
if self.result_json and "column_summaries" in self.result_json:
|
|
2898
|
+
return self.result_json["column_summaries"]
|
|
2899
|
+
return []
|
|
2900
|
+
|
|
2901
|
+
def mark_started(self) -> None:
|
|
2902
|
+
"""Mark detection as started."""
|
|
2903
|
+
self.status = AnomalyDetectionStatus.RUNNING.value
|
|
2904
|
+
self.started_at = datetime.utcnow()
|
|
2905
|
+
|
|
2906
|
+
def mark_completed(
|
|
2907
|
+
self,
|
|
2908
|
+
anomaly_count: int,
|
|
2909
|
+
anomaly_rate: float,
|
|
2910
|
+
result: dict[str, Any],
|
|
2911
|
+
) -> None:
|
|
2912
|
+
"""Mark detection as completed with results."""
|
|
2913
|
+
self.status = AnomalyDetectionStatus.SUCCESS.value
|
|
2914
|
+
self.anomaly_count = anomaly_count
|
|
2915
|
+
self.anomaly_rate = anomaly_rate
|
|
2916
|
+
self.result_json = result
|
|
2917
|
+
self.completed_at = datetime.utcnow()
|
|
2918
|
+
|
|
2919
|
+
if self.started_at:
|
|
2920
|
+
delta = self.completed_at - self.started_at
|
|
2921
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
2922
|
+
|
|
2923
|
+
def mark_error(self, message: str) -> None:
|
|
2924
|
+
"""Mark detection as errored."""
|
|
2925
|
+
self.status = AnomalyDetectionStatus.ERROR.value
|
|
2926
|
+
self.error_message = message
|
|
2927
|
+
self.completed_at = datetime.utcnow()
|
|
2928
|
+
|
|
2929
|
+
if self.started_at:
|
|
2930
|
+
delta = self.completed_at - self.started_at
|
|
2931
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
2932
|
+
|
|
2933
|
+
|
|
2934
|
+
class AnomalyExplanation(Base, UUIDMixin):
|
|
2935
|
+
"""SHAP/LIME explanations for anomaly detection results.
|
|
2936
|
+
|
|
2937
|
+
Stores feature-level explanations for individual anomalous rows,
|
|
2938
|
+
providing interpretability for ML-based anomaly detection.
|
|
2939
|
+
|
|
2940
|
+
Attributes:
|
|
2941
|
+
id: Unique identifier (UUID).
|
|
2942
|
+
detection_id: Reference to parent AnomalyDetection.
|
|
2943
|
+
row_index: Row index in the original dataset.
|
|
2944
|
+
anomaly_score: Anomaly score for this row.
|
|
2945
|
+
feature_contributions: JSON array of feature contributions.
|
|
2946
|
+
total_shap: Sum of all SHAP values.
|
|
2947
|
+
summary: Human-readable explanation summary.
|
|
2948
|
+
generated_at: When the explanation was generated.
|
|
2949
|
+
"""
|
|
2950
|
+
|
|
2951
|
+
__tablename__ = "anomaly_explanations"
|
|
2952
|
+
|
|
2953
|
+
__table_args__ = (
|
|
2954
|
+
Index("idx_anomaly_explanations_detection", "detection_id"),
|
|
2955
|
+
Index("idx_anomaly_explanations_row", "detection_id", "row_index"),
|
|
2956
|
+
)
|
|
2957
|
+
|
|
2958
|
+
detection_id: Mapped[str] = mapped_column(
|
|
2959
|
+
String(36),
|
|
2960
|
+
ForeignKey("anomaly_detections.id", ondelete="CASCADE"),
|
|
2961
|
+
nullable=False,
|
|
2962
|
+
index=True,
|
|
2963
|
+
)
|
|
2964
|
+
|
|
2965
|
+
# Row identification
|
|
2966
|
+
row_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
2967
|
+
|
|
2968
|
+
# Anomaly information
|
|
2969
|
+
anomaly_score: Mapped[float] = mapped_column(Float, nullable=False)
|
|
2970
|
+
|
|
2971
|
+
# SHAP/LIME explanation data
|
|
2972
|
+
feature_contributions: Mapped[list[dict[str, Any]]] = mapped_column(
|
|
2973
|
+
JSON,
|
|
2974
|
+
nullable=False,
|
|
2975
|
+
default=list,
|
|
2976
|
+
)
|
|
2977
|
+
total_shap: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
|
2978
|
+
|
|
2979
|
+
# Human-readable summary
|
|
2980
|
+
summary: Mapped[str] = mapped_column(Text, nullable=False)
|
|
2981
|
+
|
|
2982
|
+
# Timestamp
|
|
2983
|
+
generated_at: Mapped[datetime] = mapped_column(
|
|
2984
|
+
DateTime,
|
|
2985
|
+
default=datetime.utcnow,
|
|
2986
|
+
nullable=False,
|
|
2987
|
+
)
|
|
2988
|
+
|
|
2989
|
+
# Relationships
|
|
2990
|
+
detection: Mapped[AnomalyDetection] = relationship(
|
|
2991
|
+
"AnomalyDetection",
|
|
2992
|
+
backref="explanations",
|
|
2993
|
+
)
|
|
2994
|
+
|
|
2995
|
+
@property
|
|
2996
|
+
def top_features(self) -> list[str]:
|
|
2997
|
+
"""Get names of top contributing features."""
|
|
2998
|
+
if not self.feature_contributions:
|
|
2999
|
+
return []
|
|
3000
|
+
return [fc.get("feature", "") for fc in self.feature_contributions[:5]]
|
|
3001
|
+
|
|
3002
|
+
@property
|
|
3003
|
+
def contribution_count(self) -> int:
|
|
3004
|
+
"""Get number of feature contributions stored."""
|
|
3005
|
+
return len(self.feature_contributions) if self.feature_contributions else 0
|
|
3006
|
+
|
|
3007
|
+
|
|
3008
|
+
class BatchDetectionStatus(str, Enum):
|
|
3009
|
+
"""Status of a batch anomaly detection job."""
|
|
3010
|
+
|
|
3011
|
+
PENDING = "pending"
|
|
3012
|
+
RUNNING = "running"
|
|
3013
|
+
COMPLETED = "completed"
|
|
3014
|
+
PARTIAL = "partial" # Some sources completed with errors
|
|
3015
|
+
ERROR = "error"
|
|
3016
|
+
CANCELLED = "cancelled"
|
|
3017
|
+
|
|
3018
|
+
|
|
3019
|
+
class AnomalyBatchJob(Base, UUIDMixin, TimestampMixin):
|
|
3020
|
+
"""Batch anomaly detection job model.
|
|
3021
|
+
|
|
3022
|
+
Stores configuration and results for batch anomaly detection
|
|
3023
|
+
across multiple data sources.
|
|
3024
|
+
|
|
3025
|
+
Attributes:
|
|
3026
|
+
id: Unique identifier (UUID).
|
|
3027
|
+
name: Optional job name.
|
|
3028
|
+
status: Current status of the batch job.
|
|
3029
|
+
algorithm: Detection algorithm to use.
|
|
3030
|
+
config: Algorithm-specific configuration.
|
|
3031
|
+
source_ids: List of source IDs to process.
|
|
3032
|
+
total_sources: Total number of sources.
|
|
3033
|
+
completed_sources: Number of completed sources.
|
|
3034
|
+
failed_sources: Number of failed sources.
|
|
3035
|
+
total_anomalies: Total anomalies found across all sources.
|
|
3036
|
+
results_json: Detection results per source.
|
|
3037
|
+
error_message: Error message if job failed.
|
|
3038
|
+
duration_ms: Total execution time in milliseconds.
|
|
3039
|
+
"""
|
|
3040
|
+
|
|
3041
|
+
__tablename__ = "anomaly_batch_jobs"
|
|
3042
|
+
|
|
3043
|
+
__table_args__ = (
|
|
3044
|
+
Index("idx_anomaly_batch_jobs_status", "status"),
|
|
3045
|
+
Index("idx_anomaly_batch_jobs_created", "created_at"),
|
|
3046
|
+
)
|
|
3047
|
+
|
|
3048
|
+
name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
3049
|
+
|
|
3050
|
+
# Status tracking
|
|
3051
|
+
status: Mapped[str] = mapped_column(
|
|
3052
|
+
String(20),
|
|
3053
|
+
nullable=False,
|
|
3054
|
+
default=BatchDetectionStatus.PENDING.value,
|
|
3055
|
+
index=True,
|
|
3056
|
+
)
|
|
3057
|
+
|
|
3058
|
+
# Algorithm configuration
|
|
3059
|
+
algorithm: Mapped[str] = mapped_column(
|
|
3060
|
+
String(30),
|
|
3061
|
+
nullable=False,
|
|
3062
|
+
default=AnomalyAlgorithm.ISOLATION_FOREST.value,
|
|
3063
|
+
)
|
|
3064
|
+
config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
3065
|
+
|
|
3066
|
+
# Sources to process
|
|
3067
|
+
source_ids: Mapped[list[str]] = mapped_column(JSON, nullable=False)
|
|
3068
|
+
|
|
3069
|
+
# Progress tracking
|
|
3070
|
+
total_sources: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
3071
|
+
completed_sources: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
3072
|
+
failed_sources: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
3073
|
+
current_source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
|
3074
|
+
|
|
3075
|
+
# Aggregate results
|
|
3076
|
+
total_anomalies: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
3077
|
+
total_rows_analyzed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
3078
|
+
|
|
3079
|
+
# Per-source results: {source_id: {detection_id, status, anomaly_count, anomaly_rate, ...}}
|
|
3080
|
+
results_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
3081
|
+
|
|
3082
|
+
# Error handling
|
|
3083
|
+
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
3084
|
+
|
|
3085
|
+
# Timing
|
|
3086
|
+
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
3087
|
+
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3088
|
+
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3089
|
+
|
|
3090
|
+
@property
|
|
3091
|
+
def progress_percent(self) -> float:
|
|
3092
|
+
"""Get progress as percentage."""
|
|
3093
|
+
if self.total_sources == 0:
|
|
3094
|
+
return 0.0
|
|
3095
|
+
return ((self.completed_sources + self.failed_sources) / self.total_sources) * 100
|
|
3096
|
+
|
|
3097
|
+
@property
|
|
3098
|
+
def average_anomaly_rate(self) -> float:
|
|
3099
|
+
"""Get average anomaly rate across all successful sources."""
|
|
3100
|
+
if not self.results_json:
|
|
3101
|
+
return 0.0
|
|
3102
|
+
rates = [
|
|
3103
|
+
r["anomaly_rate"]
|
|
3104
|
+
for r in self.results_json.values()
|
|
3105
|
+
if r.get("status") == "success" and r.get("anomaly_rate") is not None
|
|
3106
|
+
]
|
|
3107
|
+
return sum(rates) / len(rates) if rates else 0.0
|
|
3108
|
+
|
|
3109
|
+
@property
|
|
3110
|
+
def is_complete(self) -> bool:
|
|
3111
|
+
"""Check if batch job has completed."""
|
|
3112
|
+
return self.status in (
|
|
3113
|
+
BatchDetectionStatus.COMPLETED.value,
|
|
3114
|
+
BatchDetectionStatus.PARTIAL.value,
|
|
3115
|
+
BatchDetectionStatus.ERROR.value,
|
|
3116
|
+
BatchDetectionStatus.CANCELLED.value,
|
|
3117
|
+
)
|
|
3118
|
+
|
|
3119
|
+
def mark_started(self) -> None:
|
|
3120
|
+
"""Mark batch job as started."""
|
|
3121
|
+
self.status = BatchDetectionStatus.RUNNING.value
|
|
3122
|
+
self.started_at = datetime.utcnow()
|
|
3123
|
+
|
|
3124
|
+
def update_progress(
|
|
3125
|
+
self,
|
|
3126
|
+
source_id: str,
|
|
3127
|
+
detection_id: str,
|
|
3128
|
+
status: str,
|
|
3129
|
+
anomaly_count: int = 0,
|
|
3130
|
+
anomaly_rate: float = 0.0,
|
|
3131
|
+
total_rows: int = 0,
|
|
3132
|
+
error_message: str | None = None,
|
|
3133
|
+
) -> None:
|
|
3134
|
+
"""Update progress for a single source."""
|
|
3135
|
+
if self.results_json is None:
|
|
3136
|
+
self.results_json = {}
|
|
3137
|
+
|
|
3138
|
+
self.results_json[source_id] = {
|
|
3139
|
+
"detection_id": detection_id,
|
|
3140
|
+
"status": status,
|
|
3141
|
+
"anomaly_count": anomaly_count,
|
|
3142
|
+
"anomaly_rate": anomaly_rate,
|
|
3143
|
+
"total_rows": total_rows,
|
|
3144
|
+
"error_message": error_message,
|
|
3145
|
+
}
|
|
3146
|
+
|
|
3147
|
+
if status == "success":
|
|
3148
|
+
self.completed_sources += 1
|
|
3149
|
+
self.total_anomalies += anomaly_count
|
|
3150
|
+
self.total_rows_analyzed += total_rows
|
|
3151
|
+
elif status == "error":
|
|
3152
|
+
self.failed_sources += 1
|
|
3153
|
+
|
|
3154
|
+
def mark_completed(self) -> None:
|
|
3155
|
+
"""Mark batch job as completed."""
|
|
3156
|
+
if self.failed_sources > 0 and self.completed_sources > 0:
|
|
3157
|
+
self.status = BatchDetectionStatus.PARTIAL.value
|
|
3158
|
+
elif self.failed_sources == self.total_sources:
|
|
3159
|
+
self.status = BatchDetectionStatus.ERROR.value
|
|
3160
|
+
else:
|
|
3161
|
+
self.status = BatchDetectionStatus.COMPLETED.value
|
|
3162
|
+
|
|
3163
|
+
self.completed_at = datetime.utcnow()
|
|
3164
|
+
self.current_source_id = None
|
|
3165
|
+
|
|
3166
|
+
if self.started_at:
|
|
3167
|
+
delta = self.completed_at - self.started_at
|
|
3168
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
3169
|
+
|
|
3170
|
+
def mark_error(self, message: str) -> None:
|
|
3171
|
+
"""Mark batch job as errored."""
|
|
3172
|
+
self.status = BatchDetectionStatus.ERROR.value
|
|
3173
|
+
self.error_message = message
|
|
3174
|
+
self.completed_at = datetime.utcnow()
|
|
3175
|
+
self.current_source_id = None
|
|
3176
|
+
|
|
3177
|
+
if self.started_at:
|
|
3178
|
+
delta = self.completed_at - self.started_at
|
|
3179
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
3180
|
+
|
|
3181
|
+
def mark_cancelled(self) -> None:
|
|
3182
|
+
"""Mark batch job as cancelled."""
|
|
3183
|
+
self.status = BatchDetectionStatus.CANCELLED.value
|
|
3184
|
+
self.completed_at = datetime.utcnow()
|
|
3185
|
+
self.current_source_id = None
|
|
3186
|
+
|
|
3187
|
+
if self.started_at:
|
|
3188
|
+
delta = self.completed_at - self.started_at
|
|
3189
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
3190
|
+
|
|
3191
|
+
|
|
3192
|
+
# =============================================================================
|
|
3193
|
+
# Phase 10: Model Monitoring Models
|
|
3194
|
+
# =============================================================================
|
|
3195
|
+
|
|
3196
|
+
|
|
3197
|
+
class ModelStatus(str, Enum):
|
|
3198
|
+
"""Status of a monitored model."""
|
|
3199
|
+
|
|
3200
|
+
ACTIVE = "active"
|
|
3201
|
+
PAUSED = "paused"
|
|
3202
|
+
DEGRADED = "degraded"
|
|
3203
|
+
ERROR = "error"
|
|
3204
|
+
|
|
3205
|
+
|
|
3206
|
+
class AlertSeverityLevel(str, Enum):
|
|
3207
|
+
"""Alert severity levels for model monitoring."""
|
|
3208
|
+
|
|
3209
|
+
CRITICAL = "critical"
|
|
3210
|
+
WARNING = "warning"
|
|
3211
|
+
INFO = "info"
|
|
3212
|
+
|
|
3213
|
+
|
|
3214
|
+
class MetricTypeEnum(str, Enum):
|
|
3215
|
+
"""Types of metrics collected."""
|
|
3216
|
+
|
|
3217
|
+
LATENCY = "latency"
|
|
3218
|
+
THROUGHPUT = "throughput"
|
|
3219
|
+
ERROR_RATE = "error_rate"
|
|
3220
|
+
NULL_RATE = "null_rate"
|
|
3221
|
+
TYPE_VIOLATION = "type_violation"
|
|
3222
|
+
DRIFT_SCORE = "drift_score"
|
|
3223
|
+
CUSTOM = "custom"
|
|
3224
|
+
|
|
3225
|
+
|
|
3226
|
+
class AlertRuleTypeEnum(str, Enum):
|
|
3227
|
+
"""Types of alert rules."""
|
|
3228
|
+
|
|
3229
|
+
THRESHOLD = "threshold"
|
|
3230
|
+
STATISTICAL = "statistical"
|
|
3231
|
+
TREND = "trend"
|
|
3232
|
+
|
|
3233
|
+
|
|
3234
|
+
class AlertHandlerTypeEnum(str, Enum):
|
|
3235
|
+
"""Types of alert handlers."""
|
|
3236
|
+
|
|
3237
|
+
SLACK = "slack"
|
|
3238
|
+
WEBHOOK = "webhook"
|
|
3239
|
+
EMAIL = "email"
|
|
3240
|
+
|
|
3241
|
+
|
|
3242
|
+
class MonitoredModel(Base, UUIDMixin, TimestampMixin):
|
|
3243
|
+
"""Monitored ML model registration model.
|
|
3244
|
+
|
|
3245
|
+
Represents a registered ML model for monitoring performance,
|
|
3246
|
+
drift, and data quality metrics.
|
|
3247
|
+
|
|
3248
|
+
Attributes:
|
|
3249
|
+
id: Unique identifier (UUID).
|
|
3250
|
+
name: Model name/identifier.
|
|
3251
|
+
version: Model version string.
|
|
3252
|
+
description: Model description.
|
|
3253
|
+
status: Monitoring status (active, paused, degraded, error).
|
|
3254
|
+
config: JSON configuration for monitoring settings.
|
|
3255
|
+
metadata_json: Additional model metadata.
|
|
3256
|
+
prediction_count: Total predictions recorded.
|
|
3257
|
+
last_prediction_at: Timestamp of last prediction.
|
|
3258
|
+
current_drift_score: Current drift score.
|
|
3259
|
+
health_score: Model health score (0-100).
|
|
3260
|
+
"""
|
|
3261
|
+
|
|
3262
|
+
__tablename__ = "monitored_models"
|
|
3263
|
+
|
|
3264
|
+
__table_args__ = (
|
|
3265
|
+
Index("idx_monitored_models_name", "name"),
|
|
3266
|
+
Index("idx_monitored_models_status", "status"),
|
|
3267
|
+
)
|
|
3268
|
+
|
|
3269
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
3270
|
+
version: Mapped[str] = mapped_column(String(50), nullable=False, default="1.0.0")
|
|
3271
|
+
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
3272
|
+
status: Mapped[str] = mapped_column(
|
|
3273
|
+
String(20),
|
|
3274
|
+
nullable=False,
|
|
3275
|
+
default=ModelStatus.ACTIVE.value,
|
|
3276
|
+
index=True,
|
|
3277
|
+
)
|
|
3278
|
+
config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
|
|
3279
|
+
metadata_json: Mapped[dict[str, Any] | None] = mapped_column(
|
|
3280
|
+
"metadata", JSON, nullable=True
|
|
3281
|
+
)
|
|
3282
|
+
prediction_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
3283
|
+
last_prediction_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3284
|
+
current_drift_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
3285
|
+
health_score: Mapped[float] = mapped_column(Float, default=100.0, nullable=False)
|
|
3286
|
+
|
|
3287
|
+
# Relationships
|
|
3288
|
+
predictions: Mapped[list[ModelPrediction]] = relationship(
|
|
3289
|
+
"ModelPrediction",
|
|
3290
|
+
back_populates="model",
|
|
3291
|
+
cascade="all, delete-orphan",
|
|
3292
|
+
lazy="selectin",
|
|
3293
|
+
)
|
|
3294
|
+
metrics: Mapped[list[ModelMetric]] = relationship(
|
|
3295
|
+
"ModelMetric",
|
|
3296
|
+
back_populates="model",
|
|
3297
|
+
cascade="all, delete-orphan",
|
|
3298
|
+
lazy="selectin",
|
|
3299
|
+
)
|
|
3300
|
+
alert_rules: Mapped[list[ModelAlertRule]] = relationship(
|
|
3301
|
+
"ModelAlertRule",
|
|
3302
|
+
back_populates="model",
|
|
3303
|
+
cascade="all, delete-orphan",
|
|
3304
|
+
lazy="selectin",
|
|
3305
|
+
)
|
|
3306
|
+
alerts: Mapped[list[ModelAlert]] = relationship(
|
|
3307
|
+
"ModelAlert",
|
|
3308
|
+
back_populates="model",
|
|
3309
|
+
cascade="all, delete-orphan",
|
|
3310
|
+
lazy="selectin",
|
|
3311
|
+
)
|
|
3312
|
+
|
|
3313
|
+
@property
|
|
3314
|
+
def is_active(self) -> bool:
|
|
3315
|
+
"""Check if model is actively monitored."""
|
|
3316
|
+
return self.status == ModelStatus.ACTIVE.value
|
|
3317
|
+
|
|
3318
|
+
@property
|
|
3319
|
+
def is_healthy(self) -> bool:
|
|
3320
|
+
"""Check if model is in healthy state."""
|
|
3321
|
+
return self.health_score >= 70.0
|
|
3322
|
+
|
|
3323
|
+
@property
|
|
3324
|
+
def has_drift(self) -> bool:
|
|
3325
|
+
"""Check if model has significant drift."""
|
|
3326
|
+
threshold = self.config.get("drift_threshold", 0.1)
|
|
3327
|
+
return (self.current_drift_score or 0.0) > threshold
|
|
3328
|
+
|
|
3329
|
+
def record_prediction(self) -> None:
|
|
3330
|
+
"""Record a new prediction."""
|
|
3331
|
+
self.prediction_count += 1
|
|
3332
|
+
self.last_prediction_at = datetime.utcnow()
|
|
3333
|
+
|
|
3334
|
+
def update_drift_score(self, score: float) -> None:
|
|
3335
|
+
"""Update the drift score."""
|
|
3336
|
+
self.current_drift_score = score
|
|
3337
|
+
# Update health score based on drift
|
|
3338
|
+
if score > 0.3:
|
|
3339
|
+
self.status = ModelStatus.DEGRADED.value
|
|
3340
|
+
self.health_score = max(0.0, 100.0 - score * 100)
|
|
3341
|
+
elif score > 0.1:
|
|
3342
|
+
self.health_score = max(50.0, 100.0 - score * 50)
|
|
3343
|
+
|
|
3344
|
+
def pause(self) -> None:
|
|
3345
|
+
"""Pause monitoring for this model."""
|
|
3346
|
+
self.status = ModelStatus.PAUSED.value
|
|
3347
|
+
|
|
3348
|
+
def resume(self) -> None:
|
|
3349
|
+
"""Resume monitoring for this model."""
|
|
3350
|
+
self.status = ModelStatus.ACTIVE.value
|
|
3351
|
+
|
|
3352
|
+
|
|
3353
|
+
class ModelPrediction(Base, UUIDMixin):
|
|
3354
|
+
"""Model prediction record for monitoring.
|
|
3355
|
+
|
|
3356
|
+
Stores individual predictions made by a model for tracking
|
|
3357
|
+
performance and drift over time.
|
|
3358
|
+
|
|
3359
|
+
Attributes:
|
|
3360
|
+
id: Unique identifier (UUID).
|
|
3361
|
+
model_id: Reference to MonitoredModel.
|
|
3362
|
+
features: JSON of input features.
|
|
3363
|
+
prediction: The model's output prediction.
|
|
3364
|
+
actual: The actual value (if available).
|
|
3365
|
+
latency_ms: Prediction latency in milliseconds.
|
|
3366
|
+
metadata_json: Additional prediction metadata.
|
|
3367
|
+
recorded_at: When the prediction was recorded.
|
|
3368
|
+
"""
|
|
3369
|
+
|
|
3370
|
+
__tablename__ = "model_predictions"
|
|
3371
|
+
|
|
3372
|
+
__table_args__ = (
|
|
3373
|
+
Index("idx_model_predictions_model", "model_id", "recorded_at"),
|
|
3374
|
+
Index("idx_model_predictions_time", "recorded_at"),
|
|
3375
|
+
)
|
|
3376
|
+
|
|
3377
|
+
model_id: Mapped[str] = mapped_column(
|
|
3378
|
+
String(36),
|
|
3379
|
+
ForeignKey("monitored_models.id", ondelete="CASCADE"),
|
|
3380
|
+
nullable=False,
|
|
3381
|
+
index=True,
|
|
3382
|
+
)
|
|
3383
|
+
features: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
|
3384
|
+
prediction: Mapped[Any] = mapped_column(JSON, nullable=False)
|
|
3385
|
+
actual: Mapped[Any | None] = mapped_column(JSON, nullable=True)
|
|
3386
|
+
latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
3387
|
+
metadata_json: Mapped[dict[str, Any] | None] = mapped_column(
|
|
3388
|
+
"metadata", JSON, nullable=True
|
|
3389
|
+
)
|
|
3390
|
+
recorded_at: Mapped[datetime] = mapped_column(
|
|
3391
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
3392
|
+
)
|
|
3393
|
+
|
|
3394
|
+
# Relationships
|
|
3395
|
+
model: Mapped[MonitoredModel] = relationship(
|
|
3396
|
+
"MonitoredModel",
|
|
3397
|
+
back_populates="predictions",
|
|
3398
|
+
)
|
|
3399
|
+
|
|
3400
|
+
@property
|
|
3401
|
+
def has_actual(self) -> bool:
|
|
3402
|
+
"""Check if actual value is available."""
|
|
3403
|
+
return self.actual is not None
|
|
3404
|
+
|
|
3405
|
+
|
|
3406
|
+
class ModelMetric(Base, UUIDMixin):
|
|
3407
|
+
"""Model performance metric record.
|
|
3408
|
+
|
|
3409
|
+
Stores aggregated metric values for a monitored model.
|
|
3410
|
+
|
|
3411
|
+
Attributes:
|
|
3412
|
+
id: Unique identifier (UUID).
|
|
3413
|
+
model_id: Reference to MonitoredModel.
|
|
3414
|
+
metric_type: Type of metric (latency, throughput, etc.).
|
|
3415
|
+
metric_name: Name of the metric.
|
|
3416
|
+
value: Metric value.
|
|
3417
|
+
labels: JSON labels for the metric.
|
|
3418
|
+
recorded_at: When the metric was recorded.
|
|
3419
|
+
"""
|
|
3420
|
+
|
|
3421
|
+
__tablename__ = "model_metrics"
|
|
3422
|
+
|
|
3423
|
+
__table_args__ = (
|
|
3424
|
+
Index("idx_model_metrics_model", "model_id", "recorded_at"),
|
|
3425
|
+
Index("idx_model_metrics_type", "metric_type"),
|
|
3426
|
+
Index("idx_model_metrics_name", "metric_name"),
|
|
3427
|
+
)
|
|
3428
|
+
|
|
3429
|
+
model_id: Mapped[str] = mapped_column(
|
|
3430
|
+
String(36),
|
|
3431
|
+
ForeignKey("monitored_models.id", ondelete="CASCADE"),
|
|
3432
|
+
nullable=False,
|
|
3433
|
+
index=True,
|
|
3434
|
+
)
|
|
3435
|
+
metric_type: Mapped[str] = mapped_column(
|
|
3436
|
+
String(30),
|
|
3437
|
+
nullable=False,
|
|
3438
|
+
index=True,
|
|
3439
|
+
)
|
|
3440
|
+
metric_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
3441
|
+
value: Mapped[float] = mapped_column(Float, nullable=False)
|
|
3442
|
+
labels: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True)
|
|
3443
|
+
recorded_at: Mapped[datetime] = mapped_column(
|
|
3444
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
3445
|
+
)
|
|
3446
|
+
|
|
3447
|
+
# Relationships
|
|
3448
|
+
model: Mapped[MonitoredModel] = relationship(
|
|
3449
|
+
"MonitoredModel",
|
|
3450
|
+
back_populates="metrics",
|
|
3451
|
+
)
|
|
3452
|
+
|
|
3453
|
+
|
|
3454
|
+
class ModelAlertRule(Base, UUIDMixin, TimestampMixin):
|
|
3455
|
+
"""Alert rule for model monitoring.
|
|
3456
|
+
|
|
3457
|
+
Defines conditions that trigger alerts for a monitored model.
|
|
3458
|
+
|
|
3459
|
+
Attributes:
|
|
3460
|
+
id: Unique identifier (UUID).
|
|
3461
|
+
model_id: Reference to MonitoredModel.
|
|
3462
|
+
name: Rule name.
|
|
3463
|
+
rule_type: Type of rule (threshold, statistical, trend).
|
|
3464
|
+
severity: Alert severity level.
|
|
3465
|
+
config: Rule-specific configuration.
|
|
3466
|
+
is_active: Whether rule is active.
|
|
3467
|
+
last_triggered_at: When last triggered.
|
|
3468
|
+
trigger_count: Total trigger count.
|
|
3469
|
+
"""
|
|
3470
|
+
|
|
3471
|
+
__tablename__ = "model_alert_rules"
|
|
3472
|
+
|
|
3473
|
+
__table_args__ = (
|
|
3474
|
+
Index("idx_model_alert_rules_model", "model_id"),
|
|
3475
|
+
Index("idx_model_alert_rules_type", "rule_type"),
|
|
3476
|
+
)
|
|
3477
|
+
|
|
3478
|
+
model_id: Mapped[str] = mapped_column(
|
|
3479
|
+
String(36),
|
|
3480
|
+
ForeignKey("monitored_models.id", ondelete="CASCADE"),
|
|
3481
|
+
nullable=False,
|
|
3482
|
+
index=True,
|
|
3483
|
+
)
|
|
3484
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
3485
|
+
rule_type: Mapped[str] = mapped_column(
|
|
3486
|
+
String(20),
|
|
3487
|
+
nullable=False,
|
|
3488
|
+
index=True,
|
|
3489
|
+
)
|
|
3490
|
+
severity: Mapped[str] = mapped_column(
|
|
3491
|
+
String(20),
|
|
3492
|
+
nullable=False,
|
|
3493
|
+
default=AlertSeverityLevel.WARNING.value,
|
|
3494
|
+
)
|
|
3495
|
+
config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
|
3496
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
3497
|
+
last_triggered_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3498
|
+
trigger_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
3499
|
+
|
|
3500
|
+
# Relationships
|
|
3501
|
+
model: Mapped[MonitoredModel] = relationship(
|
|
3502
|
+
"MonitoredModel",
|
|
3503
|
+
back_populates="alert_rules",
|
|
3504
|
+
)
|
|
3505
|
+
alerts: Mapped[list[ModelAlert]] = relationship(
|
|
3506
|
+
"ModelAlert",
|
|
3507
|
+
back_populates="rule",
|
|
3508
|
+
cascade="all, delete-orphan",
|
|
3509
|
+
lazy="selectin",
|
|
3510
|
+
)
|
|
3511
|
+
|
|
3512
|
+
def trigger(self) -> None:
|
|
3513
|
+
"""Record a trigger of this rule."""
|
|
3514
|
+
self.last_triggered_at = datetime.utcnow()
|
|
3515
|
+
self.trigger_count += 1
|
|
3516
|
+
|
|
3517
|
+
def activate(self) -> None:
|
|
3518
|
+
"""Activate this rule."""
|
|
3519
|
+
self.is_active = True
|
|
3520
|
+
|
|
3521
|
+
def deactivate(self) -> None:
|
|
3522
|
+
"""Deactivate this rule."""
|
|
3523
|
+
self.is_active = False
|
|
3524
|
+
|
|
3525
|
+
|
|
3526
|
+
class ModelAlertHandler(Base, UUIDMixin, TimestampMixin):
|
|
3527
|
+
"""Alert handler for model monitoring.
|
|
3528
|
+
|
|
3529
|
+
Defines where and how alerts are sent.
|
|
3530
|
+
|
|
3531
|
+
Attributes:
|
|
3532
|
+
id: Unique identifier (UUID).
|
|
3533
|
+
name: Handler name.
|
|
3534
|
+
handler_type: Type of handler (slack, webhook, email).
|
|
3535
|
+
config: Handler-specific configuration.
|
|
3536
|
+
is_active: Whether handler is active.
|
|
3537
|
+
last_sent_at: When last alert was sent.
|
|
3538
|
+
send_count: Total alerts sent.
|
|
3539
|
+
failure_count: Total send failures.
|
|
3540
|
+
"""
|
|
3541
|
+
|
|
3542
|
+
__tablename__ = "model_alert_handlers"
|
|
3543
|
+
|
|
3544
|
+
__table_args__ = (Index("idx_model_alert_handlers_type", "handler_type"),)
|
|
3545
|
+
|
|
3546
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
3547
|
+
handler_type: Mapped[str] = mapped_column(
|
|
3548
|
+
String(20),
|
|
3549
|
+
nullable=False,
|
|
3550
|
+
index=True,
|
|
3551
|
+
)
|
|
3552
|
+
config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
|
3553
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
3554
|
+
last_sent_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3555
|
+
send_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
3556
|
+
failure_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
3557
|
+
|
|
3558
|
+
def record_send(self, success: bool = True) -> None:
|
|
3559
|
+
"""Record a send attempt."""
|
|
3560
|
+
self.last_sent_at = datetime.utcnow()
|
|
3561
|
+
if success:
|
|
3562
|
+
self.send_count += 1
|
|
3563
|
+
else:
|
|
3564
|
+
self.failure_count += 1
|
|
3565
|
+
|
|
3566
|
+
def activate(self) -> None:
|
|
3567
|
+
"""Activate this handler."""
|
|
3568
|
+
self.is_active = True
|
|
3569
|
+
|
|
3570
|
+
def deactivate(self) -> None:
|
|
3571
|
+
"""Deactivate this handler."""
|
|
3572
|
+
self.is_active = False
|
|
3573
|
+
|
|
3574
|
+
|
|
3575
|
+
class ModelAlert(Base, UUIDMixin):
|
|
3576
|
+
"""Alert instance for model monitoring.
|
|
3577
|
+
|
|
3578
|
+
Represents a triggered alert from an alert rule.
|
|
3579
|
+
|
|
3580
|
+
Attributes:
|
|
3581
|
+
id: Unique identifier (UUID).
|
|
3582
|
+
model_id: Reference to MonitoredModel.
|
|
3583
|
+
rule_id: Reference to ModelAlertRule.
|
|
3584
|
+
severity: Alert severity.
|
|
3585
|
+
message: Alert message.
|
|
3586
|
+
metric_value: Value that triggered the alert.
|
|
3587
|
+
threshold_value: Threshold that was exceeded.
|
|
3588
|
+
acknowledged: Whether alert is acknowledged.
|
|
3589
|
+
acknowledged_by: Who acknowledged.
|
|
3590
|
+
acknowledged_at: When acknowledged.
|
|
3591
|
+
resolved: Whether alert is resolved.
|
|
3592
|
+
resolved_at: When resolved.
|
|
3593
|
+
created_at: When alert was created.
|
|
3594
|
+
"""
|
|
3595
|
+
|
|
3596
|
+
__tablename__ = "model_alerts"
|
|
3597
|
+
|
|
3598
|
+
__table_args__ = (
|
|
3599
|
+
Index("idx_model_alerts_model", "model_id", "created_at"),
|
|
3600
|
+
Index("idx_model_alerts_rule", "rule_id"),
|
|
3601
|
+
Index("idx_model_alerts_resolved", "resolved"),
|
|
3602
|
+
)
|
|
3603
|
+
|
|
3604
|
+
model_id: Mapped[str] = mapped_column(
|
|
3605
|
+
String(36),
|
|
3606
|
+
ForeignKey("monitored_models.id", ondelete="CASCADE"),
|
|
3607
|
+
nullable=False,
|
|
3608
|
+
index=True,
|
|
3609
|
+
)
|
|
3610
|
+
rule_id: Mapped[str] = mapped_column(
|
|
3611
|
+
String(36),
|
|
3612
|
+
ForeignKey("model_alert_rules.id", ondelete="CASCADE"),
|
|
3613
|
+
nullable=False,
|
|
3614
|
+
index=True,
|
|
3615
|
+
)
|
|
3616
|
+
severity: Mapped[str] = mapped_column(
|
|
3617
|
+
String(20),
|
|
3618
|
+
nullable=False,
|
|
3619
|
+
default=AlertSeverityLevel.WARNING.value,
|
|
3620
|
+
)
|
|
3621
|
+
message: Mapped[str] = mapped_column(Text, nullable=False)
|
|
3622
|
+
metric_value: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
3623
|
+
threshold_value: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
3624
|
+
acknowledged: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
3625
|
+
acknowledged_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
3626
|
+
acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3627
|
+
resolved: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
3628
|
+
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3629
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
3630
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
3631
|
+
)
|
|
3632
|
+
updated_at: Mapped[datetime] = mapped_column(
|
|
3633
|
+
DateTime,
|
|
3634
|
+
default=datetime.utcnow,
|
|
3635
|
+
onupdate=datetime.utcnow,
|
|
3636
|
+
nullable=False,
|
|
3637
|
+
)
|
|
3638
|
+
|
|
3639
|
+
# Relationships
|
|
3640
|
+
model: Mapped[MonitoredModel] = relationship(
|
|
3641
|
+
"MonitoredModel",
|
|
3642
|
+
back_populates="alerts",
|
|
3643
|
+
)
|
|
3644
|
+
rule: Mapped[ModelAlertRule] = relationship(
|
|
3645
|
+
"ModelAlertRule",
|
|
3646
|
+
back_populates="alerts",
|
|
3647
|
+
)
|
|
3648
|
+
|
|
3649
|
+
@property
|
|
3650
|
+
def is_active(self) -> bool:
|
|
3651
|
+
"""Check if alert is active (not resolved)."""
|
|
3652
|
+
return not self.resolved
|
|
3653
|
+
|
|
3654
|
+
def acknowledge(self, by: str) -> None:
|
|
3655
|
+
"""Acknowledge this alert."""
|
|
3656
|
+
self.acknowledged = True
|
|
3657
|
+
self.acknowledged_by = by
|
|
3658
|
+
self.acknowledged_at = datetime.utcnow()
|
|
3659
|
+
|
|
3660
|
+
def resolve(self) -> None:
|
|
3661
|
+
"""Resolve this alert."""
|
|
3662
|
+
self.resolved = True
|
|
3663
|
+
self.resolved_at = datetime.utcnow()
|
|
3664
|
+
|
|
3665
|
+
|
|
3666
|
+
# =============================================================================
|
|
3667
|
+
# Phase 10: Drift Monitor Models
|
|
3668
|
+
# =============================================================================
|
|
3669
|
+
|
|
3670
|
+
|
|
3671
|
+
class DriftMonitorStatus(str, Enum):
|
|
3672
|
+
"""Status of a drift monitor."""
|
|
3673
|
+
|
|
3674
|
+
ACTIVE = "active"
|
|
3675
|
+
PAUSED = "paused"
|
|
3676
|
+
ERROR = "error"
|
|
3677
|
+
|
|
3678
|
+
|
|
3679
|
+
class DriftAlertStatus(str, Enum):
|
|
3680
|
+
"""Status of a drift alert."""
|
|
3681
|
+
|
|
3682
|
+
ACTIVE = "active"
|
|
3683
|
+
ACKNOWLEDGED = "acknowledged"
|
|
3684
|
+
RESOLVED = "resolved"
|
|
3685
|
+
|
|
3686
|
+
|
|
3687
|
+
class DriftMonitor(Base, UUIDMixin, TimestampMixin):
|
|
3688
|
+
"""Drift monitoring configuration model.
|
|
3689
|
+
|
|
3690
|
+
Defines a drift monitor that compares baseline and current data sources.
|
|
3691
|
+
|
|
3692
|
+
Attributes:
|
|
3693
|
+
id: Unique identifier (UUID).
|
|
3694
|
+
name: Monitor name.
|
|
3695
|
+
baseline_source_id: Reference to baseline Source.
|
|
3696
|
+
current_source_id: Reference to current Source.
|
|
3697
|
+
status: Monitor status.
|
|
3698
|
+
method: Drift detection method.
|
|
3699
|
+
threshold_critical: Critical drift threshold.
|
|
3700
|
+
threshold_high: High drift threshold.
|
|
3701
|
+
columns: Columns to monitor (None = all).
|
|
3702
|
+
schedule_cron: Optional cron expression for scheduling.
|
|
3703
|
+
last_run_at: Last run timestamp.
|
|
3704
|
+
next_run_at: Next scheduled run.
|
|
3705
|
+
config: Additional configuration.
|
|
3706
|
+
"""
|
|
3707
|
+
|
|
3708
|
+
__tablename__ = "drift_monitors"
|
|
3709
|
+
|
|
3710
|
+
__table_args__ = (
|
|
3711
|
+
Index("idx_drift_monitors_status", "status"),
|
|
3712
|
+
Index("idx_drift_monitors_baseline", "baseline_source_id"),
|
|
3713
|
+
Index("idx_drift_monitors_current", "current_source_id"),
|
|
3714
|
+
)
|
|
3715
|
+
|
|
3716
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
3717
|
+
baseline_source_id: Mapped[str] = mapped_column(
|
|
3718
|
+
String(36),
|
|
3719
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
3720
|
+
nullable=False,
|
|
3721
|
+
index=True,
|
|
3722
|
+
)
|
|
3723
|
+
current_source_id: Mapped[str] = mapped_column(
|
|
3724
|
+
String(36),
|
|
3725
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
3726
|
+
nullable=False,
|
|
3727
|
+
index=True,
|
|
3728
|
+
)
|
|
3729
|
+
status: Mapped[str] = mapped_column(
|
|
3730
|
+
String(20),
|
|
3731
|
+
nullable=False,
|
|
3732
|
+
default=DriftMonitorStatus.ACTIVE.value,
|
|
3733
|
+
index=True,
|
|
3734
|
+
)
|
|
3735
|
+
method: Mapped[str] = mapped_column(String(30), nullable=False, default="auto")
|
|
3736
|
+
threshold_critical: Mapped[float] = mapped_column(Float, default=0.3, nullable=False)
|
|
3737
|
+
threshold_high: Mapped[float] = mapped_column(Float, default=0.1, nullable=False)
|
|
3738
|
+
columns: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
3739
|
+
schedule_cron: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
|
3740
|
+
last_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3741
|
+
next_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3742
|
+
config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
3743
|
+
|
|
3744
|
+
# Relationships
|
|
3745
|
+
baseline_source: Mapped[Source] = relationship(
|
|
3746
|
+
"Source",
|
|
3747
|
+
foreign_keys=[baseline_source_id],
|
|
3748
|
+
lazy="selectin",
|
|
3749
|
+
)
|
|
3750
|
+
current_source: Mapped[Source] = relationship(
|
|
3751
|
+
"Source",
|
|
3752
|
+
foreign_keys=[current_source_id],
|
|
3753
|
+
lazy="selectin",
|
|
3754
|
+
)
|
|
3755
|
+
runs: Mapped[list[DriftMonitorRun]] = relationship(
|
|
3756
|
+
"DriftMonitorRun",
|
|
3757
|
+
back_populates="monitor",
|
|
3758
|
+
cascade="all, delete-orphan",
|
|
3759
|
+
lazy="selectin",
|
|
3760
|
+
order_by="desc(DriftMonitorRun.created_at)",
|
|
3761
|
+
)
|
|
3762
|
+
alerts: Mapped[list[DriftAlert]] = relationship(
|
|
3763
|
+
"DriftAlert",
|
|
3764
|
+
back_populates="monitor",
|
|
3765
|
+
cascade="all, delete-orphan",
|
|
3766
|
+
lazy="selectin",
|
|
3767
|
+
)
|
|
3768
|
+
|
|
3769
|
+
@property
|
|
3770
|
+
def is_active(self) -> bool:
|
|
3771
|
+
"""Check if monitor is active."""
|
|
3772
|
+
return self.status == DriftMonitorStatus.ACTIVE.value
|
|
3773
|
+
|
|
3774
|
+
@property
|
|
3775
|
+
def latest_run(self) -> DriftMonitorRun | None:
|
|
3776
|
+
"""Get the most recent run."""
|
|
3777
|
+
return self.runs[0] if self.runs else None
|
|
3778
|
+
|
|
3779
|
+
def pause(self) -> None:
|
|
3780
|
+
"""Pause this monitor."""
|
|
3781
|
+
self.status = DriftMonitorStatus.PAUSED.value
|
|
3782
|
+
|
|
3783
|
+
def resume(self) -> None:
|
|
3784
|
+
"""Resume this monitor."""
|
|
3785
|
+
self.status = DriftMonitorStatus.ACTIVE.value
|
|
3786
|
+
|
|
3787
|
+
def mark_run(self, next_run: datetime | None = None) -> None:
|
|
3788
|
+
"""Mark as run and update next run time."""
|
|
3789
|
+
self.last_run_at = datetime.utcnow()
|
|
3790
|
+
self.next_run_at = next_run
|
|
3791
|
+
|
|
3792
|
+
|
|
3793
|
+
class DriftMonitorRun(Base, UUIDMixin):
|
|
3794
|
+
"""Drift monitor execution record.
|
|
3795
|
+
|
|
3796
|
+
Stores results from a drift monitor run.
|
|
3797
|
+
|
|
3798
|
+
Attributes:
|
|
3799
|
+
id: Unique identifier (UUID).
|
|
3800
|
+
monitor_id: Reference to DriftMonitor.
|
|
3801
|
+
status: Run status.
|
|
3802
|
+
has_drift: Whether drift was detected.
|
|
3803
|
+
max_drift_score: Maximum drift score across columns.
|
|
3804
|
+
total_columns: Total columns compared.
|
|
3805
|
+
drifted_columns: Number of columns with drift.
|
|
3806
|
+
column_results: JSON of per-column results.
|
|
3807
|
+
root_cause_analysis: JSON of root cause analysis.
|
|
3808
|
+
duration_ms: Run duration in milliseconds.
|
|
3809
|
+
error_message: Error message if failed.
|
|
3810
|
+
created_at: When run started.
|
|
3811
|
+
completed_at: When run completed.
|
|
3812
|
+
"""
|
|
3813
|
+
|
|
3814
|
+
__tablename__ = "drift_monitor_runs"
|
|
3815
|
+
|
|
3816
|
+
__table_args__ = (
|
|
3817
|
+
Index("idx_drift_monitor_runs_monitor", "monitor_id", "created_at"),
|
|
3818
|
+
Index("idx_drift_monitor_runs_status", "status"),
|
|
3819
|
+
)
|
|
3820
|
+
|
|
3821
|
+
monitor_id: Mapped[str] = mapped_column(
|
|
3822
|
+
String(36),
|
|
3823
|
+
ForeignKey("drift_monitors.id", ondelete="CASCADE"),
|
|
3824
|
+
nullable=False,
|
|
3825
|
+
index=True,
|
|
3826
|
+
)
|
|
3827
|
+
status: Mapped[str] = mapped_column(
|
|
3828
|
+
String(20),
|
|
3829
|
+
nullable=False,
|
|
3830
|
+
default="pending",
|
|
3831
|
+
index=True,
|
|
3832
|
+
)
|
|
3833
|
+
has_drift: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
3834
|
+
max_drift_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
3835
|
+
total_columns: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
3836
|
+
drifted_columns: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
3837
|
+
column_results: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
3838
|
+
root_cause_analysis: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
3839
|
+
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
3840
|
+
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
3841
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
3842
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
3843
|
+
)
|
|
3844
|
+
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3845
|
+
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3846
|
+
|
|
3847
|
+
# Relationships
|
|
3848
|
+
monitor: Mapped[DriftMonitor] = relationship(
|
|
3849
|
+
"DriftMonitor",
|
|
3850
|
+
back_populates="runs",
|
|
3851
|
+
)
|
|
3852
|
+
|
|
3853
|
+
@property
|
|
3854
|
+
def is_complete(self) -> bool:
|
|
3855
|
+
"""Check if run has completed."""
|
|
3856
|
+
return self.status in ("success", "failed", "error")
|
|
3857
|
+
|
|
3858
|
+
@property
|
|
3859
|
+
def drift_percentage(self) -> float:
|
|
3860
|
+
"""Calculate percentage of columns with drift."""
|
|
3861
|
+
if self.total_columns and self.total_columns > 0:
|
|
3862
|
+
return (self.drifted_columns or 0) / self.total_columns * 100
|
|
3863
|
+
return 0.0
|
|
3864
|
+
|
|
3865
|
+
def mark_started(self) -> None:
|
|
3866
|
+
"""Mark run as started."""
|
|
3867
|
+
self.status = "running"
|
|
3868
|
+
self.started_at = datetime.utcnow()
|
|
3869
|
+
|
|
3870
|
+
def mark_completed(
|
|
3871
|
+
self,
|
|
3872
|
+
has_drift: bool,
|
|
3873
|
+
max_drift_score: float,
|
|
3874
|
+
total_columns: int,
|
|
3875
|
+
drifted_columns: int,
|
|
3876
|
+
column_results: dict[str, Any],
|
|
3877
|
+
root_cause_analysis: dict[str, Any] | None = None,
|
|
3878
|
+
) -> None:
|
|
3879
|
+
"""Mark run as completed with results."""
|
|
3880
|
+
self.status = "success"
|
|
3881
|
+
self.has_drift = has_drift
|
|
3882
|
+
self.max_drift_score = max_drift_score
|
|
3883
|
+
self.total_columns = total_columns
|
|
3884
|
+
self.drifted_columns = drifted_columns
|
|
3885
|
+
self.column_results = column_results
|
|
3886
|
+
self.root_cause_analysis = root_cause_analysis
|
|
3887
|
+
self.completed_at = datetime.utcnow()
|
|
3888
|
+
|
|
3889
|
+
if self.started_at:
|
|
3890
|
+
delta = self.completed_at - self.started_at
|
|
3891
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
3892
|
+
|
|
3893
|
+
def mark_error(self, message: str) -> None:
|
|
3894
|
+
"""Mark run as errored."""
|
|
3895
|
+
self.status = "error"
|
|
3896
|
+
self.error_message = message
|
|
3897
|
+
self.completed_at = datetime.utcnow()
|
|
3898
|
+
|
|
3899
|
+
if self.started_at:
|
|
3900
|
+
delta = self.completed_at - self.started_at
|
|
3901
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
3902
|
+
|
|
3903
|
+
|
|
3904
|
+
class DriftAlert(Base, UUIDMixin):
|
|
3905
|
+
"""Drift alert model.
|
|
3906
|
+
|
|
3907
|
+
Represents an alert triggered by drift detection.
|
|
3908
|
+
|
|
3909
|
+
Attributes:
|
|
3910
|
+
id: Unique identifier (UUID).
|
|
3911
|
+
monitor_id: Reference to DriftMonitor.
|
|
3912
|
+
run_id: Reference to DriftMonitorRun.
|
|
3913
|
+
severity: Alert severity (critical, high, medium, low).
|
|
3914
|
+
status: Alert status.
|
|
3915
|
+
drift_score: Drift score that triggered alert.
|
|
3916
|
+
affected_columns: List of affected columns.
|
|
3917
|
+
message: Alert message.
|
|
3918
|
+
acknowledged_by: Who acknowledged.
|
|
3919
|
+
acknowledged_at: When acknowledged.
|
|
3920
|
+
resolved_at: When resolved.
|
|
3921
|
+
created_at: When alert was created.
|
|
3922
|
+
"""
|
|
3923
|
+
|
|
3924
|
+
__tablename__ = "drift_alerts"
|
|
3925
|
+
|
|
3926
|
+
__table_args__ = (
|
|
3927
|
+
Index("idx_drift_alerts_monitor", "monitor_id", "created_at"),
|
|
3928
|
+
Index("idx_drift_alerts_status", "status"),
|
|
3929
|
+
)
|
|
3930
|
+
|
|
3931
|
+
monitor_id: Mapped[str] = mapped_column(
|
|
3932
|
+
String(36),
|
|
3933
|
+
ForeignKey("drift_monitors.id", ondelete="CASCADE"),
|
|
3934
|
+
nullable=False,
|
|
3935
|
+
index=True,
|
|
3936
|
+
)
|
|
3937
|
+
run_id: Mapped[str | None] = mapped_column(
|
|
3938
|
+
String(36),
|
|
3939
|
+
ForeignKey("drift_monitor_runs.id", ondelete="SET NULL"),
|
|
3940
|
+
nullable=True,
|
|
3941
|
+
)
|
|
3942
|
+
severity: Mapped[str] = mapped_column(String(20), nullable=False, default="high")
|
|
3943
|
+
status: Mapped[str] = mapped_column(
|
|
3944
|
+
String(20),
|
|
3945
|
+
nullable=False,
|
|
3946
|
+
default=DriftAlertStatus.ACTIVE.value,
|
|
3947
|
+
index=True,
|
|
3948
|
+
)
|
|
3949
|
+
drift_score: Mapped[float] = mapped_column(Float, nullable=False)
|
|
3950
|
+
affected_columns: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
3951
|
+
message: Mapped[str] = mapped_column(Text, nullable=False)
|
|
3952
|
+
acknowledged_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
3953
|
+
acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3954
|
+
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
3955
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
3956
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
3957
|
+
)
|
|
3958
|
+
|
|
3959
|
+
# Relationships
|
|
3960
|
+
monitor: Mapped[DriftMonitor] = relationship(
|
|
3961
|
+
"DriftMonitor",
|
|
3962
|
+
back_populates="alerts",
|
|
3963
|
+
)
|
|
3964
|
+
run: Mapped[DriftMonitorRun | None] = relationship(
|
|
3965
|
+
"DriftMonitorRun",
|
|
3966
|
+
lazy="selectin",
|
|
3967
|
+
)
|
|
3968
|
+
|
|
3969
|
+
@property
|
|
3970
|
+
def is_active(self) -> bool:
|
|
3971
|
+
"""Check if alert is active."""
|
|
3972
|
+
return self.status == DriftAlertStatus.ACTIVE.value
|
|
3973
|
+
|
|
3974
|
+
def acknowledge(self, by: str) -> None:
|
|
3975
|
+
"""Acknowledge this alert."""
|
|
3976
|
+
self.status = DriftAlertStatus.ACKNOWLEDGED.value
|
|
3977
|
+
self.acknowledged_by = by
|
|
3978
|
+
self.acknowledged_at = datetime.utcnow()
|
|
3979
|
+
|
|
3980
|
+
def resolve(self) -> None:
|
|
3981
|
+
"""Resolve this alert."""
|
|
3982
|
+
self.status = DriftAlertStatus.RESOLVED.value
|
|
3983
|
+
self.resolved_at = datetime.utcnow()
|
|
3984
|
+
|
|
3985
|
+
|
|
3986
|
+
# =============================================================================
|
|
3987
|
+
# Phase 9: Plugin System Models
|
|
3988
|
+
# =============================================================================
|
|
3989
|
+
|
|
3990
|
+
|
|
3991
|
+
class PluginType(str, Enum):
|
|
3992
|
+
"""Type of plugin."""
|
|
3993
|
+
|
|
3994
|
+
VALIDATOR = "validator"
|
|
3995
|
+
REPORTER = "reporter"
|
|
3996
|
+
CONNECTOR = "connector"
|
|
3997
|
+
TRANSFORMER = "transformer"
|
|
3998
|
+
|
|
3999
|
+
|
|
4000
|
+
class PluginStatus(str, Enum):
|
|
4001
|
+
"""Installation status of a plugin."""
|
|
4002
|
+
|
|
4003
|
+
AVAILABLE = "available"
|
|
4004
|
+
INSTALLED = "installed"
|
|
4005
|
+
ENABLED = "enabled"
|
|
4006
|
+
DISABLED = "disabled"
|
|
4007
|
+
UPDATE_AVAILABLE = "update_available"
|
|
4008
|
+
ERROR = "error"
|
|
4009
|
+
|
|
4010
|
+
|
|
4011
|
+
class PluginSource(str, Enum):
|
|
4012
|
+
"""Source of the plugin."""
|
|
4013
|
+
|
|
4014
|
+
OFFICIAL = "official"
|
|
4015
|
+
COMMUNITY = "community"
|
|
4016
|
+
LOCAL = "local"
|
|
4017
|
+
PRIVATE = "private"
|
|
4018
|
+
|
|
4019
|
+
|
|
4020
|
+
class SecurityLevel(str, Enum):
|
|
4021
|
+
"""Security level of the plugin."""
|
|
4022
|
+
|
|
4023
|
+
TRUSTED = "trusted"
|
|
4024
|
+
VERIFIED = "verified"
|
|
4025
|
+
UNVERIFIED = "unverified"
|
|
4026
|
+
SANDBOXED = "sandboxed"
|
|
4027
|
+
|
|
4028
|
+
|
|
4029
|
+
class Plugin(Base, UUIDMixin, TimestampMixin):
|
|
4030
|
+
"""Plugin model.
|
|
4031
|
+
|
|
4032
|
+
Represents an installable plugin in the plugin marketplace.
|
|
4033
|
+
|
|
4034
|
+
Attributes:
|
|
4035
|
+
id: Unique identifier (UUID).
|
|
4036
|
+
name: Plugin name (unique identifier).
|
|
4037
|
+
display_name: Human-readable display name.
|
|
4038
|
+
description: Plugin description.
|
|
4039
|
+
version: Current/installed version.
|
|
4040
|
+
latest_version: Latest available version.
|
|
4041
|
+
type: Plugin type (validator, reporter, connector, transformer).
|
|
4042
|
+
source: Plugin source (official, community, local, private).
|
|
4043
|
+
status: Installation status.
|
|
4044
|
+
security_level: Security verification level.
|
|
4045
|
+
author: Author information (JSON).
|
|
4046
|
+
license: License identifier.
|
|
4047
|
+
homepage: Plugin homepage URL.
|
|
4048
|
+
repository: Repository URL.
|
|
4049
|
+
keywords: Search keywords (JSON array).
|
|
4050
|
+
categories: Plugin categories (JSON array).
|
|
4051
|
+
dependencies: Plugin dependencies (JSON array).
|
|
4052
|
+
permissions: Required permissions (JSON array).
|
|
4053
|
+
python_version: Required Python version.
|
|
4054
|
+
dashboard_version: Required dashboard version.
|
|
4055
|
+
icon_url: Plugin icon URL.
|
|
4056
|
+
banner_url: Plugin banner URL.
|
|
4057
|
+
documentation_url: Documentation URL.
|
|
4058
|
+
changelog: Changelog markdown.
|
|
4059
|
+
readme: README markdown.
|
|
4060
|
+
package_url: URL to download plugin package.
|
|
4061
|
+
signature: Plugin signature info (JSON).
|
|
4062
|
+
sandbox_config: Sandbox configuration (JSON).
|
|
4063
|
+
is_enabled: Whether plugin is enabled.
|
|
4064
|
+
install_count: Total installation count.
|
|
4065
|
+
rating: Average rating (1-5).
|
|
4066
|
+
rating_count: Number of ratings.
|
|
4067
|
+
validators_count: Number of validators provided.
|
|
4068
|
+
reporters_count: Number of reporters provided.
|
|
4069
|
+
installed_at: When plugin was installed.
|
|
4070
|
+
last_updated: When plugin was last updated.
|
|
4071
|
+
"""
|
|
4072
|
+
|
|
4073
|
+
__tablename__ = "plugins"
|
|
4074
|
+
|
|
4075
|
+
__table_args__ = (
|
|
4076
|
+
Index("idx_plugins_name", "name", unique=True),
|
|
4077
|
+
Index("idx_plugins_type", "type"),
|
|
4078
|
+
Index("idx_plugins_source", "source"),
|
|
4079
|
+
Index("idx_plugins_status", "status"),
|
|
4080
|
+
)
|
|
4081
|
+
|
|
4082
|
+
name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
|
|
4083
|
+
display_name: Mapped[str] = mapped_column(String(200), nullable=False)
|
|
4084
|
+
description: Mapped[str] = mapped_column(Text, nullable=False)
|
|
4085
|
+
version: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
4086
|
+
latest_version: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
|
4087
|
+
type: Mapped[str] = mapped_column(
|
|
4088
|
+
SQLEnum(PluginType, native_enum=False, length=20),
|
|
4089
|
+
nullable=False,
|
|
4090
|
+
index=True,
|
|
4091
|
+
)
|
|
4092
|
+
source: Mapped[str] = mapped_column(
|
|
4093
|
+
SQLEnum(PluginSource, native_enum=False, length=20),
|
|
4094
|
+
nullable=False,
|
|
4095
|
+
default=PluginSource.COMMUNITY.value,
|
|
4096
|
+
)
|
|
4097
|
+
status: Mapped[str] = mapped_column(
|
|
4098
|
+
SQLEnum(PluginStatus, native_enum=False, length=20),
|
|
4099
|
+
nullable=False,
|
|
4100
|
+
default=PluginStatus.AVAILABLE.value,
|
|
4101
|
+
index=True,
|
|
4102
|
+
)
|
|
4103
|
+
security_level: Mapped[str] = mapped_column(
|
|
4104
|
+
SQLEnum(SecurityLevel, native_enum=False, length=20),
|
|
4105
|
+
nullable=False,
|
|
4106
|
+
default=SecurityLevel.UNVERIFIED.value,
|
|
4107
|
+
)
|
|
4108
|
+
author: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
4109
|
+
license: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
|
4110
|
+
homepage: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4111
|
+
repository: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4112
|
+
keywords: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
4113
|
+
categories: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
4114
|
+
dependencies: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
|
|
4115
|
+
permissions: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
4116
|
+
python_version: Mapped[str | None] = mapped_column(String(20), nullable=True)
|
|
4117
|
+
dashboard_version: Mapped[str | None] = mapped_column(String(20), nullable=True)
|
|
4118
|
+
icon_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4119
|
+
banner_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4120
|
+
documentation_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4121
|
+
changelog: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4122
|
+
readme: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4123
|
+
package_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4124
|
+
signature: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
4125
|
+
sandbox_config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
4126
|
+
is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4127
|
+
install_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4128
|
+
rating: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
4129
|
+
rating_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4130
|
+
validators_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4131
|
+
reporters_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4132
|
+
installed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4133
|
+
last_updated: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4134
|
+
|
|
4135
|
+
# Relationships
|
|
4136
|
+
custom_validators: Mapped[list["CustomValidator"]] = relationship(
|
|
4137
|
+
"CustomValidator",
|
|
4138
|
+
back_populates="plugin",
|
|
4139
|
+
cascade="all, delete-orphan",
|
|
4140
|
+
lazy="selectin",
|
|
4141
|
+
)
|
|
4142
|
+
custom_reporters: Mapped[list["CustomReporter"]] = relationship(
|
|
4143
|
+
"CustomReporter",
|
|
4144
|
+
back_populates="plugin",
|
|
4145
|
+
cascade="all, delete-orphan",
|
|
4146
|
+
lazy="selectin",
|
|
4147
|
+
)
|
|
4148
|
+
ratings: Mapped[list["PluginRating"]] = relationship(
|
|
4149
|
+
"PluginRating",
|
|
4150
|
+
back_populates="plugin",
|
|
4151
|
+
cascade="all, delete-orphan",
|
|
4152
|
+
lazy="selectin",
|
|
4153
|
+
)
|
|
4154
|
+
signatures: Mapped[list["PluginSignature"]] = relationship(
|
|
4155
|
+
"PluginSignature",
|
|
4156
|
+
back_populates="plugin",
|
|
4157
|
+
cascade="all, delete-orphan",
|
|
4158
|
+
lazy="selectin",
|
|
4159
|
+
)
|
|
4160
|
+
hot_reload_config: Mapped["HotReloadConfig | None"] = relationship(
|
|
4161
|
+
"HotReloadConfig",
|
|
4162
|
+
back_populates="plugin",
|
|
4163
|
+
cascade="all, delete-orphan",
|
|
4164
|
+
uselist=False,
|
|
4165
|
+
lazy="selectin",
|
|
4166
|
+
)
|
|
4167
|
+
hooks: Mapped[list["PluginHook"]] = relationship(
|
|
4168
|
+
"PluginHook",
|
|
4169
|
+
back_populates="plugin",
|
|
4170
|
+
cascade="all, delete-orphan",
|
|
4171
|
+
lazy="selectin",
|
|
4172
|
+
)
|
|
4173
|
+
|
|
4174
|
+
def install(self) -> None:
|
|
4175
|
+
"""Mark plugin as installed."""
|
|
4176
|
+
self.status = PluginStatus.INSTALLED.value
|
|
4177
|
+
self.installed_at = datetime.utcnow()
|
|
4178
|
+
self.install_count += 1
|
|
4179
|
+
|
|
4180
|
+
def enable(self) -> None:
|
|
4181
|
+
"""Enable the plugin."""
|
|
4182
|
+
self.status = PluginStatus.ENABLED.value
|
|
4183
|
+
self.is_enabled = True
|
|
4184
|
+
|
|
4185
|
+
def disable(self) -> None:
|
|
4186
|
+
"""Disable the plugin."""
|
|
4187
|
+
self.status = PluginStatus.DISABLED.value
|
|
4188
|
+
self.is_enabled = False
|
|
4189
|
+
|
|
4190
|
+
def uninstall(self) -> None:
|
|
4191
|
+
"""Mark plugin as available (uninstalled)."""
|
|
4192
|
+
self.status = PluginStatus.AVAILABLE.value
|
|
4193
|
+
self.is_enabled = False
|
|
4194
|
+
self.installed_at = None
|
|
4195
|
+
|
|
4196
|
+
def update_rating(self, new_rating: float) -> None:
|
|
4197
|
+
"""Update average rating with a new rating."""
|
|
4198
|
+
if self.rating is None:
|
|
4199
|
+
self.rating = new_rating
|
|
4200
|
+
else:
|
|
4201
|
+
total = self.rating * self.rating_count + new_rating
|
|
4202
|
+
self.rating = total / (self.rating_count + 1)
|
|
4203
|
+
self.rating_count += 1
|
|
4204
|
+
|
|
4205
|
+
|
|
4206
|
+
class CustomValidator(Base, UUIDMixin, TimestampMixin):
|
|
4207
|
+
"""Custom validator model.
|
|
4208
|
+
|
|
4209
|
+
Represents a user-defined validator that can be used for data validation.
|
|
4210
|
+
|
|
4211
|
+
Attributes:
|
|
4212
|
+
id: Unique identifier (UUID).
|
|
4213
|
+
plugin_id: Optional reference to parent plugin.
|
|
4214
|
+
name: Validator name (unique).
|
|
4215
|
+
display_name: Human-readable display name.
|
|
4216
|
+
description: Validator description.
|
|
4217
|
+
category: Validator category.
|
|
4218
|
+
severity: Default severity (error, warning, info).
|
|
4219
|
+
tags: Search/filter tags (JSON array).
|
|
4220
|
+
parameters: Parameter definitions (JSON array).
|
|
4221
|
+
code: Python code implementing the validator.
|
|
4222
|
+
test_cases: Test cases for validation (JSON array).
|
|
4223
|
+
is_enabled: Whether validator is enabled.
|
|
4224
|
+
is_verified: Whether validator is security-verified.
|
|
4225
|
+
usage_count: Number of times validator has been used.
|
|
4226
|
+
last_used_at: When validator was last used.
|
|
4227
|
+
"""
|
|
4228
|
+
|
|
4229
|
+
__tablename__ = "custom_validators"
|
|
4230
|
+
|
|
4231
|
+
__table_args__ = (
|
|
4232
|
+
Index("idx_custom_validators_name", "name", unique=True),
|
|
4233
|
+
Index("idx_custom_validators_plugin", "plugin_id"),
|
|
4234
|
+
Index("idx_custom_validators_category", "category"),
|
|
4235
|
+
)
|
|
4236
|
+
|
|
4237
|
+
plugin_id: Mapped[str | None] = mapped_column(
|
|
4238
|
+
String(36),
|
|
4239
|
+
ForeignKey("plugins.id", ondelete="CASCADE"),
|
|
4240
|
+
nullable=True,
|
|
4241
|
+
index=True,
|
|
4242
|
+
)
|
|
4243
|
+
name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
|
|
4244
|
+
display_name: Mapped[str] = mapped_column(String(200), nullable=False)
|
|
4245
|
+
description: Mapped[str] = mapped_column(Text, nullable=False)
|
|
4246
|
+
category: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
|
4247
|
+
severity: Mapped[str] = mapped_column(String(20), nullable=False, default="error")
|
|
4248
|
+
tags: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
4249
|
+
parameters: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
|
|
4250
|
+
code: Mapped[str] = mapped_column(Text, nullable=False)
|
|
4251
|
+
test_cases: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
|
|
4252
|
+
is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
4253
|
+
is_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4254
|
+
usage_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4255
|
+
last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4256
|
+
|
|
4257
|
+
# Relationships
|
|
4258
|
+
plugin: Mapped[Plugin | None] = relationship(
|
|
4259
|
+
"Plugin",
|
|
4260
|
+
back_populates="custom_validators",
|
|
4261
|
+
)
|
|
4262
|
+
|
|
4263
|
+
def increment_usage(self) -> None:
|
|
4264
|
+
"""Increment usage count and update last used timestamp."""
|
|
4265
|
+
self.usage_count += 1
|
|
4266
|
+
self.last_used_at = datetime.utcnow()
|
|
4267
|
+
|
|
4268
|
+
|
|
4269
|
+
class CustomReporter(Base, UUIDMixin, TimestampMixin):
|
|
4270
|
+
"""Custom reporter model.
|
|
4271
|
+
|
|
4272
|
+
Represents a user-defined reporter for generating custom reports.
|
|
4273
|
+
|
|
4274
|
+
Attributes:
|
|
4275
|
+
id: Unique identifier (UUID).
|
|
4276
|
+
plugin_id: Optional reference to parent plugin.
|
|
4277
|
+
name: Reporter name (unique).
|
|
4278
|
+
display_name: Human-readable display name.
|
|
4279
|
+
description: Reporter description.
|
|
4280
|
+
output_formats: Supported output formats (JSON array).
|
|
4281
|
+
config_fields: Configuration field definitions (JSON array).
|
|
4282
|
+
template: Jinja2 template for HTML/text reports.
|
|
4283
|
+
code: Python code for custom report generation.
|
|
4284
|
+
preview_image_url: Preview image URL.
|
|
4285
|
+
is_enabled: Whether reporter is enabled.
|
|
4286
|
+
is_verified: Whether reporter is security-verified.
|
|
4287
|
+
usage_count: Number of times reporter has been used.
|
|
4288
|
+
"""
|
|
4289
|
+
|
|
4290
|
+
__tablename__ = "custom_reporters"
|
|
4291
|
+
|
|
4292
|
+
__table_args__ = (
|
|
4293
|
+
Index("idx_custom_reporters_name", "name", unique=True),
|
|
4294
|
+
Index("idx_custom_reporters_plugin", "plugin_id"),
|
|
4295
|
+
)
|
|
4296
|
+
|
|
4297
|
+
plugin_id: Mapped[str | None] = mapped_column(
|
|
4298
|
+
String(36),
|
|
4299
|
+
ForeignKey("plugins.id", ondelete="CASCADE"),
|
|
4300
|
+
nullable=True,
|
|
4301
|
+
index=True,
|
|
4302
|
+
)
|
|
4303
|
+
name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
|
|
4304
|
+
display_name: Mapped[str] = mapped_column(String(200), nullable=False)
|
|
4305
|
+
description: Mapped[str] = mapped_column(Text, nullable=False)
|
|
4306
|
+
output_formats: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
4307
|
+
config_fields: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
|
|
4308
|
+
template: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4309
|
+
code: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4310
|
+
preview_image_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4311
|
+
is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
4312
|
+
is_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4313
|
+
usage_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4314
|
+
|
|
4315
|
+
# Relationships
|
|
4316
|
+
plugin: Mapped[Plugin | None] = relationship(
|
|
4317
|
+
"Plugin",
|
|
4318
|
+
back_populates="custom_reporters",
|
|
4319
|
+
)
|
|
4320
|
+
|
|
4321
|
+
def increment_usage(self) -> None:
|
|
4322
|
+
"""Increment usage count."""
|
|
4323
|
+
self.usage_count += 1
|
|
4324
|
+
|
|
4325
|
+
|
|
4326
|
+
class ReportFormatType(str, Enum):
|
|
4327
|
+
"""Report output format types."""
|
|
4328
|
+
|
|
4329
|
+
HTML = "html"
|
|
4330
|
+
PDF = "pdf"
|
|
4331
|
+
CSV = "csv"
|
|
4332
|
+
JSON = "json"
|
|
4333
|
+
MARKDOWN = "markdown"
|
|
4334
|
+
JUNIT = "junit"
|
|
4335
|
+
EXCEL = "excel"
|
|
4336
|
+
CUSTOM = "custom"
|
|
4337
|
+
|
|
4338
|
+
|
|
4339
|
+
class ReportStatus(str, Enum):
|
|
4340
|
+
"""Status of report generation."""
|
|
4341
|
+
|
|
4342
|
+
PENDING = "pending"
|
|
4343
|
+
GENERATING = "generating"
|
|
4344
|
+
COMPLETED = "completed"
|
|
4345
|
+
FAILED = "failed"
|
|
4346
|
+
EXPIRED = "expired"
|
|
4347
|
+
|
|
4348
|
+
|
|
4349
|
+
class GeneratedReport(Base, UUIDMixin, TimestampMixin):
|
|
4350
|
+
"""Generated report history model.
|
|
4351
|
+
|
|
4352
|
+
Stores metadata and content of generated reports for history tracking,
|
|
4353
|
+
caching, and audit purposes.
|
|
4354
|
+
|
|
4355
|
+
Attributes:
|
|
4356
|
+
id: Unique identifier (UUID).
|
|
4357
|
+
validation_id: Reference to the validation run.
|
|
4358
|
+
source_id: Reference to the data source.
|
|
4359
|
+
reporter_id: Optional reference to custom reporter used.
|
|
4360
|
+
name: Human-readable report name.
|
|
4361
|
+
description: Optional description.
|
|
4362
|
+
format: Report output format (html, pdf, csv, etc.).
|
|
4363
|
+
theme: Theme used for HTML/PDF reports.
|
|
4364
|
+
locale: Language locale used.
|
|
4365
|
+
status: Generation status.
|
|
4366
|
+
file_path: Path to stored report file (if persisted).
|
|
4367
|
+
file_size: Size of the report file in bytes.
|
|
4368
|
+
content_hash: Hash of report content for deduplication.
|
|
4369
|
+
config: Configuration used for generation (JSON).
|
|
4370
|
+
metadata: Additional metadata (JSON).
|
|
4371
|
+
error_message: Error message if generation failed.
|
|
4372
|
+
generation_time_ms: Time taken to generate in milliseconds.
|
|
4373
|
+
expires_at: When the report expires and can be cleaned up.
|
|
4374
|
+
downloaded_count: Number of times the report was downloaded.
|
|
4375
|
+
last_downloaded_at: Last download timestamp.
|
|
4376
|
+
"""
|
|
4377
|
+
|
|
4378
|
+
__tablename__ = "generated_reports"
|
|
4379
|
+
|
|
4380
|
+
__table_args__ = (
|
|
4381
|
+
Index("idx_generated_reports_validation", "validation_id"),
|
|
4382
|
+
Index("idx_generated_reports_source", "source_id"),
|
|
4383
|
+
Index("idx_generated_reports_reporter", "reporter_id"),
|
|
4384
|
+
Index("idx_generated_reports_status", "status"),
|
|
4385
|
+
Index("idx_generated_reports_format", "format"),
|
|
4386
|
+
Index("idx_generated_reports_created", "created_at"),
|
|
4387
|
+
Index("idx_generated_reports_expires", "expires_at"),
|
|
4388
|
+
)
|
|
4389
|
+
|
|
4390
|
+
validation_id: Mapped[str | None] = mapped_column(
|
|
4391
|
+
String(36),
|
|
4392
|
+
ForeignKey("validations.id", ondelete="SET NULL"),
|
|
4393
|
+
nullable=True,
|
|
4394
|
+
index=True,
|
|
4395
|
+
)
|
|
4396
|
+
source_id: Mapped[str | None] = mapped_column(
|
|
4397
|
+
String(36),
|
|
4398
|
+
ForeignKey("sources.id", ondelete="SET NULL"),
|
|
4399
|
+
nullable=True,
|
|
4400
|
+
index=True,
|
|
4401
|
+
)
|
|
4402
|
+
reporter_id: Mapped[str | None] = mapped_column(
|
|
4403
|
+
String(36),
|
|
4404
|
+
ForeignKey("custom_reporters.id", ondelete="SET NULL"),
|
|
4405
|
+
nullable=True,
|
|
4406
|
+
index=True,
|
|
4407
|
+
)
|
|
4408
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
4409
|
+
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4410
|
+
format: Mapped[str] = mapped_column(
|
|
4411
|
+
SQLEnum(ReportFormatType), nullable=False, default=ReportFormatType.HTML
|
|
4412
|
+
)
|
|
4413
|
+
theme: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
|
4414
|
+
locale: Mapped[str] = mapped_column(String(10), nullable=False, default="en")
|
|
4415
|
+
status: Mapped[str] = mapped_column(
|
|
4416
|
+
SQLEnum(ReportStatus), nullable=False, default=ReportStatus.PENDING
|
|
4417
|
+
)
|
|
4418
|
+
file_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
4419
|
+
file_size: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
4420
|
+
content_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
4421
|
+
config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
4422
|
+
report_metadata: Mapped[dict[str, Any] | None] = mapped_column(
|
|
4423
|
+
"metadata", JSON, nullable=True
|
|
4424
|
+
)
|
|
4425
|
+
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4426
|
+
generation_time_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
4427
|
+
expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4428
|
+
downloaded_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4429
|
+
last_downloaded_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4430
|
+
|
|
4431
|
+
# Relationships
|
|
4432
|
+
validation: Mapped["Validation | None"] = relationship(
|
|
4433
|
+
"Validation",
|
|
4434
|
+
back_populates="generated_reports",
|
|
4435
|
+
)
|
|
4436
|
+
source: Mapped["Source | None"] = relationship(
|
|
4437
|
+
"Source",
|
|
4438
|
+
back_populates="generated_reports",
|
|
4439
|
+
)
|
|
4440
|
+
reporter: Mapped[CustomReporter | None] = relationship(
|
|
4441
|
+
"CustomReporter",
|
|
4442
|
+
backref="generated_reports",
|
|
4443
|
+
)
|
|
4444
|
+
|
|
4445
|
+
def increment_download(self) -> None:
|
|
4446
|
+
"""Increment download count and update last download timestamp."""
|
|
4447
|
+
self.downloaded_count += 1
|
|
4448
|
+
self.last_downloaded_at = datetime.utcnow()
|
|
4449
|
+
|
|
4450
|
+
def mark_completed(self, file_path: str, file_size: int, generation_time_ms: float) -> None:
|
|
4451
|
+
"""Mark report as completed."""
|
|
4452
|
+
self.status = ReportStatus.COMPLETED
|
|
4453
|
+
self.file_path = file_path
|
|
4454
|
+
self.file_size = file_size
|
|
4455
|
+
self.generation_time_ms = generation_time_ms
|
|
4456
|
+
|
|
4457
|
+
def mark_failed(self, error_message: str) -> None:
|
|
4458
|
+
"""Mark report as failed."""
|
|
4459
|
+
self.status = ReportStatus.FAILED
|
|
4460
|
+
self.error_message = error_message
|
|
4461
|
+
|
|
4462
|
+
def is_expired(self) -> bool:
|
|
4463
|
+
"""Check if report has expired."""
|
|
4464
|
+
if self.expires_at is None:
|
|
4465
|
+
return False
|
|
4466
|
+
return datetime.utcnow() > self.expires_at
|
|
4467
|
+
|
|
4468
|
+
|
|
4469
|
+
class PluginRating(Base, UUIDMixin, TimestampMixin):
|
|
4470
|
+
"""Plugin rating model.
|
|
4471
|
+
|
|
4472
|
+
Stores user ratings and reviews for plugins.
|
|
4473
|
+
|
|
4474
|
+
Attributes:
|
|
4475
|
+
id: Unique identifier (UUID).
|
|
4476
|
+
plugin_id: Reference to Plugin.
|
|
4477
|
+
user_id: User identifier (could be from auth system).
|
|
4478
|
+
rating: Rating value (1-5).
|
|
4479
|
+
review: Optional review text.
|
|
4480
|
+
"""
|
|
4481
|
+
|
|
4482
|
+
__tablename__ = "plugin_ratings"
|
|
4483
|
+
|
|
4484
|
+
__table_args__ = (
|
|
4485
|
+
Index("idx_plugin_ratings_plugin", "plugin_id"),
|
|
4486
|
+
Index("idx_plugin_ratings_user", "user_id"),
|
|
4487
|
+
)
|
|
4488
|
+
|
|
4489
|
+
plugin_id: Mapped[str] = mapped_column(
|
|
4490
|
+
String(36),
|
|
4491
|
+
ForeignKey("plugins.id", ondelete="CASCADE"),
|
|
4492
|
+
nullable=False,
|
|
4493
|
+
index=True,
|
|
4494
|
+
)
|
|
4495
|
+
user_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
|
|
4496
|
+
rating: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
4497
|
+
review: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4498
|
+
|
|
4499
|
+
# Relationships
|
|
4500
|
+
plugin: Mapped[Plugin] = relationship(
|
|
4501
|
+
"Plugin",
|
|
4502
|
+
back_populates="ratings",
|
|
4503
|
+
)
|
|
4504
|
+
|
|
4505
|
+
|
|
4506
|
+
class PluginExecutionLog(Base, UUIDMixin):
|
|
4507
|
+
"""Plugin execution log model.
|
|
4508
|
+
|
|
4509
|
+
Tracks plugin executions for monitoring and debugging.
|
|
4510
|
+
|
|
4511
|
+
Attributes:
|
|
4512
|
+
id: Unique identifier (UUID).
|
|
4513
|
+
plugin_id: Reference to Plugin.
|
|
4514
|
+
validator_id: Optional reference to CustomValidator.
|
|
4515
|
+
reporter_id: Optional reference to CustomReporter.
|
|
4516
|
+
execution_id: Unique execution identifier.
|
|
4517
|
+
source_id: Optional reference to data source.
|
|
4518
|
+
status: Execution status (pending, running, success, failed, error).
|
|
4519
|
+
execution_time_ms: Execution duration in milliseconds.
|
|
4520
|
+
memory_used_mb: Memory used in MB.
|
|
4521
|
+
result: Execution result (JSON).
|
|
4522
|
+
error_message: Error message if failed.
|
|
4523
|
+
logs: Execution logs (JSON array).
|
|
4524
|
+
started_at: When execution started.
|
|
4525
|
+
completed_at: When execution completed.
|
|
4526
|
+
"""
|
|
4527
|
+
|
|
4528
|
+
__tablename__ = "plugin_execution_logs"
|
|
4529
|
+
|
|
4530
|
+
__table_args__ = (
|
|
4531
|
+
Index("idx_plugin_exec_logs_plugin", "plugin_id", "started_at"),
|
|
4532
|
+
Index("idx_plugin_exec_logs_status", "status"),
|
|
4533
|
+
)
|
|
4534
|
+
|
|
4535
|
+
plugin_id: Mapped[str] = mapped_column(
|
|
4536
|
+
String(36),
|
|
4537
|
+
ForeignKey("plugins.id", ondelete="CASCADE"),
|
|
4538
|
+
nullable=False,
|
|
4539
|
+
index=True,
|
|
4540
|
+
)
|
|
4541
|
+
validator_id: Mapped[str | None] = mapped_column(
|
|
4542
|
+
String(36),
|
|
4543
|
+
ForeignKey("custom_validators.id", ondelete="SET NULL"),
|
|
4544
|
+
nullable=True,
|
|
4545
|
+
)
|
|
4546
|
+
reporter_id: Mapped[str | None] = mapped_column(
|
|
4547
|
+
String(36),
|
|
4548
|
+
ForeignKey("custom_reporters.id", ondelete="SET NULL"),
|
|
4549
|
+
nullable=True,
|
|
4550
|
+
)
|
|
4551
|
+
execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
|
4552
|
+
source_id: Mapped[str | None] = mapped_column(
|
|
4553
|
+
String(36),
|
|
4554
|
+
ForeignKey("sources.id", ondelete="SET NULL"),
|
|
4555
|
+
nullable=True,
|
|
4556
|
+
)
|
|
4557
|
+
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")
|
|
4558
|
+
execution_time_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
4559
|
+
memory_used_mb: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
4560
|
+
result: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
4561
|
+
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4562
|
+
logs: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
4563
|
+
started_at: Mapped[datetime] = mapped_column(
|
|
4564
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
4565
|
+
)
|
|
4566
|
+
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4567
|
+
|
|
4568
|
+
def mark_started(self) -> None:
|
|
4569
|
+
"""Mark execution as started."""
|
|
4570
|
+
self.status = "running"
|
|
4571
|
+
self.started_at = datetime.utcnow()
|
|
4572
|
+
|
|
4573
|
+
def mark_completed(
|
|
4574
|
+
self,
|
|
4575
|
+
result: dict[str, Any] | None = None,
|
|
4576
|
+
memory_used_mb: float | None = None,
|
|
4577
|
+
) -> None:
|
|
4578
|
+
"""Mark execution as completed."""
|
|
4579
|
+
self.status = "success"
|
|
4580
|
+
self.result = result
|
|
4581
|
+
self.memory_used_mb = memory_used_mb
|
|
4582
|
+
self.completed_at = datetime.utcnow()
|
|
4583
|
+
if self.started_at:
|
|
4584
|
+
delta = self.completed_at - self.started_at
|
|
4585
|
+
self.execution_time_ms = int(delta.total_seconds() * 1000)
|
|
4586
|
+
|
|
4587
|
+
def mark_failed(self, error_message: str) -> None:
|
|
4588
|
+
"""Mark execution as failed."""
|
|
4589
|
+
self.status = "failed"
|
|
4590
|
+
self.error_message = error_message
|
|
4591
|
+
self.completed_at = datetime.utcnow()
|
|
4592
|
+
if self.started_at:
|
|
4593
|
+
delta = self.completed_at - self.started_at
|
|
4594
|
+
self.execution_time_ms = int(delta.total_seconds() * 1000)
|
|
4595
|
+
|
|
4596
|
+
|
|
4597
|
+
# =============================================================================
|
|
4598
|
+
# Phase 9: Trust Store and Security Models
|
|
4599
|
+
# =============================================================================
|
|
4600
|
+
|
|
4601
|
+
|
|
4602
|
+
class SignatureAlgorithmType(str, Enum):
|
|
4603
|
+
"""Supported signature algorithms."""
|
|
4604
|
+
|
|
4605
|
+
HMAC_SHA256 = "hmac_sha256"
|
|
4606
|
+
HMAC_SHA512 = "hmac_sha512"
|
|
4607
|
+
RSA_SHA256 = "rsa_sha256"
|
|
4608
|
+
ED25519 = "ed25519"
|
|
4609
|
+
|
|
4610
|
+
|
|
4611
|
+
class TrustLevelType(str, Enum):
|
|
4612
|
+
"""Trust levels for signers."""
|
|
4613
|
+
|
|
4614
|
+
TRUSTED = "trusted"
|
|
4615
|
+
VERIFIED = "verified"
|
|
4616
|
+
UNVERIFIED = "unverified"
|
|
4617
|
+
REVOKED = "revoked"
|
|
4618
|
+
|
|
4619
|
+
|
|
4620
|
+
class IsolationLevelType(str, Enum):
|
|
4621
|
+
"""Sandbox isolation levels."""
|
|
4622
|
+
|
|
4623
|
+
NONE = "none"
|
|
4624
|
+
PROCESS = "process"
|
|
4625
|
+
CONTAINER = "container"
|
|
4626
|
+
|
|
4627
|
+
|
|
4628
|
+
class TrustedSigner(Base, UUIDMixin, TimestampMixin):
|
|
4629
|
+
"""Trusted signer model for plugin signature verification.
|
|
4630
|
+
|
|
4631
|
+
Represents a trusted entity that can sign plugins.
|
|
4632
|
+
|
|
4633
|
+
Attributes:
|
|
4634
|
+
id: Unique identifier (UUID).
|
|
4635
|
+
signer_id: Unique signer identifier (e.g., email, domain).
|
|
4636
|
+
name: Display name of the signer.
|
|
4637
|
+
organization: Organization name.
|
|
4638
|
+
email: Contact email.
|
|
4639
|
+
public_key: PEM-encoded public key.
|
|
4640
|
+
algorithm: Signature algorithm used.
|
|
4641
|
+
fingerprint: Key fingerprint for quick identification.
|
|
4642
|
+
trust_level: Current trust level.
|
|
4643
|
+
plugins_signed: Count of plugins signed.
|
|
4644
|
+
expires_at: When the trust expires.
|
|
4645
|
+
revoked_at: When the signer was revoked.
|
|
4646
|
+
revocation_reason: Reason for revocation.
|
|
4647
|
+
metadata: Additional signer metadata (JSON).
|
|
4648
|
+
"""
|
|
4649
|
+
|
|
4650
|
+
__tablename__ = "trusted_signers"
|
|
4651
|
+
|
|
4652
|
+
__table_args__ = (
|
|
4653
|
+
Index("idx_trusted_signers_signer_id", "signer_id", unique=True),
|
|
4654
|
+
Index("idx_trusted_signers_fingerprint", "fingerprint"),
|
|
4655
|
+
Index("idx_trusted_signers_trust_level", "trust_level"),
|
|
4656
|
+
)
|
|
4657
|
+
|
|
4658
|
+
signer_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
|
4659
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
4660
|
+
organization: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
4661
|
+
email: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
4662
|
+
public_key: Mapped[str] = mapped_column(Text, nullable=False)
|
|
4663
|
+
algorithm: Mapped[str] = mapped_column(
|
|
4664
|
+
String(20),
|
|
4665
|
+
nullable=False,
|
|
4666
|
+
default=SignatureAlgorithmType.ED25519.value,
|
|
4667
|
+
)
|
|
4668
|
+
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
|
4669
|
+
trust_level: Mapped[str] = mapped_column(
|
|
4670
|
+
String(20),
|
|
4671
|
+
nullable=False,
|
|
4672
|
+
default=TrustLevelType.VERIFIED.value,
|
|
4673
|
+
)
|
|
4674
|
+
plugins_signed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4675
|
+
expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4676
|
+
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4677
|
+
revocation_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4678
|
+
signer_metadata: Mapped[dict[str, Any]] = mapped_column(
|
|
4679
|
+
"metadata", JSON, nullable=False, default=dict
|
|
4680
|
+
)
|
|
4681
|
+
|
|
4682
|
+
def is_valid(self) -> bool:
|
|
4683
|
+
"""Check if the signer is currently valid."""
|
|
4684
|
+
if self.trust_level == TrustLevelType.REVOKED.value:
|
|
4685
|
+
return False
|
|
4686
|
+
if self.revoked_at is not None:
|
|
4687
|
+
return False
|
|
4688
|
+
if self.expires_at and self.expires_at < datetime.utcnow():
|
|
4689
|
+
return False
|
|
4690
|
+
return True
|
|
4691
|
+
|
|
4692
|
+
def revoke(self, reason: str) -> None:
|
|
4693
|
+
"""Revoke this signer."""
|
|
4694
|
+
self.trust_level = TrustLevelType.REVOKED.value
|
|
4695
|
+
self.revoked_at = datetime.utcnow()
|
|
4696
|
+
self.revocation_reason = reason
|
|
4697
|
+
|
|
4698
|
+
def increment_signed_count(self) -> None:
|
|
4699
|
+
"""Increment the plugins signed count."""
|
|
4700
|
+
self.plugins_signed += 1
|
|
4701
|
+
|
|
4702
|
+
|
|
4703
|
+
class SecurityPolicy(Base, UUIDMixin, TimestampMixin):
|
|
4704
|
+
"""Security policy model for plugin execution.
|
|
4705
|
+
|
|
4706
|
+
Defines security policies for plugin sandbox execution.
|
|
4707
|
+
|
|
4708
|
+
Attributes:
|
|
4709
|
+
id: Unique identifier (UUID).
|
|
4710
|
+
name: Policy name.
|
|
4711
|
+
description: Policy description.
|
|
4712
|
+
is_default: Whether this is the default policy.
|
|
4713
|
+
is_active: Whether the policy is active.
|
|
4714
|
+
isolation_level: Sandbox isolation level.
|
|
4715
|
+
memory_limit_mb: Maximum memory in MB.
|
|
4716
|
+
cpu_time_limit_sec: Maximum CPU time in seconds.
|
|
4717
|
+
wall_time_limit_sec: Maximum wall clock time in seconds.
|
|
4718
|
+
network_enabled: Whether network access is allowed.
|
|
4719
|
+
file_read_enabled: Whether file read is allowed.
|
|
4720
|
+
file_write_enabled: Whether file write is allowed.
|
|
4721
|
+
allowed_modules: List of allowed Python modules.
|
|
4722
|
+
blocked_modules: List of blocked Python modules.
|
|
4723
|
+
allowed_builtins: List of allowed Python builtins.
|
|
4724
|
+
require_signature: Whether plugin signature is required.
|
|
4725
|
+
min_trust_level: Minimum trust level required.
|
|
4726
|
+
max_processes: Maximum number of processes.
|
|
4727
|
+
container_image: Docker image for container isolation.
|
|
4728
|
+
extra_options: Additional sandbox options (JSON).
|
|
4729
|
+
"""
|
|
4730
|
+
|
|
4731
|
+
__tablename__ = "security_policies"
|
|
4732
|
+
|
|
4733
|
+
__table_args__ = (
|
|
4734
|
+
Index("idx_security_policies_name", "name", unique=True),
|
|
4735
|
+
Index("idx_security_policies_default", "is_default"),
|
|
4736
|
+
)
|
|
4737
|
+
|
|
4738
|
+
name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
|
|
4739
|
+
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4740
|
+
is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4741
|
+
is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
4742
|
+
isolation_level: Mapped[str] = mapped_column(
|
|
4743
|
+
String(20),
|
|
4744
|
+
nullable=False,
|
|
4745
|
+
default=IsolationLevelType.PROCESS.value,
|
|
4746
|
+
)
|
|
4747
|
+
memory_limit_mb: Mapped[int] = mapped_column(Integer, nullable=False, default=256)
|
|
4748
|
+
cpu_time_limit_sec: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
|
|
4749
|
+
wall_time_limit_sec: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
|
|
4750
|
+
network_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4751
|
+
file_read_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
4752
|
+
file_write_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4753
|
+
allowed_modules: Mapped[list[str]] = mapped_column(
|
|
4754
|
+
JSON,
|
|
4755
|
+
nullable=False,
|
|
4756
|
+
default=lambda: [
|
|
4757
|
+
"math", "statistics", "decimal", "fractions",
|
|
4758
|
+
"random", "re", "json", "datetime",
|
|
4759
|
+
"collections", "itertools", "functools",
|
|
4760
|
+
"operator", "string", "typing", "dataclasses",
|
|
4761
|
+
],
|
|
4762
|
+
)
|
|
4763
|
+
blocked_modules: Mapped[list[str]] = mapped_column(
|
|
4764
|
+
JSON,
|
|
4765
|
+
nullable=False,
|
|
4766
|
+
default=lambda: [
|
|
4767
|
+
"os", "sys", "subprocess", "socket", "shutil",
|
|
4768
|
+
"importlib", "ctypes", "multiprocessing",
|
|
4769
|
+
],
|
|
4770
|
+
)
|
|
4771
|
+
allowed_builtins: Mapped[list[str]] = mapped_column(
|
|
4772
|
+
JSON,
|
|
4773
|
+
nullable=False,
|
|
4774
|
+
default=lambda: [
|
|
4775
|
+
"abs", "all", "any", "ascii", "bin", "bool", "chr",
|
|
4776
|
+
"dict", "divmod", "enumerate", "filter", "float",
|
|
4777
|
+
"format", "frozenset", "getattr", "hasattr", "hash",
|
|
4778
|
+
"hex", "int", "isinstance", "issubclass", "iter",
|
|
4779
|
+
"len", "list", "map", "max", "min", "next", "oct",
|
|
4780
|
+
"ord", "pow", "print", "range", "repr", "reversed",
|
|
4781
|
+
"round", "set", "slice", "sorted", "str", "sum",
|
|
4782
|
+
"tuple", "type", "zip",
|
|
4783
|
+
],
|
|
4784
|
+
)
|
|
4785
|
+
require_signature: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4786
|
+
min_trust_level: Mapped[str] = mapped_column(
|
|
4787
|
+
String(20),
|
|
4788
|
+
nullable=False,
|
|
4789
|
+
default=TrustLevelType.UNVERIFIED.value,
|
|
4790
|
+
)
|
|
4791
|
+
max_processes: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
|
4792
|
+
container_image: Mapped[str] = mapped_column(
|
|
4793
|
+
String(255),
|
|
4794
|
+
nullable=False,
|
|
4795
|
+
default="python:3.11-slim",
|
|
4796
|
+
)
|
|
4797
|
+
extra_options: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
|
|
4798
|
+
|
|
4799
|
+
@classmethod
|
|
4800
|
+
def get_preset(cls, preset_name: str) -> "SecurityPolicy":
|
|
4801
|
+
"""Create a security policy from a preset.
|
|
4802
|
+
|
|
4803
|
+
Args:
|
|
4804
|
+
preset_name: Name of the preset (development, testing, standard, enterprise, strict).
|
|
4805
|
+
|
|
4806
|
+
Returns:
|
|
4807
|
+
SecurityPolicy with preset configuration.
|
|
4808
|
+
"""
|
|
4809
|
+
presets = {
|
|
4810
|
+
"development": {
|
|
4811
|
+
"isolation_level": IsolationLevelType.NONE.value,
|
|
4812
|
+
"network_enabled": True,
|
|
4813
|
+
"file_read_enabled": True,
|
|
4814
|
+
"file_write_enabled": True,
|
|
4815
|
+
"require_signature": False,
|
|
4816
|
+
"memory_limit_mb": 1024,
|
|
4817
|
+
"wall_time_limit_sec": 300,
|
|
4818
|
+
},
|
|
4819
|
+
"testing": {
|
|
4820
|
+
"isolation_level": IsolationLevelType.PROCESS.value,
|
|
4821
|
+
"network_enabled": True,
|
|
4822
|
+
"file_read_enabled": True,
|
|
4823
|
+
"file_write_enabled": False,
|
|
4824
|
+
"require_signature": False,
|
|
4825
|
+
"memory_limit_mb": 512,
|
|
4826
|
+
"wall_time_limit_sec": 120,
|
|
4827
|
+
},
|
|
4828
|
+
"standard": {
|
|
4829
|
+
"isolation_level": IsolationLevelType.PROCESS.value,
|
|
4830
|
+
"network_enabled": False,
|
|
4831
|
+
"file_read_enabled": True,
|
|
4832
|
+
"file_write_enabled": False,
|
|
4833
|
+
"require_signature": False,
|
|
4834
|
+
"min_trust_level": TrustLevelType.VERIFIED.value,
|
|
4835
|
+
"memory_limit_mb": 256,
|
|
4836
|
+
"wall_time_limit_sec": 60,
|
|
4837
|
+
},
|
|
4838
|
+
"enterprise": {
|
|
4839
|
+
"isolation_level": IsolationLevelType.PROCESS.value,
|
|
4840
|
+
"network_enabled": False,
|
|
4841
|
+
"file_read_enabled": True,
|
|
4842
|
+
"file_write_enabled": False,
|
|
4843
|
+
"require_signature": True,
|
|
4844
|
+
"min_trust_level": TrustLevelType.TRUSTED.value,
|
|
4845
|
+
"memory_limit_mb": 512,
|
|
4846
|
+
"wall_time_limit_sec": 120,
|
|
4847
|
+
},
|
|
4848
|
+
"strict": {
|
|
4849
|
+
"isolation_level": IsolationLevelType.CONTAINER.value,
|
|
4850
|
+
"network_enabled": False,
|
|
4851
|
+
"file_read_enabled": False,
|
|
4852
|
+
"file_write_enabled": False,
|
|
4853
|
+
"require_signature": True,
|
|
4854
|
+
"min_trust_level": TrustLevelType.TRUSTED.value,
|
|
4855
|
+
"memory_limit_mb": 128,
|
|
4856
|
+
"wall_time_limit_sec": 30,
|
|
4857
|
+
},
|
|
4858
|
+
}
|
|
4859
|
+
|
|
4860
|
+
config = presets.get(preset_name, presets["standard"])
|
|
4861
|
+
return cls(
|
|
4862
|
+
name=preset_name,
|
|
4863
|
+
description=f"{preset_name.capitalize()} security policy preset",
|
|
4864
|
+
**config,
|
|
4865
|
+
)
|
|
4866
|
+
|
|
4867
|
+
|
|
4868
|
+
class PluginSignature(Base, UUIDMixin, TimestampMixin):
|
|
4869
|
+
"""Plugin signature model.
|
|
4870
|
+
|
|
4871
|
+
Stores signature information for verified plugins.
|
|
4872
|
+
|
|
4873
|
+
Attributes:
|
|
4874
|
+
id: Unique identifier (UUID).
|
|
4875
|
+
plugin_id: Reference to Plugin.
|
|
4876
|
+
signer_id: Reference to TrustedSigner.
|
|
4877
|
+
algorithm: Signature algorithm used.
|
|
4878
|
+
signature: Base64-encoded signature.
|
|
4879
|
+
signed_hash: Hash of the signed content.
|
|
4880
|
+
signed_at: When the signature was created.
|
|
4881
|
+
verified_at: When the signature was last verified.
|
|
4882
|
+
is_valid: Whether the signature is currently valid.
|
|
4883
|
+
metadata: Additional signature metadata.
|
|
4884
|
+
"""
|
|
4885
|
+
|
|
4886
|
+
__tablename__ = "plugin_signatures"
|
|
4887
|
+
|
|
4888
|
+
__table_args__ = (
|
|
4889
|
+
Index("idx_plugin_signatures_plugin", "plugin_id"),
|
|
4890
|
+
Index("idx_plugin_signatures_signer", "signer_id"),
|
|
4891
|
+
)
|
|
4892
|
+
|
|
4893
|
+
plugin_id: Mapped[str] = mapped_column(
|
|
4894
|
+
String(36),
|
|
4895
|
+
ForeignKey("plugins.id", ondelete="CASCADE"),
|
|
4896
|
+
nullable=False,
|
|
4897
|
+
index=True,
|
|
4898
|
+
)
|
|
4899
|
+
signer_id: Mapped[str] = mapped_column(
|
|
4900
|
+
String(36),
|
|
4901
|
+
ForeignKey("trusted_signers.id", ondelete="SET NULL"),
|
|
4902
|
+
nullable=True,
|
|
4903
|
+
index=True,
|
|
4904
|
+
)
|
|
4905
|
+
algorithm: Mapped[str] = mapped_column(
|
|
4906
|
+
String(20),
|
|
4907
|
+
nullable=False,
|
|
4908
|
+
default=SignatureAlgorithmType.ED25519.value,
|
|
4909
|
+
)
|
|
4910
|
+
signature: Mapped[str] = mapped_column(Text, nullable=False)
|
|
4911
|
+
signed_hash: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
4912
|
+
signed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
|
|
4913
|
+
verified_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4914
|
+
is_valid: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
4915
|
+
signature_metadata: Mapped[dict[str, Any]] = mapped_column(
|
|
4916
|
+
"metadata", JSON, nullable=False, default=dict
|
|
4917
|
+
)
|
|
4918
|
+
|
|
4919
|
+
# Relationships
|
|
4920
|
+
plugin: Mapped["Plugin"] = relationship("Plugin", back_populates="signatures")
|
|
4921
|
+
signer: Mapped["TrustedSigner"] = relationship("TrustedSigner")
|
|
4922
|
+
|
|
4923
|
+
def mark_verified(self) -> None:
|
|
4924
|
+
"""Mark the signature as verified."""
|
|
4925
|
+
self.verified_at = datetime.utcnow()
|
|
4926
|
+
self.is_valid = True
|
|
4927
|
+
|
|
4928
|
+
def invalidate(self) -> None:
|
|
4929
|
+
"""Invalidate the signature."""
|
|
4930
|
+
self.is_valid = False
|
|
4931
|
+
|
|
4932
|
+
|
|
4933
|
+
class HotReloadConfig(Base, UUIDMixin, TimestampMixin):
|
|
4934
|
+
"""Hot reload configuration model.
|
|
4935
|
+
|
|
4936
|
+
Stores hot reload settings for plugins.
|
|
4937
|
+
|
|
4938
|
+
Attributes:
|
|
4939
|
+
id: Unique identifier (UUID).
|
|
4940
|
+
plugin_id: Reference to Plugin.
|
|
4941
|
+
enabled: Whether hot reload is enabled.
|
|
4942
|
+
watch_paths: Paths to watch for changes (JSON array).
|
|
4943
|
+
debounce_ms: Debounce time in milliseconds.
|
|
4944
|
+
reload_strategy: Reload strategy (graceful, immediate, scheduled).
|
|
4945
|
+
max_reload_attempts: Maximum reload attempts before disabling.
|
|
4946
|
+
backup_on_reload: Whether to backup before reload.
|
|
4947
|
+
last_reload_at: When the last reload occurred.
|
|
4948
|
+
reload_count: Total reload count.
|
|
4949
|
+
last_error: Last error message.
|
|
4950
|
+
"""
|
|
4951
|
+
|
|
4952
|
+
__tablename__ = "hot_reload_configs"
|
|
4953
|
+
|
|
4954
|
+
__table_args__ = (
|
|
4955
|
+
Index("idx_hot_reload_configs_plugin", "plugin_id", unique=True),
|
|
4956
|
+
)
|
|
4957
|
+
|
|
4958
|
+
plugin_id: Mapped[str] = mapped_column(
|
|
4959
|
+
String(36),
|
|
4960
|
+
ForeignKey("plugins.id", ondelete="CASCADE"),
|
|
4961
|
+
nullable=False,
|
|
4962
|
+
unique=True,
|
|
4963
|
+
)
|
|
4964
|
+
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
4965
|
+
watch_paths: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list)
|
|
4966
|
+
debounce_ms: Mapped[int] = mapped_column(Integer, nullable=False, default=500)
|
|
4967
|
+
reload_strategy: Mapped[str] = mapped_column(
|
|
4968
|
+
String(20),
|
|
4969
|
+
nullable=False,
|
|
4970
|
+
default="graceful",
|
|
4971
|
+
)
|
|
4972
|
+
max_reload_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)
|
|
4973
|
+
backup_on_reload: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
4974
|
+
last_reload_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
4975
|
+
reload_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
4976
|
+
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
4977
|
+
|
|
4978
|
+
# Relationships
|
|
4979
|
+
plugin: Mapped["Plugin"] = relationship("Plugin", back_populates="hot_reload_config")
|
|
4980
|
+
|
|
4981
|
+
def record_reload(self, success: bool = True, error: str | None = None) -> None:
|
|
4982
|
+
"""Record a reload attempt."""
|
|
4983
|
+
self.last_reload_at = datetime.utcnow()
|
|
4984
|
+
self.reload_count += 1
|
|
4985
|
+
if not success:
|
|
4986
|
+
self.last_error = error
|
|
4987
|
+
|
|
4988
|
+
|
|
4989
|
+
class PluginHook(Base, UUIDMixin, TimestampMixin):
|
|
4990
|
+
"""Plugin hook registration model.
|
|
4991
|
+
|
|
4992
|
+
Stores registered hooks for plugins.
|
|
4993
|
+
|
|
4994
|
+
Attributes:
|
|
4995
|
+
id: Unique identifier (UUID).
|
|
4996
|
+
plugin_id: Reference to Plugin.
|
|
4997
|
+
hook_type: Type of hook (pre_validation, post_validation, etc.).
|
|
4998
|
+
callback_name: Name of the callback function.
|
|
4999
|
+
priority: Hook priority (lower runs first).
|
|
5000
|
+
enabled: Whether the hook is enabled.
|
|
5001
|
+
last_triggered_at: When the hook was last triggered.
|
|
5002
|
+
trigger_count: Total trigger count.
|
|
5003
|
+
total_execution_ms: Total execution time in ms.
|
|
5004
|
+
last_error: Last error message.
|
|
5005
|
+
"""
|
|
5006
|
+
|
|
5007
|
+
__tablename__ = "plugin_hooks"
|
|
5008
|
+
|
|
5009
|
+
__table_args__ = (
|
|
5010
|
+
Index("idx_plugin_hooks_plugin", "plugin_id"),
|
|
5011
|
+
Index("idx_plugin_hooks_type", "hook_type"),
|
|
5012
|
+
Index("idx_plugin_hooks_priority", "priority"),
|
|
5013
|
+
)
|
|
5014
|
+
|
|
5015
|
+
plugin_id: Mapped[str] = mapped_column(
|
|
5016
|
+
String(36),
|
|
5017
|
+
ForeignKey("plugins.id", ondelete="CASCADE"),
|
|
5018
|
+
nullable=False,
|
|
5019
|
+
index=True,
|
|
5020
|
+
)
|
|
5021
|
+
hook_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
|
5022
|
+
callback_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
5023
|
+
priority: Mapped[int] = mapped_column(Integer, nullable=False, default=50)
|
|
5024
|
+
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
5025
|
+
last_triggered_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
5026
|
+
trigger_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
5027
|
+
total_execution_ms: Mapped[float] = mapped_column(Float, nullable=False, default=0)
|
|
5028
|
+
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
5029
|
+
|
|
5030
|
+
# Relationships
|
|
5031
|
+
plugin: Mapped["Plugin"] = relationship("Plugin", back_populates="hooks")
|
|
5032
|
+
|
|
5033
|
+
@property
|
|
5034
|
+
def average_execution_ms(self) -> float:
|
|
5035
|
+
"""Calculate average execution time."""
|
|
5036
|
+
if self.trigger_count == 0:
|
|
5037
|
+
return 0
|
|
5038
|
+
return self.total_execution_ms / self.trigger_count
|
|
5039
|
+
|
|
5040
|
+
def record_trigger(self, execution_ms: float, error: str | None = None) -> None:
|
|
5041
|
+
"""Record a hook trigger."""
|
|
5042
|
+
self.last_triggered_at = datetime.utcnow()
|
|
5043
|
+
self.trigger_count += 1
|
|
5044
|
+
self.total_execution_ms += execution_ms
|
|
5045
|
+
if error:
|
|
5046
|
+
self.last_error = error
|