truthound-dashboard 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. truthound_dashboard/api/alerts.py +258 -0
  2. truthound_dashboard/api/anomaly.py +1302 -0
  3. truthound_dashboard/api/cross_alerts.py +352 -0
  4. truthound_dashboard/api/deps.py +143 -0
  5. truthound_dashboard/api/drift_monitor.py +540 -0
  6. truthound_dashboard/api/lineage.py +1151 -0
  7. truthound_dashboard/api/maintenance.py +363 -0
  8. truthound_dashboard/api/middleware.py +373 -1
  9. truthound_dashboard/api/model_monitoring.py +805 -0
  10. truthound_dashboard/api/notifications_advanced.py +2452 -0
  11. truthound_dashboard/api/plugins.py +2096 -0
  12. truthound_dashboard/api/profile.py +211 -14
  13. truthound_dashboard/api/reports.py +853 -0
  14. truthound_dashboard/api/router.py +147 -0
  15. truthound_dashboard/api/rule_suggestions.py +310 -0
  16. truthound_dashboard/api/schema_evolution.py +231 -0
  17. truthound_dashboard/api/sources.py +47 -3
  18. truthound_dashboard/api/triggers.py +190 -0
  19. truthound_dashboard/api/validations.py +13 -0
  20. truthound_dashboard/api/validators.py +333 -4
  21. truthound_dashboard/api/versioning.py +309 -0
  22. truthound_dashboard/api/websocket.py +301 -0
  23. truthound_dashboard/core/__init__.py +27 -0
  24. truthound_dashboard/core/anomaly.py +1395 -0
  25. truthound_dashboard/core/anomaly_explainer.py +633 -0
  26. truthound_dashboard/core/cache.py +206 -0
  27. truthound_dashboard/core/cached_services.py +422 -0
  28. truthound_dashboard/core/charts.py +352 -0
  29. truthound_dashboard/core/connections.py +1069 -42
  30. truthound_dashboard/core/cross_alerts.py +837 -0
  31. truthound_dashboard/core/drift_monitor.py +1477 -0
  32. truthound_dashboard/core/drift_sampling.py +669 -0
  33. truthound_dashboard/core/i18n/__init__.py +42 -0
  34. truthound_dashboard/core/i18n/detector.py +173 -0
  35. truthound_dashboard/core/i18n/messages.py +564 -0
  36. truthound_dashboard/core/lineage.py +971 -0
  37. truthound_dashboard/core/maintenance.py +443 -5
  38. truthound_dashboard/core/model_monitoring.py +1043 -0
  39. truthound_dashboard/core/notifications/channels.py +1020 -1
  40. truthound_dashboard/core/notifications/deduplication/__init__.py +143 -0
  41. truthound_dashboard/core/notifications/deduplication/policies.py +274 -0
  42. truthound_dashboard/core/notifications/deduplication/service.py +400 -0
  43. truthound_dashboard/core/notifications/deduplication/stores.py +2365 -0
  44. truthound_dashboard/core/notifications/deduplication/strategies.py +422 -0
  45. truthound_dashboard/core/notifications/dispatcher.py +43 -0
  46. truthound_dashboard/core/notifications/escalation/__init__.py +149 -0
  47. truthound_dashboard/core/notifications/escalation/backends.py +1384 -0
  48. truthound_dashboard/core/notifications/escalation/engine.py +429 -0
  49. truthound_dashboard/core/notifications/escalation/models.py +336 -0
  50. truthound_dashboard/core/notifications/escalation/scheduler.py +1187 -0
  51. truthound_dashboard/core/notifications/escalation/state_machine.py +330 -0
  52. truthound_dashboard/core/notifications/escalation/stores.py +2896 -0
  53. truthound_dashboard/core/notifications/events.py +49 -0
  54. truthound_dashboard/core/notifications/metrics/__init__.py +115 -0
  55. truthound_dashboard/core/notifications/metrics/base.py +528 -0
  56. truthound_dashboard/core/notifications/metrics/collectors.py +583 -0
  57. truthound_dashboard/core/notifications/routing/__init__.py +169 -0
  58. truthound_dashboard/core/notifications/routing/combinators.py +184 -0
  59. truthound_dashboard/core/notifications/routing/config.py +375 -0
  60. truthound_dashboard/core/notifications/routing/config_parser.py +867 -0
  61. truthound_dashboard/core/notifications/routing/engine.py +382 -0
  62. truthound_dashboard/core/notifications/routing/expression_engine.py +1269 -0
  63. truthound_dashboard/core/notifications/routing/jinja2_engine.py +774 -0
  64. truthound_dashboard/core/notifications/routing/rules.py +625 -0
  65. truthound_dashboard/core/notifications/routing/validator.py +678 -0
  66. truthound_dashboard/core/notifications/service.py +2 -0
  67. truthound_dashboard/core/notifications/stats_aggregator.py +850 -0
  68. truthound_dashboard/core/notifications/throttling/__init__.py +83 -0
  69. truthound_dashboard/core/notifications/throttling/builder.py +311 -0
  70. truthound_dashboard/core/notifications/throttling/stores.py +1859 -0
  71. truthound_dashboard/core/notifications/throttling/throttlers.py +633 -0
  72. truthound_dashboard/core/openlineage.py +1028 -0
  73. truthound_dashboard/core/plugins/__init__.py +39 -0
  74. truthound_dashboard/core/plugins/docs/__init__.py +39 -0
  75. truthound_dashboard/core/plugins/docs/extractor.py +703 -0
  76. truthound_dashboard/core/plugins/docs/renderers.py +804 -0
  77. truthound_dashboard/core/plugins/hooks/__init__.py +63 -0
  78. truthound_dashboard/core/plugins/hooks/decorators.py +367 -0
  79. truthound_dashboard/core/plugins/hooks/manager.py +403 -0
  80. truthound_dashboard/core/plugins/hooks/protocols.py +265 -0
  81. truthound_dashboard/core/plugins/lifecycle/__init__.py +41 -0
  82. truthound_dashboard/core/plugins/lifecycle/hot_reload.py +584 -0
  83. truthound_dashboard/core/plugins/lifecycle/machine.py +419 -0
  84. truthound_dashboard/core/plugins/lifecycle/states.py +266 -0
  85. truthound_dashboard/core/plugins/loader.py +504 -0
  86. truthound_dashboard/core/plugins/registry.py +810 -0
  87. truthound_dashboard/core/plugins/reporter_executor.py +588 -0
  88. truthound_dashboard/core/plugins/sandbox/__init__.py +59 -0
  89. truthound_dashboard/core/plugins/sandbox/code_validator.py +243 -0
  90. truthound_dashboard/core/plugins/sandbox/engines.py +770 -0
  91. truthound_dashboard/core/plugins/sandbox/protocols.py +194 -0
  92. truthound_dashboard/core/plugins/sandbox.py +617 -0
  93. truthound_dashboard/core/plugins/security/__init__.py +68 -0
  94. truthound_dashboard/core/plugins/security/analyzer.py +535 -0
  95. truthound_dashboard/core/plugins/security/policies.py +311 -0
  96. truthound_dashboard/core/plugins/security/protocols.py +296 -0
  97. truthound_dashboard/core/plugins/security/signing.py +842 -0
  98. truthound_dashboard/core/plugins/security.py +446 -0
  99. truthound_dashboard/core/plugins/validator_executor.py +401 -0
  100. truthound_dashboard/core/plugins/versioning/__init__.py +51 -0
  101. truthound_dashboard/core/plugins/versioning/constraints.py +377 -0
  102. truthound_dashboard/core/plugins/versioning/dependencies.py +541 -0
  103. truthound_dashboard/core/plugins/versioning/semver.py +266 -0
  104. truthound_dashboard/core/profile_comparison.py +601 -0
  105. truthound_dashboard/core/report_history.py +570 -0
  106. truthound_dashboard/core/reporters/__init__.py +57 -0
  107. truthound_dashboard/core/reporters/base.py +296 -0
  108. truthound_dashboard/core/reporters/csv_reporter.py +155 -0
  109. truthound_dashboard/core/reporters/html_reporter.py +598 -0
  110. truthound_dashboard/core/reporters/i18n/__init__.py +65 -0
  111. truthound_dashboard/core/reporters/i18n/base.py +494 -0
  112. truthound_dashboard/core/reporters/i18n/catalogs.py +930 -0
  113. truthound_dashboard/core/reporters/json_reporter.py +160 -0
  114. truthound_dashboard/core/reporters/junit_reporter.py +233 -0
  115. truthound_dashboard/core/reporters/markdown_reporter.py +207 -0
  116. truthound_dashboard/core/reporters/pdf_reporter.py +209 -0
  117. truthound_dashboard/core/reporters/registry.py +272 -0
  118. truthound_dashboard/core/rule_generator.py +2088 -0
  119. truthound_dashboard/core/scheduler.py +822 -12
  120. truthound_dashboard/core/schema_evolution.py +858 -0
  121. truthound_dashboard/core/services.py +152 -9
  122. truthound_dashboard/core/statistics.py +718 -0
  123. truthound_dashboard/core/streaming_anomaly.py +883 -0
  124. truthound_dashboard/core/triggers/__init__.py +45 -0
  125. truthound_dashboard/core/triggers/base.py +226 -0
  126. truthound_dashboard/core/triggers/evaluators.py +609 -0
  127. truthound_dashboard/core/triggers/factory.py +363 -0
  128. truthound_dashboard/core/unified_alerts.py +870 -0
  129. truthound_dashboard/core/validation_limits.py +509 -0
  130. truthound_dashboard/core/versioning.py +709 -0
  131. truthound_dashboard/core/websocket/__init__.py +59 -0
  132. truthound_dashboard/core/websocket/manager.py +512 -0
  133. truthound_dashboard/core/websocket/messages.py +130 -0
  134. truthound_dashboard/db/__init__.py +30 -0
  135. truthound_dashboard/db/models.py +3375 -3
  136. truthound_dashboard/main.py +22 -0
  137. truthound_dashboard/schemas/__init__.py +396 -1
  138. truthound_dashboard/schemas/anomaly.py +1258 -0
  139. truthound_dashboard/schemas/base.py +4 -0
  140. truthound_dashboard/schemas/cross_alerts.py +334 -0
  141. truthound_dashboard/schemas/drift_monitor.py +890 -0
  142. truthound_dashboard/schemas/lineage.py +428 -0
  143. truthound_dashboard/schemas/maintenance.py +154 -0
  144. truthound_dashboard/schemas/model_monitoring.py +374 -0
  145. truthound_dashboard/schemas/notifications_advanced.py +1363 -0
  146. truthound_dashboard/schemas/openlineage.py +704 -0
  147. truthound_dashboard/schemas/plugins.py +1293 -0
  148. truthound_dashboard/schemas/profile.py +420 -34
  149. truthound_dashboard/schemas/profile_comparison.py +242 -0
  150. truthound_dashboard/schemas/reports.py +285 -0
  151. truthound_dashboard/schemas/rule_suggestion.py +434 -0
  152. truthound_dashboard/schemas/schema_evolution.py +164 -0
  153. truthound_dashboard/schemas/source.py +117 -2
  154. truthound_dashboard/schemas/triggers.py +511 -0
  155. truthound_dashboard/schemas/unified_alerts.py +223 -0
  156. truthound_dashboard/schemas/validation.py +25 -1
  157. truthound_dashboard/schemas/validators/__init__.py +11 -0
  158. truthound_dashboard/schemas/validators/base.py +151 -0
  159. truthound_dashboard/schemas/versioning.py +152 -0
  160. truthound_dashboard/static/index.html +2 -2
  161. {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/METADATA +142 -18
  162. truthound_dashboard-1.4.0.dist-info/RECORD +239 -0
  163. truthound_dashboard/static/assets/index-BCA8H1hO.js +0 -574
  164. truthound_dashboard/static/assets/index-BNsSQ2fN.css +0 -1
  165. truthound_dashboard/static/assets/unmerged_dictionaries-CsJWCRx9.js +0 -1
  166. truthound_dashboard-1.3.0.dist-info/RECORD +0 -110
  167. {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/WHEEL +0 -0
  168. {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/entry_points.txt +0 -0
  169. {truthound_dashboard-1.3.0.dist-info → truthound_dashboard-1.4.0.dist-info}/licenses/LICENSE +0 -0
@@ -109,6 +109,43 @@ class ActivityAction(str, Enum):
109
109
  UNMAPPED = "unmapped"
110
110
 
111
111
 
112
+ # =============================================================================
113
+ # Schema Evolution Enums
114
+ # =============================================================================
115
+
116
+
117
+ class SchemaChangeType(str, Enum):
118
+ """Type of schema change detected."""
119
+
120
+ COLUMN_ADDED = "column_added"
121
+ COLUMN_REMOVED = "column_removed"
122
+ TYPE_CHANGED = "type_changed"
123
+
124
+
125
+ class SchemaChangeSeverity(str, Enum):
126
+ """Severity level of schema change."""
127
+
128
+ BREAKING = "breaking"
129
+ NON_BREAKING = "non_breaking"
130
+
131
+
132
+ # =============================================================================
133
+ # Schedule Trigger Enums
134
+ # =============================================================================
135
+
136
+
137
+ class TriggerType(str, Enum):
138
+ """Type of schedule trigger."""
139
+
140
+ CRON = "cron"
141
+ INTERVAL = "interval"
142
+ DATA_CHANGE = "data_change"
143
+ COMPOSITE = "composite"
144
+ EVENT = "event"
145
+ MANUAL = "manual"
146
+ WEBHOOK = "webhook" # External webhook triggers
147
+
148
+
112
149
  class Source(Base, UUIDMixin, TimestampMixin):
113
150
  """Data source model.
114
151
 
@@ -178,6 +215,13 @@ class Source(Base, UUIDMixin, TimestampMixin):
178
215
  back_populates="source",
179
216
  lazy="selectin",
180
217
  )
218
+ # Generated reports for this source
219
+ generated_reports: Mapped[list["GeneratedReport"]] = relationship(
220
+ "GeneratedReport",
221
+ back_populates="source",
222
+ lazy="selectin",
223
+ order_by="desc(GeneratedReport.created_at)",
224
+ )
181
225
 
182
226
  @property
183
227
  def source_path(self) -> str | None:
@@ -399,6 +443,12 @@ class Validation(Base, UUIDMixin):
399
443
 
400
444
  # Relationships
401
445
  source: Mapped[Source] = relationship("Source", back_populates="validations")
446
+ generated_reports: Mapped[list["GeneratedReport"]] = relationship(
447
+ "GeneratedReport",
448
+ back_populates="validation",
449
+ lazy="selectin",
450
+ order_by="desc(GeneratedReport.created_at)",
451
+ )
402
452
 
403
453
  @property
404
454
  def issues(self) -> list[dict[str, Any]]:
@@ -484,17 +534,22 @@ class Profile(Base, UUIDMixin, TimestampMixin):
484
534
  class Schedule(Base, UUIDMixin, TimestampMixin):
485
535
  """Validation schedule model.
486
536
 
487
- Manages scheduled validation runs using cron expressions.
537
+ Manages scheduled validation runs with flexible trigger types.
538
+ Supports cron, interval, data change detection, and composite triggers.
488
539
 
489
540
  Attributes:
490
541
  id: Unique identifier (UUID).
491
542
  name: Human-readable schedule name.
492
543
  source_id: Reference to Source to validate.
493
- cron_expression: Cron expression for scheduling.
544
+ trigger_type: Type of trigger (cron, interval, data_change, composite).
545
+ trigger_config: JSON configuration specific to trigger type.
546
+ cron_expression: Legacy cron expression (for backward compatibility).
494
547
  is_active: Whether schedule is active.
495
548
  notify_on_failure: Send notification on validation failure.
496
549
  last_run_at: Timestamp of last execution.
497
550
  next_run_at: Timestamp of next scheduled run.
551
+ trigger_count: Total number of times this schedule has triggered.
552
+ last_trigger_result: Result of last trigger evaluation.
498
553
  config: Additional configuration (validators, schema_path, etc.).
499
554
  """
500
555
 
@@ -507,13 +562,33 @@ class Schedule(Base, UUIDMixin, TimestampMixin):
507
562
  nullable=False,
508
563
  index=True,
509
564
  )
510
- cron_expression: Mapped[str] = mapped_column(String(100), nullable=False)
565
+ # New flexible trigger system
566
+ trigger_type: Mapped[str] = mapped_column(
567
+ String(50),
568
+ nullable=False,
569
+ default=TriggerType.CRON.value,
570
+ index=True,
571
+ )
572
+ trigger_config: Mapped[dict[str, Any] | None] = mapped_column(
573
+ JSON,
574
+ nullable=True,
575
+ comment="Trigger-specific configuration (threshold, metrics, etc.)",
576
+ )
577
+ # Legacy field for backward compatibility
578
+ cron_expression: Mapped[str | None] = mapped_column(String(100), nullable=True)
511
579
  is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
512
580
  notify_on_failure: Mapped[bool] = mapped_column(
513
581
  Boolean, default=True, nullable=False
514
582
  )
515
583
  last_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
516
584
  next_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
585
+ # Trigger state tracking
586
+ trigger_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
587
+ last_trigger_result: Mapped[dict[str, Any] | None] = mapped_column(
588
+ JSON,
589
+ nullable=True,
590
+ comment="Result of last trigger evaluation",
591
+ )
517
592
  config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
518
593
 
519
594
  # Relationships
@@ -531,6 +606,78 @@ class Schedule(Base, UUIDMixin, TimestampMixin):
531
606
  """Mark schedule as run and update next run time."""
532
607
  self.last_run_at = datetime.utcnow()
533
608
  self.next_run_at = next_run
609
+ self.trigger_count += 1
610
+
611
+ def update_trigger_result(self, result: dict[str, Any]) -> None:
612
+ """Update the last trigger evaluation result."""
613
+ self.last_trigger_result = result
614
+
615
+ @property
616
+ def effective_trigger_type(self) -> TriggerType:
617
+ """Get the effective trigger type.
618
+
619
+ Falls back to CRON if trigger_type is not set (legacy schedules).
620
+ """
621
+ try:
622
+ return TriggerType(self.trigger_type)
623
+ except ValueError:
624
+ return TriggerType.CRON
625
+
626
+ @property
627
+ def effective_cron_expression(self) -> str | None:
628
+ """Get the effective cron expression.
629
+
630
+ For CRON triggers, returns the expression from trigger_config or legacy field.
631
+ """
632
+ if self.trigger_type == TriggerType.CRON.value:
633
+ if self.trigger_config and "expression" in self.trigger_config:
634
+ return self.trigger_config["expression"]
635
+ return self.cron_expression
636
+ return None
637
+
638
+ def get_trigger_summary(self) -> str:
639
+ """Get a human-readable summary of the trigger configuration."""
640
+ trigger_type = self.effective_trigger_type
641
+
642
+ if trigger_type == TriggerType.CRON:
643
+ expr = self.effective_cron_expression
644
+ return f"Cron: {expr}" if expr else "Cron: (not configured)"
645
+
646
+ elif trigger_type == TriggerType.INTERVAL:
647
+ if self.trigger_config:
648
+ parts = []
649
+ if self.trigger_config.get("days"):
650
+ parts.append(f"{self.trigger_config['days']}d")
651
+ if self.trigger_config.get("hours"):
652
+ parts.append(f"{self.trigger_config['hours']}h")
653
+ if self.trigger_config.get("minutes"):
654
+ parts.append(f"{self.trigger_config['minutes']}m")
655
+ if self.trigger_config.get("seconds"):
656
+ parts.append(f"{self.trigger_config['seconds']}s")
657
+ return f"Every {' '.join(parts)}" if parts else "Interval: (not configured)"
658
+ return "Interval: (not configured)"
659
+
660
+ elif trigger_type == TriggerType.DATA_CHANGE:
661
+ threshold = self.trigger_config.get("change_threshold", 0.05) if self.trigger_config else 0.05
662
+ return f"Data change >= {threshold * 100:.0f}%"
663
+
664
+ elif trigger_type == TriggerType.COMPOSITE:
665
+ if self.trigger_config:
666
+ operator = self.trigger_config.get("operator", "and").upper()
667
+ count = len(self.trigger_config.get("triggers", []))
668
+ return f"Composite: {count} triggers ({operator})"
669
+ return "Composite: (not configured)"
670
+
671
+ elif trigger_type == TriggerType.EVENT:
672
+ if self.trigger_config:
673
+ events = self.trigger_config.get("event_types", [])
674
+ return f"Events: {', '.join(events[:2])}{'...' if len(events) > 2 else ''}"
675
+ return "Event: (not configured)"
676
+
677
+ elif trigger_type == TriggerType.MANUAL:
678
+ return "Manual trigger only"
679
+
680
+ return f"{trigger_type.value}: (unknown)"
534
681
 
535
682
 
536
683
  class DriftComparison(Base, UUIDMixin, TimestampMixin):
@@ -1672,3 +1819,3228 @@ class Activity(Base, UUIDMixin):
1672
1819
  def resource_key(self) -> str:
1673
1820
  """Get unique resource key."""
1674
1821
  return f"{self.resource_type}:{self.resource_id}"
1822
+
1823
+
1824
+ # =============================================================================
1825
+ # Schema Evolution Models
1826
+ # =============================================================================
1827
+
1828
+
1829
+ class SchemaVersion(Base, UUIDMixin, TimestampMixin):
1830
+ """Schema version snapshot for evolution tracking.
1831
+
1832
+ Stores a snapshot of schema structure at a point in time
1833
+ to enable schema change detection and history tracking.
1834
+
1835
+ Attributes:
1836
+ id: Unique identifier (UUID).
1837
+ source_id: Reference to parent Source.
1838
+ schema_id: Reference to the Schema record.
1839
+ version_number: Sequential version number for this source.
1840
+ schema_hash: SHA256 hash of normalized schema structure.
1841
+ column_snapshot: JSON snapshot of column definitions.
1842
+ """
1843
+
1844
+ __tablename__ = "schema_versions"
1845
+
1846
+ __table_args__ = (
1847
+ Index("idx_schema_versions_source", "source_id", "version_number"),
1848
+ Index("idx_schema_versions_hash", "schema_hash"),
1849
+ )
1850
+
1851
+ source_id: Mapped[str] = mapped_column(
1852
+ String(36),
1853
+ ForeignKey("sources.id", ondelete="CASCADE"),
1854
+ nullable=False,
1855
+ index=True,
1856
+ )
1857
+ schema_id: Mapped[str] = mapped_column(
1858
+ String(36),
1859
+ ForeignKey("schemas.id", ondelete="CASCADE"),
1860
+ nullable=False,
1861
+ )
1862
+ version_number: Mapped[int] = mapped_column(Integer, nullable=False)
1863
+ schema_hash: Mapped[str] = mapped_column(String(64), nullable=False)
1864
+ column_snapshot: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
1865
+
1866
+ # Relationships
1867
+ source: Mapped[Source] = relationship("Source", lazy="selectin")
1868
+ schema: Mapped[Schema] = relationship("Schema", lazy="selectin")
1869
+
1870
+ @property
1871
+ def column_names(self) -> list[str]:
1872
+ """Get list of column names from snapshot."""
1873
+ return list(self.column_snapshot.keys())
1874
+
1875
+ @property
1876
+ def column_count(self) -> int:
1877
+ """Get number of columns in this version."""
1878
+ return len(self.column_snapshot)
1879
+
1880
+
1881
+ class SchemaChange(Base, UUIDMixin):
1882
+ """Individual schema change record.
1883
+
1884
+ Records a single change detected between two schema versions.
1885
+
1886
+ Attributes:
1887
+ id: Unique identifier (UUID).
1888
+ source_id: Reference to parent Source.
1889
+ from_version_id: Reference to previous SchemaVersion (null for first version).
1890
+ to_version_id: Reference to new SchemaVersion.
1891
+ change_type: Type of change (column_added, column_removed, type_changed).
1892
+ column_name: Name of the affected column.
1893
+ old_value: Previous value (for type changes).
1894
+ new_value: New value (for type changes or additions).
1895
+ severity: Severity of the change (breaking/non_breaking).
1896
+ created_at: When the change was detected.
1897
+ """
1898
+
1899
+ __tablename__ = "schema_changes"
1900
+
1901
+ __table_args__ = (
1902
+ Index("idx_schema_changes_source", "source_id", "created_at"),
1903
+ Index("idx_schema_changes_type", "change_type"),
1904
+ )
1905
+
1906
+ source_id: Mapped[str] = mapped_column(
1907
+ String(36),
1908
+ ForeignKey("sources.id", ondelete="CASCADE"),
1909
+ nullable=False,
1910
+ index=True,
1911
+ )
1912
+ from_version_id: Mapped[str | None] = mapped_column(
1913
+ String(36),
1914
+ ForeignKey("schema_versions.id", ondelete="SET NULL"),
1915
+ nullable=True,
1916
+ )
1917
+ to_version_id: Mapped[str] = mapped_column(
1918
+ String(36),
1919
+ ForeignKey("schema_versions.id", ondelete="CASCADE"),
1920
+ nullable=False,
1921
+ )
1922
+ change_type: Mapped[str] = mapped_column(
1923
+ String(30),
1924
+ nullable=False,
1925
+ index=True,
1926
+ )
1927
+ column_name: Mapped[str] = mapped_column(String(255), nullable=False)
1928
+ old_value: Mapped[str | None] = mapped_column(Text, nullable=True)
1929
+ new_value: Mapped[str | None] = mapped_column(Text, nullable=True)
1930
+ severity: Mapped[str] = mapped_column(
1931
+ String(20),
1932
+ nullable=False,
1933
+ default="non_breaking",
1934
+ )
1935
+ created_at: Mapped[datetime] = mapped_column(
1936
+ DateTime,
1937
+ default=datetime.utcnow,
1938
+ nullable=False,
1939
+ )
1940
+
1941
+ # Relationships
1942
+ source: Mapped[Source] = relationship("Source", lazy="selectin")
1943
+ from_version: Mapped[SchemaVersion | None] = relationship(
1944
+ "SchemaVersion",
1945
+ foreign_keys=[from_version_id],
1946
+ lazy="selectin",
1947
+ )
1948
+ to_version: Mapped[SchemaVersion] = relationship(
1949
+ "SchemaVersion",
1950
+ foreign_keys=[to_version_id],
1951
+ lazy="selectin",
1952
+ )
1953
+
1954
+ @property
1955
+ def is_breaking(self) -> bool:
1956
+ """Check if this is a breaking change."""
1957
+ return self.severity == "breaking"
1958
+
1959
+ @property
1960
+ def description(self) -> str:
1961
+ """Generate human-readable description of the change."""
1962
+ if self.change_type == "column_added":
1963
+ return f"Column '{self.column_name}' added with type {self.new_value}"
1964
+ elif self.change_type == "column_removed":
1965
+ return f"Column '{self.column_name}' removed (was {self.old_value})"
1966
+ elif self.change_type == "type_changed":
1967
+ return f"Column '{self.column_name}' type changed from {self.old_value} to {self.new_value}"
1968
+ return f"Unknown change on '{self.column_name}'"
1969
+
1970
+
1971
+ # =============================================================================
1972
+ # Phase 14: Advanced Notification Models
1973
+ # =============================================================================
1974
+
1975
+
1976
+ class DeduplicationStrategyEnum(str, Enum):
1977
+ """Deduplication window strategies."""
1978
+
1979
+ SLIDING = "sliding"
1980
+ TUMBLING = "tumbling"
1981
+ SESSION = "session"
1982
+ ADAPTIVE = "adaptive"
1983
+
1984
+
1985
+ class DeduplicationPolicyEnum(str, Enum):
1986
+ """Deduplication policies."""
1987
+
1988
+ NONE = "none"
1989
+ BASIC = "basic"
1990
+ SEVERITY = "severity"
1991
+ ISSUE_BASED = "issue_based"
1992
+ STRICT = "strict"
1993
+ CUSTOM = "custom"
1994
+
1995
+
1996
+ class EscalationStateEnum(str, Enum):
1997
+ """Escalation incident states."""
1998
+
1999
+ PENDING = "pending"
2000
+ TRIGGERED = "triggered"
2001
+ ACKNOWLEDGED = "acknowledged"
2002
+ ESCALATED = "escalated"
2003
+ RESOLVED = "resolved"
2004
+
2005
+
2006
+ class TargetTypeEnum(str, Enum):
2007
+ """Escalation target types."""
2008
+
2009
+ USER = "user"
2010
+ GROUP = "group"
2011
+ ONCALL = "oncall"
2012
+ CHANNEL = "channel"
2013
+
2014
+
2015
+ class RoutingRuleModel(Base, UUIDMixin, TimestampMixin):
2016
+ """Advanced routing rule model.
2017
+
2018
+ Stores rule-based routing configuration for directing
2019
+ notifications to appropriate channels based on conditions.
2020
+
2021
+ Attributes:
2022
+ id: Unique identifier (UUID).
2023
+ name: Human-readable rule name.
2024
+ rule_config: JSON configuration defining the rule logic.
2025
+ actions: List of channel IDs to notify when rule matches.
2026
+ priority: Priority for rule evaluation (higher = evaluated first).
2027
+ is_active: Whether rule is active.
2028
+ stop_on_match: Stop processing after this rule matches.
2029
+ metadata: Additional metadata.
2030
+ """
2031
+
2032
+ __tablename__ = "routing_rules"
2033
+
2034
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2035
+ rule_config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
2036
+ actions: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list)
2037
+ priority: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
2038
+ is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
2039
+ stop_on_match: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
2040
+ routing_metadata: Mapped[dict[str, Any] | None] = mapped_column(
2041
+ "metadata", JSON, nullable=True
2042
+ )
2043
+
2044
+ @property
2045
+ def rule_type(self) -> str:
2046
+ """Get the rule type from config."""
2047
+ return self.rule_config.get("type", "unknown")
2048
+
2049
+
2050
+ class DeduplicationConfig(Base, UUIDMixin, TimestampMixin):
2051
+ """Deduplication configuration model.
2052
+
2053
+ Stores configuration for notification deduplication
2054
+ to prevent duplicate notifications within time windows.
2055
+
2056
+ Attributes:
2057
+ id: Unique identifier (UUID).
2058
+ name: Configuration name.
2059
+ strategy: Window strategy (sliding, tumbling, session, adaptive).
2060
+ policy: Deduplication policy (basic, severity, issue_based, etc.).
2061
+ window_seconds: Duration of deduplication window.
2062
+ is_active: Whether config is active.
2063
+ """
2064
+
2065
+ __tablename__ = "deduplication_configs"
2066
+
2067
+ __table_args__ = (
2068
+ Index("idx_dedup_config_is_active", "is_active"),
2069
+ Index("idx_dedup_config_strategy", "strategy"),
2070
+ Index("idx_dedup_config_policy", "policy"),
2071
+ Index("idx_dedup_config_created_at", "created_at"),
2072
+ )
2073
+
2074
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2075
+ strategy: Mapped[str] = mapped_column(
2076
+ String(20),
2077
+ nullable=False,
2078
+ default=DeduplicationStrategyEnum.SLIDING.value,
2079
+ )
2080
+ policy: Mapped[str] = mapped_column(
2081
+ String(20),
2082
+ nullable=False,
2083
+ default=DeduplicationPolicyEnum.BASIC.value,
2084
+ )
2085
+ window_seconds: Mapped[int] = mapped_column(Integer, default=300, nullable=False)
2086
+ is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
2087
+
2088
+
2089
+ class ThrottlingConfig(Base, UUIDMixin, TimestampMixin):
2090
+ """Throttling configuration model.
2091
+
2092
+ Stores rate limiting configuration for notifications
2093
+ to prevent overwhelming notification channels.
2094
+
2095
+ Attributes:
2096
+ id: Unique identifier (UUID).
2097
+ name: Configuration name.
2098
+ per_minute: Max notifications per minute.
2099
+ per_hour: Max notifications per hour.
2100
+ per_day: Max notifications per day.
2101
+ burst_allowance: Factor to allow temporary bursts.
2102
+ channel_id: Optional channel ID for per-channel throttling.
2103
+ is_active: Whether config is active.
2104
+ """
2105
+
2106
+ __tablename__ = "throttling_configs"
2107
+
2108
+ __table_args__ = (
2109
+ Index("idx_throttle_config_is_active", "is_active"),
2110
+ Index("idx_throttle_config_created_at", "created_at"),
2111
+ )
2112
+
2113
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2114
+ per_minute: Mapped[int | None] = mapped_column(Integer, nullable=True)
2115
+ per_hour: Mapped[int | None] = mapped_column(Integer, nullable=True)
2116
+ per_day: Mapped[int | None] = mapped_column(Integer, nullable=True)
2117
+ burst_allowance: Mapped[float] = mapped_column(Float, default=1.5, nullable=False)
2118
+ channel_id: Mapped[str | None] = mapped_column(
2119
+ String(36),
2120
+ ForeignKey("notification_channels.id", ondelete="SET NULL"),
2121
+ nullable=True,
2122
+ index=True,
2123
+ )
2124
+ is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
2125
+
2126
+ # Relationships
2127
+ channel: Mapped[NotificationChannel | None] = relationship(
2128
+ "NotificationChannel",
2129
+ lazy="selectin",
2130
+ )
2131
+
2132
+
2133
+ class EscalationPolicyModel(Base, UUIDMixin, TimestampMixin):
2134
+ """Escalation policy model.
2135
+
2136
+ Stores multi-level escalation policy configuration
2137
+ for handling unacknowledged alerts.
2138
+
2139
+ Attributes:
2140
+ id: Unique identifier (UUID).
2141
+ name: Policy name.
2142
+ description: Policy description.
2143
+ levels: JSON array of escalation levels.
2144
+ auto_resolve_on_success: Whether to auto-resolve on validation success.
2145
+ max_escalations: Maximum number of escalation attempts.
2146
+ is_active: Whether policy is active.
2147
+ """
2148
+
2149
+ __tablename__ = "escalation_policies"
2150
+
2151
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2152
+ description: Mapped[str | None] = mapped_column(Text, nullable=True)
2153
+ levels: Mapped[list[dict[str, Any]]] = mapped_column(JSON, nullable=False)
2154
+ auto_resolve_on_success: Mapped[bool] = mapped_column(
2155
+ Boolean, default=True, nullable=False
2156
+ )
2157
+ max_escalations: Mapped[int] = mapped_column(Integer, default=3, nullable=False)
2158
+ is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
2159
+
2160
+ # Relationships
2161
+ incidents: Mapped[list[EscalationIncidentModel]] = relationship(
2162
+ "EscalationIncidentModel",
2163
+ back_populates="policy",
2164
+ cascade="all, delete-orphan",
2165
+ lazy="selectin",
2166
+ )
2167
+
2168
+ @property
2169
+ def level_count(self) -> int:
2170
+ """Get number of escalation levels."""
2171
+ return len(self.levels) if self.levels else 0
2172
+
2173
+
2174
+ class EscalationIncidentModel(Base, UUIDMixin):
2175
+ """Escalation incident model.
2176
+
2177
+ Tracks the lifecycle of an escalation from trigger to resolution.
2178
+
2179
+ Attributes:
2180
+ id: Unique identifier (UUID).
2181
+ policy_id: Reference to escalation policy.
2182
+ incident_ref: External reference (e.g., validation ID).
2183
+ state: Current incident state.
2184
+ current_level: Current escalation level.
2185
+ escalation_count: Number of escalation attempts.
2186
+ context: JSON context data.
2187
+ acknowledged_by: Who acknowledged the incident.
2188
+ acknowledged_at: When acknowledged.
2189
+ resolved_by: Who resolved the incident.
2190
+ resolved_at: When resolved.
2191
+ events: JSON array of state transition events.
2192
+ next_escalation_at: When next escalation will occur.
2193
+ """
2194
+
2195
+ __tablename__ = "escalation_incidents"
2196
+
2197
+ __table_args__ = (
2198
+ Index("idx_escalation_incidents_policy", "policy_id"),
2199
+ Index("idx_escalation_incidents_ref", "incident_ref"),
2200
+ Index("idx_escalation_incidents_state", "state"),
2201
+ Index("idx_escalation_incidents_created_at", "created_at"),
2202
+ Index("idx_escalation_incidents_state_created", "state", "created_at"),
2203
+ )
2204
+
2205
+ policy_id: Mapped[str] = mapped_column(
2206
+ String(36),
2207
+ ForeignKey("escalation_policies.id", ondelete="CASCADE"),
2208
+ nullable=False,
2209
+ index=True,
2210
+ )
2211
+ incident_ref: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2212
+ state: Mapped[str] = mapped_column(
2213
+ String(20),
2214
+ nullable=False,
2215
+ default=EscalationStateEnum.PENDING.value,
2216
+ index=True,
2217
+ )
2218
+ current_level: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
2219
+ escalation_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
2220
+ context: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
2221
+ acknowledged_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
2222
+ acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
2223
+ resolved_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
2224
+ resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
2225
+ events: Mapped[list[dict[str, Any]]] = mapped_column(JSON, nullable=False, default=list)
2226
+ next_escalation_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
2227
+
2228
+ # Timestamps
2229
+ created_at: Mapped[datetime] = mapped_column(
2230
+ DateTime, default=datetime.utcnow, nullable=False
2231
+ )
2232
+ updated_at: Mapped[datetime] = mapped_column(
2233
+ DateTime,
2234
+ default=datetime.utcnow,
2235
+ onupdate=datetime.utcnow,
2236
+ nullable=False,
2237
+ )
2238
+
2239
+ # Relationships
2240
+ policy: Mapped[EscalationPolicyModel] = relationship(
2241
+ "EscalationPolicyModel",
2242
+ back_populates="incidents",
2243
+ )
2244
+
2245
+ @property
2246
+ def is_active(self) -> bool:
2247
+ """Check if incident is active (not resolved)."""
2248
+ return self.state != EscalationStateEnum.RESOLVED.value
2249
+
2250
+ @property
2251
+ def is_acknowledged(self) -> bool:
2252
+ """Check if incident has been acknowledged."""
2253
+ return self.state == EscalationStateEnum.ACKNOWLEDGED.value
2254
+
2255
+ @property
2256
+ def is_resolved(self) -> bool:
2257
+ """Check if incident is resolved."""
2258
+ return self.state == EscalationStateEnum.RESOLVED.value
2259
+
2260
+ def can_escalate(self, max_escalations: int) -> bool:
2261
+ """Check if incident can be escalated further.
2262
+
2263
+ Args:
2264
+ max_escalations: Maximum allowed escalations from policy.
2265
+
2266
+ Returns:
2267
+ True if escalation is allowed, False otherwise.
2268
+ """
2269
+ # Cannot escalate resolved or acknowledged incidents
2270
+ if self.state in (
2271
+ EscalationStateEnum.RESOLVED.value,
2272
+ EscalationStateEnum.ACKNOWLEDGED.value,
2273
+ ):
2274
+ return False
2275
+
2276
+ # Check escalation count limit
2277
+ if self.escalation_count >= max_escalations:
2278
+ return False
2279
+
2280
+ return True
2281
+
2282
+ def escalate(
2283
+ self,
2284
+ next_level: int,
2285
+ next_escalation_at: datetime | None,
2286
+ max_escalations: int,
2287
+ ) -> bool:
2288
+ """Attempt to escalate the incident.
2289
+
2290
+ Thread-safe escalation with validation. Returns False if
2291
+ escalation is not allowed.
2292
+
2293
+ Args:
2294
+ next_level: The new escalation level.
2295
+ next_escalation_at: When the next escalation should occur.
2296
+ max_escalations: Maximum allowed escalations.
2297
+
2298
+ Returns:
2299
+ True if escalation succeeded, False if not allowed.
2300
+ """
2301
+ if not self.can_escalate(max_escalations):
2302
+ return False
2303
+
2304
+ old_state = self.state
2305
+ self.state = EscalationStateEnum.ESCALATED.value
2306
+ self.current_level = next_level
2307
+ self.escalation_count += 1
2308
+ self.next_escalation_at = next_escalation_at
2309
+ self.updated_at = datetime.utcnow()
2310
+
2311
+ self.add_event(
2312
+ from_state=old_state,
2313
+ to_state=EscalationStateEnum.ESCALATED.value,
2314
+ actor="system",
2315
+ message=f"Escalated to level {next_level}",
2316
+ )
2317
+
2318
+ return True
2319
+
2320
+ def add_event(
2321
+ self,
2322
+ from_state: str | None,
2323
+ to_state: str,
2324
+ actor: str | None = None,
2325
+ message: str = "",
2326
+ ) -> None:
2327
+ """Add a state transition event."""
2328
+ event = {
2329
+ "from_state": from_state,
2330
+ "to_state": to_state,
2331
+ "actor": actor,
2332
+ "message": message,
2333
+ "timestamp": datetime.utcnow().isoformat(),
2334
+ }
2335
+ if self.events is None:
2336
+ self.events = []
2337
+ self.events.append(event)
2338
+
2339
+
2340
+ # =============================================================================
2341
+ # Scheduler Job Model (Persistent Job Storage)
2342
+ # =============================================================================
2343
+
2344
+
2345
+ class SchedulerJobState(str, Enum):
2346
+ """State of a scheduled job."""
2347
+
2348
+ PENDING = "pending"
2349
+ RUNNING = "running"
2350
+ COMPLETED = "completed"
2351
+ FAILED = "failed"
2352
+ MISFIRED = "misfired"
2353
+ PAUSED = "paused"
2354
+
2355
+
2356
+ class SchedulerJob(Base, UUIDMixin):
2357
+ """Persistent scheduler job model.
2358
+
2359
+ Stores scheduled jobs for APScheduler with SQLAlchemy backend,
2360
+ enabling job persistence across restarts.
2361
+
2362
+ Features:
2363
+ - Survives process restarts
2364
+ - Supports exponential backoff retries
2365
+ - Tracks execution history
2366
+ - Enables job recovery on startup
2367
+
2368
+ Attributes:
2369
+ id: Unique job identifier (UUID).
2370
+ name: Human-readable job name.
2371
+ func_ref: Reference to the function to execute (module:function).
2372
+ trigger_type: Type of trigger (interval, cron, date).
2373
+ trigger_args: Arguments for the trigger configuration.
2374
+ args: Positional arguments for the function.
2375
+ kwargs: Keyword arguments for the function.
2376
+ next_run_time: Next scheduled execution time.
2377
+ state: Current job state.
2378
+ retry_count: Number of retry attempts.
2379
+ last_run_time: Last execution time.
2380
+ last_error: Last error message.
2381
+ job_metadata: Additional job metadata.
2382
+ created_at: When the job was created.
2383
+ updated_at: Last update timestamp.
2384
+ """
2385
+
2386
+ __tablename__ = "scheduler_jobs"
2387
+
2388
+ __table_args__ = (
2389
+ Index("idx_scheduler_jobs_state", "state"),
2390
+ Index("idx_scheduler_jobs_next_run", "next_run_time"),
2391
+ Index("idx_scheduler_jobs_state_next_run", "state", "next_run_time"),
2392
+ )
2393
+
2394
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2395
+ func_ref: Mapped[str] = mapped_column(String(512), nullable=False)
2396
+ trigger_type: Mapped[str] = mapped_column(
2397
+ String(20),
2398
+ nullable=False,
2399
+ default="interval",
2400
+ )
2401
+ trigger_args: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
2402
+ args: Mapped[list[Any] | None] = mapped_column(JSON, nullable=True)
2403
+ kwargs: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
2404
+ next_run_time: Mapped[datetime | None] = mapped_column(
2405
+ DateTime,
2406
+ nullable=True,
2407
+ index=True,
2408
+ )
2409
+ state: Mapped[str] = mapped_column(
2410
+ String(20),
2411
+ nullable=False,
2412
+ default=SchedulerJobState.PENDING.value,
2413
+ index=True,
2414
+ )
2415
+ retry_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
2416
+ last_run_time: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
2417
+ last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
2418
+ job_metadata: Mapped[dict[str, Any] | None] = mapped_column(
2419
+ "metadata",
2420
+ JSON,
2421
+ nullable=True,
2422
+ )
2423
+
2424
+ # Timestamps
2425
+ created_at: Mapped[datetime] = mapped_column(
2426
+ DateTime,
2427
+ default=datetime.utcnow,
2428
+ nullable=False,
2429
+ )
2430
+ updated_at: Mapped[datetime] = mapped_column(
2431
+ DateTime,
2432
+ default=datetime.utcnow,
2433
+ onupdate=datetime.utcnow,
2434
+ nullable=False,
2435
+ )
2436
+
2437
+ @property
2438
+ def is_due(self) -> bool:
2439
+ """Check if job is due for execution."""
2440
+ if not self.next_run_time:
2441
+ return False
2442
+ return (
2443
+ self.state in (SchedulerJobState.PENDING.value, SchedulerJobState.MISFIRED.value)
2444
+ and datetime.utcnow() >= self.next_run_time
2445
+ )
2446
+
2447
+ @property
2448
+ def is_running(self) -> bool:
2449
+ """Check if job is currently running."""
2450
+ return self.state == SchedulerJobState.RUNNING.value
2451
+
2452
+ @property
2453
+ def is_completed(self) -> bool:
2454
+ """Check if job has completed successfully."""
2455
+ return self.state == SchedulerJobState.COMPLETED.value
2456
+
2457
+ @property
2458
+ def is_failed(self) -> bool:
2459
+ """Check if job has failed permanently."""
2460
+ return self.state == SchedulerJobState.FAILED.value
2461
+
2462
+ def mark_running(self) -> None:
2463
+ """Mark job as running."""
2464
+ self.state = SchedulerJobState.RUNNING.value
2465
+ self.last_run_time = datetime.utcnow()
2466
+ self.updated_at = datetime.utcnow()
2467
+
2468
+ def mark_completed(self, next_run_time: datetime | None = None) -> None:
2469
+ """Mark job as completed.
2470
+
2471
+ Args:
2472
+ next_run_time: Next scheduled run time (for recurring jobs).
2473
+ """
2474
+ if next_run_time:
2475
+ self.state = SchedulerJobState.PENDING.value
2476
+ self.next_run_time = next_run_time
2477
+ else:
2478
+ self.state = SchedulerJobState.COMPLETED.value
2479
+ self.retry_count = 0
2480
+ self.last_error = None
2481
+ self.updated_at = datetime.utcnow()
2482
+
2483
+ def mark_failed(self, error: str, can_retry: bool = True) -> None:
2484
+ """Mark job as failed.
2485
+
2486
+ Args:
2487
+ error: Error message.
2488
+ can_retry: Whether job can be retried.
2489
+ """
2490
+ self.last_error = error
2491
+ self.updated_at = datetime.utcnow()
2492
+
2493
+ if can_retry:
2494
+ self.state = SchedulerJobState.PENDING.value
2495
+ self.retry_count += 1
2496
+ else:
2497
+ self.state = SchedulerJobState.FAILED.value
2498
+
2499
+ def mark_misfired(self) -> None:
2500
+ """Mark job as misfired."""
2501
+ self.state = SchedulerJobState.MISFIRED.value
2502
+ self.updated_at = datetime.utcnow()
2503
+ if self.job_metadata is None:
2504
+ self.job_metadata = {}
2505
+ self.job_metadata["misfire_count"] = self.job_metadata.get("misfire_count", 0) + 1
2506
+ self.job_metadata["last_misfire_at"] = datetime.utcnow().isoformat()
2507
+
2508
+
2509
+ # =============================================================================
2510
+ # Phase 10: Data Lineage Models
2511
+ # =============================================================================
2512
+
2513
+
2514
+ class LineageNodeType(str, Enum):
2515
+ """Type of lineage node."""
2516
+
2517
+ SOURCE = "source"
2518
+ TRANSFORM = "transform"
2519
+ SINK = "sink"
2520
+
2521
+
2522
+ class LineageEdgeType(str, Enum):
2523
+ """Type of lineage edge."""
2524
+
2525
+ DERIVES_FROM = "derives_from"
2526
+ TRANSFORMS_TO = "transforms_to"
2527
+ JOINS_WITH = "joins_with"
2528
+ FILTERS_FROM = "filters_from"
2529
+
2530
+
2531
+ class LineageNode(Base, UUIDMixin, TimestampMixin):
2532
+ """Data lineage node model.
2533
+
2534
+ Represents a node in the data lineage graph (source, transformation, or sink).
2535
+
2536
+ Attributes:
2537
+ id: Unique identifier (UUID).
2538
+ name: Human-readable node name.
2539
+ node_type: Type of node (source, transform, sink).
2540
+ source_id: Optional reference to a data source.
2541
+ metadata_json: Additional metadata for the node.
2542
+ position_x: X coordinate for graph visualization.
2543
+ position_y: Y coordinate for graph visualization.
2544
+ """
2545
+
2546
+ __tablename__ = "lineage_nodes"
2547
+
2548
+ __table_args__ = (
2549
+ Index("idx_lineage_nodes_type", "node_type"),
2550
+ Index("idx_lineage_nodes_source", "source_id"),
2551
+ )
2552
+
2553
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2554
+ node_type: Mapped[str] = mapped_column(
2555
+ String(20),
2556
+ nullable=False,
2557
+ default=LineageNodeType.SOURCE.value,
2558
+ index=True,
2559
+ )
2560
+ source_id: Mapped[str | None] = mapped_column(
2561
+ String(36),
2562
+ ForeignKey("sources.id", ondelete="SET NULL"),
2563
+ nullable=True,
2564
+ index=True,
2565
+ )
2566
+ metadata_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
2567
+ position_x: Mapped[float | None] = mapped_column(Float, nullable=True)
2568
+ position_y: Mapped[float | None] = mapped_column(Float, nullable=True)
2569
+
2570
+ # Relationships
2571
+ source: Mapped[Source | None] = relationship(
2572
+ "Source",
2573
+ lazy="selectin",
2574
+ )
2575
+ outgoing_edges: Mapped[list[LineageEdge]] = relationship(
2576
+ "LineageEdge",
2577
+ foreign_keys="LineageEdge.source_node_id",
2578
+ back_populates="source_node",
2579
+ cascade="all, delete-orphan",
2580
+ lazy="selectin",
2581
+ )
2582
+ incoming_edges: Mapped[list[LineageEdge]] = relationship(
2583
+ "LineageEdge",
2584
+ foreign_keys="LineageEdge.target_node_id",
2585
+ back_populates="target_node",
2586
+ cascade="all, delete-orphan",
2587
+ lazy="selectin",
2588
+ )
2589
+
2590
+ @property
2591
+ def upstream_count(self) -> int:
2592
+ """Get number of upstream (incoming) connections."""
2593
+ return len(self.incoming_edges)
2594
+
2595
+ @property
2596
+ def downstream_count(self) -> int:
2597
+ """Get number of downstream (outgoing) connections."""
2598
+ return len(self.outgoing_edges)
2599
+
2600
+ @property
2601
+ def is_source_type(self) -> bool:
2602
+ """Check if this is a source node."""
2603
+ return self.node_type == LineageNodeType.SOURCE.value
2604
+
2605
+ @property
2606
+ def is_transform_type(self) -> bool:
2607
+ """Check if this is a transform node."""
2608
+ return self.node_type == LineageNodeType.TRANSFORM.value
2609
+
2610
+ @property
2611
+ def is_sink_type(self) -> bool:
2612
+ """Check if this is a sink node."""
2613
+ return self.node_type == LineageNodeType.SINK.value
2614
+
2615
+
2616
+ class LineageEdge(Base, UUIDMixin):
2617
+ """Data lineage edge model.
2618
+
2619
+ Represents a connection (data flow) between two lineage nodes.
2620
+
2621
+ Attributes:
2622
+ id: Unique identifier (UUID).
2623
+ source_node_id: ID of the source node (origin of data flow).
2624
+ target_node_id: ID of the target node (destination of data flow).
2625
+ edge_type: Type of relationship.
2626
+ metadata_json: Additional metadata for the edge.
2627
+ created_at: When the edge was created.
2628
+ """
2629
+
2630
+ __tablename__ = "lineage_edges"
2631
+
2632
+ __table_args__ = (
2633
+ Index("idx_lineage_edges_source", "source_node_id"),
2634
+ Index("idx_lineage_edges_target", "target_node_id"),
2635
+ Index(
2636
+ "idx_lineage_edges_unique",
2637
+ "source_node_id",
2638
+ "target_node_id",
2639
+ "edge_type",
2640
+ unique=True,
2641
+ ),
2642
+ )
2643
+
2644
+ source_node_id: Mapped[str] = mapped_column(
2645
+ String(36),
2646
+ ForeignKey("lineage_nodes.id", ondelete="CASCADE"),
2647
+ nullable=False,
2648
+ index=True,
2649
+ )
2650
+ target_node_id: Mapped[str] = mapped_column(
2651
+ String(36),
2652
+ ForeignKey("lineage_nodes.id", ondelete="CASCADE"),
2653
+ nullable=False,
2654
+ index=True,
2655
+ )
2656
+ edge_type: Mapped[str] = mapped_column(
2657
+ String(30),
2658
+ nullable=False,
2659
+ default=LineageEdgeType.DERIVES_FROM.value,
2660
+ index=True,
2661
+ )
2662
+ metadata_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
2663
+ created_at: Mapped[datetime] = mapped_column(
2664
+ DateTime,
2665
+ default=datetime.utcnow,
2666
+ nullable=False,
2667
+ )
2668
+
2669
+ # Relationships
2670
+ source_node: Mapped[LineageNode] = relationship(
2671
+ "LineageNode",
2672
+ foreign_keys=[source_node_id],
2673
+ back_populates="outgoing_edges",
2674
+ )
2675
+ target_node: Mapped[LineageNode] = relationship(
2676
+ "LineageNode",
2677
+ foreign_keys=[target_node_id],
2678
+ back_populates="incoming_edges",
2679
+ )
2680
+
2681
+
2682
+ # =============================================================================
2683
+ # OpenLineage Webhook Models
2684
+ # =============================================================================
2685
+
2686
+
2687
+ class OpenLineageEventType(str, Enum):
2688
+ """Types of OpenLineage events that can be emitted."""
2689
+
2690
+ JOB = "job"
2691
+ DATASET = "dataset"
2692
+ ALL = "all"
2693
+
2694
+
2695
+ class OpenLineageWebhook(Base, UUIDMixin, TimestampMixin):
2696
+ """OpenLineage webhook configuration model.
2697
+
2698
+ Stores configuration for emitting OpenLineage events to external endpoints.
2699
+
2700
+ Attributes:
2701
+ id: Unique identifier (UUID).
2702
+ name: Human-readable name for the webhook.
2703
+ url: Target URL for the webhook.
2704
+ is_active: Whether the webhook is enabled.
2705
+ headers_json: Custom headers as JSON (excluding auth).
2706
+ api_key: Optional API key for authentication.
2707
+ event_types: Types of events to emit (job, dataset, all).
2708
+ batch_size: Number of events per batch.
2709
+ timeout_seconds: Request timeout.
2710
+ last_sent_at: Timestamp of last successful emission.
2711
+ success_count: Total successful emissions.
2712
+ failure_count: Total failed emissions.
2713
+ last_error: Last error message if any.
2714
+ """
2715
+
2716
+ __tablename__ = "openlineage_webhooks"
2717
+
2718
+ __table_args__ = (
2719
+ Index("idx_openlineage_webhooks_active", "is_active"),
2720
+ )
2721
+
2722
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
2723
+ url: Mapped[str] = mapped_column(Text, nullable=False)
2724
+ is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
2725
+ headers_json: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True)
2726
+ api_key: Mapped[str | None] = mapped_column(Text, nullable=True)
2727
+ event_types: Mapped[str] = mapped_column(
2728
+ String(20),
2729
+ nullable=False,
2730
+ default=OpenLineageEventType.ALL.value,
2731
+ )
2732
+ batch_size: Mapped[int] = mapped_column(Integer, nullable=False, default=100)
2733
+ timeout_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
2734
+
2735
+ # Statistics
2736
+ last_sent_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
2737
+ success_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
2738
+ failure_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
2739
+ last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
2740
+
2741
+ @property
2742
+ def headers(self) -> dict[str, str]:
2743
+ """Get headers dictionary."""
2744
+ return self.headers_json or {}
2745
+
2746
+ @property
2747
+ def total_emissions(self) -> int:
2748
+ """Get total emission attempts."""
2749
+ return self.success_count + self.failure_count
2750
+
2751
+ @property
2752
+ def success_rate(self) -> float:
2753
+ """Get success rate as percentage."""
2754
+ if self.total_emissions == 0:
2755
+ return 100.0
2756
+ return (self.success_count / self.total_emissions) * 100
2757
+
2758
+ def record_success(self) -> None:
2759
+ """Record a successful emission."""
2760
+ self.success_count += 1
2761
+ self.last_sent_at = datetime.utcnow()
2762
+ self.last_error = None
2763
+
2764
+ def record_failure(self, error: str) -> None:
2765
+ """Record a failed emission."""
2766
+ self.failure_count += 1
2767
+ self.last_error = error
2768
+
2769
+ def activate(self) -> None:
2770
+ """Enable the webhook."""
2771
+ self.is_active = True
2772
+
2773
+ def deactivate(self) -> None:
2774
+ """Disable the webhook."""
2775
+ self.is_active = False
2776
+
2777
+
2778
+ # =============================================================================
2779
+ # Phase 10: Anomaly Detection Models
2780
+ # =============================================================================
2781
+
2782
+
2783
+ class AnomalyAlgorithm(str, Enum):
2784
+ """Supported anomaly detection algorithms."""
2785
+
2786
+ ISOLATION_FOREST = "isolation_forest"
2787
+ LOF = "lof"
2788
+ ONE_CLASS_SVM = "one_class_svm"
2789
+ DBSCAN = "dbscan"
2790
+ STATISTICAL = "statistical"
2791
+ AUTOENCODER = "autoencoder"
2792
+
2793
+
2794
+ class AnomalyDetectionStatus(str, Enum):
2795
+ """Status of an anomaly detection run."""
2796
+
2797
+ PENDING = "pending"
2798
+ RUNNING = "running"
2799
+ SUCCESS = "success"
2800
+ ERROR = "error"
2801
+
2802
+
2803
+ class AnomalyDetection(Base, UUIDMixin):
2804
+ """Anomaly detection result model.
2805
+
2806
+ Stores results from ML-based anomaly detection runs.
2807
+
2808
+ Attributes:
2809
+ id: Unique identifier (UUID).
2810
+ source_id: Reference to parent Source.
2811
+ status: Current status (pending, running, success, error).
2812
+ algorithm: Detection algorithm used.
2813
+ config: Algorithm-specific configuration.
2814
+ total_rows: Total rows analyzed.
2815
+ anomaly_count: Number of anomalies found.
2816
+ anomaly_rate: Rate of anomalies (0-1).
2817
+ columns_analyzed: List of columns that were analyzed.
2818
+ result_json: Full detection result as JSON.
2819
+ duration_ms: Execution time in milliseconds.
2820
+ error_message: Error message if failed.
2821
+ """
2822
+
2823
+ __tablename__ = "anomaly_detections"
2824
+
2825
+ __table_args__ = (
2826
+ Index("idx_anomaly_detections_source_created", "source_id", "created_at"),
2827
+ Index("idx_anomaly_detections_status", "status"),
2828
+ Index("idx_anomaly_detections_algorithm", "algorithm"),
2829
+ )
2830
+
2831
+ source_id: Mapped[str] = mapped_column(
2832
+ String(36),
2833
+ ForeignKey("sources.id", ondelete="CASCADE"),
2834
+ nullable=False,
2835
+ index=True,
2836
+ )
2837
+
2838
+ # Status tracking
2839
+ status: Mapped[str] = mapped_column(
2840
+ String(20),
2841
+ nullable=False,
2842
+ default=AnomalyDetectionStatus.PENDING.value,
2843
+ index=True,
2844
+ )
2845
+
2846
+ # Algorithm configuration
2847
+ algorithm: Mapped[str] = mapped_column(
2848
+ String(30),
2849
+ nullable=False,
2850
+ default=AnomalyAlgorithm.ISOLATION_FOREST.value,
2851
+ index=True,
2852
+ )
2853
+ config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
2854
+
2855
+ # Results summary
2856
+ total_rows: Mapped[int | None] = mapped_column(Integer, nullable=True)
2857
+ anomaly_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
2858
+ anomaly_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
2859
+ columns_analyzed: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
2860
+
2861
+ # Full result and timing
2862
+ result_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
2863
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
2864
+ duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
2865
+
2866
+ # Timestamps
2867
+ created_at: Mapped[datetime] = mapped_column(
2868
+ DateTime, default=datetime.utcnow, nullable=False
2869
+ )
2870
+ started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
2871
+ completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
2872
+
2873
+ # Relationships
2874
+ source: Mapped[Source] = relationship(
2875
+ "Source",
2876
+ backref="anomaly_detections",
2877
+ )
2878
+
2879
+ @property
2880
+ def is_complete(self) -> bool:
2881
+ """Check if detection has completed (success or error)."""
2882
+ return self.status in (
2883
+ AnomalyDetectionStatus.SUCCESS.value,
2884
+ AnomalyDetectionStatus.ERROR.value,
2885
+ )
2886
+
2887
+ @property
2888
+ def anomalies(self) -> list[dict[str, Any]]:
2889
+ """Get list of anomaly records from result JSON."""
2890
+ if self.result_json and "anomalies" in self.result_json:
2891
+ return self.result_json["anomalies"]
2892
+ return []
2893
+
2894
+ @property
2895
+ def column_summaries(self) -> list[dict[str, Any]]:
2896
+ """Get per-column anomaly summaries from result JSON."""
2897
+ if self.result_json and "column_summaries" in self.result_json:
2898
+ return self.result_json["column_summaries"]
2899
+ return []
2900
+
2901
+ def mark_started(self) -> None:
2902
+ """Mark detection as started."""
2903
+ self.status = AnomalyDetectionStatus.RUNNING.value
2904
+ self.started_at = datetime.utcnow()
2905
+
2906
+ def mark_completed(
2907
+ self,
2908
+ anomaly_count: int,
2909
+ anomaly_rate: float,
2910
+ result: dict[str, Any],
2911
+ ) -> None:
2912
+ """Mark detection as completed with results."""
2913
+ self.status = AnomalyDetectionStatus.SUCCESS.value
2914
+ self.anomaly_count = anomaly_count
2915
+ self.anomaly_rate = anomaly_rate
2916
+ self.result_json = result
2917
+ self.completed_at = datetime.utcnow()
2918
+
2919
+ if self.started_at:
2920
+ delta = self.completed_at - self.started_at
2921
+ self.duration_ms = int(delta.total_seconds() * 1000)
2922
+
2923
+ def mark_error(self, message: str) -> None:
2924
+ """Mark detection as errored."""
2925
+ self.status = AnomalyDetectionStatus.ERROR.value
2926
+ self.error_message = message
2927
+ self.completed_at = datetime.utcnow()
2928
+
2929
+ if self.started_at:
2930
+ delta = self.completed_at - self.started_at
2931
+ self.duration_ms = int(delta.total_seconds() * 1000)
2932
+
2933
+
2934
+ class AnomalyExplanation(Base, UUIDMixin):
2935
+ """SHAP/LIME explanations for anomaly detection results.
2936
+
2937
+ Stores feature-level explanations for individual anomalous rows,
2938
+ providing interpretability for ML-based anomaly detection.
2939
+
2940
+ Attributes:
2941
+ id: Unique identifier (UUID).
2942
+ detection_id: Reference to parent AnomalyDetection.
2943
+ row_index: Row index in the original dataset.
2944
+ anomaly_score: Anomaly score for this row.
2945
+ feature_contributions: JSON array of feature contributions.
2946
+ total_shap: Sum of all SHAP values.
2947
+ summary: Human-readable explanation summary.
2948
+ generated_at: When the explanation was generated.
2949
+ """
2950
+
2951
+ __tablename__ = "anomaly_explanations"
2952
+
2953
+ __table_args__ = (
2954
+ Index("idx_anomaly_explanations_detection", "detection_id"),
2955
+ Index("idx_anomaly_explanations_row", "detection_id", "row_index"),
2956
+ )
2957
+
2958
+ detection_id: Mapped[str] = mapped_column(
2959
+ String(36),
2960
+ ForeignKey("anomaly_detections.id", ondelete="CASCADE"),
2961
+ nullable=False,
2962
+ index=True,
2963
+ )
2964
+
2965
+ # Row identification
2966
+ row_index: Mapped[int] = mapped_column(Integer, nullable=False)
2967
+
2968
+ # Anomaly information
2969
+ anomaly_score: Mapped[float] = mapped_column(Float, nullable=False)
2970
+
2971
+ # SHAP/LIME explanation data
2972
+ feature_contributions: Mapped[list[dict[str, Any]]] = mapped_column(
2973
+ JSON,
2974
+ nullable=False,
2975
+ default=list,
2976
+ )
2977
+ total_shap: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
2978
+
2979
+ # Human-readable summary
2980
+ summary: Mapped[str] = mapped_column(Text, nullable=False)
2981
+
2982
+ # Timestamp
2983
+ generated_at: Mapped[datetime] = mapped_column(
2984
+ DateTime,
2985
+ default=datetime.utcnow,
2986
+ nullable=False,
2987
+ )
2988
+
2989
+ # Relationships
2990
+ detection: Mapped[AnomalyDetection] = relationship(
2991
+ "AnomalyDetection",
2992
+ backref="explanations",
2993
+ )
2994
+
2995
+ @property
2996
+ def top_features(self) -> list[str]:
2997
+ """Get names of top contributing features."""
2998
+ if not self.feature_contributions:
2999
+ return []
3000
+ return [fc.get("feature", "") for fc in self.feature_contributions[:5]]
3001
+
3002
+ @property
3003
+ def contribution_count(self) -> int:
3004
+ """Get number of feature contributions stored."""
3005
+ return len(self.feature_contributions) if self.feature_contributions else 0
3006
+
3007
+
3008
+ class BatchDetectionStatus(str, Enum):
3009
+ """Status of a batch anomaly detection job."""
3010
+
3011
+ PENDING = "pending"
3012
+ RUNNING = "running"
3013
+ COMPLETED = "completed"
3014
+ PARTIAL = "partial" # Some sources completed with errors
3015
+ ERROR = "error"
3016
+ CANCELLED = "cancelled"
3017
+
3018
+
3019
+ class AnomalyBatchJob(Base, UUIDMixin, TimestampMixin):
3020
+ """Batch anomaly detection job model.
3021
+
3022
+ Stores configuration and results for batch anomaly detection
3023
+ across multiple data sources.
3024
+
3025
+ Attributes:
3026
+ id: Unique identifier (UUID).
3027
+ name: Optional job name.
3028
+ status: Current status of the batch job.
3029
+ algorithm: Detection algorithm to use.
3030
+ config: Algorithm-specific configuration.
3031
+ source_ids: List of source IDs to process.
3032
+ total_sources: Total number of sources.
3033
+ completed_sources: Number of completed sources.
3034
+ failed_sources: Number of failed sources.
3035
+ total_anomalies: Total anomalies found across all sources.
3036
+ results_json: Detection results per source.
3037
+ error_message: Error message if job failed.
3038
+ duration_ms: Total execution time in milliseconds.
3039
+ """
3040
+
3041
+ __tablename__ = "anomaly_batch_jobs"
3042
+
3043
+ __table_args__ = (
3044
+ Index("idx_anomaly_batch_jobs_status", "status"),
3045
+ Index("idx_anomaly_batch_jobs_created", "created_at"),
3046
+ )
3047
+
3048
+ name: Mapped[str | None] = mapped_column(String(255), nullable=True)
3049
+
3050
+ # Status tracking
3051
+ status: Mapped[str] = mapped_column(
3052
+ String(20),
3053
+ nullable=False,
3054
+ default=BatchDetectionStatus.PENDING.value,
3055
+ index=True,
3056
+ )
3057
+
3058
+ # Algorithm configuration
3059
+ algorithm: Mapped[str] = mapped_column(
3060
+ String(30),
3061
+ nullable=False,
3062
+ default=AnomalyAlgorithm.ISOLATION_FOREST.value,
3063
+ )
3064
+ config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
3065
+
3066
+ # Sources to process
3067
+ source_ids: Mapped[list[str]] = mapped_column(JSON, nullable=False)
3068
+
3069
+ # Progress tracking
3070
+ total_sources: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
3071
+ completed_sources: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
3072
+ failed_sources: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
3073
+ current_source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
3074
+
3075
+ # Aggregate results
3076
+ total_anomalies: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
3077
+ total_rows_analyzed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
3078
+
3079
+ # Per-source results: {source_id: {detection_id, status, anomaly_count, anomaly_rate, ...}}
3080
+ results_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
3081
+
3082
+ # Error handling
3083
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
3084
+
3085
+ # Timing
3086
+ duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
3087
+ started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3088
+ completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3089
+
3090
+ @property
3091
+ def progress_percent(self) -> float:
3092
+ """Get progress as percentage."""
3093
+ if self.total_sources == 0:
3094
+ return 0.0
3095
+ return ((self.completed_sources + self.failed_sources) / self.total_sources) * 100
3096
+
3097
+ @property
3098
+ def average_anomaly_rate(self) -> float:
3099
+ """Get average anomaly rate across all successful sources."""
3100
+ if not self.results_json:
3101
+ return 0.0
3102
+ rates = [
3103
+ r["anomaly_rate"]
3104
+ for r in self.results_json.values()
3105
+ if r.get("status") == "success" and r.get("anomaly_rate") is not None
3106
+ ]
3107
+ return sum(rates) / len(rates) if rates else 0.0
3108
+
3109
+ @property
3110
+ def is_complete(self) -> bool:
3111
+ """Check if batch job has completed."""
3112
+ return self.status in (
3113
+ BatchDetectionStatus.COMPLETED.value,
3114
+ BatchDetectionStatus.PARTIAL.value,
3115
+ BatchDetectionStatus.ERROR.value,
3116
+ BatchDetectionStatus.CANCELLED.value,
3117
+ )
3118
+
3119
+ def mark_started(self) -> None:
3120
+ """Mark batch job as started."""
3121
+ self.status = BatchDetectionStatus.RUNNING.value
3122
+ self.started_at = datetime.utcnow()
3123
+
3124
+ def update_progress(
3125
+ self,
3126
+ source_id: str,
3127
+ detection_id: str,
3128
+ status: str,
3129
+ anomaly_count: int = 0,
3130
+ anomaly_rate: float = 0.0,
3131
+ total_rows: int = 0,
3132
+ error_message: str | None = None,
3133
+ ) -> None:
3134
+ """Update progress for a single source."""
3135
+ if self.results_json is None:
3136
+ self.results_json = {}
3137
+
3138
+ self.results_json[source_id] = {
3139
+ "detection_id": detection_id,
3140
+ "status": status,
3141
+ "anomaly_count": anomaly_count,
3142
+ "anomaly_rate": anomaly_rate,
3143
+ "total_rows": total_rows,
3144
+ "error_message": error_message,
3145
+ }
3146
+
3147
+ if status == "success":
3148
+ self.completed_sources += 1
3149
+ self.total_anomalies += anomaly_count
3150
+ self.total_rows_analyzed += total_rows
3151
+ elif status == "error":
3152
+ self.failed_sources += 1
3153
+
3154
+ def mark_completed(self) -> None:
3155
+ """Mark batch job as completed."""
3156
+ if self.failed_sources > 0 and self.completed_sources > 0:
3157
+ self.status = BatchDetectionStatus.PARTIAL.value
3158
+ elif self.failed_sources == self.total_sources:
3159
+ self.status = BatchDetectionStatus.ERROR.value
3160
+ else:
3161
+ self.status = BatchDetectionStatus.COMPLETED.value
3162
+
3163
+ self.completed_at = datetime.utcnow()
3164
+ self.current_source_id = None
3165
+
3166
+ if self.started_at:
3167
+ delta = self.completed_at - self.started_at
3168
+ self.duration_ms = int(delta.total_seconds() * 1000)
3169
+
3170
+ def mark_error(self, message: str) -> None:
3171
+ """Mark batch job as errored."""
3172
+ self.status = BatchDetectionStatus.ERROR.value
3173
+ self.error_message = message
3174
+ self.completed_at = datetime.utcnow()
3175
+ self.current_source_id = None
3176
+
3177
+ if self.started_at:
3178
+ delta = self.completed_at - self.started_at
3179
+ self.duration_ms = int(delta.total_seconds() * 1000)
3180
+
3181
+ def mark_cancelled(self) -> None:
3182
+ """Mark batch job as cancelled."""
3183
+ self.status = BatchDetectionStatus.CANCELLED.value
3184
+ self.completed_at = datetime.utcnow()
3185
+ self.current_source_id = None
3186
+
3187
+ if self.started_at:
3188
+ delta = self.completed_at - self.started_at
3189
+ self.duration_ms = int(delta.total_seconds() * 1000)
3190
+
3191
+
3192
+ # =============================================================================
3193
+ # Phase 10: Model Monitoring Models
3194
+ # =============================================================================
3195
+
3196
+
3197
+ class ModelStatus(str, Enum):
3198
+ """Status of a monitored model."""
3199
+
3200
+ ACTIVE = "active"
3201
+ PAUSED = "paused"
3202
+ DEGRADED = "degraded"
3203
+ ERROR = "error"
3204
+
3205
+
3206
+ class AlertSeverityLevel(str, Enum):
3207
+ """Alert severity levels for model monitoring."""
3208
+
3209
+ CRITICAL = "critical"
3210
+ WARNING = "warning"
3211
+ INFO = "info"
3212
+
3213
+
3214
+ class MetricTypeEnum(str, Enum):
3215
+ """Types of metrics collected."""
3216
+
3217
+ LATENCY = "latency"
3218
+ THROUGHPUT = "throughput"
3219
+ ERROR_RATE = "error_rate"
3220
+ NULL_RATE = "null_rate"
3221
+ TYPE_VIOLATION = "type_violation"
3222
+ DRIFT_SCORE = "drift_score"
3223
+ CUSTOM = "custom"
3224
+
3225
+
3226
+ class AlertRuleTypeEnum(str, Enum):
3227
+ """Types of alert rules."""
3228
+
3229
+ THRESHOLD = "threshold"
3230
+ STATISTICAL = "statistical"
3231
+ TREND = "trend"
3232
+
3233
+
3234
+ class AlertHandlerTypeEnum(str, Enum):
3235
+ """Types of alert handlers."""
3236
+
3237
+ SLACK = "slack"
3238
+ WEBHOOK = "webhook"
3239
+ EMAIL = "email"
3240
+
3241
+
3242
+ class MonitoredModel(Base, UUIDMixin, TimestampMixin):
3243
+ """Monitored ML model registration model.
3244
+
3245
+ Represents a registered ML model for monitoring performance,
3246
+ drift, and data quality metrics.
3247
+
3248
+ Attributes:
3249
+ id: Unique identifier (UUID).
3250
+ name: Model name/identifier.
3251
+ version: Model version string.
3252
+ description: Model description.
3253
+ status: Monitoring status (active, paused, degraded, error).
3254
+ config: JSON configuration for monitoring settings.
3255
+ metadata_json: Additional model metadata.
3256
+ prediction_count: Total predictions recorded.
3257
+ last_prediction_at: Timestamp of last prediction.
3258
+ current_drift_score: Current drift score.
3259
+ health_score: Model health score (0-100).
3260
+ """
3261
+
3262
+ __tablename__ = "monitored_models"
3263
+
3264
+ __table_args__ = (
3265
+ Index("idx_monitored_models_name", "name"),
3266
+ Index("idx_monitored_models_status", "status"),
3267
+ )
3268
+
3269
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
3270
+ version: Mapped[str] = mapped_column(String(50), nullable=False, default="1.0.0")
3271
+ description: Mapped[str | None] = mapped_column(Text, nullable=True)
3272
+ status: Mapped[str] = mapped_column(
3273
+ String(20),
3274
+ nullable=False,
3275
+ default=ModelStatus.ACTIVE.value,
3276
+ index=True,
3277
+ )
3278
+ config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
3279
+ metadata_json: Mapped[dict[str, Any] | None] = mapped_column(
3280
+ "metadata", JSON, nullable=True
3281
+ )
3282
+ prediction_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
3283
+ last_prediction_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3284
+ current_drift_score: Mapped[float | None] = mapped_column(Float, nullable=True)
3285
+ health_score: Mapped[float] = mapped_column(Float, default=100.0, nullable=False)
3286
+
3287
+ # Relationships
3288
+ predictions: Mapped[list[ModelPrediction]] = relationship(
3289
+ "ModelPrediction",
3290
+ back_populates="model",
3291
+ cascade="all, delete-orphan",
3292
+ lazy="selectin",
3293
+ )
3294
+ metrics: Mapped[list[ModelMetric]] = relationship(
3295
+ "ModelMetric",
3296
+ back_populates="model",
3297
+ cascade="all, delete-orphan",
3298
+ lazy="selectin",
3299
+ )
3300
+ alert_rules: Mapped[list[ModelAlertRule]] = relationship(
3301
+ "ModelAlertRule",
3302
+ back_populates="model",
3303
+ cascade="all, delete-orphan",
3304
+ lazy="selectin",
3305
+ )
3306
+ alerts: Mapped[list[ModelAlert]] = relationship(
3307
+ "ModelAlert",
3308
+ back_populates="model",
3309
+ cascade="all, delete-orphan",
3310
+ lazy="selectin",
3311
+ )
3312
+
3313
+ @property
3314
+ def is_active(self) -> bool:
3315
+ """Check if model is actively monitored."""
3316
+ return self.status == ModelStatus.ACTIVE.value
3317
+
3318
+ @property
3319
+ def is_healthy(self) -> bool:
3320
+ """Check if model is in healthy state."""
3321
+ return self.health_score >= 70.0
3322
+
3323
+ @property
3324
+ def has_drift(self) -> bool:
3325
+ """Check if model has significant drift."""
3326
+ threshold = self.config.get("drift_threshold", 0.1)
3327
+ return (self.current_drift_score or 0.0) > threshold
3328
+
3329
+ def record_prediction(self) -> None:
3330
+ """Record a new prediction."""
3331
+ self.prediction_count += 1
3332
+ self.last_prediction_at = datetime.utcnow()
3333
+
3334
+ def update_drift_score(self, score: float) -> None:
3335
+ """Update the drift score."""
3336
+ self.current_drift_score = score
3337
+ # Update health score based on drift
3338
+ if score > 0.3:
3339
+ self.status = ModelStatus.DEGRADED.value
3340
+ self.health_score = max(0.0, 100.0 - score * 100)
3341
+ elif score > 0.1:
3342
+ self.health_score = max(50.0, 100.0 - score * 50)
3343
+
3344
+ def pause(self) -> None:
3345
+ """Pause monitoring for this model."""
3346
+ self.status = ModelStatus.PAUSED.value
3347
+
3348
+ def resume(self) -> None:
3349
+ """Resume monitoring for this model."""
3350
+ self.status = ModelStatus.ACTIVE.value
3351
+
3352
+
3353
+ class ModelPrediction(Base, UUIDMixin):
3354
+ """Model prediction record for monitoring.
3355
+
3356
+ Stores individual predictions made by a model for tracking
3357
+ performance and drift over time.
3358
+
3359
+ Attributes:
3360
+ id: Unique identifier (UUID).
3361
+ model_id: Reference to MonitoredModel.
3362
+ features: JSON of input features.
3363
+ prediction: The model's output prediction.
3364
+ actual: The actual value (if available).
3365
+ latency_ms: Prediction latency in milliseconds.
3366
+ metadata_json: Additional prediction metadata.
3367
+ recorded_at: When the prediction was recorded.
3368
+ """
3369
+
3370
+ __tablename__ = "model_predictions"
3371
+
3372
+ __table_args__ = (
3373
+ Index("idx_model_predictions_model", "model_id", "recorded_at"),
3374
+ Index("idx_model_predictions_time", "recorded_at"),
3375
+ )
3376
+
3377
+ model_id: Mapped[str] = mapped_column(
3378
+ String(36),
3379
+ ForeignKey("monitored_models.id", ondelete="CASCADE"),
3380
+ nullable=False,
3381
+ index=True,
3382
+ )
3383
+ features: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
3384
+ prediction: Mapped[Any] = mapped_column(JSON, nullable=False)
3385
+ actual: Mapped[Any | None] = mapped_column(JSON, nullable=True)
3386
+ latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
3387
+ metadata_json: Mapped[dict[str, Any] | None] = mapped_column(
3388
+ "metadata", JSON, nullable=True
3389
+ )
3390
+ recorded_at: Mapped[datetime] = mapped_column(
3391
+ DateTime, default=datetime.utcnow, nullable=False
3392
+ )
3393
+
3394
+ # Relationships
3395
+ model: Mapped[MonitoredModel] = relationship(
3396
+ "MonitoredModel",
3397
+ back_populates="predictions",
3398
+ )
3399
+
3400
+ @property
3401
+ def has_actual(self) -> bool:
3402
+ """Check if actual value is available."""
3403
+ return self.actual is not None
3404
+
3405
+
3406
+ class ModelMetric(Base, UUIDMixin):
3407
+ """Model performance metric record.
3408
+
3409
+ Stores aggregated metric values for a monitored model.
3410
+
3411
+ Attributes:
3412
+ id: Unique identifier (UUID).
3413
+ model_id: Reference to MonitoredModel.
3414
+ metric_type: Type of metric (latency, throughput, etc.).
3415
+ metric_name: Name of the metric.
3416
+ value: Metric value.
3417
+ labels: JSON labels for the metric.
3418
+ recorded_at: When the metric was recorded.
3419
+ """
3420
+
3421
+ __tablename__ = "model_metrics"
3422
+
3423
+ __table_args__ = (
3424
+ Index("idx_model_metrics_model", "model_id", "recorded_at"),
3425
+ Index("idx_model_metrics_type", "metric_type"),
3426
+ Index("idx_model_metrics_name", "metric_name"),
3427
+ )
3428
+
3429
+ model_id: Mapped[str] = mapped_column(
3430
+ String(36),
3431
+ ForeignKey("monitored_models.id", ondelete="CASCADE"),
3432
+ nullable=False,
3433
+ index=True,
3434
+ )
3435
+ metric_type: Mapped[str] = mapped_column(
3436
+ String(30),
3437
+ nullable=False,
3438
+ index=True,
3439
+ )
3440
+ metric_name: Mapped[str] = mapped_column(String(100), nullable=False)
3441
+ value: Mapped[float] = mapped_column(Float, nullable=False)
3442
+ labels: Mapped[dict[str, str] | None] = mapped_column(JSON, nullable=True)
3443
+ recorded_at: Mapped[datetime] = mapped_column(
3444
+ DateTime, default=datetime.utcnow, nullable=False
3445
+ )
3446
+
3447
+ # Relationships
3448
+ model: Mapped[MonitoredModel] = relationship(
3449
+ "MonitoredModel",
3450
+ back_populates="metrics",
3451
+ )
3452
+
3453
+
3454
+ class ModelAlertRule(Base, UUIDMixin, TimestampMixin):
3455
+ """Alert rule for model monitoring.
3456
+
3457
+ Defines conditions that trigger alerts for a monitored model.
3458
+
3459
+ Attributes:
3460
+ id: Unique identifier (UUID).
3461
+ model_id: Reference to MonitoredModel.
3462
+ name: Rule name.
3463
+ rule_type: Type of rule (threshold, statistical, trend).
3464
+ severity: Alert severity level.
3465
+ config: Rule-specific configuration.
3466
+ is_active: Whether rule is active.
3467
+ last_triggered_at: When last triggered.
3468
+ trigger_count: Total trigger count.
3469
+ """
3470
+
3471
+ __tablename__ = "model_alert_rules"
3472
+
3473
+ __table_args__ = (
3474
+ Index("idx_model_alert_rules_model", "model_id"),
3475
+ Index("idx_model_alert_rules_type", "rule_type"),
3476
+ )
3477
+
3478
+ model_id: Mapped[str] = mapped_column(
3479
+ String(36),
3480
+ ForeignKey("monitored_models.id", ondelete="CASCADE"),
3481
+ nullable=False,
3482
+ index=True,
3483
+ )
3484
+ name: Mapped[str] = mapped_column(String(255), nullable=False)
3485
+ rule_type: Mapped[str] = mapped_column(
3486
+ String(20),
3487
+ nullable=False,
3488
+ index=True,
3489
+ )
3490
+ severity: Mapped[str] = mapped_column(
3491
+ String(20),
3492
+ nullable=False,
3493
+ default=AlertSeverityLevel.WARNING.value,
3494
+ )
3495
+ config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
3496
+ is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
3497
+ last_triggered_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3498
+ trigger_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
3499
+
3500
+ # Relationships
3501
+ model: Mapped[MonitoredModel] = relationship(
3502
+ "MonitoredModel",
3503
+ back_populates="alert_rules",
3504
+ )
3505
+ alerts: Mapped[list[ModelAlert]] = relationship(
3506
+ "ModelAlert",
3507
+ back_populates="rule",
3508
+ cascade="all, delete-orphan",
3509
+ lazy="selectin",
3510
+ )
3511
+
3512
+ def trigger(self) -> None:
3513
+ """Record a trigger of this rule."""
3514
+ self.last_triggered_at = datetime.utcnow()
3515
+ self.trigger_count += 1
3516
+
3517
+ def activate(self) -> None:
3518
+ """Activate this rule."""
3519
+ self.is_active = True
3520
+
3521
+ def deactivate(self) -> None:
3522
+ """Deactivate this rule."""
3523
+ self.is_active = False
3524
+
3525
+
3526
+ class ModelAlertHandler(Base, UUIDMixin, TimestampMixin):
3527
+ """Alert handler for model monitoring.
3528
+
3529
+ Defines where and how alerts are sent.
3530
+
3531
+ Attributes:
3532
+ id: Unique identifier (UUID).
3533
+ name: Handler name.
3534
+ handler_type: Type of handler (slack, webhook, email).
3535
+ config: Handler-specific configuration.
3536
+ is_active: Whether handler is active.
3537
+ last_sent_at: When last alert was sent.
3538
+ send_count: Total alerts sent.
3539
+ failure_count: Total send failures.
3540
+ """
3541
+
3542
+ __tablename__ = "model_alert_handlers"
3543
+
3544
+ __table_args__ = (Index("idx_model_alert_handlers_type", "handler_type"),)
3545
+
3546
+ name: Mapped[str] = mapped_column(String(255), nullable=False)
3547
+ handler_type: Mapped[str] = mapped_column(
3548
+ String(20),
3549
+ nullable=False,
3550
+ index=True,
3551
+ )
3552
+ config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
3553
+ is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
3554
+ last_sent_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3555
+ send_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
3556
+ failure_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
3557
+
3558
+ def record_send(self, success: bool = True) -> None:
3559
+ """Record a send attempt."""
3560
+ self.last_sent_at = datetime.utcnow()
3561
+ if success:
3562
+ self.send_count += 1
3563
+ else:
3564
+ self.failure_count += 1
3565
+
3566
+ def activate(self) -> None:
3567
+ """Activate this handler."""
3568
+ self.is_active = True
3569
+
3570
+ def deactivate(self) -> None:
3571
+ """Deactivate this handler."""
3572
+ self.is_active = False
3573
+
3574
+
3575
+ class ModelAlert(Base, UUIDMixin):
3576
+ """Alert instance for model monitoring.
3577
+
3578
+ Represents a triggered alert from an alert rule.
3579
+
3580
+ Attributes:
3581
+ id: Unique identifier (UUID).
3582
+ model_id: Reference to MonitoredModel.
3583
+ rule_id: Reference to ModelAlertRule.
3584
+ severity: Alert severity.
3585
+ message: Alert message.
3586
+ metric_value: Value that triggered the alert.
3587
+ threshold_value: Threshold that was exceeded.
3588
+ acknowledged: Whether alert is acknowledged.
3589
+ acknowledged_by: Who acknowledged.
3590
+ acknowledged_at: When acknowledged.
3591
+ resolved: Whether alert is resolved.
3592
+ resolved_at: When resolved.
3593
+ created_at: When alert was created.
3594
+ """
3595
+
3596
+ __tablename__ = "model_alerts"
3597
+
3598
+ __table_args__ = (
3599
+ Index("idx_model_alerts_model", "model_id", "created_at"),
3600
+ Index("idx_model_alerts_rule", "rule_id"),
3601
+ Index("idx_model_alerts_resolved", "resolved"),
3602
+ )
3603
+
3604
+ model_id: Mapped[str] = mapped_column(
3605
+ String(36),
3606
+ ForeignKey("monitored_models.id", ondelete="CASCADE"),
3607
+ nullable=False,
3608
+ index=True,
3609
+ )
3610
+ rule_id: Mapped[str] = mapped_column(
3611
+ String(36),
3612
+ ForeignKey("model_alert_rules.id", ondelete="CASCADE"),
3613
+ nullable=False,
3614
+ index=True,
3615
+ )
3616
+ severity: Mapped[str] = mapped_column(
3617
+ String(20),
3618
+ nullable=False,
3619
+ default=AlertSeverityLevel.WARNING.value,
3620
+ )
3621
+ message: Mapped[str] = mapped_column(Text, nullable=False)
3622
+ metric_value: Mapped[float | None] = mapped_column(Float, nullable=True)
3623
+ threshold_value: Mapped[float | None] = mapped_column(Float, nullable=True)
3624
+ acknowledged: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
3625
+ acknowledged_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
3626
+ acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3627
+ resolved: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
3628
+ resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3629
+ created_at: Mapped[datetime] = mapped_column(
3630
+ DateTime, default=datetime.utcnow, nullable=False
3631
+ )
3632
+ updated_at: Mapped[datetime] = mapped_column(
3633
+ DateTime,
3634
+ default=datetime.utcnow,
3635
+ onupdate=datetime.utcnow,
3636
+ nullable=False,
3637
+ )
3638
+
3639
+ # Relationships
3640
+ model: Mapped[MonitoredModel] = relationship(
3641
+ "MonitoredModel",
3642
+ back_populates="alerts",
3643
+ )
3644
+ rule: Mapped[ModelAlertRule] = relationship(
3645
+ "ModelAlertRule",
3646
+ back_populates="alerts",
3647
+ )
3648
+
3649
+ @property
3650
+ def is_active(self) -> bool:
3651
+ """Check if alert is active (not resolved)."""
3652
+ return not self.resolved
3653
+
3654
+ def acknowledge(self, by: str) -> None:
3655
+ """Acknowledge this alert."""
3656
+ self.acknowledged = True
3657
+ self.acknowledged_by = by
3658
+ self.acknowledged_at = datetime.utcnow()
3659
+
3660
+ def resolve(self) -> None:
3661
+ """Resolve this alert."""
3662
+ self.resolved = True
3663
+ self.resolved_at = datetime.utcnow()
3664
+
3665
+
3666
+ # =============================================================================
3667
+ # Phase 10: Drift Monitor Models
3668
+ # =============================================================================
3669
+
3670
+
3671
+ class DriftMonitorStatus(str, Enum):
3672
+ """Status of a drift monitor."""
3673
+
3674
+ ACTIVE = "active"
3675
+ PAUSED = "paused"
3676
+ ERROR = "error"
3677
+
3678
+
3679
+ class DriftAlertStatus(str, Enum):
3680
+ """Status of a drift alert."""
3681
+
3682
+ ACTIVE = "active"
3683
+ ACKNOWLEDGED = "acknowledged"
3684
+ RESOLVED = "resolved"
3685
+
3686
+
3687
+ class DriftMonitor(Base, UUIDMixin, TimestampMixin):
3688
+ """Drift monitoring configuration model.
3689
+
3690
+ Defines a drift monitor that compares baseline and current data sources.
3691
+
3692
+ Attributes:
3693
+ id: Unique identifier (UUID).
3694
+ name: Monitor name.
3695
+ baseline_source_id: Reference to baseline Source.
3696
+ current_source_id: Reference to current Source.
3697
+ status: Monitor status.
3698
+ method: Drift detection method.
3699
+ threshold_critical: Critical drift threshold.
3700
+ threshold_high: High drift threshold.
3701
+ columns: Columns to monitor (None = all).
3702
+ schedule_cron: Optional cron expression for scheduling.
3703
+ last_run_at: Last run timestamp.
3704
+ next_run_at: Next scheduled run.
3705
+ config: Additional configuration.
3706
+ """
3707
+
3708
+ __tablename__ = "drift_monitors"
3709
+
3710
+ __table_args__ = (
3711
+ Index("idx_drift_monitors_status", "status"),
3712
+ Index("idx_drift_monitors_baseline", "baseline_source_id"),
3713
+ Index("idx_drift_monitors_current", "current_source_id"),
3714
+ )
3715
+
3716
+ name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
3717
+ baseline_source_id: Mapped[str] = mapped_column(
3718
+ String(36),
3719
+ ForeignKey("sources.id", ondelete="CASCADE"),
3720
+ nullable=False,
3721
+ index=True,
3722
+ )
3723
+ current_source_id: Mapped[str] = mapped_column(
3724
+ String(36),
3725
+ ForeignKey("sources.id", ondelete="CASCADE"),
3726
+ nullable=False,
3727
+ index=True,
3728
+ )
3729
+ status: Mapped[str] = mapped_column(
3730
+ String(20),
3731
+ nullable=False,
3732
+ default=DriftMonitorStatus.ACTIVE.value,
3733
+ index=True,
3734
+ )
3735
+ method: Mapped[str] = mapped_column(String(30), nullable=False, default="auto")
3736
+ threshold_critical: Mapped[float] = mapped_column(Float, default=0.3, nullable=False)
3737
+ threshold_high: Mapped[float] = mapped_column(Float, default=0.1, nullable=False)
3738
+ columns: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
3739
+ schedule_cron: Mapped[str | None] = mapped_column(String(100), nullable=True)
3740
+ last_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3741
+ next_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3742
+ config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
3743
+
3744
+ # Relationships
3745
+ baseline_source: Mapped[Source] = relationship(
3746
+ "Source",
3747
+ foreign_keys=[baseline_source_id],
3748
+ lazy="selectin",
3749
+ )
3750
+ current_source: Mapped[Source] = relationship(
3751
+ "Source",
3752
+ foreign_keys=[current_source_id],
3753
+ lazy="selectin",
3754
+ )
3755
+ runs: Mapped[list[DriftMonitorRun]] = relationship(
3756
+ "DriftMonitorRun",
3757
+ back_populates="monitor",
3758
+ cascade="all, delete-orphan",
3759
+ lazy="selectin",
3760
+ order_by="desc(DriftMonitorRun.created_at)",
3761
+ )
3762
+ alerts: Mapped[list[DriftAlert]] = relationship(
3763
+ "DriftAlert",
3764
+ back_populates="monitor",
3765
+ cascade="all, delete-orphan",
3766
+ lazy="selectin",
3767
+ )
3768
+
3769
+ @property
3770
+ def is_active(self) -> bool:
3771
+ """Check if monitor is active."""
3772
+ return self.status == DriftMonitorStatus.ACTIVE.value
3773
+
3774
+ @property
3775
+ def latest_run(self) -> DriftMonitorRun | None:
3776
+ """Get the most recent run."""
3777
+ return self.runs[0] if self.runs else None
3778
+
3779
+ def pause(self) -> None:
3780
+ """Pause this monitor."""
3781
+ self.status = DriftMonitorStatus.PAUSED.value
3782
+
3783
+ def resume(self) -> None:
3784
+ """Resume this monitor."""
3785
+ self.status = DriftMonitorStatus.ACTIVE.value
3786
+
3787
+ def mark_run(self, next_run: datetime | None = None) -> None:
3788
+ """Mark as run and update next run time."""
3789
+ self.last_run_at = datetime.utcnow()
3790
+ self.next_run_at = next_run
3791
+
3792
+
3793
+ class DriftMonitorRun(Base, UUIDMixin):
3794
+ """Drift monitor execution record.
3795
+
3796
+ Stores results from a drift monitor run.
3797
+
3798
+ Attributes:
3799
+ id: Unique identifier (UUID).
3800
+ monitor_id: Reference to DriftMonitor.
3801
+ status: Run status.
3802
+ has_drift: Whether drift was detected.
3803
+ max_drift_score: Maximum drift score across columns.
3804
+ total_columns: Total columns compared.
3805
+ drifted_columns: Number of columns with drift.
3806
+ column_results: JSON of per-column results.
3807
+ root_cause_analysis: JSON of root cause analysis.
3808
+ duration_ms: Run duration in milliseconds.
3809
+ error_message: Error message if failed.
3810
+ created_at: When run started.
3811
+ completed_at: When run completed.
3812
+ """
3813
+
3814
+ __tablename__ = "drift_monitor_runs"
3815
+
3816
+ __table_args__ = (
3817
+ Index("idx_drift_monitor_runs_monitor", "monitor_id", "created_at"),
3818
+ Index("idx_drift_monitor_runs_status", "status"),
3819
+ )
3820
+
3821
+ monitor_id: Mapped[str] = mapped_column(
3822
+ String(36),
3823
+ ForeignKey("drift_monitors.id", ondelete="CASCADE"),
3824
+ nullable=False,
3825
+ index=True,
3826
+ )
3827
+ status: Mapped[str] = mapped_column(
3828
+ String(20),
3829
+ nullable=False,
3830
+ default="pending",
3831
+ index=True,
3832
+ )
3833
+ has_drift: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
3834
+ max_drift_score: Mapped[float | None] = mapped_column(Float, nullable=True)
3835
+ total_columns: Mapped[int | None] = mapped_column(Integer, nullable=True)
3836
+ drifted_columns: Mapped[int | None] = mapped_column(Integer, nullable=True)
3837
+ column_results: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
3838
+ root_cause_analysis: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
3839
+ duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
3840
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
3841
+ created_at: Mapped[datetime] = mapped_column(
3842
+ DateTime, default=datetime.utcnow, nullable=False
3843
+ )
3844
+ started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3845
+ completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3846
+
3847
+ # Relationships
3848
+ monitor: Mapped[DriftMonitor] = relationship(
3849
+ "DriftMonitor",
3850
+ back_populates="runs",
3851
+ )
3852
+
3853
+ @property
3854
+ def is_complete(self) -> bool:
3855
+ """Check if run has completed."""
3856
+ return self.status in ("success", "failed", "error")
3857
+
3858
+ @property
3859
+ def drift_percentage(self) -> float:
3860
+ """Calculate percentage of columns with drift."""
3861
+ if self.total_columns and self.total_columns > 0:
3862
+ return (self.drifted_columns or 0) / self.total_columns * 100
3863
+ return 0.0
3864
+
3865
+ def mark_started(self) -> None:
3866
+ """Mark run as started."""
3867
+ self.status = "running"
3868
+ self.started_at = datetime.utcnow()
3869
+
3870
+ def mark_completed(
3871
+ self,
3872
+ has_drift: bool,
3873
+ max_drift_score: float,
3874
+ total_columns: int,
3875
+ drifted_columns: int,
3876
+ column_results: dict[str, Any],
3877
+ root_cause_analysis: dict[str, Any] | None = None,
3878
+ ) -> None:
3879
+ """Mark run as completed with results."""
3880
+ self.status = "success"
3881
+ self.has_drift = has_drift
3882
+ self.max_drift_score = max_drift_score
3883
+ self.total_columns = total_columns
3884
+ self.drifted_columns = drifted_columns
3885
+ self.column_results = column_results
3886
+ self.root_cause_analysis = root_cause_analysis
3887
+ self.completed_at = datetime.utcnow()
3888
+
3889
+ if self.started_at:
3890
+ delta = self.completed_at - self.started_at
3891
+ self.duration_ms = int(delta.total_seconds() * 1000)
3892
+
3893
+ def mark_error(self, message: str) -> None:
3894
+ """Mark run as errored."""
3895
+ self.status = "error"
3896
+ self.error_message = message
3897
+ self.completed_at = datetime.utcnow()
3898
+
3899
+ if self.started_at:
3900
+ delta = self.completed_at - self.started_at
3901
+ self.duration_ms = int(delta.total_seconds() * 1000)
3902
+
3903
+
3904
+ class DriftAlert(Base, UUIDMixin):
3905
+ """Drift alert model.
3906
+
3907
+ Represents an alert triggered by drift detection.
3908
+
3909
+ Attributes:
3910
+ id: Unique identifier (UUID).
3911
+ monitor_id: Reference to DriftMonitor.
3912
+ run_id: Reference to DriftMonitorRun.
3913
+ severity: Alert severity (critical, high, medium, low).
3914
+ status: Alert status.
3915
+ drift_score: Drift score that triggered alert.
3916
+ affected_columns: List of affected columns.
3917
+ message: Alert message.
3918
+ acknowledged_by: Who acknowledged.
3919
+ acknowledged_at: When acknowledged.
3920
+ resolved_at: When resolved.
3921
+ created_at: When alert was created.
3922
+ """
3923
+
3924
+ __tablename__ = "drift_alerts"
3925
+
3926
+ __table_args__ = (
3927
+ Index("idx_drift_alerts_monitor", "monitor_id", "created_at"),
3928
+ Index("idx_drift_alerts_status", "status"),
3929
+ )
3930
+
3931
+ monitor_id: Mapped[str] = mapped_column(
3932
+ String(36),
3933
+ ForeignKey("drift_monitors.id", ondelete="CASCADE"),
3934
+ nullable=False,
3935
+ index=True,
3936
+ )
3937
+ run_id: Mapped[str | None] = mapped_column(
3938
+ String(36),
3939
+ ForeignKey("drift_monitor_runs.id", ondelete="SET NULL"),
3940
+ nullable=True,
3941
+ )
3942
+ severity: Mapped[str] = mapped_column(String(20), nullable=False, default="high")
3943
+ status: Mapped[str] = mapped_column(
3944
+ String(20),
3945
+ nullable=False,
3946
+ default=DriftAlertStatus.ACTIVE.value,
3947
+ index=True,
3948
+ )
3949
+ drift_score: Mapped[float] = mapped_column(Float, nullable=False)
3950
+ affected_columns: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
3951
+ message: Mapped[str] = mapped_column(Text, nullable=False)
3952
+ acknowledged_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
3953
+ acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3954
+ resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
3955
+ created_at: Mapped[datetime] = mapped_column(
3956
+ DateTime, default=datetime.utcnow, nullable=False
3957
+ )
3958
+
3959
+ # Relationships
3960
+ monitor: Mapped[DriftMonitor] = relationship(
3961
+ "DriftMonitor",
3962
+ back_populates="alerts",
3963
+ )
3964
+ run: Mapped[DriftMonitorRun | None] = relationship(
3965
+ "DriftMonitorRun",
3966
+ lazy="selectin",
3967
+ )
3968
+
3969
+ @property
3970
+ def is_active(self) -> bool:
3971
+ """Check if alert is active."""
3972
+ return self.status == DriftAlertStatus.ACTIVE.value
3973
+
3974
+ def acknowledge(self, by: str) -> None:
3975
+ """Acknowledge this alert."""
3976
+ self.status = DriftAlertStatus.ACKNOWLEDGED.value
3977
+ self.acknowledged_by = by
3978
+ self.acknowledged_at = datetime.utcnow()
3979
+
3980
+ def resolve(self) -> None:
3981
+ """Resolve this alert."""
3982
+ self.status = DriftAlertStatus.RESOLVED.value
3983
+ self.resolved_at = datetime.utcnow()
3984
+
3985
+
3986
+ # =============================================================================
3987
+ # Phase 9: Plugin System Models
3988
+ # =============================================================================
3989
+
3990
+
3991
+ class PluginType(str, Enum):
3992
+ """Type of plugin."""
3993
+
3994
+ VALIDATOR = "validator"
3995
+ REPORTER = "reporter"
3996
+ CONNECTOR = "connector"
3997
+ TRANSFORMER = "transformer"
3998
+
3999
+
4000
+ class PluginStatus(str, Enum):
4001
+ """Installation status of a plugin."""
4002
+
4003
+ AVAILABLE = "available"
4004
+ INSTALLED = "installed"
4005
+ ENABLED = "enabled"
4006
+ DISABLED = "disabled"
4007
+ UPDATE_AVAILABLE = "update_available"
4008
+ ERROR = "error"
4009
+
4010
+
4011
+ class PluginSource(str, Enum):
4012
+ """Source of the plugin."""
4013
+
4014
+ OFFICIAL = "official"
4015
+ COMMUNITY = "community"
4016
+ LOCAL = "local"
4017
+ PRIVATE = "private"
4018
+
4019
+
4020
+ class SecurityLevel(str, Enum):
4021
+ """Security level of the plugin."""
4022
+
4023
+ TRUSTED = "trusted"
4024
+ VERIFIED = "verified"
4025
+ UNVERIFIED = "unverified"
4026
+ SANDBOXED = "sandboxed"
4027
+
4028
+
4029
+ class Plugin(Base, UUIDMixin, TimestampMixin):
4030
+ """Plugin model.
4031
+
4032
+ Represents an installable plugin in the plugin marketplace.
4033
+
4034
+ Attributes:
4035
+ id: Unique identifier (UUID).
4036
+ name: Plugin name (unique identifier).
4037
+ display_name: Human-readable display name.
4038
+ description: Plugin description.
4039
+ version: Current/installed version.
4040
+ latest_version: Latest available version.
4041
+ type: Plugin type (validator, reporter, connector, transformer).
4042
+ source: Plugin source (official, community, local, private).
4043
+ status: Installation status.
4044
+ security_level: Security verification level.
4045
+ author: Author information (JSON).
4046
+ license: License identifier.
4047
+ homepage: Plugin homepage URL.
4048
+ repository: Repository URL.
4049
+ keywords: Search keywords (JSON array).
4050
+ categories: Plugin categories (JSON array).
4051
+ dependencies: Plugin dependencies (JSON array).
4052
+ permissions: Required permissions (JSON array).
4053
+ python_version: Required Python version.
4054
+ dashboard_version: Required dashboard version.
4055
+ icon_url: Plugin icon URL.
4056
+ banner_url: Plugin banner URL.
4057
+ documentation_url: Documentation URL.
4058
+ changelog: Changelog markdown.
4059
+ readme: README markdown.
4060
+ package_url: URL to download plugin package.
4061
+ signature: Plugin signature info (JSON).
4062
+ sandbox_config: Sandbox configuration (JSON).
4063
+ is_enabled: Whether plugin is enabled.
4064
+ install_count: Total installation count.
4065
+ rating: Average rating (1-5).
4066
+ rating_count: Number of ratings.
4067
+ validators_count: Number of validators provided.
4068
+ reporters_count: Number of reporters provided.
4069
+ installed_at: When plugin was installed.
4070
+ last_updated: When plugin was last updated.
4071
+ """
4072
+
4073
+ __tablename__ = "plugins"
4074
+
4075
+ __table_args__ = (
4076
+ Index("idx_plugins_name", "name", unique=True),
4077
+ Index("idx_plugins_type", "type"),
4078
+ Index("idx_plugins_source", "source"),
4079
+ Index("idx_plugins_status", "status"),
4080
+ )
4081
+
4082
+ name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
4083
+ display_name: Mapped[str] = mapped_column(String(200), nullable=False)
4084
+ description: Mapped[str] = mapped_column(Text, nullable=False)
4085
+ version: Mapped[str] = mapped_column(String(50), nullable=False)
4086
+ latest_version: Mapped[str | None] = mapped_column(String(50), nullable=True)
4087
+ type: Mapped[str] = mapped_column(
4088
+ SQLEnum(PluginType, native_enum=False, length=20),
4089
+ nullable=False,
4090
+ index=True,
4091
+ )
4092
+ source: Mapped[str] = mapped_column(
4093
+ SQLEnum(PluginSource, native_enum=False, length=20),
4094
+ nullable=False,
4095
+ default=PluginSource.COMMUNITY.value,
4096
+ )
4097
+ status: Mapped[str] = mapped_column(
4098
+ SQLEnum(PluginStatus, native_enum=False, length=20),
4099
+ nullable=False,
4100
+ default=PluginStatus.AVAILABLE.value,
4101
+ index=True,
4102
+ )
4103
+ security_level: Mapped[str] = mapped_column(
4104
+ SQLEnum(SecurityLevel, native_enum=False, length=20),
4105
+ nullable=False,
4106
+ default=SecurityLevel.UNVERIFIED.value,
4107
+ )
4108
+ author: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
4109
+ license: Mapped[str | None] = mapped_column(String(50), nullable=True)
4110
+ homepage: Mapped[str | None] = mapped_column(String(500), nullable=True)
4111
+ repository: Mapped[str | None] = mapped_column(String(500), nullable=True)
4112
+ keywords: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
4113
+ categories: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
4114
+ dependencies: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
4115
+ permissions: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
4116
+ python_version: Mapped[str | None] = mapped_column(String(20), nullable=True)
4117
+ dashboard_version: Mapped[str | None] = mapped_column(String(20), nullable=True)
4118
+ icon_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
4119
+ banner_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
4120
+ documentation_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
4121
+ changelog: Mapped[str | None] = mapped_column(Text, nullable=True)
4122
+ readme: Mapped[str | None] = mapped_column(Text, nullable=True)
4123
+ package_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
4124
+ signature: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
4125
+ sandbox_config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
4126
+ is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4127
+ install_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4128
+ rating: Mapped[float | None] = mapped_column(Float, nullable=True)
4129
+ rating_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4130
+ validators_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4131
+ reporters_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4132
+ installed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4133
+ last_updated: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4134
+
4135
+ # Relationships
4136
+ custom_validators: Mapped[list["CustomValidator"]] = relationship(
4137
+ "CustomValidator",
4138
+ back_populates="plugin",
4139
+ cascade="all, delete-orphan",
4140
+ lazy="selectin",
4141
+ )
4142
+ custom_reporters: Mapped[list["CustomReporter"]] = relationship(
4143
+ "CustomReporter",
4144
+ back_populates="plugin",
4145
+ cascade="all, delete-orphan",
4146
+ lazy="selectin",
4147
+ )
4148
+ ratings: Mapped[list["PluginRating"]] = relationship(
4149
+ "PluginRating",
4150
+ back_populates="plugin",
4151
+ cascade="all, delete-orphan",
4152
+ lazy="selectin",
4153
+ )
4154
+ signatures: Mapped[list["PluginSignature"]] = relationship(
4155
+ "PluginSignature",
4156
+ back_populates="plugin",
4157
+ cascade="all, delete-orphan",
4158
+ lazy="selectin",
4159
+ )
4160
+ hot_reload_config: Mapped["HotReloadConfig | None"] = relationship(
4161
+ "HotReloadConfig",
4162
+ back_populates="plugin",
4163
+ cascade="all, delete-orphan",
4164
+ uselist=False,
4165
+ lazy="selectin",
4166
+ )
4167
+ hooks: Mapped[list["PluginHook"]] = relationship(
4168
+ "PluginHook",
4169
+ back_populates="plugin",
4170
+ cascade="all, delete-orphan",
4171
+ lazy="selectin",
4172
+ )
4173
+
4174
+ def install(self) -> None:
4175
+ """Mark plugin as installed."""
4176
+ self.status = PluginStatus.INSTALLED.value
4177
+ self.installed_at = datetime.utcnow()
4178
+ self.install_count += 1
4179
+
4180
+ def enable(self) -> None:
4181
+ """Enable the plugin."""
4182
+ self.status = PluginStatus.ENABLED.value
4183
+ self.is_enabled = True
4184
+
4185
+ def disable(self) -> None:
4186
+ """Disable the plugin."""
4187
+ self.status = PluginStatus.DISABLED.value
4188
+ self.is_enabled = False
4189
+
4190
+ def uninstall(self) -> None:
4191
+ """Mark plugin as available (uninstalled)."""
4192
+ self.status = PluginStatus.AVAILABLE.value
4193
+ self.is_enabled = False
4194
+ self.installed_at = None
4195
+
4196
+ def update_rating(self, new_rating: float) -> None:
4197
+ """Update average rating with a new rating."""
4198
+ if self.rating is None:
4199
+ self.rating = new_rating
4200
+ else:
4201
+ total = self.rating * self.rating_count + new_rating
4202
+ self.rating = total / (self.rating_count + 1)
4203
+ self.rating_count += 1
4204
+
4205
+
4206
+ class CustomValidator(Base, UUIDMixin, TimestampMixin):
4207
+ """Custom validator model.
4208
+
4209
+ Represents a user-defined validator that can be used for data validation.
4210
+
4211
+ Attributes:
4212
+ id: Unique identifier (UUID).
4213
+ plugin_id: Optional reference to parent plugin.
4214
+ name: Validator name (unique).
4215
+ display_name: Human-readable display name.
4216
+ description: Validator description.
4217
+ category: Validator category.
4218
+ severity: Default severity (error, warning, info).
4219
+ tags: Search/filter tags (JSON array).
4220
+ parameters: Parameter definitions (JSON array).
4221
+ code: Python code implementing the validator.
4222
+ test_cases: Test cases for validation (JSON array).
4223
+ is_enabled: Whether validator is enabled.
4224
+ is_verified: Whether validator is security-verified.
4225
+ usage_count: Number of times validator has been used.
4226
+ last_used_at: When validator was last used.
4227
+ """
4228
+
4229
+ __tablename__ = "custom_validators"
4230
+
4231
+ __table_args__ = (
4232
+ Index("idx_custom_validators_name", "name", unique=True),
4233
+ Index("idx_custom_validators_plugin", "plugin_id"),
4234
+ Index("idx_custom_validators_category", "category"),
4235
+ )
4236
+
4237
+ plugin_id: Mapped[str | None] = mapped_column(
4238
+ String(36),
4239
+ ForeignKey("plugins.id", ondelete="CASCADE"),
4240
+ nullable=True,
4241
+ index=True,
4242
+ )
4243
+ name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
4244
+ display_name: Mapped[str] = mapped_column(String(200), nullable=False)
4245
+ description: Mapped[str] = mapped_column(Text, nullable=False)
4246
+ category: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
4247
+ severity: Mapped[str] = mapped_column(String(20), nullable=False, default="error")
4248
+ tags: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
4249
+ parameters: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
4250
+ code: Mapped[str] = mapped_column(Text, nullable=False)
4251
+ test_cases: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
4252
+ is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
4253
+ is_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4254
+ usage_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4255
+ last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4256
+
4257
+ # Relationships
4258
+ plugin: Mapped[Plugin | None] = relationship(
4259
+ "Plugin",
4260
+ back_populates="custom_validators",
4261
+ )
4262
+
4263
+ def increment_usage(self) -> None:
4264
+ """Increment usage count and update last used timestamp."""
4265
+ self.usage_count += 1
4266
+ self.last_used_at = datetime.utcnow()
4267
+
4268
+
4269
+ class CustomReporter(Base, UUIDMixin, TimestampMixin):
4270
+ """Custom reporter model.
4271
+
4272
+ Represents a user-defined reporter for generating custom reports.
4273
+
4274
+ Attributes:
4275
+ id: Unique identifier (UUID).
4276
+ plugin_id: Optional reference to parent plugin.
4277
+ name: Reporter name (unique).
4278
+ display_name: Human-readable display name.
4279
+ description: Reporter description.
4280
+ output_formats: Supported output formats (JSON array).
4281
+ config_fields: Configuration field definitions (JSON array).
4282
+ template: Jinja2 template for HTML/text reports.
4283
+ code: Python code for custom report generation.
4284
+ preview_image_url: Preview image URL.
4285
+ is_enabled: Whether reporter is enabled.
4286
+ is_verified: Whether reporter is security-verified.
4287
+ usage_count: Number of times reporter has been used.
4288
+ """
4289
+
4290
+ __tablename__ = "custom_reporters"
4291
+
4292
+ __table_args__ = (
4293
+ Index("idx_custom_reporters_name", "name", unique=True),
4294
+ Index("idx_custom_reporters_plugin", "plugin_id"),
4295
+ )
4296
+
4297
+ plugin_id: Mapped[str | None] = mapped_column(
4298
+ String(36),
4299
+ ForeignKey("plugins.id", ondelete="CASCADE"),
4300
+ nullable=True,
4301
+ index=True,
4302
+ )
4303
+ name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
4304
+ display_name: Mapped[str] = mapped_column(String(200), nullable=False)
4305
+ description: Mapped[str] = mapped_column(Text, nullable=False)
4306
+ output_formats: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
4307
+ config_fields: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON, nullable=True)
4308
+ template: Mapped[str | None] = mapped_column(Text, nullable=True)
4309
+ code: Mapped[str | None] = mapped_column(Text, nullable=True)
4310
+ preview_image_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
4311
+ is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
4312
+ is_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4313
+ usage_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4314
+
4315
+ # Relationships
4316
+ plugin: Mapped[Plugin | None] = relationship(
4317
+ "Plugin",
4318
+ back_populates="custom_reporters",
4319
+ )
4320
+
4321
+ def increment_usage(self) -> None:
4322
+ """Increment usage count."""
4323
+ self.usage_count += 1
4324
+
4325
+
4326
+ class ReportFormatType(str, Enum):
4327
+ """Report output format types."""
4328
+
4329
+ HTML = "html"
4330
+ PDF = "pdf"
4331
+ CSV = "csv"
4332
+ JSON = "json"
4333
+ MARKDOWN = "markdown"
4334
+ JUNIT = "junit"
4335
+ EXCEL = "excel"
4336
+ CUSTOM = "custom"
4337
+
4338
+
4339
+ class ReportStatus(str, Enum):
4340
+ """Status of report generation."""
4341
+
4342
+ PENDING = "pending"
4343
+ GENERATING = "generating"
4344
+ COMPLETED = "completed"
4345
+ FAILED = "failed"
4346
+ EXPIRED = "expired"
4347
+
4348
+
4349
+ class GeneratedReport(Base, UUIDMixin, TimestampMixin):
4350
+ """Generated report history model.
4351
+
4352
+ Stores metadata and content of generated reports for history tracking,
4353
+ caching, and audit purposes.
4354
+
4355
+ Attributes:
4356
+ id: Unique identifier (UUID).
4357
+ validation_id: Reference to the validation run.
4358
+ source_id: Reference to the data source.
4359
+ reporter_id: Optional reference to custom reporter used.
4360
+ name: Human-readable report name.
4361
+ description: Optional description.
4362
+ format: Report output format (html, pdf, csv, etc.).
4363
+ theme: Theme used for HTML/PDF reports.
4364
+ locale: Language locale used.
4365
+ status: Generation status.
4366
+ file_path: Path to stored report file (if persisted).
4367
+ file_size: Size of the report file in bytes.
4368
+ content_hash: Hash of report content for deduplication.
4369
+ config: Configuration used for generation (JSON).
4370
+ metadata: Additional metadata (JSON).
4371
+ error_message: Error message if generation failed.
4372
+ generation_time_ms: Time taken to generate in milliseconds.
4373
+ expires_at: When the report expires and can be cleaned up.
4374
+ downloaded_count: Number of times the report was downloaded.
4375
+ last_downloaded_at: Last download timestamp.
4376
+ """
4377
+
4378
+ __tablename__ = "generated_reports"
4379
+
4380
+ __table_args__ = (
4381
+ Index("idx_generated_reports_validation", "validation_id"),
4382
+ Index("idx_generated_reports_source", "source_id"),
4383
+ Index("idx_generated_reports_reporter", "reporter_id"),
4384
+ Index("idx_generated_reports_status", "status"),
4385
+ Index("idx_generated_reports_format", "format"),
4386
+ Index("idx_generated_reports_created", "created_at"),
4387
+ Index("idx_generated_reports_expires", "expires_at"),
4388
+ )
4389
+
4390
+ validation_id: Mapped[str | None] = mapped_column(
4391
+ String(36),
4392
+ ForeignKey("validations.id", ondelete="SET NULL"),
4393
+ nullable=True,
4394
+ index=True,
4395
+ )
4396
+ source_id: Mapped[str | None] = mapped_column(
4397
+ String(36),
4398
+ ForeignKey("sources.id", ondelete="SET NULL"),
4399
+ nullable=True,
4400
+ index=True,
4401
+ )
4402
+ reporter_id: Mapped[str | None] = mapped_column(
4403
+ String(36),
4404
+ ForeignKey("custom_reporters.id", ondelete="SET NULL"),
4405
+ nullable=True,
4406
+ index=True,
4407
+ )
4408
+ name: Mapped[str] = mapped_column(String(255), nullable=False)
4409
+ description: Mapped[str | None] = mapped_column(Text, nullable=True)
4410
+ format: Mapped[str] = mapped_column(
4411
+ SQLEnum(ReportFormatType), nullable=False, default=ReportFormatType.HTML
4412
+ )
4413
+ theme: Mapped[str | None] = mapped_column(String(50), nullable=True)
4414
+ locale: Mapped[str] = mapped_column(String(10), nullable=False, default="en")
4415
+ status: Mapped[str] = mapped_column(
4416
+ SQLEnum(ReportStatus), nullable=False, default=ReportStatus.PENDING
4417
+ )
4418
+ file_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
4419
+ file_size: Mapped[int | None] = mapped_column(Integer, nullable=True)
4420
+ content_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
4421
+ config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
4422
+ report_metadata: Mapped[dict[str, Any] | None] = mapped_column(
4423
+ "metadata", JSON, nullable=True
4424
+ )
4425
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
4426
+ generation_time_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
4427
+ expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4428
+ downloaded_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4429
+ last_downloaded_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4430
+
4431
+ # Relationships
4432
+ validation: Mapped["Validation | None"] = relationship(
4433
+ "Validation",
4434
+ back_populates="generated_reports",
4435
+ )
4436
+ source: Mapped["Source | None"] = relationship(
4437
+ "Source",
4438
+ back_populates="generated_reports",
4439
+ )
4440
+ reporter: Mapped[CustomReporter | None] = relationship(
4441
+ "CustomReporter",
4442
+ backref="generated_reports",
4443
+ )
4444
+
4445
+ def increment_download(self) -> None:
4446
+ """Increment download count and update last download timestamp."""
4447
+ self.downloaded_count += 1
4448
+ self.last_downloaded_at = datetime.utcnow()
4449
+
4450
+ def mark_completed(self, file_path: str, file_size: int, generation_time_ms: float) -> None:
4451
+ """Mark report as completed."""
4452
+ self.status = ReportStatus.COMPLETED
4453
+ self.file_path = file_path
4454
+ self.file_size = file_size
4455
+ self.generation_time_ms = generation_time_ms
4456
+
4457
+ def mark_failed(self, error_message: str) -> None:
4458
+ """Mark report as failed."""
4459
+ self.status = ReportStatus.FAILED
4460
+ self.error_message = error_message
4461
+
4462
+ def is_expired(self) -> bool:
4463
+ """Check if report has expired."""
4464
+ if self.expires_at is None:
4465
+ return False
4466
+ return datetime.utcnow() > self.expires_at
4467
+
4468
+
4469
+ class PluginRating(Base, UUIDMixin, TimestampMixin):
4470
+ """Plugin rating model.
4471
+
4472
+ Stores user ratings and reviews for plugins.
4473
+
4474
+ Attributes:
4475
+ id: Unique identifier (UUID).
4476
+ plugin_id: Reference to Plugin.
4477
+ user_id: User identifier (could be from auth system).
4478
+ rating: Rating value (1-5).
4479
+ review: Optional review text.
4480
+ """
4481
+
4482
+ __tablename__ = "plugin_ratings"
4483
+
4484
+ __table_args__ = (
4485
+ Index("idx_plugin_ratings_plugin", "plugin_id"),
4486
+ Index("idx_plugin_ratings_user", "user_id"),
4487
+ )
4488
+
4489
+ plugin_id: Mapped[str] = mapped_column(
4490
+ String(36),
4491
+ ForeignKey("plugins.id", ondelete="CASCADE"),
4492
+ nullable=False,
4493
+ index=True,
4494
+ )
4495
+ user_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
4496
+ rating: Mapped[int] = mapped_column(Integer, nullable=False)
4497
+ review: Mapped[str | None] = mapped_column(Text, nullable=True)
4498
+
4499
+ # Relationships
4500
+ plugin: Mapped[Plugin] = relationship(
4501
+ "Plugin",
4502
+ back_populates="ratings",
4503
+ )
4504
+
4505
+
4506
+ class PluginExecutionLog(Base, UUIDMixin):
4507
+ """Plugin execution log model.
4508
+
4509
+ Tracks plugin executions for monitoring and debugging.
4510
+
4511
+ Attributes:
4512
+ id: Unique identifier (UUID).
4513
+ plugin_id: Reference to Plugin.
4514
+ validator_id: Optional reference to CustomValidator.
4515
+ reporter_id: Optional reference to CustomReporter.
4516
+ execution_id: Unique execution identifier.
4517
+ source_id: Optional reference to data source.
4518
+ status: Execution status (pending, running, success, failed, error).
4519
+ execution_time_ms: Execution duration in milliseconds.
4520
+ memory_used_mb: Memory used in MB.
4521
+ result: Execution result (JSON).
4522
+ error_message: Error message if failed.
4523
+ logs: Execution logs (JSON array).
4524
+ started_at: When execution started.
4525
+ completed_at: When execution completed.
4526
+ """
4527
+
4528
+ __tablename__ = "plugin_execution_logs"
4529
+
4530
+ __table_args__ = (
4531
+ Index("idx_plugin_exec_logs_plugin", "plugin_id", "started_at"),
4532
+ Index("idx_plugin_exec_logs_status", "status"),
4533
+ )
4534
+
4535
+ plugin_id: Mapped[str] = mapped_column(
4536
+ String(36),
4537
+ ForeignKey("plugins.id", ondelete="CASCADE"),
4538
+ nullable=False,
4539
+ index=True,
4540
+ )
4541
+ validator_id: Mapped[str | None] = mapped_column(
4542
+ String(36),
4543
+ ForeignKey("custom_validators.id", ondelete="SET NULL"),
4544
+ nullable=True,
4545
+ )
4546
+ reporter_id: Mapped[str | None] = mapped_column(
4547
+ String(36),
4548
+ ForeignKey("custom_reporters.id", ondelete="SET NULL"),
4549
+ nullable=True,
4550
+ )
4551
+ execution_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
4552
+ source_id: Mapped[str | None] = mapped_column(
4553
+ String(36),
4554
+ ForeignKey("sources.id", ondelete="SET NULL"),
4555
+ nullable=True,
4556
+ )
4557
+ status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")
4558
+ execution_time_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
4559
+ memory_used_mb: Mapped[float | None] = mapped_column(Float, nullable=True)
4560
+ result: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
4561
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
4562
+ logs: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
4563
+ started_at: Mapped[datetime] = mapped_column(
4564
+ DateTime, default=datetime.utcnow, nullable=False
4565
+ )
4566
+ completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4567
+
4568
+ def mark_started(self) -> None:
4569
+ """Mark execution as started."""
4570
+ self.status = "running"
4571
+ self.started_at = datetime.utcnow()
4572
+
4573
+ def mark_completed(
4574
+ self,
4575
+ result: dict[str, Any] | None = None,
4576
+ memory_used_mb: float | None = None,
4577
+ ) -> None:
4578
+ """Mark execution as completed."""
4579
+ self.status = "success"
4580
+ self.result = result
4581
+ self.memory_used_mb = memory_used_mb
4582
+ self.completed_at = datetime.utcnow()
4583
+ if self.started_at:
4584
+ delta = self.completed_at - self.started_at
4585
+ self.execution_time_ms = int(delta.total_seconds() * 1000)
4586
+
4587
+ def mark_failed(self, error_message: str) -> None:
4588
+ """Mark execution as failed."""
4589
+ self.status = "failed"
4590
+ self.error_message = error_message
4591
+ self.completed_at = datetime.utcnow()
4592
+ if self.started_at:
4593
+ delta = self.completed_at - self.started_at
4594
+ self.execution_time_ms = int(delta.total_seconds() * 1000)
4595
+
4596
+
4597
+ # =============================================================================
4598
+ # Phase 9: Trust Store and Security Models
4599
+ # =============================================================================
4600
+
4601
+
4602
+ class SignatureAlgorithmType(str, Enum):
4603
+ """Supported signature algorithms."""
4604
+
4605
+ HMAC_SHA256 = "hmac_sha256"
4606
+ HMAC_SHA512 = "hmac_sha512"
4607
+ RSA_SHA256 = "rsa_sha256"
4608
+ ED25519 = "ed25519"
4609
+
4610
+
4611
+ class TrustLevelType(str, Enum):
4612
+ """Trust levels for signers."""
4613
+
4614
+ TRUSTED = "trusted"
4615
+ VERIFIED = "verified"
4616
+ UNVERIFIED = "unverified"
4617
+ REVOKED = "revoked"
4618
+
4619
+
4620
+ class IsolationLevelType(str, Enum):
4621
+ """Sandbox isolation levels."""
4622
+
4623
+ NONE = "none"
4624
+ PROCESS = "process"
4625
+ CONTAINER = "container"
4626
+
4627
+
4628
+ class TrustedSigner(Base, UUIDMixin, TimestampMixin):
4629
+ """Trusted signer model for plugin signature verification.
4630
+
4631
+ Represents a trusted entity that can sign plugins.
4632
+
4633
+ Attributes:
4634
+ id: Unique identifier (UUID).
4635
+ signer_id: Unique signer identifier (e.g., email, domain).
4636
+ name: Display name of the signer.
4637
+ organization: Organization name.
4638
+ email: Contact email.
4639
+ public_key: PEM-encoded public key.
4640
+ algorithm: Signature algorithm used.
4641
+ fingerprint: Key fingerprint for quick identification.
4642
+ trust_level: Current trust level.
4643
+ plugins_signed: Count of plugins signed.
4644
+ expires_at: When the trust expires.
4645
+ revoked_at: When the signer was revoked.
4646
+ revocation_reason: Reason for revocation.
4647
+ metadata: Additional signer metadata (JSON).
4648
+ """
4649
+
4650
+ __tablename__ = "trusted_signers"
4651
+
4652
+ __table_args__ = (
4653
+ Index("idx_trusted_signers_signer_id", "signer_id", unique=True),
4654
+ Index("idx_trusted_signers_fingerprint", "fingerprint"),
4655
+ Index("idx_trusted_signers_trust_level", "trust_level"),
4656
+ )
4657
+
4658
+ signer_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
4659
+ name: Mapped[str] = mapped_column(String(255), nullable=False)
4660
+ organization: Mapped[str | None] = mapped_column(String(255), nullable=True)
4661
+ email: Mapped[str | None] = mapped_column(String(255), nullable=True)
4662
+ public_key: Mapped[str] = mapped_column(Text, nullable=False)
4663
+ algorithm: Mapped[str] = mapped_column(
4664
+ String(20),
4665
+ nullable=False,
4666
+ default=SignatureAlgorithmType.ED25519.value,
4667
+ )
4668
+ fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
4669
+ trust_level: Mapped[str] = mapped_column(
4670
+ String(20),
4671
+ nullable=False,
4672
+ default=TrustLevelType.VERIFIED.value,
4673
+ )
4674
+ plugins_signed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4675
+ expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4676
+ revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4677
+ revocation_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
4678
+ signer_metadata: Mapped[dict[str, Any]] = mapped_column(
4679
+ "metadata", JSON, nullable=False, default=dict
4680
+ )
4681
+
4682
+ def is_valid(self) -> bool:
4683
+ """Check if the signer is currently valid."""
4684
+ if self.trust_level == TrustLevelType.REVOKED.value:
4685
+ return False
4686
+ if self.revoked_at is not None:
4687
+ return False
4688
+ if self.expires_at and self.expires_at < datetime.utcnow():
4689
+ return False
4690
+ return True
4691
+
4692
+ def revoke(self, reason: str) -> None:
4693
+ """Revoke this signer."""
4694
+ self.trust_level = TrustLevelType.REVOKED.value
4695
+ self.revoked_at = datetime.utcnow()
4696
+ self.revocation_reason = reason
4697
+
4698
+ def increment_signed_count(self) -> None:
4699
+ """Increment the plugins signed count."""
4700
+ self.plugins_signed += 1
4701
+
4702
+
4703
+ class SecurityPolicy(Base, UUIDMixin, TimestampMixin):
4704
+ """Security policy model for plugin execution.
4705
+
4706
+ Defines security policies for plugin sandbox execution.
4707
+
4708
+ Attributes:
4709
+ id: Unique identifier (UUID).
4710
+ name: Policy name.
4711
+ description: Policy description.
4712
+ is_default: Whether this is the default policy.
4713
+ is_active: Whether the policy is active.
4714
+ isolation_level: Sandbox isolation level.
4715
+ memory_limit_mb: Maximum memory in MB.
4716
+ cpu_time_limit_sec: Maximum CPU time in seconds.
4717
+ wall_time_limit_sec: Maximum wall clock time in seconds.
4718
+ network_enabled: Whether network access is allowed.
4719
+ file_read_enabled: Whether file read is allowed.
4720
+ file_write_enabled: Whether file write is allowed.
4721
+ allowed_modules: List of allowed Python modules.
4722
+ blocked_modules: List of blocked Python modules.
4723
+ allowed_builtins: List of allowed Python builtins.
4724
+ require_signature: Whether plugin signature is required.
4725
+ min_trust_level: Minimum trust level required.
4726
+ max_processes: Maximum number of processes.
4727
+ container_image: Docker image for container isolation.
4728
+ extra_options: Additional sandbox options (JSON).
4729
+ """
4730
+
4731
+ __tablename__ = "security_policies"
4732
+
4733
+ __table_args__ = (
4734
+ Index("idx_security_policies_name", "name", unique=True),
4735
+ Index("idx_security_policies_default", "is_default"),
4736
+ )
4737
+
4738
+ name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
4739
+ description: Mapped[str | None] = mapped_column(Text, nullable=True)
4740
+ is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4741
+ is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
4742
+ isolation_level: Mapped[str] = mapped_column(
4743
+ String(20),
4744
+ nullable=False,
4745
+ default=IsolationLevelType.PROCESS.value,
4746
+ )
4747
+ memory_limit_mb: Mapped[int] = mapped_column(Integer, nullable=False, default=256)
4748
+ cpu_time_limit_sec: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
4749
+ wall_time_limit_sec: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
4750
+ network_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4751
+ file_read_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
4752
+ file_write_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4753
+ allowed_modules: Mapped[list[str]] = mapped_column(
4754
+ JSON,
4755
+ nullable=False,
4756
+ default=lambda: [
4757
+ "math", "statistics", "decimal", "fractions",
4758
+ "random", "re", "json", "datetime",
4759
+ "collections", "itertools", "functools",
4760
+ "operator", "string", "typing", "dataclasses",
4761
+ ],
4762
+ )
4763
+ blocked_modules: Mapped[list[str]] = mapped_column(
4764
+ JSON,
4765
+ nullable=False,
4766
+ default=lambda: [
4767
+ "os", "sys", "subprocess", "socket", "shutil",
4768
+ "importlib", "ctypes", "multiprocessing",
4769
+ ],
4770
+ )
4771
+ allowed_builtins: Mapped[list[str]] = mapped_column(
4772
+ JSON,
4773
+ nullable=False,
4774
+ default=lambda: [
4775
+ "abs", "all", "any", "ascii", "bin", "bool", "chr",
4776
+ "dict", "divmod", "enumerate", "filter", "float",
4777
+ "format", "frozenset", "getattr", "hasattr", "hash",
4778
+ "hex", "int", "isinstance", "issubclass", "iter",
4779
+ "len", "list", "map", "max", "min", "next", "oct",
4780
+ "ord", "pow", "print", "range", "repr", "reversed",
4781
+ "round", "set", "slice", "sorted", "str", "sum",
4782
+ "tuple", "type", "zip",
4783
+ ],
4784
+ )
4785
+ require_signature: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4786
+ min_trust_level: Mapped[str] = mapped_column(
4787
+ String(20),
4788
+ nullable=False,
4789
+ default=TrustLevelType.UNVERIFIED.value,
4790
+ )
4791
+ max_processes: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
4792
+ container_image: Mapped[str] = mapped_column(
4793
+ String(255),
4794
+ nullable=False,
4795
+ default="python:3.11-slim",
4796
+ )
4797
+ extra_options: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
4798
+
4799
+ @classmethod
4800
+ def get_preset(cls, preset_name: str) -> "SecurityPolicy":
4801
+ """Create a security policy from a preset.
4802
+
4803
+ Args:
4804
+ preset_name: Name of the preset (development, testing, standard, enterprise, strict).
4805
+
4806
+ Returns:
4807
+ SecurityPolicy with preset configuration.
4808
+ """
4809
+ presets = {
4810
+ "development": {
4811
+ "isolation_level": IsolationLevelType.NONE.value,
4812
+ "network_enabled": True,
4813
+ "file_read_enabled": True,
4814
+ "file_write_enabled": True,
4815
+ "require_signature": False,
4816
+ "memory_limit_mb": 1024,
4817
+ "wall_time_limit_sec": 300,
4818
+ },
4819
+ "testing": {
4820
+ "isolation_level": IsolationLevelType.PROCESS.value,
4821
+ "network_enabled": True,
4822
+ "file_read_enabled": True,
4823
+ "file_write_enabled": False,
4824
+ "require_signature": False,
4825
+ "memory_limit_mb": 512,
4826
+ "wall_time_limit_sec": 120,
4827
+ },
4828
+ "standard": {
4829
+ "isolation_level": IsolationLevelType.PROCESS.value,
4830
+ "network_enabled": False,
4831
+ "file_read_enabled": True,
4832
+ "file_write_enabled": False,
4833
+ "require_signature": False,
4834
+ "min_trust_level": TrustLevelType.VERIFIED.value,
4835
+ "memory_limit_mb": 256,
4836
+ "wall_time_limit_sec": 60,
4837
+ },
4838
+ "enterprise": {
4839
+ "isolation_level": IsolationLevelType.PROCESS.value,
4840
+ "network_enabled": False,
4841
+ "file_read_enabled": True,
4842
+ "file_write_enabled": False,
4843
+ "require_signature": True,
4844
+ "min_trust_level": TrustLevelType.TRUSTED.value,
4845
+ "memory_limit_mb": 512,
4846
+ "wall_time_limit_sec": 120,
4847
+ },
4848
+ "strict": {
4849
+ "isolation_level": IsolationLevelType.CONTAINER.value,
4850
+ "network_enabled": False,
4851
+ "file_read_enabled": False,
4852
+ "file_write_enabled": False,
4853
+ "require_signature": True,
4854
+ "min_trust_level": TrustLevelType.TRUSTED.value,
4855
+ "memory_limit_mb": 128,
4856
+ "wall_time_limit_sec": 30,
4857
+ },
4858
+ }
4859
+
4860
+ config = presets.get(preset_name, presets["standard"])
4861
+ return cls(
4862
+ name=preset_name,
4863
+ description=f"{preset_name.capitalize()} security policy preset",
4864
+ **config,
4865
+ )
4866
+
4867
+
4868
+ class PluginSignature(Base, UUIDMixin, TimestampMixin):
4869
+ """Plugin signature model.
4870
+
4871
+ Stores signature information for verified plugins.
4872
+
4873
+ Attributes:
4874
+ id: Unique identifier (UUID).
4875
+ plugin_id: Reference to Plugin.
4876
+ signer_id: Reference to TrustedSigner.
4877
+ algorithm: Signature algorithm used.
4878
+ signature: Base64-encoded signature.
4879
+ signed_hash: Hash of the signed content.
4880
+ signed_at: When the signature was created.
4881
+ verified_at: When the signature was last verified.
4882
+ is_valid: Whether the signature is currently valid.
4883
+ metadata: Additional signature metadata.
4884
+ """
4885
+
4886
+ __tablename__ = "plugin_signatures"
4887
+
4888
+ __table_args__ = (
4889
+ Index("idx_plugin_signatures_plugin", "plugin_id"),
4890
+ Index("idx_plugin_signatures_signer", "signer_id"),
4891
+ )
4892
+
4893
+ plugin_id: Mapped[str] = mapped_column(
4894
+ String(36),
4895
+ ForeignKey("plugins.id", ondelete="CASCADE"),
4896
+ nullable=False,
4897
+ index=True,
4898
+ )
4899
+ signer_id: Mapped[str] = mapped_column(
4900
+ String(36),
4901
+ ForeignKey("trusted_signers.id", ondelete="SET NULL"),
4902
+ nullable=True,
4903
+ index=True,
4904
+ )
4905
+ algorithm: Mapped[str] = mapped_column(
4906
+ String(20),
4907
+ nullable=False,
4908
+ default=SignatureAlgorithmType.ED25519.value,
4909
+ )
4910
+ signature: Mapped[str] = mapped_column(Text, nullable=False)
4911
+ signed_hash: Mapped[str] = mapped_column(String(128), nullable=False)
4912
+ signed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
4913
+ verified_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4914
+ is_valid: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
4915
+ signature_metadata: Mapped[dict[str, Any]] = mapped_column(
4916
+ "metadata", JSON, nullable=False, default=dict
4917
+ )
4918
+
4919
+ # Relationships
4920
+ plugin: Mapped["Plugin"] = relationship("Plugin", back_populates="signatures")
4921
+ signer: Mapped["TrustedSigner"] = relationship("TrustedSigner")
4922
+
4923
+ def mark_verified(self) -> None:
4924
+ """Mark the signature as verified."""
4925
+ self.verified_at = datetime.utcnow()
4926
+ self.is_valid = True
4927
+
4928
+ def invalidate(self) -> None:
4929
+ """Invalidate the signature."""
4930
+ self.is_valid = False
4931
+
4932
+
4933
+ class HotReloadConfig(Base, UUIDMixin, TimestampMixin):
4934
+ """Hot reload configuration model.
4935
+
4936
+ Stores hot reload settings for plugins.
4937
+
4938
+ Attributes:
4939
+ id: Unique identifier (UUID).
4940
+ plugin_id: Reference to Plugin.
4941
+ enabled: Whether hot reload is enabled.
4942
+ watch_paths: Paths to watch for changes (JSON array).
4943
+ debounce_ms: Debounce time in milliseconds.
4944
+ reload_strategy: Reload strategy (graceful, immediate, scheduled).
4945
+ max_reload_attempts: Maximum reload attempts before disabling.
4946
+ backup_on_reload: Whether to backup before reload.
4947
+ last_reload_at: When the last reload occurred.
4948
+ reload_count: Total reload count.
4949
+ last_error: Last error message.
4950
+ """
4951
+
4952
+ __tablename__ = "hot_reload_configs"
4953
+
4954
+ __table_args__ = (
4955
+ Index("idx_hot_reload_configs_plugin", "plugin_id", unique=True),
4956
+ )
4957
+
4958
+ plugin_id: Mapped[str] = mapped_column(
4959
+ String(36),
4960
+ ForeignKey("plugins.id", ondelete="CASCADE"),
4961
+ nullable=False,
4962
+ unique=True,
4963
+ )
4964
+ enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
4965
+ watch_paths: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list)
4966
+ debounce_ms: Mapped[int] = mapped_column(Integer, nullable=False, default=500)
4967
+ reload_strategy: Mapped[str] = mapped_column(
4968
+ String(20),
4969
+ nullable=False,
4970
+ default="graceful",
4971
+ )
4972
+ max_reload_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)
4973
+ backup_on_reload: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
4974
+ last_reload_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
4975
+ reload_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
4976
+ last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
4977
+
4978
+ # Relationships
4979
+ plugin: Mapped["Plugin"] = relationship("Plugin", back_populates="hot_reload_config")
4980
+
4981
+ def record_reload(self, success: bool = True, error: str | None = None) -> None:
4982
+ """Record a reload attempt."""
4983
+ self.last_reload_at = datetime.utcnow()
4984
+ self.reload_count += 1
4985
+ if not success:
4986
+ self.last_error = error
4987
+
4988
+
4989
+ class PluginHook(Base, UUIDMixin, TimestampMixin):
4990
+ """Plugin hook registration model.
4991
+
4992
+ Stores registered hooks for plugins.
4993
+
4994
+ Attributes:
4995
+ id: Unique identifier (UUID).
4996
+ plugin_id: Reference to Plugin.
4997
+ hook_type: Type of hook (pre_validation, post_validation, etc.).
4998
+ callback_name: Name of the callback function.
4999
+ priority: Hook priority (lower runs first).
5000
+ enabled: Whether the hook is enabled.
5001
+ last_triggered_at: When the hook was last triggered.
5002
+ trigger_count: Total trigger count.
5003
+ total_execution_ms: Total execution time in ms.
5004
+ last_error: Last error message.
5005
+ """
5006
+
5007
+ __tablename__ = "plugin_hooks"
5008
+
5009
+ __table_args__ = (
5010
+ Index("idx_plugin_hooks_plugin", "plugin_id"),
5011
+ Index("idx_plugin_hooks_type", "hook_type"),
5012
+ Index("idx_plugin_hooks_priority", "priority"),
5013
+ )
5014
+
5015
+ plugin_id: Mapped[str] = mapped_column(
5016
+ String(36),
5017
+ ForeignKey("plugins.id", ondelete="CASCADE"),
5018
+ nullable=False,
5019
+ index=True,
5020
+ )
5021
+ hook_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
5022
+ callback_name: Mapped[str] = mapped_column(String(255), nullable=False)
5023
+ priority: Mapped[int] = mapped_column(Integer, nullable=False, default=50)
5024
+ enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
5025
+ last_triggered_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
5026
+ trigger_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
5027
+ total_execution_ms: Mapped[float] = mapped_column(Float, nullable=False, default=0)
5028
+ last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
5029
+
5030
+ # Relationships
5031
+ plugin: Mapped["Plugin"] = relationship("Plugin", back_populates="hooks")
5032
+
5033
+ @property
5034
+ def average_execution_ms(self) -> float:
5035
+ """Calculate average execution time."""
5036
+ if self.trigger_count == 0:
5037
+ return 0
5038
+ return self.total_execution_ms / self.trigger_count
5039
+
5040
+ def record_trigger(self, execution_ms: float, error: str | None = None) -> None:
5041
+ """Record a hook trigger."""
5042
+ self.last_triggered_at = datetime.utcnow()
5043
+ self.trigger_count += 1
5044
+ self.total_execution_ms += execution_ms
5045
+ if error:
5046
+ self.last_error = error