truthound-dashboard 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- truthound_dashboard/__init__.py +11 -0
- truthound_dashboard/__main__.py +6 -0
- truthound_dashboard/api/__init__.py +15 -0
- truthound_dashboard/api/deps.py +153 -0
- truthound_dashboard/api/drift.py +179 -0
- truthound_dashboard/api/error_handlers.py +287 -0
- truthound_dashboard/api/health.py +78 -0
- truthound_dashboard/api/history.py +62 -0
- truthound_dashboard/api/middleware.py +626 -0
- truthound_dashboard/api/notifications.py +561 -0
- truthound_dashboard/api/profile.py +52 -0
- truthound_dashboard/api/router.py +83 -0
- truthound_dashboard/api/rules.py +277 -0
- truthound_dashboard/api/schedules.py +329 -0
- truthound_dashboard/api/schemas.py +136 -0
- truthound_dashboard/api/sources.py +229 -0
- truthound_dashboard/api/validations.py +125 -0
- truthound_dashboard/cli.py +226 -0
- truthound_dashboard/config.py +132 -0
- truthound_dashboard/core/__init__.py +264 -0
- truthound_dashboard/core/base.py +185 -0
- truthound_dashboard/core/cache.py +479 -0
- truthound_dashboard/core/connections.py +331 -0
- truthound_dashboard/core/encryption.py +409 -0
- truthound_dashboard/core/exceptions.py +627 -0
- truthound_dashboard/core/logging.py +488 -0
- truthound_dashboard/core/maintenance.py +542 -0
- truthound_dashboard/core/notifications/__init__.py +56 -0
- truthound_dashboard/core/notifications/base.py +390 -0
- truthound_dashboard/core/notifications/channels.py +557 -0
- truthound_dashboard/core/notifications/dispatcher.py +453 -0
- truthound_dashboard/core/notifications/events.py +155 -0
- truthound_dashboard/core/notifications/service.py +744 -0
- truthound_dashboard/core/sampling.py +626 -0
- truthound_dashboard/core/scheduler.py +311 -0
- truthound_dashboard/core/services.py +1531 -0
- truthound_dashboard/core/truthound_adapter.py +659 -0
- truthound_dashboard/db/__init__.py +67 -0
- truthound_dashboard/db/base.py +108 -0
- truthound_dashboard/db/database.py +196 -0
- truthound_dashboard/db/models.py +732 -0
- truthound_dashboard/db/repository.py +237 -0
- truthound_dashboard/main.py +309 -0
- truthound_dashboard/schemas/__init__.py +150 -0
- truthound_dashboard/schemas/base.py +96 -0
- truthound_dashboard/schemas/drift.py +118 -0
- truthound_dashboard/schemas/history.py +74 -0
- truthound_dashboard/schemas/profile.py +91 -0
- truthound_dashboard/schemas/rule.py +199 -0
- truthound_dashboard/schemas/schedule.py +88 -0
- truthound_dashboard/schemas/schema.py +121 -0
- truthound_dashboard/schemas/source.py +138 -0
- truthound_dashboard/schemas/validation.py +192 -0
- truthound_dashboard/static/assets/index-BqJMyAHX.js +110 -0
- truthound_dashboard/static/assets/index-DMDxHCTs.js +465 -0
- truthound_dashboard/static/assets/index-Dm2D11TK.css +1 -0
- truthound_dashboard/static/index.html +15 -0
- truthound_dashboard/static/mockServiceWorker.js +349 -0
- truthound_dashboard-1.0.0.dist-info/METADATA +218 -0
- truthound_dashboard-1.0.0.dist-info/RECORD +62 -0
- truthound_dashboard-1.0.0.dist-info/WHEEL +4 -0
- truthound_dashboard-1.0.0.dist-info/entry_points.txt +5 -0
|
@@ -0,0 +1,732 @@
|
|
|
1
|
+
"""SQLAlchemy database models.
|
|
2
|
+
|
|
3
|
+
This module defines all database models for the dashboard.
|
|
4
|
+
Models use mixins from base.py for consistent behavior.
|
|
5
|
+
|
|
6
|
+
Models:
|
|
7
|
+
- Source: Data source configuration
|
|
8
|
+
- Schema: Learned schemas from th.learn
|
|
9
|
+
- Rule: Custom validation rules for sources
|
|
10
|
+
- Validation: Validation run results
|
|
11
|
+
- AppSettings: Application-level settings
|
|
12
|
+
- NotificationChannel: Notification channel configuration
|
|
13
|
+
- NotificationRule: Notification trigger rules
|
|
14
|
+
- NotificationLog: Notification delivery log
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from datetime import datetime
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from sqlalchemy import (
|
|
23
|
+
JSON,
|
|
24
|
+
Boolean,
|
|
25
|
+
DateTime,
|
|
26
|
+
Enum as SQLEnum,
|
|
27
|
+
ForeignKey,
|
|
28
|
+
Index,
|
|
29
|
+
Integer,
|
|
30
|
+
String,
|
|
31
|
+
Text,
|
|
32
|
+
)
|
|
33
|
+
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|
34
|
+
|
|
35
|
+
from .base import Base, TimestampMixin, UUIDMixin
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Source(Base, UUIDMixin, TimestampMixin):
|
|
39
|
+
"""Data source model.
|
|
40
|
+
|
|
41
|
+
Represents a data source that can be validated.
|
|
42
|
+
Supports various types: file, postgresql, mysql, snowflake, bigquery.
|
|
43
|
+
|
|
44
|
+
Attributes:
|
|
45
|
+
id: Unique identifier (UUID).
|
|
46
|
+
name: Human-readable name for the source.
|
|
47
|
+
type: Source type (file, postgresql, etc.).
|
|
48
|
+
config: JSON configuration specific to source type.
|
|
49
|
+
is_active: Whether the source is active.
|
|
50
|
+
last_validated_at: Timestamp of last validation.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
__tablename__ = "sources"
|
|
54
|
+
|
|
55
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
56
|
+
type: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
57
|
+
config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
|
|
58
|
+
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
59
|
+
is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
60
|
+
last_validated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
61
|
+
|
|
62
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
63
|
+
if "is_active" not in kwargs:
|
|
64
|
+
kwargs["is_active"] = True
|
|
65
|
+
super().__init__(**kwargs)
|
|
66
|
+
|
|
67
|
+
# Relationships
|
|
68
|
+
schemas: Mapped[list[Schema]] = relationship(
|
|
69
|
+
"Schema",
|
|
70
|
+
back_populates="source",
|
|
71
|
+
cascade="all, delete-orphan",
|
|
72
|
+
lazy="selectin",
|
|
73
|
+
)
|
|
74
|
+
rules: Mapped[list[Rule]] = relationship(
|
|
75
|
+
"Rule",
|
|
76
|
+
back_populates="source",
|
|
77
|
+
cascade="all, delete-orphan",
|
|
78
|
+
lazy="selectin",
|
|
79
|
+
order_by="desc(Rule.created_at)",
|
|
80
|
+
)
|
|
81
|
+
validations: Mapped[list[Validation]] = relationship(
|
|
82
|
+
"Validation",
|
|
83
|
+
back_populates="source",
|
|
84
|
+
cascade="all, delete-orphan",
|
|
85
|
+
lazy="selectin",
|
|
86
|
+
order_by="desc(Validation.created_at)",
|
|
87
|
+
)
|
|
88
|
+
profiles: Mapped[list[Profile]] = relationship(
|
|
89
|
+
"Profile",
|
|
90
|
+
back_populates="source",
|
|
91
|
+
cascade="all, delete-orphan",
|
|
92
|
+
lazy="selectin",
|
|
93
|
+
order_by="desc(Profile.created_at)",
|
|
94
|
+
)
|
|
95
|
+
schedules: Mapped[list[Schedule]] = relationship(
|
|
96
|
+
"Schedule",
|
|
97
|
+
back_populates="source",
|
|
98
|
+
cascade="all, delete-orphan",
|
|
99
|
+
lazy="selectin",
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def source_path(self) -> str | None:
|
|
104
|
+
"""Get the data path from config."""
|
|
105
|
+
return self.config.get("path") or self.config.get("connection_string")
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def latest_schema(self) -> Schema | None:
|
|
109
|
+
"""Get the most recent schema."""
|
|
110
|
+
if self.schemas:
|
|
111
|
+
return max(self.schemas, key=lambda s: s.created_at)
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def latest_validation(self) -> Validation | None:
|
|
116
|
+
"""Get the most recent validation."""
|
|
117
|
+
if self.validations:
|
|
118
|
+
return self.validations[0]
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def active_rules(self) -> list[Rule]:
|
|
123
|
+
"""Get all active rules for this source."""
|
|
124
|
+
return [r for r in self.rules if r.is_active]
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def active_rule(self) -> Rule | None:
|
|
128
|
+
"""Get the active rule for this source (most recent)."""
|
|
129
|
+
active = self.active_rules
|
|
130
|
+
return active[0] if active else None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class Rule(Base, UUIDMixin, TimestampMixin):
|
|
134
|
+
"""Custom validation rules model.
|
|
135
|
+
|
|
136
|
+
Stores custom validation rules for data sources in YAML format.
|
|
137
|
+
Rules are used by truthound validators during validation runs.
|
|
138
|
+
|
|
139
|
+
Attributes:
|
|
140
|
+
id: Unique identifier (UUID).
|
|
141
|
+
source_id: Reference to parent Source.
|
|
142
|
+
name: Human-readable rule name.
|
|
143
|
+
description: Optional description of the rule.
|
|
144
|
+
rules_yaml: YAML content defining validation rules.
|
|
145
|
+
rules_json: Parsed rules as JSON for programmatic access.
|
|
146
|
+
is_active: Whether this rule set is currently active.
|
|
147
|
+
version: Optional version string for tracking changes.
|
|
148
|
+
|
|
149
|
+
Example rules_yaml format:
|
|
150
|
+
columns:
|
|
151
|
+
user_id:
|
|
152
|
+
not_null: true
|
|
153
|
+
unique: true
|
|
154
|
+
email:
|
|
155
|
+
pattern: "^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$"
|
|
156
|
+
age:
|
|
157
|
+
min: 0
|
|
158
|
+
max: 150
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
__tablename__ = "rules"
|
|
162
|
+
|
|
163
|
+
source_id: Mapped[str] = mapped_column(
|
|
164
|
+
String(36),
|
|
165
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
166
|
+
nullable=False,
|
|
167
|
+
index=True,
|
|
168
|
+
)
|
|
169
|
+
name: Mapped[str] = mapped_column(
|
|
170
|
+
String(255),
|
|
171
|
+
nullable=False,
|
|
172
|
+
default="Default Rules",
|
|
173
|
+
)
|
|
174
|
+
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
175
|
+
rules_yaml: Mapped[str] = mapped_column(Text, nullable=False)
|
|
176
|
+
rules_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
177
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
178
|
+
version: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
|
179
|
+
|
|
180
|
+
# Relationships
|
|
181
|
+
source: Mapped[Source] = relationship("Source", back_populates="rules")
|
|
182
|
+
|
|
183
|
+
@property
|
|
184
|
+
def column_rules(self) -> dict[str, Any]:
|
|
185
|
+
"""Get column-level rules from JSON."""
|
|
186
|
+
if self.rules_json and "columns" in self.rules_json:
|
|
187
|
+
return self.rules_json["columns"]
|
|
188
|
+
return {}
|
|
189
|
+
|
|
190
|
+
@property
|
|
191
|
+
def table_rules(self) -> dict[str, Any]:
|
|
192
|
+
"""Get table-level rules from JSON."""
|
|
193
|
+
if self.rules_json and "table" in self.rules_json:
|
|
194
|
+
return self.rules_json["table"]
|
|
195
|
+
return {}
|
|
196
|
+
|
|
197
|
+
@property
|
|
198
|
+
def column_count(self) -> int:
|
|
199
|
+
"""Get number of columns with rules defined."""
|
|
200
|
+
return len(self.column_rules)
|
|
201
|
+
|
|
202
|
+
def deactivate(self) -> None:
|
|
203
|
+
"""Mark this rule as inactive."""
|
|
204
|
+
self.is_active = False
|
|
205
|
+
|
|
206
|
+
def activate(self) -> None:
|
|
207
|
+
"""Mark this rule as active."""
|
|
208
|
+
self.is_active = True
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class Schema(Base, UUIDMixin, TimestampMixin):
|
|
212
|
+
"""Learned schema model.
|
|
213
|
+
|
|
214
|
+
Stores truthound Schema objects (from th.learn) which contain:
|
|
215
|
+
- Column definitions with dtype, nullable, unique, constraints
|
|
216
|
+
- Row count and statistics
|
|
217
|
+
- Version information
|
|
218
|
+
|
|
219
|
+
Attributes:
|
|
220
|
+
id: Unique identifier (UUID).
|
|
221
|
+
source_id: Reference to parent Source.
|
|
222
|
+
schema_yaml: YAML representation for display/editing.
|
|
223
|
+
schema_json: Full schema as JSON for programmatic access.
|
|
224
|
+
row_count: Number of rows when schema was learned.
|
|
225
|
+
version: Schema version string.
|
|
226
|
+
is_active: Whether this is the active schema for the source.
|
|
227
|
+
"""
|
|
228
|
+
|
|
229
|
+
__tablename__ = "schemas"
|
|
230
|
+
|
|
231
|
+
source_id: Mapped[str] = mapped_column(
|
|
232
|
+
String(36),
|
|
233
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
234
|
+
nullable=False,
|
|
235
|
+
index=True,
|
|
236
|
+
)
|
|
237
|
+
schema_yaml: Mapped[str] = mapped_column(Text, nullable=False)
|
|
238
|
+
schema_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
239
|
+
row_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
240
|
+
column_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
241
|
+
version: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
|
242
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
243
|
+
|
|
244
|
+
# Relationships
|
|
245
|
+
source: Mapped[Source] = relationship("Source", back_populates="schemas")
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def columns(self) -> list[str]:
|
|
249
|
+
"""Get list of column names from schema."""
|
|
250
|
+
if self.schema_json and "columns" in self.schema_json:
|
|
251
|
+
return list(self.schema_json["columns"].keys())
|
|
252
|
+
return []
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class Validation(Base, UUIDMixin):
|
|
256
|
+
"""Validation result model.
|
|
257
|
+
|
|
258
|
+
Stores results from th.check validation runs.
|
|
259
|
+
|
|
260
|
+
Attributes:
|
|
261
|
+
id: Unique identifier (UUID).
|
|
262
|
+
source_id: Reference to parent Source.
|
|
263
|
+
status: Current status (pending, running, success, failed, error).
|
|
264
|
+
passed: Whether validation passed (no issues).
|
|
265
|
+
has_critical: Whether critical issues were found.
|
|
266
|
+
has_high: Whether high severity issues were found.
|
|
267
|
+
total_issues: Total number of issues found.
|
|
268
|
+
result_json: Full validation result as JSON.
|
|
269
|
+
duration_ms: Validation duration in milliseconds.
|
|
270
|
+
"""
|
|
271
|
+
|
|
272
|
+
__tablename__ = "validations"
|
|
273
|
+
|
|
274
|
+
# Composite index for efficient history queries (source + time ordering)
|
|
275
|
+
__table_args__ = (
|
|
276
|
+
Index("idx_validations_source_created", "source_id", "created_at"),
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
source_id: Mapped[str] = mapped_column(
|
|
280
|
+
String(36),
|
|
281
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
282
|
+
nullable=False,
|
|
283
|
+
index=True,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
# Status tracking
|
|
287
|
+
status: Mapped[str] = mapped_column(
|
|
288
|
+
String(20),
|
|
289
|
+
nullable=False,
|
|
290
|
+
default="pending",
|
|
291
|
+
index=True,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
# Validation results summary
|
|
295
|
+
passed: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
296
|
+
has_critical: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
297
|
+
has_high: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
298
|
+
total_issues: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
299
|
+
critical_issues: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
300
|
+
high_issues: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
301
|
+
medium_issues: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
302
|
+
low_issues: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
303
|
+
|
|
304
|
+
# Data statistics
|
|
305
|
+
row_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
306
|
+
column_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
307
|
+
|
|
308
|
+
# Full result and timing
|
|
309
|
+
result_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
310
|
+
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
311
|
+
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
312
|
+
|
|
313
|
+
# Timestamps
|
|
314
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
315
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
316
|
+
)
|
|
317
|
+
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
318
|
+
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
319
|
+
|
|
320
|
+
# Relationships
|
|
321
|
+
source: Mapped[Source] = relationship("Source", back_populates="validations")
|
|
322
|
+
|
|
323
|
+
@property
|
|
324
|
+
def issues(self) -> list[dict[str, Any]]:
|
|
325
|
+
"""Get list of issues from result JSON."""
|
|
326
|
+
if self.result_json and "issues" in self.result_json:
|
|
327
|
+
return self.result_json["issues"]
|
|
328
|
+
return []
|
|
329
|
+
|
|
330
|
+
@property
|
|
331
|
+
def is_complete(self) -> bool:
|
|
332
|
+
"""Check if validation has completed (success, failed, or error)."""
|
|
333
|
+
return self.status in ("success", "failed", "error")
|
|
334
|
+
|
|
335
|
+
def mark_started(self) -> None:
|
|
336
|
+
"""Mark validation as started."""
|
|
337
|
+
self.status = "running"
|
|
338
|
+
self.started_at = datetime.utcnow()
|
|
339
|
+
|
|
340
|
+
def mark_completed(
|
|
341
|
+
self,
|
|
342
|
+
passed: bool,
|
|
343
|
+
result: dict[str, Any],
|
|
344
|
+
) -> None:
|
|
345
|
+
"""Mark validation as completed with results."""
|
|
346
|
+
self.status = "success" if passed else "failed"
|
|
347
|
+
self.passed = passed
|
|
348
|
+
self.result_json = result
|
|
349
|
+
self.completed_at = datetime.utcnow()
|
|
350
|
+
|
|
351
|
+
if self.started_at:
|
|
352
|
+
delta = self.completed_at - self.started_at
|
|
353
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
354
|
+
|
|
355
|
+
def mark_error(self, message: str) -> None:
|
|
356
|
+
"""Mark validation as errored."""
|
|
357
|
+
self.status = "error"
|
|
358
|
+
self.error_message = message
|
|
359
|
+
self.completed_at = datetime.utcnow()
|
|
360
|
+
|
|
361
|
+
if self.started_at:
|
|
362
|
+
delta = self.completed_at - self.started_at
|
|
363
|
+
self.duration_ms = int(delta.total_seconds() * 1000)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
class Profile(Base, UUIDMixin, TimestampMixin):
|
|
367
|
+
"""Data profile model.
|
|
368
|
+
|
|
369
|
+
Stores profiling results from th.profile() for historical tracking.
|
|
370
|
+
|
|
371
|
+
Attributes:
|
|
372
|
+
id: Unique identifier (UUID).
|
|
373
|
+
source_id: Reference to parent Source.
|
|
374
|
+
profile_json: Full profile result as JSON.
|
|
375
|
+
row_count: Number of rows profiled.
|
|
376
|
+
column_count: Number of columns.
|
|
377
|
+
size_bytes: Data size in bytes.
|
|
378
|
+
"""
|
|
379
|
+
|
|
380
|
+
__tablename__ = "profiles"
|
|
381
|
+
|
|
382
|
+
source_id: Mapped[str] = mapped_column(
|
|
383
|
+
String(36),
|
|
384
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
385
|
+
nullable=False,
|
|
386
|
+
index=True,
|
|
387
|
+
)
|
|
388
|
+
profile_json: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
|
389
|
+
row_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
390
|
+
column_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
391
|
+
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
392
|
+
|
|
393
|
+
# Relationships
|
|
394
|
+
source: Mapped[Source] = relationship("Source", back_populates="profiles")
|
|
395
|
+
|
|
396
|
+
@property
|
|
397
|
+
def columns(self) -> list[dict[str, Any]]:
|
|
398
|
+
"""Get column profiles from JSON."""
|
|
399
|
+
if self.profile_json and "columns" in self.profile_json:
|
|
400
|
+
return self.profile_json["columns"]
|
|
401
|
+
return []
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
class Schedule(Base, UUIDMixin, TimestampMixin):
|
|
405
|
+
"""Validation schedule model.
|
|
406
|
+
|
|
407
|
+
Manages scheduled validation runs using cron expressions.
|
|
408
|
+
|
|
409
|
+
Attributes:
|
|
410
|
+
id: Unique identifier (UUID).
|
|
411
|
+
name: Human-readable schedule name.
|
|
412
|
+
source_id: Reference to Source to validate.
|
|
413
|
+
cron_expression: Cron expression for scheduling.
|
|
414
|
+
is_active: Whether schedule is active.
|
|
415
|
+
notify_on_failure: Send notification on validation failure.
|
|
416
|
+
last_run_at: Timestamp of last execution.
|
|
417
|
+
next_run_at: Timestamp of next scheduled run.
|
|
418
|
+
config: Additional configuration (validators, schema_path, etc.).
|
|
419
|
+
"""
|
|
420
|
+
|
|
421
|
+
__tablename__ = "schedules"
|
|
422
|
+
|
|
423
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
424
|
+
source_id: Mapped[str] = mapped_column(
|
|
425
|
+
String(36),
|
|
426
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
427
|
+
nullable=False,
|
|
428
|
+
index=True,
|
|
429
|
+
)
|
|
430
|
+
cron_expression: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
431
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
432
|
+
notify_on_failure: Mapped[bool] = mapped_column(
|
|
433
|
+
Boolean, default=True, nullable=False
|
|
434
|
+
)
|
|
435
|
+
last_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
436
|
+
next_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
437
|
+
config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
438
|
+
|
|
439
|
+
# Relationships
|
|
440
|
+
source: Mapped[Source] = relationship("Source", back_populates="schedules")
|
|
441
|
+
|
|
442
|
+
def pause(self) -> None:
|
|
443
|
+
"""Pause this schedule."""
|
|
444
|
+
self.is_active = False
|
|
445
|
+
|
|
446
|
+
def resume(self) -> None:
|
|
447
|
+
"""Resume this schedule."""
|
|
448
|
+
self.is_active = True
|
|
449
|
+
|
|
450
|
+
def mark_run(self, next_run: datetime | None = None) -> None:
|
|
451
|
+
"""Mark schedule as run and update next run time."""
|
|
452
|
+
self.last_run_at = datetime.utcnow()
|
|
453
|
+
self.next_run_at = next_run
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
class DriftComparison(Base, UUIDMixin, TimestampMixin):
|
|
457
|
+
"""Drift comparison result model.
|
|
458
|
+
|
|
459
|
+
Stores results from th.compare() drift detection.
|
|
460
|
+
|
|
461
|
+
Attributes:
|
|
462
|
+
id: Unique identifier (UUID).
|
|
463
|
+
baseline_source_id: Reference to baseline Source.
|
|
464
|
+
current_source_id: Reference to current Source.
|
|
465
|
+
has_drift: Whether drift was detected.
|
|
466
|
+
has_high_drift: Whether high-severity drift was detected.
|
|
467
|
+
total_columns: Total columns compared.
|
|
468
|
+
drifted_columns: Number of columns with drift.
|
|
469
|
+
result_json: Full comparison result as JSON.
|
|
470
|
+
config: Comparison configuration (method, threshold, etc.).
|
|
471
|
+
"""
|
|
472
|
+
|
|
473
|
+
__tablename__ = "drift_comparisons"
|
|
474
|
+
|
|
475
|
+
baseline_source_id: Mapped[str] = mapped_column(
|
|
476
|
+
String(36),
|
|
477
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
478
|
+
nullable=False,
|
|
479
|
+
index=True,
|
|
480
|
+
)
|
|
481
|
+
current_source_id: Mapped[str] = mapped_column(
|
|
482
|
+
String(36),
|
|
483
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
484
|
+
nullable=False,
|
|
485
|
+
index=True,
|
|
486
|
+
)
|
|
487
|
+
has_drift: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
488
|
+
has_high_drift: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
|
489
|
+
total_columns: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
490
|
+
drifted_columns: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
491
|
+
result_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
492
|
+
config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
493
|
+
|
|
494
|
+
# Relationships
|
|
495
|
+
baseline_source: Mapped[Source] = relationship(
|
|
496
|
+
"Source",
|
|
497
|
+
foreign_keys=[baseline_source_id],
|
|
498
|
+
backref="baseline_comparisons",
|
|
499
|
+
)
|
|
500
|
+
current_source: Mapped[Source] = relationship(
|
|
501
|
+
"Source",
|
|
502
|
+
foreign_keys=[current_source_id],
|
|
503
|
+
backref="current_comparisons",
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
@property
|
|
507
|
+
def drift_percentage(self) -> float:
|
|
508
|
+
"""Calculate percentage of columns with drift."""
|
|
509
|
+
if self.total_columns and self.total_columns > 0:
|
|
510
|
+
return (self.drifted_columns or 0) / self.total_columns * 100
|
|
511
|
+
return 0.0
|
|
512
|
+
|
|
513
|
+
@property
|
|
514
|
+
def column_results(self) -> list[dict[str, Any]]:
|
|
515
|
+
"""Get per-column drift results."""
|
|
516
|
+
if self.result_json and "columns" in self.result_json:
|
|
517
|
+
return self.result_json["columns"]
|
|
518
|
+
return []
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
class AppSettings(Base):
|
|
522
|
+
"""Application settings model.
|
|
523
|
+
|
|
524
|
+
Stores key-value configuration that can be modified at runtime.
|
|
525
|
+
|
|
526
|
+
Attributes:
|
|
527
|
+
key: Setting key (primary key).
|
|
528
|
+
value: JSON value for the setting.
|
|
529
|
+
description: Human-readable description.
|
|
530
|
+
"""
|
|
531
|
+
|
|
532
|
+
__tablename__ = "app_settings"
|
|
533
|
+
|
|
534
|
+
key: Mapped[str] = mapped_column(String(100), primary_key=True)
|
|
535
|
+
value: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
|
536
|
+
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
537
|
+
updated_at: Mapped[datetime] = mapped_column(
|
|
538
|
+
DateTime,
|
|
539
|
+
default=datetime.utcnow,
|
|
540
|
+
onupdate=datetime.utcnow,
|
|
541
|
+
nullable=False,
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
# =============================================================================
|
|
546
|
+
# Phase 3: Notification Models
|
|
547
|
+
# =============================================================================
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
class NotificationChannel(Base, UUIDMixin, TimestampMixin):
|
|
551
|
+
"""Notification channel configuration model.
|
|
552
|
+
|
|
553
|
+
Represents a configured notification channel (Slack, Email, Webhook, etc.).
|
|
554
|
+
Uses a polymorphic pattern where channel-specific configuration is stored
|
|
555
|
+
in the 'config' JSON field.
|
|
556
|
+
|
|
557
|
+
Attributes:
|
|
558
|
+
id: Unique identifier (UUID).
|
|
559
|
+
type: Channel type (slack, email, webhook, etc.).
|
|
560
|
+
name: Human-readable channel name.
|
|
561
|
+
config: JSON configuration specific to channel type.
|
|
562
|
+
is_active: Whether channel is active.
|
|
563
|
+
|
|
564
|
+
Example configs:
|
|
565
|
+
Slack: {"webhook_url": "https://hooks.slack.com/..."}
|
|
566
|
+
Email: {"smtp_host": "...", "smtp_port": 587, "recipients": [...]}
|
|
567
|
+
Webhook: {"url": "...", "headers": {...}, "method": "POST"}
|
|
568
|
+
"""
|
|
569
|
+
|
|
570
|
+
__tablename__ = "notification_channels"
|
|
571
|
+
|
|
572
|
+
type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
|
573
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
574
|
+
config: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
|
|
575
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
576
|
+
|
|
577
|
+
# Relationships
|
|
578
|
+
logs: Mapped[list[NotificationLog]] = relationship(
|
|
579
|
+
"NotificationLog",
|
|
580
|
+
back_populates="channel",
|
|
581
|
+
cascade="all, delete-orphan",
|
|
582
|
+
lazy="selectin",
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
def activate(self) -> None:
|
|
586
|
+
"""Activate this channel."""
|
|
587
|
+
self.is_active = True
|
|
588
|
+
|
|
589
|
+
def deactivate(self) -> None:
|
|
590
|
+
"""Deactivate this channel."""
|
|
591
|
+
self.is_active = False
|
|
592
|
+
|
|
593
|
+
def get_config_summary(self) -> str:
|
|
594
|
+
"""Get a safe summary of channel configuration (hides sensitive data)."""
|
|
595
|
+
if self.type == "slack":
|
|
596
|
+
url = self.config.get("webhook_url", "")
|
|
597
|
+
return f"Webhook: ...{url[-20:]}" if len(url) > 20 else f"Webhook: {url}"
|
|
598
|
+
elif self.type == "email":
|
|
599
|
+
recipients = self.config.get("recipients", [])
|
|
600
|
+
preview = ", ".join(recipients[:2])
|
|
601
|
+
suffix = "..." if len(recipients) > 2 else ""
|
|
602
|
+
return f"Recipients: {preview}{suffix}"
|
|
603
|
+
elif self.type == "webhook":
|
|
604
|
+
url = self.config.get("url", "")
|
|
605
|
+
return f"URL: {url[:50]}..." if len(url) > 50 else f"URL: {url}"
|
|
606
|
+
return "Configured"
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
class NotificationRule(Base, UUIDMixin, TimestampMixin):
|
|
610
|
+
"""Notification trigger rules model.
|
|
611
|
+
|
|
612
|
+
Defines when and how notifications should be triggered based on
|
|
613
|
+
validation events and conditions.
|
|
614
|
+
|
|
615
|
+
Attributes:
|
|
616
|
+
id: Unique identifier (UUID).
|
|
617
|
+
name: Human-readable rule name.
|
|
618
|
+
condition: Trigger condition type.
|
|
619
|
+
condition_config: Additional condition configuration.
|
|
620
|
+
channel_ids: List of channel IDs to notify.
|
|
621
|
+
source_ids: Optional list of source IDs to filter (null = all sources).
|
|
622
|
+
is_active: Whether rule is active.
|
|
623
|
+
|
|
624
|
+
Condition types:
|
|
625
|
+
- validation_failed: Any validation failure
|
|
626
|
+
- critical_issues: Validation has critical issues
|
|
627
|
+
- high_issues: Validation has high severity issues
|
|
628
|
+
- schedule_failed: Scheduled validation failed
|
|
629
|
+
- drift_detected: Drift detected in comparison
|
|
630
|
+
"""
|
|
631
|
+
|
|
632
|
+
__tablename__ = "notification_rules"
|
|
633
|
+
|
|
634
|
+
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
635
|
+
condition: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
|
636
|
+
condition_config: Mapped[dict[str, Any] | None] = mapped_column(
|
|
637
|
+
JSON, nullable=True, default=dict
|
|
638
|
+
)
|
|
639
|
+
channel_ids: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list)
|
|
640
|
+
source_ids: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
|
641
|
+
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
|
642
|
+
|
|
643
|
+
def activate(self) -> None:
|
|
644
|
+
"""Activate this rule."""
|
|
645
|
+
self.is_active = True
|
|
646
|
+
|
|
647
|
+
def deactivate(self) -> None:
|
|
648
|
+
"""Deactivate this rule."""
|
|
649
|
+
self.is_active = False
|
|
650
|
+
|
|
651
|
+
def matches_source(self, source_id: str) -> bool:
|
|
652
|
+
"""Check if this rule applies to a given source.
|
|
653
|
+
|
|
654
|
+
Args:
|
|
655
|
+
source_id: Source ID to check.
|
|
656
|
+
|
|
657
|
+
Returns:
|
|
658
|
+
True if rule applies to this source.
|
|
659
|
+
"""
|
|
660
|
+
if self.source_ids is None:
|
|
661
|
+
return True # Applies to all sources
|
|
662
|
+
return source_id in self.source_ids
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
class NotificationLog(Base, UUIDMixin):
|
|
666
|
+
"""Notification delivery log model.
|
|
667
|
+
|
|
668
|
+
Records all notification delivery attempts for auditing and debugging.
|
|
669
|
+
|
|
670
|
+
Attributes:
|
|
671
|
+
id: Unique identifier (UUID).
|
|
672
|
+
channel_id: Reference to NotificationChannel.
|
|
673
|
+
rule_id: Optional reference to NotificationRule that triggered this.
|
|
674
|
+
event_type: Type of event that triggered notification.
|
|
675
|
+
event_data: JSON data about the triggering event.
|
|
676
|
+
status: Delivery status (pending, sent, failed).
|
|
677
|
+
error_message: Error message if delivery failed.
|
|
678
|
+
sent_at: Timestamp when notification was sent.
|
|
679
|
+
"""
|
|
680
|
+
|
|
681
|
+
__tablename__ = "notification_logs"
|
|
682
|
+
|
|
683
|
+
# Composite index for efficient queries
|
|
684
|
+
__table_args__ = (
|
|
685
|
+
Index("idx_notification_logs_channel_created", "channel_id", "created_at"),
|
|
686
|
+
Index("idx_notification_logs_status", "status"),
|
|
687
|
+
)
|
|
688
|
+
|
|
689
|
+
channel_id: Mapped[str] = mapped_column(
|
|
690
|
+
String(36),
|
|
691
|
+
ForeignKey("notification_channels.id", ondelete="CASCADE"),
|
|
692
|
+
nullable=False,
|
|
693
|
+
index=True,
|
|
694
|
+
)
|
|
695
|
+
rule_id: Mapped[str | None] = mapped_column(
|
|
696
|
+
String(36),
|
|
697
|
+
ForeignKey("notification_rules.id", ondelete="SET NULL"),
|
|
698
|
+
nullable=True,
|
|
699
|
+
index=True,
|
|
700
|
+
)
|
|
701
|
+
event_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
702
|
+
event_data: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
|
703
|
+
message: Mapped[str] = mapped_column(Text, nullable=False)
|
|
704
|
+
status: Mapped[str] = mapped_column(
|
|
705
|
+
String(20), nullable=False, default="pending", index=True
|
|
706
|
+
)
|
|
707
|
+
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
708
|
+
|
|
709
|
+
# Timestamps
|
|
710
|
+
created_at: Mapped[datetime] = mapped_column(
|
|
711
|
+
DateTime, default=datetime.utcnow, nullable=False
|
|
712
|
+
)
|
|
713
|
+
sent_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
|
714
|
+
|
|
715
|
+
# Relationships
|
|
716
|
+
channel: Mapped[NotificationChannel] = relationship(
|
|
717
|
+
"NotificationChannel", back_populates="logs"
|
|
718
|
+
)
|
|
719
|
+
rule: Mapped[NotificationRule | None] = relationship(
|
|
720
|
+
"NotificationRule", backref="logs"
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
def mark_sent(self) -> None:
|
|
724
|
+
"""Mark notification as successfully sent."""
|
|
725
|
+
self.status = "sent"
|
|
726
|
+
self.sent_at = datetime.utcnow()
|
|
727
|
+
|
|
728
|
+
def mark_failed(self, error: str) -> None:
|
|
729
|
+
"""Mark notification as failed with error message."""
|
|
730
|
+
self.status = "failed"
|
|
731
|
+
self.error_message = error
|
|
732
|
+
self.sent_at = datetime.utcnow()
|