dqvouch 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dqvouch/__init__.py +1 -0
- dqvouch/alerts.py +129 -0
- dqvouch/changes.py +507 -0
- dqvouch/cli.py +283 -0
- dqvouch/config.py +181 -0
- dqvouch/context.py +229 -0
- dqvouch/llm.py +231 -0
- dqvouch/pipeline.py +350 -0
- dqvouch/profiling.py +1018 -0
- dqvouch/render_sql.py +711 -0
- dqvouch/rules.py +176 -0
- dqvouch/selector.py +132 -0
- dqvouch/server.py +493 -0
- dqvouch/signals.py +542 -0
- dqvouch/specifier.py +544 -0
- dqvouch/static/assets/index-6eimbKv1.css +1 -0
- dqvouch/static/assets/index-CaNRKsuh.js +3 -0
- dqvouch/static/assets/inter-cyrillic-ext-wght-normal-BOeWTOD4.woff2 +0 -0
- dqvouch/static/assets/inter-cyrillic-wght-normal-DqGufNeO.woff2 +0 -0
- dqvouch/static/assets/inter-greek-ext-wght-normal-DlzME5K_.woff2 +0 -0
- dqvouch/static/assets/inter-greek-wght-normal-CkhJZR-_.woff2 +0 -0
- dqvouch/static/assets/inter-latin-ext-wght-normal-DO1Apj_S.woff2 +0 -0
- dqvouch/static/assets/inter-latin-wght-normal-Dx4kXJAl.woff2 +0 -0
- dqvouch/static/assets/inter-vietnamese-wght-normal-CBcvBZtf.woff2 +0 -0
- dqvouch/static/icon.svg +8 -0
- dqvouch/static/index.html +15 -0
- dqvouch/store.py +163 -0
- dqvouch/translate.py +114 -0
- dqvouch/validator.py +1394 -0
- dqvouch/warehouse.py +214 -0
- dqvouch-0.2.1.dist-info/METADATA +350 -0
- dqvouch-0.2.1.dist-info/RECORD +35 -0
- dqvouch-0.2.1.dist-info/WHEEL +4 -0
- dqvouch-0.2.1.dist-info/entry_points.txt +2 -0
- dqvouch-0.2.1.dist-info/licenses/LICENSE +21 -0
dqvouch/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""dqvouch: an LLM proposes data quality tests, code validates them, you vouch for them."""
|
dqvouch/alerts.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Alerts when an execution of a run's tests gets worse: a test starts failing or erroring, or the execution itself
|
|
2
|
+
cannot run after one that did. Sent by webhook ({"text": ...}: Slack, Mattermost, Google Chat, Rocket.Chat, Teams
|
|
3
|
+
workflows; Discord with /slack appended) and/or by email (SMTP), as configured once in the .env file. A known problem
|
|
4
|
+
is said once, not at every execution, and a failed send never fails the execution: it is recorded instead."""
|
|
5
|
+
import json
|
|
6
|
+
import smtplib
|
|
7
|
+
import ssl
|
|
8
|
+
import urllib.request
|
|
9
|
+
from email.message import EmailMessage
|
|
10
|
+
|
|
11
|
+
from dqvouch.config import ENGINE_LABELS
|
|
12
|
+
|
|
13
|
+
WEBHOOK = "DQVOUCH_ALERT_WEBHOOK_URL"
|
|
14
|
+
BAD = ("fail", "error")
|
|
15
|
+
TRIGGERS = {"schedule": "a schedule", "command": "dqvouch rerun", "manual": "a person"}
|
|
16
|
+
MAX_LISTED = 20
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def worse(entry: dict, history: list) -> dict | None:
|
|
20
|
+
"""What got worse in `entry` against `history` (the executions before it, newest first): {"error": ...} when it
|
|
21
|
+
could not run and the one before could, {"new": [check ids]} for tests failing or erroring now that did not in
|
|
22
|
+
the last execution with verdicts; None otherwise (the first execution has nothing to be compared with)."""
|
|
23
|
+
if "error" in entry:
|
|
24
|
+
return {"error": entry["error"]} if not history or "error" not in history[0] else None
|
|
25
|
+
before = next((h for h in history if h.get("verdicts")), None)
|
|
26
|
+
if before is None or not entry.get("verdicts"):
|
|
27
|
+
return None
|
|
28
|
+
new = [i for i, v in entry["verdicts"].items() if v in BAD and before["verdicts"].get(i) not in BAD]
|
|
29
|
+
return {"new": new} if new else None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def message(run_info: dict, entry: dict, problem: dict, rows: list) -> tuple[str, str]:
|
|
33
|
+
"""(subject, text) of the alert."""
|
|
34
|
+
db = run_info["settings"]["database"]
|
|
35
|
+
where = (f"{ENGINE_LABELS.get(db['engine'], db['engine'])} at {db['host']}:{db['port']} "
|
|
36
|
+
f"({', '.join(db['schemas'])})")
|
|
37
|
+
when = f"started by {TRIGGERS.get(entry.get('trigger'), entry.get('trigger'))} at {entry.get('at')}"
|
|
38
|
+
if "error" in problem:
|
|
39
|
+
return (f"dqvouch: run {run_info['id']} could not execute its tests",
|
|
40
|
+
f"The tests of dqvouch run {run_info['id']} on {where} could not run ({when}):\n{problem['error']}")
|
|
41
|
+
new, by_id = problem["new"], {r["check_id"]: r for r in rows}
|
|
42
|
+
counts = ", ".join(f"{n} {v.replace('_', ' ')}" for v, n in sorted((entry.get("counts") or {}).items()))
|
|
43
|
+
lines = [f"{len(new)} test(s) of dqvouch run {run_info['id']} on {where} started failing ({when}).",
|
|
44
|
+
f"This execution: {counts}.", ""]
|
|
45
|
+
for check_id in new[:MAX_LISTED]:
|
|
46
|
+
r = by_id.get(check_id, {})
|
|
47
|
+
title = r.get("title") or str(check_id).split("__")[-1]
|
|
48
|
+
detail = (f"error: {r['error']}" if r.get("error")
|
|
49
|
+
else f"observed {r.get('observed')}, expected {r.get('expected')}")
|
|
50
|
+
lines.append(f"- {title} [{check_id}] on {r.get('asset', '?')}: {r.get('verdict', entry['verdicts'][check_id])}"
|
|
51
|
+
f", {detail}")
|
|
52
|
+
if len(new) > MAX_LISTED:
|
|
53
|
+
lines.append(f"... and {len(new) - MAX_LISTED} more")
|
|
54
|
+
return f"dqvouch: {len(new)} test(s) started failing in run {run_info['id']}", "\n".join(lines)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def send_webhook(url: str, text: str, timeout: float = 10) -> None:
|
|
58
|
+
request = urllib.request.Request(url, data=json.dumps({"text": text}).encode("utf-8"), method="POST",
|
|
59
|
+
headers={"Content-Type": "application/json"})
|
|
60
|
+
with urllib.request.urlopen(request, timeout=timeout):
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def send_email(values: dict, subject: str, text: str) -> None:
|
|
65
|
+
host, port = values["DQVOUCH_SMTP_HOST"], int(values.get("DQVOUCH_SMTP_PORT") or 587)
|
|
66
|
+
user = values.get("DQVOUCH_SMTP_USER") or ""
|
|
67
|
+
msg = EmailMessage()
|
|
68
|
+
msg["Subject"], msg["From"] = subject, values.get("DQVOUCH_SMTP_FROM") or user
|
|
69
|
+
msg["To"] = ", ".join(a.strip() for a in values["DQVOUCH_SMTP_TO"].split(",") if a.strip())
|
|
70
|
+
msg.set_content(text)
|
|
71
|
+
context = ssl.create_default_context()
|
|
72
|
+
if port == 465:
|
|
73
|
+
server = smtplib.SMTP_SSL(host, port, timeout=10, context=context)
|
|
74
|
+
else:
|
|
75
|
+
server = smtplib.SMTP(host, port, timeout=10)
|
|
76
|
+
with server:
|
|
77
|
+
encrypted = port == 465
|
|
78
|
+
if not encrypted:
|
|
79
|
+
server.ehlo()
|
|
80
|
+
if server.has_extn("starttls"):
|
|
81
|
+
server.starttls(context=context)
|
|
82
|
+
server.ehlo()
|
|
83
|
+
encrypted = True
|
|
84
|
+
if user:
|
|
85
|
+
if not encrypted:
|
|
86
|
+
raise RuntimeError(f"{host}:{port} does not offer STARTTLS, and dqvouch does not send the SMTP password "
|
|
87
|
+
f"unencrypted: use port 465 or a server with STARTTLS")
|
|
88
|
+
server.login(user, values.get("DQVOUCH_SMTP_PASSWORD") or "")
|
|
89
|
+
server.send_message(msg)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _email_configured(values: dict) -> bool:
|
|
93
|
+
return bool(values.get("DQVOUCH_SMTP_HOST") and values.get("DQVOUCH_SMTP_TO"))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def configured(values: dict) -> bool:
|
|
97
|
+
return bool(values.get(WEBHOOK)) or _email_configured(values)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def deliver(values: dict, subject: str, text: str, redact) -> dict:
|
|
101
|
+
"""Sends to every configured channel: {"alerted": [channels]} and/or {"alert_error": text}, never raises."""
|
|
102
|
+
sent, errors = [], []
|
|
103
|
+
if values.get(WEBHOOK):
|
|
104
|
+
try:
|
|
105
|
+
send_webhook(values[WEBHOOK], text)
|
|
106
|
+
sent.append("webhook")
|
|
107
|
+
except Exception as e:
|
|
108
|
+
errors.append(f"webhook: {e or type(e).__name__}")
|
|
109
|
+
if _email_configured(values):
|
|
110
|
+
try:
|
|
111
|
+
send_email(values, subject, text)
|
|
112
|
+
sent.append("email")
|
|
113
|
+
except Exception as e:
|
|
114
|
+
errors.append(f"email: {e or type(e).__name__}")
|
|
115
|
+
result = {"alerted": sent} if sent else {}
|
|
116
|
+
if errors:
|
|
117
|
+
result["alert_error"] = redact("; ".join(errors))
|
|
118
|
+
return result
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def notifier(values: dict, redact, run_info: dict):
|
|
122
|
+
"""The `notify(entry, history, rows)` that record_execution calls: alerts when the execution got worse."""
|
|
123
|
+
def notify(entry: dict, history: list, rows: list) -> dict:
|
|
124
|
+
problem = worse(entry, history) if configured(values) else None
|
|
125
|
+
if not problem:
|
|
126
|
+
return {}
|
|
127
|
+
subject, text = message(run_info, entry, problem, rows)
|
|
128
|
+
return deliver(values, subject, redact(text), redact)
|
|
129
|
+
return notify
|
dqvouch/changes.py
ADDED
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""Structural changes between a run and the database now, and what dqvouch does about them.
|
|
2
|
+
|
|
3
|
+
A structural change is a change in what a table is: its columns (names, types, descriptions), its description, the
|
|
4
|
+
SQL that builds it and the tables it reads from or feeds. What dqvouch derives from those (dq_signals, semantic
|
|
5
|
+
roles, usage facts) is not part of it, so a new dqvouch release never looks like a change; values (row counts,
|
|
6
|
+
nulls, distributions) never are either: catching a value going wrong is the tests' job. Each table gets its own
|
|
7
|
+
signature, so only the tables that changed go back to the model."""
|
|
8
|
+
import copy
|
|
9
|
+
import datetime
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import shutil
|
|
14
|
+
import time
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from dqvouch import alerts, pipeline
|
|
18
|
+
from dqvouch.profiling import DQ_DIMENSIONS
|
|
19
|
+
from dqvouch.rules import run_invariant_extractor, with_invariant_items
|
|
20
|
+
from dqvouch.selector import select_dimensions
|
|
21
|
+
from dqvouch.store import Run, RunBusyError
|
|
22
|
+
from dqvouch.validator import (
|
|
23
|
+
validate_dimension_output, validate_reviewed_metric_thresholds, validate_reviewed_test_specifications,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _structure(ctx: dict) -> dict:
|
|
28
|
+
"""The facts of an AssetContext that come from the database and the SQL files, not from dqvouch's code."""
|
|
29
|
+
lineage = ctx.get("lineage") or {}
|
|
30
|
+
|
|
31
|
+
def names(value):
|
|
32
|
+
return sorted(value) if isinstance(value, list) else value
|
|
33
|
+
return {"description": (ctx.get("asset") or {}).get("description"),
|
|
34
|
+
"columns": sorted([[c.get("name"), c.get("type"), c.get("description")] for c in ctx.get("columns") or []],
|
|
35
|
+
key=lambda c: str(c[0])),
|
|
36
|
+
"sql": (ctx.get("transformation") or {}).get("raw_sql"),
|
|
37
|
+
"upstream": names(lineage.get("upstream_assets")), "downstream": names(lineage.get("downstream_assets"))}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def table_signatures(asset_contexts: dict) -> dict[str, str]:
|
|
41
|
+
"""{table: sha256 of its structure}: equal signatures, the table is what it was."""
|
|
42
|
+
return {table: hashlib.sha256(json.dumps(_structure(ctx), sort_keys=True, default=str).encode("utf-8")).hexdigest()
|
|
43
|
+
for table, ctx in asset_contexts.items()}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _columns(ctx: dict) -> dict:
|
|
47
|
+
return {c["name"]: c for c in ctx.get("columns") or []}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _detail(old: dict, new: dict) -> dict:
|
|
51
|
+
"""What changed on one table, in the terms a person reads."""
|
|
52
|
+
before, after = _columns(old), _columns(new)
|
|
53
|
+
common = [name for name in before if name in after]
|
|
54
|
+
detail = {
|
|
55
|
+
"columns_added": [{"column": name, "type": after[name].get("type")} for name in after if name not in before],
|
|
56
|
+
"columns_dropped": [name for name in before if name not in after],
|
|
57
|
+
"columns_retyped": [{"column": name, "from": before[name].get("type"), "to": after[name].get("type")}
|
|
58
|
+
for name in common if before[name].get("type") != after[name].get("type")],
|
|
59
|
+
"descriptions_changed": ((old.get("asset") or {}).get("description")
|
|
60
|
+
!= (new.get("asset") or {}).get("description")
|
|
61
|
+
or any(before[n].get("description") != after[n].get("description") for n in common)),
|
|
62
|
+
"sql_changed": (old.get("transformation") or {}).get("raw_sql")
|
|
63
|
+
!= (new.get("transformation") or {}).get("raw_sql"),
|
|
64
|
+
"lineage_changed": ((_structure(old)["upstream"], _structure(old)["downstream"])
|
|
65
|
+
!= (_structure(new)["upstream"], _structure(new)["downstream"])),
|
|
66
|
+
}
|
|
67
|
+
return detail
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def diff_contexts(old_contexts: dict, old_available: list[str], new_contexts: dict) -> dict:
|
|
71
|
+
"""{"changed": {table: detail}, "new": [tables], "dropped": [tables]} between a run's AssetContexts and fresh
|
|
72
|
+
ones. A table the run left out on purpose (in `old_available` but not in its contexts) is neither."""
|
|
73
|
+
before, after = table_signatures(old_contexts), table_signatures(new_contexts)
|
|
74
|
+
return {
|
|
75
|
+
"changed": {t: _detail(old_contexts[t], new_contexts[t])
|
|
76
|
+
for t in sorted(before) if t in after and before[t] != after[t]},
|
|
77
|
+
"new": sorted(t for t in after if t not in set(old_available)),
|
|
78
|
+
"dropped": sorted(t for t in before if t not in after),
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def has_changes(changes: dict) -> bool:
|
|
83
|
+
return bool(changes["changed"] or changes["new"] or changes["dropped"])
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def targets(changes: dict) -> list[str]:
|
|
87
|
+
"""The tables the model proposes tests for again: the changed ones and the new ones."""
|
|
88
|
+
return sorted(set(changes["changed"]) | set(changes["new"]))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _line(table: str, detail: dict) -> str:
|
|
92
|
+
parts = [f"+ {a['column']} ({a['type']})" for a in detail["columns_added"]]
|
|
93
|
+
parts += [f"- {c}" for c in detail["columns_dropped"]]
|
|
94
|
+
parts += [f"{r['column']}: {r['from']} -> {r['to']}" for r in detail["columns_retyped"]]
|
|
95
|
+
for key, words in (("descriptions_changed", "descriptions changed"), ("sql_changed", "SQL changed"),
|
|
96
|
+
("lineage_changed", "lineage changed")):
|
|
97
|
+
if detail[key]:
|
|
98
|
+
parts.append(words)
|
|
99
|
+
return f"{table}: {', '.join(parts)}"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def describe_changes(changes: dict) -> list[str]:
|
|
103
|
+
"""One line per table, sorted by table: what a person reads in the alert and on the Review screen."""
|
|
104
|
+
lines = {t: _line(t, d) for t, d in changes["changed"].items()}
|
|
105
|
+
lines.update({t: f"{t}: new table" for t in changes["new"]})
|
|
106
|
+
lines.update({t: f"{t}: dropped" for t in changes["dropped"]})
|
|
107
|
+
return [lines[t] for t in sorted(lines)]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def detect(run: Run, wh) -> dict:
|
|
111
|
+
"""{"changes": diff, "context": stage 1's result now} for the run's tables still there plus the tables that
|
|
112
|
+
appeared since (not the ones the run left out with --tables). Reads only metadata and the SQL files."""
|
|
113
|
+
settings, old = run.info()["settings"], run.load("context")
|
|
114
|
+
sql_dir = settings.get("sql_dir")
|
|
115
|
+
if settings.get("context_level") == "C3" and sql_dir and not (Path(sql_dir).is_dir()
|
|
116
|
+
and any(Path(sql_dir).rglob("*.sql"))):
|
|
117
|
+
# without it every table's SQL would look gone: a moved folder is not a change of the database
|
|
118
|
+
raise ValueError(f"cannot check the structure: the SQL folder {sql_dir} is missing or has no .sql file")
|
|
119
|
+
now = pipeline.build_context(settings, wh) # one read of the metadata: tables_available and every context
|
|
120
|
+
known = set(old["tables_available"])
|
|
121
|
+
wanted = [t for t in now["tables_available"] if t in old["asset_contexts"] or t not in known]
|
|
122
|
+
if not wanted: # every table of the run is gone and nothing new appeared
|
|
123
|
+
fresh = {**now, "asset_contexts": {}}
|
|
124
|
+
elif set(wanted) == set(now["asset_contexts"]):
|
|
125
|
+
fresh = now
|
|
126
|
+
else: # the run covered some tables only: their contexts must be built the same way (lineage, usage)
|
|
127
|
+
fresh = pipeline.build_context(settings, wh, wanted)
|
|
128
|
+
return {"changes": diff_contexts(old["asset_contexts"], old["tables_available"], fresh["asset_contexts"]),
|
|
129
|
+
"context": fresh}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _without_rationale(item: dict) -> str:
|
|
133
|
+
return json.dumps({k: v for k, v in item.items() if k != "rationale"}, sort_keys=True, default=str)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def merge_dimensions(old: dict, new: dict, targets: set[str], tables: set[str]) -> dict:
|
|
137
|
+
"""The dimensions of the new run: the old run's for every table that did not change, the union of old and new
|
|
138
|
+
for the target tables (so nothing a person already relied on is lost), and the cross-table checks of both, new
|
|
139
|
+
ones only when they involve a target. Tables no longer in `tables` are dropped."""
|
|
140
|
+
def items(dims: dict, table: str) -> list:
|
|
141
|
+
return next((list(t.get("dimensions") or []) for t in dims.get("tables", []) if t.get("table") == table), [])
|
|
142
|
+
|
|
143
|
+
out_tables = []
|
|
144
|
+
for table in [t.get("table") for t in old.get("tables", [])] + [t.get("table") for t in new.get("tables", [])]:
|
|
145
|
+
if table not in tables or any(e["table"] == table for e in out_tables):
|
|
146
|
+
continue
|
|
147
|
+
merged, seen = [], set()
|
|
148
|
+
for item in items(old, table) + (items(new, table) if table in targets else []):
|
|
149
|
+
key = (str(item.get("dimension", "")).lower(), tuple(item.get("columns") or []))
|
|
150
|
+
if key not in seen:
|
|
151
|
+
seen.add(key)
|
|
152
|
+
merged.append(item)
|
|
153
|
+
if merged:
|
|
154
|
+
out_tables.append({"table": table, "dimensions": merged})
|
|
155
|
+
cross, seen = [], set()
|
|
156
|
+
candidates = list(old.get("cross_table_checks") or []) + [
|
|
157
|
+
c for c in new.get("cross_table_checks") or [] if set(c.get("tables_involved") or []) & targets]
|
|
158
|
+
for check in candidates:
|
|
159
|
+
key = _without_rationale(check)
|
|
160
|
+
if set(check.get("tables_involved") or []) <= tables and key not in seen:
|
|
161
|
+
seen.add(key)
|
|
162
|
+
cross.append(check)
|
|
163
|
+
return {"tables": out_tables, "cross_table_checks": cross}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def merge_tests(kept: list[dict], proposed: list[dict]) -> list[dict]:
|
|
167
|
+
"""The kept tests first, then each proposal that checks something new; a proposal whose id is taken gets the
|
|
168
|
+
first free _2, _3... suffix."""
|
|
169
|
+
out = list(kept)
|
|
170
|
+
checks = {pipeline.signature(s) for s in kept}
|
|
171
|
+
ids = {s["id"] for s in kept}
|
|
172
|
+
for spec in proposed:
|
|
173
|
+
if pipeline.signature(spec) in checks:
|
|
174
|
+
continue
|
|
175
|
+
spec_id, n = spec["id"], 2
|
|
176
|
+
while spec_id in ids:
|
|
177
|
+
spec_id, n = f"{spec['id']}_{n}", n + 1
|
|
178
|
+
out.append({**spec, "id": spec_id})
|
|
179
|
+
checks.add(pipeline.signature(spec))
|
|
180
|
+
ids.add(spec_id)
|
|
181
|
+
return out
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _with_free_ids(kept: list[dict], proposed: list[dict], same) -> list[dict]:
|
|
185
|
+
"""`kept`, then the proposals `same` does not match with a kept item, ids made unique with _2, _3..."""
|
|
186
|
+
out, ids = list(kept), {i["id"] for i in kept}
|
|
187
|
+
for item in proposed:
|
|
188
|
+
if any(same(item, k) for k in kept):
|
|
189
|
+
continue
|
|
190
|
+
item_id, n = item["id"], 2
|
|
191
|
+
while item_id in ids:
|
|
192
|
+
item_id, n = f"{item['id']}_{n}", n + 1
|
|
193
|
+
out.append({**item, "id": item_id})
|
|
194
|
+
ids.add(item_id)
|
|
195
|
+
return out
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _same_threshold(a: dict, b: dict) -> bool:
|
|
199
|
+
return (a.get("asset"), a.get("check_id"), a.get("metric")) == (b.get("asset"), b.get("check_id"), b.get("metric"))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _set_aside(rejected: list, asset_of: dict) -> list:
|
|
203
|
+
"""The validator's rejections as a person reads them: which test, on which table, and why."""
|
|
204
|
+
out = []
|
|
205
|
+
for r in rejected:
|
|
206
|
+
detail = r.get("detail")
|
|
207
|
+
reason = r.get("reason") or "rejected by the validator"
|
|
208
|
+
if detail and detail != asset_of.get(r.get("scope")):
|
|
209
|
+
reason = f"{reason}: {detail if isinstance(detail, str) else json.dumps(detail, default=str)}"
|
|
210
|
+
out.append({"id": r.get("scope"), "asset": asset_of.get(r.get("scope")), "reason": reason})
|
|
211
|
+
return out
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def carry_over(old_run: Run, contexts: dict) -> tuple[list, list, list]:
|
|
215
|
+
"""The old run's confirmed tests and metric thresholds, checked again against the tables as they are now:
|
|
216
|
+
(kept tests, kept thresholds, set aside). Only the structure can set one aside (a column or table gone): the
|
|
217
|
+
validator measures them against the values the review saw, and a kept item comes back exactly as reviewed
|
|
218
|
+
(threshold, edits, warnings), starting confirmed and saying which run it comes from. A test that is failing
|
|
219
|
+
on a real defect today must not lose its limit because the structure changed."""
|
|
220
|
+
review = old_run.load("review") or {}
|
|
221
|
+
summary = (old_run.load("tests") or {}).get("profiling_summary") or {} # what the review was made against
|
|
222
|
+
specs = [dict(s, status="confirmed") for s in review.get("test_specifications") or []
|
|
223
|
+
if s.get("status") != "excluded"]
|
|
224
|
+
thresholds = [dict(t, status="confirmed") for t in review.get("metric_thresholds") or []
|
|
225
|
+
if t.get("status") != "excluded"]
|
|
226
|
+
spec_report = validate_reviewed_test_specifications(copy.deepcopy(specs), contexts, pipeline._ASSERTION_TYPES,
|
|
227
|
+
profiling_summary=summary)
|
|
228
|
+
threshold_report = validate_reviewed_metric_thresholds(copy.deepcopy(thresholds), contexts, summary)
|
|
229
|
+
asset_of = {i.get("id"): i.get("asset") for i in specs + thresholds}
|
|
230
|
+
reviewed = {i.get("id"): i for i in specs + thresholds}
|
|
231
|
+
|
|
232
|
+
def mark(items):
|
|
233
|
+
return [{**reviewed[i["id"]], "status": "confirmed", "_kept_from": old_run.id} for i in items]
|
|
234
|
+
return (mark(spec_report.accepted), mark(threshold_report.accepted),
|
|
235
|
+
_set_aside(spec_report.rejected + threshold_report.rejected, asset_of))
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _neighbours(contexts: dict, goal: set) -> set:
|
|
239
|
+
"""Tables one lineage step from a target, either way: the other side of the checks across two tables."""
|
|
240
|
+
linked = set()
|
|
241
|
+
for table, ctx in contexts.items():
|
|
242
|
+
lineage = ctx.get("lineage") or {}
|
|
243
|
+
near = set()
|
|
244
|
+
for key in ("upstream_assets", "downstream_assets"):
|
|
245
|
+
if isinstance(lineage.get(key), list):
|
|
246
|
+
near |= set(lineage[key])
|
|
247
|
+
if table in goal:
|
|
248
|
+
linked |= near
|
|
249
|
+
elif near & goal:
|
|
250
|
+
linked.add(table)
|
|
251
|
+
return (linked & set(contexts)) - goal
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _has_sql(ctx: dict) -> bool:
|
|
255
|
+
return (ctx.get("transformation") or {}).get("raw_sql") not in (None, "", "unknown")
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _source_of(guarantee_id: str, tables) -> str | None:
|
|
259
|
+
"""The table whose transformation a SQL guarantee was read from: its id is "<that table, as a name>_<n>"."""
|
|
260
|
+
return next((t for t in tables if re.fullmatch(rf"{re.escape(re.sub(r'[^0-9A-Za-z]+', '_', t))}_\d+",
|
|
261
|
+
guarantee_id)), None)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def rule_contexts(contexts: dict, old_contexts: dict, reread: set) -> dict:
|
|
265
|
+
"""The contexts the SQL rules stage leaves for a change's run, before re-reading `reread`: every table as it is
|
|
266
|
+
now, plus the old guarantees whose transformation still exists, still has SQL and is not read again (an input
|
|
267
|
+
requirement stays on its input table even when that table is the one that changed)."""
|
|
268
|
+
out = {t: copy.deepcopy(ctx) for t, ctx in contexts.items()}
|
|
269
|
+
for table, ctx in old_contexts.items():
|
|
270
|
+
if table not in out:
|
|
271
|
+
continue
|
|
272
|
+
for guarantee_id, guarantee in (ctx.get("sql_invariants") or {}).items():
|
|
273
|
+
source = _source_of(guarantee_id, old_contexts)
|
|
274
|
+
if source in contexts and source not in reread and _has_sql(contexts[source]):
|
|
275
|
+
out[table].setdefault("sql_invariants", {})[guarantee_id] = copy.deepcopy(guarantee)
|
|
276
|
+
return out
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def propose(old_run: Run, wh, llm, detected: dict, progress=None) -> Run:
|
|
280
|
+
"""A new run (stages 1-5) for a structural change: the model only works on the changed and new tables, every
|
|
281
|
+
other table's dimensions and SQL rules come from `old_run`, profiling is measured again in full, and every
|
|
282
|
+
confirmed test of `old_run` is carried (or set aside, with the reason). Nothing runs: it waits for review.
|
|
283
|
+
If anything fails, the new run's folder is removed and the error raised."""
|
|
284
|
+
new = Run.create(old_run.path.parent, old_run.info()["settings"])
|
|
285
|
+
try:
|
|
286
|
+
_build(old_run, new, wh, llm, detected, progress or (lambda message: None))
|
|
287
|
+
except BaseException:
|
|
288
|
+
shutil.rmtree(new.path, ignore_errors=True)
|
|
289
|
+
raise
|
|
290
|
+
return new
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _build(old_run: Run, new: Run, wh, llm, detected: dict, progress) -> None:
|
|
294
|
+
settings, changes = old_run.info()["settings"], detected["changes"]
|
|
295
|
+
contexts = detected["context"]["asset_contexts"]
|
|
296
|
+
goal = targets(changes)
|
|
297
|
+
new.save("context", detected["context"])
|
|
298
|
+
|
|
299
|
+
# 2 dimensions: one call on the targets (and their neighbours, for checks across two tables), merged with the
|
|
300
|
+
# old run's, then checked against the tables as they are now (a dropped column leaves its dimensions)
|
|
301
|
+
t0, calls, prompt, raw, rejected = time.monotonic(), [], None, None, []
|
|
302
|
+
found = {"tables": [], "cross_table_checks": []}
|
|
303
|
+
if goal:
|
|
304
|
+
scoped = {t: contexts[t] for t in sorted(set(goal) | _neighbours(contexts, set(goal)))}
|
|
305
|
+
counted, calls = pipeline._counted(llm)
|
|
306
|
+
progress(f"Dimension Selector: {settings['model']} is reading the {len(goal)} changed table(s)")
|
|
307
|
+
dq_json, raw, prompt = select_dimensions(scoped, settings["context_level"], counted,
|
|
308
|
+
pipeline._engine(settings))
|
|
309
|
+
report = validate_dimension_output(dq_json, scoped, DQ_DIMENSIONS)
|
|
310
|
+
found, rejected = report.accepted, report.rejected
|
|
311
|
+
old_dims = (old_run.load("dimensions") or {}).get("dimensions") or {"tables": [], "cross_table_checks": []}
|
|
312
|
+
merged = validate_dimension_output(merge_dimensions(old_dims, found, set(goal), set(contexts)), contexts,
|
|
313
|
+
DQ_DIMENSIONS)
|
|
314
|
+
new.save("dimensions", {"prompt": prompt, "raw_response": raw, "dimensions": merged.accepted,
|
|
315
|
+
"rejected": rejected + merged.rejected, "llm_calls": len(calls),
|
|
316
|
+
"seconds": round(time.monotonic() - t0, 2), "scoped_to": goal})
|
|
317
|
+
|
|
318
|
+
# 3 profiling, in full: the carried tests cite these measurements and their thresholds measure these checks
|
|
319
|
+
pipeline.stage_profiling(new, wh, progress)
|
|
320
|
+
|
|
321
|
+
# 4 SQL rules: re-read only the targets' transformations; the other tables keep the old run's rules
|
|
322
|
+
t0 = time.monotonic()
|
|
323
|
+
reason = pipeline.sql_rules_skip_reason(settings, contexts)
|
|
324
|
+
if reason:
|
|
325
|
+
new.save("sql_rules", {"skipped": reason, "seconds": 0.0})
|
|
326
|
+
else:
|
|
327
|
+
old_contexts = (old_run.load("sql_rules") or {}).get("asset_contexts") or {}
|
|
328
|
+
reread = {t for t in goal if _has_sql(contexts[t])}
|
|
329
|
+
rule_ctx = rule_contexts(contexts, old_contexts, reread)
|
|
330
|
+
record, calls = {"calls": [], "accepted": [], "rejected": []}, []
|
|
331
|
+
if reread:
|
|
332
|
+
counted, calls = pipeline._counted(llm)
|
|
333
|
+
record = run_invariant_extractor(rule_ctx, counted, progress, only=reread)
|
|
334
|
+
dims = new.load("profiling")["dimensions"]
|
|
335
|
+
new.save("sql_rules", {"asset_contexts": rule_ctx,
|
|
336
|
+
"checklist_dimensions": with_invariant_items(dims, rule_ctx),
|
|
337
|
+
"calls": record["calls"], "accepted": record["accepted"],
|
|
338
|
+
"rejected": record["rejected"], "llm_calls": len(calls),
|
|
339
|
+
"seconds": round(time.monotonic() - t0, 2)})
|
|
340
|
+
|
|
341
|
+
# 5 tests: proposals for the targets only, then every confirmed test of the old run, checked again
|
|
342
|
+
t0 = time.monotonic()
|
|
343
|
+
rules, profiling = new.load("sql_rules"), new.load("profiling")
|
|
344
|
+
test_contexts = rules.get("asset_contexts") or contexts
|
|
345
|
+
checklist = rules.get("checklist_dimensions") or profiling["dimensions"]
|
|
346
|
+
scoped_checklist = {
|
|
347
|
+
"tables": [t for t in checklist.get("tables", []) if t.get("table") in goal],
|
|
348
|
+
"cross_table_checks": [c for c in checklist.get("cross_table_checks") or []
|
|
349
|
+
if set(c.get("tables_involved") or []) & set(goal)]}
|
|
350
|
+
if scoped_checklist["tables"] or scoped_checklist["cross_table_checks"]:
|
|
351
|
+
proposed = pipeline.propose_tests(test_contexts, scoped_checklist, profiling["profiling_summary"], llm, wh,
|
|
352
|
+
settings, progress)
|
|
353
|
+
else:
|
|
354
|
+
proposed = {"calls": [], "test_specifications": [], "rejected": [], "metric_thresholds": [],
|
|
355
|
+
"metric_thresholds_rejected": [],
|
|
356
|
+
"profiling_summary": copy.deepcopy(profiling["profiling_summary"]), "llm_calls": 0}
|
|
357
|
+
kept_specs, kept_thresholds, aside = carry_over(old_run, test_contexts)
|
|
358
|
+
specs = merge_tests(kept_specs, proposed["test_specifications"])
|
|
359
|
+
thresholds = _with_free_ids(kept_thresholds, proposed["metric_thresholds"], _same_threshold)
|
|
360
|
+
new.save("tests", {**proposed, "test_specifications": specs, "metric_thresholds": thresholds,
|
|
361
|
+
"kept_from": old_run.id, "set_aside": aside, "seconds": round(time.monotonic() - t0, 2)})
|
|
362
|
+
new.update_info(origin={
|
|
363
|
+
"kind": "change", "from_run": old_run.id,
|
|
364
|
+
"detected_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
365
|
+
"changes": changes, "summary": describe_changes(changes),
|
|
366
|
+
"kept": len(kept_specs) + len(kept_thresholds),
|
|
367
|
+
"proposed": (len(specs) - len(kept_specs)) + (len(thresholds) - len(kept_thresholds)),
|
|
368
|
+
"set_aside": aside})
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
SCHEDULE_MODEL_ATTEMPTS = 8 # a busy model gets about half a minute; then the proposal fails and is tried next time
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def read_state(run: Run) -> dict:
|
|
375
|
+
"""The run's changes.json: its pending run, what it was proposed for, a failed attempt, who superseded it."""
|
|
376
|
+
path = run.path / "changes.json"
|
|
377
|
+
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def write_state(run: Run, **fields) -> None:
|
|
381
|
+
"""Sets (or, with None, removes) keys of the run's changes.json."""
|
|
382
|
+
state = {k: v for k, v in {**read_state(run), **fields}.items() if v is not None}
|
|
383
|
+
(run.path / "changes.json").write_text(json.dumps(state, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _alert(values: dict, redact, subject: str, lines: list[str]) -> list[str]:
|
|
387
|
+
"""Sends one alert (subject first, then the lines) to the configured channels; the channels it reached."""
|
|
388
|
+
if not alerts.configured(values):
|
|
389
|
+
return []
|
|
390
|
+
sent = alerts.deliver(values, subject, redact("\n".join([subject, ""] + lines)), redact)
|
|
391
|
+
return sent.get("alerted", [])
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def after_execution(run: Run, wh, make_llm, alert_values: dict, redact, progress=None) -> dict:
|
|
395
|
+
"""What a scheduled execution does once its tests ran: look for structural changes and, at most once per
|
|
396
|
+
change, propose tests for them in a new run that waits for review, and alert. `make_llm()` gives the model
|
|
397
|
+
(it may raise: no key, no server). Returns {"status", "pending_run", "lines", "alerted", "error"}, status
|
|
398
|
+
one of unchanged, pending_exists, proposed, replaced, detected_only, failed, busy. Never runs a test."""
|
|
399
|
+
outcome = {"status": "unchanged", "pending_run": None, "lines": [], "alerted": [], "error": None}
|
|
400
|
+
try:
|
|
401
|
+
with run.proposal_lock():
|
|
402
|
+
return _check(run, wh, make_llm, alert_values, redact, progress, outcome)
|
|
403
|
+
except RunBusyError: # the in-app schedule and a `dqvouch rerun` noticed the same change: one proposes
|
|
404
|
+
return {**outcome, "status": "busy"}
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _check(run: Run, wh, make_llm, alert_values: dict, redact, progress, outcome: dict) -> dict:
|
|
408
|
+
"""after_execution, with the proposal lock held."""
|
|
409
|
+
detected = detect(run, wh)
|
|
410
|
+
state = read_state(run)
|
|
411
|
+
pending = state.get("pending_run")
|
|
412
|
+
if not has_changes(detected["changes"]):
|
|
413
|
+
if pending: # the structure went back to what was reviewed: the pending run has nothing left to do
|
|
414
|
+
_replace(run, pending, None)
|
|
415
|
+
if pending or state.get("failed") or state.get("detected"):
|
|
416
|
+
write_state(run, pending_run=None, signatures=None, failed=None, detected=None)
|
|
417
|
+
return outcome
|
|
418
|
+
lines = describe_changes(detected["changes"])
|
|
419
|
+
sigs = table_signatures(detected["context"]["asset_contexts"])
|
|
420
|
+
outcome.update(lines=lines, pending_run=pending)
|
|
421
|
+
if pending and state.get("signatures") == sigs:
|
|
422
|
+
return {**outcome, "status": "pending_exists"}
|
|
423
|
+
n = len(lines)
|
|
424
|
+
if not run.info()["settings"].get("propose_on_change", True):
|
|
425
|
+
outcome["status"] = "detected_only"
|
|
426
|
+
if (state.get("detected") or {}).get("signatures") != sigs:
|
|
427
|
+
outcome["alerted"] = _alert(alert_values, redact, f"dqvouch: structure changed in {n} table(s)",
|
|
428
|
+
lines + ["", "Proposing is off (propose_on_change = false): start a new "
|
|
429
|
+
"run to cover it."])
|
|
430
|
+
write_state(run, detected={"signatures": sigs, "lines": lines})
|
|
431
|
+
return outcome
|
|
432
|
+
try:
|
|
433
|
+
llm = make_llm() if targets(detected["changes"]) else None
|
|
434
|
+
if hasattr(llm, "max_attempts"): # no one waits on a schedule: an outage must end in "failed", not hang
|
|
435
|
+
llm.max_attempts = SCHEDULE_MODEL_ATTEMPTS
|
|
436
|
+
new = propose(run, wh, llm, detected, progress)
|
|
437
|
+
except Exception as e:
|
|
438
|
+
error = redact(str(e) or type(e).__name__)
|
|
439
|
+
outcome.update(status="failed", error=error)
|
|
440
|
+
if (state.get("failed") or {}).get("signatures") != sigs:
|
|
441
|
+
outcome["alerted"] = _alert(
|
|
442
|
+
alert_values, redact, f"dqvouch: structure changed in {n} table(s), but proposing new tests failed",
|
|
443
|
+
lines + ["", f"{error}. dqvouch will try again at the next execution."])
|
|
444
|
+
write_state(run, failed={"signatures": sigs, "error": error, "lines": lines})
|
|
445
|
+
return outcome
|
|
446
|
+
if pending:
|
|
447
|
+
_replace(run, pending, new.id)
|
|
448
|
+
origin = new.info()["origin"]
|
|
449
|
+
write_state(run, pending_run=new.id, signatures=sigs, failed=None, detected=None,
|
|
450
|
+
detected_at=origin["detected_at"], proposed=origin["proposed"])
|
|
451
|
+
aside = origin["set_aside"]
|
|
452
|
+
body = lines + ["", f"{origin['kept']} confirmed test(s) kept, {len(aside)} set aside, {origin['proposed']} new. "
|
|
453
|
+
f"Review them in run {new.id} (dqvouch ui)."]
|
|
454
|
+
body += [f"- {a['id']}: {a['reason']}" for a in aside]
|
|
455
|
+
outcome.update(status="replaced" if pending else "proposed", pending_run=new.id,
|
|
456
|
+
alerted=_alert(alert_values, redact, f"dqvouch: structure changed in {n} table(s), "
|
|
457
|
+
f"{origin['proposed']} new test(s) to review", body))
|
|
458
|
+
return outcome
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _replace(run: Run, pending: str, by: str | None) -> None:
|
|
462
|
+
"""Marks an earlier pending run as replaced (kept on disk, shown as replaced), if it is still there."""
|
|
463
|
+
path = run.path.parent / pending
|
|
464
|
+
if (path / "run.json").is_file():
|
|
465
|
+
Run.open(path).update_info(replaced_by=by or "the reviewed structure")
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def hand_over(new_run: Run) -> bool:
|
|
469
|
+
"""Once a change's run is reviewed, the daily executions move to it: the old run's schedule moves over, its
|
|
470
|
+
history comes first in the new run's (so the trends of the kept tests, same ids, continue), and the old run
|
|
471
|
+
is marked superseded (which `current` follows, so cron lines naming the old run keep working). Only for a
|
|
472
|
+
reviewed run made by a change that is still the one waiting (not replaced by a later change). True when the
|
|
473
|
+
executions moved (or had already moved) to `new_run`."""
|
|
474
|
+
info = new_run.info()
|
|
475
|
+
origin = info.get("origin") or {}
|
|
476
|
+
old_path = new_run.path.parent / str(origin.get("from_run"))
|
|
477
|
+
if origin.get("kind") != "change" or "review" not in new_run.completed() or not (old_path / "run.json").is_file():
|
|
478
|
+
return False
|
|
479
|
+
old = Run.open(old_path)
|
|
480
|
+
state = read_state(old)
|
|
481
|
+
if state.get("superseded_by") == new_run.id:
|
|
482
|
+
return True
|
|
483
|
+
if info.get("replaced_by") or (state.get("pending_run") and state["pending_run"] != new_run.id):
|
|
484
|
+
return False # a later change replaced it: the run that replaced it is the one to review
|
|
485
|
+
schedule = old.path / "schedule.json"
|
|
486
|
+
if schedule.exists() and not (new_run.path / "schedule.json").exists():
|
|
487
|
+
schedule.replace(new_run.path / "schedule.json")
|
|
488
|
+
history = old.path / "history.jsonl"
|
|
489
|
+
if history.exists():
|
|
490
|
+
mine = new_run.path / "history.jsonl"
|
|
491
|
+
own = mine.read_text(encoding="utf-8") if mine.exists() else ""
|
|
492
|
+
mine.write_text(history.read_text(encoding="utf-8") + own, encoding="utf-8")
|
|
493
|
+
write_state(old, superseded_by=new_run.id, pending_run=None, signatures=None)
|
|
494
|
+
return True
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def current(run: Run) -> Run:
|
|
498
|
+
"""The run whose tests are executed in place of `run`: follows superseded_by while the next run exists and
|
|
499
|
+
was reviewed (a run still waiting for review is never executed)."""
|
|
500
|
+
seen = {run.id}
|
|
501
|
+
while True:
|
|
502
|
+
nxt = read_state(run).get("superseded_by")
|
|
503
|
+
path = run.path.parent / str(nxt)
|
|
504
|
+
if not nxt or nxt in seen or not (path / "run.json").is_file() or "review" not in Run.open(path).completed():
|
|
505
|
+
return run
|
|
506
|
+
run = Run.open(path)
|
|
507
|
+
seen.add(run.id)
|