dqvouch 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dqvouch/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """dqvouch: an LLM proposes data quality tests, code validates them, you vouch for them."""
dqvouch/alerts.py ADDED
@@ -0,0 +1,129 @@
1
+ """Alerts when an execution of a run's tests gets worse: a test starts failing or erroring, or the execution itself
2
+ cannot run after one that did. Sent by webhook ({"text": ...}: Slack, Mattermost, Google Chat, Rocket.Chat, Teams
3
+ workflows; Discord with /slack appended) and/or by email (SMTP), as configured once in the .env file. A known problem
4
+ is said once, not at every execution, and a failed send never fails the execution: it is recorded instead."""
5
+ import json
6
+ import smtplib
7
+ import ssl
8
+ import urllib.request
9
+ from email.message import EmailMessage
10
+
11
+ from dqvouch.config import ENGINE_LABELS
12
+
13
+ WEBHOOK = "DQVOUCH_ALERT_WEBHOOK_URL"
14
+ BAD = ("fail", "error")
15
+ TRIGGERS = {"schedule": "a schedule", "command": "dqvouch rerun", "manual": "a person"}
16
+ MAX_LISTED = 20
17
+
18
+
19
+ def worse(entry: dict, history: list) -> dict | None:
20
+ """What got worse in `entry` against `history` (the executions before it, newest first): {"error": ...} when it
21
+ could not run and the one before could, {"new": [check ids]} for tests failing or erroring now that did not in
22
+ the last execution with verdicts; None otherwise (the first execution has nothing to be compared with)."""
23
+ if "error" in entry:
24
+ return {"error": entry["error"]} if not history or "error" not in history[0] else None
25
+ before = next((h for h in history if h.get("verdicts")), None)
26
+ if before is None or not entry.get("verdicts"):
27
+ return None
28
+ new = [i for i, v in entry["verdicts"].items() if v in BAD and before["verdicts"].get(i) not in BAD]
29
+ return {"new": new} if new else None
30
+
31
+
32
+ def message(run_info: dict, entry: dict, problem: dict, rows: list) -> tuple[str, str]:
33
+ """(subject, text) of the alert."""
34
+ db = run_info["settings"]["database"]
35
+ where = (f"{ENGINE_LABELS.get(db['engine'], db['engine'])} at {db['host']}:{db['port']} "
36
+ f"({', '.join(db['schemas'])})")
37
+ when = f"started by {TRIGGERS.get(entry.get('trigger'), entry.get('trigger'))} at {entry.get('at')}"
38
+ if "error" in problem:
39
+ return (f"dqvouch: run {run_info['id']} could not execute its tests",
40
+ f"The tests of dqvouch run {run_info['id']} on {where} could not run ({when}):\n{problem['error']}")
41
+ new, by_id = problem["new"], {r["check_id"]: r for r in rows}
42
+ counts = ", ".join(f"{n} {v.replace('_', ' ')}" for v, n in sorted((entry.get("counts") or {}).items()))
43
+ lines = [f"{len(new)} test(s) of dqvouch run {run_info['id']} on {where} started failing ({when}).",
44
+ f"This execution: {counts}.", ""]
45
+ for check_id in new[:MAX_LISTED]:
46
+ r = by_id.get(check_id, {})
47
+ title = r.get("title") or str(check_id).split("__")[-1]
48
+ detail = (f"error: {r['error']}" if r.get("error")
49
+ else f"observed {r.get('observed')}, expected {r.get('expected')}")
50
+ lines.append(f"- {title} [{check_id}] on {r.get('asset', '?')}: {r.get('verdict', entry['verdicts'][check_id])}"
51
+ f", {detail}")
52
+ if len(new) > MAX_LISTED:
53
+ lines.append(f"... and {len(new) - MAX_LISTED} more")
54
+ return f"dqvouch: {len(new)} test(s) started failing in run {run_info['id']}", "\n".join(lines)
55
+
56
+
57
+ def send_webhook(url: str, text: str, timeout: float = 10) -> None:
58
+ request = urllib.request.Request(url, data=json.dumps({"text": text}).encode("utf-8"), method="POST",
59
+ headers={"Content-Type": "application/json"})
60
+ with urllib.request.urlopen(request, timeout=timeout):
61
+ pass
62
+
63
+
64
+ def send_email(values: dict, subject: str, text: str) -> None:
65
+ host, port = values["DQVOUCH_SMTP_HOST"], int(values.get("DQVOUCH_SMTP_PORT") or 587)
66
+ user = values.get("DQVOUCH_SMTP_USER") or ""
67
+ msg = EmailMessage()
68
+ msg["Subject"], msg["From"] = subject, values.get("DQVOUCH_SMTP_FROM") or user
69
+ msg["To"] = ", ".join(a.strip() for a in values["DQVOUCH_SMTP_TO"].split(",") if a.strip())
70
+ msg.set_content(text)
71
+ context = ssl.create_default_context()
72
+ if port == 465:
73
+ server = smtplib.SMTP_SSL(host, port, timeout=10, context=context)
74
+ else:
75
+ server = smtplib.SMTP(host, port, timeout=10)
76
+ with server:
77
+ encrypted = port == 465
78
+ if not encrypted:
79
+ server.ehlo()
80
+ if server.has_extn("starttls"):
81
+ server.starttls(context=context)
82
+ server.ehlo()
83
+ encrypted = True
84
+ if user:
85
+ if not encrypted:
86
+ raise RuntimeError(f"{host}:{port} does not offer STARTTLS, and dqvouch does not send the SMTP password "
87
+ f"unencrypted: use port 465 or a server with STARTTLS")
88
+ server.login(user, values.get("DQVOUCH_SMTP_PASSWORD") or "")
89
+ server.send_message(msg)
90
+
91
+
92
+ def _email_configured(values: dict) -> bool:
93
+ return bool(values.get("DQVOUCH_SMTP_HOST") and values.get("DQVOUCH_SMTP_TO"))
94
+
95
+
96
+ def configured(values: dict) -> bool:
97
+ return bool(values.get(WEBHOOK)) or _email_configured(values)
98
+
99
+
100
+ def deliver(values: dict, subject: str, text: str, redact) -> dict:
101
+ """Sends to every configured channel: {"alerted": [channels]} and/or {"alert_error": text}, never raises."""
102
+ sent, errors = [], []
103
+ if values.get(WEBHOOK):
104
+ try:
105
+ send_webhook(values[WEBHOOK], text)
106
+ sent.append("webhook")
107
+ except Exception as e:
108
+ errors.append(f"webhook: {e or type(e).__name__}")
109
+ if _email_configured(values):
110
+ try:
111
+ send_email(values, subject, text)
112
+ sent.append("email")
113
+ except Exception as e:
114
+ errors.append(f"email: {e or type(e).__name__}")
115
+ result = {"alerted": sent} if sent else {}
116
+ if errors:
117
+ result["alert_error"] = redact("; ".join(errors))
118
+ return result
119
+
120
+
121
+ def notifier(values: dict, redact, run_info: dict):
122
+ """The `notify(entry, history, rows)` that record_execution calls: alerts when the execution got worse."""
123
+ def notify(entry: dict, history: list, rows: list) -> dict:
124
+ problem = worse(entry, history) if configured(values) else None
125
+ if not problem:
126
+ return {}
127
+ subject, text = message(run_info, entry, problem, rows)
128
+ return deliver(values, subject, redact(text), redact)
129
+ return notify
dqvouch/changes.py ADDED
@@ -0,0 +1,507 @@
1
+ """Structural changes between a run and the database now, and what dqvouch does about them.
2
+
3
+ A structural change is a change in what a table is: its columns (names, types, descriptions), its description, the
4
+ SQL that builds it and the tables it reads from or feeds. What dqvouch derives from those (dq_signals, semantic
5
+ roles, usage facts) is not part of it, so a new dqvouch release never looks like a change; values (row counts,
6
+ nulls, distributions) never are either: catching a value going wrong is the tests' job. Each table gets its own
7
+ signature, so only the tables that changed go back to the model."""
8
+ import copy
9
+ import datetime
10
+ import hashlib
11
+ import json
12
+ import re
13
+ import shutil
14
+ import time
15
+ from pathlib import Path
16
+
17
+ from dqvouch import alerts, pipeline
18
+ from dqvouch.profiling import DQ_DIMENSIONS
19
+ from dqvouch.rules import run_invariant_extractor, with_invariant_items
20
+ from dqvouch.selector import select_dimensions
21
+ from dqvouch.store import Run, RunBusyError
22
+ from dqvouch.validator import (
23
+ validate_dimension_output, validate_reviewed_metric_thresholds, validate_reviewed_test_specifications,
24
+ )
25
+
26
+
27
+ def _structure(ctx: dict) -> dict:
28
+ """The facts of an AssetContext that come from the database and the SQL files, not from dqvouch's code."""
29
+ lineage = ctx.get("lineage") or {}
30
+
31
+ def names(value):
32
+ return sorted(value) if isinstance(value, list) else value
33
+ return {"description": (ctx.get("asset") or {}).get("description"),
34
+ "columns": sorted([[c.get("name"), c.get("type"), c.get("description")] for c in ctx.get("columns") or []],
35
+ key=lambda c: str(c[0])),
36
+ "sql": (ctx.get("transformation") or {}).get("raw_sql"),
37
+ "upstream": names(lineage.get("upstream_assets")), "downstream": names(lineage.get("downstream_assets"))}
38
+
39
+
40
+ def table_signatures(asset_contexts: dict) -> dict[str, str]:
41
+ """{table: sha256 of its structure}: equal signatures, the table is what it was."""
42
+ return {table: hashlib.sha256(json.dumps(_structure(ctx), sort_keys=True, default=str).encode("utf-8")).hexdigest()
43
+ for table, ctx in asset_contexts.items()}
44
+
45
+
46
+ def _columns(ctx: dict) -> dict:
47
+ return {c["name"]: c for c in ctx.get("columns") or []}
48
+
49
+
50
+ def _detail(old: dict, new: dict) -> dict:
51
+ """What changed on one table, in the terms a person reads."""
52
+ before, after = _columns(old), _columns(new)
53
+ common = [name for name in before if name in after]
54
+ detail = {
55
+ "columns_added": [{"column": name, "type": after[name].get("type")} for name in after if name not in before],
56
+ "columns_dropped": [name for name in before if name not in after],
57
+ "columns_retyped": [{"column": name, "from": before[name].get("type"), "to": after[name].get("type")}
58
+ for name in common if before[name].get("type") != after[name].get("type")],
59
+ "descriptions_changed": ((old.get("asset") or {}).get("description")
60
+ != (new.get("asset") or {}).get("description")
61
+ or any(before[n].get("description") != after[n].get("description") for n in common)),
62
+ "sql_changed": (old.get("transformation") or {}).get("raw_sql")
63
+ != (new.get("transformation") or {}).get("raw_sql"),
64
+ "lineage_changed": ((_structure(old)["upstream"], _structure(old)["downstream"])
65
+ != (_structure(new)["upstream"], _structure(new)["downstream"])),
66
+ }
67
+ return detail
68
+
69
+
70
+ def diff_contexts(old_contexts: dict, old_available: list[str], new_contexts: dict) -> dict:
71
+ """{"changed": {table: detail}, "new": [tables], "dropped": [tables]} between a run's AssetContexts and fresh
72
+ ones. A table the run left out on purpose (in `old_available` but not in its contexts) is neither."""
73
+ before, after = table_signatures(old_contexts), table_signatures(new_contexts)
74
+ return {
75
+ "changed": {t: _detail(old_contexts[t], new_contexts[t])
76
+ for t in sorted(before) if t in after and before[t] != after[t]},
77
+ "new": sorted(t for t in after if t not in set(old_available)),
78
+ "dropped": sorted(t for t in before if t not in after),
79
+ }
80
+
81
+
82
+ def has_changes(changes: dict) -> bool:
83
+ return bool(changes["changed"] or changes["new"] or changes["dropped"])
84
+
85
+
86
+ def targets(changes: dict) -> list[str]:
87
+ """The tables the model proposes tests for again: the changed ones and the new ones."""
88
+ return sorted(set(changes["changed"]) | set(changes["new"]))
89
+
90
+
91
+ def _line(table: str, detail: dict) -> str:
92
+ parts = [f"+ {a['column']} ({a['type']})" for a in detail["columns_added"]]
93
+ parts += [f"- {c}" for c in detail["columns_dropped"]]
94
+ parts += [f"{r['column']}: {r['from']} -> {r['to']}" for r in detail["columns_retyped"]]
95
+ for key, words in (("descriptions_changed", "descriptions changed"), ("sql_changed", "SQL changed"),
96
+ ("lineage_changed", "lineage changed")):
97
+ if detail[key]:
98
+ parts.append(words)
99
+ return f"{table}: {', '.join(parts)}"
100
+
101
+
102
+ def describe_changes(changes: dict) -> list[str]:
103
+ """One line per table, sorted by table: what a person reads in the alert and on the Review screen."""
104
+ lines = {t: _line(t, d) for t, d in changes["changed"].items()}
105
+ lines.update({t: f"{t}: new table" for t in changes["new"]})
106
+ lines.update({t: f"{t}: dropped" for t in changes["dropped"]})
107
+ return [lines[t] for t in sorted(lines)]
108
+
109
+
110
+ def detect(run: Run, wh) -> dict:
111
+ """{"changes": diff, "context": stage 1's result now} for the run's tables still there plus the tables that
112
+ appeared since (not the ones the run left out with --tables). Reads only metadata and the SQL files."""
113
+ settings, old = run.info()["settings"], run.load("context")
114
+ sql_dir = settings.get("sql_dir")
115
+ if settings.get("context_level") == "C3" and sql_dir and not (Path(sql_dir).is_dir()
116
+ and any(Path(sql_dir).rglob("*.sql"))):
117
+ # without it every table's SQL would look gone: a moved folder is not a change of the database
118
+ raise ValueError(f"cannot check the structure: the SQL folder {sql_dir} is missing or has no .sql file")
119
+ now = pipeline.build_context(settings, wh) # one read of the metadata: tables_available and every context
120
+ known = set(old["tables_available"])
121
+ wanted = [t for t in now["tables_available"] if t in old["asset_contexts"] or t not in known]
122
+ if not wanted: # every table of the run is gone and nothing new appeared
123
+ fresh = {**now, "asset_contexts": {}}
124
+ elif set(wanted) == set(now["asset_contexts"]):
125
+ fresh = now
126
+ else: # the run covered some tables only: their contexts must be built the same way (lineage, usage)
127
+ fresh = pipeline.build_context(settings, wh, wanted)
128
+ return {"changes": diff_contexts(old["asset_contexts"], old["tables_available"], fresh["asset_contexts"]),
129
+ "context": fresh}
130
+
131
+
132
+ def _without_rationale(item: dict) -> str:
133
+ return json.dumps({k: v for k, v in item.items() if k != "rationale"}, sort_keys=True, default=str)
134
+
135
+
136
+ def merge_dimensions(old: dict, new: dict, targets: set[str], tables: set[str]) -> dict:
137
+ """The dimensions of the new run: the old run's for every table that did not change, the union of old and new
138
+ for the target tables (so nothing a person already relied on is lost), and the cross-table checks of both, new
139
+ ones only when they involve a target. Tables no longer in `tables` are dropped."""
140
+ def items(dims: dict, table: str) -> list:
141
+ return next((list(t.get("dimensions") or []) for t in dims.get("tables", []) if t.get("table") == table), [])
142
+
143
+ out_tables = []
144
+ for table in [t.get("table") for t in old.get("tables", [])] + [t.get("table") for t in new.get("tables", [])]:
145
+ if table not in tables or any(e["table"] == table for e in out_tables):
146
+ continue
147
+ merged, seen = [], set()
148
+ for item in items(old, table) + (items(new, table) if table in targets else []):
149
+ key = (str(item.get("dimension", "")).lower(), tuple(item.get("columns") or []))
150
+ if key not in seen:
151
+ seen.add(key)
152
+ merged.append(item)
153
+ if merged:
154
+ out_tables.append({"table": table, "dimensions": merged})
155
+ cross, seen = [], set()
156
+ candidates = list(old.get("cross_table_checks") or []) + [
157
+ c for c in new.get("cross_table_checks") or [] if set(c.get("tables_involved") or []) & targets]
158
+ for check in candidates:
159
+ key = _without_rationale(check)
160
+ if set(check.get("tables_involved") or []) <= tables and key not in seen:
161
+ seen.add(key)
162
+ cross.append(check)
163
+ return {"tables": out_tables, "cross_table_checks": cross}
164
+
165
+
166
+ def merge_tests(kept: list[dict], proposed: list[dict]) -> list[dict]:
167
+ """The kept tests first, then each proposal that checks something new; a proposal whose id is taken gets the
168
+ first free _2, _3... suffix."""
169
+ out = list(kept)
170
+ checks = {pipeline.signature(s) for s in kept}
171
+ ids = {s["id"] for s in kept}
172
+ for spec in proposed:
173
+ if pipeline.signature(spec) in checks:
174
+ continue
175
+ spec_id, n = spec["id"], 2
176
+ while spec_id in ids:
177
+ spec_id, n = f"{spec['id']}_{n}", n + 1
178
+ out.append({**spec, "id": spec_id})
179
+ checks.add(pipeline.signature(spec))
180
+ ids.add(spec_id)
181
+ return out
182
+
183
+
184
+ def _with_free_ids(kept: list[dict], proposed: list[dict], same) -> list[dict]:
185
+ """`kept`, then the proposals `same` does not match with a kept item, ids made unique with _2, _3..."""
186
+ out, ids = list(kept), {i["id"] for i in kept}
187
+ for item in proposed:
188
+ if any(same(item, k) for k in kept):
189
+ continue
190
+ item_id, n = item["id"], 2
191
+ while item_id in ids:
192
+ item_id, n = f"{item['id']}_{n}", n + 1
193
+ out.append({**item, "id": item_id})
194
+ ids.add(item_id)
195
+ return out
196
+
197
+
198
+ def _same_threshold(a: dict, b: dict) -> bool:
199
+ return (a.get("asset"), a.get("check_id"), a.get("metric")) == (b.get("asset"), b.get("check_id"), b.get("metric"))
200
+
201
+
202
+ def _set_aside(rejected: list, asset_of: dict) -> list:
203
+ """The validator's rejections as a person reads them: which test, on which table, and why."""
204
+ out = []
205
+ for r in rejected:
206
+ detail = r.get("detail")
207
+ reason = r.get("reason") or "rejected by the validator"
208
+ if detail and detail != asset_of.get(r.get("scope")):
209
+ reason = f"{reason}: {detail if isinstance(detail, str) else json.dumps(detail, default=str)}"
210
+ out.append({"id": r.get("scope"), "asset": asset_of.get(r.get("scope")), "reason": reason})
211
+ return out
212
+
213
+
214
+ def carry_over(old_run: Run, contexts: dict) -> tuple[list, list, list]:
215
+ """The old run's confirmed tests and metric thresholds, checked again against the tables as they are now:
216
+ (kept tests, kept thresholds, set aside). Only the structure can set one aside (a column or table gone): the
217
+ validator measures them against the values the review saw, and a kept item comes back exactly as reviewed
218
+ (threshold, edits, warnings), starting confirmed and saying which run it comes from. A test that is failing
219
+ on a real defect today must not lose its limit because the structure changed."""
220
+ review = old_run.load("review") or {}
221
+ summary = (old_run.load("tests") or {}).get("profiling_summary") or {} # what the review was made against
222
+ specs = [dict(s, status="confirmed") for s in review.get("test_specifications") or []
223
+ if s.get("status") != "excluded"]
224
+ thresholds = [dict(t, status="confirmed") for t in review.get("metric_thresholds") or []
225
+ if t.get("status") != "excluded"]
226
+ spec_report = validate_reviewed_test_specifications(copy.deepcopy(specs), contexts, pipeline._ASSERTION_TYPES,
227
+ profiling_summary=summary)
228
+ threshold_report = validate_reviewed_metric_thresholds(copy.deepcopy(thresholds), contexts, summary)
229
+ asset_of = {i.get("id"): i.get("asset") for i in specs + thresholds}
230
+ reviewed = {i.get("id"): i for i in specs + thresholds}
231
+
232
+ def mark(items):
233
+ return [{**reviewed[i["id"]], "status": "confirmed", "_kept_from": old_run.id} for i in items]
234
+ return (mark(spec_report.accepted), mark(threshold_report.accepted),
235
+ _set_aside(spec_report.rejected + threshold_report.rejected, asset_of))
236
+
237
+
238
+ def _neighbours(contexts: dict, goal: set) -> set:
239
+ """Tables one lineage step from a target, either way: the other side of the checks across two tables."""
240
+ linked = set()
241
+ for table, ctx in contexts.items():
242
+ lineage = ctx.get("lineage") or {}
243
+ near = set()
244
+ for key in ("upstream_assets", "downstream_assets"):
245
+ if isinstance(lineage.get(key), list):
246
+ near |= set(lineage[key])
247
+ if table in goal:
248
+ linked |= near
249
+ elif near & goal:
250
+ linked.add(table)
251
+ return (linked & set(contexts)) - goal
252
+
253
+
254
+ def _has_sql(ctx: dict) -> bool:
255
+ return (ctx.get("transformation") or {}).get("raw_sql") not in (None, "", "unknown")
256
+
257
+
258
+ def _source_of(guarantee_id: str, tables) -> str | None:
259
+ """The table whose transformation a SQL guarantee was read from: its id is "<that table, as a name>_<n>"."""
260
+ return next((t for t in tables if re.fullmatch(rf"{re.escape(re.sub(r'[^0-9A-Za-z]+', '_', t))}_\d+",
261
+ guarantee_id)), None)
262
+
263
+
264
+ def rule_contexts(contexts: dict, old_contexts: dict, reread: set) -> dict:
265
+ """The contexts the SQL rules stage leaves for a change's run, before re-reading `reread`: every table as it is
266
+ now, plus the old guarantees whose transformation still exists, still has SQL and is not read again (an input
267
+ requirement stays on its input table even when that table is the one that changed)."""
268
+ out = {t: copy.deepcopy(ctx) for t, ctx in contexts.items()}
269
+ for table, ctx in old_contexts.items():
270
+ if table not in out:
271
+ continue
272
+ for guarantee_id, guarantee in (ctx.get("sql_invariants") or {}).items():
273
+ source = _source_of(guarantee_id, old_contexts)
274
+ if source in contexts and source not in reread and _has_sql(contexts[source]):
275
+ out[table].setdefault("sql_invariants", {})[guarantee_id] = copy.deepcopy(guarantee)
276
+ return out
277
+
278
+
279
+ def propose(old_run: Run, wh, llm, detected: dict, progress=None) -> Run:
280
+ """A new run (stages 1-5) for a structural change: the model only works on the changed and new tables, every
281
+ other table's dimensions and SQL rules come from `old_run`, profiling is measured again in full, and every
282
+ confirmed test of `old_run` is carried (or set aside, with the reason). Nothing runs: it waits for review.
283
+ If anything fails, the new run's folder is removed and the error raised."""
284
+ new = Run.create(old_run.path.parent, old_run.info()["settings"])
285
+ try:
286
+ _build(old_run, new, wh, llm, detected, progress or (lambda message: None))
287
+ except BaseException:
288
+ shutil.rmtree(new.path, ignore_errors=True)
289
+ raise
290
+ return new
291
+
292
+
293
+ def _build(old_run: Run, new: Run, wh, llm, detected: dict, progress) -> None:
294
+ settings, changes = old_run.info()["settings"], detected["changes"]
295
+ contexts = detected["context"]["asset_contexts"]
296
+ goal = targets(changes)
297
+ new.save("context", detected["context"])
298
+
299
+ # 2 dimensions: one call on the targets (and their neighbours, for checks across two tables), merged with the
300
+ # old run's, then checked against the tables as they are now (a dropped column leaves its dimensions)
301
+ t0, calls, prompt, raw, rejected = time.monotonic(), [], None, None, []
302
+ found = {"tables": [], "cross_table_checks": []}
303
+ if goal:
304
+ scoped = {t: contexts[t] for t in sorted(set(goal) | _neighbours(contexts, set(goal)))}
305
+ counted, calls = pipeline._counted(llm)
306
+ progress(f"Dimension Selector: {settings['model']} is reading the {len(goal)} changed table(s)")
307
+ dq_json, raw, prompt = select_dimensions(scoped, settings["context_level"], counted,
308
+ pipeline._engine(settings))
309
+ report = validate_dimension_output(dq_json, scoped, DQ_DIMENSIONS)
310
+ found, rejected = report.accepted, report.rejected
311
+ old_dims = (old_run.load("dimensions") or {}).get("dimensions") or {"tables": [], "cross_table_checks": []}
312
+ merged = validate_dimension_output(merge_dimensions(old_dims, found, set(goal), set(contexts)), contexts,
313
+ DQ_DIMENSIONS)
314
+ new.save("dimensions", {"prompt": prompt, "raw_response": raw, "dimensions": merged.accepted,
315
+ "rejected": rejected + merged.rejected, "llm_calls": len(calls),
316
+ "seconds": round(time.monotonic() - t0, 2), "scoped_to": goal})
317
+
318
+ # 3 profiling, in full: the carried tests cite these measurements and their thresholds measure these checks
319
+ pipeline.stage_profiling(new, wh, progress)
320
+
321
+ # 4 SQL rules: re-read only the targets' transformations; the other tables keep the old run's rules
322
+ t0 = time.monotonic()
323
+ reason = pipeline.sql_rules_skip_reason(settings, contexts)
324
+ if reason:
325
+ new.save("sql_rules", {"skipped": reason, "seconds": 0.0})
326
+ else:
327
+ old_contexts = (old_run.load("sql_rules") or {}).get("asset_contexts") or {}
328
+ reread = {t for t in goal if _has_sql(contexts[t])}
329
+ rule_ctx = rule_contexts(contexts, old_contexts, reread)
330
+ record, calls = {"calls": [], "accepted": [], "rejected": []}, []
331
+ if reread:
332
+ counted, calls = pipeline._counted(llm)
333
+ record = run_invariant_extractor(rule_ctx, counted, progress, only=reread)
334
+ dims = new.load("profiling")["dimensions"]
335
+ new.save("sql_rules", {"asset_contexts": rule_ctx,
336
+ "checklist_dimensions": with_invariant_items(dims, rule_ctx),
337
+ "calls": record["calls"], "accepted": record["accepted"],
338
+ "rejected": record["rejected"], "llm_calls": len(calls),
339
+ "seconds": round(time.monotonic() - t0, 2)})
340
+
341
+ # 5 tests: proposals for the targets only, then every confirmed test of the old run, checked again
342
+ t0 = time.monotonic()
343
+ rules, profiling = new.load("sql_rules"), new.load("profiling")
344
+ test_contexts = rules.get("asset_contexts") or contexts
345
+ checklist = rules.get("checklist_dimensions") or profiling["dimensions"]
346
+ scoped_checklist = {
347
+ "tables": [t for t in checklist.get("tables", []) if t.get("table") in goal],
348
+ "cross_table_checks": [c for c in checklist.get("cross_table_checks") or []
349
+ if set(c.get("tables_involved") or []) & set(goal)]}
350
+ if scoped_checklist["tables"] or scoped_checklist["cross_table_checks"]:
351
+ proposed = pipeline.propose_tests(test_contexts, scoped_checklist, profiling["profiling_summary"], llm, wh,
352
+ settings, progress)
353
+ else:
354
+ proposed = {"calls": [], "test_specifications": [], "rejected": [], "metric_thresholds": [],
355
+ "metric_thresholds_rejected": [],
356
+ "profiling_summary": copy.deepcopy(profiling["profiling_summary"]), "llm_calls": 0}
357
+ kept_specs, kept_thresholds, aside = carry_over(old_run, test_contexts)
358
+ specs = merge_tests(kept_specs, proposed["test_specifications"])
359
+ thresholds = _with_free_ids(kept_thresholds, proposed["metric_thresholds"], _same_threshold)
360
+ new.save("tests", {**proposed, "test_specifications": specs, "metric_thresholds": thresholds,
361
+ "kept_from": old_run.id, "set_aside": aside, "seconds": round(time.monotonic() - t0, 2)})
362
+ new.update_info(origin={
363
+ "kind": "change", "from_run": old_run.id,
364
+ "detected_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
365
+ "changes": changes, "summary": describe_changes(changes),
366
+ "kept": len(kept_specs) + len(kept_thresholds),
367
+ "proposed": (len(specs) - len(kept_specs)) + (len(thresholds) - len(kept_thresholds)),
368
+ "set_aside": aside})
369
+
370
+
371
+ SCHEDULE_MODEL_ATTEMPTS = 8 # a busy model gets about half a minute; then the proposal fails and is tried next time
372
+
373
+
374
+ def read_state(run: Run) -> dict:
375
+ """The run's changes.json: its pending run, what it was proposed for, a failed attempt, who superseded it."""
376
+ path = run.path / "changes.json"
377
+ return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
378
+
379
+
380
+ def write_state(run: Run, **fields) -> None:
381
+ """Sets (or, with None, removes) keys of the run's changes.json."""
382
+ state = {k: v for k, v in {**read_state(run), **fields}.items() if v is not None}
383
+ (run.path / "changes.json").write_text(json.dumps(state, indent=2, ensure_ascii=False), encoding="utf-8")
384
+
385
+
386
+ def _alert(values: dict, redact, subject: str, lines: list[str]) -> list[str]:
387
+ """Sends one alert (subject first, then the lines) to the configured channels; the channels it reached."""
388
+ if not alerts.configured(values):
389
+ return []
390
+ sent = alerts.deliver(values, subject, redact("\n".join([subject, ""] + lines)), redact)
391
+ return sent.get("alerted", [])
392
+
393
+
394
+ def after_execution(run: Run, wh, make_llm, alert_values: dict, redact, progress=None) -> dict:
395
+ """What a scheduled execution does once its tests ran: look for structural changes and, at most once per
396
+ change, propose tests for them in a new run that waits for review, and alert. `make_llm()` gives the model
397
+ (it may raise: no key, no server). Returns {"status", "pending_run", "lines", "alerted", "error"}, status
398
+ one of unchanged, pending_exists, proposed, replaced, detected_only, failed, busy. Never runs a test."""
399
+ outcome = {"status": "unchanged", "pending_run": None, "lines": [], "alerted": [], "error": None}
400
+ try:
401
+ with run.proposal_lock():
402
+ return _check(run, wh, make_llm, alert_values, redact, progress, outcome)
403
+ except RunBusyError: # the in-app schedule and a `dqvouch rerun` noticed the same change: one proposes
404
+ return {**outcome, "status": "busy"}
405
+
406
+
407
+ def _check(run: Run, wh, make_llm, alert_values: dict, redact, progress, outcome: dict) -> dict:
408
+ """after_execution, with the proposal lock held."""
409
+ detected = detect(run, wh)
410
+ state = read_state(run)
411
+ pending = state.get("pending_run")
412
+ if not has_changes(detected["changes"]):
413
+ if pending: # the structure went back to what was reviewed: the pending run has nothing left to do
414
+ _replace(run, pending, None)
415
+ if pending or state.get("failed") or state.get("detected"):
416
+ write_state(run, pending_run=None, signatures=None, failed=None, detected=None)
417
+ return outcome
418
+ lines = describe_changes(detected["changes"])
419
+ sigs = table_signatures(detected["context"]["asset_contexts"])
420
+ outcome.update(lines=lines, pending_run=pending)
421
+ if pending and state.get("signatures") == sigs:
422
+ return {**outcome, "status": "pending_exists"}
423
+ n = len(lines)
424
+ if not run.info()["settings"].get("propose_on_change", True):
425
+ outcome["status"] = "detected_only"
426
+ if (state.get("detected") or {}).get("signatures") != sigs:
427
+ outcome["alerted"] = _alert(alert_values, redact, f"dqvouch: structure changed in {n} table(s)",
428
+ lines + ["", "Proposing is off (propose_on_change = false): start a new "
429
+ "run to cover it."])
430
+ write_state(run, detected={"signatures": sigs, "lines": lines})
431
+ return outcome
432
+ try:
433
+ llm = make_llm() if targets(detected["changes"]) else None
434
+ if hasattr(llm, "max_attempts"): # no one waits on a schedule: an outage must end in "failed", not hang
435
+ llm.max_attempts = SCHEDULE_MODEL_ATTEMPTS
436
+ new = propose(run, wh, llm, detected, progress)
437
+ except Exception as e:
438
+ error = redact(str(e) or type(e).__name__)
439
+ outcome.update(status="failed", error=error)
440
+ if (state.get("failed") or {}).get("signatures") != sigs:
441
+ outcome["alerted"] = _alert(
442
+ alert_values, redact, f"dqvouch: structure changed in {n} table(s), but proposing new tests failed",
443
+ lines + ["", f"{error}. dqvouch will try again at the next execution."])
444
+ write_state(run, failed={"signatures": sigs, "error": error, "lines": lines})
445
+ return outcome
446
+ if pending:
447
+ _replace(run, pending, new.id)
448
+ origin = new.info()["origin"]
449
+ write_state(run, pending_run=new.id, signatures=sigs, failed=None, detected=None,
450
+ detected_at=origin["detected_at"], proposed=origin["proposed"])
451
+ aside = origin["set_aside"]
452
+ body = lines + ["", f"{origin['kept']} confirmed test(s) kept, {len(aside)} set aside, {origin['proposed']} new. "
453
+ f"Review them in run {new.id} (dqvouch ui)."]
454
+ body += [f"- {a['id']}: {a['reason']}" for a in aside]
455
+ outcome.update(status="replaced" if pending else "proposed", pending_run=new.id,
456
+ alerted=_alert(alert_values, redact, f"dqvouch: structure changed in {n} table(s), "
457
+ f"{origin['proposed']} new test(s) to review", body))
458
+ return outcome
459
+
460
+
461
+ def _replace(run: Run, pending: str, by: str | None) -> None:
462
+ """Marks an earlier pending run as replaced (kept on disk, shown as replaced), if it is still there."""
463
+ path = run.path.parent / pending
464
+ if (path / "run.json").is_file():
465
+ Run.open(path).update_info(replaced_by=by or "the reviewed structure")
466
+
467
+
468
+ def hand_over(new_run: Run) -> bool:
469
+ """Once a change's run is reviewed, the daily executions move to it: the old run's schedule moves over, its
470
+ history comes first in the new run's (so the trends of the kept tests, same ids, continue), and the old run
471
+ is marked superseded (which `current` follows, so cron lines naming the old run keep working). Only for a
472
+ reviewed run made by a change that is still the one waiting (not replaced by a later change). True when the
473
+ executions moved (or had already moved) to `new_run`."""
474
+ info = new_run.info()
475
+ origin = info.get("origin") or {}
476
+ old_path = new_run.path.parent / str(origin.get("from_run"))
477
+ if origin.get("kind") != "change" or "review" not in new_run.completed() or not (old_path / "run.json").is_file():
478
+ return False
479
+ old = Run.open(old_path)
480
+ state = read_state(old)
481
+ if state.get("superseded_by") == new_run.id:
482
+ return True
483
+ if info.get("replaced_by") or (state.get("pending_run") and state["pending_run"] != new_run.id):
484
+ return False # a later change replaced it: the run that replaced it is the one to review
485
+ schedule = old.path / "schedule.json"
486
+ if schedule.exists() and not (new_run.path / "schedule.json").exists():
487
+ schedule.replace(new_run.path / "schedule.json")
488
+ history = old.path / "history.jsonl"
489
+ if history.exists():
490
+ mine = new_run.path / "history.jsonl"
491
+ own = mine.read_text(encoding="utf-8") if mine.exists() else ""
492
+ mine.write_text(history.read_text(encoding="utf-8") + own, encoding="utf-8")
493
+ write_state(old, superseded_by=new_run.id, pending_run=None, signatures=None)
494
+ return True
495
+
496
+
497
+ def current(run: Run) -> Run:
498
+ """The run whose tests are executed in place of `run`: follows superseded_by while the next run exists and
499
+ was reviewed (a run still waiting for review is never executed)."""
500
+ seen = {run.id}
501
+ while True:
502
+ nxt = read_state(run).get("superseded_by")
503
+ path = run.path.parent / str(nxt)
504
+ if not nxt or nxt in seen or not (path / "run.json").is_file() or "review" not in Run.open(path).completed():
505
+ return run
506
+ run = Run.open(path)
507
+ seen.add(run.id)