bricksentry 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bricksentry/__init__.py +26 -0
- bricksentry/agent_tools.py +139 -0
- bricksentry/assess.py +165 -0
- bricksentry/backends/__init__.py +3 -0
- bricksentry/backends/base.py +96 -0
- bricksentry/backends/databricks_backend.py +161 -0
- bricksentry/backends/duckdb_backend.py +52 -0
- bricksentry/changes.py +34 -0
- bricksentry/cli.py +260 -0
- bricksentry/config.py +123 -0
- bricksentry/diff.py +194 -0
- bricksentry/explain.py +91 -0
- bricksentry/github.py +52 -0
- bricksentry/lineage.py +224 -0
- bricksentry/mcp_server.py +47 -0
- bricksentry/models.py +212 -0
- bricksentry/policy.py +221 -0
- bricksentry/preview.py +79 -0
- bricksentry/py.typed +0 -0
- bricksentry/report.py +88 -0
- bricksentry-0.1.0.dist-info/METADATA +186 -0
- bricksentry-0.1.0.dist-info/RECORD +25 -0
- bricksentry-0.1.0.dist-info/WHEEL +4 -0
- bricksentry-0.1.0.dist-info/entry_points.txt +2 -0
- bricksentry-0.1.0.dist-info/licenses/LICENSE +202 -0
bricksentry/cli.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""bricksentry CLI.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 = verdict below --fail-on, 1 = verdict at or above --fail-on,
|
|
4
|
+
2 = usage / configuration error.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import click
|
|
15
|
+
|
|
16
|
+
from bricksentry import __version__
|
|
17
|
+
from bricksentry.config import Config
|
|
18
|
+
from bricksentry.models import Verdict
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _sentry(config_path):
|
|
22
|
+
from bricksentry.assess import Sentry
|
|
23
|
+
|
|
24
|
+
return Sentry(Config.load(config_path))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _emit(obj, fmt: str, markdown_fn=None):
|
|
28
|
+
if fmt == "json":
|
|
29
|
+
click.echo(json.dumps(obj if isinstance(obj, (dict, list)) else obj.to_dict(), indent=2, default=str))
|
|
30
|
+
elif fmt == "markdown" and markdown_fn:
|
|
31
|
+
click.echo(markdown_fn())
|
|
32
|
+
else:
|
|
33
|
+
click.echo(obj if isinstance(obj, str) else json.dumps(obj, indent=2, default=str))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _exit_for(verdict: Verdict, fail_on: str) -> None:
|
|
37
|
+
if verdict.rank >= Verdict(fail_on).rank:
|
|
38
|
+
sys.exit(1)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
config_opt = click.option("--config", "-c", "config_path", type=click.Path(), default=None,
|
|
42
|
+
help="Path to bricksentry.yml (default: ./bricksentry.yml or $BRICKSENTRY_CONFIG)")
|
|
43
|
+
fail_opt = click.option("--fail-on", type=click.Choice([v.value for v in Verdict]), default="block",
|
|
44
|
+
show_default=True, help="Exit 1 when the verdict is at least this severe")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@click.group()
|
|
48
|
+
@click.version_option(__version__, prog_name="bricksentry")
|
|
49
|
+
def main():
|
|
50
|
+
"""Change-safety for Databricks pipelines and the AI agents that edit them."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@main.command()
|
|
54
|
+
@click.argument("base")
|
|
55
|
+
@click.argument("candidate")
|
|
56
|
+
@click.option("--key", "-k", "keys", multiple=True, help="Key column (repeatable). Defaults to config.")
|
|
57
|
+
@click.option("--sample-pct", type=float, default=None, help="Diff a deterministic hash sample (0-100).")
|
|
58
|
+
@click.option("--format", "fmt", type=click.Choice(["text", "json"]), default="text")
|
|
59
|
+
@config_opt
|
|
60
|
+
def diff(base, candidate, keys, sample_pct, fmt, config_path):
|
|
61
|
+
"""Diff BASE (production) against CANDIDATE (your change)."""
|
|
62
|
+
from bricksentry.explain import explain_diff
|
|
63
|
+
|
|
64
|
+
s = _sentry(config_path)
|
|
65
|
+
d = s.diff(base, candidate, keys=list(keys) or None, table=base, sample_pct=sample_pct)
|
|
66
|
+
_emit(d.to_dict() if fmt == "json" else explain_diff(d), fmt)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@main.command()
|
|
70
|
+
@click.argument("table")
|
|
71
|
+
@click.option("--depth", type=int, default=None)
|
|
72
|
+
@click.option("--format", "fmt", type=click.Choice(["text", "json"]), default="text")
|
|
73
|
+
@config_opt
|
|
74
|
+
def impact(table, depth, fmt, config_path):
|
|
75
|
+
"""Show the downstream blast radius of TABLE."""
|
|
76
|
+
from bricksentry.explain import explain_impact
|
|
77
|
+
|
|
78
|
+
s = _sentry(config_path)
|
|
79
|
+
nodes = s.downstream(table, depth)
|
|
80
|
+
if fmt == "json":
|
|
81
|
+
_emit({"table": table, "downstream": [n.to_dict() for n in nodes]}, fmt)
|
|
82
|
+
return
|
|
83
|
+
click.echo(explain_impact(table, nodes))
|
|
84
|
+
for n in sorted(nodes, key=lambda n: (n.depth, n.kind, n.name)):
|
|
85
|
+
tags = ",".join(n.tags) if n.tags else ""
|
|
86
|
+
click.echo(f" {' ' * (n.depth - 1)}└ [{n.kind}] {n.name} {n.owner or ''} {tags}".rstrip())
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@main.command()
|
|
90
|
+
@click.option("--table", "-t", "tables", multiple=True, help="Table to assess (repeatable).")
|
|
91
|
+
@click.option("--from-git", "git_base", default=None, help="Detect changed tables from `git diff BASE...HEAD`.")
|
|
92
|
+
@click.option("--candidate", "cand_pairs", multiple=True, help="Override: TABLE=CANDIDATE_TABLE")
|
|
93
|
+
@click.option("--pr", default=None, help="PR number for candidate_pattern {pr}.")
|
|
94
|
+
@click.option("--format", "fmt", type=click.Choice(["text", "json", "markdown"]), default="text")
|
|
95
|
+
@click.option("--output", "-o", type=click.Path(), default=None, help="Also write the markdown report here.")
|
|
96
|
+
@click.option("--comment", is_flag=True, help="Post/update the report as a GitHub PR comment.")
|
|
97
|
+
@fail_opt
|
|
98
|
+
@config_opt
|
|
99
|
+
def assess(tables, git_base, cand_pairs, pr, fmt, output, comment, fail_on, config_path):
|
|
100
|
+
"""Assess a change: data diff + blast radius + policy verdict + rollback plan."""
|
|
101
|
+
from bricksentry.changes import changed_files, tables_for_files
|
|
102
|
+
from bricksentry.report import to_markdown
|
|
103
|
+
|
|
104
|
+
s = _sentry(config_path)
|
|
105
|
+
tables = list(tables)
|
|
106
|
+
if git_base:
|
|
107
|
+
files = changed_files(git_base)
|
|
108
|
+
found = tables_for_files(s.config, files)
|
|
109
|
+
click.echo(f"{len(files)} changed file(s) -> {len(found)} table(s): {', '.join(found) or 'none'}", err=True)
|
|
110
|
+
tables += [t for t in found if t not in tables]
|
|
111
|
+
if not tables:
|
|
112
|
+
click.echo("Nothing to assess: no tables given or matched.", err=True)
|
|
113
|
+
sys.exit(0)
|
|
114
|
+
candidates = dict(p.split("=", 1) for p in cand_pairs)
|
|
115
|
+
pr = pr or os.environ.get("BRICKSENTRY_PR")
|
|
116
|
+
a = s.assess(tables, candidates=candidates, pr=pr)
|
|
117
|
+
md = to_markdown(a)
|
|
118
|
+
if output:
|
|
119
|
+
Path(output).write_text(md)
|
|
120
|
+
if comment:
|
|
121
|
+
from bricksentry.github import upsert_pr_comment
|
|
122
|
+
|
|
123
|
+
url = upsert_pr_comment(md, pr=pr)
|
|
124
|
+
click.echo(f"PR comment: {url}", err=True)
|
|
125
|
+
if fmt == "text":
|
|
126
|
+
click.echo(f"{a.verdict.value.upper()} (risk {a.verdict.risk_tier})")
|
|
127
|
+
for t in a.tables:
|
|
128
|
+
click.echo(f"- {t.summary}")
|
|
129
|
+
for h in t.hits:
|
|
130
|
+
click.echo(f" [{h.verdict.value}] {h.rule}: {h.reason}")
|
|
131
|
+
if a.notify:
|
|
132
|
+
click.echo(f"Notify: {', '.join(a.notify)}")
|
|
133
|
+
else:
|
|
134
|
+
_emit(a, fmt, lambda: md)
|
|
135
|
+
_exit_for(a.verdict, fail_on)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@main.command()
|
|
139
|
+
@click.argument("target")
|
|
140
|
+
@click.option("--sql", "sql_text", default=None, help="The write statement. Use {{target}} or the table name.")
|
|
141
|
+
@click.option("--file", "sql_file", type=click.Path(exists=True), default=None)
|
|
142
|
+
@click.option("--key", "-k", "keys", multiple=True)
|
|
143
|
+
@click.option("--format", "fmt", type=click.Choice(["text", "json"]), default="text")
|
|
144
|
+
@fail_opt
|
|
145
|
+
@config_opt
|
|
146
|
+
def preview(target, sql_text, sql_file, keys, fmt, fail_on, config_path):
|
|
147
|
+
"""Run a write against a sandbox clone of TARGET and show what it would change."""
|
|
148
|
+
if not (sql_text or sql_file):
|
|
149
|
+
raise click.UsageError("pass --sql or --file")
|
|
150
|
+
sql = sql_text or Path(sql_file).read_text()
|
|
151
|
+
s = _sentry(config_path)
|
|
152
|
+
out = s.preview_write(sql, target, keys=list(keys) or None)
|
|
153
|
+
if fmt == "json":
|
|
154
|
+
_emit(out, fmt)
|
|
155
|
+
else:
|
|
156
|
+
click.echo(f"{out['verdict'].upper()} (risk {out['risk_tier']})")
|
|
157
|
+
for t in out["tables"]:
|
|
158
|
+
click.echo(f"- {t['summary']}")
|
|
159
|
+
for h in t["rule_hits"]:
|
|
160
|
+
click.echo(f" [{h['verdict']}] {h['rule']}: {h['reason']}")
|
|
161
|
+
_exit_for(Verdict(out["verdict"]), fail_on)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@main.command()
|
|
165
|
+
@click.argument("table")
|
|
166
|
+
@config_opt
|
|
167
|
+
def rollback(table, config_path):
|
|
168
|
+
"""Print the SQL that restores TABLE to its current version."""
|
|
169
|
+
click.echo(_sentry(config_path).rollback_plan(table))
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@main.command()
|
|
173
|
+
@click.option("--transport", type=click.Choice(["stdio", "sse", "streamable-http"]), default="stdio")
|
|
174
|
+
@config_opt
|
|
175
|
+
def mcp(transport, config_path):
|
|
176
|
+
"""Start the MCP server for AI agents."""
|
|
177
|
+
if config_path:
|
|
178
|
+
os.environ["BRICKSENTRY_CONFIG"] = str(Path(config_path).resolve())
|
|
179
|
+
from bricksentry.mcp_server import serve
|
|
180
|
+
|
|
181
|
+
serve(transport)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
_INIT_CONFIG = """# bricksentry configuration
|
|
185
|
+
backend:
|
|
186
|
+
type: databricks # or duckdb for local work
|
|
187
|
+
# warehouse_id: <sql-warehouse-id> # or env DATABRICKS_WAREHOUSE_ID
|
|
188
|
+
# profile: DEFAULT # ~/.databrickscfg profile (optional)
|
|
189
|
+
|
|
190
|
+
sandbox_schema: main.bricksentry_sandbox # where preview_write puts disposable clones
|
|
191
|
+
|
|
192
|
+
# Where your CI build writes the candidate (changed) version of each table.
|
|
193
|
+
candidate_pattern: "{catalog}.{schema}_pr_{pr}.{table}"
|
|
194
|
+
|
|
195
|
+
# sample_pct: 10 # diff a deterministic 10% key sample on big tables
|
|
196
|
+
|
|
197
|
+
lineage:
|
|
198
|
+
depth: 3
|
|
199
|
+
lookback_days: 90
|
|
200
|
+
|
|
201
|
+
tables:
|
|
202
|
+
main.sales.orders:
|
|
203
|
+
keys: [order_id]
|
|
204
|
+
segment_by: [region]
|
|
205
|
+
paths: ["src/pipelines/orders/**", "resources/orders_job.yml"]
|
|
206
|
+
|
|
207
|
+
policy: policy.yml
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
_INIT_POLICY = """default: pass
|
|
211
|
+
notify_owners_on: needs_human
|
|
212
|
+
rules:
|
|
213
|
+
- {name: diff-failed, when: {diff_error: true}, then: needs_human}
|
|
214
|
+
- {name: duplicate-keys, when: {duplicate_keys: true}, then: block}
|
|
215
|
+
- {name: dropped-column, when: {column_removed: true}, then: needs_human}
|
|
216
|
+
- {name: type-change, when: {column_type_changed: true}, then: warn}
|
|
217
|
+
- {name: certified-downstream, when: {downstream_tag: certified}, then: needs_human}
|
|
218
|
+
- {name: pii-table, when: {tag: pii}, then: needs_human}
|
|
219
|
+
- {name: row-loss, when: {removed_pct_gt: 1}, then: needs_human}
|
|
220
|
+
- {name: many-rows-changed, when: {changed_pct_gt: 10}, then: warn}
|
|
221
|
+
- {name: numeric-total-moved, when: {sum_change_pct_gt: 5}, then: warn}
|
|
222
|
+
- {name: wide-blast-radius, when: {downstream_count_gt: 25}, then: warn}
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
@main.command()
|
|
227
|
+
@click.option("--force", is_flag=True)
|
|
228
|
+
def init(force):
|
|
229
|
+
"""Write a starter bricksentry.yml and policy.yml."""
|
|
230
|
+
for name, body in (("bricksentry.yml", _INIT_CONFIG), ("policy.yml", _INIT_POLICY)):
|
|
231
|
+
p = Path(name)
|
|
232
|
+
if p.exists() and not force:
|
|
233
|
+
click.echo(f"skip {name} (exists; use --force)")
|
|
234
|
+
continue
|
|
235
|
+
p.write_text(body)
|
|
236
|
+
click.echo(f"wrote {name}")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def entry() -> None:
|
|
242
|
+
"""Console entry point: turn unexpected errors into a clean message and exit code 2."""
|
|
243
|
+
try:
|
|
244
|
+
main(standalone_mode=False)
|
|
245
|
+
except click.exceptions.Exit as e:
|
|
246
|
+
sys.exit(e.exit_code)
|
|
247
|
+
except click.ClickException as e:
|
|
248
|
+
e.show()
|
|
249
|
+
sys.exit(2)
|
|
250
|
+
except click.exceptions.Abort:
|
|
251
|
+
sys.exit(2)
|
|
252
|
+
except SystemExit:
|
|
253
|
+
raise
|
|
254
|
+
except Exception as e:
|
|
255
|
+
click.echo(f"bricksentry: error: {e}", err=True)
|
|
256
|
+
sys.exit(2)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
if __name__ == "__main__": # pragma: no cover
|
|
260
|
+
entry()
|
bricksentry/config.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Project configuration, loaded from ``bricksentry.yml``.
|
|
2
|
+
|
|
3
|
+
Example::
|
|
4
|
+
|
|
5
|
+
backend:
|
|
6
|
+
type: databricks # or duckdb
|
|
7
|
+
warehouse_id: abc123 # env DATABRICKS_WAREHOUSE_ID also works
|
|
8
|
+
sandbox_schema: main.bricksentry_sandbox
|
|
9
|
+
candidate_pattern: "{catalog}.{schema}_pr_{pr}.{table}"
|
|
10
|
+
sample_pct: 10 # diff a deterministic 10% hash sample (omit for full diff)
|
|
11
|
+
lineage:
|
|
12
|
+
depth: 3
|
|
13
|
+
lookback_days: 90
|
|
14
|
+
tables:
|
|
15
|
+
main.sales.orders:
|
|
16
|
+
keys: [order_id]
|
|
17
|
+
segment_by: [region]
|
|
18
|
+
paths: ["pipelines/orders/**", "resources/orders_job.yml"]
|
|
19
|
+
policy: policy.yml # or inline mapping
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
import yaml
|
|
30
|
+
|
|
31
|
+
DEFAULT_CONFIG_NAMES = ("bricksentry.yml", "bricksentry.yaml", ".bricksentry.yml")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class TableSpec:
|
|
36
|
+
name: str
|
|
37
|
+
keys: list[str]
|
|
38
|
+
segment_by: list[str] = field(default_factory=list)
|
|
39
|
+
paths: list[str] = field(default_factory=list)
|
|
40
|
+
candidate: str | None = None # explicit candidate table, overrides candidate_pattern
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class Config:
|
|
45
|
+
backend: dict[str, Any] = field(default_factory=lambda: {"type": "duckdb"})
|
|
46
|
+
sandbox_schema: str = "bricksentry_sandbox"
|
|
47
|
+
candidate_pattern: str = "{catalog}.{schema}_ci.{table}"
|
|
48
|
+
sample_pct: float | None = None
|
|
49
|
+
max_samples: int = 5
|
|
50
|
+
lineage: dict[str, Any] = field(default_factory=dict)
|
|
51
|
+
tables: dict[str, TableSpec] = field(default_factory=dict)
|
|
52
|
+
policy: dict[str, Any] | None = None
|
|
53
|
+
root: Path = field(default_factory=Path.cwd)
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def from_dict(cls, data: dict[str, Any], root: Path | None = None) -> "Config":
|
|
57
|
+
root = root or Path.cwd()
|
|
58
|
+
tables = {}
|
|
59
|
+
for name, spec in (data.get("tables") or {}).items():
|
|
60
|
+
spec = spec or {}
|
|
61
|
+
keys = spec.get("keys") or spec.get("key") or []
|
|
62
|
+
if isinstance(keys, str):
|
|
63
|
+
keys = [keys]
|
|
64
|
+
seg = spec.get("segment_by") or []
|
|
65
|
+
if isinstance(seg, str):
|
|
66
|
+
seg = [seg]
|
|
67
|
+
tables[name] = TableSpec(
|
|
68
|
+
name=name,
|
|
69
|
+
keys=list(keys),
|
|
70
|
+
segment_by=list(seg),
|
|
71
|
+
paths=list(spec.get("paths") or []),
|
|
72
|
+
candidate=spec.get("candidate"),
|
|
73
|
+
)
|
|
74
|
+
policy = data.get("policy")
|
|
75
|
+
if isinstance(policy, str):
|
|
76
|
+
policy = yaml.safe_load((root / policy).read_text()) or {}
|
|
77
|
+
return cls(
|
|
78
|
+
backend=data.get("backend") or {"type": "duckdb"},
|
|
79
|
+
sandbox_schema=data.get("sandbox_schema", "bricksentry_sandbox"),
|
|
80
|
+
candidate_pattern=data.get("candidate_pattern", "{catalog}.{schema}_ci.{table}"),
|
|
81
|
+
sample_pct=data.get("sample_pct"),
|
|
82
|
+
max_samples=int(data.get("max_samples", 5)),
|
|
83
|
+
lineage=data.get("lineage") or {},
|
|
84
|
+
tables=tables,
|
|
85
|
+
policy=policy,
|
|
86
|
+
root=root,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def load(cls, path: str | os.PathLike | None = None) -> "Config":
|
|
91
|
+
if path is None:
|
|
92
|
+
path = os.environ.get("BRICKSENTRY_CONFIG")
|
|
93
|
+
if path is None:
|
|
94
|
+
for name in DEFAULT_CONFIG_NAMES:
|
|
95
|
+
if Path(name).exists():
|
|
96
|
+
path = name
|
|
97
|
+
break
|
|
98
|
+
if path is None:
|
|
99
|
+
return cls()
|
|
100
|
+
p = Path(path)
|
|
101
|
+
data = yaml.safe_load(p.read_text()) or {}
|
|
102
|
+
return cls.from_dict(data, root=p.resolve().parent)
|
|
103
|
+
|
|
104
|
+
def spec(self, table: str) -> TableSpec:
|
|
105
|
+
if table in self.tables:
|
|
106
|
+
return self.tables[table]
|
|
107
|
+
return TableSpec(name=table, keys=[])
|
|
108
|
+
|
|
109
|
+
def candidate_for(self, table: str, pr: str | None = None) -> str:
|
|
110
|
+
spec = self.spec(table)
|
|
111
|
+
if spec.candidate:
|
|
112
|
+
return spec.candidate.format(pr=pr or os.environ.get("BRICKSENTRY_PR", "local"))
|
|
113
|
+
parts = table.split(".")
|
|
114
|
+
while len(parts) < 3:
|
|
115
|
+
parts.insert(0, "")
|
|
116
|
+
catalog, schema, name = parts[-3], parts[-2], parts[-1]
|
|
117
|
+
out = self.candidate_pattern.format(
|
|
118
|
+
catalog=catalog,
|
|
119
|
+
schema=schema,
|
|
120
|
+
table=name,
|
|
121
|
+
pr=pr or os.environ.get("BRICKSENTRY_PR", "local"),
|
|
122
|
+
)
|
|
123
|
+
return out.lstrip(".")
|
bricksentry/diff.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Key-based data diff, pushed down to the engine as SQL.
|
|
2
|
+
|
|
3
|
+
Compares a *base* table (what production has now) with a *candidate* table
|
|
4
|
+
(what the change produces) on primary-key columns and reports:
|
|
5
|
+
|
|
6
|
+
* schema changes (added / removed / type-changed columns)
|
|
7
|
+
* row counts, added / removed / changed / unchanged rows
|
|
8
|
+
* per-column change counts, null counts and numeric sums
|
|
9
|
+
* duplicate keys on either side
|
|
10
|
+
* sample changed values per column
|
|
11
|
+
* segment attribution ("most changes are in region = APAC")
|
|
12
|
+
|
|
13
|
+
Sampling uses a deterministic hash on the keys, so both sides sample the same keys.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from bricksentry.backends.base import Backend
|
|
19
|
+
from bricksentry.models import ColumnChange, SchemaChange, Segment, TableDiff
|
|
20
|
+
|
|
21
|
+
MAX_SAMPLE_COLUMNS = 10
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class DiffError(ValueError):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def diff_tables(
|
|
29
|
+
backend: Backend,
|
|
30
|
+
base: str,
|
|
31
|
+
candidate: str,
|
|
32
|
+
keys: list[str],
|
|
33
|
+
*,
|
|
34
|
+
table: str | None = None,
|
|
35
|
+
sample_pct: float | None = None,
|
|
36
|
+
segment_by: list[str] | None = None,
|
|
37
|
+
max_samples: int = 5,
|
|
38
|
+
columns: list[str] | None = None,
|
|
39
|
+
) -> TableDiff:
|
|
40
|
+
if not keys:
|
|
41
|
+
raise DiffError(
|
|
42
|
+
f"No key columns for {table or base}. Add `keys:` for it in bricksentry.yml "
|
|
43
|
+
"(or pass --key) so rows can be matched."
|
|
44
|
+
)
|
|
45
|
+
q = backend.quote
|
|
46
|
+
b_cols = backend.columns(base)
|
|
47
|
+
c_cols = backend.columns(candidate)
|
|
48
|
+
|
|
49
|
+
missing = [k for k in keys if k not in b_cols or k not in c_cols]
|
|
50
|
+
if missing:
|
|
51
|
+
raise DiffError(f"Key column(s) {missing} not present in both {base} and {candidate}")
|
|
52
|
+
|
|
53
|
+
schema = SchemaChange(
|
|
54
|
+
added={k: v for k, v in c_cols.items() if k not in b_cols},
|
|
55
|
+
removed={k: v for k, v in b_cols.items() if k not in c_cols},
|
|
56
|
+
type_changed={
|
|
57
|
+
k: (b_cols[k], c_cols[k]) for k in b_cols if k in c_cols and b_cols[k] != c_cols[k]
|
|
58
|
+
},
|
|
59
|
+
)
|
|
60
|
+
common = [c for c in b_cols if c in c_cols and c not in keys]
|
|
61
|
+
if columns:
|
|
62
|
+
common = [c for c in common if c in columns]
|
|
63
|
+
|
|
64
|
+
def side(tbl: str, alias: str) -> str:
|
|
65
|
+
where = f" WHERE {backend.sample_predicate(keys, sample_pct)}" if sample_pct else ""
|
|
66
|
+
return f"{alias} AS (SELECT * FROM {backend.qualified(tbl)}{where})"
|
|
67
|
+
|
|
68
|
+
ctes = f"WITH {side(base, 'b_src')}, {side(candidate, 'c_src')}"
|
|
69
|
+
join_on = " AND ".join(f"b.{q(k)} = c.{q(k)}" for k in keys)
|
|
70
|
+
k0 = q(keys[0])
|
|
71
|
+
|
|
72
|
+
def cmp(col: str) -> str:
|
|
73
|
+
bc, cc = f"b.{q(col)}", f"c.{q(col)}"
|
|
74
|
+
if col in schema.type_changed:
|
|
75
|
+
st = backend.string_type
|
|
76
|
+
bc, cc = f"CAST({bc} AS {st})", f"CAST({cc} AS {st})"
|
|
77
|
+
return f"{bc} IS DISTINCT FROM {cc}"
|
|
78
|
+
|
|
79
|
+
both = f"b.{k0} IS NOT NULL AND c.{k0} IS NOT NULL"
|
|
80
|
+
any_diff = " OR ".join(f"({cmp(c)})" for c in common) or "FALSE"
|
|
81
|
+
|
|
82
|
+
selects = [
|
|
83
|
+
f"SUM(CASE WHEN b.{k0} IS NULL THEN 1 ELSE 0 END) AS added_rows",
|
|
84
|
+
f"SUM(CASE WHEN c.{k0} IS NULL THEN 1 ELSE 0 END) AS removed_rows",
|
|
85
|
+
f"SUM(CASE WHEN {both} AND ({any_diff}) THEN 1 ELSE 0 END) AS changed_rows",
|
|
86
|
+
f"SUM(CASE WHEN {both} AND NOT ({any_diff}) THEN 1 ELSE 0 END) AS unchanged_rows",
|
|
87
|
+
]
|
|
88
|
+
for i, col in enumerate(common):
|
|
89
|
+
selects.append(f"SUM(CASE WHEN {both} AND ({cmp(col)}) THEN 1 ELSE 0 END) AS chg_{i}")
|
|
90
|
+
|
|
91
|
+
join_sql = (
|
|
92
|
+
f"{ctes} SELECT {', '.join(selects)} "
|
|
93
|
+
f"FROM b_src b FULL OUTER JOIN c_src c ON {join_on}"
|
|
94
|
+
)
|
|
95
|
+
j = backend.query(join_sql)[0]
|
|
96
|
+
|
|
97
|
+
# Row counts, nulls, numeric sums per side.
|
|
98
|
+
def profile(tbl_cte: str, cols: dict[str, str]) -> dict:
|
|
99
|
+
parts = ["COUNT(*) AS n_rows"]
|
|
100
|
+
for i, col in enumerate(common):
|
|
101
|
+
parts.append(f"SUM(CASE WHEN {q(col)} IS NULL THEN 1 ELSE 0 END) AS nul_{i}")
|
|
102
|
+
if backend.is_numeric(cols[col]):
|
|
103
|
+
parts.append(f"SUM(CAST({q(col)} AS DOUBLE)) AS sum_{i}")
|
|
104
|
+
return backend.query(f"{ctes} SELECT {', '.join(parts)} FROM {tbl_cte}")[0]
|
|
105
|
+
|
|
106
|
+
pb = profile("b_src", b_cols)
|
|
107
|
+
pc = profile("c_src", c_cols)
|
|
108
|
+
|
|
109
|
+
def dupes(tbl_cte: str) -> int:
|
|
110
|
+
kc = ", ".join(q(k) for k in keys)
|
|
111
|
+
sql = (
|
|
112
|
+
f"{ctes} SELECT COUNT(*) AS n FROM "
|
|
113
|
+
f"(SELECT {kc} FROM {tbl_cte} GROUP BY {kc} HAVING COUNT(*) > 1) d"
|
|
114
|
+
)
|
|
115
|
+
return int(backend.scalar(sql) or 0)
|
|
116
|
+
|
|
117
|
+
col_changes: list[ColumnChange] = []
|
|
118
|
+
for i, col in enumerate(common):
|
|
119
|
+
changed = int(j.get(f"chg_{i}") or 0)
|
|
120
|
+
cc = ColumnChange(
|
|
121
|
+
column=col,
|
|
122
|
+
changed_rows=changed,
|
|
123
|
+
base_nulls=_int(pb.get(f"nul_{i}")),
|
|
124
|
+
candidate_nulls=_int(pc.get(f"nul_{i}")),
|
|
125
|
+
base_sum=_float(pb.get(f"sum_{i}")),
|
|
126
|
+
candidate_sum=_float(pc.get(f"sum_{i}")),
|
|
127
|
+
)
|
|
128
|
+
col_changes.append(cc)
|
|
129
|
+
|
|
130
|
+
# Samples for the most-changed columns.
|
|
131
|
+
key_sel = ", ".join(f"COALESCE(b.{q(k)}, c.{q(k)}) AS {q(k)}" for k in keys)
|
|
132
|
+
for cc in sorted(col_changes, key=lambda c: -c.changed_rows)[:MAX_SAMPLE_COLUMNS]:
|
|
133
|
+
if cc.changed_rows == 0 or max_samples <= 0:
|
|
134
|
+
continue
|
|
135
|
+
col = cc.column
|
|
136
|
+
sql = (
|
|
137
|
+
f"{ctes} SELECT {key_sel}, b.{q(col)} AS base_value, c.{q(col)} AS candidate_value "
|
|
138
|
+
f"FROM b_src b JOIN c_src c ON {join_on} WHERE {cmp(col)} "
|
|
139
|
+
f"ORDER BY {', '.join(f'b.{q(k)}' for k in keys)} LIMIT {int(max_samples)}"
|
|
140
|
+
)
|
|
141
|
+
cc.samples = [_jsonable(r) for r in backend.query(sql)]
|
|
142
|
+
|
|
143
|
+
# Segment attribution over added + removed + changed rows.
|
|
144
|
+
segments: list[Segment] = []
|
|
145
|
+
for seg in segment_by or []:
|
|
146
|
+
if seg not in b_cols and seg not in c_cols:
|
|
147
|
+
continue
|
|
148
|
+
b_seg = f"b.{q(seg)}" if seg in b_cols else "NULL"
|
|
149
|
+
c_seg = f"c.{q(seg)}" if seg in c_cols else "NULL"
|
|
150
|
+
affected = f"(b.{k0} IS NULL OR c.{k0} IS NULL OR ({any_diff}))"
|
|
151
|
+
sql = (
|
|
152
|
+
f"{ctes} SELECT COALESCE({c_seg}, {b_seg}) AS seg_value, COUNT(*) AS n "
|
|
153
|
+
f"FROM b_src b FULL OUTER JOIN c_src c ON {join_on} WHERE {affected} "
|
|
154
|
+
f"GROUP BY 1 ORDER BY n DESC LIMIT 5"
|
|
155
|
+
)
|
|
156
|
+
for r in backend.query(sql):
|
|
157
|
+
segments.append(Segment(column=seg, value=_jsonable_value(r["seg_value"]), affected_rows=int(r["n"])))
|
|
158
|
+
|
|
159
|
+
return TableDiff(
|
|
160
|
+
table=table or base,
|
|
161
|
+
base=base,
|
|
162
|
+
candidate=candidate,
|
|
163
|
+
keys=list(keys),
|
|
164
|
+
base_rows=int(pb.get("n_rows") or 0),
|
|
165
|
+
candidate_rows=int(pc.get("n_rows") or 0),
|
|
166
|
+
added_rows=int(j.get("added_rows") or 0),
|
|
167
|
+
removed_rows=int(j.get("removed_rows") or 0),
|
|
168
|
+
changed_rows=int(j.get("changed_rows") or 0),
|
|
169
|
+
unchanged_rows=int(j.get("unchanged_rows") or 0),
|
|
170
|
+
schema=schema,
|
|
171
|
+
columns=col_changes,
|
|
172
|
+
duplicate_keys_base=dupes("b_src"),
|
|
173
|
+
duplicate_keys_candidate=dupes("c_src"),
|
|
174
|
+
segments=segments,
|
|
175
|
+
sampled_pct=sample_pct,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _int(v):
|
|
180
|
+
return None if v is None else int(v)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _float(v):
|
|
184
|
+
return None if v is None else float(v)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _jsonable_value(v):
|
|
188
|
+
if v is None or isinstance(v, (str, int, float, bool)):
|
|
189
|
+
return v
|
|
190
|
+
return str(v)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _jsonable(row: dict) -> dict:
|
|
194
|
+
return {k: _jsonable_value(v) for k, v in row.items()}
|
bricksentry/explain.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Plain-English summaries, built from templates (deterministic, no LLM needed).
|
|
2
|
+
|
|
3
|
+
An agent framework can pass this text to its own LLM if it wants richer prose,
|
|
4
|
+
but the verdict never depends on it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from bricksentry.models import ImpactNode, TableDiff
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _n(x: int) -> str:
|
|
13
|
+
return f"{x:,}"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def explain_diff(d: TableDiff) -> str:
|
|
17
|
+
if d.error:
|
|
18
|
+
return f"{d.table}: the diff could not run ({d.error})."
|
|
19
|
+
if d.is_identical:
|
|
20
|
+
scope = f" in a {d.sampled_pct:g}% sample" if d.sampled_pct else ""
|
|
21
|
+
return f"{d.table}: no data or schema changes{scope}."
|
|
22
|
+
|
|
23
|
+
parts: list[str] = []
|
|
24
|
+
rc = d.candidate_rows - d.base_rows
|
|
25
|
+
if rc:
|
|
26
|
+
parts.append(f"row count {'+' if rc > 0 else ''}{_n(rc)} ({d.row_count_change_pct:+.2f}%)")
|
|
27
|
+
if d.added_rows:
|
|
28
|
+
parts.append(f"{_n(d.added_rows)} rows added")
|
|
29
|
+
if d.removed_rows:
|
|
30
|
+
parts.append(f"{_n(d.removed_rows)} rows removed ({d.removed_pct:.2f}%)")
|
|
31
|
+
if d.changed_rows:
|
|
32
|
+
parts.append(f"{_n(d.changed_rows)} rows changed ({d.changed_pct:.2f}%)")
|
|
33
|
+
s = f"{d.table}: " + ", ".join(parts) if parts else f"{d.table}: schema-only change"
|
|
34
|
+
|
|
35
|
+
sch = []
|
|
36
|
+
if d.schema.removed:
|
|
37
|
+
sch.append("dropped " + ", ".join(d.schema.removed))
|
|
38
|
+
if d.schema.added:
|
|
39
|
+
sch.append("added " + ", ".join(d.schema.added))
|
|
40
|
+
if d.schema.type_changed:
|
|
41
|
+
sch.append("retyped " + ", ".join(f"{k} ({a} -> {b})" for k, (a, b) in d.schema.type_changed.items()))
|
|
42
|
+
if sch:
|
|
43
|
+
s += ". Schema: " + "; ".join(sch)
|
|
44
|
+
|
|
45
|
+
top = [c for c in sorted(d.columns, key=lambda c: -c.changed_rows) if c.changed_rows][:3]
|
|
46
|
+
if top:
|
|
47
|
+
s += ". Most changed: " + ", ".join(f"{c.column} ({_n(c.changed_rows)})" for c in top)
|
|
48
|
+
|
|
49
|
+
moved = [c for c in d.columns if c.sum_change_pct is not None and abs(c.sum_change_pct) >= 0.01]
|
|
50
|
+
moved.sort(key=lambda c: -abs(c.sum_change_pct or 0))
|
|
51
|
+
if moved:
|
|
52
|
+
s += ". Totals: " + ", ".join(f"sum({c.column}) {c.sum_change_pct:+.2f}%" for c in moved[:3])
|
|
53
|
+
|
|
54
|
+
nulls = [
|
|
55
|
+
c for c in d.columns
|
|
56
|
+
if c.base_nulls is not None and c.candidate_nulls is not None and c.candidate_nulls > c.base_nulls
|
|
57
|
+
]
|
|
58
|
+
if nulls:
|
|
59
|
+
s += ". New nulls: " + ", ".join(f"{c.column} +{_n(c.candidate_nulls - c.base_nulls)}" for c in nulls[:3])
|
|
60
|
+
|
|
61
|
+
if d.segments:
|
|
62
|
+
by_col: dict[str, list] = {}
|
|
63
|
+
for seg in d.segments:
|
|
64
|
+
by_col.setdefault(seg.column, []).append(seg)
|
|
65
|
+
total = max(d.added_rows + d.removed_rows + d.changed_rows, 1)
|
|
66
|
+
for col, segs in by_col.items():
|
|
67
|
+
lead = segs[0]
|
|
68
|
+
share = lead.affected_rows / total * 100
|
|
69
|
+
if share >= 50:
|
|
70
|
+
s += f". {share:.0f}% of affected rows are {col} = {lead.value}"
|
|
71
|
+
else:
|
|
72
|
+
s += f". Spread across {col}: " + ", ".join(f"{x.value} ({_n(x.affected_rows)})" for x in segs[:3])
|
|
73
|
+
|
|
74
|
+
if d.duplicate_keys_candidate > d.duplicate_keys_base:
|
|
75
|
+
s += f". WARNING: {_n(d.duplicate_keys_candidate)} duplicate key(s) on {', '.join(d.keys)}"
|
|
76
|
+
if d.sampled_pct:
|
|
77
|
+
s += f" (based on a {d.sampled_pct:g}% key sample)"
|
|
78
|
+
return s + "."
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def explain_impact(table: str, nodes: list[ImpactNode]) -> str:
|
|
82
|
+
if not nodes:
|
|
83
|
+
return f"{table}: nothing downstream found in lineage."
|
|
84
|
+
kinds: dict[str, int] = {}
|
|
85
|
+
for n in nodes:
|
|
86
|
+
kinds[n.kind] = kinds.get(n.kind, 0) + 1
|
|
87
|
+
owners = sorted({n.owner for n in nodes if n.owner})
|
|
88
|
+
s = f"{table} feeds " + ", ".join(f"{v} {k}{'s' if v > 1 else ''}" for k, v in sorted(kinds.items()))
|
|
89
|
+
if owners:
|
|
90
|
+
s += f"; owners: {', '.join(owners[:5])}"
|
|
91
|
+
return s + "."
|