rowproof 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rowproof/__init__.py +1 -0
- rowproof/cli/__init__.py +0 -0
- rowproof/cli/main.py +548 -0
- rowproof/cli/render.py +132 -0
- rowproof/cli/spec.py +99 -0
- rowproof/config/__init__.py +167 -0
- rowproof/connectors/__init__.py +0 -0
- rowproof/connectors/_chwire.py +160 -0
- rowproof/connectors/_pgwire.py +165 -0
- rowproof/connectors/_sfwire.py +157 -0
- rowproof/connectors/base.py +120 -0
- rowproof/connectors/clickhouse.py +580 -0
- rowproof/connectors/postgres.py +470 -0
- rowproof/connectors/snowflake.py +477 -0
- rowproof/core/__init__.py +0 -0
- rowproof/core/column_matching.py +217 -0
- rowproof/core/errors.py +37 -0
- rowproof/core/hashdiff.py +933 -0
- rowproof/core/joindiff.py +304 -0
- rowproof/core/models.py +192 -0
- rowproof/core/normalisation.py +119 -0
- rowproof/core/segmentation.py +159 -0
- rowproof/report/__init__.py +6 -0
- rowproof/report/html.py +167 -0
- rowproof-0.1.0.dist-info/METADATA +172 -0
- rowproof-0.1.0.dist-info/RECORD +30 -0
- rowproof-0.1.0.dist-info/WHEEL +5 -0
- rowproof-0.1.0.dist-info/entry_points.txt +2 -0
- rowproof-0.1.0.dist-info/licenses/LICENSE +201 -0
- rowproof-0.1.0.dist-info/top_level.txt +1 -0
rowproof/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
rowproof/cli/__init__.py
ADDED
|
File without changes
|
rowproof/cli/main.py
ADDED
|
@@ -0,0 +1,548 @@
|
|
|
1
|
+
"""rowproof CLI — argparse stand-in for `typer` (see docs/DEV_ENVIRONMENT.md).
|
|
2
|
+
Flags, behavior and exit codes follow spec §7 exactly; only the argument
|
|
3
|
+
*parsing library* differs from the spec's chosen stack.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
import traceback
|
|
12
|
+
|
|
13
|
+
import rowproof
|
|
14
|
+
from rowproof.cli.render import render_json, render_terminal
|
|
15
|
+
from rowproof.cli.spec import parse_source_spec, resolve_source_spec
|
|
16
|
+
from rowproof.config import load_config
|
|
17
|
+
from rowproof.connectors.clickhouse import ClickHouseConnector
|
|
18
|
+
from rowproof.connectors.postgres import PostgresConnector
|
|
19
|
+
from rowproof.core.errors import RowProofError
|
|
20
|
+
from rowproof.core.hashdiff import diff as run_hashdiff
|
|
21
|
+
from rowproof.core.hashdiff import explain as run_explain
|
|
22
|
+
from rowproof.core.joindiff import diff as run_joindiff
|
|
23
|
+
from rowproof.report.html import render_html
|
|
24
|
+
|
|
25
|
+
EXIT_MATCH = 0
|
|
26
|
+
EXIT_DIFFERENT = 1
|
|
27
|
+
EXIT_COULD_NOT_COMPARE = 2
|
|
28
|
+
|
|
29
|
+
# M4 hardening: "Connection retry (3 attempts, backoff) on transient
|
|
30
|
+
# failures; clean exit 2 after." Each connector's own wire module
|
|
31
|
+
# (_pgwire/_chwire/_sfwire) raises its own ConnectionFailedError type for
|
|
32
|
+
# both a genuinely transient failure (network blip, DNS hiccup, timeout)
|
|
33
|
+
# and a permanent one (bad credentials, unknown database) alike -- there's
|
|
34
|
+
# no cheap, reliable way from here to tell those apart without engine-
|
|
35
|
+
# specific error-code inspection, so every connect() failure gets the same
|
|
36
|
+
# short retry rather than none at all. Backoff is deliberately small
|
|
37
|
+
# (0.5s, 1s) so a genuinely permanent failure like a bad password doesn't
|
|
38
|
+
# feel sluggish -- worst case adds ~1.5s before the clean one-line error.
|
|
39
|
+
_CONNECT_RETRY_ATTEMPTS = 3
|
|
40
|
+
_CONNECT_RETRY_BACKOFF_SECONDS = 0.5
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _connect_with_retry(connector, dsn: str) -> None:
|
|
44
|
+
last_error: Exception | None = None
|
|
45
|
+
for attempt in range(1, _CONNECT_RETRY_ATTEMPTS + 1):
|
|
46
|
+
try:
|
|
47
|
+
connector.connect(dsn)
|
|
48
|
+
return
|
|
49
|
+
except Exception as e: # noqa: BLE001 - retried uniformly, see module note above
|
|
50
|
+
last_error = e
|
|
51
|
+
if attempt < _CONNECT_RETRY_ATTEMPTS:
|
|
52
|
+
time.sleep(_CONNECT_RETRY_BACKOFF_SECONDS * attempt)
|
|
53
|
+
raise last_error # noqa: RSE102 - last_error is always set: the loop only exits this way after >=1 failed attempt
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _report_error(e: Exception, verbose: bool, prefix: str = "error") -> None:
|
|
57
|
+
"""spec §10 + M4: "Every error path produces a one-line human message;
|
|
58
|
+
traceback only under --verbose." The one-liner always goes to stderr;
|
|
59
|
+
the traceback (when requested) is appended after it, never instead of
|
|
60
|
+
it, so a --verbose run is strictly more informative, not different.
|
|
61
|
+
"""
|
|
62
|
+
print(f"{prefix}: {e}", file=sys.stderr)
|
|
63
|
+
if verbose:
|
|
64
|
+
traceback.print_exc()
|
|
65
|
+
|
|
66
|
+
def _snowflake_factory():
|
|
67
|
+
# Imported lazily, not at module scope like Postgres/ClickHouse above
|
|
68
|
+
# — spec §3/§9: Snowflake support is an optional extra
|
|
69
|
+
# (`rowproof[snowflake]`) precisely because its driver is a "heavy
|
|
70
|
+
# dependency" a Postgres/ClickHouse-only install shouldn't be forced
|
|
71
|
+
# to carry. An eager top-level import would defeat that: it'd make
|
|
72
|
+
# `snowflake-connector-python` a hard dependency of the whole CLI,
|
|
73
|
+
# breaking every command for a user who installed the base package.
|
|
74
|
+
try:
|
|
75
|
+
from rowproof.connectors.snowflake import SnowflakeConnector
|
|
76
|
+
except ImportError as e:
|
|
77
|
+
raise RowProofError(
|
|
78
|
+
"Snowflake support requires the optional extra -- install with "
|
|
79
|
+
"`pip install rowproof[snowflake]` (or `pipx install rowproof[snowflake]`)"
|
|
80
|
+
) from e
|
|
81
|
+
return SnowflakeConnector()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
_CONNECTOR_FACTORIES = {
|
|
85
|
+
"postgres": PostgresConnector,
|
|
86
|
+
"postgresql": PostgresConnector,
|
|
87
|
+
"clickhouse": ClickHouseConnector,
|
|
88
|
+
"ch": ClickHouseConnector,
|
|
89
|
+
"snowflake": _snowflake_factory,
|
|
90
|
+
"sf": _snowflake_factory,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _make_connector(engine: str, verbose: bool):
|
|
95
|
+
factory = _CONNECTOR_FACTORIES.get(engine)
|
|
96
|
+
if factory is None:
|
|
97
|
+
supported = ", ".join(sorted(set(_CONNECTOR_FACTORIES) - {"postgresql", "ch", "sf"}))
|
|
98
|
+
raise RowProofError(f"unsupported engine '{engine}' (supported: {supported})")
|
|
99
|
+
connector = factory()
|
|
100
|
+
if verbose and hasattr(connector, "on_query"):
|
|
101
|
+
connector.on_query = lambda sql: print(f"[sql] {sql}", file=sys.stderr)
|
|
102
|
+
return connector
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _parse_sample_pct(value: str) -> float:
|
|
106
|
+
"""spec §7: `--sample 1%` (the `%` is optional -- `--sample 1` means
|
|
107
|
+
the same thing)."""
|
|
108
|
+
text = value.strip().rstrip("%").strip()
|
|
109
|
+
try:
|
|
110
|
+
pct = float(text)
|
|
111
|
+
except ValueError as e:
|
|
112
|
+
raise argparse.ArgumentTypeError(f"--sample expects a percentage like '1%%', got {value!r}") from e
|
|
113
|
+
if not (0 < pct <= 100):
|
|
114
|
+
raise argparse.ArgumentTypeError(f"--sample must be between 0 and 100, got {value!r}")
|
|
115
|
+
return pct
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _split_cols(value: str | None) -> list[str] | None:
|
|
119
|
+
if not value:
|
|
120
|
+
return None
|
|
121
|
+
return [c.strip() for c in value.split(",") if c.strip()]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _parse_column_map(value: str | None) -> dict | None:
|
|
125
|
+
"""spec §7: `--column-map a:b,c:d`."""
|
|
126
|
+
if not value:
|
|
127
|
+
return None
|
|
128
|
+
result = {}
|
|
129
|
+
for pair in value.split(","):
|
|
130
|
+
pair = pair.strip()
|
|
131
|
+
if not pair:
|
|
132
|
+
continue
|
|
133
|
+
src, _, tgt = pair.partition(":")
|
|
134
|
+
result[src.strip()] = tgt.strip()
|
|
135
|
+
return result or None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _connect_pair(args, sql_log: list[str] | None = None):
|
|
139
|
+
src_spec = parse_source_spec(args.source)
|
|
140
|
+
tgt_spec = parse_source_spec(args.target)
|
|
141
|
+
source = _make_connector(src_spec.engine, args.verbose)
|
|
142
|
+
target = _make_connector(tgt_spec.engine, args.verbose)
|
|
143
|
+
if sql_log is not None:
|
|
144
|
+
for connector in (source, target):
|
|
145
|
+
if hasattr(connector, "on_query"):
|
|
146
|
+
previous = connector.on_query
|
|
147
|
+
# spec §8.2: JSON output carries every generated SQL
|
|
148
|
+
# statement too — reuse the exact same on_query hook
|
|
149
|
+
# --verbose already installs (chaining onto it, so
|
|
150
|
+
# --verbose logging to stderr keeps working unchanged)
|
|
151
|
+
# rather than a second SQL-capturing mechanism.
|
|
152
|
+
connector.on_query = (
|
|
153
|
+
lambda sql, _prev=previous: (_prev(sql) if _prev else None, sql_log.append(sql))
|
|
154
|
+
)
|
|
155
|
+
_connect_with_retry(source, src_spec.connect_dsn)
|
|
156
|
+
_connect_with_retry(target, tgt_spec.connect_dsn)
|
|
157
|
+
|
|
158
|
+
# spec §7: `--threads N` — extra, already-connected connectors per
|
|
159
|
+
# side for core.hashdiff's parallel segment path (see its own
|
|
160
|
+
# docstring on why genuine parallelism needs genuine extra
|
|
161
|
+
# connections, and why this is still "one [fixed] connection per
|
|
162
|
+
# side" in spirit, not a general pool). `explain` never sets
|
|
163
|
+
# --threads, and `getattr` covers that instead of giving every
|
|
164
|
+
# subcommand's argparser a `--threads` flag it doesn't use.
|
|
165
|
+
threads = getattr(args, "threads", 1)
|
|
166
|
+
source_pool: list = []
|
|
167
|
+
target_pool: list = []
|
|
168
|
+
if threads > 1:
|
|
169
|
+
source_pool = [_make_connector(src_spec.engine, args.verbose) for _ in range(threads)]
|
|
170
|
+
target_pool = [_make_connector(tgt_spec.engine, args.verbose) for _ in range(threads)]
|
|
171
|
+
for connector in source_pool:
|
|
172
|
+
_connect_with_retry(connector, src_spec.connect_dsn)
|
|
173
|
+
for connector in target_pool:
|
|
174
|
+
_connect_with_retry(connector, tgt_spec.connect_dsn)
|
|
175
|
+
|
|
176
|
+
return source, target, src_spec.table_ref, tgt_spec.table_ref, source_pool, target_pool
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def resolve_algorithm(requested: str, args) -> str:
|
|
180
|
+
"""spec §4.2: "Use [joindiff] automatically when both sides resolve to
|
|
181
|
+
the same connection; force with --algorithm joindiff." "same
|
|
182
|
+
connection" is decided from the parsed DSN (engine + host/port/user/
|
|
183
|
+
database), not from table identity — two different tables in the same
|
|
184
|
+
database is exactly the case joindiff is for.
|
|
185
|
+
"""
|
|
186
|
+
if requested != "auto":
|
|
187
|
+
return requested
|
|
188
|
+
src_spec = parse_source_spec(args.source)
|
|
189
|
+
tgt_spec = parse_source_spec(args.target)
|
|
190
|
+
if src_spec.engine == tgt_spec.engine and src_spec.connect_dsn == tgt_spec.connect_dsn:
|
|
191
|
+
return "joindiff"
|
|
192
|
+
return "hashdiff"
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _normalise_kwargs(args) -> dict:
|
|
196
|
+
return dict(
|
|
197
|
+
trim=args.trim,
|
|
198
|
+
case_insensitive=args.case_insensitive,
|
|
199
|
+
float_precision=args.float_precision,
|
|
200
|
+
assume_tz=args.assume_tz,
|
|
201
|
+
column_map=_parse_column_map(args.column_map),
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def cmd_diff(args) -> int:
|
|
206
|
+
source = target = None
|
|
207
|
+
source_pool: list = []
|
|
208
|
+
target_pool: list = []
|
|
209
|
+
try:
|
|
210
|
+
sql_log: list[str] = []
|
|
211
|
+
source, target, source_ref, target_ref, source_pool, target_pool = _connect_pair(args, sql_log)
|
|
212
|
+
key_columns = _split_cols(args.key)
|
|
213
|
+
columns = _split_cols(args.columns)
|
|
214
|
+
exclude = _split_cols(args.exclude)
|
|
215
|
+
algorithm = resolve_algorithm(args.algorithm, args)
|
|
216
|
+
sample = getattr(args, "sample", None)
|
|
217
|
+
sample_rows = getattr(args, "sample_rows", None)
|
|
218
|
+
|
|
219
|
+
if algorithm == "joindiff":
|
|
220
|
+
if sample is not None or sample_rows is not None:
|
|
221
|
+
# spec §4.3: "apply the same sampling predicate to both
|
|
222
|
+
# sides ... then run hashdiff on the sample" -- sampling
|
|
223
|
+
# is defined in terms of hashdiff's segmented approach,
|
|
224
|
+
# not joindiff's single exact query; --algorithm auto
|
|
225
|
+
# would have picked hashdiff already for two different
|
|
226
|
+
# connections, so this only fires when the user forced
|
|
227
|
+
# joindiff (or both sides really are the same connection)
|
|
228
|
+
# while also asking to sample -- a clear error beats
|
|
229
|
+
# silently ignoring the flag.
|
|
230
|
+
raise RowProofError(
|
|
231
|
+
"--sample/--sample-rows requires hashdiff (spec §4.3) -- "
|
|
232
|
+
"pass --algorithm hashdiff, or diff two different connections "
|
|
233
|
+
"so hashdiff is auto-selected"
|
|
234
|
+
)
|
|
235
|
+
result = run_joindiff(
|
|
236
|
+
source, target, source_ref, target_ref,
|
|
237
|
+
key_columns=key_columns, columns=columns, exclude=exclude,
|
|
238
|
+
max_diff_rows=args.max_diff_rows,
|
|
239
|
+
where=args.where, where_source=args.where_source, where_target=args.where_target,
|
|
240
|
+
**_normalise_kwargs(args),
|
|
241
|
+
)
|
|
242
|
+
else:
|
|
243
|
+
result = run_hashdiff(
|
|
244
|
+
source,
|
|
245
|
+
target,
|
|
246
|
+
source_ref,
|
|
247
|
+
target_ref,
|
|
248
|
+
key_columns=key_columns,
|
|
249
|
+
columns=columns,
|
|
250
|
+
exclude=exclude,
|
|
251
|
+
row_threshold=args.row_threshold,
|
|
252
|
+
max_diff_rows=args.max_diff_rows,
|
|
253
|
+
where=args.where,
|
|
254
|
+
where_source=args.where_source,
|
|
255
|
+
where_target=args.where_target,
|
|
256
|
+
threads=args.threads,
|
|
257
|
+
source_pool=source_pool,
|
|
258
|
+
target_pool=target_pool,
|
|
259
|
+
sample=sample,
|
|
260
|
+
sample_rows=sample_rows,
|
|
261
|
+
**_normalise_kwargs(args),
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
outputs = args.output or ["terminal"]
|
|
265
|
+
for fmt in outputs:
|
|
266
|
+
if fmt == "terminal":
|
|
267
|
+
print(render_terminal(result))
|
|
268
|
+
elif fmt == "json":
|
|
269
|
+
text = render_json(result, sql_statements=sql_log)
|
|
270
|
+
if args.json_path:
|
|
271
|
+
with open(args.json_path, "w") as f:
|
|
272
|
+
f.write(text)
|
|
273
|
+
else:
|
|
274
|
+
print(text)
|
|
275
|
+
elif fmt == "html":
|
|
276
|
+
# spec §8.3: DSNs shown in the report must be redacted --
|
|
277
|
+
# render_html does that itself (via cli.spec.redact_dsn),
|
|
278
|
+
# so the *raw* connect_dsn is passed through here, same as
|
|
279
|
+
# what actually connected (re-parsed rather than plumbed
|
|
280
|
+
# out of _connect_pair, since parse_source_spec is pure
|
|
281
|
+
# and cheap, and no other caller needs the DSN back).
|
|
282
|
+
html_text = render_html(
|
|
283
|
+
result,
|
|
284
|
+
source_dsn=parse_source_spec(args.source).connect_dsn,
|
|
285
|
+
target_dsn=parse_source_spec(args.target).connect_dsn,
|
|
286
|
+
sql_statements=sql_log,
|
|
287
|
+
)
|
|
288
|
+
if not args.html_path:
|
|
289
|
+
raise RowProofError("--output html requires --html-path PATH")
|
|
290
|
+
with open(args.html_path, "w", encoding="utf-8") as f:
|
|
291
|
+
f.write(html_text)
|
|
292
|
+
|
|
293
|
+
if args.fail_on == "none":
|
|
294
|
+
return EXIT_MATCH
|
|
295
|
+
if args.fail_on == "count":
|
|
296
|
+
return EXIT_MATCH if result.source_count == result.target_count else EXIT_DIFFERENT
|
|
297
|
+
return result.exit_code()
|
|
298
|
+
|
|
299
|
+
except RowProofError as e:
|
|
300
|
+
_report_error(e, args.verbose)
|
|
301
|
+
return EXIT_COULD_NOT_COMPARE
|
|
302
|
+
except Exception as e: # noqa: BLE001 - CLI boundary: never leak an unhandled traceback
|
|
303
|
+
# NOTE: this used to re-raise when --verbose was set, meaning to
|
|
304
|
+
# show a traceback for debugging. That was wrong: re-raising here
|
|
305
|
+
# doesn't add a traceback to a clean report, it crashes the process
|
|
306
|
+
# — Python's own unhandled-exception handler then prints the
|
|
307
|
+
# traceback AND exits with status 1, not 2. A real network-kill
|
|
308
|
+
# test (tests/integration/test_m0_acceptance.py) caught this doing
|
|
309
|
+
# exactly that. Spec §13 M0's "kill network mid-run" bullet requires
|
|
310
|
+
# exit 2 UNCONDITIONALLY, --verbose or not -- `_report_error` always
|
|
311
|
+
# returns normally (never raises), printing the one-line message
|
|
312
|
+
# unconditionally and the traceback ONLY when --verbose is set
|
|
313
|
+
# (M4: "traceback only under --verbose"), so exit 2 below always
|
|
314
|
+
# still happens either way.
|
|
315
|
+
_report_error(e, args.verbose)
|
|
316
|
+
return EXIT_COULD_NOT_COMPARE
|
|
317
|
+
finally:
|
|
318
|
+
if source is not None:
|
|
319
|
+
source.close()
|
|
320
|
+
if target is not None:
|
|
321
|
+
target.close()
|
|
322
|
+
for connector in (*source_pool, *target_pool):
|
|
323
|
+
connector.close()
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def cmd_explain(args) -> int:
|
|
327
|
+
source = target = None
|
|
328
|
+
try:
|
|
329
|
+
source, target, source_ref, target_ref, _source_pool, _target_pool = _connect_pair(args)
|
|
330
|
+
key_columns = _split_cols(args.key)
|
|
331
|
+
columns = _split_cols(args.columns)
|
|
332
|
+
exclude = _split_cols(args.exclude)
|
|
333
|
+
statements = run_explain(
|
|
334
|
+
source, target, source_ref, target_ref,
|
|
335
|
+
key_columns=key_columns, columns=columns, exclude=exclude,
|
|
336
|
+
where=args.where, where_source=args.where_source, where_target=args.where_target,
|
|
337
|
+
**_normalise_kwargs(args),
|
|
338
|
+
)
|
|
339
|
+
for stmt in statements:
|
|
340
|
+
print(stmt)
|
|
341
|
+
return EXIT_MATCH
|
|
342
|
+
except RowProofError as e:
|
|
343
|
+
_report_error(e, args.verbose)
|
|
344
|
+
return EXIT_COULD_NOT_COMPARE
|
|
345
|
+
except Exception as e: # noqa: BLE001
|
|
346
|
+
_report_error(e, args.verbose)
|
|
347
|
+
return EXIT_COULD_NOT_COMPARE
|
|
348
|
+
finally:
|
|
349
|
+
if source is not None:
|
|
350
|
+
source.close()
|
|
351
|
+
if target is not None:
|
|
352
|
+
target.close()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _run_one_job(job, connections: dict, verbose: bool) -> int:
|
|
356
|
+
"""One table pair from a `run` config — spec §7: "run many table
|
|
357
|
+
pairs; one report." Each job prints its own terminal block (so the
|
|
358
|
+
"one report" is the concatenation, in config order) and the run's
|
|
359
|
+
overall exit code is the worst of every job's own (spec's exit-code
|
|
360
|
+
semantics — 0 match / 1 different / 2 could-not-compare — apply the
|
|
361
|
+
same way per table; "worst" means 2 beats 1 beats 0, the same
|
|
362
|
+
ordering the exit codes already have).
|
|
363
|
+
"""
|
|
364
|
+
source = target = None
|
|
365
|
+
try:
|
|
366
|
+
src_spec = resolve_source_spec(job.source, connections)
|
|
367
|
+
tgt_spec = resolve_source_spec(job.target, connections)
|
|
368
|
+
source = _make_connector(src_spec.engine, verbose)
|
|
369
|
+
target = _make_connector(tgt_spec.engine, verbose)
|
|
370
|
+
_connect_with_retry(source, src_spec.connect_dsn)
|
|
371
|
+
_connect_with_retry(target, tgt_spec.connect_dsn)
|
|
372
|
+
|
|
373
|
+
algorithm = job.algorithm
|
|
374
|
+
if algorithm == "auto":
|
|
375
|
+
algorithm = (
|
|
376
|
+
"joindiff"
|
|
377
|
+
if src_spec.engine == tgt_spec.engine and src_spec.connect_dsn == tgt_spec.connect_dsn
|
|
378
|
+
else "hashdiff"
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
shared_kwargs = dict(
|
|
382
|
+
key_columns=job.key, columns=job.columns, exclude=job.exclude,
|
|
383
|
+
where=job.where, where_source=job.where_source, where_target=job.where_target,
|
|
384
|
+
trim=job.trim, case_insensitive=job.case_insensitive,
|
|
385
|
+
float_precision=job.float_precision, assume_tz=job.assume_tz, column_map=job.column_map,
|
|
386
|
+
)
|
|
387
|
+
if algorithm == "joindiff":
|
|
388
|
+
result = run_joindiff(
|
|
389
|
+
source, target, src_spec.table_ref, tgt_spec.table_ref,
|
|
390
|
+
max_diff_rows=job.max_diff_rows, **shared_kwargs,
|
|
391
|
+
)
|
|
392
|
+
else:
|
|
393
|
+
result = run_hashdiff(
|
|
394
|
+
source, target, src_spec.table_ref, tgt_spec.table_ref,
|
|
395
|
+
row_threshold=job.row_threshold, max_diff_rows=job.max_diff_rows, **shared_kwargs,
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
print(render_terminal(result))
|
|
399
|
+
print()
|
|
400
|
+
if job.fail_on == "none":
|
|
401
|
+
return EXIT_MATCH
|
|
402
|
+
if job.fail_on == "count":
|
|
403
|
+
return EXIT_MATCH if result.source_count == result.target_count else EXIT_DIFFERENT
|
|
404
|
+
return result.exit_code()
|
|
405
|
+
except RowProofError as e:
|
|
406
|
+
_report_error(e, verbose, prefix=f"error ({job.source} -> {job.target})")
|
|
407
|
+
return EXIT_COULD_NOT_COMPARE
|
|
408
|
+
except Exception as e: # noqa: BLE001 - never leak a traceback (see cmd_diff's note)
|
|
409
|
+
_report_error(e, verbose, prefix=f"error ({job.source} -> {job.target})")
|
|
410
|
+
return EXIT_COULD_NOT_COMPARE
|
|
411
|
+
finally:
|
|
412
|
+
if source is not None:
|
|
413
|
+
source.close()
|
|
414
|
+
if target is not None:
|
|
415
|
+
target.close()
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def cmd_run(args) -> int:
|
|
419
|
+
try:
|
|
420
|
+
config = load_config(args.config)
|
|
421
|
+
except RowProofError as e:
|
|
422
|
+
_report_error(e, args.verbose)
|
|
423
|
+
return EXIT_COULD_NOT_COMPARE
|
|
424
|
+
|
|
425
|
+
if not config.tables:
|
|
426
|
+
print("error: config has no 'tables' entries", file=sys.stderr)
|
|
427
|
+
return EXIT_COULD_NOT_COMPARE
|
|
428
|
+
|
|
429
|
+
worst = EXIT_MATCH
|
|
430
|
+
for job in config.tables:
|
|
431
|
+
code = _run_one_job(job, config.connections, args.verbose)
|
|
432
|
+
worst = max(worst, code)
|
|
433
|
+
return worst
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def cmd_connections_test(args) -> int:
|
|
437
|
+
try:
|
|
438
|
+
config = load_config(args.config)
|
|
439
|
+
except RowProofError as e:
|
|
440
|
+
_report_error(e, args.verbose)
|
|
441
|
+
return EXIT_COULD_NOT_COMPARE
|
|
442
|
+
|
|
443
|
+
if args.name not in config.connections:
|
|
444
|
+
print(
|
|
445
|
+
f"error: unknown connection '{args.name}' (known: {', '.join(sorted(config.connections))})",
|
|
446
|
+
file=sys.stderr,
|
|
447
|
+
)
|
|
448
|
+
return EXIT_COULD_NOT_COMPARE
|
|
449
|
+
|
|
450
|
+
dsn = config.connections[args.name]
|
|
451
|
+
engine = dsn.split("://", 1)[0] if "://" in dsn else ""
|
|
452
|
+
connector = None
|
|
453
|
+
try:
|
|
454
|
+
connector = _make_connector(engine, args.verbose)
|
|
455
|
+
_connect_with_retry(connector, dsn)
|
|
456
|
+
connector.query("SELECT 1")
|
|
457
|
+
except RowProofError as e:
|
|
458
|
+
_report_error(e, args.verbose, prefix=f"error: connection '{args.name}' failed")
|
|
459
|
+
return EXIT_COULD_NOT_COMPARE
|
|
460
|
+
except Exception as e: # noqa: BLE001 - never leak a traceback (see cmd_diff's note)
|
|
461
|
+
_report_error(e, args.verbose, prefix=f"error: connection '{args.name}' failed")
|
|
462
|
+
return EXIT_COULD_NOT_COMPARE
|
|
463
|
+
finally:
|
|
464
|
+
if connector is not None:
|
|
465
|
+
connector.close()
|
|
466
|
+
|
|
467
|
+
print(f"{args.name}: ok")
|
|
468
|
+
return EXIT_MATCH
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _add_common_args(p: argparse.ArgumentParser) -> None:
|
|
472
|
+
p.add_argument("source")
|
|
473
|
+
p.add_argument("target")
|
|
474
|
+
p.add_argument("--key", help="comma-separated key columns (default: detected PK)")
|
|
475
|
+
p.add_argument("--columns", help="only compare these columns")
|
|
476
|
+
p.add_argument("--exclude", help="skip these columns")
|
|
477
|
+
p.add_argument("--where", help="SQL filter applied to both sides before comparing")
|
|
478
|
+
p.add_argument("--where-source", help="SQL filter for the source side only (overrides --where)")
|
|
479
|
+
p.add_argument("--where-target", help="SQL filter for the target side only (overrides --where)")
|
|
480
|
+
p.add_argument("--verbose", action="store_true", help="log every generated SQL statement")
|
|
481
|
+
p.add_argument("--trim", action="store_true", help="STR-2: strip trailing whitespace before comparing text")
|
|
482
|
+
p.add_argument("--case-insensitive", action="store_true", help="STR-2: lower-case text before comparing")
|
|
483
|
+
p.add_argument("--float-precision", type=int, default=15, help="FLT-1: significant digits (default 15)")
|
|
484
|
+
p.add_argument("--assume-tz", default="UTC", help="TS-3: timezone assumed for naive timestamps (default UTC)")
|
|
485
|
+
p.add_argument("--column-map", help="rename columns when matching sides: source_col:target_col,...")
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
489
|
+
parser = argparse.ArgumentParser(prog="rowproof")
|
|
490
|
+
parser.add_argument(
|
|
491
|
+
"--version", action="version", version=f"rowproof {rowproof.__version__}",
|
|
492
|
+
)
|
|
493
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
494
|
+
|
|
495
|
+
diff_p = sub.add_parser("diff", help="verify two tables match")
|
|
496
|
+
_add_common_args(diff_p)
|
|
497
|
+
diff_p.add_argument("--algorithm", choices=["auto", "hashdiff", "joindiff"], default="auto")
|
|
498
|
+
diff_p.add_argument(
|
|
499
|
+
"--threads", type=int, default=4,
|
|
500
|
+
help="parallel segment queries per side, hashdiff only (default 4)",
|
|
501
|
+
)
|
|
502
|
+
diff_p.add_argument("--row-threshold", type=int, default=1000)
|
|
503
|
+
diff_p.add_argument("--max-diff-rows", type=int, default=10_000)
|
|
504
|
+
sample_group = diff_p.add_mutually_exclusive_group()
|
|
505
|
+
sample_group.add_argument(
|
|
506
|
+
"--sample", type=_parse_sample_pct, metavar="PCT",
|
|
507
|
+
help="diff a deterministic sample of rows, e.g. --sample 1%% (spec §4.3)",
|
|
508
|
+
)
|
|
509
|
+
sample_group.add_argument(
|
|
510
|
+
"--sample-rows", type=int, metavar="N",
|
|
511
|
+
help="diff a deterministic sample of approximately N rows",
|
|
512
|
+
)
|
|
513
|
+
diff_p.add_argument("--output", action="append", choices=["terminal", "json", "html"], default=None)
|
|
514
|
+
diff_p.add_argument("--json-path")
|
|
515
|
+
diff_p.add_argument("--html-path", help="file to write the HTML sign-off report to (spec §8.3)")
|
|
516
|
+
diff_p.add_argument("--fail-on", choices=["none", "any", "count"], default="any")
|
|
517
|
+
diff_p.set_defaults(func=cmd_diff)
|
|
518
|
+
|
|
519
|
+
explain_p = sub.add_parser("explain", help="print the SQL rowproof would run; execute nothing")
|
|
520
|
+
_add_common_args(explain_p)
|
|
521
|
+
explain_p.set_defaults(func=cmd_explain)
|
|
522
|
+
|
|
523
|
+
run_p = sub.add_parser("run", help="run every table pair in a config file; one report")
|
|
524
|
+
run_p.add_argument("config", help="path to a rowproof YAML config file")
|
|
525
|
+
run_p.add_argument("--verbose", action="store_true", help="log every generated SQL statement")
|
|
526
|
+
run_p.set_defaults(func=cmd_run)
|
|
527
|
+
|
|
528
|
+
connections_p = sub.add_parser("connections", help="work with a config file's named connections")
|
|
529
|
+
connections_sub = connections_p.add_subparsers(dest="connections_command", required=True)
|
|
530
|
+
connections_test_p = connections_sub.add_parser("test", help="check credentials for a named connection")
|
|
531
|
+
connections_test_p.add_argument("name", help="connection name, as declared in the config file")
|
|
532
|
+
connections_test_p.add_argument(
|
|
533
|
+
"--config", default="rowproof.yaml", help="path to a rowproof YAML config file (default: rowproof.yaml)"
|
|
534
|
+
)
|
|
535
|
+
connections_test_p.add_argument("--verbose", action="store_true", help="log every generated SQL statement")
|
|
536
|
+
connections_test_p.set_defaults(func=cmd_connections_test)
|
|
537
|
+
|
|
538
|
+
return parser
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def main(argv: list[str] | None = None) -> int:
|
|
542
|
+
parser = build_parser()
|
|
543
|
+
args = parser.parse_args(argv)
|
|
544
|
+
return args.func(args)
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
if __name__ == "__main__":
|
|
548
|
+
sys.exit(main())
|
rowproof/cli/render.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Terminal + JSON rendering — plain-text stand-in for `rich` (see
|
|
2
|
+
docs/DEV_ENVIRONMENT.md). Behavior/content matches spec §8; only the
|
|
3
|
+
"tables and progress bars" styling is simplified.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
|
|
10
|
+
from rowproof.core.models import DiffResult, TableRef
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _display(value):
|
|
14
|
+
"""A value as shown in row-level output (terminal and JSON): the
|
|
15
|
+
NULL-1 canonical string (spec §6.1: "the literal `\\N`") is the
|
|
16
|
+
internal SQL-comparison marker every normalise_expr wraps a NULL in —
|
|
17
|
+
exactly what hashing/equality needs it to be — but showing a data
|
|
18
|
+
engineer a raw `\\N` in a report reads as a mangled value, not a
|
|
19
|
+
deliberate marker. Row *output* (this function) shows "NULL" instead;
|
|
20
|
+
the underlying RowDiff/DiffResult model, the hash, and the
|
|
21
|
+
IS DISTINCT FROM comparison all keep using the real `\\N` string
|
|
22
|
+
completely unchanged — only this display step ever runs the swap, and
|
|
23
|
+
only on a value that is exactly the marker, never a substring match.
|
|
24
|
+
"""
|
|
25
|
+
if value == "\\N":
|
|
26
|
+
return "NULL"
|
|
27
|
+
return value
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def render_terminal(result: DiffResult) -> str:
|
|
31
|
+
lines = []
|
|
32
|
+
lines.append(f"{result.source.qualified_name} {result.source} -> {result.target}")
|
|
33
|
+
if result.sample_pct is not None:
|
|
34
|
+
# spec §4.3: "Report results as 'of the sampled rows' with the
|
|
35
|
+
# sample size stated. Never silently sample." — on its own line,
|
|
36
|
+
# right under the header, so it can't be missed or mistaken for
|
|
37
|
+
# a full-table result.
|
|
38
|
+
lines.append(f" SAMPLED {result.sample_pct:g}% of rows — counts below are of the sample only")
|
|
39
|
+
lines.append(f" rows {result.source_count:<20} {result.target_count:<20}")
|
|
40
|
+
lines.append(f" key {', '.join(result.key_columns)}")
|
|
41
|
+
if result.excluded_columns:
|
|
42
|
+
lines.append(f" columns {len(result.excluded_columns)} excluded ({', '.join(result.excluded_columns)})")
|
|
43
|
+
lines.append(
|
|
44
|
+
f" algorithm {result.algorithm.value} · {result.segments_examined} segments · "
|
|
45
|
+
f"{result.queries_per_side} queries/side · {result.elapsed_seconds:.1f}s"
|
|
46
|
+
)
|
|
47
|
+
lines.append("")
|
|
48
|
+
lines.append(f" missing in target {result.missing_in_target}")
|
|
49
|
+
lines.append(f" extra in target {result.extra_in_target}")
|
|
50
|
+
lines.append(f" changed {result.changed}")
|
|
51
|
+
|
|
52
|
+
if result.row_diffs:
|
|
53
|
+
lines.append("")
|
|
54
|
+
for rd in result.row_diffs:
|
|
55
|
+
key_str = ",".join(str(k) for k in rd.key)
|
|
56
|
+
if rd.kind == "changed":
|
|
57
|
+
for col, (sv, tv, rule) in rd.changes.items():
|
|
58
|
+
rule_note = f" ({rule.value})" if rule else ""
|
|
59
|
+
lines.append(f" key={key_str} {col} {_display(sv)!r} -> {_display(tv)!r}{rule_note}")
|
|
60
|
+
else:
|
|
61
|
+
lines.append(f" key={key_str} {rd.kind}")
|
|
62
|
+
if result.truncated:
|
|
63
|
+
lines.append(f" ... row list truncated at {len(result.row_diffs)} rows (counts above are exact)")
|
|
64
|
+
|
|
65
|
+
for w in result.warnings:
|
|
66
|
+
lines.append(f" warning: {w.message}")
|
|
67
|
+
|
|
68
|
+
lines.append("")
|
|
69
|
+
verdict = "MATCH" if result.is_match else "DIFFERENT"
|
|
70
|
+
lines.append(f" result {verdict} exit {result.exit_code()}")
|
|
71
|
+
return "\n".join(lines)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def render_json(
|
|
75
|
+
result: DiffResult,
|
|
76
|
+
sql_statements: list[str] | None = None,
|
|
77
|
+
) -> str:
|
|
78
|
+
"""spec §8.2: "Stable schema, versioned (schema_version: 1). Contains
|
|
79
|
+
everything in the terminal output plus every generated SQL statement
|
|
80
|
+
(redacted), timings, and the warnings list."
|
|
81
|
+
|
|
82
|
+
`sql_statements` is the caller's own query log (the CLI already wires
|
|
83
|
+
every connector's `on_query` hook for `--verbose`; JSON output reuses
|
|
84
|
+
exactly that same hook to collect this list rather than adding a
|
|
85
|
+
second SQL-capturing mechanism) — SQL text itself never contains
|
|
86
|
+
connection secrets (those live in the DSN, which no generated
|
|
87
|
+
statement ever embeds), so "redacted" here is automatically satisfied
|
|
88
|
+
by construction, not by a separate scrub step.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
def default(o):
|
|
92
|
+
if hasattr(o, "value"): # Enum
|
|
93
|
+
return o.value
|
|
94
|
+
if isinstance(o, TableRef):
|
|
95
|
+
return {"engine": o.engine, "database": o.database, "schema": o.schema, "table": o.table}
|
|
96
|
+
raise TypeError(f"not JSON serialisable: {o!r}")
|
|
97
|
+
|
|
98
|
+
payload = {
|
|
99
|
+
"schema_version": 1,
|
|
100
|
+
"tool": "rowproof",
|
|
101
|
+
"source": result.source,
|
|
102
|
+
"target": result.target,
|
|
103
|
+
"key_columns": list(result.key_columns),
|
|
104
|
+
"algorithm": result.algorithm,
|
|
105
|
+
"sample_pct": result.sample_pct,
|
|
106
|
+
"source_count": result.source_count,
|
|
107
|
+
"target_count": result.target_count,
|
|
108
|
+
"missing_in_target": result.missing_in_target,
|
|
109
|
+
"extra_in_target": result.extra_in_target,
|
|
110
|
+
"changed": result.changed,
|
|
111
|
+
"truncated": result.truncated,
|
|
112
|
+
"is_match": result.is_match,
|
|
113
|
+
"exit_code": result.exit_code(),
|
|
114
|
+
"excluded_columns": list(result.excluded_columns),
|
|
115
|
+
"segments_examined": result.segments_examined,
|
|
116
|
+
"queries_per_side": result.queries_per_side,
|
|
117
|
+
"timings": {"total_seconds": round(result.elapsed_seconds, 3)},
|
|
118
|
+
"sql_statements": list(sql_statements or []),
|
|
119
|
+
"warnings": [w.message for w in result.warnings],
|
|
120
|
+
"row_diffs": [
|
|
121
|
+
{
|
|
122
|
+
"key": list(rd.key),
|
|
123
|
+
"kind": rd.kind,
|
|
124
|
+
"changes": {
|
|
125
|
+
col: {"source": _display(sv), "target": _display(tv), "rule": rule.value if rule else None}
|
|
126
|
+
for col, (sv, tv, rule) in rd.changes.items()
|
|
127
|
+
},
|
|
128
|
+
}
|
|
129
|
+
for rd in result.row_diffs
|
|
130
|
+
],
|
|
131
|
+
}
|
|
132
|
+
return json.dumps(payload, default=default, indent=2)
|