table-validator 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- table_validator/__init__.py +46 -0
- table_validator/auth/__init__.py +1 -0
- table_validator/auth/azure_auth.py +52 -0
- table_validator/auth/databricks_auth.py +31 -0
- table_validator/cli/__init__.py +1 -0
- table_validator/cli/main.py +722 -0
- table_validator/cli/partition_prompt.py +78 -0
- table_validator/cli/summary_table.py +146 -0
- table_validator/cli/wizard.py +429 -0
- table_validator/config/__init__.py +1 -0
- table_validator/config/manager.py +84 -0
- table_validator/config/schema.py +179 -0
- table_validator/connectors/__init__.py +1 -0
- table_validator/connectors/azure_connector.py +809 -0
- table_validator/connectors/databricks_connector.py +1230 -0
- table_validator/engine/__init__.py +1 -0
- table_validator/engine/comparison_engine.py +645 -0
- table_validator/models.py +952 -0
- table_validator/reports/__init__.py +1 -0
- table_validator/reports/excel_report.py +953 -0
- table_validator/validators/__init__.py +1 -0
- table_validator/validators/blob_discovery.py +467 -0
- table_validator/validators/catalog_validator.py +1863 -0
- table_validator/validators/row_validator.py +1727 -0
- table_validator-0.1.0.dist-info/METADATA +190 -0
- table_validator-0.1.0.dist-info/RECORD +30 -0
- table_validator-0.1.0.dist-info/WHEEL +5 -0
- table_validator-0.1.0.dist-info/entry_points.txt +2 -0
- table_validator-0.1.0.dist-info/licenses/LICENSE +21 -0
- table_validator-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,722 @@
|
|
|
1
|
+
"""CLI entry point: `tablevalidator` console script."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import platform
|
|
6
|
+
import subprocess
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Dict, Optional
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from table_validator.auth.azure_auth import get_azure_credential
|
|
13
|
+
from table_validator.auth.databricks_auth import get_databricks_token
|
|
14
|
+
from table_validator.cli.summary_table import (
|
|
15
|
+
print_summary_table,
|
|
16
|
+
summary_from_excel,
|
|
17
|
+
summary_from_response,
|
|
18
|
+
)
|
|
19
|
+
from table_validator.cli.partition_prompt import build_partition_prompt
|
|
20
|
+
from table_validator.cli.wizard import run_configure_wizard
|
|
21
|
+
from table_validator.config.manager import CONFIG_PATH, ConfigNotFoundError, require_config
|
|
22
|
+
from table_validator.config.schema import SourceType, ValidationType, ValidatorConfig
|
|
23
|
+
from table_validator.connectors.azure_connector import AzureConnector, AzureSqlConnector
|
|
24
|
+
from table_validator.connectors.databricks_connector import DatabricksConnector
|
|
25
|
+
from table_validator.models import (
|
|
26
|
+
AzureSqlValidationRequest,
|
|
27
|
+
CatalogValidationRequest,
|
|
28
|
+
CatalogValidationResponse,
|
|
29
|
+
DataCompareMode,
|
|
30
|
+
ValidationStatus,
|
|
31
|
+
ValidationTier,
|
|
32
|
+
)
|
|
33
|
+
from table_validator.reports.excel_report import generate_excel_report
|
|
34
|
+
from table_validator.validators.blob_discovery import BlobCatalogValidator
|
|
35
|
+
from table_validator.validators.catalog_validator import CatalogValidator
|
|
36
|
+
from table_validator.validators.row_validator import AzureSqlValidator
|
|
37
|
+
|
|
38
|
+
app = typer.Typer(
|
|
39
|
+
name="tablevalidator",
|
|
40
|
+
help="Validate data migrations between Azure and Databricks Delta Lake.",
|
|
41
|
+
no_args_is_help=True,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
logger = logging.getLogger(__name__)
|
|
45
|
+
|
|
46
|
+
DEFAULT_OUTPUT_PATH = "validation_report.xlsx"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _configure_logging(verbose: bool, quiet: bool) -> None:
|
|
50
|
+
"""
|
|
51
|
+
Print progress to the console as validation runs, instead of going
|
|
52
|
+
silent for the entire duration of a large catalog-wide run (the
|
|
53
|
+
validators already log per-schema/per-table progress via `logging`;
|
|
54
|
+
without this, none of it reaches the console and a multi-minute run
|
|
55
|
+
looks indistinguishable from a hang).
|
|
56
|
+
|
|
57
|
+
Default: plain "Validating table 'x.y' ..." progress lines only.
|
|
58
|
+
--verbose: also show the detailed [row-hash]/statistics diagnostic
|
|
59
|
+
logs the validators already emit at INFO level.
|
|
60
|
+
--quiet: suppress progress logging entirely (only the final summary
|
|
61
|
+
prints) - useful for CI/scripted runs where log noise is unwanted.
|
|
62
|
+
"""
|
|
63
|
+
logger = logging.getLogger("table_validator")
|
|
64
|
+
logger.propagate = False
|
|
65
|
+
|
|
66
|
+
if quiet:
|
|
67
|
+
# No handler at all (not even a raised level) - otherwise Python's
|
|
68
|
+
# logging module falls back to its own stderr "lastResort" handler
|
|
69
|
+
# for any record with no handler in its logger chain, which would
|
|
70
|
+
# leak WARNING+ messages (e.g. missing/extra table notices)
|
|
71
|
+
# despite --quiet asking for none of that.
|
|
72
|
+
logger.addHandler(logging.NullHandler())
|
|
73
|
+
logger.setLevel(logging.CRITICAL + 1)
|
|
74
|
+
return
|
|
75
|
+
|
|
76
|
+
handler = logging.StreamHandler()
|
|
77
|
+
handler.setFormatter(logging.Formatter("%(message)s"))
|
|
78
|
+
|
|
79
|
+
logger.setLevel(logging.INFO)
|
|
80
|
+
logger.addHandler(handler)
|
|
81
|
+
|
|
82
|
+
if not verbose:
|
|
83
|
+
# Quiet the noisier tagged diagnostic logs by default; the plain
|
|
84
|
+
# per-table progress line added in _validate_table still comes
|
|
85
|
+
# through since it's logged at INFO on the same logger tree -
|
|
86
|
+
# only these specific detailed diagnostics are filtered out.
|
|
87
|
+
_VERBOSE_ONLY_TAGS = ("[row-hash]", "[compare_data]", "[data-mismatch]")
|
|
88
|
+
|
|
89
|
+
class _HideVerboseDiagnostics(logging.Filter):
|
|
90
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
91
|
+
message = record.getMessage()
|
|
92
|
+
return not any(tag in message for tag in _VERBOSE_ONLY_TAGS)
|
|
93
|
+
|
|
94
|
+
handler.addFilter(_HideVerboseDiagnostics())
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@app.command()
|
|
98
|
+
def info() -> None:
|
|
99
|
+
"""Show what this tool does, which platforms it supports, and what
|
|
100
|
+
each command is for - a quick orientation for first-time use."""
|
|
101
|
+
typer.echo(
|
|
102
|
+
"\n"
|
|
103
|
+
"tablevalidator - data migration validator\n"
|
|
104
|
+
"------------------------------------------\n"
|
|
105
|
+
"Compares a source table against a target Databricks table/catalog "
|
|
106
|
+
"and reports whether the migration is correct: matching schema, "
|
|
107
|
+
"row counts, statistics, and (where a difference is found) the "
|
|
108
|
+
"exact row/column that changed.\n"
|
|
109
|
+
"\n"
|
|
110
|
+
"Supported sources (target is always Databricks):\n"
|
|
111
|
+
" - Databricks catalog -> Databricks catalog\n"
|
|
112
|
+
" - Azure Blob Storage -> Databricks catalog\n"
|
|
113
|
+
" - Azure SQL Database -> Databricks catalog\n"
|
|
114
|
+
"\n"
|
|
115
|
+
"Typical workflow, in order:\n"
|
|
116
|
+
"\n"
|
|
117
|
+
" 1. tablevalidator configure\n"
|
|
118
|
+
" Interactively set up credentials and which source/target "
|
|
119
|
+
"table(s) to compare. Run this first, and again any time you "
|
|
120
|
+
"need to change settings (source type, tables, primary key, "
|
|
121
|
+
"which validations to run).\n"
|
|
122
|
+
"\n"
|
|
123
|
+
" 2. tablevalidator validate\n"
|
|
124
|
+
" Runs the comparison using the saved configuration and "
|
|
125
|
+
"writes an Excel report (validation_report.xlsx by default). "
|
|
126
|
+
"Prints a pass/fail summary in the terminal when it finishes.\n"
|
|
127
|
+
"\n"
|
|
128
|
+
" 3. tablevalidator open\n"
|
|
129
|
+
" Opens the most recently generated report in your default "
|
|
130
|
+
"spreadsheet app (Excel/LibreOffice/etc.), so you can inspect "
|
|
131
|
+
"exactly what passed, failed, or mismatched.\n"
|
|
132
|
+
"\n"
|
|
133
|
+
"Other commands:\n"
|
|
134
|
+
" tablevalidator report Print the summary table from an "
|
|
135
|
+
"existing report without opening it.\n"
|
|
136
|
+
"\n"
|
|
137
|
+
"Run 'tablevalidator <command> --help' for a command's full "
|
|
138
|
+
"options (e.g. tablevalidator validate --help)."
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@app.command()
|
|
143
|
+
def configure() -> None:
|
|
144
|
+
"""Interactively configure Azure and Databricks credentials."""
|
|
145
|
+
run_configure_wizard()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@app.command()
|
|
149
|
+
def validate(
|
|
150
|
+
config_path: Path = typer.Option(
|
|
151
|
+
CONFIG_PATH,
|
|
152
|
+
"--config-path",
|
|
153
|
+
help="Path to config.yaml (default: ~/.table_validator/config.yaml).",
|
|
154
|
+
),
|
|
155
|
+
output: Path = typer.Option(
|
|
156
|
+
DEFAULT_OUTPUT_PATH,
|
|
157
|
+
"--output",
|
|
158
|
+
help="Path to write the Excel validation report to.",
|
|
159
|
+
),
|
|
160
|
+
verbose: bool = typer.Option(
|
|
161
|
+
False,
|
|
162
|
+
"--verbose",
|
|
163
|
+
help="Show detailed per-table row-hash/statistics diagnostic logs.",
|
|
164
|
+
),
|
|
165
|
+
quiet: bool = typer.Option(
|
|
166
|
+
False,
|
|
167
|
+
"--quiet",
|
|
168
|
+
help="Suppress progress logging entirely (only the final summary prints).",
|
|
169
|
+
),
|
|
170
|
+
mode: str = typer.Option(
|
|
171
|
+
"full",
|
|
172
|
+
"--mode",
|
|
173
|
+
help=(
|
|
174
|
+
"Databricks-to-Databricks only: how far the tiered fail-fast "
|
|
175
|
+
"funnel is allowed to go. 'stats' stops after row count/null/"
|
|
176
|
+
"distinct/min-max statistics (never runs a fingerprint or "
|
|
177
|
+
"row-hash comparison, even on a match). 'full' (default) lets "
|
|
178
|
+
"the funnel continue through the whole-table fingerprint and, "
|
|
179
|
+
"if that disagrees, row-hash/column-level diff - but only if "
|
|
180
|
+
"cheaper tiers couldn't already prove the tables equal or "
|
|
181
|
+
"different. Ignored for azure_blob/azure_sql source types."
|
|
182
|
+
),
|
|
183
|
+
),
|
|
184
|
+
yes: bool = typer.Option(
|
|
185
|
+
False,
|
|
186
|
+
"--yes",
|
|
187
|
+
help=(
|
|
188
|
+
"Never prompt interactively. A large table (Databricks-to-"
|
|
189
|
+
"Databricks only) with a confirmed mismatch is normally "
|
|
190
|
+
"offered a partition-column choice before row-hash "
|
|
191
|
+
"comparison; --yes skips that prompt and always compares the "
|
|
192
|
+
"whole table unpartitioned instead. Use for CI/non-interactive "
|
|
193
|
+
"runs - a run with no attached terminal already skips the "
|
|
194
|
+
"prompt on its own, but --yes makes that explicit."
|
|
195
|
+
),
|
|
196
|
+
),
|
|
197
|
+
) -> None:
|
|
198
|
+
"""Run validation and produce an Excel validation report."""
|
|
199
|
+
|
|
200
|
+
if mode not in ("stats", "full"):
|
|
201
|
+
typer.secho(f"Invalid --mode '{mode}' - must be 'stats' or 'full'.", fg=typer.colors.RED)
|
|
202
|
+
raise typer.Exit(code=1)
|
|
203
|
+
|
|
204
|
+
# Print progress as validation runs (which table it's on, etc.) -
|
|
205
|
+
# without this, a large catalog-wide run goes completely silent
|
|
206
|
+
# until the final summary, indistinguishable from a hang.
|
|
207
|
+
_configure_logging(verbose=verbose, quiet=quiet)
|
|
208
|
+
|
|
209
|
+
# ------------------------------------------------------------------
|
|
210
|
+
# 1. Load config. require_config() is the single place that decides
|
|
211
|
+
# "config is missing" - anything else (e.g. the wizard's own
|
|
212
|
+
# load_config() pre-populate call) is a different, legitimate use of
|
|
213
|
+
# a missing file and must not duplicate this check.
|
|
214
|
+
# ------------------------------------------------------------------
|
|
215
|
+
try:
|
|
216
|
+
config = require_config(config_path)
|
|
217
|
+
except ConfigNotFoundError as exc:
|
|
218
|
+
typer.secho(str(exc), fg=typer.colors.RED)
|
|
219
|
+
raise typer.Exit(code=1)
|
|
220
|
+
|
|
221
|
+
missing = _missing_config_fields(config)
|
|
222
|
+
if missing:
|
|
223
|
+
typer.secho(
|
|
224
|
+
"Config is incomplete - missing: " + ", ".join(missing) + ". "
|
|
225
|
+
"Run 'tablevalidator configure' to fill these in.",
|
|
226
|
+
fg=typer.colors.RED,
|
|
227
|
+
)
|
|
228
|
+
raise typer.Exit(code=1)
|
|
229
|
+
|
|
230
|
+
# ------------------------------------------------------------------
|
|
231
|
+
# 2. Load secrets from ~/.table_validator/.env via the auth
|
|
232
|
+
# abstraction, and 3. build the Databricks connector from them - every
|
|
233
|
+
# source_type targets Databricks, so this connector is always needed.
|
|
234
|
+
# ------------------------------------------------------------------
|
|
235
|
+
token = get_databricks_token(config)
|
|
236
|
+
if not token:
|
|
237
|
+
typer.secho(
|
|
238
|
+
"No Databricks token found in ~/.table_validator/.env. "
|
|
239
|
+
"Run 'tablevalidator configure' first.",
|
|
240
|
+
fg=typer.colors.RED,
|
|
241
|
+
)
|
|
242
|
+
raise typer.Exit(code=1)
|
|
243
|
+
|
|
244
|
+
try:
|
|
245
|
+
databricks = DatabricksConnector(
|
|
246
|
+
host=_host_from_workspace_url(config.databricks.workspace_url),
|
|
247
|
+
token=token,
|
|
248
|
+
http_path=config.databricks.http_path,
|
|
249
|
+
)
|
|
250
|
+
except ValueError as exc:
|
|
251
|
+
typer.secho(f"Configuration error: {exc}", fg=typer.colors.RED)
|
|
252
|
+
raise typer.Exit(code=1)
|
|
253
|
+
|
|
254
|
+
azure_credential = get_azure_credential(config)
|
|
255
|
+
|
|
256
|
+
scope_desc = _describe_scope(config)
|
|
257
|
+
typer.echo(f"Validating ({config.source_type.value}) {scope_desc} ...")
|
|
258
|
+
|
|
259
|
+
# ------------------------------------------------------------------
|
|
260
|
+
# 4/5/6. Determine scope and run the source-type-specific comparison
|
|
261
|
+
# path. Each branch resolves its own connector(s) and calls its own
|
|
262
|
+
# validator, but all three converge on the same CatalogValidationResponse
|
|
263
|
+
# shape, so report generation and summary printing (7/8) are identical
|
|
264
|
+
# regardless of source_type.
|
|
265
|
+
# ------------------------------------------------------------------
|
|
266
|
+
try:
|
|
267
|
+
if config.source_type == SourceType.AZURE_BLOB:
|
|
268
|
+
result = _run_blob_validation(config, azure_credential, databricks)
|
|
269
|
+
elif config.source_type == SourceType.AZURE_SQL:
|
|
270
|
+
result = _run_sql_validation(config, azure_credential, databricks)
|
|
271
|
+
else:
|
|
272
|
+
result = _run_databricks_validation(config, databricks, mode=mode, yes=yes)
|
|
273
|
+
except ValueError as exc:
|
|
274
|
+
typer.secho(f"Configuration error: {exc}", fg=typer.colors.RED)
|
|
275
|
+
raise typer.Exit(code=1)
|
|
276
|
+
|
|
277
|
+
# ------------------------------------------------------------------
|
|
278
|
+
# 7. Generate the Excel report (one row per table across every
|
|
279
|
+
# matched schema, already aggregated onto a single
|
|
280
|
+
# CatalogValidationResponse). generate_excel_report itself doesn't
|
|
281
|
+
# create the parent directory (openpyxl's wb.save() requires it to
|
|
282
|
+
# already exist), so --output pointing at a not-yet-created directory
|
|
283
|
+
# is handled here rather than failing with a raw FileNotFoundError.
|
|
284
|
+
# ------------------------------------------------------------------
|
|
285
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
286
|
+
# enabled_validations only actually gates behavior for source_type ==
|
|
287
|
+
# databricks today (CatalogValidator) - azure_blob/azure_sql always
|
|
288
|
+
# run their full fixed pipeline regardless of config.validations, so
|
|
289
|
+
# filtering their report would hide data that was genuinely computed.
|
|
290
|
+
report_enabled_validations = (
|
|
291
|
+
{v.value for v in config.validations}
|
|
292
|
+
if config.source_type == SourceType.DATABRICKS
|
|
293
|
+
else None
|
|
294
|
+
)
|
|
295
|
+
generate_excel_report(
|
|
296
|
+
result, str(output),
|
|
297
|
+
source_type=config.source_type.value,
|
|
298
|
+
enabled_validations=report_enabled_validations,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
# ------------------------------------------------------------------
|
|
302
|
+
# 8. Print a summary - per-table PASS/FAIL lines, then the same
|
|
303
|
+
# aggregate summary table used by `tablevalidator report` and by the
|
|
304
|
+
# Excel report's own Summary sheet (built from the same
|
|
305
|
+
# _build_summary_metrics() call), so the two never disagree.
|
|
306
|
+
# ------------------------------------------------------------------
|
|
307
|
+
_print_summary(result, config, output)
|
|
308
|
+
|
|
309
|
+
validations_run_str = (
|
|
310
|
+
", ".join(sorted(report_enabled_validations))
|
|
311
|
+
if report_enabled_validations is not None
|
|
312
|
+
else None
|
|
313
|
+
)
|
|
314
|
+
print_summary_table(
|
|
315
|
+
summary_from_response(result, config.source_type.value, validations_run_str)
|
|
316
|
+
)
|
|
317
|
+
typer.echo(f"Report written to: {output.resolve()}")
|
|
318
|
+
typer.echo("Run 'tablevalidator open' to open it now.")
|
|
319
|
+
|
|
320
|
+
if result.status in (ValidationStatus.FAIL, ValidationStatus.ERROR):
|
|
321
|
+
raise typer.Exit(code=1)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
@app.command()
|
|
325
|
+
def report(
|
|
326
|
+
path: Optional[Path] = typer.Option(
|
|
327
|
+
None,
|
|
328
|
+
"--path",
|
|
329
|
+
help=(
|
|
330
|
+
"Path to a validation_report.xlsx to read. Defaults to "
|
|
331
|
+
"./validation_report.xlsx (the same default 'validate' writes to)."
|
|
332
|
+
),
|
|
333
|
+
),
|
|
334
|
+
) -> None:
|
|
335
|
+
"""Print the summary table from the most recently generated (or a
|
|
336
|
+
specified) validation report."""
|
|
337
|
+
report_path = (path or Path(DEFAULT_OUTPUT_PATH)).resolve()
|
|
338
|
+
|
|
339
|
+
if not report_path.exists():
|
|
340
|
+
typer.secho(
|
|
341
|
+
"No validation report found. Run 'tablevalidator validate' first.",
|
|
342
|
+
fg=typer.colors.RED,
|
|
343
|
+
)
|
|
344
|
+
raise typer.Exit(code=1)
|
|
345
|
+
|
|
346
|
+
try:
|
|
347
|
+
data = summary_from_excel(report_path)
|
|
348
|
+
except Exception as exc:
|
|
349
|
+
typer.secho(f"Unable to read report at {report_path}: {exc}", fg=typer.colors.RED)
|
|
350
|
+
raise typer.Exit(code=1)
|
|
351
|
+
|
|
352
|
+
print_summary_table(data)
|
|
353
|
+
# str(report_path) on Windows already uses backslashes consistently
|
|
354
|
+
# (Path.resolve() normalizes separators for the current OS), so this
|
|
355
|
+
# prints an absolute, OS-native path that's easy to Ctrl+click or
|
|
356
|
+
# copy straight into Explorer/another terminal.
|
|
357
|
+
typer.echo(f"Report file: {report_path}")
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
@app.command(name="open")
|
|
361
|
+
def open_report_command(
|
|
362
|
+
path: Optional[Path] = typer.Option(
|
|
363
|
+
None,
|
|
364
|
+
"--path",
|
|
365
|
+
help=(
|
|
366
|
+
"Path to a validation_report.xlsx to open. Defaults to "
|
|
367
|
+
"./validation_report.xlsx (the same default 'validate' writes to)."
|
|
368
|
+
),
|
|
369
|
+
),
|
|
370
|
+
) -> None:
|
|
371
|
+
"""Open the most recently generated (or a specified) validation
|
|
372
|
+
report in the OS's default application (Excel/LibreOffice/etc.)."""
|
|
373
|
+
report_path = (path or Path(DEFAULT_OUTPUT_PATH)).resolve()
|
|
374
|
+
|
|
375
|
+
if not report_path.exists():
|
|
376
|
+
typer.secho(
|
|
377
|
+
"No validation report found. Run 'tablevalidator validate' first.",
|
|
378
|
+
fg=typer.colors.RED,
|
|
379
|
+
)
|
|
380
|
+
raise typer.Exit(code=1)
|
|
381
|
+
|
|
382
|
+
_open_in_default_app(report_path)
|
|
383
|
+
typer.echo(f"Opening: {report_path}")
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
# ---------------------------------------------------------------------------
|
|
387
|
+
# Source-type-specific comparison paths
|
|
388
|
+
# ---------------------------------------------------------------------------
|
|
389
|
+
def _run_databricks_validation(
|
|
390
|
+
config: ValidatorConfig,
|
|
391
|
+
databricks: DatabricksConnector,
|
|
392
|
+
mode: str = "full",
|
|
393
|
+
yes: bool = False,
|
|
394
|
+
) -> CatalogValidationResponse:
|
|
395
|
+
"""source_type == databricks: source is another Databricks catalog.
|
|
396
|
+
A blank schema/table on EITHER side means "compare everything
|
|
397
|
+
matching" for that level - CatalogValidator already performs the
|
|
398
|
+
list-and-intersect discovery internally whenever request.schemas/
|
|
399
|
+
tables is left unrestricted (None), so leaving the restriction off
|
|
400
|
+
IS the discovery trigger; there is no separate discovery step to call."""
|
|
401
|
+
schemas_restriction = None
|
|
402
|
+
if config.source_table.schema_name and config.target_table.schema_name:
|
|
403
|
+
schemas_restriction = [config.target_table.schema_name]
|
|
404
|
+
|
|
405
|
+
tables_restriction = None
|
|
406
|
+
if config.source_table.table and config.target_table.table:
|
|
407
|
+
tables_restriction = [config.target_table.table]
|
|
408
|
+
|
|
409
|
+
# A configured primary key only applies to the single named table
|
|
410
|
+
# (not a catalog-wide sweep) - CatalogValidator looks it up by
|
|
411
|
+
# "schema.table" first, falling back to a bare table name, so provide
|
|
412
|
+
# both forms when a schema is known; falls back to row-number matching
|
|
413
|
+
# as before when unset.
|
|
414
|
+
primary_keys: Dict[str, list] = {}
|
|
415
|
+
if config.primary_key and config.target_table.table:
|
|
416
|
+
primary_keys[config.target_table.table] = config.primary_key
|
|
417
|
+
if config.target_table.schema_name:
|
|
418
|
+
key = f"{config.target_table.schema_name}.{config.target_table.table}"
|
|
419
|
+
primary_keys[key] = config.primary_key
|
|
420
|
+
|
|
421
|
+
max_tier = ValidationTier.STATISTICAL if mode == "stats" else ValidationTier.COLUMN_DIFF
|
|
422
|
+
|
|
423
|
+
request = CatalogValidationRequest(
|
|
424
|
+
source_catalog=config.source_table.catalog or "",
|
|
425
|
+
target_catalog=config.target_table.catalog or "",
|
|
426
|
+
schemas=schemas_restriction,
|
|
427
|
+
tables=tables_restriction,
|
|
428
|
+
enabled_validations=set(config.validations),
|
|
429
|
+
primary_keys=primary_keys,
|
|
430
|
+
max_tier=max_tier,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
partition_prompt = build_partition_prompt(yes=yes)
|
|
434
|
+
validator = CatalogValidator(databricks, partition_prompt=partition_prompt)
|
|
435
|
+
return validator.compare_catalogs(request)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _run_blob_validation(
|
|
439
|
+
config: ValidatorConfig,
|
|
440
|
+
azure_credential,
|
|
441
|
+
databricks: DatabricksConnector,
|
|
442
|
+
) -> CatalogValidationResponse:
|
|
443
|
+
"""source_type == azure_blob: discover blobs matching folder_prefix/
|
|
444
|
+
file_pattern, match them to Databricks tables by inferred filename,
|
|
445
|
+
and compare each matched pair (row count + column name/type only -
|
|
446
|
+
see validators/blob_discovery.py for why row-hash comparison is out
|
|
447
|
+
of scope for this multi-blob-match path)."""
|
|
448
|
+
if not azure_credential.storage_account_key:
|
|
449
|
+
raise ValueError(
|
|
450
|
+
"No Azure Storage account key found in ~/.table_validator/.env. "
|
|
451
|
+
"Run 'tablevalidator configure' first."
|
|
452
|
+
)
|
|
453
|
+
if not config.azure.storage_account or not config.blob_source.container:
|
|
454
|
+
raise ValueError(
|
|
455
|
+
"azure.storage_account and blob_source.container are required "
|
|
456
|
+
"for an Azure Blob source. Run 'tablevalidator configure' first."
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
azure = AzureConnector(
|
|
460
|
+
account_name=config.azure.storage_account,
|
|
461
|
+
account_key=azure_credential.storage_account_key,
|
|
462
|
+
container_name=config.blob_source.container,
|
|
463
|
+
)
|
|
464
|
+
|
|
465
|
+
validator = BlobCatalogValidator(azure, databricks)
|
|
466
|
+
return validator.validate(
|
|
467
|
+
target_catalog=config.target_table.catalog or "",
|
|
468
|
+
# None (not "") when left blank - BlobCatalogValidator treats
|
|
469
|
+
# None as "match blobs against every schema in the catalog".
|
|
470
|
+
target_schema=config.target_table.schema_name,
|
|
471
|
+
folder_prefix=config.blob_source.folder_prefix,
|
|
472
|
+
file_pattern=config.blob_source.file_pattern,
|
|
473
|
+
# If BOTH an exact blob and target table are named, compare that
|
|
474
|
+
# pair directly - bypasses filename-to-table discovery entirely,
|
|
475
|
+
# even if the names don't match.
|
|
476
|
+
blob_path=config.blob_source.blob_path,
|
|
477
|
+
target_table=config.target_table.table,
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _run_sql_validation(
|
|
482
|
+
config: ValidatorConfig,
|
|
483
|
+
azure_credential,
|
|
484
|
+
databricks: DatabricksConnector,
|
|
485
|
+
) -> CatalogValidationResponse:
|
|
486
|
+
"""source_type == azure_sql: structurally identical to the Databricks
|
|
487
|
+
catalog-to-catalog path (schema discovery, table discovery, row/
|
|
488
|
+
column comparison) - AzureSqlValidator already implements this
|
|
489
|
+
against AzureSqlConnector's schema/table listing methods, so it's
|
|
490
|
+
reused directly rather than duplicated."""
|
|
491
|
+
if not azure_credential.sql_username or not azure_credential.sql_password:
|
|
492
|
+
raise ValueError(
|
|
493
|
+
"No Azure SQL username/password found in ~/.table_validator/.env. "
|
|
494
|
+
"Run 'tablevalidator configure' first."
|
|
495
|
+
)
|
|
496
|
+
if not config.azure.sql_server or not config.azure.sql_database:
|
|
497
|
+
raise ValueError(
|
|
498
|
+
"azure.sql_server and azure.sql_database are required for an "
|
|
499
|
+
"Azure SQL source. Run 'tablevalidator configure' first."
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
azure_sql = AzureSqlConnector(
|
|
503
|
+
server=config.azure.sql_server,
|
|
504
|
+
database=config.azure.sql_database,
|
|
505
|
+
username=azure_credential.sql_username,
|
|
506
|
+
password=azure_credential.sql_password,
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
# If the user explicitly named BOTH a source (SQL) and target
|
|
510
|
+
# (Databricks) schema/table, compare that exact pair directly -
|
|
511
|
+
# schema_map/table_map bypass name-based matching entirely, so the
|
|
512
|
+
# two sides don't need to share a name (e.g. SQL's 'dbo' vs a
|
|
513
|
+
# purpose-named Databricks schema). schemas/tables restrict by the
|
|
514
|
+
# SOURCE (Azure SQL) name, since that's what _compare_schemas/
|
|
515
|
+
# _compare_tables filter against.
|
|
516
|
+
schema_map: Dict[str, str] = {}
|
|
517
|
+
schemas_restriction = None
|
|
518
|
+
if config.sql_source.schema_name and config.target_table.schema_name:
|
|
519
|
+
schema_map = {config.sql_source.schema_name: config.target_table.schema_name}
|
|
520
|
+
schemas_restriction = [config.sql_source.schema_name]
|
|
521
|
+
|
|
522
|
+
table_map: Dict[str, str] = {}
|
|
523
|
+
tables_restriction = None
|
|
524
|
+
if config.sql_source.table and config.target_table.table:
|
|
525
|
+
table_map = {config.sql_source.table: config.target_table.table}
|
|
526
|
+
tables_restriction = [config.sql_source.table]
|
|
527
|
+
|
|
528
|
+
# Mirrors _run_databricks_validation's primary_keys construction: a
|
|
529
|
+
# configured key only applies to the single named table (not a
|
|
530
|
+
# catalog-wide sweep), looked up by "schema.table" first, falling
|
|
531
|
+
# back to a bare table name. Without this, config.primary_key was
|
|
532
|
+
# silently dropped and Data Mismatches could never populate even
|
|
533
|
+
# when FULL mode found real mismatches.
|
|
534
|
+
primary_keys: Dict[str, list] = {}
|
|
535
|
+
if config.primary_key and config.target_table.table:
|
|
536
|
+
primary_keys[config.target_table.table] = config.primary_key
|
|
537
|
+
if config.target_table.schema_name:
|
|
538
|
+
key = f"{config.target_table.schema_name}.{config.target_table.table}"
|
|
539
|
+
primary_keys[key] = config.primary_key
|
|
540
|
+
|
|
541
|
+
# FULL mode is what actually populates Data Mismatches/row-level
|
|
542
|
+
# detail (see AzureSqlValidator's `mode == DataCompareMode.FULL and
|
|
543
|
+
# mismatch_count > 0 and not using_row_number_fallback` gate) - the
|
|
544
|
+
# model's own default (STATISTICS) never triggers that path, so this
|
|
545
|
+
# was silently unreachable from the CLI before.
|
|
546
|
+
request = AzureSqlValidationRequest(
|
|
547
|
+
target_catalog=config.target_table.catalog or "",
|
|
548
|
+
schemas=schemas_restriction,
|
|
549
|
+
schema_map=schema_map,
|
|
550
|
+
tables=tables_restriction,
|
|
551
|
+
table_map=table_map,
|
|
552
|
+
primary_keys=primary_keys,
|
|
553
|
+
data_compare_mode=DataCompareMode.FULL,
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
validator = AzureSqlValidator(azure_sql, databricks)
|
|
557
|
+
return validator.validate(request)
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
# ---------------------------------------------------------------------------
|
|
561
|
+
# Helpers
|
|
562
|
+
# ---------------------------------------------------------------------------
|
|
563
|
+
def _missing_config_fields(config: ValidatorConfig) -> list:
|
|
564
|
+
"""
|
|
565
|
+
Only the Databricks connection, the target catalog, and whatever the
|
|
566
|
+
selected source_type hard-requires are checked here. schema_name/
|
|
567
|
+
table being blank (on either source or target) is intentionally
|
|
568
|
+
optional everywhere (Phase 4 Part 2): it means "compare everything
|
|
569
|
+
matching" rather than "config incomplete".
|
|
570
|
+
"""
|
|
571
|
+
missing = []
|
|
572
|
+
if not config.databricks.workspace_url:
|
|
573
|
+
missing.append("databricks.workspace_url")
|
|
574
|
+
if not config.databricks.http_path:
|
|
575
|
+
missing.append("databricks.http_path")
|
|
576
|
+
if not config.target_table.catalog:
|
|
577
|
+
missing.append("target_table.catalog")
|
|
578
|
+
|
|
579
|
+
if config.source_type == SourceType.AZURE_BLOB:
|
|
580
|
+
if not config.azure.storage_account:
|
|
581
|
+
missing.append("azure.storage_account")
|
|
582
|
+
if not config.blob_source.container:
|
|
583
|
+
missing.append("blob_source.container")
|
|
584
|
+
elif config.source_type == SourceType.AZURE_SQL:
|
|
585
|
+
if not config.azure.sql_server:
|
|
586
|
+
missing.append("azure.sql_server")
|
|
587
|
+
if not config.azure.sql_database:
|
|
588
|
+
missing.append("azure.sql_database")
|
|
589
|
+
else:
|
|
590
|
+
if not config.source_table.catalog:
|
|
591
|
+
missing.append("source_table.catalog")
|
|
592
|
+
|
|
593
|
+
return missing
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _host_from_workspace_url(workspace_url: Optional[str]) -> Optional[str]:
|
|
597
|
+
"""DatabricksConnector wants a bare hostname; the wizard stores a full
|
|
598
|
+
https:// workspace URL, so strip the scheme and any trailing path."""
|
|
599
|
+
if not workspace_url:
|
|
600
|
+
return None
|
|
601
|
+
host = workspace_url.replace("https://", "").replace("http://", "")
|
|
602
|
+
return host.split("/")[0]
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _open_in_default_app(path: Path) -> None:
|
|
606
|
+
"""
|
|
607
|
+
Launch `path` in whatever application the OS has associated with its
|
|
608
|
+
file extension (Excel/LibreOffice/etc. for .xlsx) - so a user running
|
|
609
|
+
this as an installed package gets the report opened for them instead
|
|
610
|
+
of having to go find the file themselves. Best-effort only: any
|
|
611
|
+
failure (no GUI, no associated app, sandboxed environment) is logged
|
|
612
|
+
and swallowed rather than failing the whole `validate` run, since the
|
|
613
|
+
report was already written successfully at this point.
|
|
614
|
+
"""
|
|
615
|
+
try:
|
|
616
|
+
system = platform.system()
|
|
617
|
+
if system == "Windows":
|
|
618
|
+
os.startfile(str(path)) # type: ignore[attr-defined]
|
|
619
|
+
elif system == "Darwin":
|
|
620
|
+
subprocess.run(["open", str(path)], check=True)
|
|
621
|
+
else:
|
|
622
|
+
subprocess.run(["xdg-open", str(path)], check=True)
|
|
623
|
+
except Exception as exc:
|
|
624
|
+
logger.debug("Could not auto-open report at %s: %s", path, exc)
|
|
625
|
+
typer.echo(
|
|
626
|
+
f"(Could not open the report automatically - open it manually: {path})"
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _describe_scope(config: ValidatorConfig) -> str:
|
|
631
|
+
"""Human-readable description of what's being compared, for the
|
|
632
|
+
'Validating ...' progress line - reflects blank schema/table as
|
|
633
|
+
'all schemas'/'all tables' rather than printing an empty segment."""
|
|
634
|
+
|
|
635
|
+
def _side(ref) -> str:
|
|
636
|
+
catalog = ref.catalog or "?"
|
|
637
|
+
if not ref.schema_name:
|
|
638
|
+
return f"{catalog} (all schemas)"
|
|
639
|
+
if not ref.table:
|
|
640
|
+
return f"{catalog}.{ref.schema_name} (all tables)"
|
|
641
|
+
return f"{catalog}.{ref.schema_name}.{ref.table}"
|
|
642
|
+
|
|
643
|
+
target_desc = _side(config.target_table)
|
|
644
|
+
|
|
645
|
+
if config.source_type == SourceType.AZURE_BLOB:
|
|
646
|
+
container = config.blob_source.container or "?"
|
|
647
|
+
scope_bits = []
|
|
648
|
+
if config.blob_source.folder_prefix:
|
|
649
|
+
scope_bits.append(config.blob_source.folder_prefix)
|
|
650
|
+
if config.blob_source.file_pattern:
|
|
651
|
+
scope_bits.append(config.blob_source.file_pattern)
|
|
652
|
+
scope_desc = f" ({', '.join(scope_bits)})" if scope_bits else ""
|
|
653
|
+
source_desc = f"blob:{container}{scope_desc}"
|
|
654
|
+
elif config.source_type == SourceType.AZURE_SQL:
|
|
655
|
+
database = config.azure.sql_database or "?"
|
|
656
|
+
if not config.sql_source.schema_name:
|
|
657
|
+
source_desc = f"sql:{database} (all schemas)"
|
|
658
|
+
elif not config.sql_source.table:
|
|
659
|
+
source_desc = f"sql:{database}.{config.sql_source.schema_name} (all tables)"
|
|
660
|
+
else:
|
|
661
|
+
source_desc = f"sql:{database}.{config.sql_source.schema_name}.{config.sql_source.table}"
|
|
662
|
+
else:
|
|
663
|
+
source_desc = _side(config.source_table)
|
|
664
|
+
|
|
665
|
+
return f"{source_desc} -> {target_desc}"
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def _print_summary(result, config: ValidatorConfig, output: Path) -> None:
|
|
669
|
+
total = passed = failed = errors = skipped = 0
|
|
670
|
+
per_table_lines = []
|
|
671
|
+
|
|
672
|
+
for schema in result.schemas:
|
|
673
|
+
for table in schema.tables:
|
|
674
|
+
total += 1
|
|
675
|
+
if table.status == ValidationStatus.PASS:
|
|
676
|
+
passed += 1
|
|
677
|
+
elif table.status == ValidationStatus.ERROR:
|
|
678
|
+
errors += 1
|
|
679
|
+
elif table.status == ValidationStatus.SKIPPED:
|
|
680
|
+
skipped += 1
|
|
681
|
+
else:
|
|
682
|
+
failed += 1
|
|
683
|
+
per_table_lines.append(
|
|
684
|
+
(f"{schema.schema_name}.{table.table}", table.status)
|
|
685
|
+
)
|
|
686
|
+
|
|
687
|
+
status_color = {
|
|
688
|
+
ValidationStatus.PASS: typer.colors.GREEN,
|
|
689
|
+
ValidationStatus.FAIL: typer.colors.RED,
|
|
690
|
+
ValidationStatus.ERROR: typer.colors.YELLOW,
|
|
691
|
+
ValidationStatus.SKIPPED: typer.colors.WHITE,
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
typer.echo("")
|
|
695
|
+
if total > 1:
|
|
696
|
+
typer.echo("Per-table results:")
|
|
697
|
+
for name, status in per_table_lines:
|
|
698
|
+
typer.secho(
|
|
699
|
+
f" {name}: {status.value}",
|
|
700
|
+
fg=status_color.get(status, typer.colors.WHITE),
|
|
701
|
+
)
|
|
702
|
+
typer.echo("")
|
|
703
|
+
|
|
704
|
+
overall_color = status_color.get(result.status, typer.colors.WHITE)
|
|
705
|
+
typer.secho(f"Overall status: {result.status.value}", fg=overall_color, bold=True)
|
|
706
|
+
typer.echo(f"Tables: {total} total, {passed} passed, {failed} failed, "
|
|
707
|
+
f"{errors} error, {skipped} skipped")
|
|
708
|
+
if result.error:
|
|
709
|
+
typer.secho(f"Error: {result.error}", fg=typer.colors.RED)
|
|
710
|
+
|
|
711
|
+
enabled = ", ".join(v.value for v in config.validations) or "none"
|
|
712
|
+
typer.echo(f"Validations requested: {enabled}")
|
|
713
|
+
if ValidationType.ROW not in config.validations and config.source_type != SourceType.DATABRICKS:
|
|
714
|
+
typer.echo(
|
|
715
|
+
"Note: for this source type, row-level comparison always runs "
|
|
716
|
+
"when a table exists (it cannot be disabled independently in "
|
|
717
|
+
"that pipeline yet); 'row' being unselected is informational only."
|
|
718
|
+
)
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
if __name__ == "__main__":
|
|
722
|
+
app()
|