tablevalidator-databricks 0.1.1__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/PKG-INFO +15 -6
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/README.md +14 -5
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/templates/widget_notebook.py.tmpl +103 -25
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/PKG-INFO +15 -6
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/scm_version.json +2 -2
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tests/test_generated_notebook_execution.py +186 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/pyproject.toml +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/setup.cfg +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/__init__.py +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/cli/__init__.py +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/cli/main.py +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/generator/__init__.py +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/generator/generator.py +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/SOURCES.txt +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/dependency_links.txt +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/entry_points.txt +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/requires.txt +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/scm_file_list.json +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/top_level.txt +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tests/test_cli.py +0 -0
- {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tests/test_generator.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablevalidator-databricks
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
|
|
5
5
|
License: MIT
|
|
6
6
|
Requires-Python: >=3.9
|
|
@@ -45,11 +45,12 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
|
|
|
45
45
|
tablevalidator-databricks init --output MyValidation.py --mode full
|
|
46
46
|
```
|
|
47
47
|
|
|
48
|
-
- `--mode basic` - only the check-group
|
|
48
|
+
- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
|
|
49
49
|
- `--mode full` (default) - also shows optional filter widgets (only/
|
|
50
|
-
ignore columns, row filter, primary key).
|
|
51
|
-
- `--mode schema` -
|
|
52
|
-
|
|
50
|
+
ignore columns, row filter, primary key). Every check defaults to "Yes".
|
|
51
|
+
- `--mode schema` - same widgets as `basic`, but Column/Row default to
|
|
52
|
+
"No" - a lightweight schema-shape-only comparison out of the box (still
|
|
53
|
+
changeable per-run).
|
|
53
54
|
|
|
54
55
|
Then, in your Databricks workspace: **Workspace -> Import**, format
|
|
55
56
|
"Source", and select the generated file. Databricks recognizes the
|
|
@@ -62,7 +63,7 @@ markers and opens it as a real notebook.
|
|
|
62
63
|
2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
|
|
63
64
|
`SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
|
|
64
65
|
populate Source/Target Catalog, Schema, and Table dropdowns.
|
|
65
|
-
3. Shows a
|
|
66
|
+
3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
|
|
66
67
|
Row - the three independently-selectable check groups the engine
|
|
67
68
|
actually supports) plus, in `full` mode, optional text widgets for
|
|
68
69
|
column filtering, a row filter, and a primary key.
|
|
@@ -86,6 +87,14 @@ isn't showing up (a permissions lag, sort order, anything). It's used
|
|
|
86
87
|
as-is, not checked against the live catalog listing. Leave every override
|
|
87
88
|
blank to just use the dropdowns as normal.
|
|
88
89
|
|
|
90
|
+
**Validate every table in a schema, not just one**: pick
|
|
91
|
+
"(all tables in schema)" as the Table dropdown (Source and/or Target
|
|
92
|
+
independently) to compare every identically-named table in that schema in
|
|
93
|
+
one run - the same schema-wide sweep `validate_tables()` itself supports.
|
|
94
|
+
A primary key can't be set in this mode (leave the Primary Key widget
|
|
95
|
+
blank). Comparing every schema in a catalog isn't supported yet - Schema
|
|
96
|
+
must be a specific choice, even when Table is left as "(all ... in ...)".
|
|
97
|
+
|
|
89
98
|
## Why a separate package
|
|
90
99
|
|
|
91
100
|
`table-validator` stays focused on the actual comparison engine (CLI +
|
|
@@ -31,11 +31,12 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
|
|
|
31
31
|
tablevalidator-databricks init --output MyValidation.py --mode full
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
-
- `--mode basic` - only the check-group
|
|
34
|
+
- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
|
|
35
35
|
- `--mode full` (default) - also shows optional filter widgets (only/
|
|
36
|
-
ignore columns, row filter, primary key).
|
|
37
|
-
- `--mode schema` -
|
|
38
|
-
|
|
36
|
+
ignore columns, row filter, primary key). Every check defaults to "Yes".
|
|
37
|
+
- `--mode schema` - same widgets as `basic`, but Column/Row default to
|
|
38
|
+
"No" - a lightweight schema-shape-only comparison out of the box (still
|
|
39
|
+
changeable per-run).
|
|
39
40
|
|
|
40
41
|
Then, in your Databricks workspace: **Workspace -> Import**, format
|
|
41
42
|
"Source", and select the generated file. Databricks recognizes the
|
|
@@ -48,7 +49,7 @@ markers and opens it as a real notebook.
|
|
|
48
49
|
2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
|
|
49
50
|
`SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
|
|
50
51
|
populate Source/Target Catalog, Schema, and Table dropdowns.
|
|
51
|
-
3. Shows a
|
|
52
|
+
3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
|
|
52
53
|
Row - the three independently-selectable check groups the engine
|
|
53
54
|
actually supports) plus, in `full` mode, optional text widgets for
|
|
54
55
|
column filtering, a row filter, and a primary key.
|
|
@@ -72,6 +73,14 @@ isn't showing up (a permissions lag, sort order, anything). It's used
|
|
|
72
73
|
as-is, not checked against the live catalog listing. Leave every override
|
|
73
74
|
blank to just use the dropdowns as normal.
|
|
74
75
|
|
|
76
|
+
**Validate every table in a schema, not just one**: pick
|
|
77
|
+
"(all tables in schema)" as the Table dropdown (Source and/or Target
|
|
78
|
+
independently) to compare every identically-named table in that schema in
|
|
79
|
+
one run - the same schema-wide sweep `validate_tables()` itself supports.
|
|
80
|
+
A primary key can't be set in this mode (leave the Primary Key widget
|
|
81
|
+
blank). Comparing every schema in a catalog isn't supported yet - Schema
|
|
82
|
+
must be a specific choice, even when Table is left as "(all ... in ...)".
|
|
83
|
+
|
|
75
84
|
## Why a separate package
|
|
76
85
|
|
|
77
86
|
`table-validator` stays focused on the actual comparison engine (CLI +
|
|
@@ -22,6 +22,12 @@
|
|
|
22
22
|
# MAGIC something isn't showing up (a permissions lag, a name that doesn't sort
|
|
23
23
|
# MAGIC into view, etc.). Leave every override blank to just use the dropdowns
|
|
24
24
|
# MAGIC as normal.
|
|
25
|
+
# MAGIC
|
|
26
|
+
# MAGIC **Want to validate every table in a schema instead of just one?** Pick
|
|
27
|
+
# MAGIC "(all tables in schema)" as the Table (both Source and Target) - every
|
|
28
|
+
# MAGIC identically-named table in that schema is compared in one run. A
|
|
29
|
+
# MAGIC primary key can't be set in this mode (a single key can't apply to
|
|
30
|
+
# MAGIC every table) - leave the Primary Key widget blank if you use this.
|
|
25
31
|
|
|
26
32
|
# COMMAND ----------
|
|
27
33
|
|
|
@@ -66,6 +72,30 @@ def _current_or_first(options, current):
|
|
|
66
72
|
return current
|
|
67
73
|
return options[0] if options else ""
|
|
68
74
|
|
|
75
|
+
|
|
76
|
+
def _current_or_first_real(options, current, sentinel):
|
|
77
|
+
"""Like _current_or_first, but when falling back (no valid current
|
|
78
|
+
selection), prefers the first REAL option over `sentinel` if one
|
|
79
|
+
exists - so a brand-new notebook defaults to an actual Schema/Table
|
|
80
|
+
rather than immediately landing on "(all ... in ...)". The sentinel
|
|
81
|
+
stays fully selectable (and sticky once picked) via the current-
|
|
82
|
+
value check above; this only changes what a first-time default
|
|
83
|
+
lands on."""
|
|
84
|
+
if current in options:
|
|
85
|
+
return current
|
|
86
|
+
real_options = [o for o in options if o != sentinel]
|
|
87
|
+
return real_options[0] if real_options else (options[0] if options else "")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# Sentinel choices meaning "leave this blank" - selecting one of these
|
|
91
|
+
# skips the level(s) below it and triggers validate_tables()'s own
|
|
92
|
+
# schema-wide sweep mode (comparing every identically-named table)
|
|
93
|
+
# instead of a single table pair. Real Databricks catalog/schema/table
|
|
94
|
+
# names can't contain parentheses, so these can never collide with a
|
|
95
|
+
# genuine name.
|
|
96
|
+
_ALL_SCHEMAS = "(all schemas in catalog)"
|
|
97
|
+
_ALL_TABLES = "(all tables in schema)"
|
|
98
|
+
|
|
69
99
|
# COMMAND ----------
|
|
70
100
|
|
|
71
101
|
# MAGIC %md ### Create / refresh widgets
|
|
@@ -103,18 +133,34 @@ def _build_cascading_widgets(prefix: str, label_prefix: str) -> None:
|
|
|
103
133
|
now, not a stale snapshot from an earlier run. Also creates a
|
|
104
134
|
"(manual override)" text widget next to each dropdown, for a name
|
|
105
135
|
that isn't showing up in the dropdown for any reason - see
|
|
106
|
-
_effective_value() below for how the override takes priority.
|
|
136
|
+
_effective_value() below for how the override takes priority.
|
|
137
|
+
|
|
138
|
+
Schema and Table each also get an "(all ... in ...)" sentinel choice
|
|
139
|
+
as their first option, for a schema-wide sweep (leave Table blank to
|
|
140
|
+
compare every identically-named table in the schema) or a catalog-
|
|
141
|
+
wide sweep (leave Schema, and therefore Table too, blank to compare
|
|
142
|
+
every identically-named table across every schema) - see
|
|
143
|
+
_ALL_SCHEMAS/_ALL_TABLES above and how the "Run validation" cell
|
|
144
|
+
below builds validate_tables()'s "catalog.schema" (no table) /
|
|
145
|
+
"catalog" (no schema) sweep-mode strings from this choice."""
|
|
107
146
|
catalog_current = _current_or_first(catalogs, _effective_value(f"{prefix}_catalog"))
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
147
|
+
|
|
148
|
+
schema_choices = [_ALL_SCHEMAS] + _list_schemas(catalog_current)
|
|
149
|
+
schema_current = _current_or_first_real(schema_choices, _effective_value(f"{prefix}_schema"), _ALL_SCHEMAS)
|
|
150
|
+
sweeping_schema = schema_current == _ALL_SCHEMAS
|
|
151
|
+
|
|
152
|
+
if sweeping_schema:
|
|
153
|
+
table_choices = [_ALL_TABLES]
|
|
154
|
+
table_current = _ALL_TABLES
|
|
155
|
+
else:
|
|
156
|
+
table_choices = [_ALL_TABLES] + _list_tables(catalog_current, schema_current)
|
|
157
|
+
table_current = _current_or_first_real(table_choices, _effective_value(f"{prefix}_table"), _ALL_TABLES)
|
|
112
158
|
|
|
113
159
|
dbutils.widgets.dropdown(f"{prefix}_catalog", catalog_current, catalogs or [""], f"{label_prefix} Catalog")
|
|
114
160
|
dbutils.widgets.text(f"{prefix}_catalog_override", "", f"{label_prefix} Catalog (manual override)")
|
|
115
|
-
dbutils.widgets.dropdown(f"{prefix}_schema", schema_current,
|
|
161
|
+
dbutils.widgets.dropdown(f"{prefix}_schema", schema_current, schema_choices, f"{label_prefix} Schema")
|
|
116
162
|
dbutils.widgets.text(f"{prefix}_schema_override", "", f"{label_prefix} Schema (manual override)")
|
|
117
|
-
dbutils.widgets.dropdown(f"{prefix}_table", table_current,
|
|
163
|
+
dbutils.widgets.dropdown(f"{prefix}_table", table_current, table_choices, f"{label_prefix} Table")
|
|
118
164
|
dbutils.widgets.text(f"{prefix}_table_override", "", f"{label_prefix} Table (manual override)")
|
|
119
165
|
|
|
120
166
|
|
|
@@ -134,11 +180,20 @@ print(
|
|
|
134
180
|
|
|
135
181
|
# COMMAND ----------
|
|
136
182
|
|
|
137
|
-
|
|
138
|
-
dbutils.widgets.multiselect
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
)
|
|
183
|
+
# One Yes/No dropdown per check group, instead of a single multiselect -
|
|
184
|
+
# dbutils.widgets.multiselect enforces an internal "selection sequence"
|
|
185
|
+
# validation against its own default that was observed to reject even a
|
|
186
|
+
# same-length, same-items default on some Databricks runtimes/existing-
|
|
187
|
+
# widget states (DefaultValueNotInChoicesList), with no reliable
|
|
188
|
+
# workaround. Three independent dropdowns have no such cross-widget
|
|
189
|
+
# validation and are the more robust primitive for this.
|
|
190
|
+
_YES_NO = ["Yes", "No"]
|
|
191
|
+
_schema_default = "Yes"
|
|
192
|
+
_column_default = "No" if MODE == "schema" else "Yes"
|
|
193
|
+
_row_default = "No" if MODE == "schema" else "Yes"
|
|
194
|
+
dbutils.widgets.dropdown("check_catalog_schema", _schema_default, _YES_NO, "Run Catalog & Schema checks?")
|
|
195
|
+
dbutils.widgets.dropdown("check_column", _column_default, _YES_NO, "Run Column checks?")
|
|
196
|
+
dbutils.widgets.dropdown("check_row", _row_default, _YES_NO, "Run Row checks?")
|
|
142
197
|
|
|
143
198
|
if SHOW_OPTIONAL_WIDGETS:
|
|
144
199
|
dbutils.widgets.text("only_columns", "", "Only compare these columns (comma-separated, optional)")
|
|
@@ -161,14 +216,15 @@ def _validate_current_selection(prefix: str, label: str) -> None:
|
|
|
161
216
|
manual override set is trusted as typed and skipped here - the whole
|
|
162
217
|
point of the override is to name something the live catalog listing
|
|
163
218
|
doesn't (currently) show, so it can never pass this check by
|
|
164
|
-
definition.
|
|
219
|
+
definition. _ALL_SCHEMAS/_ALL_TABLES (leave-blank sweep sentinels)
|
|
220
|
+
are always valid and skip the corresponding check."""
|
|
165
221
|
catalog = _effective_value(f"{prefix}_catalog")
|
|
166
222
|
schema_overridden = bool(_get_widget(f"{prefix}_schema_override").strip())
|
|
167
223
|
table_overridden = bool(_get_widget(f"{prefix}_table_override").strip())
|
|
168
224
|
schema = _effective_value(f"{prefix}_schema")
|
|
169
225
|
table = _effective_value(f"{prefix}_table")
|
|
170
226
|
|
|
171
|
-
if not schema_overridden:
|
|
227
|
+
if not schema_overridden and schema != _ALL_SCHEMAS:
|
|
172
228
|
live_schemas = _list_schemas(catalog)
|
|
173
229
|
if schema not in live_schemas:
|
|
174
230
|
raise ValueError(
|
|
@@ -181,7 +237,7 @@ def _validate_current_selection(prefix: str, label: str) -> None:
|
|
|
181
237
|
"the dropdown, type it into the Schema manual override "
|
|
182
238
|
"widget instead.)"
|
|
183
239
|
)
|
|
184
|
-
if not table_overridden:
|
|
240
|
+
if not table_overridden and schema != _ALL_SCHEMAS and table != _ALL_TABLES:
|
|
185
241
|
live_tables = _list_tables(catalog, schema)
|
|
186
242
|
if table not in live_tables:
|
|
187
243
|
raise ValueError(
|
|
@@ -198,18 +254,40 @@ def _validate_current_selection(prefix: str, label: str) -> None:
|
|
|
198
254
|
_validate_current_selection("source", "Source")
|
|
199
255
|
_validate_current_selection("target", "Target")
|
|
200
256
|
|
|
201
|
-
source = f"{_effective_value('source_catalog')}.{_effective_value('source_schema')}.{_effective_value('source_table')}"
|
|
202
|
-
target = f"{_effective_value('target_catalog')}.{_effective_value('target_schema')}.{_effective_value('target_table')}"
|
|
203
257
|
|
|
204
|
-
|
|
205
|
-
"
|
|
206
|
-
"
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
258
|
+
def _source_or_target_string(prefix: str) -> str:
|
|
259
|
+
"""Builds the "catalog.schema.table" (single table) or
|
|
260
|
+
"catalog.schema" (schema-wide sweep, Schema chosen but Table left as
|
|
261
|
+
_ALL_TABLES) string validate_tables() expects. Catalog-wide sweeps
|
|
262
|
+
(Schema also left blank) aren't supported by validate_tables() today
|
|
263
|
+
- _ALL_SCHEMAS is caught with a clear error below instead of being
|
|
264
|
+
silently passed through as a literal, broken source/target string."""
|
|
265
|
+
catalog = _effective_value(f"{prefix}_catalog")
|
|
266
|
+
schema = _effective_value(f"{prefix}_schema")
|
|
267
|
+
table = _effective_value(f"{prefix}_table")
|
|
268
|
+
|
|
269
|
+
if schema == _ALL_SCHEMAS:
|
|
270
|
+
raise ValueError(
|
|
271
|
+
f"{prefix.capitalize()} Schema is set to {_ALL_SCHEMAS!r} - "
|
|
272
|
+
"comparing every schema in a catalog isn't supported yet. "
|
|
273
|
+
"Pick a specific Schema (Table can still be left as "
|
|
274
|
+
f"{_ALL_TABLES!r} to sweep every table within it)."
|
|
275
|
+
)
|
|
276
|
+
if table == _ALL_TABLES:
|
|
277
|
+
return f"{catalog}.{schema}"
|
|
278
|
+
return f"{catalog}.{schema}.{table}"
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
source = _source_or_target_string("source")
|
|
282
|
+
target = _source_or_target_string("target")
|
|
283
|
+
|
|
210
284
|
enabled_validations = set()
|
|
211
|
-
|
|
212
|
-
enabled_validations |=
|
|
285
|
+
if dbutils.widgets.get("check_catalog_schema") == "Yes":
|
|
286
|
+
enabled_validations |= {ValidationType.CATALOG, ValidationType.SCHEMA}
|
|
287
|
+
if dbutils.widgets.get("check_column") == "Yes":
|
|
288
|
+
enabled_validations |= {ValidationType.COLUMN}
|
|
289
|
+
if dbutils.widgets.get("check_row") == "Yes":
|
|
290
|
+
enabled_validations |= {ValidationType.ROW}
|
|
213
291
|
if not enabled_validations:
|
|
214
292
|
enabled_validations = None # fall back to validate_tables()'s own "run everything" default
|
|
215
293
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablevalidator-databricks
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
|
|
5
5
|
License: MIT
|
|
6
6
|
Requires-Python: >=3.9
|
|
@@ -45,11 +45,12 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
|
|
|
45
45
|
tablevalidator-databricks init --output MyValidation.py --mode full
|
|
46
46
|
```
|
|
47
47
|
|
|
48
|
-
- `--mode basic` - only the check-group
|
|
48
|
+
- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
|
|
49
49
|
- `--mode full` (default) - also shows optional filter widgets (only/
|
|
50
|
-
ignore columns, row filter, primary key).
|
|
51
|
-
- `--mode schema` -
|
|
52
|
-
|
|
50
|
+
ignore columns, row filter, primary key). Every check defaults to "Yes".
|
|
51
|
+
- `--mode schema` - same widgets as `basic`, but Column/Row default to
|
|
52
|
+
"No" - a lightweight schema-shape-only comparison out of the box (still
|
|
53
|
+
changeable per-run).
|
|
53
54
|
|
|
54
55
|
Then, in your Databricks workspace: **Workspace -> Import**, format
|
|
55
56
|
"Source", and select the generated file. Databricks recognizes the
|
|
@@ -62,7 +63,7 @@ markers and opens it as a real notebook.
|
|
|
62
63
|
2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
|
|
63
64
|
`SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
|
|
64
65
|
populate Source/Target Catalog, Schema, and Table dropdowns.
|
|
65
|
-
3. Shows a
|
|
66
|
+
3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
|
|
66
67
|
Row - the three independently-selectable check groups the engine
|
|
67
68
|
actually supports) plus, in `full` mode, optional text widgets for
|
|
68
69
|
column filtering, a row filter, and a primary key.
|
|
@@ -86,6 +87,14 @@ isn't showing up (a permissions lag, sort order, anything). It's used
|
|
|
86
87
|
as-is, not checked against the live catalog listing. Leave every override
|
|
87
88
|
blank to just use the dropdowns as normal.
|
|
88
89
|
|
|
90
|
+
**Validate every table in a schema, not just one**: pick
|
|
91
|
+
"(all tables in schema)" as the Table dropdown (Source and/or Target
|
|
92
|
+
independently) to compare every identically-named table in that schema in
|
|
93
|
+
one run - the same schema-wide sweep `validate_tables()` itself supports.
|
|
94
|
+
A primary key can't be set in this mode (leave the Primary Key widget
|
|
95
|
+
blank). Comparing every schema in a catalog isn't supported yet - Schema
|
|
96
|
+
must be a specific choice, even when Table is left as "(all ... in ...)".
|
|
97
|
+
|
|
89
98
|
## Why a separate package
|
|
90
99
|
|
|
91
100
|
`table-validator` stays focused on the actual comparison engine (CLI +
|
|
@@ -15,6 +15,7 @@ from unittest.mock import MagicMock
|
|
|
15
15
|
|
|
16
16
|
import pytest
|
|
17
17
|
|
|
18
|
+
from table_validator.config.schema import ValidationType
|
|
18
19
|
from tablevalidator_databricks.generator.generator import generate_notebook
|
|
19
20
|
|
|
20
21
|
|
|
@@ -29,9 +30,15 @@ def _code_from_notebook(path: Path) -> str:
|
|
|
29
30
|
return "\n".join(code_lines)
|
|
30
31
|
|
|
31
32
|
|
|
33
|
+
class _DefaultValueNotInChoicesList(Exception):
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
32
37
|
class _FakeWidgets:
|
|
33
38
|
def __init__(self, initial=None):
|
|
34
39
|
self._values = dict(initial or {})
|
|
40
|
+
self.dropdown_choices = {}
|
|
41
|
+
self.multiselect_choices = {}
|
|
35
42
|
|
|
36
43
|
def get(self, name):
|
|
37
44
|
return self._values.get(name, "")
|
|
@@ -43,9 +50,25 @@ class _FakeWidgets:
|
|
|
43
50
|
# Real dbutils.widgets.dropdown does NOT overwrite an existing
|
|
44
51
|
# value just because the cell re-ran with a different `default` -
|
|
45
52
|
# it only sets it the first time. Match that.
|
|
53
|
+
if default not in choices:
|
|
54
|
+
raise _DefaultValueNotInChoicesList(f"{name}: {default!r} not in {choices!r}")
|
|
55
|
+
self.dropdown_choices[name] = choices
|
|
46
56
|
self._values.setdefault(name, default)
|
|
47
57
|
|
|
48
58
|
def multiselect(self, name, default, choices, label):
|
|
59
|
+
# Real Databricks dbutils.widgets.multiselect requires the
|
|
60
|
+
# default's comma-separated items to all be present in choices -
|
|
61
|
+
# observed in production to reject a narrower/reordered subset
|
|
62
|
+
# with "DefaultValueNotInChoicesList: Selection sequence must
|
|
63
|
+
# include <choices>" even when every individual item IS a valid
|
|
64
|
+
# choice. Replicate that constraint here so this suite would have
|
|
65
|
+
# caught the real bug (a "schema"-mode default of just
|
|
66
|
+
# "Catalog & Schema" against choices=["Catalog & Schema",
|
|
67
|
+
# "Column", "Row"]) before it ever reached a live workspace.
|
|
68
|
+
default_items = default.split(",") if default else []
|
|
69
|
+
if any(item not in choices for item in default_items):
|
|
70
|
+
raise _DefaultValueNotInChoicesList(f"{name}: {default!r} not a subset of {choices!r}")
|
|
71
|
+
self.multiselect_choices[name] = choices
|
|
49
72
|
self._values.setdefault(name, default)
|
|
50
73
|
|
|
51
74
|
|
|
@@ -229,3 +252,166 @@ def test_catalog_override_drives_schema_and_table_lookup(notebook_path):
|
|
|
229
252
|
)
|
|
230
253
|
|
|
231
254
|
assert captured["source"] == "hidden_catalog.sales.orders"
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
# ---------------------------------------------------------------------------
|
|
258
|
+
# Validation-checks selection - three independent Yes/No dropdowns
|
|
259
|
+
# (check_catalog_schema / check_column / check_row), replacing an earlier
|
|
260
|
+
# single dbutils.widgets.multiselect that was observed in production to
|
|
261
|
+
# raise DefaultValueNotInChoicesList even with a default that appeared to
|
|
262
|
+
# satisfy multiselect's own documented constraints - three independent
|
|
263
|
+
# dropdowns have no equivalent cross-widget validation.
|
|
264
|
+
# ---------------------------------------------------------------------------
|
|
265
|
+
def test_full_and_basic_mode_default_all_checks_to_yes(tmp_path: Path) -> None:
|
|
266
|
+
for mode in ("full", "basic"):
|
|
267
|
+
path = tmp_path / f"{mode}.py"
|
|
268
|
+
generate_notebook(path, mode)
|
|
269
|
+
spark = _fake_spark(
|
|
270
|
+
catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
|
|
271
|
+
tables_by_schema={("main", "sales"): ["orders"]},
|
|
272
|
+
)
|
|
273
|
+
captured = {}
|
|
274
|
+
|
|
275
|
+
def fake_validate_tables(source, target, **kwargs):
|
|
276
|
+
captured["enabled_validations"] = kwargs.get("enabled_validations")
|
|
277
|
+
m = MagicMock()
|
|
278
|
+
m.__str__ = MagicMock(return_value="Overall status: PASS")
|
|
279
|
+
return m
|
|
280
|
+
|
|
281
|
+
_run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
|
|
282
|
+
|
|
283
|
+
assert captured["enabled_validations"] == {
|
|
284
|
+
ValidationType.CATALOG, ValidationType.SCHEMA,
|
|
285
|
+
ValidationType.COLUMN, ValidationType.ROW,
|
|
286
|
+
}, f"mode={mode}"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def test_schema_mode_defaults_column_and_row_checks_to_no(tmp_path: Path) -> None:
|
|
290
|
+
path = tmp_path / "schema.py"
|
|
291
|
+
generate_notebook(path, "schema")
|
|
292
|
+
spark = _fake_spark(
|
|
293
|
+
catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
|
|
294
|
+
tables_by_schema={("main", "sales"): ["orders"]},
|
|
295
|
+
)
|
|
296
|
+
captured = {}
|
|
297
|
+
|
|
298
|
+
def fake_validate_tables(source, target, **kwargs):
|
|
299
|
+
captured["enabled_validations"] = kwargs.get("enabled_validations")
|
|
300
|
+
m = MagicMock()
|
|
301
|
+
m.__str__ = MagicMock(return_value="Overall status: PASS")
|
|
302
|
+
return m
|
|
303
|
+
|
|
304
|
+
_run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
|
|
305
|
+
|
|
306
|
+
assert captured["enabled_validations"] == {ValidationType.CATALOG, ValidationType.SCHEMA}
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def test_deselecting_every_check_falls_back_to_run_everything(notebook_path):
|
|
310
|
+
"""All three Yes/No dropdowns set to "No" must fall back to
|
|
311
|
+
validate_tables()'s own "run everything" default (enabled_validations=
|
|
312
|
+
None), not silently run nothing."""
|
|
313
|
+
spark = _fake_spark(
|
|
314
|
+
catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
|
|
315
|
+
tables_by_schema={("main", "sales"): ["orders"]},
|
|
316
|
+
)
|
|
317
|
+
captured = {}
|
|
318
|
+
|
|
319
|
+
def fake_validate_tables(source, target, **kwargs):
|
|
320
|
+
captured["enabled_validations"] = kwargs.get("enabled_validations")
|
|
321
|
+
m = MagicMock()
|
|
322
|
+
m.__str__ = MagicMock(return_value="Overall status: PASS")
|
|
323
|
+
return m
|
|
324
|
+
|
|
325
|
+
_run_notebook(
|
|
326
|
+
notebook_path,
|
|
327
|
+
widget_values={
|
|
328
|
+
"check_catalog_schema": "No", "check_column": "No", "check_row": "No",
|
|
329
|
+
},
|
|
330
|
+
spark=spark,
|
|
331
|
+
validate_tables_mock=fake_validate_tables,
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
assert captured["enabled_validations"] is None
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# ---------------------------------------------------------------------------
|
|
338
|
+
# Schema-wide sweep mode - leaving Table as "(all tables in schema)".
|
|
339
|
+
# ---------------------------------------------------------------------------
|
|
340
|
+
def test_default_selection_lands_on_a_real_table_not_the_sweep_sentinel(notebook_path):
|
|
341
|
+
"""A brand-new notebook (no prior widget values) must default to a
|
|
342
|
+
real Catalog/Schema/Table, not silently land on the "(all ... in
|
|
343
|
+
...)" sweep sentinel and immediately error - the sentinel must be
|
|
344
|
+
reachable but never the unprompted default."""
|
|
345
|
+
spark = _fake_spark(
|
|
346
|
+
catalogs=["main"],
|
|
347
|
+
schemas_by_catalog={"main": ["sales"]},
|
|
348
|
+
tables_by_schema={("main", "sales"): ["orders"]},
|
|
349
|
+
)
|
|
350
|
+
captured = {}
|
|
351
|
+
|
|
352
|
+
def fake_validate_tables(source, target, **kwargs):
|
|
353
|
+
captured["source"] = source
|
|
354
|
+
captured["target"] = target
|
|
355
|
+
m = MagicMock()
|
|
356
|
+
m.__str__ = MagicMock(return_value="Overall status: PASS")
|
|
357
|
+
return m
|
|
358
|
+
|
|
359
|
+
_run_notebook(notebook_path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
|
|
360
|
+
|
|
361
|
+
assert captured["source"] == "main.sales.orders"
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def test_table_left_as_sweep_sentinel_builds_two_part_source_string(notebook_path):
|
|
365
|
+
"""Selecting "(all tables in schema)" for Table must produce a
|
|
366
|
+
"catalog.schema" (no table) string - validate_tables()'s own
|
|
367
|
+
schema-wide sweep trigger - comparing every identically-named table."""
|
|
368
|
+
spark = _fake_spark(
|
|
369
|
+
catalogs=["main"],
|
|
370
|
+
schemas_by_catalog={"main": ["bronze"], },
|
|
371
|
+
tables_by_schema={("main", "bronze"): ["orders", "customers"]},
|
|
372
|
+
)
|
|
373
|
+
captured = {}
|
|
374
|
+
|
|
375
|
+
def fake_validate_tables(source, target, **kwargs):
|
|
376
|
+
captured["source"] = source
|
|
377
|
+
captured["target"] = target
|
|
378
|
+
captured["kwargs"] = kwargs
|
|
379
|
+
m = MagicMock()
|
|
380
|
+
m.__str__ = MagicMock(return_value="Overall status: PASS")
|
|
381
|
+
return m
|
|
382
|
+
|
|
383
|
+
_run_notebook(
|
|
384
|
+
notebook_path,
|
|
385
|
+
widget_values={
|
|
386
|
+
"source_catalog": "main", "source_schema": "bronze", "source_table": "(all tables in schema)",
|
|
387
|
+
"target_catalog": "main", "target_schema": "bronze", "target_table": "(all tables in schema)",
|
|
388
|
+
},
|
|
389
|
+
spark=spark,
|
|
390
|
+
validate_tables_mock=fake_validate_tables,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
assert captured["source"] == "main.bronze"
|
|
394
|
+
assert captured["target"] == "main.bronze"
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def test_catalog_wide_sweep_raises_clear_error(notebook_path):
|
|
398
|
+
"""Schema left as "(all schemas in catalog)" isn't supported by
|
|
399
|
+
validate_tables() today - must fail with a clear, actionable error
|
|
400
|
+
rather than building a broken "catalog" (no schema) string."""
|
|
401
|
+
spark = _fake_spark(
|
|
402
|
+
catalogs=["main"],
|
|
403
|
+
schemas_by_catalog={"main": ["sales"]},
|
|
404
|
+
tables_by_schema={("main", "sales"): ["orders"]},
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
with pytest.raises(ValueError, match="all schemas in catalog"):
|
|
408
|
+
_run_notebook(
|
|
409
|
+
notebook_path,
|
|
410
|
+
widget_values={
|
|
411
|
+
"source_catalog": "main", "source_schema": "(all schemas in catalog)",
|
|
412
|
+
"source_table": "(all tables in schema)",
|
|
413
|
+
"target_catalog": "main", "target_schema": "sales", "target_table": "orders",
|
|
414
|
+
},
|
|
415
|
+
spark=spark,
|
|
416
|
+
validate_tables_mock=MagicMock(),
|
|
417
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|