tablevalidator-databricks 0.1.1__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/PKG-INFO +15 -6
  2. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/README.md +14 -5
  3. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/templates/widget_notebook.py.tmpl +103 -25
  4. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/PKG-INFO +15 -6
  5. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/scm_version.json +2 -2
  6. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tests/test_generated_notebook_execution.py +186 -0
  7. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/pyproject.toml +0 -0
  8. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/setup.cfg +0 -0
  9. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/__init__.py +0 -0
  10. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/cli/__init__.py +0 -0
  11. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/cli/main.py +0 -0
  12. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/generator/__init__.py +0 -0
  13. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks/generator/generator.py +0 -0
  14. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/SOURCES.txt +0 -0
  15. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/dependency_links.txt +0 -0
  16. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/entry_points.txt +0 -0
  17. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/requires.txt +0 -0
  18. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/scm_file_list.json +0 -0
  19. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tablevalidator_databricks.egg-info/top_level.txt +0 -0
  20. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tests/test_cli.py +0 -0
  21. {tablevalidator_databricks-0.1.1 → tablevalidator_databricks-0.1.3}/tests/test_generator.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablevalidator-databricks
3
- Version: 0.1.1
3
+ Version: 0.1.3
4
4
  Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
5
5
  License: MIT
6
6
  Requires-Python: >=3.9
@@ -45,11 +45,12 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
45
45
  tablevalidator-databricks init --output MyValidation.py --mode full
46
46
  ```
47
47
 
48
- - `--mode basic` - only the check-group multiselect and table pickers.
48
+ - `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
49
49
  - `--mode full` (default) - also shows optional filter widgets (only/
50
- ignore columns, row filter, primary key).
51
- - `--mode schema` - pre-selects only the "Catalog & Schema" check group,
52
- for a lightweight schema-shape-only comparison.
50
+ ignore columns, row filter, primary key). Every check defaults to "Yes".
51
+ - `--mode schema` - same widgets as `basic`, but Column/Row default to
52
+ "No" - a lightweight schema-shape-only comparison out of the box (still
53
+ changeable per-run).
53
54
 
54
55
  Then, in your Databricks workspace: **Workspace -> Import**, format
55
56
  "Source", and select the generated file. Databricks recognizes the
@@ -62,7 +63,7 @@ markers and opens it as a real notebook.
62
63
  2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
63
64
  `SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
64
65
  populate Source/Target Catalog, Schema, and Table dropdowns.
65
- 3. Shows a "Validation Checks" multiselect (Catalog & Schema / Column /
66
+ 3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
66
67
  Row - the three independently-selectable check groups the engine
67
68
  actually supports) plus, in `full` mode, optional text widgets for
68
69
  column filtering, a row filter, and a primary key.
@@ -86,6 +87,14 @@ isn't showing up (a permissions lag, sort order, anything). It's used
86
87
  as-is, not checked against the live catalog listing. Leave every override
87
88
  blank to just use the dropdowns as normal.
88
89
 
90
+ **Validate every table in a schema, not just one**: pick
91
+ "(all tables in schema)" as the Table dropdown (Source and/or Target
92
+ independently) to compare every identically-named table in that schema in
93
+ one run - the same schema-wide sweep `validate_tables()` itself supports.
94
+ A primary key can't be set in this mode (leave the Primary Key widget
95
+ blank). Comparing every schema in a catalog isn't supported yet - Schema
96
+ must be a specific choice, even when Table is left as "(all ... in ...)".
97
+
89
98
  ## Why a separate package
90
99
 
91
100
  `table-validator` stays focused on the actual comparison engine (CLI +
@@ -31,11 +31,12 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
31
31
  tablevalidator-databricks init --output MyValidation.py --mode full
32
32
  ```
33
33
 
34
- - `--mode basic` - only the check-group multiselect and table pickers.
34
+ - `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
35
35
  - `--mode full` (default) - also shows optional filter widgets (only/
36
- ignore columns, row filter, primary key).
37
- - `--mode schema` - pre-selects only the "Catalog & Schema" check group,
38
- for a lightweight schema-shape-only comparison.
36
+ ignore columns, row filter, primary key). Every check defaults to "Yes".
37
+ - `--mode schema` - same widgets as `basic`, but Column/Row default to
38
+ "No" - a lightweight schema-shape-only comparison out of the box (still
39
+ changeable per-run).
39
40
 
40
41
  Then, in your Databricks workspace: **Workspace -> Import**, format
41
42
  "Source", and select the generated file. Databricks recognizes the
@@ -48,7 +49,7 @@ markers and opens it as a real notebook.
48
49
  2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
49
50
  `SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
50
51
  populate Source/Target Catalog, Schema, and Table dropdowns.
51
- 3. Shows a "Validation Checks" multiselect (Catalog & Schema / Column /
52
+ 3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
52
53
  Row - the three independently-selectable check groups the engine
53
54
  actually supports) plus, in `full` mode, optional text widgets for
54
55
  column filtering, a row filter, and a primary key.
@@ -72,6 +73,14 @@ isn't showing up (a permissions lag, sort order, anything). It's used
72
73
  as-is, not checked against the live catalog listing. Leave every override
73
74
  blank to just use the dropdowns as normal.
74
75
 
76
+ **Validate every table in a schema, not just one**: pick
77
+ "(all tables in schema)" as the Table dropdown (Source and/or Target
78
+ independently) to compare every identically-named table in that schema in
79
+ one run - the same schema-wide sweep `validate_tables()` itself supports.
80
+ A primary key can't be set in this mode (leave the Primary Key widget
81
+ blank). Comparing every schema in a catalog isn't supported yet - Schema
82
+ must be a specific choice, even when Table is left as "(all ... in ...)".
83
+
75
84
  ## Why a separate package
76
85
 
77
86
  `table-validator` stays focused on the actual comparison engine (CLI +
@@ -22,6 +22,12 @@
22
22
  # MAGIC something isn't showing up (a permissions lag, a name that doesn't sort
23
23
  # MAGIC into view, etc.). Leave every override blank to just use the dropdowns
24
24
  # MAGIC as normal.
25
+ # MAGIC
26
+ # MAGIC **Want to validate every table in a schema instead of just one?** Pick
27
+ # MAGIC "(all tables in schema)" as the Table (both Source and Target) - every
28
+ # MAGIC identically-named table in that schema is compared in one run. A
29
+ # MAGIC primary key can't be set in this mode (a single key can't apply to
30
+ # MAGIC every table) - leave the Primary Key widget blank if you use this.
25
31
 
26
32
  # COMMAND ----------
27
33
 
@@ -66,6 +72,30 @@ def _current_or_first(options, current):
66
72
  return current
67
73
  return options[0] if options else ""
68
74
 
75
+
76
+ def _current_or_first_real(options, current, sentinel):
77
+ """Like _current_or_first, but when falling back (no valid current
78
+ selection), prefers the first REAL option over `sentinel` if one
79
+ exists - so a brand-new notebook defaults to an actual Schema/Table
80
+ rather than immediately landing on "(all ... in ...)". The sentinel
81
+ stays fully selectable (and sticky once picked) via the current-
82
+ value check above; this only changes what a first-time default
83
+ lands on."""
84
+ if current in options:
85
+ return current
86
+ real_options = [o for o in options if o != sentinel]
87
+ return real_options[0] if real_options else (options[0] if options else "")
88
+
89
+
90
+ # Sentinel choices meaning "leave this blank" - selecting one of these
91
+ # skips the level(s) below it and triggers validate_tables()'s own
92
+ # schema-wide sweep mode (comparing every identically-named table)
93
+ # instead of a single table pair. Real Databricks catalog/schema/table
94
+ # names can't contain parentheses, so these can never collide with a
95
+ # genuine name.
96
+ _ALL_SCHEMAS = "(all schemas in catalog)"
97
+ _ALL_TABLES = "(all tables in schema)"
98
+
69
99
  # COMMAND ----------
70
100
 
71
101
  # MAGIC %md ### Create / refresh widgets
@@ -103,18 +133,34 @@ def _build_cascading_widgets(prefix: str, label_prefix: str) -> None:
103
133
  now, not a stale snapshot from an earlier run. Also creates a
104
134
  "(manual override)" text widget next to each dropdown, for a name
105
135
  that isn't showing up in the dropdown for any reason - see
106
- _effective_value() below for how the override takes priority."""
136
+ _effective_value() below for how the override takes priority.
137
+
138
+ Schema and Table each also get an "(all ... in ...)" sentinel choice
139
+ as their first option, for a schema-wide sweep (leave Table blank to
140
+ compare every identically-named table in the schema) or a catalog-
141
+ wide sweep (leave Schema, and therefore Table too, blank to compare
142
+ every identically-named table across every schema) - see
143
+ _ALL_SCHEMAS/_ALL_TABLES above and how the "Run validation" cell
144
+ below builds validate_tables()'s "catalog.schema" (no table) /
145
+ "catalog" (no schema) sweep-mode strings from this choice."""
107
146
  catalog_current = _current_or_first(catalogs, _effective_value(f"{prefix}_catalog"))
108
- schemas = _list_schemas(catalog_current)
109
- schema_current = _current_or_first(schemas, _effective_value(f"{prefix}_schema"))
110
- tables = _list_tables(catalog_current, schema_current)
111
- table_current = _current_or_first(tables, _effective_value(f"{prefix}_table"))
147
+
148
+ schema_choices = [_ALL_SCHEMAS] + _list_schemas(catalog_current)
149
+ schema_current = _current_or_first_real(schema_choices, _effective_value(f"{prefix}_schema"), _ALL_SCHEMAS)
150
+ sweeping_schema = schema_current == _ALL_SCHEMAS
151
+
152
+ if sweeping_schema:
153
+ table_choices = [_ALL_TABLES]
154
+ table_current = _ALL_TABLES
155
+ else:
156
+ table_choices = [_ALL_TABLES] + _list_tables(catalog_current, schema_current)
157
+ table_current = _current_or_first_real(table_choices, _effective_value(f"{prefix}_table"), _ALL_TABLES)
112
158
 
113
159
  dbutils.widgets.dropdown(f"{prefix}_catalog", catalog_current, catalogs or [""], f"{label_prefix} Catalog")
114
160
  dbutils.widgets.text(f"{prefix}_catalog_override", "", f"{label_prefix} Catalog (manual override)")
115
- dbutils.widgets.dropdown(f"{prefix}_schema", schema_current, schemas or [""], f"{label_prefix} Schema")
161
+ dbutils.widgets.dropdown(f"{prefix}_schema", schema_current, schema_choices, f"{label_prefix} Schema")
116
162
  dbutils.widgets.text(f"{prefix}_schema_override", "", f"{label_prefix} Schema (manual override)")
117
- dbutils.widgets.dropdown(f"{prefix}_table", table_current, tables or [""], f"{label_prefix} Table")
163
+ dbutils.widgets.dropdown(f"{prefix}_table", table_current, table_choices, f"{label_prefix} Table")
118
164
  dbutils.widgets.text(f"{prefix}_table_override", "", f"{label_prefix} Table (manual override)")
119
165
 
120
166
 
@@ -134,11 +180,20 @@ print(
134
180
 
135
181
  # COMMAND ----------
136
182
 
137
- _default_checks = ["Catalog & Schema", "Column", "Row"] if MODE != "schema" else ["Catalog & Schema"]
138
- dbutils.widgets.multiselect(
139
- "checks", ",".join(_default_checks),
140
- ["Catalog & Schema", "Column", "Row"], "Validation Checks",
141
- )
183
+ # One Yes/No dropdown per check group, instead of a single multiselect -
184
+ # dbutils.widgets.multiselect enforces an internal "selection sequence"
185
+ # validation against its own default that was observed to reject even a
186
+ # same-length, same-items default on some Databricks runtimes/existing-
187
+ # widget states (DefaultValueNotInChoicesList), with no reliable
188
+ # workaround. Three independent dropdowns have no such cross-widget
189
+ # validation and are the more robust primitive for this.
190
+ _YES_NO = ["Yes", "No"]
191
+ _schema_default = "Yes"
192
+ _column_default = "No" if MODE == "schema" else "Yes"
193
+ _row_default = "No" if MODE == "schema" else "Yes"
194
+ dbutils.widgets.dropdown("check_catalog_schema", _schema_default, _YES_NO, "Run Catalog & Schema checks?")
195
+ dbutils.widgets.dropdown("check_column", _column_default, _YES_NO, "Run Column checks?")
196
+ dbutils.widgets.dropdown("check_row", _row_default, _YES_NO, "Run Row checks?")
142
197
 
143
198
  if SHOW_OPTIONAL_WIDGETS:
144
199
  dbutils.widgets.text("only_columns", "", "Only compare these columns (comma-separated, optional)")
@@ -161,14 +216,15 @@ def _validate_current_selection(prefix: str, label: str) -> None:
161
216
  manual override set is trusted as typed and skipped here - the whole
162
217
  point of the override is to name something the live catalog listing
163
218
  doesn't (currently) show, so it can never pass this check by
164
- definition."""
219
+ definition. _ALL_SCHEMAS/_ALL_TABLES (leave-blank sweep sentinels)
220
+ are always valid and skip the corresponding check."""
165
221
  catalog = _effective_value(f"{prefix}_catalog")
166
222
  schema_overridden = bool(_get_widget(f"{prefix}_schema_override").strip())
167
223
  table_overridden = bool(_get_widget(f"{prefix}_table_override").strip())
168
224
  schema = _effective_value(f"{prefix}_schema")
169
225
  table = _effective_value(f"{prefix}_table")
170
226
 
171
- if not schema_overridden:
227
+ if not schema_overridden and schema != _ALL_SCHEMAS:
172
228
  live_schemas = _list_schemas(catalog)
173
229
  if schema not in live_schemas:
174
230
  raise ValueError(
@@ -181,7 +237,7 @@ def _validate_current_selection(prefix: str, label: str) -> None:
181
237
  "the dropdown, type it into the Schema manual override "
182
238
  "widget instead.)"
183
239
  )
184
- if not table_overridden:
240
+ if not table_overridden and schema != _ALL_SCHEMAS and table != _ALL_TABLES:
185
241
  live_tables = _list_tables(catalog, schema)
186
242
  if table not in live_tables:
187
243
  raise ValueError(
@@ -198,18 +254,40 @@ def _validate_current_selection(prefix: str, label: str) -> None:
198
254
  _validate_current_selection("source", "Source")
199
255
  _validate_current_selection("target", "Target")
200
256
 
201
- source = f"{_effective_value('source_catalog')}.{_effective_value('source_schema')}.{_effective_value('source_table')}"
202
- target = f"{_effective_value('target_catalog')}.{_effective_value('target_schema')}.{_effective_value('target_table')}"
203
257
 
204
- _CHECK_MAP = {
205
- "Catalog & Schema": {ValidationType.CATALOG, ValidationType.SCHEMA},
206
- "Column": {ValidationType.COLUMN},
207
- "Row": {ValidationType.ROW},
208
- }
209
- selected_checks = [c for c in dbutils.widgets.get("checks").split(",") if c]
258
+ def _source_or_target_string(prefix: str) -> str:
259
+ """Builds the "catalog.schema.table" (single table) or
260
+ "catalog.schema" (schema-wide sweep, Schema chosen but Table left as
261
+ _ALL_TABLES) string validate_tables() expects. Catalog-wide sweeps
262
+ (Schema also left blank) aren't supported by validate_tables() today
263
+ - _ALL_SCHEMAS is caught with a clear error below instead of being
264
+ silently passed through as a literal, broken source/target string."""
265
+ catalog = _effective_value(f"{prefix}_catalog")
266
+ schema = _effective_value(f"{prefix}_schema")
267
+ table = _effective_value(f"{prefix}_table")
268
+
269
+ if schema == _ALL_SCHEMAS:
270
+ raise ValueError(
271
+ f"{prefix.capitalize()} Schema is set to {_ALL_SCHEMAS!r} - "
272
+ "comparing every schema in a catalog isn't supported yet. "
273
+ "Pick a specific Schema (Table can still be left as "
274
+ f"{_ALL_TABLES!r} to sweep every table within it)."
275
+ )
276
+ if table == _ALL_TABLES:
277
+ return f"{catalog}.{schema}"
278
+ return f"{catalog}.{schema}.{table}"
279
+
280
+
281
+ source = _source_or_target_string("source")
282
+ target = _source_or_target_string("target")
283
+
210
284
  enabled_validations = set()
211
- for check in selected_checks:
212
- enabled_validations |= _CHECK_MAP.get(check, set())
285
+ if dbutils.widgets.get("check_catalog_schema") == "Yes":
286
+ enabled_validations |= {ValidationType.CATALOG, ValidationType.SCHEMA}
287
+ if dbutils.widgets.get("check_column") == "Yes":
288
+ enabled_validations |= {ValidationType.COLUMN}
289
+ if dbutils.widgets.get("check_row") == "Yes":
290
+ enabled_validations |= {ValidationType.ROW}
213
291
  if not enabled_validations:
214
292
  enabled_validations = None # fall back to validate_tables()'s own "run everything" default
215
293
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablevalidator-databricks
3
- Version: 0.1.1
3
+ Version: 0.1.3
4
4
  Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
5
5
  License: MIT
6
6
  Requires-Python: >=3.9
@@ -45,11 +45,12 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
45
45
  tablevalidator-databricks init --output MyValidation.py --mode full
46
46
  ```
47
47
 
48
- - `--mode basic` - only the check-group multiselect and table pickers.
48
+ - `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
49
49
  - `--mode full` (default) - also shows optional filter widgets (only/
50
- ignore columns, row filter, primary key).
51
- - `--mode schema` - pre-selects only the "Catalog & Schema" check group,
52
- for a lightweight schema-shape-only comparison.
50
+ ignore columns, row filter, primary key). Every check defaults to "Yes".
51
+ - `--mode schema` - same widgets as `basic`, but Column/Row default to
52
+ "No" - a lightweight schema-shape-only comparison out of the box (still
53
+ changeable per-run).
53
54
 
54
55
  Then, in your Databricks workspace: **Workspace -> Import**, format
55
56
  "Source", and select the generated file. Databricks recognizes the
@@ -62,7 +63,7 @@ markers and opens it as a real notebook.
62
63
  2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
63
64
  `SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
64
65
  populate Source/Target Catalog, Schema, and Table dropdowns.
65
- 3. Shows a "Validation Checks" multiselect (Catalog & Schema / Column /
66
+ 3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
66
67
  Row - the three independently-selectable check groups the engine
67
68
  actually supports) plus, in `full` mode, optional text widgets for
68
69
  column filtering, a row filter, and a primary key.
@@ -86,6 +87,14 @@ isn't showing up (a permissions lag, sort order, anything). It's used
86
87
  as-is, not checked against the live catalog listing. Leave every override
87
88
  blank to just use the dropdowns as normal.
88
89
 
90
+ **Validate every table in a schema, not just one**: pick
91
+ "(all tables in schema)" as the Table dropdown (Source and/or Target
92
+ independently) to compare every identically-named table in that schema in
93
+ one run - the same schema-wide sweep `validate_tables()` itself supports.
94
+ A primary key can't be set in this mode (leave the Primary Key widget
95
+ blank). Comparing every schema in a catalog isn't supported yet - Schema
96
+ must be a specific choice, even when Table is left as "(all ... in ...)".
97
+
89
98
  ## Why a separate package
90
99
 
91
100
  `table-validator` stays focused on the actual comparison engine (CLI +
@@ -1,7 +1,7 @@
1
1
  {
2
- "tag": "0.1.1",
2
+ "tag": "0.1.3",
3
3
  "distance": 0,
4
- "node": "g1f85a99",
4
+ "node": "gd05d199",
5
5
  "dirty": false,
6
6
  "branch": "HEAD",
7
7
  "node_date": "2026-09-08"
@@ -15,6 +15,7 @@ from unittest.mock import MagicMock
15
15
 
16
16
  import pytest
17
17
 
18
+ from table_validator.config.schema import ValidationType
18
19
  from tablevalidator_databricks.generator.generator import generate_notebook
19
20
 
20
21
 
@@ -29,9 +30,15 @@ def _code_from_notebook(path: Path) -> str:
29
30
  return "\n".join(code_lines)
30
31
 
31
32
 
33
+ class _DefaultValueNotInChoicesList(Exception):
34
+ pass
35
+
36
+
32
37
  class _FakeWidgets:
33
38
  def __init__(self, initial=None):
34
39
  self._values = dict(initial or {})
40
+ self.dropdown_choices = {}
41
+ self.multiselect_choices = {}
35
42
 
36
43
  def get(self, name):
37
44
  return self._values.get(name, "")
@@ -43,9 +50,25 @@ class _FakeWidgets:
43
50
  # Real dbutils.widgets.dropdown does NOT overwrite an existing
44
51
  # value just because the cell re-ran with a different `default` -
45
52
  # it only sets it the first time. Match that.
53
+ if default not in choices:
54
+ raise _DefaultValueNotInChoicesList(f"{name}: {default!r} not in {choices!r}")
55
+ self.dropdown_choices[name] = choices
46
56
  self._values.setdefault(name, default)
47
57
 
48
58
  def multiselect(self, name, default, choices, label):
59
+ # Real Databricks dbutils.widgets.multiselect requires the
60
+ # default's comma-separated items to all be present in choices -
61
+ # observed in production to reject a narrower/reordered subset
62
+ # with "DefaultValueNotInChoicesList: Selection sequence must
63
+ # include <choices>" even when every individual item IS a valid
64
+ # choice. Replicate that constraint here so this suite would have
65
+ # caught the real bug (a "schema"-mode default of just
66
+ # "Catalog & Schema" against choices=["Catalog & Schema",
67
+ # "Column", "Row"]) before it ever reached a live workspace.
68
+ default_items = default.split(",") if default else []
69
+ if any(item not in choices for item in default_items):
70
+ raise _DefaultValueNotInChoicesList(f"{name}: {default!r} not a subset of {choices!r}")
71
+ self.multiselect_choices[name] = choices
49
72
  self._values.setdefault(name, default)
50
73
 
51
74
 
@@ -229,3 +252,166 @@ def test_catalog_override_drives_schema_and_table_lookup(notebook_path):
229
252
  )
230
253
 
231
254
  assert captured["source"] == "hidden_catalog.sales.orders"
255
+
256
+
257
+ # ---------------------------------------------------------------------------
258
+ # Validation-checks selection - three independent Yes/No dropdowns
259
+ # (check_catalog_schema / check_column / check_row), replacing an earlier
260
+ # single dbutils.widgets.multiselect that was observed in production to
261
+ # raise DefaultValueNotInChoicesList even with a default that appeared to
262
+ # satisfy multiselect's own documented constraints - three independent
263
+ # dropdowns have no equivalent cross-widget validation.
264
+ # ---------------------------------------------------------------------------
265
+ def test_full_and_basic_mode_default_all_checks_to_yes(tmp_path: Path) -> None:
266
+ for mode in ("full", "basic"):
267
+ path = tmp_path / f"{mode}.py"
268
+ generate_notebook(path, mode)
269
+ spark = _fake_spark(
270
+ catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
271
+ tables_by_schema={("main", "sales"): ["orders"]},
272
+ )
273
+ captured = {}
274
+
275
+ def fake_validate_tables(source, target, **kwargs):
276
+ captured["enabled_validations"] = kwargs.get("enabled_validations")
277
+ m = MagicMock()
278
+ m.__str__ = MagicMock(return_value="Overall status: PASS")
279
+ return m
280
+
281
+ _run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
282
+
283
+ assert captured["enabled_validations"] == {
284
+ ValidationType.CATALOG, ValidationType.SCHEMA,
285
+ ValidationType.COLUMN, ValidationType.ROW,
286
+ }, f"mode={mode}"
287
+
288
+
289
+ def test_schema_mode_defaults_column_and_row_checks_to_no(tmp_path: Path) -> None:
290
+ path = tmp_path / "schema.py"
291
+ generate_notebook(path, "schema")
292
+ spark = _fake_spark(
293
+ catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
294
+ tables_by_schema={("main", "sales"): ["orders"]},
295
+ )
296
+ captured = {}
297
+
298
+ def fake_validate_tables(source, target, **kwargs):
299
+ captured["enabled_validations"] = kwargs.get("enabled_validations")
300
+ m = MagicMock()
301
+ m.__str__ = MagicMock(return_value="Overall status: PASS")
302
+ return m
303
+
304
+ _run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
305
+
306
+ assert captured["enabled_validations"] == {ValidationType.CATALOG, ValidationType.SCHEMA}
307
+
308
+
309
+ def test_deselecting_every_check_falls_back_to_run_everything(notebook_path):
310
+ """All three Yes/No dropdowns set to "No" must fall back to
311
+ validate_tables()'s own "run everything" default (enabled_validations=
312
+ None), not silently run nothing."""
313
+ spark = _fake_spark(
314
+ catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
315
+ tables_by_schema={("main", "sales"): ["orders"]},
316
+ )
317
+ captured = {}
318
+
319
+ def fake_validate_tables(source, target, **kwargs):
320
+ captured["enabled_validations"] = kwargs.get("enabled_validations")
321
+ m = MagicMock()
322
+ m.__str__ = MagicMock(return_value="Overall status: PASS")
323
+ return m
324
+
325
+ _run_notebook(
326
+ notebook_path,
327
+ widget_values={
328
+ "check_catalog_schema": "No", "check_column": "No", "check_row": "No",
329
+ },
330
+ spark=spark,
331
+ validate_tables_mock=fake_validate_tables,
332
+ )
333
+
334
+ assert captured["enabled_validations"] is None
335
+
336
+
337
+ # ---------------------------------------------------------------------------
338
+ # Schema-wide sweep mode - leaving Table as "(all tables in schema)".
339
+ # ---------------------------------------------------------------------------
340
+ def test_default_selection_lands_on_a_real_table_not_the_sweep_sentinel(notebook_path):
341
+ """A brand-new notebook (no prior widget values) must default to a
342
+ real Catalog/Schema/Table, not silently land on the "(all ... in
343
+ ...)" sweep sentinel and immediately error - the sentinel must be
344
+ reachable but never the unprompted default."""
345
+ spark = _fake_spark(
346
+ catalogs=["main"],
347
+ schemas_by_catalog={"main": ["sales"]},
348
+ tables_by_schema={("main", "sales"): ["orders"]},
349
+ )
350
+ captured = {}
351
+
352
+ def fake_validate_tables(source, target, **kwargs):
353
+ captured["source"] = source
354
+ captured["target"] = target
355
+ m = MagicMock()
356
+ m.__str__ = MagicMock(return_value="Overall status: PASS")
357
+ return m
358
+
359
+ _run_notebook(notebook_path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
360
+
361
+ assert captured["source"] == "main.sales.orders"
362
+
363
+
364
+ def test_table_left_as_sweep_sentinel_builds_two_part_source_string(notebook_path):
365
+ """Selecting "(all tables in schema)" for Table must produce a
366
+ "catalog.schema" (no table) string - validate_tables()'s own
367
+ schema-wide sweep trigger - comparing every identically-named table."""
368
+ spark = _fake_spark(
369
+ catalogs=["main"],
370
+ schemas_by_catalog={"main": ["bronze"], },
371
+ tables_by_schema={("main", "bronze"): ["orders", "customers"]},
372
+ )
373
+ captured = {}
374
+
375
+ def fake_validate_tables(source, target, **kwargs):
376
+ captured["source"] = source
377
+ captured["target"] = target
378
+ captured["kwargs"] = kwargs
379
+ m = MagicMock()
380
+ m.__str__ = MagicMock(return_value="Overall status: PASS")
381
+ return m
382
+
383
+ _run_notebook(
384
+ notebook_path,
385
+ widget_values={
386
+ "source_catalog": "main", "source_schema": "bronze", "source_table": "(all tables in schema)",
387
+ "target_catalog": "main", "target_schema": "bronze", "target_table": "(all tables in schema)",
388
+ },
389
+ spark=spark,
390
+ validate_tables_mock=fake_validate_tables,
391
+ )
392
+
393
+ assert captured["source"] == "main.bronze"
394
+ assert captured["target"] == "main.bronze"
395
+
396
+
397
+ def test_catalog_wide_sweep_raises_clear_error(notebook_path):
398
+ """Schema left as "(all schemas in catalog)" isn't supported by
399
+ validate_tables() today - must fail with a clear, actionable error
400
+ rather than building a broken "catalog" (no schema) string."""
401
+ spark = _fake_spark(
402
+ catalogs=["main"],
403
+ schemas_by_catalog={"main": ["sales"]},
404
+ tables_by_schema={("main", "sales"): ["orders"]},
405
+ )
406
+
407
+ with pytest.raises(ValueError, match="all schemas in catalog"):
408
+ _run_notebook(
409
+ notebook_path,
410
+ widget_values={
411
+ "source_catalog": "main", "source_schema": "(all schemas in catalog)",
412
+ "source_table": "(all tables in schema)",
413
+ "target_catalog": "main", "target_schema": "sales", "target_table": "orders",
414
+ },
415
+ spark=spark,
416
+ validate_tables_mock=MagicMock(),
417
+ )