tablevalidator-databricks 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/PKG-INFO +9 -1
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/README.md +8 -0
- tablevalidator_databricks-0.1.6/tablevalidator_databricks/cli/main.py +219 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks/generator/generator.py +3 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks/templates/widget_notebook.py.tmpl +51 -34
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks.egg-info/PKG-INFO +9 -1
- tablevalidator_databricks-0.1.6/tablevalidator_databricks.egg-info/scm_version.json +8 -0
- tablevalidator_databricks-0.1.6/tests/test_cli.py +123 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tests/test_generator.py +15 -0
- tablevalidator_databricks-0.1.4/tablevalidator_databricks/cli/main.py +0 -101
- tablevalidator_databricks-0.1.4/tablevalidator_databricks.egg-info/scm_version.json +0 -8
- tablevalidator_databricks-0.1.4/tests/test_cli.py +0 -56
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/pyproject.toml +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/setup.cfg +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks/__init__.py +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks/cli/__init__.py +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks/generator/__init__.py +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks.egg-info/SOURCES.txt +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks.egg-info/dependency_links.txt +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks.egg-info/entry_points.txt +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks.egg-info/requires.txt +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks.egg-info/scm_file_list.json +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tablevalidator_databricks.egg-info/top_level.txt +0 -0
- {tablevalidator_databricks-0.1.4 → tablevalidator_databricks-0.1.6}/tests/test_generated_notebook_execution.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablevalidator-databricks
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
|
|
5
5
|
License: MIT
|
|
6
6
|
Requires-Python: >=3.9
|
|
@@ -35,6 +35,14 @@ there's nothing else to install separately.
|
|
|
35
35
|
|
|
36
36
|
## Usage
|
|
37
37
|
|
|
38
|
+
```bash
|
|
39
|
+
tablevalidator-databricks info
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Prints a full usage guide - the steps, the three modes, what each check
|
|
43
|
+
group actually does, schema-wide sweeps, the cascading-dropdown caveat,
|
|
44
|
+
manual overrides, and version pinning. Start here if you're new to it.
|
|
45
|
+
|
|
38
46
|
```bash
|
|
39
47
|
tablevalidator-databricks init
|
|
40
48
|
```
|
|
@@ -21,6 +21,14 @@ there's nothing else to install separately.
|
|
|
21
21
|
|
|
22
22
|
## Usage
|
|
23
23
|
|
|
24
|
+
```bash
|
|
25
|
+
tablevalidator-databricks info
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Prints a full usage guide - the steps, the three modes, what each check
|
|
29
|
+
group actually does, schema-wide sweeps, the cascading-dropdown caveat,
|
|
30
|
+
manual overrides, and version pinning. Start here if you're new to it.
|
|
31
|
+
|
|
24
32
|
```bash
|
|
25
33
|
tablevalidator-databricks init
|
|
26
34
|
```
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""CLI entry point: `tablevalidator-databricks` console script."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from tablevalidator_databricks import __version__
|
|
9
|
+
from tablevalidator_databricks.generator.generator import (
|
|
10
|
+
VALID_MODES,
|
|
11
|
+
_min_core_version,
|
|
12
|
+
generate_notebook,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
app = typer.Typer(
|
|
16
|
+
name="tablevalidator-databricks",
|
|
17
|
+
help=(
|
|
18
|
+
"Generate a widget-driven Databricks notebook UI for table-validator. "
|
|
19
|
+
"Run 'tablevalidator-databricks info' for a full usage guide."
|
|
20
|
+
),
|
|
21
|
+
no_args_is_help=True,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
DEFAULT_OUTPUT_PATH = Path("TableValidator.py")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _version_callback(value: bool) -> None:
|
|
28
|
+
if value:
|
|
29
|
+
typer.echo(f"tablevalidator-databricks {__version__}")
|
|
30
|
+
raise typer.Exit()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@app.callback()
|
|
34
|
+
def _main(
|
|
35
|
+
version: Optional[bool] = typer.Option(
|
|
36
|
+
None,
|
|
37
|
+
"--version",
|
|
38
|
+
callback=_version_callback,
|
|
39
|
+
is_eager=True,
|
|
40
|
+
help="Show the installed version and exit.",
|
|
41
|
+
),
|
|
42
|
+
) -> None:
|
|
43
|
+
"""Generate a widget-driven Databricks notebook UI for table-validator."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@app.command()
|
|
47
|
+
def init(
|
|
48
|
+
output: Path = typer.Option(
|
|
49
|
+
DEFAULT_OUTPUT_PATH,
|
|
50
|
+
"--output",
|
|
51
|
+
help="Path to write the generated notebook to (default: ./TableValidator.py).",
|
|
52
|
+
),
|
|
53
|
+
mode: str = typer.Option(
|
|
54
|
+
"full",
|
|
55
|
+
"--mode",
|
|
56
|
+
help=(
|
|
57
|
+
"basic: only the check-group Yes/No dropdowns and table pickers. "
|
|
58
|
+
"full (default): also shows optional filter widgets (only/ "
|
|
59
|
+
"ignore columns, row filter, primary key). "
|
|
60
|
+
"schema: same widgets as basic, but the Row checks default to No "
|
|
61
|
+
"(schema-shape-only comparison)."
|
|
62
|
+
),
|
|
63
|
+
),
|
|
64
|
+
) -> None:
|
|
65
|
+
"""Generate a Databricks notebook with dbutils widgets for source/
|
|
66
|
+
target catalog.schema.table selection and validation checks - fill in
|
|
67
|
+
the widgets and run the notebook, no code to write.
|
|
68
|
+
|
|
69
|
+
Import the generated file into a Databricks workspace via
|
|
70
|
+
Workspace -> Import, with format set to "Source" (or "File" -> upload
|
|
71
|
+
directly, depending on your workspace UI version) - Databricks
|
|
72
|
+
recognizes the "# Databricks notebook source" header and cell
|
|
73
|
+
(# COMMAND ----------) markers and opens it as a real notebook, not a
|
|
74
|
+
plain text file.
|
|
75
|
+
"""
|
|
76
|
+
if mode not in VALID_MODES:
|
|
77
|
+
typer.secho(
|
|
78
|
+
f"Invalid --mode {mode!r} - must be one of {', '.join(VALID_MODES)}.",
|
|
79
|
+
fg=typer.colors.RED,
|
|
80
|
+
)
|
|
81
|
+
raise typer.Exit(code=1)
|
|
82
|
+
|
|
83
|
+
generate_notebook(output, mode)
|
|
84
|
+
|
|
85
|
+
typer.echo(f"Generated notebook: {output.resolve()}")
|
|
86
|
+
typer.echo(f" generator version: {__version__} (mode: {mode})")
|
|
87
|
+
typer.echo(
|
|
88
|
+
"Import it into Databricks (Workspace -> Import, format 'Source'), "
|
|
89
|
+
"fill in the widgets, and Run All."
|
|
90
|
+
)
|
|
91
|
+
typer.echo(
|
|
92
|
+
"Re-import over the old notebook if you are updating one - widgets are "
|
|
93
|
+
"baked in at generation time, so an existing notebook keeps its old "
|
|
94
|
+
"code until you replace it."
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
_INFO_TEXT = """
|
|
99
|
+
tablevalidator-databricks - Databricks notebook UI for table-validator
|
|
100
|
+
------------------------------------------------------------------------
|
|
101
|
+
Generates a Databricks notebook driven entirely by dbutils widgets -
|
|
102
|
+
Catalog/Schema/Table dropdowns populated live from Unity Catalog, plus
|
|
103
|
+
Yes/No dropdowns for which checks to run - as a wrapper around the
|
|
104
|
+
table-validator package's validate_tables() API. No Python to write, and
|
|
105
|
+
no separate credentials to configure (the notebook reuses its own
|
|
106
|
+
ambient Spark session, same as validate_tables() itself).
|
|
107
|
+
|
|
108
|
+
This package contains NO validation logic of its own. Every generated
|
|
109
|
+
notebook calls straight into table-validator's engine, so results are
|
|
110
|
+
identical to calling validate_tables() yourself in a code cell.
|
|
111
|
+
|
|
112
|
+
STEPS
|
|
113
|
+
-----
|
|
114
|
+
1. tablevalidator-databricks init
|
|
115
|
+
Writes a ready-to-run notebook (./TableValidator.py by default).
|
|
116
|
+
--output PATH write somewhere else
|
|
117
|
+
--mode MODE basic | full (default) | schema
|
|
118
|
+
See 'tablevalidator-databricks init --help' for details.
|
|
119
|
+
|
|
120
|
+
2. Import it into Databricks
|
|
121
|
+
Workspace -> Import -> format "Source", pick the generated file.
|
|
122
|
+
Databricks reads the '# Databricks notebook source' header and
|
|
123
|
+
'# COMMAND ----------' markers and opens it as a real notebook.
|
|
124
|
+
|
|
125
|
+
3. Fill in the widgets, then Run All
|
|
126
|
+
Pick Source and Target Catalog / Schema / Table, choose which
|
|
127
|
+
checks to run, and run the notebook top to bottom.
|
|
128
|
+
|
|
129
|
+
MODES
|
|
130
|
+
-----
|
|
131
|
+
basic Check-group Yes/No dropdowns and the table pickers only.
|
|
132
|
+
full (default) Also shows optional filter widgets: only-columns,
|
|
133
|
+
ignore-columns, row filter, primary key.
|
|
134
|
+
schema Same widgets as basic, but the Row checks default to "No" -
|
|
135
|
+
a lightweight schema-shape-only comparison.
|
|
136
|
+
|
|
137
|
+
THE CHECK GROUPS
|
|
138
|
+
----------------
|
|
139
|
+
Three independent Yes/No dropdowns:
|
|
140
|
+
|
|
141
|
+
Catalog & Schema Do the catalogs/schemas/tables exist and match by
|
|
142
|
+
name? This produces NO per-table PASS/FAIL on its
|
|
143
|
+
own - running it alone reports every table as
|
|
144
|
+
SKIPPED. That is expected, not an error.
|
|
145
|
+
Column Column names, data types, nullability, null and
|
|
146
|
+
distinct counts, min/max. This is where schema-
|
|
147
|
+
shape problems are actually caught.
|
|
148
|
+
Row Row counts and row-level data comparison. The most
|
|
149
|
+
expensive group.
|
|
150
|
+
|
|
151
|
+
Turning every group off falls back to running everything, rather than
|
|
152
|
+
silently validating nothing.
|
|
153
|
+
|
|
154
|
+
VALIDATING A WHOLE SCHEMA
|
|
155
|
+
-------------------------
|
|
156
|
+
Set the Table dropdown to "(all tables in schema)" on BOTH sides to
|
|
157
|
+
compare every identically-named table in that schema in one run - the
|
|
158
|
+
same schema-wide sweep validate_tables() itself supports. A primary key
|
|
159
|
+
can't be used in this mode (a single key can't apply to every table), so
|
|
160
|
+
leave the Primary Key widget blank. Comparing every schema in a catalog
|
|
161
|
+
is not supported yet - Schema must be a specific pick.
|
|
162
|
+
|
|
163
|
+
CASCADING DROPDOWNS
|
|
164
|
+
-------------------
|
|
165
|
+
Databricks widgets have no on-change event, so choosing a new Catalog
|
|
166
|
+
does NOT repopulate the Schema dropdown below it (same for Schema ->
|
|
167
|
+
Table). After changing a Catalog or Schema, re-run the notebook's
|
|
168
|
+
"Create / refresh widgets" cell (Shift+Enter) before continuing. This is
|
|
169
|
+
a Databricks platform limitation, not a bug in the notebook - and if you
|
|
170
|
+
skip it, the notebook fails with a clear error naming the cell to re-run
|
|
171
|
+
rather than silently comparing the wrong table.
|
|
172
|
+
|
|
173
|
+
NAME NOT IN A DROPDOWN?
|
|
174
|
+
-----------------------
|
|
175
|
+
Every Catalog/Schema/Table dropdown has a "(manual override)" text
|
|
176
|
+
widget beside it. Type an exact name there and it is used instead of the
|
|
177
|
+
dropdown - handy for a permissions lag, an odd sort order, anything the
|
|
178
|
+
listing misses. It is used as-is, not checked against the live catalog.
|
|
179
|
+
Leave the overrides blank to just use the dropdowns.
|
|
180
|
+
|
|
181
|
+
VERSION PINNING (IMPORTANT)
|
|
182
|
+
---------------------------
|
|
183
|
+
The generated notebook installs the engine with a floor:
|
|
184
|
+
%pip install "table-validator>={MIN_CORE_VERSION}"
|
|
185
|
+
Do NOT lower that pin, and do not confuse it with this package's own
|
|
186
|
+
version number - the two packages are versioned completely
|
|
187
|
+
independently. validate_tables() only exists in table-validator 0.1.9
|
|
188
|
+
and later, so pinning the engine to this generator's version installs
|
|
189
|
+
something far too old and fails with:
|
|
190
|
+
ImportError: cannot import name 'validate_tables'
|
|
191
|
+
|
|
192
|
+
Re-run 'tablevalidator-databricks init' after upgrading this package -
|
|
193
|
+
an already-generated notebook keeps whatever code it was written with.
|
|
194
|
+
|
|
195
|
+
A WIDGET IS MISSING FROM MY NOTEBOOK
|
|
196
|
+
------------------------------------
|
|
197
|
+
Widgets are baked into the .py file when it is generated. Upgrading this
|
|
198
|
+
package does NOT change a notebook that already exists - you have to
|
|
199
|
+
generate a new one and re-import it.
|
|
200
|
+
|
|
201
|
+
Check what generated the notebook you are looking at: the title cell at
|
|
202
|
+
the top names the version. Compare it against your installed version:
|
|
203
|
+
tablevalidator-databricks --version
|
|
204
|
+
|
|
205
|
+
Then regenerate and re-import:
|
|
206
|
+
pip install --upgrade tablevalidator-databricks
|
|
207
|
+
tablevalidator-databricks init
|
|
208
|
+
|
|
209
|
+
For reference, widgets arrived in these versions:
|
|
210
|
+
0.1.1 "(manual override)" text box beside each dropdown
|
|
211
|
+
0.1.2 "(all tables in schema)" schema-wide sweep option
|
|
212
|
+
0.1.3 Yes/No dropdowns per check group (replacing a multiselect)
|
|
213
|
+
""".strip("\n")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@app.command()
|
|
217
|
+
def info() -> None:
|
|
218
|
+
"""Show what this tool does and how to use the generated notebook."""
|
|
219
|
+
typer.echo("\n" + _INFO_TEXT.replace("{MIN_CORE_VERSION}", _min_core_version()) + "\n")
|
|
@@ -56,10 +56,13 @@ def generate_notebook(output: Path, mode: str) -> None:
|
|
|
56
56
|
.joinpath("widget_notebook.py.tmpl")
|
|
57
57
|
.read_text(encoding="utf-8")
|
|
58
58
|
)
|
|
59
|
+
from tablevalidator_databricks import __version__
|
|
60
|
+
|
|
59
61
|
rendered = (
|
|
60
62
|
template.replace("{{MODE}}", mode)
|
|
61
63
|
.replace("{{SHOW_OPTIONAL_WIDGETS}}", "True" if mode == "full" else "False")
|
|
62
64
|
.replace("{{MIN_CORE_VERSION}}", _min_core_version())
|
|
65
|
+
.replace("{{GENERATOR_VERSION}}", __version__)
|
|
63
66
|
)
|
|
64
67
|
|
|
65
68
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -1,12 +1,22 @@
|
|
|
1
1
|
# Databricks notebook source
|
|
2
2
|
# MAGIC %md
|
|
3
3
|
# MAGIC # Table Validator
|
|
4
|
-
# MAGIC Generated by `tablevalidator-databricks
|
|
4
|
+
# MAGIC Generated by `tablevalidator-databricks` **v{{GENERATOR_VERSION}}**
|
|
5
|
+
# MAGIC (`init --mode {{MODE}}`).
|
|
5
6
|
# MAGIC
|
|
6
|
-
# MAGIC
|
|
7
|
-
# MAGIC
|
|
8
|
-
# MAGIC
|
|
9
|
-
# MAGIC `
|
|
7
|
+
# MAGIC Widgets are baked in when this file is generated - upgrading the
|
|
8
|
+
# MAGIC package does NOT change a notebook that already exists. If a widget
|
|
9
|
+
# MAGIC described below is missing here, this notebook was generated by an
|
|
10
|
+
# MAGIC older version: run `pip install --upgrade tablevalidator-databricks`
|
|
11
|
+
# MAGIC then `tablevalidator-databricks init` again, and re-import the result.
|
|
12
|
+
# MAGIC
|
|
13
|
+
# MAGIC **First time here?** Choose **Run All**. Widgets are created by running
|
|
14
|
+
# MAGIC cells, so on a fresh notebook they only appear once the run reaches the
|
|
15
|
+
# MAGIC "Create / refresh widgets" cell - that is normal, not a fault. Once they
|
|
16
|
+
# MAGIC are there, fill them in and run again.
|
|
17
|
+
# MAGIC
|
|
18
|
+
# MAGIC No code needs to be written or edited - every cell below just reads the
|
|
19
|
+
# MAGIC widgets and calls the `table_validator` package's `validate_tables()` API.
|
|
10
20
|
# MAGIC
|
|
11
21
|
# MAGIC **Changing Catalog or Schema after the dropdowns are already built?**
|
|
12
22
|
# MAGIC Databricks widgets have no on-change event, so picking a new Catalog
|
|
@@ -176,9 +186,44 @@ def _build_cascading_widgets(prefix: str, label_prefix: str) -> None:
|
|
|
176
186
|
dbutils.widgets.text(f"{prefix}_table_override", "", f"{label_prefix} Table (manual override)")
|
|
177
187
|
|
|
178
188
|
|
|
189
|
+
def _build_check_widgets() -> None:
|
|
190
|
+
"""The three check-group Yes/No dropdowns, plus (in full mode) the
|
|
191
|
+
optional filter/column text boxes.
|
|
192
|
+
|
|
193
|
+
One Yes/No dropdown per check group, rather than a single multiselect -
|
|
194
|
+
dbutils.widgets.multiselect enforces an internal "selection sequence"
|
|
195
|
+
validation against its own default that was observed to reject even a
|
|
196
|
+
same-length, same-items default on some Databricks runtimes/existing-
|
|
197
|
+
widget states (DefaultValueNotInChoicesList), with no reliable
|
|
198
|
+
workaround. Three independent dropdowns have no such cross-widget
|
|
199
|
+
validation and are the more robust primitive for this."""
|
|
200
|
+
yes_no = ["Yes", "No"]
|
|
201
|
+
# --mode schema means "schema SHAPE only" - that shape (column names,
|
|
202
|
+
# types, nullability) is validated by the COLUMN checks, so Column
|
|
203
|
+
# stays on and only the expensive Row checks (row counts + row-level
|
|
204
|
+
# data) are off by default. Turning Column off too would leave nothing
|
|
205
|
+
# producing a per-table verdict, reporting every table as SKIPPED.
|
|
206
|
+
row_default = "No" if MODE == "schema" else "Yes"
|
|
207
|
+
|
|
208
|
+
dbutils.widgets.dropdown("check_catalog_schema", "Yes", yes_no, "Run Catalog & Schema checks?")
|
|
209
|
+
dbutils.widgets.dropdown("check_column", "Yes", yes_no, "Run Column checks?")
|
|
210
|
+
dbutils.widgets.dropdown("check_row", row_default, yes_no, "Run Row checks?")
|
|
211
|
+
|
|
212
|
+
if SHOW_OPTIONAL_WIDGETS:
|
|
213
|
+
dbutils.widgets.text("primary_key", "", "Primary key column(s) (comma-separated, optional)")
|
|
214
|
+
dbutils.widgets.text("only_columns", "", "Only compare these columns (comma-separated, optional)")
|
|
215
|
+
dbutils.widgets.text("ignore_columns", "", "Skip these columns entirely (comma-separated, optional)")
|
|
216
|
+
dbutils.widgets.text("row_filter", "", "Row filter - SQL WHERE-fragment (optional)")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
# Every widget is created here, in ONE cell, so they all appear together
|
|
220
|
+
# and in a predictable order - Databricks lays widgets out in creation
|
|
221
|
+
# order, so splitting these across cells made them interleave oddly and
|
|
222
|
+
# made later ones seem to "appear" partway through a Run All.
|
|
179
223
|
catalogs = _list_catalogs()
|
|
180
224
|
_build_cascading_widgets("source", "Source")
|
|
181
225
|
_build_cascading_widgets("target", "Target")
|
|
226
|
+
_build_check_widgets()
|
|
182
227
|
|
|
183
228
|
print(
|
|
184
229
|
"Widgets refreshed for the CURRENT Catalog/Schema selection.\n"
|
|
@@ -188,7 +233,7 @@ print(
|
|
|
188
233
|
|
|
189
234
|
# COMMAND ----------
|
|
190
235
|
|
|
191
|
-
# MAGIC %md ###
|
|
236
|
+
# MAGIC %md ### About the check groups
|
|
192
237
|
# MAGIC "Catalog & Schema" only checks that the catalogs/schemas/tables exist
|
|
193
238
|
# MAGIC and match by name - it produces no per-table PASS/FAIL on its own, so
|
|
194
239
|
# MAGIC running it alone reports every table as **SKIPPED**. That's expected,
|
|
@@ -199,34 +244,6 @@ print(
|
|
|
199
244
|
|
|
200
245
|
# COMMAND ----------
|
|
201
246
|
|
|
202
|
-
# One Yes/No dropdown per check group, instead of a single multiselect -
|
|
203
|
-
# dbutils.widgets.multiselect enforces an internal "selection sequence"
|
|
204
|
-
# validation against its own default that was observed to reject even a
|
|
205
|
-
# same-length, same-items default on some Databricks runtimes/existing-
|
|
206
|
-
# widget states (DefaultValueNotInChoicesList), with no reliable
|
|
207
|
-
# workaround. Three independent dropdowns have no such cross-widget
|
|
208
|
-
# validation and are the more robust primitive for this.
|
|
209
|
-
_YES_NO = ["Yes", "No"]
|
|
210
|
-
_schema_default = "Yes"
|
|
211
|
-
# --mode schema means "schema SHAPE only" - that shape (column names,
|
|
212
|
-
# types, nullability) is validated by the COLUMN checks, so Column stays
|
|
213
|
-
# on and only the expensive Row checks (row counts + row-level data) are
|
|
214
|
-
# off by default. Turning Column off too would leave nothing producing a
|
|
215
|
-
# per-table verdict, reporting every table as SKIPPED.
|
|
216
|
-
_column_default = "Yes"
|
|
217
|
-
_row_default = "No" if MODE == "schema" else "Yes"
|
|
218
|
-
dbutils.widgets.dropdown("check_catalog_schema", _schema_default, _YES_NO, "Run Catalog & Schema checks?")
|
|
219
|
-
dbutils.widgets.dropdown("check_column", _column_default, _YES_NO, "Run Column checks?")
|
|
220
|
-
dbutils.widgets.dropdown("check_row", _row_default, _YES_NO, "Run Row checks?")
|
|
221
|
-
|
|
222
|
-
if SHOW_OPTIONAL_WIDGETS:
|
|
223
|
-
dbutils.widgets.text("only_columns", "", "Only compare these columns (comma-separated, optional)")
|
|
224
|
-
dbutils.widgets.text("ignore_columns", "", "Skip these columns entirely (comma-separated, optional)")
|
|
225
|
-
dbutils.widgets.text("row_filter", "", "Row filter - SQL WHERE-fragment (optional)")
|
|
226
|
-
dbutils.widgets.text("primary_key", "", "Primary key column(s) (comma-separated, optional)")
|
|
227
|
-
|
|
228
|
-
# COMMAND ----------
|
|
229
|
-
|
|
230
247
|
# MAGIC %md ### Run validation
|
|
231
248
|
|
|
232
249
|
# COMMAND ----------
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablevalidator-databricks
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
|
|
5
5
|
License: MIT
|
|
6
6
|
Requires-Python: >=3.9
|
|
@@ -35,6 +35,14 @@ there's nothing else to install separately.
|
|
|
35
35
|
|
|
36
36
|
## Usage
|
|
37
37
|
|
|
38
|
+
```bash
|
|
39
|
+
tablevalidator-databricks info
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Prints a full usage guide - the steps, the three modes, what each check
|
|
43
|
+
group actually does, schema-wide sweeps, the cascading-dropdown caveat,
|
|
44
|
+
manual overrides, and version pinning. Start here if you're new to it.
|
|
45
|
+
|
|
38
46
|
```bash
|
|
39
47
|
tablevalidator-databricks init
|
|
40
48
|
```
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Tests for the tablevalidator-databricks CLI (cli/main.py)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from typer.testing import CliRunner
|
|
8
|
+
|
|
9
|
+
from tablevalidator_databricks.cli.main import app
|
|
10
|
+
|
|
11
|
+
runner = CliRunner()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_app_is_importable() -> None:
|
|
15
|
+
assert app is not None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_help_lists_init_command() -> None:
|
|
19
|
+
result = runner.invoke(app, ["--help"])
|
|
20
|
+
assert result.exit_code == 0
|
|
21
|
+
assert "init" in result.output
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_init_writes_notebook_to_default_path(tmp_path: Path) -> None:
|
|
25
|
+
output = tmp_path / "TableValidator.py"
|
|
26
|
+
|
|
27
|
+
result = runner.invoke(app, ["init", "--output", str(output)])
|
|
28
|
+
|
|
29
|
+
assert result.exit_code == 0
|
|
30
|
+
assert output.exists()
|
|
31
|
+
assert "Generated notebook" in result.output
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_init_respects_mode_flag(tmp_path: Path) -> None:
|
|
35
|
+
output = tmp_path / "basic.py"
|
|
36
|
+
|
|
37
|
+
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "basic"])
|
|
38
|
+
|
|
39
|
+
assert result.exit_code == 0
|
|
40
|
+
content = output.read_text(encoding="utf-8")
|
|
41
|
+
assert 'MODE = "basic"' in content
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_init_rejects_invalid_mode(tmp_path: Path) -> None:
|
|
45
|
+
output = tmp_path / "x.py"
|
|
46
|
+
|
|
47
|
+
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "bogus"])
|
|
48
|
+
|
|
49
|
+
assert result.exit_code == 1
|
|
50
|
+
assert not output.exists()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_info_command_runs() -> None:
|
|
54
|
+
result = runner.invoke(app, ["info"])
|
|
55
|
+
assert result.exit_code == 0
|
|
56
|
+
assert "tablevalidator-databricks" in result.output
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_info_covers_every_documented_behaviour() -> None:
|
|
60
|
+
"""`info` is the single place a user is pointed at to learn this tool,
|
|
61
|
+
so it must stay in step with what the generated notebook actually
|
|
62
|
+
does - each of these corresponds to a real behaviour (or a real
|
|
63
|
+
reported confusion) the notebook has."""
|
|
64
|
+
result = runner.invoke(app, ["info"])
|
|
65
|
+
|
|
66
|
+
for expected in (
|
|
67
|
+
"init", # the generate step
|
|
68
|
+
"Workspace -> Import", # how to get it into Databricks
|
|
69
|
+
"basic", # the three modes
|
|
70
|
+
"full",
|
|
71
|
+
"schema",
|
|
72
|
+
"Yes/No", # check groups are dropdowns, not a multiselect
|
|
73
|
+
"SKIPPED", # why catalog+schema alone reports SKIPPED
|
|
74
|
+
"(all tables in schema)", # schema-wide sweep
|
|
75
|
+
"Create / refresh widgets", # the cascading-dropdown caveat
|
|
76
|
+
"(manual override)", # typing a name the dropdown misses
|
|
77
|
+
"table-validator>=", # the version floor
|
|
78
|
+
):
|
|
79
|
+
assert expected in result.output, f"`info` no longer mentions {expected!r}"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_info_renders_the_real_version_floor() -> None:
|
|
83
|
+
"""The floor shown by `info` must be substituted from package
|
|
84
|
+
metadata, not left as a literal placeholder."""
|
|
85
|
+
from tablevalidator_databricks.generator.generator import _min_core_version
|
|
86
|
+
|
|
87
|
+
result = runner.invoke(app, ["info"])
|
|
88
|
+
|
|
89
|
+
assert "{MIN_CORE_VERSION}" not in result.output
|
|
90
|
+
assert f'table-validator>={_min_core_version()}' in result.output
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_version_flag_reports_installed_version() -> None:
|
|
94
|
+
"""`--version` exists so a user can check what they have installed
|
|
95
|
+
against what generated a notebook - the fastest way to diagnose a
|
|
96
|
+
"my widgets look different" report."""
|
|
97
|
+
from tablevalidator_databricks import __version__
|
|
98
|
+
|
|
99
|
+
result = runner.invoke(app, ["--version"])
|
|
100
|
+
|
|
101
|
+
assert result.exit_code == 0
|
|
102
|
+
assert __version__ in result.output
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_init_prints_the_generator_version(tmp_path: Path) -> None:
|
|
106
|
+
"""init names the version it generated with, so the number is visible
|
|
107
|
+
at the moment the file is created, not just inside it."""
|
|
108
|
+
from tablevalidator_databricks import __version__
|
|
109
|
+
|
|
110
|
+
result = runner.invoke(app, ["init", "--output", str(tmp_path / "nb.py")])
|
|
111
|
+
|
|
112
|
+
assert result.exit_code == 0
|
|
113
|
+
assert __version__ in result.output
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_info_explains_missing_widgets() -> None:
|
|
117
|
+
"""The "a widget is missing" case is a real, reported confusion -
|
|
118
|
+
info must explain that widgets are baked in at generation time."""
|
|
119
|
+
result = runner.invoke(app, ["info"])
|
|
120
|
+
|
|
121
|
+
assert "WIDGET IS MISSING" in result.output
|
|
122
|
+
assert "--version" in result.output
|
|
123
|
+
assert "0.1.3" in result.output # the version-history table
|
|
@@ -116,3 +116,18 @@ def test_min_core_version_reads_from_package_metadata() -> None:
|
|
|
116
116
|
dependency, so the two can never silently drift apart."""
|
|
117
117
|
assert _min_core_version() != "0.0.0"
|
|
118
118
|
assert re.match(r"^[0-9]+\.[0-9]+\.[0-9]+$", _min_core_version())
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_generated_notebook_stamps_the_generator_version(tmp_path: Path) -> None:
|
|
122
|
+
"""The notebook records which version generated it, so a user seeing
|
|
123
|
+
unexpected widgets can tell at a glance whether their file is stale."""
|
|
124
|
+
from tablevalidator_databricks import __version__
|
|
125
|
+
|
|
126
|
+
output = tmp_path / "TableValidator.py"
|
|
127
|
+
generate_notebook(output, "full")
|
|
128
|
+
content = output.read_text(encoding="utf-8")
|
|
129
|
+
|
|
130
|
+
assert __version__ in content
|
|
131
|
+
# And it explains that regenerating is what picks up new widgets.
|
|
132
|
+
assert "upgrading the" in content.lower()
|
|
133
|
+
assert "tablevalidator-databricks init" in content
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
"""CLI entry point: `tablevalidator-databricks` console script."""
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
import typer
|
|
6
|
-
|
|
7
|
-
from tablevalidator_databricks.generator.generator import VALID_MODES, generate_notebook
|
|
8
|
-
|
|
9
|
-
app = typer.Typer(
|
|
10
|
-
name="tablevalidator-databricks",
|
|
11
|
-
help="Generate a widget-driven Databricks notebook UI for table-validator.",
|
|
12
|
-
no_args_is_help=True,
|
|
13
|
-
)
|
|
14
|
-
|
|
15
|
-
DEFAULT_OUTPUT_PATH = Path("TableValidator.py")
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@app.command()
|
|
19
|
-
def init(
|
|
20
|
-
output: Path = typer.Option(
|
|
21
|
-
DEFAULT_OUTPUT_PATH,
|
|
22
|
-
"--output",
|
|
23
|
-
help="Path to write the generated notebook to (default: ./TableValidator.py).",
|
|
24
|
-
),
|
|
25
|
-
mode: str = typer.Option(
|
|
26
|
-
"full",
|
|
27
|
-
"--mode",
|
|
28
|
-
help=(
|
|
29
|
-
"basic: only the check-group multiselect and table pickers. "
|
|
30
|
-
"full (default): also shows optional filter widgets (only/ "
|
|
31
|
-
"ignore columns, row filter, primary key). "
|
|
32
|
-
"schema: pre-selects only the Catalog & Schema check group."
|
|
33
|
-
),
|
|
34
|
-
),
|
|
35
|
-
) -> None:
|
|
36
|
-
"""Generate a Databricks notebook with dbutils widgets for source/
|
|
37
|
-
target catalog.schema.table selection and validation checks - fill in
|
|
38
|
-
the widgets and run the notebook, no code to write.
|
|
39
|
-
|
|
40
|
-
Import the generated file into a Databricks workspace via
|
|
41
|
-
Workspace -> Import, with format set to "Source" (or "File" -> upload
|
|
42
|
-
directly, depending on your workspace UI version) - Databricks
|
|
43
|
-
recognizes the "# Databricks notebook source" header and cell
|
|
44
|
-
(# COMMAND ----------) markers and opens it as a real notebook, not a
|
|
45
|
-
plain text file.
|
|
46
|
-
"""
|
|
47
|
-
if mode not in VALID_MODES:
|
|
48
|
-
typer.secho(
|
|
49
|
-
f"Invalid --mode {mode!r} - must be one of {', '.join(VALID_MODES)}.",
|
|
50
|
-
fg=typer.colors.RED,
|
|
51
|
-
)
|
|
52
|
-
raise typer.Exit(code=1)
|
|
53
|
-
|
|
54
|
-
generate_notebook(output, mode)
|
|
55
|
-
|
|
56
|
-
typer.echo(f"Generated notebook: {output.resolve()}")
|
|
57
|
-
typer.echo(
|
|
58
|
-
"Import it into Databricks (Workspace -> Import, format 'Source'), "
|
|
59
|
-
"fill in the widgets, and Run All."
|
|
60
|
-
)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
@app.command()
|
|
64
|
-
def info() -> None:
|
|
65
|
-
"""Show what this tool does and how to use the generated notebook."""
|
|
66
|
-
typer.echo(
|
|
67
|
-
"\n"
|
|
68
|
-
"tablevalidator-databricks - Databricks notebook UI for table-validator\n"
|
|
69
|
-
"------------------------------------------------------------------------\n"
|
|
70
|
-
"Generates a Databricks notebook with dbutils widgets (dynamic "
|
|
71
|
-
"catalog/schema/table dropdowns sourced from Unity Catalog, plus a "
|
|
72
|
-
"validation-checks multiselect) as a wrapper around the "
|
|
73
|
-
"table-validator package's validate_tables() API - no Python code "
|
|
74
|
-
"to write, no separate credentials to configure (the notebook "
|
|
75
|
-
"reuses its own ambient Spark session, same as validate_tables() "
|
|
76
|
-
"itself).\n"
|
|
77
|
-
"\n"
|
|
78
|
-
"Workflow:\n"
|
|
79
|
-
"\n"
|
|
80
|
-
" 1. tablevalidator-databricks init\n"
|
|
81
|
-
" Writes a ready-to-use notebook (./TableValidator.py by "
|
|
82
|
-
"default). Use --output to write elsewhere, --mode to pick basic/"
|
|
83
|
-
"full/schema (see 'tablevalidator-databricks init --help').\n"
|
|
84
|
-
"\n"
|
|
85
|
-
" 2. Import the generated file into Databricks\n"
|
|
86
|
-
" Workspace -> Import, format 'Source'.\n"
|
|
87
|
-
"\n"
|
|
88
|
-
" 3. Fill in the widgets and Run All\n"
|
|
89
|
-
" Pick Source/Target Catalog, Schema, Table, and which checks "
|
|
90
|
-
"to run, then run the notebook top to bottom. Re-run the "
|
|
91
|
-
"'Create / refresh widgets' cell after changing a Catalog/Schema "
|
|
92
|
-
"selection, to repopulate the dropdowns below it (a Databricks "
|
|
93
|
-
"widget limitation, not a bug). Can't find a name in a dropdown? "
|
|
94
|
-
"Each dropdown has a '(manual override)' text widget next to it - "
|
|
95
|
-
"type the exact name there instead.\n"
|
|
96
|
-
"\n"
|
|
97
|
-
"This package contains no validation logic of its own - every "
|
|
98
|
-
"generated notebook calls directly into table-validator's own "
|
|
99
|
-
"validate_tables() engine, so results are identical to calling it "
|
|
100
|
-
"yourself in a code cell.\n"
|
|
101
|
-
)
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
"""Tests for the tablevalidator-databricks CLI (cli/main.py)."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
|
|
7
|
-
from typer.testing import CliRunner
|
|
8
|
-
|
|
9
|
-
from tablevalidator_databricks.cli.main import app
|
|
10
|
-
|
|
11
|
-
runner = CliRunner()
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def test_app_is_importable() -> None:
|
|
15
|
-
assert app is not None
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def test_help_lists_init_command() -> None:
|
|
19
|
-
result = runner.invoke(app, ["--help"])
|
|
20
|
-
assert result.exit_code == 0
|
|
21
|
-
assert "init" in result.output
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def test_init_writes_notebook_to_default_path(tmp_path: Path) -> None:
|
|
25
|
-
output = tmp_path / "TableValidator.py"
|
|
26
|
-
|
|
27
|
-
result = runner.invoke(app, ["init", "--output", str(output)])
|
|
28
|
-
|
|
29
|
-
assert result.exit_code == 0
|
|
30
|
-
assert output.exists()
|
|
31
|
-
assert "Generated notebook" in result.output
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def test_init_respects_mode_flag(tmp_path: Path) -> None:
|
|
35
|
-
output = tmp_path / "basic.py"
|
|
36
|
-
|
|
37
|
-
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "basic"])
|
|
38
|
-
|
|
39
|
-
assert result.exit_code == 0
|
|
40
|
-
content = output.read_text(encoding="utf-8")
|
|
41
|
-
assert 'MODE = "basic"' in content
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def test_init_rejects_invalid_mode(tmp_path: Path) -> None:
|
|
45
|
-
output = tmp_path / "x.py"
|
|
46
|
-
|
|
47
|
-
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "bogus"])
|
|
48
|
-
|
|
49
|
-
assert result.exit_code == 1
|
|
50
|
-
assert not output.exists()
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def test_info_command_runs() -> None:
|
|
54
|
-
result = runner.invoke(app, ["info"])
|
|
55
|
-
assert result.exit_code == 0
|
|
56
|
-
assert "tablevalidator-databricks" in result.output
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|