tablevalidator-databricks 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/PKG-INFO +17 -2
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/README.md +16 -1
- tablevalidator_databricks-0.1.5/tablevalidator_databricks/cli/main.py +173 -0
- tablevalidator_databricks-0.1.5/tablevalidator_databricks/generator/generator.py +66 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks/templates/widget_notebook.py.tmpl +26 -2
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks.egg-info/PKG-INFO +17 -2
- tablevalidator_databricks-0.1.5/tablevalidator_databricks.egg-info/scm_version.json +8 -0
- tablevalidator_databricks-0.1.5/tests/test_cli.py +90 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tests/test_generated_notebook_execution.py +8 -2
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tests/test_generator.py +28 -1
- tablevalidator_databricks-0.1.3/tablevalidator_databricks/cli/main.py +0 -101
- tablevalidator_databricks-0.1.3/tablevalidator_databricks/generator/generator.py +0 -39
- tablevalidator_databricks-0.1.3/tablevalidator_databricks.egg-info/scm_version.json +0 -8
- tablevalidator_databricks-0.1.3/tests/test_cli.py +0 -56
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/pyproject.toml +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/setup.cfg +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks/__init__.py +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks/cli/__init__.py +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks/generator/__init__.py +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks.egg-info/SOURCES.txt +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks.egg-info/dependency_links.txt +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks.egg-info/entry_points.txt +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks.egg-info/requires.txt +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks.egg-info/scm_file_list.json +0 -0
- {tablevalidator_databricks-0.1.3 → tablevalidator_databricks-0.1.5}/tablevalidator_databricks.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablevalidator-databricks
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
|
|
5
5
|
License: MIT
|
|
6
6
|
Requires-Python: >=3.9
|
|
@@ -35,6 +35,14 @@ there's nothing else to install separately.
|
|
|
35
35
|
|
|
36
36
|
## Usage
|
|
37
37
|
|
|
38
|
+
```bash
|
|
39
|
+
tablevalidator-databricks info
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Prints a full usage guide - the steps, the three modes, what each check
|
|
43
|
+
group actually does, schema-wide sweeps, the cascading-dropdown caveat,
|
|
44
|
+
manual overrides, and version pinning. Start here if you're new to it.
|
|
45
|
+
|
|
38
46
|
```bash
|
|
39
47
|
tablevalidator-databricks init
|
|
40
48
|
```
|
|
@@ -48,10 +56,17 @@ tablevalidator-databricks init --output MyValidation.py --mode full
|
|
|
48
56
|
- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
|
|
49
57
|
- `--mode full` (default) - also shows optional filter widgets (only/
|
|
50
58
|
ignore columns, row filter, primary key). Every check defaults to "Yes".
|
|
51
|
-
- `--mode schema` - same widgets as `basic`, but
|
|
59
|
+
- `--mode schema` - same widgets as `basic`, but the Row checks default to
|
|
52
60
|
"No" - a lightweight schema-shape-only comparison out of the box (still
|
|
53
61
|
changeable per-run).
|
|
54
62
|
|
|
63
|
+
> **Note on the check groups**: "Catalog & Schema" only verifies that
|
|
64
|
+
> catalogs/schemas/tables exist and match by name - it produces no
|
|
65
|
+
> per-table PASS/FAIL by itself, so running it alone reports every table
|
|
66
|
+
> as `SKIPPED`. The per-table verdict comes from the **Column** checks
|
|
67
|
+
> (names, types, nullability, statistics) and **Row** checks (row counts,
|
|
68
|
+
> row-level data).
|
|
69
|
+
|
|
55
70
|
Then, in your Databricks workspace: **Workspace -> Import**, format
|
|
56
71
|
"Source", and select the generated file. Databricks recognizes the
|
|
57
72
|
`# Databricks notebook source` header and `# COMMAND ----------` cell
|
|
@@ -21,6 +21,14 @@ there's nothing else to install separately.
|
|
|
21
21
|
|
|
22
22
|
## Usage
|
|
23
23
|
|
|
24
|
+
```bash
|
|
25
|
+
tablevalidator-databricks info
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Prints a full usage guide - the steps, the three modes, what each check
|
|
29
|
+
group actually does, schema-wide sweeps, the cascading-dropdown caveat,
|
|
30
|
+
manual overrides, and version pinning. Start here if you're new to it.
|
|
31
|
+
|
|
24
32
|
```bash
|
|
25
33
|
tablevalidator-databricks init
|
|
26
34
|
```
|
|
@@ -34,10 +42,17 @@ tablevalidator-databricks init --output MyValidation.py --mode full
|
|
|
34
42
|
- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
|
|
35
43
|
- `--mode full` (default) - also shows optional filter widgets (only/
|
|
36
44
|
ignore columns, row filter, primary key). Every check defaults to "Yes".
|
|
37
|
-
- `--mode schema` - same widgets as `basic`, but
|
|
45
|
+
- `--mode schema` - same widgets as `basic`, but the Row checks default to
|
|
38
46
|
"No" - a lightweight schema-shape-only comparison out of the box (still
|
|
39
47
|
changeable per-run).
|
|
40
48
|
|
|
49
|
+
> **Note on the check groups**: "Catalog & Schema" only verifies that
|
|
50
|
+
> catalogs/schemas/tables exist and match by name - it produces no
|
|
51
|
+
> per-table PASS/FAIL by itself, so running it alone reports every table
|
|
52
|
+
> as `SKIPPED`. The per-table verdict comes from the **Column** checks
|
|
53
|
+
> (names, types, nullability, statistics) and **Row** checks (row counts,
|
|
54
|
+
> row-level data).
|
|
55
|
+
|
|
41
56
|
Then, in your Databricks workspace: **Workspace -> Import**, format
|
|
42
57
|
"Source", and select the generated file. Databricks recognizes the
|
|
43
58
|
`# Databricks notebook source` header and `# COMMAND ----------` cell
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""CLI entry point: `tablevalidator-databricks` console script."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from tablevalidator_databricks.generator.generator import (
|
|
8
|
+
VALID_MODES,
|
|
9
|
+
_min_core_version,
|
|
10
|
+
generate_notebook,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(
|
|
14
|
+
name="tablevalidator-databricks",
|
|
15
|
+
help=(
|
|
16
|
+
"Generate a widget-driven Databricks notebook UI for table-validator. "
|
|
17
|
+
"Run 'tablevalidator-databricks info' for a full usage guide."
|
|
18
|
+
),
|
|
19
|
+
no_args_is_help=True,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
DEFAULT_OUTPUT_PATH = Path("TableValidator.py")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@app.command()
|
|
26
|
+
def init(
|
|
27
|
+
output: Path = typer.Option(
|
|
28
|
+
DEFAULT_OUTPUT_PATH,
|
|
29
|
+
"--output",
|
|
30
|
+
help="Path to write the generated notebook to (default: ./TableValidator.py).",
|
|
31
|
+
),
|
|
32
|
+
mode: str = typer.Option(
|
|
33
|
+
"full",
|
|
34
|
+
"--mode",
|
|
35
|
+
help=(
|
|
36
|
+
"basic: only the check-group Yes/No dropdowns and table pickers. "
|
|
37
|
+
"full (default): also shows optional filter widgets (only/ "
|
|
38
|
+
"ignore columns, row filter, primary key). "
|
|
39
|
+
"schema: same widgets as basic, but the Row checks default to No "
|
|
40
|
+
"(schema-shape-only comparison)."
|
|
41
|
+
),
|
|
42
|
+
),
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Generate a Databricks notebook with dbutils widgets for source/
|
|
45
|
+
target catalog.schema.table selection and validation checks - fill in
|
|
46
|
+
the widgets and run the notebook, no code to write.
|
|
47
|
+
|
|
48
|
+
Import the generated file into a Databricks workspace via
|
|
49
|
+
Workspace -> Import, with format set to "Source" (or "File" -> upload
|
|
50
|
+
directly, depending on your workspace UI version) - Databricks
|
|
51
|
+
recognizes the "# Databricks notebook source" header and cell
|
|
52
|
+
(# COMMAND ----------) markers and opens it as a real notebook, not a
|
|
53
|
+
plain text file.
|
|
54
|
+
"""
|
|
55
|
+
if mode not in VALID_MODES:
|
|
56
|
+
typer.secho(
|
|
57
|
+
f"Invalid --mode {mode!r} - must be one of {', '.join(VALID_MODES)}.",
|
|
58
|
+
fg=typer.colors.RED,
|
|
59
|
+
)
|
|
60
|
+
raise typer.Exit(code=1)
|
|
61
|
+
|
|
62
|
+
generate_notebook(output, mode)
|
|
63
|
+
|
|
64
|
+
typer.echo(f"Generated notebook: {output.resolve()}")
|
|
65
|
+
typer.echo(
|
|
66
|
+
"Import it into Databricks (Workspace -> Import, format 'Source'), "
|
|
67
|
+
"fill in the widgets, and Run All."
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
_INFO_TEXT = """
|
|
72
|
+
tablevalidator-databricks - Databricks notebook UI for table-validator
|
|
73
|
+
------------------------------------------------------------------------
|
|
74
|
+
Generates a Databricks notebook driven entirely by dbutils widgets -
|
|
75
|
+
Catalog/Schema/Table dropdowns populated live from Unity Catalog, plus
|
|
76
|
+
Yes/No dropdowns for which checks to run - as a wrapper around the
|
|
77
|
+
table-validator package's validate_tables() API. No Python to write, and
|
|
78
|
+
no separate credentials to configure (the notebook reuses its own
|
|
79
|
+
ambient Spark session, same as validate_tables() itself).
|
|
80
|
+
|
|
81
|
+
This package contains NO validation logic of its own. Every generated
|
|
82
|
+
notebook calls straight into table-validator's engine, so results are
|
|
83
|
+
identical to calling validate_tables() yourself in a code cell.
|
|
84
|
+
|
|
85
|
+
STEPS
|
|
86
|
+
-----
|
|
87
|
+
1. tablevalidator-databricks init
|
|
88
|
+
Writes a ready-to-run notebook (./TableValidator.py by default).
|
|
89
|
+
--output PATH write somewhere else
|
|
90
|
+
--mode MODE basic | full (default) | schema
|
|
91
|
+
See 'tablevalidator-databricks init --help' for details.
|
|
92
|
+
|
|
93
|
+
2. Import it into Databricks
|
|
94
|
+
Workspace -> Import -> format "Source", pick the generated file.
|
|
95
|
+
Databricks reads the '# Databricks notebook source' header and
|
|
96
|
+
'# COMMAND ----------' markers and opens it as a real notebook.
|
|
97
|
+
|
|
98
|
+
3. Fill in the widgets, then Run All
|
|
99
|
+
Pick Source and Target Catalog / Schema / Table, choose which
|
|
100
|
+
checks to run, and run the notebook top to bottom.
|
|
101
|
+
|
|
102
|
+
MODES
|
|
103
|
+
-----
|
|
104
|
+
basic Check-group Yes/No dropdowns and the table pickers only.
|
|
105
|
+
full (default) Also shows optional filter widgets: only-columns,
|
|
106
|
+
ignore-columns, row filter, primary key.
|
|
107
|
+
schema Same widgets as basic, but the Row checks default to "No" -
|
|
108
|
+
a lightweight schema-shape-only comparison.
|
|
109
|
+
|
|
110
|
+
THE CHECK GROUPS
|
|
111
|
+
----------------
|
|
112
|
+
Three independent Yes/No dropdowns:
|
|
113
|
+
|
|
114
|
+
Catalog & Schema Do the catalogs/schemas/tables exist and match by
|
|
115
|
+
name? This produces NO per-table PASS/FAIL on its
|
|
116
|
+
own - running it alone reports every table as
|
|
117
|
+
SKIPPED. That is expected, not an error.
|
|
118
|
+
Column Column names, data types, nullability, null and
|
|
119
|
+
distinct counts, min/max. This is where schema-
|
|
120
|
+
shape problems are actually caught.
|
|
121
|
+
Row Row counts and row-level data comparison. The most
|
|
122
|
+
expensive group.
|
|
123
|
+
|
|
124
|
+
Turning every group off falls back to running everything, rather than
|
|
125
|
+
silently validating nothing.
|
|
126
|
+
|
|
127
|
+
VALIDATING A WHOLE SCHEMA
|
|
128
|
+
-------------------------
|
|
129
|
+
Set the Table dropdown to "(all tables in schema)" on BOTH sides to
|
|
130
|
+
compare every identically-named table in that schema in one run - the
|
|
131
|
+
same schema-wide sweep validate_tables() itself supports. A primary key
|
|
132
|
+
can't be used in this mode (a single key can't apply to every table), so
|
|
133
|
+
leave the Primary Key widget blank. Comparing every schema in a catalog
|
|
134
|
+
is not supported yet - Schema must be a specific pick.
|
|
135
|
+
|
|
136
|
+
CASCADING DROPDOWNS
|
|
137
|
+
-------------------
|
|
138
|
+
Databricks widgets have no on-change event, so choosing a new Catalog
|
|
139
|
+
does NOT repopulate the Schema dropdown below it (same for Schema ->
|
|
140
|
+
Table). After changing a Catalog or Schema, re-run the notebook's
|
|
141
|
+
"Create / refresh widgets" cell (Shift+Enter) before continuing. This is
|
|
142
|
+
a Databricks platform limitation, not a bug in the notebook - and if you
|
|
143
|
+
skip it, the notebook fails with a clear error naming the cell to re-run
|
|
144
|
+
rather than silently comparing the wrong table.
|
|
145
|
+
|
|
146
|
+
NAME NOT IN A DROPDOWN?
|
|
147
|
+
-----------------------
|
|
148
|
+
Every Catalog/Schema/Table dropdown has a "(manual override)" text
|
|
149
|
+
widget beside it. Type an exact name there and it is used instead of the
|
|
150
|
+
dropdown - handy for a permissions lag, an odd sort order, anything the
|
|
151
|
+
listing misses. It is used as-is, not checked against the live catalog.
|
|
152
|
+
Leave the overrides blank to just use the dropdowns.
|
|
153
|
+
|
|
154
|
+
VERSION PINNING (IMPORTANT)
|
|
155
|
+
---------------------------
|
|
156
|
+
The generated notebook installs the engine with a floor:
|
|
157
|
+
%pip install "table-validator>={MIN_CORE_VERSION}"
|
|
158
|
+
Do NOT lower that pin, and do not confuse it with this package's own
|
|
159
|
+
version number - the two packages are versioned completely
|
|
160
|
+
independently. validate_tables() only exists in table-validator 0.1.9
|
|
161
|
+
and later, so pinning the engine to this generator's version installs
|
|
162
|
+
something far too old and fails with:
|
|
163
|
+
ImportError: cannot import name 'validate_tables'
|
|
164
|
+
|
|
165
|
+
Re-run 'tablevalidator-databricks init' after upgrading this package -
|
|
166
|
+
an already-generated notebook keeps whatever code it was written with.
|
|
167
|
+
""".strip("\n")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@app.command()
|
|
171
|
+
def info() -> None:
|
|
172
|
+
"""Show what this tool does and how to use the generated notebook."""
|
|
173
|
+
typer.echo("\n" + _INFO_TEXT.replace("{MIN_CORE_VERSION}", _min_core_version()) + "\n")
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Renders the bundled widget_notebook.py.tmpl into a real Databricks
|
|
3
|
+
notebook file. No Jinja2 - the template has only a handful of
|
|
4
|
+
placeholders, substituted via plain str.replace(), so a templating
|
|
5
|
+
dependency isn't worth adding for this.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import importlib.resources
|
|
11
|
+
import re
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, requires
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
VALID_MODES = ("basic", "full", "schema")
|
|
16
|
+
|
|
17
|
+
# Fallback floor for the generated notebook's `%pip install
|
|
18
|
+
# "table-validator>=X"` line, used only if this package's own declared
|
|
19
|
+
# dependency can't be read at runtime. validate_tables() - the API the
|
|
20
|
+
# generated notebook calls - first shipped in table-validator 0.1.9.
|
|
21
|
+
_FALLBACK_MIN_CORE_VERSION = "0.1.9"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _min_core_version() -> str:
|
|
25
|
+
"""The minimum `table-validator` version the generated notebook should
|
|
26
|
+
install, read from THIS package's own declared dependency so the two
|
|
27
|
+
can never drift apart. Falls back to a known-good floor if the
|
|
28
|
+
metadata isn't readable (e.g. running from a source tree that was
|
|
29
|
+
never installed)."""
|
|
30
|
+
try:
|
|
31
|
+
for req in requires("tablevalidator-databricks") or []:
|
|
32
|
+
# e.g. "table-validator>=0.1.19" / "table-validator >= 0.1.19"
|
|
33
|
+
match = re.match(r"^\s*table[-_]validator\s*>=\s*([0-9][^\s,;]*)", req)
|
|
34
|
+
if match:
|
|
35
|
+
return match.group(1)
|
|
36
|
+
except PackageNotFoundError:
|
|
37
|
+
pass
|
|
38
|
+
return _FALLBACK_MIN_CORE_VERSION
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def generate_notebook(output: Path, mode: str) -> None:
|
|
42
|
+
"""Write a widget-driven Databricks notebook to `output`.
|
|
43
|
+
|
|
44
|
+
mode controls which optional widgets/pre-selected checks the notebook
|
|
45
|
+
starts with - see tablevalidator_databricks/cli/main.py's `init`
|
|
46
|
+
command docstring for what each mode means. The underlying comparison
|
|
47
|
+
logic is identical regardless of mode; only the widgets shown differ,
|
|
48
|
+
so all three modes render from the same template file rather than
|
|
49
|
+
three separate ones.
|
|
50
|
+
"""
|
|
51
|
+
if mode not in VALID_MODES:
|
|
52
|
+
raise ValueError(f"mode must be one of {VALID_MODES}, got {mode!r}")
|
|
53
|
+
|
|
54
|
+
template = (
|
|
55
|
+
importlib.resources.files("tablevalidator_databricks.templates")
|
|
56
|
+
.joinpath("widget_notebook.py.tmpl")
|
|
57
|
+
.read_text(encoding="utf-8")
|
|
58
|
+
)
|
|
59
|
+
rendered = (
|
|
60
|
+
template.replace("{{MODE}}", mode)
|
|
61
|
+
.replace("{{SHOW_OPTIONAL_WIDGETS}}", "True" if mode == "full" else "False")
|
|
62
|
+
.replace("{{MIN_CORE_VERSION}}", _min_core_version())
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
output.write_text(rendered, encoding="utf-8")
|
|
@@ -31,7 +31,19 @@
|
|
|
31
31
|
|
|
32
32
|
# COMMAND ----------
|
|
33
33
|
|
|
34
|
-
# MAGIC %
|
|
34
|
+
# MAGIC %md ### Install the validation engine
|
|
35
|
+
# MAGIC Note the `>=` version floor: `validate_tables()` (the API every cell
|
|
36
|
+
# MAGIC below calls) only exists in `table-validator` {{MIN_CORE_VERSION}} and
|
|
37
|
+
# MAGIC later. **Do not pin this to a lower version** - in particular, do not
|
|
38
|
+
# MAGIC confuse it with the `tablevalidator-databricks` version that generated
|
|
39
|
+
# MAGIC this notebook; the two packages are versioned completely independently,
|
|
40
|
+
# MAGIC and pinning the engine to the generator's version number installs an
|
|
41
|
+
# MAGIC engine far too old, failing with
|
|
42
|
+
# MAGIC `ImportError: cannot import name 'validate_tables'`.
|
|
43
|
+
|
|
44
|
+
# COMMAND ----------
|
|
45
|
+
|
|
46
|
+
# MAGIC %pip install "table-validator>={{MIN_CORE_VERSION}}"
|
|
35
47
|
|
|
36
48
|
# COMMAND ----------
|
|
37
49
|
|
|
@@ -177,6 +189,13 @@ print(
|
|
|
177
189
|
# COMMAND ----------
|
|
178
190
|
|
|
179
191
|
# MAGIC %md ### Validation checks
|
|
192
|
+
# MAGIC "Catalog & Schema" only checks that the catalogs/schemas/tables exist
|
|
193
|
+
# MAGIC and match by name - it produces no per-table PASS/FAIL on its own, so
|
|
194
|
+
# MAGIC running it alone reports every table as **SKIPPED**. That's expected,
|
|
195
|
+
# MAGIC not an error: the per-table verdict comes from the **Column** checks
|
|
196
|
+
# MAGIC (names, types, nullability, statistics) and **Row** checks (row counts
|
|
197
|
+
# MAGIC and row-level data). Leave those on unless you specifically want a
|
|
198
|
+
# MAGIC name-level-only existence check.
|
|
180
199
|
|
|
181
200
|
# COMMAND ----------
|
|
182
201
|
|
|
@@ -189,7 +208,12 @@ print(
|
|
|
189
208
|
# validation and are the more robust primitive for this.
|
|
190
209
|
_YES_NO = ["Yes", "No"]
|
|
191
210
|
_schema_default = "Yes"
|
|
192
|
-
|
|
211
|
+
# --mode schema means "schema SHAPE only" - that shape (column names,
|
|
212
|
+
# types, nullability) is validated by the COLUMN checks, so Column stays
|
|
213
|
+
# on and only the expensive Row checks (row counts + row-level data) are
|
|
214
|
+
# off by default. Turning Column off too would leave nothing producing a
|
|
215
|
+
# per-table verdict, reporting every table as SKIPPED.
|
|
216
|
+
_column_default = "Yes"
|
|
193
217
|
_row_default = "No" if MODE == "schema" else "Yes"
|
|
194
218
|
dbutils.widgets.dropdown("check_catalog_schema", _schema_default, _YES_NO, "Run Catalog & Schema checks?")
|
|
195
219
|
dbutils.widgets.dropdown("check_column", _column_default, _YES_NO, "Run Column checks?")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablevalidator-databricks
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
|
|
5
5
|
License: MIT
|
|
6
6
|
Requires-Python: >=3.9
|
|
@@ -35,6 +35,14 @@ there's nothing else to install separately.
|
|
|
35
35
|
|
|
36
36
|
## Usage
|
|
37
37
|
|
|
38
|
+
```bash
|
|
39
|
+
tablevalidator-databricks info
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Prints a full usage guide - the steps, the three modes, what each check
|
|
43
|
+
group actually does, schema-wide sweeps, the cascading-dropdown caveat,
|
|
44
|
+
manual overrides, and version pinning. Start here if you're new to it.
|
|
45
|
+
|
|
38
46
|
```bash
|
|
39
47
|
tablevalidator-databricks init
|
|
40
48
|
```
|
|
@@ -48,10 +56,17 @@ tablevalidator-databricks init --output MyValidation.py --mode full
|
|
|
48
56
|
- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
|
|
49
57
|
- `--mode full` (default) - also shows optional filter widgets (only/
|
|
50
58
|
ignore columns, row filter, primary key). Every check defaults to "Yes".
|
|
51
|
-
- `--mode schema` - same widgets as `basic`, but
|
|
59
|
+
- `--mode schema` - same widgets as `basic`, but the Row checks default to
|
|
52
60
|
"No" - a lightweight schema-shape-only comparison out of the box (still
|
|
53
61
|
changeable per-run).
|
|
54
62
|
|
|
63
|
+
> **Note on the check groups**: "Catalog & Schema" only verifies that
|
|
64
|
+
> catalogs/schemas/tables exist and match by name - it produces no
|
|
65
|
+
> per-table PASS/FAIL by itself, so running it alone reports every table
|
|
66
|
+
> as `SKIPPED`. The per-table verdict comes from the **Column** checks
|
|
67
|
+
> (names, types, nullability, statistics) and **Row** checks (row counts,
|
|
68
|
+
> row-level data).
|
|
69
|
+
|
|
55
70
|
Then, in your Databricks workspace: **Workspace -> Import**, format
|
|
56
71
|
"Source", and select the generated file. Databricks recognizes the
|
|
57
72
|
`# Databricks notebook source` header and `# COMMAND ----------` cell
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Tests for the tablevalidator-databricks CLI (cli/main.py)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from typer.testing import CliRunner
|
|
8
|
+
|
|
9
|
+
from tablevalidator_databricks.cli.main import app
|
|
10
|
+
|
|
11
|
+
runner = CliRunner()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_app_is_importable() -> None:
|
|
15
|
+
assert app is not None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_help_lists_init_command() -> None:
|
|
19
|
+
result = runner.invoke(app, ["--help"])
|
|
20
|
+
assert result.exit_code == 0
|
|
21
|
+
assert "init" in result.output
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_init_writes_notebook_to_default_path(tmp_path: Path) -> None:
|
|
25
|
+
output = tmp_path / "TableValidator.py"
|
|
26
|
+
|
|
27
|
+
result = runner.invoke(app, ["init", "--output", str(output)])
|
|
28
|
+
|
|
29
|
+
assert result.exit_code == 0
|
|
30
|
+
assert output.exists()
|
|
31
|
+
assert "Generated notebook" in result.output
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_init_respects_mode_flag(tmp_path: Path) -> None:
|
|
35
|
+
output = tmp_path / "basic.py"
|
|
36
|
+
|
|
37
|
+
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "basic"])
|
|
38
|
+
|
|
39
|
+
assert result.exit_code == 0
|
|
40
|
+
content = output.read_text(encoding="utf-8")
|
|
41
|
+
assert 'MODE = "basic"' in content
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_init_rejects_invalid_mode(tmp_path: Path) -> None:
|
|
45
|
+
output = tmp_path / "x.py"
|
|
46
|
+
|
|
47
|
+
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "bogus"])
|
|
48
|
+
|
|
49
|
+
assert result.exit_code == 1
|
|
50
|
+
assert not output.exists()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_info_command_runs() -> None:
|
|
54
|
+
result = runner.invoke(app, ["info"])
|
|
55
|
+
assert result.exit_code == 0
|
|
56
|
+
assert "tablevalidator-databricks" in result.output
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_info_covers_every_documented_behaviour() -> None:
|
|
60
|
+
"""`info` is the single place a user is pointed at to learn this tool,
|
|
61
|
+
so it must stay in step with what the generated notebook actually
|
|
62
|
+
does - each of these corresponds to a real behaviour (or a real
|
|
63
|
+
reported confusion) the notebook has."""
|
|
64
|
+
result = runner.invoke(app, ["info"])
|
|
65
|
+
|
|
66
|
+
for expected in (
|
|
67
|
+
"init", # the generate step
|
|
68
|
+
"Workspace -> Import", # how to get it into Databricks
|
|
69
|
+
"basic", # the three modes
|
|
70
|
+
"full",
|
|
71
|
+
"schema",
|
|
72
|
+
"Yes/No", # check groups are dropdowns, not a multiselect
|
|
73
|
+
"SKIPPED", # why catalog+schema alone reports SKIPPED
|
|
74
|
+
"(all tables in schema)", # schema-wide sweep
|
|
75
|
+
"Create / refresh widgets", # the cascading-dropdown caveat
|
|
76
|
+
"(manual override)", # typing a name the dropdown misses
|
|
77
|
+
"table-validator>=", # the version floor
|
|
78
|
+
):
|
|
79
|
+
assert expected in result.output, f"`info` no longer mentions {expected!r}"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_info_renders_the_real_version_floor() -> None:
|
|
83
|
+
"""The floor shown by `info` must be substituted from package
|
|
84
|
+
metadata, not left as a literal placeholder."""
|
|
85
|
+
from tablevalidator_databricks.generator.generator import _min_core_version
|
|
86
|
+
|
|
87
|
+
result = runner.invoke(app, ["info"])
|
|
88
|
+
|
|
89
|
+
assert "{MIN_CORE_VERSION}" not in result.output
|
|
90
|
+
assert f'table-validator>={_min_core_version()}' in result.output
|
|
@@ -286,7 +286,11 @@ def test_full_and_basic_mode_default_all_checks_to_yes(tmp_path: Path) -> None:
|
|
|
286
286
|
}, f"mode={mode}"
|
|
287
287
|
|
|
288
288
|
|
|
289
|
-
def
|
|
289
|
+
def test_schema_mode_keeps_column_checks_on_and_turns_row_off(tmp_path: Path) -> None:
|
|
290
|
+
"""--mode schema means "schema SHAPE only". That shape is validated by
|
|
291
|
+
the COLUMN checks, so Column must stay ON - only the expensive Row
|
|
292
|
+
checks default off. Turning Column off too would leave nothing
|
|
293
|
+
producing a per-table verdict, reporting every table as SKIPPED."""
|
|
290
294
|
path = tmp_path / "schema.py"
|
|
291
295
|
generate_notebook(path, "schema")
|
|
292
296
|
spark = _fake_spark(
|
|
@@ -303,7 +307,9 @@ def test_schema_mode_defaults_column_and_row_checks_to_no(tmp_path: Path) -> Non
|
|
|
303
307
|
|
|
304
308
|
_run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
|
|
305
309
|
|
|
306
|
-
assert captured["enabled_validations"] == {
|
|
310
|
+
assert captured["enabled_validations"] == {
|
|
311
|
+
ValidationType.CATALOG, ValidationType.SCHEMA, ValidationType.COLUMN,
|
|
312
|
+
}
|
|
307
313
|
|
|
308
314
|
|
|
309
315
|
def test_deselecting_every_check_falls_back_to_run_everything(notebook_path):
|
|
@@ -3,11 +3,12 @@ no Databricks/Spark involved."""
|
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
+
import re
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
|
|
8
9
|
import pytest
|
|
9
10
|
|
|
10
|
-
from tablevalidator_databricks.generator.generator import generate_notebook
|
|
11
|
+
from tablevalidator_databricks.generator.generator import _min_core_version, generate_notebook
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
def test_generate_notebook_writes_a_databricks_source_notebook(tmp_path: Path) -> None:
|
|
@@ -89,3 +90,29 @@ def test_generate_notebook_no_leftover_placeholders(tmp_path: Path) -> None:
|
|
|
89
90
|
|
|
90
91
|
assert "{{" not in content
|
|
91
92
|
assert "}}" not in content
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_generated_notebook_pins_a_core_version_floor(tmp_path: Path) -> None:
|
|
96
|
+
"""Regression guard for a real user-reported failure: the notebook's
|
|
97
|
+
%pip install line must carry a `>=` floor for table-validator.
|
|
98
|
+
validate_tables() only exists in 0.1.9+, and the two packages are
|
|
99
|
+
versioned independently - a user who pinned the engine to THIS
|
|
100
|
+
package's version number (0.1.3) got an engine predating
|
|
101
|
+
validate_tables() entirely and an ImportError."""
|
|
102
|
+
output = tmp_path / "TableValidator.py"
|
|
103
|
+
generate_notebook(output, "full")
|
|
104
|
+
content = output.read_text(encoding="utf-8")
|
|
105
|
+
|
|
106
|
+
assert 'pip install "table-validator>=' in content
|
|
107
|
+
# The floor must be new enough to actually contain validate_tables().
|
|
108
|
+
match = re.search(r'pip install "table-validator>=([0-9]+)\.([0-9]+)\.([0-9]+)"', content)
|
|
109
|
+
assert match, "no parseable version floor in the %pip install line"
|
|
110
|
+
major, minor, patch = (int(g) for g in match.groups())
|
|
111
|
+
assert (major, minor, patch) >= (0, 1, 9)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_min_core_version_reads_from_package_metadata() -> None:
|
|
115
|
+
"""The floor comes from this package's own declared table-validator
|
|
116
|
+
dependency, so the two can never silently drift apart."""
|
|
117
|
+
assert _min_core_version() != "0.0.0"
|
|
118
|
+
assert re.match(r"^[0-9]+\.[0-9]+\.[0-9]+$", _min_core_version())
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
"""CLI entry point: `tablevalidator-databricks` console script."""
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
import typer
|
|
6
|
-
|
|
7
|
-
from tablevalidator_databricks.generator.generator import VALID_MODES, generate_notebook
|
|
8
|
-
|
|
9
|
-
app = typer.Typer(
|
|
10
|
-
name="tablevalidator-databricks",
|
|
11
|
-
help="Generate a widget-driven Databricks notebook UI for table-validator.",
|
|
12
|
-
no_args_is_help=True,
|
|
13
|
-
)
|
|
14
|
-
|
|
15
|
-
DEFAULT_OUTPUT_PATH = Path("TableValidator.py")
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@app.command()
|
|
19
|
-
def init(
|
|
20
|
-
output: Path = typer.Option(
|
|
21
|
-
DEFAULT_OUTPUT_PATH,
|
|
22
|
-
"--output",
|
|
23
|
-
help="Path to write the generated notebook to (default: ./TableValidator.py).",
|
|
24
|
-
),
|
|
25
|
-
mode: str = typer.Option(
|
|
26
|
-
"full",
|
|
27
|
-
"--mode",
|
|
28
|
-
help=(
|
|
29
|
-
"basic: only the check-group multiselect and table pickers. "
|
|
30
|
-
"full (default): also shows optional filter widgets (only/ "
|
|
31
|
-
"ignore columns, row filter, primary key). "
|
|
32
|
-
"schema: pre-selects only the Catalog & Schema check group."
|
|
33
|
-
),
|
|
34
|
-
),
|
|
35
|
-
) -> None:
|
|
36
|
-
"""Generate a Databricks notebook with dbutils widgets for source/
|
|
37
|
-
target catalog.schema.table selection and validation checks - fill in
|
|
38
|
-
the widgets and run the notebook, no code to write.
|
|
39
|
-
|
|
40
|
-
Import the generated file into a Databricks workspace via
|
|
41
|
-
Workspace -> Import, with format set to "Source" (or "File" -> upload
|
|
42
|
-
directly, depending on your workspace UI version) - Databricks
|
|
43
|
-
recognizes the "# Databricks notebook source" header and cell
|
|
44
|
-
(# COMMAND ----------) markers and opens it as a real notebook, not a
|
|
45
|
-
plain text file.
|
|
46
|
-
"""
|
|
47
|
-
if mode not in VALID_MODES:
|
|
48
|
-
typer.secho(
|
|
49
|
-
f"Invalid --mode {mode!r} - must be one of {', '.join(VALID_MODES)}.",
|
|
50
|
-
fg=typer.colors.RED,
|
|
51
|
-
)
|
|
52
|
-
raise typer.Exit(code=1)
|
|
53
|
-
|
|
54
|
-
generate_notebook(output, mode)
|
|
55
|
-
|
|
56
|
-
typer.echo(f"Generated notebook: {output.resolve()}")
|
|
57
|
-
typer.echo(
|
|
58
|
-
"Import it into Databricks (Workspace -> Import, format 'Source'), "
|
|
59
|
-
"fill in the widgets, and Run All."
|
|
60
|
-
)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
@app.command()
|
|
64
|
-
def info() -> None:
|
|
65
|
-
"""Show what this tool does and how to use the generated notebook."""
|
|
66
|
-
typer.echo(
|
|
67
|
-
"\n"
|
|
68
|
-
"tablevalidator-databricks - Databricks notebook UI for table-validator\n"
|
|
69
|
-
"------------------------------------------------------------------------\n"
|
|
70
|
-
"Generates a Databricks notebook with dbutils widgets (dynamic "
|
|
71
|
-
"catalog/schema/table dropdowns sourced from Unity Catalog, plus a "
|
|
72
|
-
"validation-checks multiselect) as a wrapper around the "
|
|
73
|
-
"table-validator package's validate_tables() API - no Python code "
|
|
74
|
-
"to write, no separate credentials to configure (the notebook "
|
|
75
|
-
"reuses its own ambient Spark session, same as validate_tables() "
|
|
76
|
-
"itself).\n"
|
|
77
|
-
"\n"
|
|
78
|
-
"Workflow:\n"
|
|
79
|
-
"\n"
|
|
80
|
-
" 1. tablevalidator-databricks init\n"
|
|
81
|
-
" Writes a ready-to-use notebook (./TableValidator.py by "
|
|
82
|
-
"default). Use --output to write elsewhere, --mode to pick basic/"
|
|
83
|
-
"full/schema (see 'tablevalidator-databricks init --help').\n"
|
|
84
|
-
"\n"
|
|
85
|
-
" 2. Import the generated file into Databricks\n"
|
|
86
|
-
" Workspace -> Import, format 'Source'.\n"
|
|
87
|
-
"\n"
|
|
88
|
-
" 3. Fill in the widgets and Run All\n"
|
|
89
|
-
" Pick Source/Target Catalog, Schema, Table, and which checks "
|
|
90
|
-
"to run, then run the notebook top to bottom. Re-run the "
|
|
91
|
-
"'Create / refresh widgets' cell after changing a Catalog/Schema "
|
|
92
|
-
"selection, to repopulate the dropdowns below it (a Databricks "
|
|
93
|
-
"widget limitation, not a bug). Can't find a name in a dropdown? "
|
|
94
|
-
"Each dropdown has a '(manual override)' text widget next to it - "
|
|
95
|
-
"type the exact name there instead.\n"
|
|
96
|
-
"\n"
|
|
97
|
-
"This package contains no validation logic of its own - every "
|
|
98
|
-
"generated notebook calls directly into table-validator's own "
|
|
99
|
-
"validate_tables() engine, so results are identical to calling it "
|
|
100
|
-
"yourself in a code cell.\n"
|
|
101
|
-
)
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Renders the bundled widget_notebook.py.tmpl into a real Databricks
|
|
3
|
-
notebook file. No Jinja2 - the template has exactly two placeholders,
|
|
4
|
-
substituted via plain str.replace(), so a templating dependency isn't
|
|
5
|
-
worth adding for this.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from __future__ import annotations
|
|
9
|
-
|
|
10
|
-
import importlib.resources
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
|
|
13
|
-
VALID_MODES = ("basic", "full", "schema")
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def generate_notebook(output: Path, mode: str) -> None:
|
|
17
|
-
"""Write a widget-driven Databricks notebook to `output`.
|
|
18
|
-
|
|
19
|
-
mode controls which optional widgets/pre-selected checks the notebook
|
|
20
|
-
starts with - see tablevalidator_databricks/cli/main.py's `init`
|
|
21
|
-
command docstring for what each mode means. The underlying comparison
|
|
22
|
-
logic is identical regardless of mode; only the widgets shown differ,
|
|
23
|
-
so all three modes render from the same template file rather than
|
|
24
|
-
three separate ones.
|
|
25
|
-
"""
|
|
26
|
-
if mode not in VALID_MODES:
|
|
27
|
-
raise ValueError(f"mode must be one of {VALID_MODES}, got {mode!r}")
|
|
28
|
-
|
|
29
|
-
template = (
|
|
30
|
-
importlib.resources.files("tablevalidator_databricks.templates")
|
|
31
|
-
.joinpath("widget_notebook.py.tmpl")
|
|
32
|
-
.read_text(encoding="utf-8")
|
|
33
|
-
)
|
|
34
|
-
rendered = template.replace("{{MODE}}", mode).replace(
|
|
35
|
-
"{{SHOW_OPTIONAL_WIDGETS}}", "True" if mode == "full" else "False",
|
|
36
|
-
)
|
|
37
|
-
|
|
38
|
-
output.parent.mkdir(parents=True, exist_ok=True)
|
|
39
|
-
output.write_text(rendered, encoding="utf-8")
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
"""Tests for the tablevalidator-databricks CLI (cli/main.py)."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
|
|
7
|
-
from typer.testing import CliRunner
|
|
8
|
-
|
|
9
|
-
from tablevalidator_databricks.cli.main import app
|
|
10
|
-
|
|
11
|
-
runner = CliRunner()
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def test_app_is_importable() -> None:
|
|
15
|
-
assert app is not None
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def test_help_lists_init_command() -> None:
|
|
19
|
-
result = runner.invoke(app, ["--help"])
|
|
20
|
-
assert result.exit_code == 0
|
|
21
|
-
assert "init" in result.output
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def test_init_writes_notebook_to_default_path(tmp_path: Path) -> None:
|
|
25
|
-
output = tmp_path / "TableValidator.py"
|
|
26
|
-
|
|
27
|
-
result = runner.invoke(app, ["init", "--output", str(output)])
|
|
28
|
-
|
|
29
|
-
assert result.exit_code == 0
|
|
30
|
-
assert output.exists()
|
|
31
|
-
assert "Generated notebook" in result.output
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def test_init_respects_mode_flag(tmp_path: Path) -> None:
|
|
35
|
-
output = tmp_path / "basic.py"
|
|
36
|
-
|
|
37
|
-
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "basic"])
|
|
38
|
-
|
|
39
|
-
assert result.exit_code == 0
|
|
40
|
-
content = output.read_text(encoding="utf-8")
|
|
41
|
-
assert 'MODE = "basic"' in content
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def test_init_rejects_invalid_mode(tmp_path: Path) -> None:
|
|
45
|
-
output = tmp_path / "x.py"
|
|
46
|
-
|
|
47
|
-
result = runner.invoke(app, ["init", "--output", str(output), "--mode", "bogus"])
|
|
48
|
-
|
|
49
|
-
assert result.exit_code == 1
|
|
50
|
-
assert not output.exists()
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def test_info_command_runs() -> None:
|
|
54
|
-
result = runner.invoke(app, ["info"])
|
|
55
|
-
assert result.exit_code == 0
|
|
56
|
-
assert "tablevalidator-databricks" in result.output
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|