tablevalidator-databricks 0.1.2__tar.gz → 0.1.4__tar.gz

{tablevalidator_databricks-0.1.2 → tablevalidator_databricks-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tablevalidator-databricks
-Version: 0.1.2
+Version: 0.1.4
 Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
 License: MIT
 Requires-Python: >=3.9
@@ -45,12 +45,19 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
 tablevalidator-databricks init --output MyValidation.py --mode full
 ```
 
-- `--mode basic` - only the check-group multiselect and table pickers.
+- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
 - `--mode full` (default) - also shows optional filter widgets (only/
-  ignore columns, row filter, primary key).
-- `--mode schema` - same widgets as `basic`; intended for a lightweight
-  schema-shape-only comparison (just deselect Column/Row in the checks
-  multiselect after opening the notebook).
+  ignore columns, row filter, primary key). Every check defaults to "Yes".
+- `--mode schema` - same widgets as `basic`, but the Row checks default to
+  "No" - a lightweight schema-shape-only comparison out of the box (still
+  changeable per-run).
+
+> **Note on the check groups**: "Catalog & Schema" only verifies that
+> catalogs/schemas/tables exist and match by name - it produces no
+> per-table PASS/FAIL by itself, so running it alone reports every table
+> as `SKIPPED`. The per-table verdict comes from the **Column** checks
+> (names, types, nullability, statistics) and **Row** checks (row counts,
+> row-level data).
 
 Then, in your Databricks workspace: **Workspace -> Import**, format
 "Source", and select the generated file. Databricks recognizes the
@@ -63,7 +70,7 @@ markers and opens it as a real notebook.
 2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
    `SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
    populate Source/Target Catalog, Schema, and Table dropdowns.
-3. Shows a "Validation Checks" multiselect (Catalog & Schema / Column /
+3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
    Row - the three independently-selectable check groups the engine
    actually supports) plus, in `full` mode, optional text widgets for
    column filtering, a row filter, and a primary key.

{tablevalidator_databricks-0.1.2 → tablevalidator_databricks-0.1.4}/README.md

@@ -31,12 +31,19 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
 tablevalidator-databricks init --output MyValidation.py --mode full
 ```
 
-- `--mode basic` - only the check-group multiselect and table pickers.
+- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
 - `--mode full` (default) - also shows optional filter widgets (only/
-  ignore columns, row filter, primary key).
-- `--mode schema` - same widgets as `basic`; intended for a lightweight
-  schema-shape-only comparison (just deselect Column/Row in the checks
-  multiselect after opening the notebook).
+  ignore columns, row filter, primary key). Every check defaults to "Yes".
+- `--mode schema` - same widgets as `basic`, but the Row checks default to
+  "No" - a lightweight schema-shape-only comparison out of the box (still
+  changeable per-run).
+
+> **Note on the check groups**: "Catalog & Schema" only verifies that
+> catalogs/schemas/tables exist and match by name - it produces no
+> per-table PASS/FAIL by itself, so running it alone reports every table
+> as `SKIPPED`. The per-table verdict comes from the **Column** checks
+> (names, types, nullability, statistics) and **Row** checks (row counts,
+> row-level data).
 
 Then, in your Databricks workspace: **Workspace -> Import**, format
 "Source", and select the generated file. Databricks recognizes the
@@ -49,7 +56,7 @@ markers and opens it as a real notebook.
 2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
    `SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
    populate Source/Target Catalog, Schema, and Table dropdowns.
-3. Shows a "Validation Checks" multiselect (Catalog & Schema / Column /
+3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
    Row - the three independently-selectable check groups the engine
    actually supports) plus, in `full` mode, optional text widgets for
    column filtering, a row filter, and a primary key.

tablevalidator_databricks-0.1.4/tablevalidator_databricks/generator/generator.py

@@ -0,0 +1,66 @@
+"""
+Renders the bundled widget_notebook.py.tmpl into a real Databricks
+notebook file. No Jinja2 - the template has only a handful of
+placeholders, substituted via plain str.replace(), so a templating
+dependency isn't worth adding for this.
+"""
+
+from __future__ import annotations
+
+import importlib.resources
+import re
+from importlib.metadata import PackageNotFoundError, requires
+from pathlib import Path
+
+VALID_MODES = ("basic", "full", "schema")
+
+# Fallback floor for the generated notebook's `%pip install
+# "table-validator>=X"` line, used only if this package's own declared
+# dependency can't be read at runtime. validate_tables() - the API the
+# generated notebook calls - first shipped in table-validator 0.1.9.
+_FALLBACK_MIN_CORE_VERSION = "0.1.9"
+
+
+def _min_core_version() -> str:
+    """The minimum `table-validator` version the generated notebook should
+    install, read from THIS package's own declared dependency so the two
+    can never drift apart. Falls back to a known-good floor if the
+    metadata isn't readable (e.g. running from a source tree that was
+    never installed)."""
+    try:
+        for req in requires("tablevalidator-databricks") or []:
+            # e.g. "table-validator>=0.1.19" / "table-validator >= 0.1.19"
+            match = re.match(r"^\s*table[-_]validator\s*>=\s*([0-9][^\s,;]*)", req)
+            if match:
+                return match.group(1)
+    except PackageNotFoundError:
+        pass
+    return _FALLBACK_MIN_CORE_VERSION
+
+
+def generate_notebook(output: Path, mode: str) -> None:
+    """Write a widget-driven Databricks notebook to `output`.
+
+    mode controls which optional widgets/pre-selected checks the notebook
+    starts with - see tablevalidator_databricks/cli/main.py's `init`
+    command docstring for what each mode means. The underlying comparison
+    logic is identical regardless of mode; only the widgets shown differ,
+    so all three modes render from the same template file rather than
+    three separate ones.
+    """
+    if mode not in VALID_MODES:
+        raise ValueError(f"mode must be one of {VALID_MODES}, got {mode!r}")
+
+    template = (
+        importlib.resources.files("tablevalidator_databricks.templates")
+        .joinpath("widget_notebook.py.tmpl")
+        .read_text(encoding="utf-8")
+    )
+    rendered = (
+        template.replace("{{MODE}}", mode)
+        .replace("{{SHOW_OPTIONAL_WIDGETS}}", "True" if mode == "full" else "False")
+        .replace("{{MIN_CORE_VERSION}}", _min_core_version())
+    )
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(rendered, encoding="utf-8")

{tablevalidator_databricks-0.1.2 → tablevalidator_databricks-0.1.4}/tablevalidator_databricks/templates/widget_notebook.py.tmpl

@@ -31,7 +31,19 @@
 
 # COMMAND ----------
 
-# MAGIC %pip install table-validator
+# MAGIC %md ### Install the validation engine
+# MAGIC Note the `>=` version floor: `validate_tables()` (the API every cell
+# MAGIC below calls) only exists in `table-validator` {{MIN_CORE_VERSION}} and
+# MAGIC later. **Do not pin this to a lower version** - in particular, do not
+# MAGIC confuse it with the `tablevalidator-databricks` version that generated
+# MAGIC this notebook; the two packages are versioned completely independently,
+# MAGIC and pinning the engine to the generator's version number installs an
+# MAGIC engine far too old, failing with
+# MAGIC `ImportError: cannot import name 'validate_tables'`.
+
+# COMMAND ----------
+
+# MAGIC %pip install "table-validator>={{MIN_CORE_VERSION}}"
 
 # COMMAND ----------
 
@@ -177,21 +189,35 @@ print(
 # COMMAND ----------
 
 # MAGIC %md ### Validation checks
+# MAGIC "Catalog & Schema" only checks that the catalogs/schemas/tables exist
+# MAGIC and match by name - it produces no per-table PASS/FAIL on its own, so
+# MAGIC running it alone reports every table as **SKIPPED**. That's expected,
+# MAGIC not an error: the per-table verdict comes from the **Column** checks
+# MAGIC (names, types, nullability, statistics) and **Row** checks (row counts
+# MAGIC and row-level data). Leave those on unless you specifically want a
+# MAGIC name-level-only existence check.
 
 # COMMAND ----------
 
-_ALL_CHECKS = ["Catalog & Schema", "Column", "Row"]
-# dbutils.widgets.multiselect's default value must itself be one of the
-# combinations Databricks is willing to accept - in practice this means
-# "all choices, comma-joined in the declared order" is the only default
-# guaranteed to work across Databricks runtime versions (a narrower
-# default, e.g. just "Catalog & Schema", was observed to fail with
-# "DefaultValueNotInChoicesList: Selection sequence must include ..." on
-# some runtimes). --mode schema still gets its narrower intent honored -
-# see the checks-selection cell below, which treats an EMPTY selection as
-# "run everything" and a real narrowing as an explicit choice; this
-# widget just always starts fully selected regardless of --mode.
-dbutils.widgets.multiselect("checks", ",".join(_ALL_CHECKS), _ALL_CHECKS, "Validation Checks")
+# One Yes/No dropdown per check group, instead of a single multiselect -
+# dbutils.widgets.multiselect enforces an internal "selection sequence"
+# validation against its own default that was observed to reject even a
+# same-length, same-items default on some Databricks runtimes/existing-
+# widget states (DefaultValueNotInChoicesList), with no reliable
+# workaround. Three independent dropdowns have no such cross-widget
+# validation and are the more robust primitive for this.
+_YES_NO = ["Yes", "No"]
+_schema_default = "Yes"
+# --mode schema means "schema SHAPE only" - that shape (column names,
+# types, nullability) is validated by the COLUMN checks, so Column stays
+# on and only the expensive Row checks (row counts + row-level data) are
+# off by default. Turning Column off too would leave nothing producing a
+# per-table verdict, reporting every table as SKIPPED.
+_column_default = "Yes"
+_row_default = "No" if MODE == "schema" else "Yes"
+dbutils.widgets.dropdown("check_catalog_schema", _schema_default, _YES_NO, "Run Catalog & Schema checks?")
+dbutils.widgets.dropdown("check_column", _column_default, _YES_NO, "Run Column checks?")
+dbutils.widgets.dropdown("check_row", _row_default, _YES_NO, "Run Row checks?")
 
 if SHOW_OPTIONAL_WIDGETS:
     dbutils.widgets.text("only_columns", "", "Only compare these columns (comma-separated, optional)")
@@ -279,15 +305,13 @@ def _source_or_target_string(prefix: str) -> str:
 source = _source_or_target_string("source")
 target = _source_or_target_string("target")
 
-_CHECK_MAP = {
-    "Catalog & Schema": {ValidationType.CATALOG, ValidationType.SCHEMA},
-    "Column": {ValidationType.COLUMN},
-    "Row": {ValidationType.ROW},
-}
-selected_checks = [c for c in dbutils.widgets.get("checks").split(",") if c]
 enabled_validations = set()
-for check in selected_checks:
-    enabled_validations |= _CHECK_MAP.get(check, set())
+if dbutils.widgets.get("check_catalog_schema") == "Yes":
+    enabled_validations |= {ValidationType.CATALOG, ValidationType.SCHEMA}
+if dbutils.widgets.get("check_column") == "Yes":
+    enabled_validations |= {ValidationType.COLUMN}
+if dbutils.widgets.get("check_row") == "Yes":
+    enabled_validations |= {ValidationType.ROW}
 if not enabled_validations:
     enabled_validations = None  # fall back to validate_tables()'s own "run everything" default
 

{tablevalidator_databricks-0.1.2 → tablevalidator_databricks-0.1.4}/tablevalidator_databricks.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tablevalidator-databricks
-Version: 0.1.2
+Version: 0.1.4
 Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
 License: MIT
 Requires-Python: >=3.9
@@ -45,12 +45,19 @@ Writes `./TableValidator.py` - a ready-to-use Databricks notebook. Options:
 tablevalidator-databricks init --output MyValidation.py --mode full
 ```
 
-- `--mode basic` - only the check-group multiselect and table pickers.
+- `--mode basic` - only the check-group Yes/No dropdowns and table pickers.
 - `--mode full` (default) - also shows optional filter widgets (only/
-  ignore columns, row filter, primary key).
-- `--mode schema` - same widgets as `basic`; intended for a lightweight
-  schema-shape-only comparison (just deselect Column/Row in the checks
-  multiselect after opening the notebook).
+  ignore columns, row filter, primary key). Every check defaults to "Yes".
+- `--mode schema` - same widgets as `basic`, but the Row checks default to
+  "No" - a lightweight schema-shape-only comparison out of the box (still
+  changeable per-run).
+
+> **Note on the check groups**: "Catalog & Schema" only verifies that
+> catalogs/schemas/tables exist and match by name - it produces no
+> per-table PASS/FAIL by itself, so running it alone reports every table
+> as `SKIPPED`. The per-table verdict comes from the **Column** checks
+> (names, types, nullability, statistics) and **Row** checks (row counts,
+> row-level data).
 
 Then, in your Databricks workspace: **Workspace -> Import**, format
 "Source", and select the generated file. Databricks recognizes the
@@ -63,7 +70,7 @@ markers and opens it as a real notebook.
 2. Queries Unity Catalog (`SHOW CATALOGS` / `SHOW SCHEMAS IN ...` /
    `SHOW TABLES IN ...` via the notebook's own ambient Spark session) to
    populate Source/Target Catalog, Schema, and Table dropdowns.
-3. Shows a "Validation Checks" multiselect (Catalog & Schema / Column /
+3. Shows a Yes/No dropdown per check group (Catalog & Schema / Column /
    Row - the three independently-selectable check groups the engine
    actually supports) plus, in `full` mode, optional text widgets for
    column filtering, a row filter, and a primary key.

{tablevalidator_databricks-0.1.2 → tablevalidator_databricks-0.1.4}/tablevalidator_databricks.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.1.2",
+  "tag": "0.1.4",
   "distance": 0,
-  "node": "gae96a50",
+  "node": "g131e781",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-09-08"

{tablevalidator_databricks-0.1.2 → tablevalidator_databricks-0.1.4}/tests/test_generated_notebook_execution.py

@@ -15,6 +15,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
+from table_validator.config.schema import ValidationType
 from tablevalidator_databricks.generator.generator import generate_notebook
 
 
@@ -254,42 +255,89 @@ def test_catalog_override_drives_schema_and_table_lookup(notebook_path):
 
 
 # ---------------------------------------------------------------------------
-# --mode schema: regression test for a real production bug. The checks
-# multiselect's default used to be a narrower subset of its choices in
-# --mode schema (just "Catalog & Schema"), which Databricks rejected with
-# DefaultValueNotInChoicesList - _FakeWidgets.multiselect above replicates
-# that exact constraint.
+# Validation-checks selection - three independent Yes/No dropdowns
+# (check_catalog_schema / check_column / check_row), replacing an earlier
+# single dbutils.widgets.multiselect that was observed in production to
+# raise DefaultValueNotInChoicesList even with a default that appeared to
+# satisfy multiselect's own documented constraints - three independent
+# dropdowns have no equivalent cross-widget validation.
 # ---------------------------------------------------------------------------
-def test_schema_mode_multiselect_default_does_not_crash(tmp_path: Path) -> None:
-    path = tmp_path / "TableValidator.py"
-    generate_notebook(path, "schema")
+def test_full_and_basic_mode_default_all_checks_to_yes(tmp_path: Path) -> None:
+    for mode in ("full", "basic"):
+        path = tmp_path / f"{mode}.py"
+        generate_notebook(path, mode)
+        spark = _fake_spark(
+            catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
+            tables_by_schema={("main", "sales"): ["orders"]},
+        )
+        captured = {}
+
+        def fake_validate_tables(source, target, **kwargs):
+            captured["enabled_validations"] = kwargs.get("enabled_validations")
+            m = MagicMock()
+            m.__str__ = MagicMock(return_value="Overall status: PASS")
+            return m
+
+        _run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
 
+        assert captured["enabled_validations"] == {
+            ValidationType.CATALOG, ValidationType.SCHEMA,
+            ValidationType.COLUMN, ValidationType.ROW,
+        }, f"mode={mode}"
+
+
+def test_schema_mode_keeps_column_checks_on_and_turns_row_off(tmp_path: Path) -> None:
+    """--mode schema means "schema SHAPE only". That shape is validated by
+    the COLUMN checks, so Column must stay ON - only the expensive Row
+    checks default off. Turning Column off too would leave nothing
+    producing a per-table verdict, reporting every table as SKIPPED."""
+    path = tmp_path / "schema.py"
+    generate_notebook(path, "schema")
     spark = _fake_spark(
-        catalogs=["main"],
-        schemas_by_catalog={"main": ["sales"]},
+        catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
         tables_by_schema={("main", "sales"): ["orders"]},
     )
+    captured = {}
+
+    def fake_validate_tables(source, target, **kwargs):
+        captured["enabled_validations"] = kwargs.get("enabled_validations")
+        m = MagicMock()
+        m.__str__ = MagicMock(return_value="Overall status: PASS")
+        return m
 
-    # Must not raise _DefaultValueNotInChoicesList.
-    _run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=MagicMock())
+    _run_notebook(path, widget_values={}, spark=spark, validate_tables_mock=fake_validate_tables)
 
+    assert captured["enabled_validations"] == {
+        ValidationType.CATALOG, ValidationType.SCHEMA, ValidationType.COLUMN,
+    }
 
-def test_multiselect_default_is_every_choice(notebook_path):
-    """The checks widget's default must always be the full choices list,
-    comma-joined in the declared order - the only default guaranteed to
-    be accepted by dbutils.widgets.multiselect across Databricks runtime
-    versions."""
+
+def test_deselecting_every_check_falls_back_to_run_everything(notebook_path):
+    """All three Yes/No dropdowns set to "No" must fall back to
+    validate_tables()'s own "run everything" default (enabled_validations=
+    None), not silently run nothing."""
     spark = _fake_spark(
-        catalogs=["main"],
-        schemas_by_catalog={"main": ["sales"]},
+        catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
         tables_by_schema={("main", "sales"): ["orders"]},
     )
+    captured = {}
+
+    def fake_validate_tables(source, target, **kwargs):
+        captured["enabled_validations"] = kwargs.get("enabled_validations")
+        m = MagicMock()
+        m.__str__ = MagicMock(return_value="Overall status: PASS")
+        return m
 
-    globs = _run_notebook(notebook_path, widget_values={}, spark=spark, validate_tables_mock=MagicMock())
+    _run_notebook(
+        notebook_path,
+        widget_values={
+            "check_catalog_schema": "No", "check_column": "No", "check_row": "No",
+        },
+        spark=spark,
+        validate_tables_mock=fake_validate_tables,
+    )
 
-    fake_widgets = globs["dbutils"].widgets
-    choices = fake_widgets.multiselect_choices["checks"]
-    assert fake_widgets.get("checks") == ",".join(choices)
+    assert captured["enabled_validations"] is None
 
 
 # ---------------------------------------------------------------------------

{tablevalidator_databricks-0.1.2 → tablevalidator_databricks-0.1.4}/tests/test_generator.py

@@ -3,11 +3,12 @@ no Databricks/Spark involved."""
 
 from __future__ import annotations
 
+import re
 from pathlib import Path
 
 import pytest
 
-from tablevalidator_databricks.generator.generator import generate_notebook
+from tablevalidator_databricks.generator.generator import _min_core_version, generate_notebook
 
 
 def test_generate_notebook_writes_a_databricks_source_notebook(tmp_path: Path) -> None:
@@ -89,3 +90,29 @@ def test_generate_notebook_no_leftover_placeholders(tmp_path: Path) -> None:
 
     assert "{{" not in content
     assert "}}" not in content
+
+
+def test_generated_notebook_pins_a_core_version_floor(tmp_path: Path) -> None:
+    """Regression guard for a real user-reported failure: the notebook's
+    %pip install line must carry a `>=` floor for table-validator.
+    validate_tables() only exists in 0.1.9+, and the two packages are
+    versioned independently - a user who pinned the engine to THIS
+    package's version number (0.1.3) got an engine predating
+    validate_tables() entirely and an ImportError."""
+    output = tmp_path / "TableValidator.py"
+    generate_notebook(output, "full")
+    content = output.read_text(encoding="utf-8")
+
+    assert 'pip install "table-validator>=' in content
+    # The floor must be new enough to actually contain validate_tables().
+    match = re.search(r'pip install "table-validator>=([0-9]+)\.([0-9]+)\.([0-9]+)"', content)
+    assert match, "no parseable version floor in the %pip install line"
+    major, minor, patch = (int(g) for g in match.groups())
+    assert (major, minor, patch) >= (0, 1, 9)
+
+
+def test_min_core_version_reads_from_package_metadata() -> None:
+    """The floor comes from this package's own declared table-validator
+    dependency, so the two can never silently drift apart."""
+    assert _min_core_version() != "0.0.0"
+    assert re.match(r"^[0-9]+\.[0-9]+\.[0-9]+$", _min_core_version())

tablevalidator_databricks-0.1.2/tablevalidator_databricks/generator/generator.py

@@ -1,39 +0,0 @@
-"""
-Renders the bundled widget_notebook.py.tmpl into a real Databricks
-notebook file. No Jinja2 - the template has exactly two placeholders,
-substituted via plain str.replace(), so a templating dependency isn't
-worth adding for this.
-"""
-
-from __future__ import annotations
-
-import importlib.resources
-from pathlib import Path
-
-VALID_MODES = ("basic", "full", "schema")
-
-
-def generate_notebook(output: Path, mode: str) -> None:
-    """Write a widget-driven Databricks notebook to `output`.
-
-    mode controls which optional widgets/pre-selected checks the notebook
-    starts with - see tablevalidator_databricks/cli/main.py's `init`
-    command docstring for what each mode means. The underlying comparison
-    logic is identical regardless of mode; only the widgets shown differ,
-    so all three modes render from the same template file rather than
-    three separate ones.
-    """
-    if mode not in VALID_MODES:
-        raise ValueError(f"mode must be one of {VALID_MODES}, got {mode!r}")
-
-    template = (
-        importlib.resources.files("tablevalidator_databricks.templates")
-        .joinpath("widget_notebook.py.tmpl")
-        .read_text(encoding="utf-8")
-    )
-    rendered = template.replace("{{MODE}}", mode).replace(
-        "{{SHOW_OPTIONAL_WIDGETS}}", "True" if mode == "full" else "False",
-    )
-
-    output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(rendered, encoding="utf-8")