tablevalidator-databricks 0.1.6__tar.gz → 0.1.8__tar.gz

{tablevalidator_databricks-0.1.6 → tablevalidator_databricks-0.1.8}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: tablevalidator-databricks
-Version: 0.1.6
+Version: 0.1.8
 Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
 License: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: typer>=0.12
-Requires-Dist: table-validator>=0.1.19
+Requires-Dist: table-validator>=0.1.22
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: ruff>=0.4; extra == "dev"

{tablevalidator_databricks-0.1.6 → tablevalidator_databricks-0.1.8}/pyproject.toml

@@ -11,7 +11,12 @@ requires-python = ">=3.9"
 license = { text = "MIT" }
 dependencies = [
     "typer>=0.12",
-    "table-validator>=0.1.19",
+    # 0.1.22 is the first release whose result sheets survive Arrow
+    # conversion, which is what Databricks' display() uses - older engines
+    # fail with "ArrowTypeError: Expected bytes, got a 'int' object" on a
+    # column mixing row counts with blanks. The generated notebook's
+    # %pip install floor is read from this line.
+    "table-validator>=0.1.22",
 ]
 
 [project.optional-dependencies]

{tablevalidator_databricks-0.1.6 → tablevalidator_databricks-0.1.8}/tablevalidator_databricks/templates/widget_notebook.py.tmpl

@@ -42,18 +42,25 @@
 # COMMAND ----------
 
 # MAGIC %md ### Install the validation engine
-# MAGIC Note the `>=` version floor: `validate_tables()` (the API every cell
-# MAGIC below calls) only exists in `table-validator` {{MIN_CORE_VERSION}} and
-# MAGIC later. **Do not pin this to a lower version** - in particular, do not
-# MAGIC confuse it with the `tablevalidator-databricks` version that generated
-# MAGIC this notebook; the two packages are versioned completely independently,
-# MAGIC and pinning the engine to the generator's version number installs an
-# MAGIC engine far too old, failing with
+# MAGIC This always pulls the **latest** `table-validator`. The flags matter:
+# MAGIC `--upgrade` replaces an engine that is already installed on the cluster
+# MAGIC (without it pip just prints "Requirement already satisfied" and keeps
+# MAGIC the old one), and `--no-cache-dir` avoids a stale cached wheel being
+# MAGIC reused. The next cell prints the version actually loaded, so you can
+# MAGIC always confirm what you are running.
+# MAGIC
+# MAGIC The `>=` floor is a minimum, not a pin: `validate_tables()` (the API
+# MAGIC every cell below calls) only exists in `table-validator`
+# MAGIC {{MIN_CORE_VERSION}} and later. **Do not pin this to a lower version** -
+# MAGIC in particular, do not confuse it with the `tablevalidator-databricks`
+# MAGIC version that generated this notebook; the two packages are versioned
+# MAGIC completely independently, and pinning the engine to the generator's
+# MAGIC version number installs an engine far too old, failing with
 # MAGIC `ImportError: cannot import name 'validate_tables'`.
 
 # COMMAND ----------
 
-# MAGIC %pip install "table-validator>={{MIN_CORE_VERSION}}"
+# MAGIC %pip install --upgrade --upgrade-strategy eager --no-cache-dir "table-validator>={{MIN_CORE_VERSION}}"
 
 # COMMAND ----------
 
@@ -61,12 +68,41 @@ dbutils.library.restartPython()
 
 # COMMAND ----------
 
+import table_validator
 from table_validator import validate_tables
 from table_validator.config.schema import ValidationType
 
 MODE = "{{MODE}}"
 SHOW_OPTIONAL_WIDGETS = {{SHOW_OPTIONAL_WIDGETS}}
 
+# Print what is actually loaded. A %pip install without --upgrade silently
+# keeps an older already-installed engine ("Requirement already
+# satisfied"), so the version that matters is the one imported here, not
+# the one the install cell asked for.
+_MIN_CORE = "{{MIN_CORE_VERSION}}"
+print(f"table-validator engine: {table_validator.__version__}  (need >= {_MIN_CORE})")
+
+
+def _version_tuple(text):
+    parts = []
+    for chunk in str(text).split(".")[:3]:
+        digits = "".join(c for c in chunk if c.isdigit())
+        parts.append(int(digits) if digits else 0)
+    while len(parts) < 3:
+        parts.append(0)
+    return tuple(parts)
+
+
+if _version_tuple(table_validator.__version__) < _version_tuple(_MIN_CORE):
+    print(
+        f"\n  WARNING: this engine ({table_validator.__version__}) is older than "
+        f"{_MIN_CORE}, which this notebook expects.\n"
+        "  You may hit errors such as 'ArrowTypeError: Expected bytes, got a "
+        "'int' object' when displaying results.\n"
+        "  Fix: run the install cell above with --upgrade, then re-run "
+        "dbutils.library.restartPython() and start again."
+    )
+
 
 def _list_catalogs():
     return sorted(r["catalog"] for r in spark.sql("SHOW CATALOGS").collect())
@@ -178,11 +214,25 @@ def _build_cascading_widgets(prefix: str, label_prefix: str) -> None:
         table_choices = [_ALL_TABLES] + _list_tables(catalog_current, schema_current)
         table_current = _current_or_first_real(table_choices, _effective_value(f"{prefix}_table"), _ALL_TABLES)
 
+    # Only the three pickers here - the matching "(manual override)" text
+    # boxes are created later, by _build_override_widgets(), so that the
+    # essential Catalog -> Schema -> Table selection isn't broken up by
+    # advanced fields. Databricks lays widgets out in creation order, so
+    # creation order IS the on-screen order.
     dbutils.widgets.dropdown(f"{prefix}_catalog", catalog_current, catalogs or [""], f"{label_prefix} Catalog")
-    dbutils.widgets.text(f"{prefix}_catalog_override", "", f"{label_prefix} Catalog (manual override)")
     dbutils.widgets.dropdown(f"{prefix}_schema", schema_current, schema_choices, f"{label_prefix} Schema")
-    dbutils.widgets.text(f"{prefix}_schema_override", "", f"{label_prefix} Schema (manual override)")
     dbutils.widgets.dropdown(f"{prefix}_table", table_current, table_choices, f"{label_prefix} Table")
+
+
+def _build_override_widgets(prefix: str, label_prefix: str) -> None:
+    """The "(manual override)" text box for each of one side's pickers.
+
+    Created after every dropdown and check widget, since these are an
+    escape hatch for a name the dropdown doesn't list - most runs leave
+    all six blank, so they belong at the end rather than interleaved with
+    the pickers people actually use."""
+    dbutils.widgets.text(f"{prefix}_catalog_override", "", f"{label_prefix} Catalog (manual override)")
+    dbutils.widgets.text(f"{prefix}_schema_override", "", f"{label_prefix} Schema (manual override)")
     dbutils.widgets.text(f"{prefix}_table_override", "", f"{label_prefix} Table (manual override)")
 
 
@@ -220,10 +270,19 @@ def _build_check_widgets() -> None:
 # and in a predictable order - Databricks lays widgets out in creation
 # order, so splitting these across cells made them interleave oddly and
 # made later ones seem to "appear" partway through a Run All.
+#
+# Order is deliberate, most-used first:
+#   1. Source Catalog / Schema / Table   - what you're comparing
+#   2. Target Catalog / Schema / Table
+#   3. Which checks to run               - the three Yes/No dropdowns
+#   4. Optional filters                  - primary key, columns, row filter
+#   5. Manual overrides                  - rarely needed escape hatch
 catalogs = _list_catalogs()
 _build_cascading_widgets("source", "Source")
 _build_cascading_widgets("target", "Target")
 _build_check_widgets()
+_build_override_widgets("source", "Source")
+_build_override_widgets("target", "Target")
 
 print(
     "Widgets refreshed for the CURRENT Catalog/Schema selection.\n"

{tablevalidator_databricks-0.1.6 → tablevalidator_databricks-0.1.8}/tablevalidator_databricks.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: tablevalidator-databricks
-Version: 0.1.6
+Version: 0.1.8
 Summary: Generates a widget-driven Databricks notebook UI on top of the table-validator package - no code required to run a comparison.
 License: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: typer>=0.12
-Requires-Dist: table-validator>=0.1.19
+Requires-Dist: table-validator>=0.1.22
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: ruff>=0.4; extra == "dev"

{tablevalidator_databricks-0.1.6 → tablevalidator_databricks-0.1.8}/tablevalidator_databricks.egg-info/requires.txt

@@ -1,5 +1,5 @@
 typer>=0.12
-table-validator>=0.1.19
+table-validator>=0.1.22
 
 [dev]
 pytest>=7.0

tablevalidator_databricks-0.1.8/tablevalidator_databricks.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.1.8",
+  "distance": 0,
+  "node": "g0e12068",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-11"
+}

{tablevalidator_databricks-0.1.6 → tablevalidator_databricks-0.1.8}/tests/test_generated_notebook_execution.py

@@ -421,3 +421,76 @@ def test_catalog_wide_sweep_raises_clear_error(notebook_path):
             spark=spark,
             validate_tables_mock=MagicMock(),
         )
+
+
+# ---------------------------------------------------------------------------
+# Widget ORDER. Databricks lays widgets out in creation order, so creation
+# order is the on-screen order. Essential pickers must come first and the
+# rarely-needed manual overrides last.
+# ---------------------------------------------------------------------------
+def _widget_creation_order(path: Path) -> list:
+    """Run the notebook and record the order widgets are created in."""
+    order = []
+
+    class _OrderRecordingWidgets(_FakeWidgets):
+        def text(self, name, default, label):
+            order.append(name)
+            super().text(name, default, label)
+
+        def dropdown(self, name, default, choices, label):
+            order.append(name)
+            super().dropdown(name, default, choices, label)
+
+        def multiselect(self, name, default, choices, label):
+            order.append(name)
+            super().multiselect(name, default, choices, label)
+
+    spark = _fake_spark(
+        catalogs=["main"], schemas_by_catalog={"main": ["sales"]},
+        tables_by_schema={("main", "sales"): ["orders"]},
+    )
+    code = _code_from_notebook(path)
+
+    fake_dbutils = MagicMock()
+    fake_dbutils.widgets = _OrderRecordingWidgets()
+    fake_dbutils.library.restartPython = MagicMock()
+
+    import table_validator
+    import table_validator.notebook as nb_mod
+    stub = MagicMock()
+    table_validator.validate_tables = stub
+    nb_mod.validate_tables = stub
+
+    exec(compile(code, str(path), "exec"), {
+        "dbutils": fake_dbutils, "spark": spark,
+        "display": MagicMock(), "__name__": "__main__",
+    })
+    return order
+
+
+def test_source_and_target_pickers_come_first(notebook_path):
+    """What you're comparing is the first thing you should see - the six
+    Catalog/Schema/Table pickers, Source then Target, before anything
+    else."""
+    order = _widget_creation_order(notebook_path)
+
+    assert order[:6] == [
+        "source_catalog", "source_schema", "source_table",
+        "target_catalog", "target_schema", "target_table",
+    ]
+
+
+def test_manual_overrides_come_last(notebook_path):
+    """The override boxes are an escape hatch most runs leave blank, so
+    they belong at the end rather than interleaved with the pickers."""
+    order = _widget_creation_order(notebook_path)
+
+    assert all("_override" in name for name in order[-6:])
+
+
+def test_checks_then_filters_sit_between_pickers_and_overrides(notebook_path):
+    order = _widget_creation_order(notebook_path)
+    middle = order[6:-6]
+
+    assert middle[:3] == ["check_catalog_schema", "check_column", "check_row"]
+    assert middle[3:] == ["primary_key", "only_columns", "ignore_columns", "row_filter"]

{tablevalidator_databricks-0.1.6 → tablevalidator_databricks-0.1.8}/tests/test_generator.py

@@ -103,9 +103,12 @@ def test_generated_notebook_pins_a_core_version_floor(tmp_path: Path) -> None:
     generate_notebook(output, "full")
     content = output.read_text(encoding="utf-8")
 
-    assert 'pip install "table-validator>=' in content
+    assert "pip install --upgrade" in content
+    assert '"table-validator>=' in content
     # The floor must be new enough to actually contain validate_tables().
-    match = re.search(r'pip install "table-validator>=([0-9]+)\.([0-9]+)\.([0-9]+)"', content)
+    match = re.search(
+        r'"table-validator>=([0-9]+)\.([0-9]+)\.([0-9]+)"', content
+    )
     assert match, "no parseable version floor in the %pip install line"
     major, minor, patch = (int(g) for g in match.groups())
     assert (major, minor, patch) >= (0, 1, 9)
@@ -131,3 +134,57 @@ def test_generated_notebook_stamps_the_generator_version(tmp_path: Path) -> None
     # And it explains that regenerating is what picks up new widgets.
     assert "upgrading the" in content.lower()
     assert "tablevalidator-databricks init" in content
+
+
+def test_pip_install_line_forces_an_upgrade(tmp_path: Path) -> None:
+    """Real user-reported failure: `%pip install "table-validator"` printed
+    "Requirement already satisfied" and kept an older engine (0.1.20),
+    which then crashed on display(). The install must force an upgrade so
+    an existing older install is actually replaced."""
+    output = tmp_path / "TableValidator.py"
+    generate_notebook(output, "full")
+    content = output.read_text(encoding="utf-8")
+
+    assert "pip install --upgrade" in content
+    assert '"table-validator>=' in content
+
+
+def test_notebook_reports_the_loaded_engine_version(tmp_path: Path) -> None:
+    """The version that matters is the one actually imported, not the one
+    the install cell asked for - the notebook must print it and warn when
+    it is below the required floor."""
+    output = tmp_path / "TableValidator.py"
+    generate_notebook(output, "full")
+    content = output.read_text(encoding="utf-8")
+
+    assert "table_validator.__version__" in content
+    assert "WARNING" in content
+    assert "ArrowTypeError" in content
+
+
+def test_core_version_floor_includes_the_arrow_fix(tmp_path: Path) -> None:
+    """The floor must be at least 0.1.22, the first release whose result
+    sheets survive Arrow conversion (what Databricks' display() uses)."""
+    output = tmp_path / "TableValidator.py"
+    generate_notebook(output, "full")
+    content = output.read_text(encoding="utf-8")
+
+    match = re.search(r'table-validator>=([0-9]+)\.([0-9]+)\.([0-9]+)"', content)
+    assert match
+    assert tuple(int(g) for g in match.groups()) >= (0, 1, 22)
+
+
+def test_install_line_always_fetches_the_latest_engine(tmp_path: Path) -> None:
+    """The install must actually land the newest release. --upgrade
+    replaces an engine already present on the cluster (without it pip
+    just reports "Requirement already satisfied"), and --no-cache-dir
+    stops a stale cached wheel being reused."""
+    output = tmp_path / "TableValidator.py"
+    generate_notebook(output, "full")
+    content = output.read_text(encoding="utf-8")
+
+    install_line = next(
+        line for line in content.splitlines() if "pip install" in line and "table-validator>=" in line
+    )
+    assert "--upgrade" in install_line
+    assert "--no-cache-dir" in install_line

tablevalidator_databricks-0.1.6/tablevalidator_databricks.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.1.6",
-  "distance": 0,
-  "node": "g85101fc",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-09-10"
-}