table-validator 0.1.20__tar.gz → 0.1.22__tar.gz

{table_validator-0.1.20/table_validator.egg-info → table_validator-0.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.20
+Version: 0.1.22
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

{table_validator-0.1.20 → table_validator-0.1.22}/table_validator/cli/main.py

@@ -170,6 +170,16 @@ def info() -> None:
         "  tablevalidator report   Print the summary table from an "
         "existing report without opening it.\n"
         "\n"
+        "Other ways to use this package:\n"
+        "  Inside a Databricks notebook, call validate_tables() directly - "
+        "no credentials to configure, it reuses the notebook's own Spark "
+        "session. Run validate_tables.help() in a cell for the full "
+        "syntax.\n"
+        "  Prefer filling in form fields over writing code? Install the "
+        "'tablevalidator-databricks' package and run "
+        "'tablevalidator-databricks info' - it generates a Databricks "
+        "notebook driven entirely by widgets.\n"
+        "\n"
         "Run 'tablevalidator <command> --help' for a command's full "
         "options (e.g. tablevalidator validate --help).\n"
         "\n"

{table_validator-0.1.20 → table_validator-0.1.22}/table_validator/notebook.py

@@ -171,7 +171,41 @@ class ResultTable:
         self.rows = rows
 
     def to_dataframe(self) -> pd.DataFrame:
-        return pd.DataFrame(self.rows, columns=self.headers)
+        """A real pandas DataFrame of this sheet.
+
+        Mixed-type columns are normalized first. A row-builder can put an
+        int in a cell for one table and "" or None in the same column for
+        another (e.g. "Row Count (Src)" is a real count for a table that
+        reached the row-count stage, and None for one that was SKIPPED
+        before it) - pandas then types that column as `object`, and
+        Databricks' display() fails converting it to Arrow with
+        "ArrowTypeError: Expected bytes, got a 'int' object", because
+        Arrow infers a string type from the object dtype and then meets
+        an int. Coercing each object column to a single consistent type
+        removes that ambiguity, so display(...) works on every sheet.
+        """
+        df = pd.DataFrame(self.rows, columns=self.headers)
+
+        for col in df.columns:
+            if df[col].dtype != object:
+                continue
+            # Blank-ish placeholders and real nulls are the same thing to
+            # a reader - unify them so a column of numbers-plus-blanks is
+            # seen as numeric rather than mixed.
+            cleaned = df[col].map(lambda v: None if v is None or v == "" else v)
+            if cleaned.isna().all():
+                df[col] = cleaned.astype("string")
+                continue
+            numeric = pd.to_numeric(cleaned, errors="coerce")
+            if numeric.notna().sum() == cleaned.notna().sum():
+                # Every non-blank value really is a number.
+                df[col] = numeric
+            else:
+                # Anything else renders as text (statuses, percentages,
+                # mixed content) - str() each value so Arrow sees one type.
+                df[col] = cleaned.map(lambda v: None if v is None else str(v)).astype("string")
+
+        return df
 
     def __len__(self) -> int:
         return len(self.rows)

{table_validator-0.1.20 → table_validator-0.1.22/table_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.20
+Version: 0.1.22
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

table_validator-0.1.22/table_validator.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.1.22",
+  "distance": 0,
+  "node": "g9bb35af8f8b4c9edbf94bf61c68df2aed7fbfe08",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-10"
+}

{table_validator-0.1.20 → table_validator-0.1.22}/tests/test_notebook.py

@@ -822,3 +822,85 @@ def test_enabled_validations_passed_through_when_given():
         )
 
     assert captured["request"].enabled_validations == {ValidationType.SCHEMA}
+
+
+# ---------------------------------------------------------------------------
+# to_dataframe() type normalization - real user-reported failure. Databricks'
+# display() converts via Arrow, and a column holding an int for one table and
+# "" / None for another gets object dtype, which Arrow rejects with
+# "ArrowTypeError: Expected bytes, got a 'int' object". Every sheet must be
+# Arrow-convertible.
+# ---------------------------------------------------------------------------
+def test_to_dataframe_normalizes_mixed_int_and_blank_column():
+    """A column mixing real ints with blank placeholders must come back as
+    a single numeric dtype, not object."""
+    from table_validator.notebook import ResultTable
+
+    df = ResultTable(["Table", "Row Count"], [["a", 100], ["b", ""]]).to_dataframe()
+
+    assert df["Row Count"].dtype != object
+    assert df["Row Count"].iloc[0] == 100
+
+
+def test_to_dataframe_normalizes_mixed_int_and_none_column():
+    from table_validator.notebook import ResultTable
+
+    df = ResultTable(["Table", "Row Count"], [["a", 5], ["b", None]]).to_dataframe()
+
+    assert df["Row Count"].dtype != object
+
+
+def test_to_dataframe_keeps_genuine_text_columns_as_text():
+    """Status columns and the like must stay readable, not be coerced to
+    NaN by the numeric path."""
+    from table_validator.notebook import ResultTable
+
+    df = ResultTable(["Status"], [["PASS"], ["FAIL"]]).to_dataframe()
+
+    assert list(df["Status"]) == ["PASS", "FAIL"]
+
+
+def test_to_dataframe_handles_a_column_of_only_blanks():
+    from table_validator.notebook import ResultTable
+
+    df = ResultTable(["Note"], [[""], [None]]).to_dataframe()
+
+    assert len(df) == 2
+
+
+def test_every_sheet_is_arrow_convertible():
+    """End-to-end guard for the reported display() crash: convert every
+    sheet exactly the way Databricks' display() does."""
+    pa = pytest.importorskip("pyarrow")
+
+    table_ok = TableValidationResult(
+        schema_name="sch", table="t1", status=ValidationStatus.PASS,
+        exists_in_source=True, exists_in_target=True,
+        row_count_source=100, row_count_target=100,
+    )
+    # A SKIPPED table leaves the row counts as None - this is what mixes
+    # types within a single column.
+    table_skipped = TableValidationResult(
+        schema_name="sch", table="t2", status=ValidationStatus.SKIPPED,
+        exists_in_source=True, exists_in_target=True,
+    )
+    response = CatalogValidationResponse(
+        source_catalog="c1", target_catalog="c2", status=ValidationStatus.PASS,
+        schemas=[SchemaValidationResult(
+            schema_name="sch", status=ValidationStatus.PASS,
+            tables=[table_ok, table_skipped],
+        )],
+    )
+
+    with patch("table_validator.notebook.SparkConnector", return_value=MagicMock()), \
+         patch(
+             "table_validator.notebook.CatalogValidator.compare_catalogs",
+             lambda self, request: response,
+         ):
+        result = validate_tables("c1.sch", "c2.sch", spark=MagicMock())
+
+    for sheet in (
+        "table_validation", "column_validation", "data_mismatches",
+        "row_hash_mismatches", "mismatch_categories", "suggestions",
+    ):
+        pa.Table.from_pandas(getattr(result, sheet).to_dataframe())

table_validator-0.1.20/table_validator.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.1.20",
-  "distance": 0,
-  "node": "g9124e37c7cb704b705524188c1a7fec5d1848039",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-09-08"
-}