table-validator 0.1.22__tar.gz → 0.1.23__tar.gz

{table_validator-0.1.22/table_validator.egg-info → table_validator-0.1.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.22
+Version: 0.1.23
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9
@@ -24,6 +24,7 @@ Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: ruff>=0.4; extra == "dev"
 Requires-Dist: black>=24.0; extra == "dev"
+Requires-Dist: duckdb>=1.0; extra == "dev"
 Provides-Extra: spark
 Requires-Dist: pyspark>=3.4; extra == "spark"
 Dynamic: license-file

{table_validator-0.1.22 → table_validator-0.1.23}/pyproject.toml

@@ -31,6 +31,10 @@ dev = [
     "pytest>=7.0",
     "ruff>=0.4",
     "black>=24.0",
+    # tests/test_end_to_end_real_engine.py runs the validator against a
+    # real SQL engine rather than mocked query results, so that every
+    # user-facing option is proven to actually change what gets validated.
+    "duckdb>=1.0",
 ]
 # Only needed to construct SparkConnector/call validate_tables() OUTSIDE a
 # real Databricks notebook (e.g. local dev/testing) - inside an actual

{table_validator-0.1.22 → table_validator-0.1.23/table_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.22
+Version: 0.1.23
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9
@@ -24,6 +24,7 @@ Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: ruff>=0.4; extra == "dev"
 Requires-Dist: black>=24.0; extra == "dev"
+Requires-Dist: duckdb>=1.0; extra == "dev"
 Provides-Extra: spark
 Requires-Dist: pyspark>=3.4; extra == "spark"
 Dynamic: license-file

{table_validator-0.1.22 → table_validator-0.1.23}/table_validator.egg-info/SOURCES.txt

@@ -43,6 +43,7 @@ tests/test_blob_discovery.py
 tests/test_catalog_validator.py
 tests/test_cli.py
 tests/test_databricks_connector.py
+tests/test_end_to_end_real_engine.py
 tests/test_excel_report.py
 tests/test_mismatch_classifier.py
 tests/test_notebook.py

{table_validator-0.1.22 → table_validator-0.1.23}/table_validator.egg-info/requires.txt

@@ -17,6 +17,7 @@ python-dotenv>=1.0
 pytest>=7.0
 ruff>=0.4
 black>=24.0
+duckdb>=1.0
 
 [spark]
 pyspark>=3.4

{table_validator-0.1.22 → table_validator-0.1.23}/table_validator.egg-info/scm_file_list.json

@@ -37,6 +37,7 @@
     "tests/test_catalog_validator.py",
     "tests/test_cli.py",
     "tests/test_databricks_connector.py",
+    "tests/test_end_to_end_real_engine.py",
     "tests/test_excel_report.py",
     "tests/test_mismatch_classifier.py",
     "tests/test_notebook.py",

{table_validator-0.1.22 → table_validator-0.1.23}/table_validator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.1.22",
+  "tag": "0.1.23",
   "distance": 0,
-  "node": "g9bb35af8f8b4c9edbf94bf61c68df2aed7fbfe08",
+  "node": "ga62949fa884aa38b3e70751533b67c62420dacb7",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-09-10"

table_validator-0.1.23/tests/test_end_to_end_real_engine.py

@@ -0,0 +1,333 @@
+"""End-to-end validation against a REAL SQL engine, not mocked results.
+
+Every other test file mocks the connector's query results, which proves
+the validator's own logic but not that a user-facing option actually
+changes what gets validated. These tests create real tables in DuckDB,
+run the real validator against them, and assert on real verdicts - so
+each option offered by the CLI wizard, the validate_tables() API and the
+Databricks widget notebook is proven to do something, not merely to be
+passed along.
+
+DuckDB is a stand-in for Databricks SQL: the validator's generated SQL is
+executed unchanged apart from a handful of dialect renames in
+_DuckDbConnector._execute_to_dataframe (sha2 -> sha256, STRING ->
+VARCHAR, and a conv() shim). Those translations are in the harness only;
+the validator itself is never special-cased.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+
+import pandas as pd
+import pytest
+
+from table_validator.config.schema import ValidationType
+from table_validator.connectors.base_sql_connector import BaseSqlConnector
+from table_validator.models import CatalogValidationRequest
+from table_validator.notebook import ValidationResult, _parse_source_target
+from table_validator.validators.catalog_validator import CatalogValidator
+
+duckdb = pytest.importorskip("duckdb", reason="real-engine tests need duckdb")
+
+
+def _conv(value: str, from_base: int, to_base: int) -> int:
+    """Databricks' conv(str, from_base, to_base), which DuckDB lacks. The
+    validator uses it to turn a hex hash slice into a number for
+    fingerprinting."""
+    try:
+        return int(str(value), int(from_base))
+    except (TypeError, ValueError):
+        return 0
+
+
+@pytest.fixture
+def con():
+    """A real database with real rows.
+
+    orders   - identical on both sides EXCEPT one changed amount (id=3)
+    customers- identical on both sides
+    products - identical on both sides
+    """
+    connection = duckdb.connect()
+    connection.create_function("conv", _conv, ["VARCHAR", "INTEGER", "INTEGER"], "BIGINT")
+    connection.execute("CREATE SCHEMA bronze")
+    connection.execute("CREATE SCHEMA silver")
+
+    connection.execute(
+        """CREATE TABLE bronze.orders AS SELECT * FROM (VALUES
+        (1,'alice',100,'active'), (2,'bob',200,'active'),
+        (3,'carol',300,'inactive'), (4,'dan',400,'active')
+        ) t(id, name, amount, status)"""
+    )
+    connection.execute(
+        """CREATE TABLE silver.orders AS SELECT * FROM (VALUES
+        (1,'alice',100,'active'), (2,'bob',200,'active'),
+        (3,'carol',999,'inactive'), (4,'dan',400,'active')
+        ) t(id, name, amount, status)"""
+    )
+    for schema in ("bronze", "silver"):
+        connection.execute(
+            f"CREATE TABLE {schema}.customers AS SELECT * FROM "
+            "(VALUES (1,'x'), (2,'y')) t(id, label)"
+        )
+        connection.execute(
+            f"CREATE TABLE {schema}.products AS SELECT 1 AS id, 'p' AS sku"
+        )
+    return connection
+
+
+class _DuckDbConnector(BaseSqlConnector):
+    """Executes the validator's real SQL against DuckDB, recording every
+    query so tests can assert on what was actually asked of the engine."""
+
+    def __init__(self, con):
+        super().__init__()
+        self._con = con
+        self.queries: List[str] = []
+
+    def connect(self) -> None: ...
+    def disconnect(self) -> None: ...
+    def test_connection(self) -> bool: return True
+
+    def _qualify(self, catalog: str, schema: str, table: str) -> str:
+        # DuckDB has no catalog layer in this harness.
+        return f'"{schema}"."{table}"'
+
+    def _execute_to_dataframe(self, query: str) -> pd.DataFrame:
+        self.queries.append(query)
+        translated = (
+            query.replace("`", '"')
+            .replace("sha2(", "sha256(")
+            .replace(", 256)", ")")
+            .replace("AS STRING", "AS VARCHAR")
+        )
+        return self._con.execute(translated).df()
+
+    def get_schemas(self, catalog: str) -> List[str]:
+        return ["bronze", "silver"]
+
+    def get_tables(self, catalog: str, schema: str) -> List[str]:
+        rows = self._con.execute(
+            "SELECT table_name FROM information_schema.tables WHERE table_schema=?",
+            [schema],
+        ).fetchall()
+        return sorted(r[0] for r in rows)
+
+    def get_table_schema(self, catalog: str, schema: str, table: str) -> pd.DataFrame:
+        rows = self._con.execute(
+            "SELECT column_name, data_type, is_nullable, ordinal_position "
+            "FROM information_schema.columns WHERE table_schema=? AND table_name=? "
+            "ORDER BY ordinal_position",
+            [schema, table],
+        ).fetchall()
+        return pd.DataFrame(
+            [
+                {
+                    "column_name": r[0],
+                    "data_type": str(r[1]).lower(),
+                    "is_nullable": r[2] == "YES",
+                    "ordinal_position": r[3],
+                }
+                for r in rows
+            ]
+        )
+
+    def catalog_exists(self, catalog: str) -> bool:
+        return True
+
+    def is_min_max_eligible(self, data_type: str) -> bool:
+        return any(k in str(data_type).lower() for k in ("int", "decimal", "double", "float"))
+
+
+def _run(con, source: str, **kwargs) -> Tuple[ValidationResult, _DuckDbConnector]:
+    """Run the real validator the same way validate_tables() does."""
+    target = kwargs.pop("target")
+    src_catalog, src_schema, src_table = _parse_source_target(source, "source")
+    tgt_catalog, tgt_schema, tgt_table = _parse_source_target(target, "target")
+
+    connector = _DuckDbConnector(con)
+    if any(kwargs.get(k) for k in ("row_filter", "source_row_filter", "target_row_filter")):
+        connector.set_row_filters(
+            common=kwargs.get("row_filter"),
+            source=kwargs.get("source_row_filter"),
+            target=kwargs.get("target_row_filter"),
+        )
+
+    primary_key = kwargs.get("primary_key")
+    request_kwargs: Dict[str, Any] = dict(
+        source_catalog=src_catalog,
+        target_catalog=tgt_catalog,
+        schemas=[src_schema],
+        schema_map={src_schema: tgt_schema} if src_schema != tgt_schema else {},
+        tables=[src_table] if src_table else None,
+        table_map={},
+        primary_keys=(
+            {tgt_table: primary_key, f"{tgt_schema}.{tgt_table}": primary_key}
+            if primary_key and tgt_table
+            else {}
+        ),
+        ignore_columns=kwargs.get("ignore_columns") or [],
+        only_columns=kwargs.get("only_columns"),
+        column_map=kwargs.get("column_map") or {},
+        ignore_datatype_columns=kwargs.get("ignore_datatype_columns") or [],
+    )
+    if kwargs.get("enabled_validations") is not None:
+        request_kwargs["enabled_validations"] = kwargs["enabled_validations"]
+
+    response = CatalogValidator(connector).compare_catalogs(
+        CatalogValidationRequest(**request_kwargs)
+    )
+    return ValidationResult(response), connector
+
+
+def _tables_in(result: ValidationResult) -> List[str]:
+    return sorted(t.table for s in result.response.schemas for t in s.tables)
+
+
+# ---------------------------------------------------------------------------
+# Source/Target Table selection
+# ---------------------------------------------------------------------------
+def test_single_table_selection_compares_only_that_table(con):
+    result, _ = _run(con, "cat.bronze.orders", target="cat.silver.orders")
+    assert _tables_in(result) == ["orders"]
+
+
+def test_a_real_data_difference_is_actually_detected(con):
+    """silver.orders has a changed amount for id=3 - the run must FAIL."""
+    result, _ = _run(con, "cat.bronze.orders", target="cat.silver.orders")
+    assert result.response.status.value == "FAIL"
+
+
+def test_identical_tables_pass(con):
+    result, _ = _run(con, "cat.bronze.customers", target="cat.silver.customers")
+    assert result.response.status.value == "PASS"
+
+
+# ---------------------------------------------------------------------------
+# Schema-wide sweep - the "(all tables in schema)" widget option
+# ---------------------------------------------------------------------------
+def test_sweep_validates_every_table_not_just_the_first(con):
+    """Reported issue: a sweep appeared to validate only the first table."""
+    result, _ = _run(con, "cat.bronze", target="cat.silver")
+    assert _tables_in(result) == ["customers", "orders", "products"]
+
+
+def test_sweep_still_flags_the_one_bad_table(con):
+    result, _ = _run(con, "cat.bronze", target="cat.silver")
+    assert result.response.status.value == "FAIL"
+
+    per_table = {t.table: t.status.value for s in result.response.schemas for t in s.tables}
+    assert per_table["orders"] == "FAIL"
+    assert per_table["customers"] == "PASS"
+
+
+# ---------------------------------------------------------------------------
+# Row filter widgets
+# ---------------------------------------------------------------------------
+def test_row_filter_reaches_the_real_sql(con):
+    _, connector = _run(con, "cat.bronze.orders", target="cat.silver.orders", row_filter="id < 3")
+    assert any("WHERE (id < 3)" in q for q in connector.queries)
+
+
+def test_row_filter_excluding_the_bad_row_turns_fail_into_pass(con):
+    """The strongest proof the filter is real: the same two tables give a
+    different verdict purely because of the filter."""
+    unfiltered, _ = _run(con, "cat.bronze.orders", target="cat.silver.orders")
+    filtered, _ = _run(con, "cat.bronze.orders", target="cat.silver.orders", row_filter="id < 3")
+
+    assert unfiltered.response.status.value == "FAIL"
+    assert filtered.response.status.value == "PASS"
+
+
+def test_per_side_row_filters_are_applied_independently(con):
+    _, connector = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders",
+        source_row_filter="id > 1", target_row_filter="id > 2",
+    )
+    assert any("id > 1" in q for q in connector.queries)
+    assert any("id > 2" in q for q in connector.queries)
+
+
+# ---------------------------------------------------------------------------
+# Primary key widget
+# ---------------------------------------------------------------------------
+def test_primary_key_is_used_for_row_hashing(con):
+    _, connector = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders", primary_key=["id"],
+    )
+    assert any("`id`" in q and "row_hash" in q.lower() for q in connector.queries)
+
+
+def test_primary_key_pinpoints_the_exact_changed_record(con):
+    result, _ = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders", primary_key=["id"],
+    )
+    table = [t for s in result.response.schemas for t in s.tables][0]
+    assert table.data.row_hash_mismatch_count == 1
+
+
+# ---------------------------------------------------------------------------
+# Column controls
+# ---------------------------------------------------------------------------
+def test_only_columns_never_queries_the_excluded_column(con):
+    _, connector = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders", only_columns=["id", "name"],
+    )
+    assert not any("amount" in q for q in connector.queries)
+
+
+def test_ignoring_the_differing_column_turns_fail_into_pass(con):
+    result, connector = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders", ignore_columns=["amount"],
+    )
+    assert result.response.status.value == "PASS"
+    assert not any("amount" in q for q in connector.queries)
+
+
+# ---------------------------------------------------------------------------
+# Check-group widgets
+# ---------------------------------------------------------------------------
+def test_disabling_check_groups_does_less_real_work(con):
+    """Turning checks off must genuinely skip queries, not just relabel
+    the result."""
+    _, few = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders",
+        enabled_validations={ValidationType.CATALOG, ValidationType.SCHEMA},
+    )
+    _, many = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders",
+        enabled_validations={
+            ValidationType.CATALOG, ValidationType.SCHEMA,
+            ValidationType.COLUMN, ValidationType.ROW,
+        },
+    )
+    assert len(few.queries) < len(many.queries)
+
+
+def test_catalog_schema_only_reports_skipped(con):
+    """Documented behaviour users hit: this combination produces no
+    per-table verdict."""
+    result, _ = _run(
+        con, "cat.bronze.orders", target="cat.silver.orders",
+        enabled_validations={ValidationType.CATALOG, ValidationType.SCHEMA},
+    )
+    table = [t for s in result.response.schemas for t in s.tables][0]
+    assert table.status.value == "SKIPPED"
+
+
+# ---------------------------------------------------------------------------
+# The results a user actually reads
+# ---------------------------------------------------------------------------
+def test_result_sheets_are_arrow_convertible_after_a_real_run(con):
+    """display() in Databricks converts via Arrow - every sheet from a
+    real run must survive that."""
+    pa = pytest.importorskip("pyarrow")
+
+    result, _ = _run(con, "cat.bronze", target="cat.silver")
+
+    for sheet in (
+        "table_validation", "column_validation", "data_mismatches",
+        "row_hash_mismatches", "mismatch_categories", "suggestions",
+    ):
+        pa.Table.from_pandas(getattr(result, sheet).to_dataframe())