table-validator 0.1.16__tar.gz → 0.1.18__tar.gz

{table_validator-0.1.16/table_validator.egg-info → table_validator-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.16
+Version: 0.1.18
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

{table_validator-0.1.16 → table_validator-0.1.18}/table_validator/cli/main.py

@@ -507,6 +507,17 @@ def _run_databricks_validation(
         column_map=config.column_map,
     )
 
+    # Row-filter predicate - lives purely as connector instance state
+    # (BaseSqlConnector.set_row_filters/_scoped_table), same mechanism as
+    # the notebook-native validate_tables() API, not a CatalogValidationRequest
+    # field. A no-op unless the user configured at least one of the three.
+    if config.row_filter or config.source_row_filter or config.target_row_filter:
+        databricks.set_row_filters(
+            common=config.row_filter,
+            source=config.source_row_filter,
+            target=config.target_row_filter,
+        )
+
     partition_prompt = build_partition_prompt(yes=yes)
     validator = CatalogValidator(databricks, partition_prompt=partition_prompt)
     return validator.compare_catalogs(request)

{table_validator-0.1.16 → table_validator-0.1.18}/table_validator/cli/wizard.py

@@ -192,6 +192,57 @@ def _prompt_column_customization(config: ValidatorConfig) -> None:
     ) or []
 
 
+def _prompt_row_filter(config: ValidatorConfig) -> None:
+    """Optional row-filter predicate, asked right after column
+    customization - same single-table scope, Databricks source only
+    (AzureSqlConnector has no row-filter mechanism, unlike
+    BaseSqlConnector's set_row_filters/_scoped_table that the Databricks
+    connector and the notebook-native validate_tables() API both share).
+    Skipped entirely (leaving any existing row_filter/source_row_filter/
+    target_row_filter untouched) unless the user opts in, so declining
+    gives identical behavior to before this feature existed."""
+    customize = questionary.confirm(
+        "Filter to a subset of rows? (only rows matching a condition are "
+        "compared, e.g. \"id > 10 and id < 30\" or \"gender = 'male'\")",
+        default=False,
+    ).ask()
+
+    if not customize:
+        return
+
+    same_condition = questionary.confirm(
+        "Apply the same condition to both source and target?",
+        default=True,
+    ).ask()
+
+    # Mutually exclusive in this UI - picking one must clear whatever
+    # stale value the OTHER carried over from an earlier configure run,
+    # same convention as _prompt_schema_scoping below.
+    if same_condition:
+        config.row_filter = _ask(
+            questionary.text(
+                "Row filter condition (SQL WHERE-fragment):",
+                default=config.row_filter or "",
+            )
+        ) or None
+        config.source_row_filter = None
+        config.target_row_filter = None
+    else:
+        config.row_filter = None
+        config.source_row_filter = _ask(
+            questionary.text(
+                "Source row filter condition:",
+                default=config.source_row_filter or "",
+            )
+        ) or None
+        config.target_row_filter = _ask(
+            questionary.text(
+                "Target row filter condition:",
+                default=config.target_row_filter or "",
+            )
+        ) or None
+
+
 def _prompt_schema_scoping(config: ValidatorConfig) -> None:
     """
     Optional schema-wide scoping, asked only when a schema is named on
@@ -725,10 +776,11 @@ def run_configure_wizard() -> None:
         # identical behavior to before this feature existed.
         typer.echo("\n== Customize validation (optional) ==")
         _prompt_column_customization(config)
-        # Column-name mapping (renamed columns) - Databricks-to-Databricks
-        # only, since column_map/CatalogValidator don't apply to the
-        # Azure Blob/SQL source paths.
+        # Row-filter predicate and column-name mapping (renamed columns) -
+        # Databricks-to-Databricks only, since set_row_filters/column_map/
+        # CatalogValidator don't apply to the Azure Blob/SQL source paths.
         if config.source_type == SourceType.DATABRICKS:
+            _prompt_row_filter(config)
             _prompt_column_mapping(config, secrets)
     else:
         config.primary_key = None
@@ -736,6 +788,9 @@ def run_configure_wizard() -> None:
         config.ignore_columns = []
         config.ignore_datatype_columns = []
         config.column_map = {}
+        config.row_filter = None
+        config.source_row_filter = None
+        config.target_row_filter = None
 
     # Schema-wide sweep scoping - only meaningful when a schema is named
     # on both sides but no specific table is (the single-table branch

{table_validator-0.1.16 → table_validator-0.1.18}/table_validator/config/schema.py

@@ -314,6 +314,38 @@ class ValidatorConfig(BaseModel):
         ),
     )
 
+    row_filter: Optional[str] = Field(
+        default=None,
+        description=(
+            "Optional SQL WHERE-fragment restricting comparison to matching "
+            "rows on BOTH sides (row count, statistics, row-hash/column "
+            "diff) - e.g. 'id > 20 and id < 100' or \"gender = 'male'\". "
+            "Only meaningful for the single named table in source_table/"
+            "target_table (Databricks source only). Combined with "
+            "source_row_filter/target_row_filter via AND if both are set. "
+            "Used as-is, not parsed or validated - a typo surfaces as a "
+            "normal SQL error when the query runs."
+        ),
+    )
+
+    source_row_filter: Optional[str] = Field(
+        default=None,
+        description=(
+            "Optional SQL WHERE-fragment applied only to the source side, "
+            "ANDed with row_filter if both are set. Use this (with "
+            "target_row_filter) instead of row_filter when the condition "
+            "must differ per side."
+        ),
+    )
+
+    target_row_filter: Optional[str] = Field(
+        default=None,
+        description=(
+            "Optional SQL WHERE-fragment applied only to the target side, "
+            "ANDed with row_filter if both are set."
+        ),
+    )
+
     blob_source: BlobSourceConfig = Field(default_factory=BlobSourceConfig)
     sql_source: SqlSourceConfig = Field(default_factory=SqlSourceConfig)
     synapse_source: SynapseSourceConfig = Field(default_factory=SynapseSourceConfig)

{table_validator-0.1.16 → table_validator-0.1.18}/table_validator/connectors/base_sql_connector.py

@@ -528,6 +528,24 @@ class BaseSqlConnector:
                 return None
             return int(value)
 
+        def _nan_to_none(value: Any) -> Any:
+            """
+            MIN/MAX over ZERO matching rows (e.g. both sides legitimately
+            empty under the same row_filter) is SQL NULL, which arrives
+            here as float NaN after the spark.sql(...).toPandas() round-
+            trip - same underlying issue as _to_int_or_none above, but
+            min/max must stay their original type (a string, date, etc.)
+            when a real value IS present, so this only strips NaN rather
+            than casting to int. Without this, two equally-empty sides
+            each report NaN for min/max, and compare_min_max's `==`
+            check treats NaN != NaN (the standard IEEE 754 quirk) as a
+            real difference - a false FAIL on a table with nothing to
+            compare, not an actual mismatch.
+            """
+            if isinstance(value, float) and value != value:
+                return None
+            return value
+
         for col in columns:
             entry: Dict[str, Any] = {
                 "null_count": _to_int_or_none(row.get(f"{col}__nulls")),
@@ -536,8 +554,8 @@ class BaseSqlConnector:
                 "max": None,
             }
             if col.lower() in min_max_set:
-                entry["min"] = row.get(f"{col}__min")
-                entry["max"] = row.get(f"{col}__max")
+                entry["min"] = _nan_to_none(row.get(f"{col}__min"))
+                entry["max"] = _nan_to_none(row.get(f"{col}__max"))
             result[col] = entry
 
         return result

{table_validator-0.1.16 → table_validator-0.1.18/table_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.16
+Version: 0.1.18
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

{table_validator-0.1.16 → table_validator-0.1.18}/table_validator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.1.16",
+  "tag": "0.1.18",
   "distance": 0,
-  "node": "g676da730fd4229d0b784a52582b61554b052b4bb",
+  "node": "g442d915637cacc0f99f5a1eb372c1cc7128f7e06",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-09-03"

{table_validator-0.1.16 → table_validator-0.1.18}/tests/test_base_sql_connector.py

@@ -198,3 +198,76 @@ def test_get_column_statistics_still_converts_real_values_to_int():
     assert stats["id"]["distinct_count"] == 42
     assert isinstance(stats["id"]["null_count"], int)
     assert isinstance(stats["id"]["distinct_count"], int)
+
+
+def test_get_column_statistics_min_max_nan_becomes_none():
+    """Real bug found via live user testing: comparing a table against
+    ITSELF with the exact same row_filter on both sides (matching zero
+    rows) still reported Overall Status FAIL, specifically Min/Max FAIL -
+    MIN/MAX over zero rows is SQL NULL -> NaN after the pandas round-
+    trip, and NaN != NaN (IEEE 754), so compare_min_max's `==` check
+    treated two equally-empty, equally-NaN sides as a real difference.
+    min/max must come back as None (not NaN) so two empty sides compare
+    equal."""
+    fake_spark = MagicMock()
+    fake_spark.sql.return_value.toPandas.return_value = pd.DataFrame(
+        {"id__nulls": [float("nan")], "id__distinct": [float("nan")],
+         "id__min": [float("nan")], "id__max": [float("nan")]}
+    )
+    connector = SparkConnector(spark=fake_spark)
+
+    stats = connector.get_column_statistics(
+        "cat", "sch", "tbl", ["id"], min_max_columns=["id"],
+    )
+
+    assert stats["id"]["min"] is None
+    assert stats["id"]["max"] is None
+
+
+def test_get_column_statistics_min_max_real_values_pass_through_unchanged():
+    """Regression guard alongside the NaN fix above - a genuine min/max
+    result (of any type - string, date, number) must pass through
+    completely unchanged, not be mistaken for NaN."""
+    fake_spark = MagicMock()
+    fake_spark.sql.return_value.toPandas.return_value = pd.DataFrame(
+        {"name__nulls": [0], "name__distinct": [5],
+         "name__min": ["Alice"], "name__max": ["Zoe"]}
+    )
+    connector = SparkConnector(spark=fake_spark)
+
+    stats = connector.get_column_statistics(
+        "cat", "sch", "tbl", ["name"], min_max_columns=["name"],
+    )
+
+    assert stats["name"]["min"] == "Alice"
+    assert stats["name"]["max"] == "Zoe"
+
+
+def test_two_identically_filtered_empty_sides_compare_as_pass():
+    """End-to-end regression test for the exact reported scenario: the
+    SAME table compared against itself with the SAME row_filter matching
+    zero rows on both sides must report min/max as equal (both None),
+    not a false mismatch."""
+    from table_validator.validators.catalog_validator import CatalogValidator
+
+    fake_spark = MagicMock()
+    connector = SparkConnector(spark=fake_spark)
+    connector.set_row_filters(common="gender = 'male'")
+
+    fake_spark.sql.return_value.toPandas.return_value = pd.DataFrame(
+        {"id__nulls": [float("nan")], "id__distinct": [float("nan")],
+         "id__min": [float("nan")], "id__max": [float("nan")]}
+    )
+    source_stats = connector.get_column_statistics(
+        "cat", "sch", "src_tbl", ["id"], min_max_columns=["id"], side="source",
+    )
+    target_stats = connector.get_column_statistics(
+        "cat", "sch", "tgt_tbl", ["id"], min_max_columns=["id"], side="target",
+    )
+
+    validator = CatalogValidator(connector)
+    status = validator.compare_min_max(
+        source_stats["id"]["min"], source_stats["id"]["max"],
+        target_stats["id"]["min"], target_stats["id"]["max"],
+    )
+    assert status.value == "PASS"

{table_validator-0.1.16 → table_validator-0.1.18}/tests/test_cli.py

@@ -1437,6 +1437,65 @@ def test_validate_databricks_wires_only_columns_ignore_columns_and_ignore_dataty
     assert request.column_map == {"cust_id": "customer_id"}
 
 
+def test_validate_databricks_wires_row_filters_to_connector(tmp_path: Path) -> None:
+    """config.row_filter/source_row_filter/target_row_filter must reach the
+    Databricks connector via set_row_filters() before compare_catalogs runs -
+    these live purely as connector instance state (same mechanism as the
+    notebook-native validate_tables() API), never as CatalogValidationRequest
+    fields."""
+    config_path = _full_config(tmp_path)
+    config = load_config(config_path)
+    config.row_filter = "status = 'active'"
+    config.source_row_filter = "id > 20"
+    config.target_row_filter = "id > 15"
+    save_config(config, config_path)
+    output_path = tmp_path / "validation_report.xlsx"
+
+    mock_connector = _mock_databricks_connector()
+
+    with patch(
+        "table_validator.cli.main.DatabricksConnector", return_value=mock_connector
+    ), patch(
+        "table_validator.cli.main.get_databricks_token", return_value="dapi_fake"
+    ), patch(
+        "table_validator.cli.main.get_azure_credential", return_value=None
+    ):
+        runner.invoke(
+            app,
+            ["validate", "--config-path", str(config_path), "--output", str(output_path)],
+        )
+
+    mock_connector.set_row_filters.assert_called_once_with(
+        common="status = 'active'", source="id > 20", target="id > 15",
+    )
+
+
+def test_validate_databricks_skips_set_row_filters_when_none_configured(
+    tmp_path: Path,
+) -> None:
+    """A config with no row filters set must never call set_row_filters at
+    all - regression guard so a table without this feature configured
+    behaves identically to before the feature existed."""
+    config_path = _full_config(tmp_path)
+    output_path = tmp_path / "validation_report.xlsx"
+
+    mock_connector = _mock_databricks_connector()
+
+    with patch(
+        "table_validator.cli.main.DatabricksConnector", return_value=mock_connector
+    ), patch(
+        "table_validator.cli.main.get_databricks_token", return_value="dapi_fake"
+    ), patch(
+        "table_validator.cli.main.get_azure_credential", return_value=None
+    ):
+        runner.invoke(
+            app,
+            ["validate", "--config-path", str(config_path), "--output", str(output_path)],
+        )
+
+    mock_connector.set_row_filters.assert_not_called()
+
+
 def test_validate_mode_stats_stops_before_fingerprint_and_row_hash(tmp_path: Path) -> None:
     """--mode stats is the CLI-flag equivalent of the tiered funnel's
     "statistical only" prompt: it must stop after Tier 1 even though the

{table_validator-0.1.16 → table_validator-0.1.18}/tests/test_wizard.py

@@ -343,6 +343,7 @@ def test_deselecting_row_only_omits_row(tmp_path, monkeypatch):
         "tgt_cat", "silver", "customers",
         "",                                    # primary key (blank)
         False,                                 # customize validation? (declined)
+        False,                                 # filter to a subset of rows? (declined)
         ["catalog & schema", "column"],        # validations checkbox: row unchecked
     ])
 
@@ -417,6 +418,7 @@ def test_databricks_source_type_prompts_for_source_table(tmp_path, monkeypatch):
         "tgt_cat", "silver", "customers",   # target table
         "",                                   # primary key (left blank)
         False,                                # customize validation? (declined)
+        False,                                # filter to a subset of rows? (declined)
         [],                                  # validations
     ])
 
@@ -442,6 +444,7 @@ def test_databricks_source_type_prompts_for_primary_key_when_both_tables_named(
         "tgt_cat", "silver", "customers",
         "id, region_id ",                    # primary key, comma-separated with stray spaces
         False,                                # customize validation? (declined)
+        False,                                # filter to a subset of rows? (declined)
         [],
     ])
 
@@ -462,6 +465,7 @@ def test_databricks_wizard_declining_customization_leaves_defaults_untouched(
         "tgt_cat", "silver", "customers",
         "",                                    # primary key (left blank)
         False,                                 # customize validation? (declined)
+        False,                                 # filter to a subset of rows? (declined)
         [],
     ])
 
@@ -486,6 +490,7 @@ def test_databricks_wizard_customize_populates_all_three_column_lists(
         "id, name ",                            # only_columns, stray spaces
         "updated_at",                           # ignore_columns
         "legacy_flag, notes",                   # ignore_datatype_columns
+        False,                                  # filter to a subset of rows? (declined)
         [],
     ])
 
@@ -508,6 +513,7 @@ def test_databricks_wizard_customize_all_three_left_blank(tmp_path, monkeypatch)
         "",                                    # only_columns left blank
         "",                                    # ignore_columns left blank
         "",                                    # ignore_datatype_columns left blank
+        False,                                 # filter to a subset of rows? (declined)
         [],
     ])
 
@@ -811,6 +817,7 @@ def test_column_mapping_skipped_when_no_token_available(tmp_path, monkeypatch):
         "tgt_cat", "silver", "customers",
         "id",
         False,  # customize validation? (declined)
+        False,  # filter to a subset of rows? (declined)
         [],
     ])
 
@@ -834,6 +841,7 @@ def test_column_mapping_live_connection_success_populates_map(tmp_path, monkeypa
             "tgt_cat", "silver", "customers",
             "id",
             False,           # customize validation? (declined)
+            False,           # filter to a subset of rows? (declined)
             "customer_id",   # column mapping: cust_id -> customer_id
             [],
         ],
@@ -859,6 +867,7 @@ def test_column_mapping_live_connection_failure_skips_gracefully(tmp_path, monke
             "tgt_cat", "silver", "customers",
             "id",
             False,  # customize validation? (declined)
+            False,  # filter to a subset of rows? (declined)
             [],
         ],
         databricks_token="dapi_fake",
@@ -889,6 +898,7 @@ def test_column_mapping_only_asks_about_unmatched_source_columns(tmp_path, monke
             "tgt_cat", "silver", "customers",
             "id",
             False,
+            False,  # filter to a subset of rows? (declined)
             "customer_id",  # only ONE column-mapping answer, for cust_id
             [],
         ],
@@ -916,6 +926,7 @@ def test_column_mapping_skip_leaves_column_unmapped(tmp_path, monkeypatch):
             "tgt_cat", "silver", "customers",
             "id",
             False,
+            False,  # filter to a subset of rows? (declined)
             "(skip this column)",
             [],
         ],
@@ -941,6 +952,7 @@ def test_column_mapping_no_unmatched_columns_asks_nothing(tmp_path, monkeypatch)
             "tgt_cat", "silver", "customers",
             "id",
             False,
+            False,  # filter to a subset of rows? (declined)
             # no column-mapping answer - none should be consumed
             [],
         ],
@@ -993,6 +1005,7 @@ def test_schema_scoping_prompt_skipped_when_a_table_is_named(tmp_path, monkeypat
         "tgt_cat", "silver", "customers",
         "",                                    # primary key (blank)
         False,                                 # customize validation? (declined)
+        False,                                 # filter to a subset of rows? (declined)
         # no schema-scoping answer here - none should be consumed
         [],
     ])