table-validator 0.1.5__tar.gz → 0.1.7__tar.gz

{table_validator-0.1.5/table_validator.egg-info → table_validator-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.5
+Version: 0.1.7
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

{table_validator-0.1.5 → table_validator-0.1.7}/table_validator/cli/main.py

@@ -148,7 +148,7 @@ def info() -> None:
         "Run 'tablevalidator <command> --help' for a command's full "
         "options (e.g. tablevalidator validate --help).\n"
         "\n"
-        "TO KEERTHIVASAN"
+        "-MS-"
     )
 
 
@@ -470,6 +470,9 @@ def _run_databricks_validation(
         enabled_validations=set(config.validations),
         primary_keys=primary_keys,
         max_tier=max_tier,
+        only_columns=config.only_columns,
+        ignore_columns=config.ignore_columns,
+        ignore_datatype_columns=config.ignore_datatype_columns,
     )
 
     partition_prompt = build_partition_prompt(yes=yes)

{table_validator-0.1.5 → table_validator-0.1.7}/table_validator/config/schema.py

@@ -166,6 +166,42 @@ class ValidatorConfig(BaseModel):
         ),
     )
 
+    only_columns: Optional[List[str]] = Field(
+        default=None,
+        description=(
+            "Optional allowlist: if set, only these columns (plus the "
+            "primary key, if any) are compared - every other common "
+            "column is excluded from every check. Only meaningful for the "
+            "single named table in source_table/target_table, same scope "
+            "as primary_key. If a column named here isn't actually common "
+            "to both sides, it's silently absent from the effective set "
+            "(same convention as tables/schemas restrictions elsewhere). "
+            "If both only_columns and ignore_columns name the same "
+            "column, ignore_columns wins - it is always excluded."
+        ),
+    )
+
+    ignore_columns: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Columns to exclude entirely from every check (name, type, "
+            "nullable, statistics, row-hash) - useful for auto-generated "
+            "columns like timestamps that are expected to always differ."
+        ),
+    )
+
+    ignore_datatype_columns: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Columns whose data-type mismatch should be ignored - the "
+            "column's other checks (nullable, statistics, row-hash) still "
+            "run normally, but a real type difference here is reported as "
+            "SKIPPED rather than FAIL and never fails the table or aborts "
+            "the schema stage, even for a cross-family type change that "
+            "would otherwise be BLOCKING."
+        ),
+    )
+
     blob_source: BlobSourceConfig = Field(default_factory=BlobSourceConfig)
     sql_source: SqlSourceConfig = Field(default_factory=SqlSourceConfig)
 

{table_validator-0.1.5 → table_validator-0.1.7}/table_validator/models.py

@@ -488,6 +488,35 @@ class CatalogValidationRequest(BaseModel):
 
     ignore_columns: List[str] = Field(default_factory=list)
 
+    only_columns: Optional[List[str]] = Field(
+        default=None,
+        description=(
+            "Optional allowlist: if set, common_cols is further restricted "
+            "to just these column names (case-insensitive) before any "
+            "check runs - every other common column is excluded from name/"
+            "type/nullable/statistics/row-hash checks entirely, the same "
+            "as ignore_columns' exclusion. A name here that isn't actually "
+            "a common column is silently absent from the effective set, "
+            "matching this class's other restriction fields (schemas/ "
+            "tables). If a column appears in both only_columns and "
+            "ignore_columns, ignore_columns wins - it is always excluded, "
+            "applied after the allowlist restriction."
+        ),
+    )
+
+    ignore_datatype_columns: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Columns (case-insensitive) whose data_type_status should be "
+            "reported as SKIPPED rather than PASS/FAIL, and excluded from "
+            "Tier 0's BLOCKING cross-family-type-change classification - "
+            "a real type difference on one of these columns never fails "
+            "the table or aborts the schema stage. The column's other "
+            "checks (nullable, null/distinct/min-max statistics, row-"
+            "hash) still run normally."
+        ),
+    )
+
     case_sensitive_columns: bool = Field(
         default=False,
         description="Case-sensitive column name comparison.",
@@ -569,7 +598,10 @@ class CatalogValidationRequest(BaseModel):
         ),
     )
 
-    @field_validator("schemas", "tables", "ignore_columns", mode="before")
+    @field_validator(
+        "schemas", "tables", "ignore_columns", "only_columns",
+        "ignore_datatype_columns", mode="before",
+    )
     @classmethod
     def _ensure_list(cls, value: Any) -> Any:
         if value is None:

{table_validator-0.1.5 → table_validator-0.1.7}/table_validator/validators/catalog_validator.py

@@ -528,15 +528,18 @@ class CatalogValidator:
         names within an already-resolved source_schema/target_schema
         pair).
         """
-        common, missing, extra = self.compare_tables(
-            request.source_catalog, request.target_catalog, source_schema
-        )
-
-        # NOTE: compare_tables() above assumes the same schema name on
-        # both sides. When source_schema != target_schema (a schema_map
-        # pair), that identical-name baseline is meaningless - re-derive
-        # it directly from each side's own schema.
-        if source_schema.lower() != target_schema.lower():
+        if source_schema.lower() == target_schema.lower():
+            common, missing, extra = self.compare_tables(
+                request.source_catalog, request.target_catalog, source_schema
+            )
+        else:
+            # compare_tables() assumes the same schema name on both
+            # sides - calling it here would query the TARGET catalog for
+            # a schema named after the SOURCE schema (a schema_map pair
+            # means these differ), which doesn't exist there and raises
+            # SCHEMA_NOT_FOUND instead of just being a wasted query.
+            # Derive the baseline directly from each side's own,
+            # correctly-named schema instead.
             try:
                 source_tables = set(self.databricks.get_tables(request.source_catalog, source_schema))
             except Exception:
@@ -983,6 +986,19 @@ class CatalogValidator:
             source_schema_df, target_schema_df, request.case_sensitive_columns, ignore
         )
 
+        # only_columns (allowlist): restrict the already-computed common
+        # set to just these names, same case-insensitive convention as
+        # ignore_columns above. A name here that isn't actually a common
+        # column is silently absent from the effective set (matching
+        # tables/schemas' existing restriction-field convention) - it is
+        # NOT reported as missing/extra, since it was never confirmed to
+        # exist on both sides in the first place.
+        if request.only_columns:
+            allowed = {c.lower() for c in request.only_columns}
+            common_cols = [c for c in common_cols if c.lower() in allowed]
+
+        ignore_datatype = {c.lower() for c in (request.ignore_datatype_columns or [])}
+
         if column_enabled:
             result.missing_columns = missing_cols
             result.extra_columns = extra_cols
@@ -1046,7 +1062,15 @@ class CatalogValidator:
                 tgt_type = str(tgt_row.get("data_type"))
                 col_result.source_data_type = src_type
                 col_result.target_data_type = tgt_type
-                col_result.data_type_status = self._classify_type_family(src_type, tgt_type)
+                if key in ignore_datatype:
+                    # User asked to ignore this column's datatype
+                    # entirely - report SKIPPED, not PASS/FAIL, so a real
+                    # type difference here is visibly disclosed as
+                    # "intentionally not checked" rather than looking
+                    # like the types happened to match.
+                    col_result.data_type_status = ValidationStatus.SKIPPED
+                else:
+                    col_result.data_type_status = self._classify_type_family(src_type, tgt_type)
                 dtype_statuses.append(col_result.data_type_status)
 
                 if request.validate_nullable:
@@ -1073,9 +1097,14 @@ class CatalogValidator:
             result.nullable_status = ValidationStatus.SKIPPED
 
         # Cross-family type change is BLOCKING even when COLUMN reporting
-        # is disabled (detection always runs; only reporting is gated).
+        # is disabled (detection always runs; only reporting is gated) -
+        # except for a column the user explicitly put in
+        # ignore_datatype_columns, which must never abort the table on a
+        # type difference it was told to disregard.
         for col in common_cols:
             key = col.lower()
+            if key in ignore_datatype:
+                continue
             src_type = str(src_by_col.get(key, {}).get("data_type"))
             tgt_type = str(tgt_by_col.get(key, {}).get("data_type"))
             if self._classify_type_family(src_type, tgt_type) == ValidationStatus.FAIL:

{table_validator-0.1.5 → table_validator-0.1.7/table_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.5
+Version: 0.1.7
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

table_validator-0.1.7/table_validator.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.1.7",
+  "distance": 0,
+  "node": "g989192ab5b13d03c174b2c33883a8569f5437e2a",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-02"
+}

{table_validator-0.1.5 → table_validator-0.1.7}/tests/test_catalog_validator.py

@@ -1574,6 +1574,48 @@ def test_schema_map_to_nonexistent_target_schema_produces_clear_error():
     assert result.status in (ValidationStatus.ERROR, ValidationStatus.FAIL)
 
 
+def test_schema_map_does_not_query_target_using_source_schema_name():
+    """Real bug: _resolve_table_pairs unconditionally called compare_tables
+    with the SOURCE schema name for both catalogs before checking whether
+    schema_map applied - querying the target catalog for a schema named
+    after the source schema, which doesn't exist there under a real
+    Databricks connector (raises SCHEMA_NOT_FOUND) rather than just being
+    a wasted no-op query. get_tables must only ever be called with each
+    catalog's own correctly-mapped schema name."""
+    connector = _make_connector()
+    connector.get_schemas.side_effect = lambda catalog: (
+        ["for_schema_validation"] if catalog == "cat_source" else ["bronze"]
+    )
+
+    def get_tables(catalog, schema):
+        if catalog == "cat_source" and schema == "for_schema_validation":
+            return ["bronze_fn_sku_add"]
+        if catalog == "cat_target" and schema == "bronze":
+            return ["bronze_fn_sku_add"]
+        raise RuntimeError(
+            f"Unable to list tables for '{catalog}.{schema}': SCHEMA_NOT_FOUND"
+        )
+
+    connector.get_tables.side_effect = get_tables
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(
+            schemas=["for_schema_validation"],
+            schema_map={"for_schema_validation": "bronze"},
+            tables=["bronze_fn_sku_add"],
+        )
+    )
+
+    assert result.status != ValidationStatus.ERROR
+    schema_result = result.schemas[0]
+    assert schema_result.schema_name == "bronze"
+    assert schema_result.status != ValidationStatus.ERROR
+    assert len(schema_result.tables) == 1
+    assert schema_result.tables[0].table == "bronze_fn_sku_add"
+    assert schema_result.tables[0].status in (ValidationStatus.PASS, ValidationStatus.FAIL)
+
+
 def test_no_map_configured_identical_name_discovery_unchanged():
     """Regression guard: with no schema_map/table_map set (the default,
     and today's only behavior), discovery must be completely unchanged -
@@ -1610,3 +1652,114 @@ def test_table_map_entry_matching_identical_name_is_not_duplicated():
 
     schema_result = result.schemas[0]
     assert len(schema_result.tables) == 1
+
+
+# ---------------------------------------------------------------------------
+# only_columns (allowlist) / ignore_datatype_columns
+# ---------------------------------------------------------------------------
+def test_only_columns_restricts_common_cols_to_allowlist():
+    """only_columns further restricts the already-computed common set -
+    a column not in the allowlist is excluded from every check (name/
+    type/nullable/statistics), even though it's genuinely common to
+    both sides."""
+    connector = _make_connector()
+    connector.get_table_schema.return_value = _schema_df(
+        [("id", "int", False), ("name", "string", True), ("email", "string", True)]
+    )
+    connector.get_column_statistics.return_value = {
+        "id": {"null_count": 0, "distinct_count": 100, "min": None, "max": None},
+        "name": {"null_count": 2, "distinct_count": 95, "min": None, "max": None},
+        "email": {"null_count": 0, "distinct_count": 100, "min": None, "max": None},
+    }
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(_request(only_columns=["id", "name"]))
+
+    table = result.schemas[0].tables[0]
+    assert [c.column for c in table.columns] == ["id", "name"]
+    assert "email" not in [c.column for c in table.columns]
+    # Not reported as missing/extra - it was never confirmed absent, just
+    # excluded by the allowlist.
+    assert table.missing_columns == []
+    assert table.extra_columns == []
+
+
+def test_only_columns_name_not_actually_common_is_silently_absent():
+    """A name in only_columns that isn't a real common column doesn't
+    error or appear anywhere - it's just absent from the effective set,
+    matching tables/schemas' existing restriction-field convention."""
+    connector = _make_connector()
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(only_columns=["id", "does_not_exist_column"])
+    )
+
+    table = result.schemas[0].tables[0]
+    assert [c.column for c in table.columns] == ["id"]
+    assert result.status != ValidationStatus.ERROR
+
+
+def test_ignore_columns_wins_over_only_columns_for_same_column():
+    """If a column appears in both only_columns and ignore_columns,
+    ignore_columns wins - it is always excluded."""
+    connector = _make_connector()
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(only_columns=["id", "name"], ignore_columns=["name"])
+    )
+
+    table = result.schemas[0].tables[0]
+    assert [c.column for c in table.columns] == ["id"]
+
+
+def test_ignore_datatype_columns_reports_skipped_not_fail():
+    """A column in ignore_datatype_columns with genuinely different types
+    across a type family (e.g. string vs int - normally BLOCKING) must
+    report SKIPPED, never FAIL, and must not abort the table."""
+    connector = _make_connector()
+    connector.get_table_schema.side_effect = lambda catalog, schema, table: (
+        _schema_df([("id", "int", False), ("legacy_flag", "string", True)])
+        if catalog == "cat_source"
+        else _schema_df([("id", "int", False), ("legacy_flag", "int", True)])
+    )
+    connector.get_column_statistics.return_value = {
+        "id": {"null_count": 0, "distinct_count": 100, "min": None, "max": None},
+        "legacy_flag": {"null_count": 0, "distinct_count": 2, "min": None, "max": None},
+    }
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(ignore_datatype_columns=["legacy_flag"])
+    )
+
+    table = result.schemas[0].tables[0]
+    # Never BLOCKING - Tier 1+ actually ran (not SCHEMA_BLOCKED).
+    assert table.tier_reached != ValidationTier.SCHEMA_BLOCKED
+    assert table.schema_blocking is not True
+    legacy_col = next(c for c in table.columns if c.column == "legacy_flag")
+    assert legacy_col.data_type_status == ValidationStatus.SKIPPED
+    # The column's OTHER checks still ran normally.
+    assert legacy_col.nullable_status in (ValidationStatus.PASS, ValidationStatus.FAIL)
+
+
+def test_ignore_datatype_columns_does_not_affect_unlisted_columns():
+    """A genuine cross-family type change on a column NOT in
+    ignore_datatype_columns still BLOCKS the table as before - the new
+    field only suppresses the columns explicitly named."""
+    connector = _make_connector()
+    connector.get_table_schema.side_effect = lambda catalog, schema, table: (
+        _schema_df([("id", "int", False), ("name", "string", True)])
+        if catalog == "cat_source"
+        else _schema_df([("id", "int", False), ("name", "int", True)])
+    )
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(ignore_datatype_columns=["some_other_column"])
+    )
+
+    table = result.schemas[0].tables[0]
+    assert table.schema_blocking is True
+    assert table.tier_reached == ValidationTier.SCHEMA_BLOCKED

{table_validator-0.1.5 → table_validator-0.1.7}/tests/test_cli.py

@@ -1107,6 +1107,51 @@ def test_validate_no_primary_key_configured_uses_row_number_fallback(tmp_path: P
     mock_connector.get_row_hashes.assert_not_called()
 
 
+def test_validate_databricks_wires_only_columns_ignore_columns_and_ignore_datatype(
+    tmp_path: Path,
+) -> None:
+    """config.only_columns/ignore_columns/ignore_datatype_columns must
+    reach CatalogValidationRequest unchanged - these were previously
+    silently dropped (the fields didn't exist on ValidatorConfig at all,
+    and _run_databricks_validation never passed them even once added)."""
+    config_path = _full_config(tmp_path)
+    config = load_config(config_path)
+    config.only_columns = ["id", "name"]
+    config.ignore_columns = ["updated_at"]
+    config.ignore_datatype_columns = ["legacy_flag"]
+    save_config(config, config_path)
+    output_path = tmp_path / "validation_report.xlsx"
+
+    mock_connector = _mock_databricks_connector()
+
+    captured_requests = []
+    from table_validator.validators.catalog_validator import CatalogValidator
+
+    real_compare_catalogs = CatalogValidator.compare_catalogs
+
+    def spy_compare_catalogs(self, request):
+        captured_requests.append(request)
+        return real_compare_catalogs(self, request)
+
+    with patch(
+        "table_validator.cli.main.DatabricksConnector", return_value=mock_connector
+    ), patch(
+        "table_validator.cli.main.get_databricks_token", return_value="dapi_fake"
+    ), patch(
+        "table_validator.cli.main.get_azure_credential", return_value=None
+    ), patch.object(CatalogValidator, "compare_catalogs", spy_compare_catalogs):
+        runner.invoke(
+            app,
+            ["validate", "--config-path", str(config_path), "--output", str(output_path)],
+        )
+
+    assert len(captured_requests) == 1
+    request = captured_requests[0]
+    assert request.only_columns == ["id", "name"]
+    assert request.ignore_columns == ["updated_at"]
+    assert request.ignore_datatype_columns == ["legacy_flag"]
+
+
 def test_validate_mode_stats_stops_before_fingerprint_and_row_hash(tmp_path: Path) -> None:
     """--mode stats is the CLI-flag equivalent of the tiered funnel's
     "statistical only" prompt: it must stop after Tier 1 even though the

table_validator-0.1.5/table_validator.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.1.5",
-  "distance": 0,
-  "node": "g343999ed2ff4c4d7ddf8d0ddcd3cc0ced0df29b2",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-08-28"
-}