table-validator 0.1.6__tar.gz → 0.1.8__tar.gz

{table_validator-0.1.6/table_validator.egg-info → table_validator-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.6
+Version: 0.1.8
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

{table_validator-0.1.6 → table_validator-0.1.8}/table_validator/cli/main.py

@@ -148,7 +148,7 @@ def info() -> None:
         "Run 'tablevalidator <command> --help' for a command's full "
         "options (e.g. tablevalidator validate --help).\n"
         "\n"
-        "TO KEERTHIVASAN"
+        "-MS-"
     )
 
 
@@ -470,6 +470,9 @@ def _run_databricks_validation(
         enabled_validations=set(config.validations),
         primary_keys=primary_keys,
         max_tier=max_tier,
+        only_columns=config.only_columns,
+        ignore_columns=config.ignore_columns,
+        ignore_datatype_columns=config.ignore_datatype_columns,
     )
 
     partition_prompt = build_partition_prompt(yes=yes)

{table_validator-0.1.6 → table_validator-0.1.8}/table_validator/cli/wizard.py

@@ -117,6 +117,52 @@ def _prompt_primary_key(existing: Optional[list]) -> Optional[list]:
     return [col.strip() for col in answer.split(",") if col.strip()]
 
 
+def _prompt_column_list(prompt_text: str, existing: Optional[list]) -> Optional[list]:
+    """Shared free-text parser for a comma-separated column list answer -
+    used by all three customization sub-options below. Returns None for
+    a blank answer (caller decides the actual default: None vs [])."""
+    default_str = ", ".join(existing) if existing else ""
+    answer = _ask(questionary.text(prompt_text, default=default_str))
+    if not answer:
+        return None
+    return [col.strip() for col in answer.split(",") if col.strip()]
+
+
+def _prompt_column_customization(config: ValidatorConfig) -> None:
+    """Optional column-level customization, asked right after the
+    primary key - only meaningful for the single named table
+    (primary_key's same scope). Skipped entirely (leaving any existing
+    only_columns/ignore_columns/ignore_datatype_columns untouched) unless
+    the user opts in, so a user who never touches this gets identical
+    behavior to before this feature existed."""
+    customize = questionary.confirm(
+        "Customize column validation? (skip specific columns, compare "
+        "only specific columns, or ignore datatype mismatches for "
+        "specific columns)",
+        default=False,
+    ).ask()
+
+    if not customize:
+        return
+
+    config.only_columns = _prompt_column_list(
+        "Compare ONLY these columns, comma-separated (leave blank to "
+        "compare every common column as usual):",
+        config.only_columns,
+    )
+    config.ignore_columns = _prompt_column_list(
+        "SKIP these columns entirely, comma-separated (leave blank to "
+        "skip none):",
+        config.ignore_columns,
+    ) or []
+    config.ignore_datatype_columns = _prompt_column_list(
+        "Ignore DATATYPE mismatches only for these columns, "
+        "comma-separated - their other checks (nullable, statistics, "
+        "row values) still run (leave blank to skip none):",
+        config.ignore_datatype_columns,
+    ) or []
+
+
 def _write_env_file(values: Dict[str, str], env_path: Optional[Path] = None) -> None:
     """
     Write secrets to env_path (default: ENV_PATH, resolved at call time)
@@ -394,8 +440,18 @@ def run_configure_wizard() -> None:
 
     if source_table_named and config.target_table.table:
         config.primary_key = _prompt_primary_key(config.primary_key)
+        # Column-level customization - same single-table scope as the
+        # primary key above (only_columns/ignore_columns/
+        # ignore_datatype_columns are only meaningful when there's one
+        # specific table to compare). Optional: a user who declines gets
+        # identical behavior to before this feature existed.
+        typer.echo("\n== Customize validation (optional) ==")
+        _prompt_column_customization(config)
     else:
         config.primary_key = None
+        config.only_columns = None
+        config.ignore_columns = []
+        config.ignore_datatype_columns = []
 
     # ------------------------------------------------------------------
     # 5. Validations to run

{table_validator-0.1.6 → table_validator-0.1.8}/table_validator/config/schema.py

@@ -166,6 +166,42 @@ class ValidatorConfig(BaseModel):
         ),
     )
 
+    only_columns: Optional[List[str]] = Field(
+        default=None,
+        description=(
+            "Optional allowlist: if set, only these columns (plus the "
+            "primary key, if any) are compared - every other common "
+            "column is excluded from every check. Only meaningful for the "
+            "single named table in source_table/target_table, same scope "
+            "as primary_key. If a column named here isn't actually common "
+            "to both sides, it's silently absent from the effective set "
+            "(same convention as tables/schemas restrictions elsewhere). "
+            "If both only_columns and ignore_columns name the same "
+            "column, ignore_columns wins - it is always excluded."
+        ),
+    )
+
+    ignore_columns: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Columns to exclude entirely from every check (name, type, "
+            "nullable, statistics, row-hash) - useful for auto-generated "
+            "columns like timestamps that are expected to always differ."
+        ),
+    )
+
+    ignore_datatype_columns: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Columns whose data-type mismatch should be ignored - the "
+            "column's other checks (nullable, statistics, row-hash) still "
+            "run normally, but a real type difference here is reported as "
+            "SKIPPED rather than FAIL and never fails the table or aborts "
+            "the schema stage, even for a cross-family type change that "
+            "would otherwise be BLOCKING."
+        ),
+    )
+
     blob_source: BlobSourceConfig = Field(default_factory=BlobSourceConfig)
     sql_source: SqlSourceConfig = Field(default_factory=SqlSourceConfig)
 

{table_validator-0.1.6 → table_validator-0.1.8}/table_validator/models.py

@@ -488,6 +488,35 @@ class CatalogValidationRequest(BaseModel):
 
     ignore_columns: List[str] = Field(default_factory=list)
 
+    only_columns: Optional[List[str]] = Field(
+        default=None,
+        description=(
+            "Optional allowlist: if set, common_cols is further restricted "
+            "to just these column names (case-insensitive) before any "
+            "check runs - every other common column is excluded from name/"
+            "type/nullable/statistics/row-hash checks entirely, the same "
+            "as ignore_columns' exclusion. A name here that isn't actually "
+            "a common column is silently absent from the effective set, "
+            "matching this class's other restriction fields (schemas/ "
+            "tables). If a column appears in both only_columns and "
+            "ignore_columns, ignore_columns wins - it is always excluded, "
+            "applied after the allowlist restriction."
+        ),
+    )
+
+    ignore_datatype_columns: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Columns (case-insensitive) whose data_type_status should be "
+            "reported as SKIPPED rather than PASS/FAIL, and excluded from "
+            "Tier 0's BLOCKING cross-family-type-change classification - "
+            "a real type difference on one of these columns never fails "
+            "the table or aborts the schema stage. The column's other "
+            "checks (nullable, null/distinct/min-max statistics, row-"
+            "hash) still run normally."
+        ),
+    )
+
     case_sensitive_columns: bool = Field(
         default=False,
         description="Case-sensitive column name comparison.",
@@ -569,7 +598,10 @@ class CatalogValidationRequest(BaseModel):
         ),
     )
 
-    @field_validator("schemas", "tables", "ignore_columns", mode="before")
+    @field_validator(
+        "schemas", "tables", "ignore_columns", "only_columns",
+        "ignore_datatype_columns", mode="before",
+    )
     @classmethod
     def _ensure_list(cls, value: Any) -> Any:
         if value is None:

{table_validator-0.1.6 → table_validator-0.1.8}/table_validator/validators/catalog_validator.py

@@ -986,6 +986,19 @@ class CatalogValidator:
             source_schema_df, target_schema_df, request.case_sensitive_columns, ignore
         )
 
+        # only_columns (allowlist): restrict the already-computed common
+        # set to just these names, same case-insensitive convention as
+        # ignore_columns above. A name here that isn't actually a common
+        # column is silently absent from the effective set (matching
+        # tables/schemas' existing restriction-field convention) - it is
+        # NOT reported as missing/extra, since it was never confirmed to
+        # exist on both sides in the first place.
+        if request.only_columns:
+            allowed = {c.lower() for c in request.only_columns}
+            common_cols = [c for c in common_cols if c.lower() in allowed]
+
+        ignore_datatype = {c.lower() for c in (request.ignore_datatype_columns or [])}
+
         if column_enabled:
             result.missing_columns = missing_cols
             result.extra_columns = extra_cols
@@ -1049,7 +1062,15 @@ class CatalogValidator:
                 tgt_type = str(tgt_row.get("data_type"))
                 col_result.source_data_type = src_type
                 col_result.target_data_type = tgt_type
-                col_result.data_type_status = self._classify_type_family(src_type, tgt_type)
+                if key in ignore_datatype:
+                    # User asked to ignore this column's datatype
+                    # entirely - report SKIPPED, not PASS/FAIL, so a real
+                    # type difference here is visibly disclosed as
+                    # "intentionally not checked" rather than looking
+                    # like the types happened to match.
+                    col_result.data_type_status = ValidationStatus.SKIPPED
+                else:
+                    col_result.data_type_status = self._classify_type_family(src_type, tgt_type)
                 dtype_statuses.append(col_result.data_type_status)
 
                 if request.validate_nullable:
@@ -1076,9 +1097,14 @@ class CatalogValidator:
             result.nullable_status = ValidationStatus.SKIPPED
 
         # Cross-family type change is BLOCKING even when COLUMN reporting
-        # is disabled (detection always runs; only reporting is gated).
+        # is disabled (detection always runs; only reporting is gated) -
+        # except for a column the user explicitly put in
+        # ignore_datatype_columns, which must never abort the table on a
+        # type difference it was told to disregard.
         for col in common_cols:
             key = col.lower()
+            if key in ignore_datatype:
+                continue
             src_type = str(src_by_col.get(key, {}).get("data_type"))
             tgt_type = str(tgt_by_col.get(key, {}).get("data_type"))
             if self._classify_type_family(src_type, tgt_type) == ValidationStatus.FAIL:

{table_validator-0.1.6 → table_validator-0.1.8/table_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.6
+Version: 0.1.8
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

table_validator-0.1.8/table_validator.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.1.8",
+  "distance": 0,
+  "node": "g5004811d73585508b803dc003e03bd1da62ede4a",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-02"
+}

{table_validator-0.1.6 → table_validator-0.1.8}/tests/test_catalog_validator.py

@@ -1652,3 +1652,114 @@ def test_table_map_entry_matching_identical_name_is_not_duplicated():
 
     schema_result = result.schemas[0]
     assert len(schema_result.tables) == 1
+
+
+# ---------------------------------------------------------------------------
+# only_columns (allowlist) / ignore_datatype_columns
+# ---------------------------------------------------------------------------
+def test_only_columns_restricts_common_cols_to_allowlist():
+    """only_columns further restricts the already-computed common set -
+    a column not in the allowlist is excluded from every check (name/
+    type/nullable/statistics), even though it's genuinely common to
+    both sides."""
+    connector = _make_connector()
+    connector.get_table_schema.return_value = _schema_df(
+        [("id", "int", False), ("name", "string", True), ("email", "string", True)]
+    )
+    connector.get_column_statistics.return_value = {
+        "id": {"null_count": 0, "distinct_count": 100, "min": None, "max": None},
+        "name": {"null_count": 2, "distinct_count": 95, "min": None, "max": None},
+        "email": {"null_count": 0, "distinct_count": 100, "min": None, "max": None},
+    }
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(_request(only_columns=["id", "name"]))
+
+    table = result.schemas[0].tables[0]
+    assert [c.column for c in table.columns] == ["id", "name"]
+    assert "email" not in [c.column for c in table.columns]
+    # Not reported as missing/extra - it was never confirmed absent, just
+    # excluded by the allowlist.
+    assert table.missing_columns == []
+    assert table.extra_columns == []
+
+
+def test_only_columns_name_not_actually_common_is_silently_absent():
+    """A name in only_columns that isn't a real common column doesn't
+    error or appear anywhere - it's just absent from the effective set,
+    matching tables/schemas' existing restriction-field convention."""
+    connector = _make_connector()
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(only_columns=["id", "does_not_exist_column"])
+    )
+
+    table = result.schemas[0].tables[0]
+    assert [c.column for c in table.columns] == ["id"]
+    assert result.status != ValidationStatus.ERROR
+
+
+def test_ignore_columns_wins_over_only_columns_for_same_column():
+    """If a column appears in both only_columns and ignore_columns,
+    ignore_columns wins - it is always excluded."""
+    connector = _make_connector()
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(only_columns=["id", "name"], ignore_columns=["name"])
+    )
+
+    table = result.schemas[0].tables[0]
+    assert [c.column for c in table.columns] == ["id"]
+
+
+def test_ignore_datatype_columns_reports_skipped_not_fail():
+    """A column in ignore_datatype_columns with genuinely different types
+    across a type family (e.g. string vs int - normally BLOCKING) must
+    report SKIPPED, never FAIL, and must not abort the table."""
+    connector = _make_connector()
+    connector.get_table_schema.side_effect = lambda catalog, schema, table: (
+        _schema_df([("id", "int", False), ("legacy_flag", "string", True)])
+        if catalog == "cat_source"
+        else _schema_df([("id", "int", False), ("legacy_flag", "int", True)])
+    )
+    connector.get_column_statistics.return_value = {
+        "id": {"null_count": 0, "distinct_count": 100, "min": None, "max": None},
+        "legacy_flag": {"null_count": 0, "distinct_count": 2, "min": None, "max": None},
+    }
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(ignore_datatype_columns=["legacy_flag"])
+    )
+
+    table = result.schemas[0].tables[0]
+    # Never BLOCKING - Tier 1+ actually ran (not SCHEMA_BLOCKED).
+    assert table.tier_reached != ValidationTier.SCHEMA_BLOCKED
+    assert table.schema_blocking is not True
+    legacy_col = next(c for c in table.columns if c.column == "legacy_flag")
+    assert legacy_col.data_type_status == ValidationStatus.SKIPPED
+    # The column's OTHER checks still ran normally.
+    assert legacy_col.nullable_status in (ValidationStatus.PASS, ValidationStatus.FAIL)
+
+
+def test_ignore_datatype_columns_does_not_affect_unlisted_columns():
+    """A genuine cross-family type change on a column NOT in
+    ignore_datatype_columns still BLOCKS the table as before - the new
+    field only suppresses the columns explicitly named."""
+    connector = _make_connector()
+    connector.get_table_schema.side_effect = lambda catalog, schema, table: (
+        _schema_df([("id", "int", False), ("name", "string", True)])
+        if catalog == "cat_source"
+        else _schema_df([("id", "int", False), ("name", "int", True)])
+    )
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(ignore_datatype_columns=["some_other_column"])
+    )
+
+    table = result.schemas[0].tables[0]
+    assert table.schema_blocking is True
+    assert table.tier_reached == ValidationTier.SCHEMA_BLOCKED

{table_validator-0.1.6 → table_validator-0.1.8}/tests/test_cli.py

@@ -1107,6 +1107,51 @@ def test_validate_no_primary_key_configured_uses_row_number_fallback(tmp_path: P
     mock_connector.get_row_hashes.assert_not_called()
 
 
+def test_validate_databricks_wires_only_columns_ignore_columns_and_ignore_datatype(
+    tmp_path: Path,
+) -> None:
+    """config.only_columns/ignore_columns/ignore_datatype_columns must
+    reach CatalogValidationRequest unchanged - these were previously
+    silently dropped (the fields didn't exist on ValidatorConfig at all,
+    and _run_databricks_validation never passed them even once added)."""
+    config_path = _full_config(tmp_path)
+    config = load_config(config_path)
+    config.only_columns = ["id", "name"]
+    config.ignore_columns = ["updated_at"]
+    config.ignore_datatype_columns = ["legacy_flag"]
+    save_config(config, config_path)
+    output_path = tmp_path / "validation_report.xlsx"
+
+    mock_connector = _mock_databricks_connector()
+
+    captured_requests = []
+    from table_validator.validators.catalog_validator import CatalogValidator
+
+    real_compare_catalogs = CatalogValidator.compare_catalogs
+
+    def spy_compare_catalogs(self, request):
+        captured_requests.append(request)
+        return real_compare_catalogs(self, request)
+
+    with patch(
+        "table_validator.cli.main.DatabricksConnector", return_value=mock_connector
+    ), patch(
+        "table_validator.cli.main.get_databricks_token", return_value="dapi_fake"
+    ), patch(
+        "table_validator.cli.main.get_azure_credential", return_value=None
+    ), patch.object(CatalogValidator, "compare_catalogs", spy_compare_catalogs):
+        runner.invoke(
+            app,
+            ["validate", "--config-path", str(config_path), "--output", str(output_path)],
+        )
+
+    assert len(captured_requests) == 1
+    request = captured_requests[0]
+    assert request.only_columns == ["id", "name"]
+    assert request.ignore_columns == ["updated_at"]
+    assert request.ignore_datatype_columns == ["legacy_flag"]
+
+
 def test_validate_mode_stats_stops_before_fingerprint_and_row_hash(tmp_path: Path) -> None:
     """--mode stats is the CLI-flag equivalent of the tiered funnel's
     "statistical only" prompt: it must stop after Tier 1 even though the

{table_validator-0.1.6 → table_validator-0.1.8}/tests/test_wizard.py

@@ -335,6 +335,7 @@ def test_databricks_source_type_prompts_for_source_table(tmp_path, monkeypatch):
         "src_cat", "bronze", "customers",   # source table
         "tgt_cat", "silver", "customers",   # target table
         "",                                   # primary key (left blank)
+        False,                                # customize validation? (declined)
         [],                                  # validations
     ])
 
@@ -359,12 +360,100 @@ def test_databricks_source_type_prompts_for_primary_key_when_both_tables_named(
         "src_cat", "bronze", "customers",
         "tgt_cat", "silver", "customers",
         "id, region_id ",                    # primary key, comma-separated with stray spaces
+        False,                                # customize validation? (declined)
         [],
     ])
 
     assert config.primary_key == ["id", "region_id"]
 
 
+def test_databricks_wizard_declining_customization_leaves_defaults_untouched(
+    tmp_path, monkeypatch
+):
+    """Answering No to the customize-validation gate must leave
+    only_columns/ignore_columns/ignore_datatype_columns at their defaults
+    - a user who never opts in gets identical behavior to before this
+    feature existed."""
+    config = _run_wizard_with_answers(tmp_path, monkeypatch, [
+        "Databricks catalog -> Databricks catalog",
+        "https://adb-1.databricks.net", "/sql/1.0/warehouses/x", "tok",
+        "src_cat", "bronze", "customers",
+        "tgt_cat", "silver", "customers",
+        "",                                    # primary key (left blank)
+        False,                                 # customize validation? (declined)
+        [],
+    ])
+
+    assert config.only_columns is None
+    assert config.ignore_columns == []
+    assert config.ignore_datatype_columns == []
+
+
+def test_databricks_wizard_customize_populates_all_three_column_lists(
+    tmp_path, monkeypatch
+):
+    """Answering Yes to the customize-validation gate must ask for and
+    save all three column lists - only_columns (allowlist),
+    ignore_columns (denylist), ignore_datatype_columns."""
+    config = _run_wizard_with_answers(tmp_path, monkeypatch, [
+        "Databricks catalog -> Databricks catalog",
+        "https://adb-1.databricks.net", "/sql/1.0/warehouses/x", "tok",
+        "src_cat", "bronze", "customers",
+        "tgt_cat", "silver", "customers",
+        "id",                                  # primary key
+        True,                                  # customize validation? (accepted)
+        "id, name ",                            # only_columns, stray spaces
+        "updated_at",                           # ignore_columns
+        "legacy_flag, notes",                   # ignore_datatype_columns
+        [],
+    ])
+
+    assert config.only_columns == ["id", "name"]
+    assert config.ignore_columns == ["updated_at"]
+    assert config.ignore_datatype_columns == ["legacy_flag", "notes"]
+
+
+def test_databricks_wizard_customize_all_three_left_blank(tmp_path, monkeypatch):
+    """Opting into customization but leaving all three prompts blank must
+    resolve to only_columns=None, ignore_columns=[], ignore_datatype_columns=[] -
+    the same effective defaults as declining outright."""
+    config = _run_wizard_with_answers(tmp_path, monkeypatch, [
+        "Databricks catalog -> Databricks catalog",
+        "https://adb-1.databricks.net", "/sql/1.0/warehouses/x", "tok",
+        "src_cat", "bronze", "customers",
+        "tgt_cat", "silver", "customers",
+        "id",
+        True,                                  # customize validation? (accepted)
+        "",                                    # only_columns left blank
+        "",                                    # ignore_columns left blank
+        "",                                    # ignore_datatype_columns left blank
+        [],
+    ])
+
+    assert config.only_columns is None
+    assert config.ignore_columns == []
+    assert config.ignore_datatype_columns == []
+
+
+def test_databricks_wizard_catalog_wide_sweep_skips_customization_prompt(
+    tmp_path, monkeypatch
+):
+    """Catalog-wide sweep (no specific table named) must skip the
+    customize-validation gate entirely, same scope rule as primary_key -
+    a single column list can't apply to many different tables."""
+    config = _run_wizard_with_answers(tmp_path, monkeypatch, [
+        "Databricks catalog -> Databricks catalog",
+        "https://adb-1.databricks.net", "/sql/1.0/warehouses/x", "tok",
+        "src_cat", "", "",                    # source: catalog only
+        "tgt_cat", "", "",                    # target: catalog only
+        [],                                    # validations (no customize prompt in between)
+    ])
+
+    assert config.only_columns is None
+    assert config.ignore_columns == []
+    assert config.ignore_datatype_columns == []
+
+
 def test_databricks_wizard_skips_primary_key_prompt_for_catalog_wide_sweep(
     tmp_path, monkeypatch
 ):
@@ -457,6 +546,7 @@ def test_azure_sql_prompts_for_primary_key_when_table_named(tmp_path, monkeypatc
         "employees",                          # sql_source table
         "tgt_cat", "dbo", "employees_sample",  # target table
         "EmployeeID",                         # primary key
+        False,                                # customize validation? (declined)
         [],                                   # validations
     ])
 
@@ -498,6 +588,7 @@ def test_azure_blob_prompts_for_primary_key_when_blob_path_named(tmp_path, monke
         "validation/customers.csv",            # blob_path (explicit)
         "tgt_cat", "bronze", "customers",      # target table
         "id",                                  # primary key
+        False,                                 # customize validation? (declined)
         [],                                    # validations
     ])
 

table_validator-0.1.6/table_validator.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.1.6",
-  "distance": 0,
-  "node": "g9357213ed68fd98fc2c232aed2bb737f273cabbf",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-08-28"
-}