table-validator 0.1.7__tar.gz → 0.1.9__tar.gz

{table_validator-0.1.7/table_validator.egg-info → table_validator-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.7
+Version: 0.1.9
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9

{table_validator-0.1.7 → table_validator-0.1.9}/table_validator/auth/databricks_auth.py

@@ -29,3 +29,18 @@ def get_databricks_token(config: ValidatorConfig, env_path: Path = ENV_PATH) ->
     """
     values = dotenv_values(env_path) if env_path.exists() else {}
     return values.get("DATABRICKS_TOKEN") or None
+
+
+def host_from_workspace_url(workspace_url: Optional[str]) -> Optional[str]:
+    """DatabricksConnector wants a bare hostname; the wizard stores a full
+    https:// workspace URL, so strip the scheme and any trailing path.
+
+    Shared by cli/main.py (building the connector for `validate`) and
+    cli/wizard.py (building a connector during `configure` for the
+    column-mapping live picker) - kept here rather than in either CLI
+    module so neither has to import from the other.
+    """
+    if not workspace_url:
+        return None
+    host = workspace_url.replace("https://", "").replace("http://", "")
+    return host.split("/")[0]

{table_validator-0.1.7 → table_validator-0.1.9}/table_validator/cli/main.py

@@ -10,7 +10,7 @@ from typing import Dict, Optional
 import typer
 
 from table_validator.auth.azure_auth import get_azure_credential
-from table_validator.auth.databricks_auth import get_databricks_token
+from table_validator.auth.databricks_auth import get_databricks_token, host_from_workspace_url
 from table_validator.cli.summary_table import (
     print_summary_table,
     summary_from_excel,
@@ -256,7 +256,7 @@ def validate(
 
     try:
         databricks = DatabricksConnector(
-            host=_host_from_workspace_url(config.databricks.workspace_url),
+            host=host_from_workspace_url(config.databricks.workspace_url),
             token=token,
             http_path=config.databricks.http_path,
         )
@@ -473,6 +473,7 @@ def _run_databricks_validation(
         only_columns=config.only_columns,
         ignore_columns=config.ignore_columns,
         ignore_datatype_columns=config.ignore_datatype_columns,
+        column_map=config.column_map,
     )
 
     partition_prompt = build_partition_prompt(yes=yes)
@@ -638,15 +639,6 @@ def _missing_config_fields(config: ValidatorConfig) -> list:
     return missing
 
 
-def _host_from_workspace_url(workspace_url: Optional[str]) -> Optional[str]:
-    """DatabricksConnector wants a bare hostname; the wizard stores a full
-    https:// workspace URL, so strip the scheme and any trailing path."""
-    if not workspace_url:
-        return None
-    host = workspace_url.replace("https://", "").replace("http://", "")
-    return host.split("/")[0]
-
-
 def _open_in_default_app(path: Path) -> None:
     """
     Launch `path` in whatever application the OS has associated with its

{table_validator-0.1.7 → table_validator-0.1.9}/table_validator/cli/wizard.py

@@ -117,6 +117,133 @@ def _prompt_primary_key(existing: Optional[list]) -> Optional[list]:
     return [col.strip() for col in answer.split(",") if col.strip()]
 
 
+def _prompt_column_list(prompt_text: str, existing: Optional[list]) -> Optional[list]:
+    """Shared free-text parser for a comma-separated column list answer -
+    used by all three customization sub-options below. Returns None for
+    a blank answer (caller decides the actual default: None vs [])."""
+    default_str = ", ".join(existing) if existing else ""
+    answer = _ask(questionary.text(prompt_text, default=default_str))
+    if not answer:
+        return None
+    return [col.strip() for col in answer.split(",") if col.strip()]
+
+
+def _prompt_column_customization(config: ValidatorConfig) -> None:
+    """Optional column-level customization, asked right after the
+    primary key - only meaningful for the single named table
+    (primary_key's same scope). Skipped entirely (leaving any existing
+    only_columns/ignore_columns/ignore_datatype_columns untouched) unless
+    the user opts in, so a user who never touches this gets identical
+    behavior to before this feature existed."""
+    customize = questionary.confirm(
+        "Customize column validation? (skip specific columns, compare "
+        "only specific columns, or ignore datatype mismatches for "
+        "specific columns)",
+        default=False,
+    ).ask()
+
+    if not customize:
+        return
+
+    config.only_columns = _prompt_column_list(
+        "Compare ONLY these columns, comma-separated (leave blank to "
+        "compare every common column as usual):",
+        config.only_columns,
+    )
+    config.ignore_columns = _prompt_column_list(
+        "SKIP these columns entirely, comma-separated (leave blank to "
+        "skip none):",
+        config.ignore_columns,
+    ) or []
+    config.ignore_datatype_columns = _prompt_column_list(
+        "Ignore DATATYPE mismatches only for these columns, "
+        "comma-separated - their other checks (nullable, statistics, "
+        "row values) still run (leave blank to skip none):",
+        config.ignore_datatype_columns,
+    ) or []
+
+
+def _prompt_column_mapping(config: ValidatorConfig, secrets: Dict[str, str]) -> None:
+    """
+    Optional column-name mapping for the single named table (same scope
+    as primary_key/column customization) - lets the user pair up
+    individual columns that were renamed between source and target (e.g.
+    source has 'cust_id', target has 'customer_id').
+
+    Connects to Databricks LIVE (the first time this wizard ever does so
+    during `configure`, rather than only at `validate` time) to fetch
+    both tables' real column lists, so the picker can be built from
+    actual columns rather than blind free-text entry. Any failure along
+    the way (missing/bad credentials, network issue, wrong table name,
+    insufficient permissions) is caught broadly and degrades to silently
+    skipping this step - `configure` must never crash just because this
+    optional, nice-to-have step couldn't reach Databricks. secrets may
+    not yet contain a freshly-typed token if the user is configuring for
+    the first time in this same run, so DATABRICKS_TOKEN is checked
+    there first, falling back to whatever's already on disk.
+    """
+    from table_validator.auth.databricks_auth import (
+        ENV_PATH,
+        get_databricks_token,
+        host_from_workspace_url,
+    )
+    from table_validator.connectors.databricks_connector import DatabricksConnector
+
+    try:
+        token = secrets.get("DATABRICKS_TOKEN") or get_databricks_token(config, ENV_PATH)
+        if not token:
+            return
+        databricks = DatabricksConnector(
+            host=host_from_workspace_url(config.databricks.workspace_url),
+            token=token,
+            http_path=config.databricks.http_path,
+        )
+        source_columns_df = databricks.get_table_schema(
+            config.source_table.catalog, config.source_table.schema_name, config.source_table.table
+        )
+        target_columns_df = databricks.get_table_schema(
+            config.target_table.catalog, config.target_table.schema_name, config.target_table.table
+        )
+    except Exception:
+        # Any connection/auth/query failure here just means the live
+        # picker isn't available this run - column_map can still be set
+        # later by re-running configure, or left unset entirely.
+        return
+
+    source_names = [str(c) for c in source_columns_df["column_name"]]
+    target_names = [str(c) for c in target_columns_df["column_name"]]
+    target_lower = {t.lower() for t in target_names}
+
+    unmatched_source = [s for s in source_names if s.lower() not in target_lower]
+    if not unmatched_source:
+        # Every source column already has an identical-name match -
+        # nothing to map, so don't ask anything at all.
+        return
+
+    source_lower = {s.lower() for s in source_names}
+    remaining_target = [t for t in target_names if t.lower() not in source_lower]
+
+    typer.echo(
+        f"\n{len(unmatched_source)} source column(s) have no identical-name "
+        "match in the target table - map any that were renamed (optional):"
+    )
+
+    new_map: Dict[str, str] = dict(config.column_map or {})
+    skip_label = "(skip this column)"
+    for src_col in unmatched_source:
+        if not remaining_target:
+            break
+        choice = questionary.select(
+            f"  Source column '{src_col}' has no matching target column - map it to:",
+            choices=remaining_target + [skip_label],
+        ).ask()
+        if choice and choice != skip_label:
+            new_map[src_col] = choice
+            remaining_target = [t for t in remaining_target if t != choice]
+
+    config.column_map = new_map
+
+
 def _write_env_file(values: Dict[str, str], env_path: Optional[Path] = None) -> None:
     """
     Write secrets to env_path (default: ENV_PATH, resolved at call time)
@@ -394,8 +521,24 @@ def run_configure_wizard() -> None:
 
     if source_table_named and config.target_table.table:
         config.primary_key = _prompt_primary_key(config.primary_key)
+        # Column-level customization - same single-table scope as the
+        # primary key above (only_columns/ignore_columns/
+        # ignore_datatype_columns are only meaningful when there's one
+        # specific table to compare). Optional: a user who declines gets
+        # identical behavior to before this feature existed.
+        typer.echo("\n== Customize validation (optional) ==")
+        _prompt_column_customization(config)
+        # Column-name mapping (renamed columns) - Databricks-to-Databricks
+        # only, since column_map/CatalogValidator don't apply to the
+        # Azure Blob/SQL source paths.
+        if config.source_type == SourceType.DATABRICKS:
+            _prompt_column_mapping(config, secrets)
     else:
         config.primary_key = None
+        config.only_columns = None
+        config.ignore_columns = []
+        config.ignore_datatype_columns = []
+        config.column_map = {}
 
     # ------------------------------------------------------------------
     # 5. Validations to run

{table_validator-0.1.7 → table_validator-0.1.9}/table_validator/config/schema.py

@@ -7,7 +7,7 @@ handled separately in Phase 4 and never appear on these models.
 from __future__ import annotations
 
 from enum import Enum
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 from pydantic import BaseModel, Field
 
@@ -202,6 +202,22 @@ class ValidatorConfig(BaseModel):
         ),
     )
 
+    column_map: Dict[str, str] = Field(
+        default_factory=dict,
+        description=(
+            "Optional map of source-table column name -> target-table "
+            "column name, for an individual column renamed between "
+            "source and target (e.g. source has 'cust_id', target has "
+            "'customer_id'). Only meaningful for the single named table "
+            "in source_table/target_table, same scope as primary_key. "
+            "Set interactively by `configure`'s column-mapping picker, "
+            "or by hand. A column configured as both a primary key and "
+            "an entry here is rejected with a clear error at validate "
+            "time - a column used as the row-level join key must have "
+            "the identical name on both sides."
+        ),
+    )
+
     blob_source: BlobSourceConfig = Field(default_factory=BlobSourceConfig)
     sql_source: SqlSourceConfig = Field(default_factory=SqlSourceConfig)
 

{table_validator-0.1.7 → table_validator-0.1.9}/table_validator/connectors/databricks_connector.py

@@ -878,6 +878,7 @@ class DatabricksConnector:
         key_values: Sequence[str],
         value_columns: Sequence[str],
         limit_samples: int = 500,
+        target_value_columns: Optional[Sequence[str]] = None,
     ) -> List[Dict[str, Any]]:
         """
         Tier 5: column-level diff for a bounded, already-known set of
@@ -887,6 +888,13 @@ class DatabricksConnector:
         column-by-column so callers can report exactly which column(s)
         differ per row. `key_values` are treated as opaque string literals
         (matching Tier 4's compare_row_hashes display-key convention).
+
+        `value_columns` is the SOURCE-side spelling; `target_value_columns`
+        (when a column_map applies), the positionally-aligned TARGET-side
+        spelling for the same columns. `key_column` itself is assumed
+        identical on both sides (a column used as a primary key must not
+        also be renamed via column_map - enforced at request-resolution
+        time, not here).
         """
         if not key_values:
             return []
@@ -909,6 +917,7 @@ class DatabricksConnector:
             value_columns=value_columns,
             changed_query=changed_query,
             limit_samples=limit_samples,
+            target_value_columns=target_value_columns,
         )
 
     def _changed_row_detail(
@@ -921,26 +930,42 @@ class DatabricksConnector:
         value_columns: Sequence[str],
         changed_query: str,
         limit_samples: int,
+        target_value_columns: Optional[Sequence[str]] = None,
     ) -> List[Dict[str, Any]]:
         """
         For a bounded sample of changed keys (from `changed_query`), fetch
         the full source and target rows (key + value columns) plus a
         whole-row hash for each side, so callers can report exactly which
         column(s) differ per row without ever collecting a full table.
+
+        `value_columns` is always the SOURCE-side column spelling.
+        `target_value_columns`, when given, is the positionally-aligned
+        TARGET-side spelling for the same columns (a column_map case) -
+        the source and target SQL each use their own side's names, and
+        the result is reconciled back to ONE canonical label per pair
+        (the target name, or the shared name when unmapped) so callers
+        never have to know which side a given result dict's keys came
+        from. When omitted, target_value_columns defaults to
+        value_columns (today's behavior, unchanged).
         """
-        value_idents = [self._quote_ident(c) for c in value_columns]
-        all_idents = key_idents + value_idents
-        select_list = ", ".join(all_idents)
-        value_concat = ", ".join(value_idents)
+        target_value_columns = list(target_value_columns or value_columns)
+        source_to_target = dict(zip(value_columns, target_value_columns))
+
+        source_value_idents = [self._quote_ident(c) for c in value_columns]
+        target_value_idents = [self._quote_ident(c) for c in target_value_columns]
+        source_select_list = ", ".join(key_idents + source_value_idents)
+        target_select_list = ", ".join(key_idents + target_value_idents)
+        source_concat = ", ".join(source_value_idents)
+        target_concat = ", ".join(target_value_idents)
 
         source_rows = self._execute_to_dataframe(f"""
-            SELECT {select_list}, hash({value_concat}) AS __row_hash
+            SELECT {source_select_list}, hash({source_concat}) AS __row_hash
             FROM {src}
             WHERE ({key_list}) IN (SELECT {key_list} FROM ({changed_query} LIMIT {int(limit_samples)}) __k)
         """).to_dict(orient="records")
 
         target_rows = self._execute_to_dataframe(f"""
-            SELECT {select_list}, hash({value_concat}) AS __row_hash
+            SELECT {target_select_list}, hash({target_concat}) AS __row_hash
             FROM {tgt}
             WHERE ({key_list}) IN (SELECT {key_list} FROM ({changed_query} LIMIT {int(limit_samples)}) __k)
         """).to_dict(orient="records")
@@ -956,9 +981,12 @@ class DatabricksConnector:
             if tgt_row is None:
                 continue
 
+            # Diff by pair, but report every result keyed by the
+            # canonical (target) column name - src_row/tgt_row are read
+            # using each side's own real column name.
             mismatched_columns = [
-                col for col in value_columns
-                if values_differ(src_row.get(col), tgt_row.get(col))
+                source_to_target[src_col] for src_col in value_columns
+                if values_differ(src_row.get(src_col), tgt_row.get(source_to_target[src_col]))
             ]
             if not mismatched_columns:
                 # SQL-side hash() flagged this row as changed, but our
@@ -967,14 +995,21 @@ class DatabricksConnector:
                 # that our value comparison normalizes away). Report the
                 # row anyway rather than silently dropping a row the
                 # mismatch count already accounts for.
-                mismatched_columns = list(value_columns)
+                mismatched_columns = list(target_value_columns)
 
             detail.append(
                 {
                     "key": {k: src_row.get(k) for k in key_columns},
                     "mismatched_columns": mismatched_columns,
-                    "source_values": {c: src_row.get(c) for c in mismatched_columns},
-                    "target_values": {c: tgt_row.get(c) for c in mismatched_columns},
+                    "source_values": {
+                        source_to_target[src_col]: src_row.get(src_col)
+                        for src_col in value_columns
+                        if source_to_target[src_col] in mismatched_columns
+                    },
+                    "target_values": {
+                        tgt_col: tgt_row.get(tgt_col)
+                        for tgt_col in mismatched_columns
+                    },
                     "source_row_hash": src_row.get("__row_hash"),
                     "target_row_hash": tgt_row.get("__row_hash"),
                 }
@@ -1131,6 +1166,8 @@ class DatabricksConnector:
         value_columns: Sequence[str],
         limit_samples: int = 500,
         bucket_predicate: Optional[Tuple[str, Any]] = None,
+        target_order_by_columns: Optional[Sequence[str]] = None,
+        target_value_columns: Optional[Sequence[str]] = None,
     ) -> List[Dict[str, Any]]:
         """
         Best-effort Tier 5 column-level diff for the ROW_NUMBER() fallback
@@ -1139,12 +1176,15 @@ class DatabricksConnector:
         used by get_row_hashes_by_row_number, filtered down to the given
         row numbers, then diffs the fetched rows column-by-column.
 
-        `order_by_columns` MUST be the exact same column list (same order)
-        passed to get_row_hashes_by_row_number for this table - that is
-        what keeps row numbers consistent between the hash-computation
-        pass (Tier 4) and this re-fetch (Tier 5). `value_columns` is
-        normally the same list too, since the row-number fallback has no
-        key to exclude.
+        `order_by_columns`/`value_columns` are the SOURCE-side spelling;
+        `target_order_by_columns`/`target_value_columns` (when a
+        column_map applies), the positionally-aligned TARGET-side
+        spelling for the same columns - each MUST be the exact same
+        per-side column list (same order) passed to
+        get_row_hashes_by_row_number for this table, which is what keeps
+        row numbers consistent between the hash-computation pass (Tier 4)
+        and this re-fetch (Tier 5). When omitted, the target lists default
+        to the source lists (today's behavior, unchanged).
 
         This is inherently best-effort, not a substitute for a real key:
         "row N" on the source and target are only the same logical record
@@ -1161,25 +1201,34 @@ class DatabricksConnector:
         Returns the same shape as _changed_row_detail: one dict per
         row with "key" (here always {"row_number": N}),
         "mismatched_columns", "source_values", "target_values",
-        "source_row_hash", "target_row_hash".
+        "source_row_hash", "target_row_hash" - all keyed/labeled by the
+        canonical (target) column name.
         """
         if not row_numbers:
             return []
 
+        target_order_by_columns = list(target_order_by_columns or order_by_columns)
+        target_value_columns = list(target_value_columns or value_columns)
+        source_to_target = dict(zip(value_columns, target_value_columns))
+
         src = self._qualify(source_catalog, schema, table)
         tgt = self._qualify(target_catalog, schema, table)
-        order_by = ", ".join(self._quote_ident(c) for c in order_by_columns)
-        value_idents = [self._quote_ident(c) for c in value_columns]
-        select_list = ", ".join(value_idents)
+        source_order_by = ", ".join(self._quote_ident(c) for c in order_by_columns)
+        target_order_by = ", ".join(self._quote_ident(c) for c in target_order_by_columns)
+        source_value_idents = [self._quote_ident(c) for c in value_columns]
+        target_value_idents = [self._quote_ident(c) for c in target_value_columns]
+        source_select_list = ", ".join(source_value_idents)
+        target_select_list = ", ".join(target_value_idents)
         where_clause = self._bucket_where_clause(bucket_predicate)
-        row_hash_expr = self._row_hash_expr(value_columns)
+        source_row_hash_expr = self._row_hash_expr(value_columns)
+        target_row_hash_expr = self._row_hash_expr(target_value_columns)
 
         # row_numbers are Python ints derived from our own prior
         # ROW_NUMBER() output (never user input) - safe to inline.
         unique_row_numbers = sorted(set(int(n) for n in row_numbers))[: int(limit_samples)]
         row_numbers_csv = ", ".join(str(n) for n in unique_row_numbers)
 
-        def _numbered_query(fqtn: str) -> str:
+        def _numbered_query(fqtn: str, order_by: str, select_list: str, row_hash_expr: str) -> str:
             return f"""
                 SELECT row_number, {select_list}, {row_hash_expr} AS __row_hash
                 FROM (
@@ -1194,10 +1243,10 @@ class DatabricksConnector:
 
         try:
             source_rows = self._execute_to_dataframe(
-                _numbered_query(src)
+                _numbered_query(src, source_order_by, source_select_list, source_row_hash_expr)
             ).to_dict(orient="records")
             target_rows = self._execute_to_dataframe(
-                _numbered_query(tgt)
+                _numbered_query(tgt, target_order_by, target_select_list, target_row_hash_expr)
             ).to_dict(orient="records")
         except Exception as exc:
             logger.exception(
@@ -1216,18 +1265,25 @@ class DatabricksConnector:
                 continue
 
             mismatched_columns = [
-                col for col in value_columns
-                if values_differ(src_row.get(col), tgt_row.get(col))
+                source_to_target[src_col] for src_col in value_columns
+                if values_differ(src_row.get(src_col), tgt_row.get(source_to_target[src_col]))
             ]
             if not mismatched_columns:
-                mismatched_columns = list(value_columns)
+                mismatched_columns = list(target_value_columns)
 
             detail.append(
                 {
                     "key": {"row_number": src_row["row_number"]},
                     "mismatched_columns": mismatched_columns,
-                    "source_values": {c: src_row.get(c) for c in mismatched_columns},
-                    "target_values": {c: tgt_row.get(c) for c in mismatched_columns},
+                    "source_values": {
+                        source_to_target[src_col]: src_row.get(src_col)
+                        for src_col in value_columns
+                        if source_to_target[src_col] in mismatched_columns
+                    },
+                    "target_values": {
+                        tgt_col: tgt_row.get(tgt_col)
+                        for tgt_col in mismatched_columns
+                    },
                     "source_row_hash": src_row.get("__row_hash"),
                     "target_row_hash": tgt_row.get("__row_hash"),
                 }

{table_validator-0.1.7 → table_validator-0.1.9}/table_validator/models.py

@@ -517,6 +517,30 @@ class CatalogValidationRequest(BaseModel):
         ),
     )
 
+    column_map: Dict[str, str] = Field(
+        default_factory=dict,
+        description=(
+            "Optional map of source-table column name -> target-table "
+            "column name, for when an individual column was renamed "
+            "between source and target (e.g. source has 'cust_id', "
+            "target has 'customer_id'). An explicit pair like this is "
+            "treated as fully equivalent through the entire pipeline - "
+            "schema/type/nullable checks, null/distinct/min-max "
+            "statistics, whole-table fingerprint, row-hash diff, and "
+            "column-level mismatch detail - bypassing name-based column "
+            "matching entirely for that pair (unlike ignore_columns/"
+            "only_columns, which still require the name to appear in the "
+            "intersection). Unmapped columns are matched by identical "
+            "name as usual. Resolved before only_columns/ignore_columns/"
+            "ignore_datatype_columns apply, so those three act on the "
+            "resolved/canonical (target-side) name. A mapped name that "
+            "doesn't actually exist on either side produces a clear "
+            "error rather than a silent no-op. Known limitation: a "
+            "column configured as a primary key, or used as a Tier 3 "
+            "partition/bucket column, must not also appear here."
+        ),
+    )
+
     case_sensitive_columns: bool = Field(
         default=False,
         description="Case-sensitive column name comparison.",
@@ -758,6 +782,12 @@ class ColumnValidationResult(BaseModel):
     column: str
     status: ValidationStatus
 
+    # Populated only when column_map actually renamed this column (source
+    # name differs from the target/canonical name shown above) - mirrors
+    # TableValidationResult.source_table_name's "only set when it
+    # differs" convention.
+    source_column: Optional[str] = None
+
     source_data_type: Optional[str] = None
     target_data_type: Optional[str] = None
     data_type_status: Optional[ValidationStatus] = None
@@ -799,6 +829,11 @@ class RowMismatchDetail(BaseModel):
     primary_key: Dict[str, Any] = Field(default_factory=dict)
     mismatch_column: str
 
+    # Populated only when column_map actually renamed this column -
+    # mismatch_column stays the canonical/target-side name (existing
+    # report rendering, existing tests, unaffected).
+    source_mismatch_column: Optional[str] = None
+
     source_value: Optional[Any] = None
     target_value: Optional[Any] = None