table-validator 0.1.2__tar.gz → 0.1.4__tar.gz

{table_validator-0.1.2/table_validator.egg-info → table_validator-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.2
+Version: 0.1.4
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9
@@ -174,6 +174,14 @@ Everything lives outside the repo, under your home directory:
   by construction, since it's written under your home directory rather
   than the working directory.
 
+Optional: `DATABRICKS_RETRY_TIMEOUT_SECONDS` (a plain environment variable,
+not part of `config.yaml`) raises the CloudFetch HTTP retry timeout above
+its 300-second default. On a slow or unstable network, downloading a
+large row-hash result set can legitimately take longer than that, causing
+a `Retry request would exceed Retry policy max retry duration` failure
+even though the query itself succeeded. Set it higher (e.g. `900` for 15
+minutes) if you hit this on large tables.
+
 A future version will replace manual credential entry with Azure CLI /
 Service Principal auth and Databricks CLI / OAuth login, without changing
 the config file format or any command usage above.

{table_validator-0.1.2 → table_validator-0.1.4}/README.md

@@ -147,6 +147,14 @@ Everything lives outside the repo, under your home directory:
   by construction, since it's written under your home directory rather
   than the working directory.
 
+Optional: `DATABRICKS_RETRY_TIMEOUT_SECONDS` (a plain environment variable,
+not part of `config.yaml`) raises the CloudFetch HTTP retry timeout above
+its 300-second default. On a slow or unstable network, downloading a
+large row-hash result set can legitimately take longer than that, causing
+a `Retry request would exceed Retry policy max retry duration` failure
+even though the query itself succeeded. Set it higher (e.g. `900` for 15
+minutes) if you hit this on large tables.
+
 A future version will replace manual credential entry with Azure CLI /
 Service Principal auth and Databricks CLI / OAuth login, without changing
 the config file format or any command usage above.

{table_validator-0.1.2 → table_validator-0.1.4}/table_validator/cli/main.py

@@ -420,20 +420,37 @@ def _run_databricks_validation(
     matching" for that level - CatalogValidator already performs the
     list-and-intersect discovery internally whenever request.schemas/
     tables is left unrestricted (None), so leaving the restriction off
-    IS the discovery trigger; there is no separate discovery step to call."""
+    IS the discovery trigger; there is no separate discovery step to call.
+
+    If the user explicitly named BOTH a source and target schema/table,
+    compare that exact pair directly - schema_map/table_map bypass
+    name-based matching entirely, so the two sides don't need to share a
+    name (a typo, a rename, different casing). schemas_restriction/
+    tables_restriction scope by the SOURCE-side name, matching
+    CatalogValidator._validate_schema/compare_catalogs' filtering
+    convention (mirrors _run_sql_validation's identical pattern below).
+    When names are identical (today's common case) schema_map/table_map
+    end up empty and behavior is unchanged."""
+    schema_map: Dict[str, str] = {}
     schemas_restriction = None
     if config.source_table.schema_name and config.target_table.schema_name:
-        schemas_restriction = [config.target_table.schema_name]
+        schemas_restriction = [config.source_table.schema_name]
+        if config.source_table.schema_name.lower() != config.target_table.schema_name.lower():
+            schema_map = {config.source_table.schema_name: config.target_table.schema_name}
 
+    table_map: Dict[str, str] = {}
     tables_restriction = None
     if config.source_table.table and config.target_table.table:
-        tables_restriction = [config.target_table.table]
+        tables_restriction = [config.source_table.table]
+        if config.source_table.table.lower() != config.target_table.table.lower():
+            table_map = {config.source_table.table: config.target_table.table}
 
     # A configured primary key only applies to the single named table
     # (not a catalog-wide sweep) - CatalogValidator looks it up by
     # "schema.table" first, falling back to a bare table name, so provide
     # both forms when a schema is known; falls back to row-number matching
-    # as before when unset.
+    # as before when unset. Keyed by the TARGET-side name, since that's
+    # what _lookup_primary_key resolves against.
     primary_keys: Dict[str, list] = {}
     if config.primary_key and config.target_table.table:
         primary_keys[config.target_table.table] = config.primary_key
@@ -447,7 +464,9 @@ def _run_databricks_validation(
         source_catalog=config.source_table.catalog or "",
         target_catalog=config.target_table.catalog or "",
         schemas=schemas_restriction,
+        schema_map=schema_map,
         tables=tables_restriction,
+        table_map=table_map,
         enabled_validations=set(config.validations),
         primary_keys=primary_keys,
         max_tier=max_tier,

{table_validator-0.1.2 → table_validator-0.1.4}/table_validator/connectors/databricks_connector.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 import datetime
 import logging
 import numbers
+import os
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple
 
 import pandas as pd
@@ -97,6 +98,7 @@ class DatabricksConnector:
         host: Optional[str] = None,
         token: Optional[str] = None,
         http_path: Optional[str] = None,
+        retry_timeout_seconds: Optional[float] = None,
     ) -> None:
         """
         host/token/http_path must be resolved by the caller before
@@ -104,11 +106,27 @@ class DatabricksConnector:
         get_databricks_token() for the token, and config.databricks.
         workspace_url/http_path for the rest. This connector does not
         read credentials from the environment itself.
+
+        retry_timeout_seconds overrides databricks-sql-connector's
+        CloudFetch HTTP retry policy (_retry_stop_after_attempts_duration),
+        which otherwise defaults to 300 seconds. On a slow or unstable
+        network, downloading a large row-hash result set can legitimately
+        take longer than that, causing a hard
+        "Retry request would exceed Retry policy max retry duration"
+        failure even though the query itself succeeded - raising this
+        gives a slow-but-working connection more time instead of giving
+        up. Falls back to the DATABRICKS_RETRY_TIMEOUT_SECONDS environment
+        variable, then the connector's own default, if not given.
         """
 
         self._host = host
         self._token = token
         self._http_path = http_path
+        self._retry_timeout_seconds = retry_timeout_seconds or (
+            float(os.environ["DATABRICKS_RETRY_TIMEOUT_SECONDS"])
+            if os.environ.get("DATABRICKS_RETRY_TIMEOUT_SECONDS")
+            else None
+        )
 
         if not self._host or not self._token:
             raise ValueError(
@@ -138,11 +156,17 @@ class DatabricksConnector:
 
         try:
 
-            self._connection = sql.connect(
-                server_hostname=self._host,
-                http_path=self._http_path,
-                access_token=self._token,
-            )
+            connect_kwargs: Dict[str, Any] = {
+                "server_hostname": self._host,
+                "http_path": self._http_path,
+                "access_token": self._token,
+            }
+            if self._retry_timeout_seconds is not None:
+                connect_kwargs["_retry_stop_after_attempts_duration"] = (
+                    self._retry_timeout_seconds
+                )
+
+            self._connection = sql.connect(**connect_kwargs)
 
             with self._connection.cursor() as cursor:
                 cursor.execute("SELECT 1")

{table_validator-0.1.2 → table_validator-0.1.4}/table_validator/models.py

@@ -451,6 +451,19 @@ class CatalogValidationRequest(BaseModel):
         ),
     )
 
+    schema_map: Dict[str, str] = Field(
+        default_factory=dict,
+        description=(
+            "Optional map of source-catalog schema name -> target-catalog "
+            "schema name, for when the user has explicitly named a source "
+            "and target schema that don't share the same name. An explicit "
+            "pair like this is compared directly, bypassing name-based "
+            "schema matching entirely (unlike `schemas`, which still "
+            "requires the name to appear in the intersection). Unmapped "
+            "schemas are matched by identical name as usual."
+        ),
+    )
+
     tables: Optional[List[str]] = Field(
         default=None,
         description=(
@@ -460,6 +473,19 @@ class CatalogValidationRequest(BaseModel):
         ),
     )
 
+    table_map: Dict[str, str] = Field(
+        default_factory=dict,
+        description=(
+            "Optional map of source-catalog table name -> target-catalog "
+            "table name, for when the user has explicitly named a source "
+            "and target table that don't share the same name - an explicit "
+            "pair like this is compared directly, bypassing name-based "
+            "table matching entirely (unlike `tables`, which still requires "
+            "the name to appear in the intersection). Unmapped tables are "
+            "matched by identical name as usual."
+        ),
+    )
+
     ignore_columns: List[str] = Field(default_factory=list)
 
     case_sensitive_columns: bool = Field(
@@ -835,6 +861,14 @@ class TableValidationResult(BaseModel):
     table: str
     status: ValidationStatus = ValidationStatus.SKIPPED
 
+    # Populated by CatalogValidator/AzureSqlValidator whenever the
+    # source-side schema/table name differs from the target-side name
+    # shown above (e.g. an explicit schema_map/table_map pair) - lets a
+    # future report enhancement show both names. schema_name/table above
+    # always reflect the TARGET-side name (the reporting convention).
+    source_schema_name: Optional[str] = None
+    source_table_name: Optional[str] = None
+
     exists_in_source: bool = True
     exists_in_target: bool = True