table-validator 0.1.1__tar.gz → 0.1.3__tar.gz

{table_validator-0.1.1/table_validator.egg-info → table_validator-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.1
+Version: 0.1.3
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9
@@ -174,6 +174,14 @@ Everything lives outside the repo, under your home directory:
   by construction, since it's written under your home directory rather
   than the working directory.
 
+Optional: `DATABRICKS_RETRY_TIMEOUT_SECONDS` (a plain environment variable,
+not part of `config.yaml`) raises the CloudFetch HTTP retry timeout above
+its 300-second default. On a slow or unstable network, downloading a
+large row-hash result set can legitimately take longer than that, causing
+a `Retry request would exceed Retry policy max retry duration` failure
+even though the query itself succeeded. Set it higher (e.g. `900` for 15
+minutes) if you hit this on large tables.
+
 A future version will replace manual credential entry with Azure CLI /
 Service Principal auth and Databricks CLI / OAuth login, without changing
 the config file format or any command usage above.

{table_validator-0.1.1 → table_validator-0.1.3}/README.md

@@ -147,6 +147,14 @@ Everything lives outside the repo, under your home directory:
   by construction, since it's written under your home directory rather
   than the working directory.
 
+Optional: `DATABRICKS_RETRY_TIMEOUT_SECONDS` (a plain environment variable,
+not part of `config.yaml`) raises the CloudFetch HTTP retry timeout above
+its 300-second default. On a slow or unstable network, downloading a
+large row-hash result set can legitimately take longer than that, causing
+a `Retry request would exceed Retry policy max retry duration` failure
+even though the query itself succeeded. Set it higher (e.g. `900` for 15
+minutes) if you hit this on large tables.
+
 A future version will replace manual credential entry with Azure CLI /
 Service Principal auth and Databricks CLI / OAuth login, without changing
 the config file format or any command usage above.

{table_validator-0.1.1 → table_validator-0.1.3}/table_validator/connectors/databricks_connector.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 import datetime
 import logging
 import numbers
+import os
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple
 
 import pandas as pd
@@ -97,6 +98,7 @@ class DatabricksConnector:
         host: Optional[str] = None,
         token: Optional[str] = None,
         http_path: Optional[str] = None,
+        retry_timeout_seconds: Optional[float] = None,
     ) -> None:
         """
         host/token/http_path must be resolved by the caller before
@@ -104,11 +106,27 @@ class DatabricksConnector:
         get_databricks_token() for the token, and config.databricks.
         workspace_url/http_path for the rest. This connector does not
         read credentials from the environment itself.
+
+        retry_timeout_seconds overrides databricks-sql-connector's
+        CloudFetch HTTP retry policy (_retry_stop_after_attempts_duration),
+        which otherwise defaults to 300 seconds. On a slow or unstable
+        network, downloading a large row-hash result set can legitimately
+        take longer than that, causing a hard
+        "Retry request would exceed Retry policy max retry duration"
+        failure even though the query itself succeeded - raising this
+        gives a slow-but-working connection more time instead of giving
+        up. Falls back to the DATABRICKS_RETRY_TIMEOUT_SECONDS environment
+        variable, then the connector's own default, if not given.
         """
 
         self._host = host
         self._token = token
         self._http_path = http_path
+        self._retry_timeout_seconds = retry_timeout_seconds or (
+            float(os.environ["DATABRICKS_RETRY_TIMEOUT_SECONDS"])
+            if os.environ.get("DATABRICKS_RETRY_TIMEOUT_SECONDS")
+            else None
+        )
 
         if not self._host or not self._token:
             raise ValueError(
@@ -138,11 +156,17 @@ class DatabricksConnector:
 
         try:
 
-            self._connection = sql.connect(
-                server_hostname=self._host,
-                http_path=self._http_path,
-                access_token=self._token,
-            )
+            connect_kwargs: Dict[str, Any] = {
+                "server_hostname": self._host,
+                "http_path": self._http_path,
+                "access_token": self._token,
+            }
+            if self._retry_timeout_seconds is not None:
+                connect_kwargs["_retry_stop_after_attempts_duration"] = (
+                    self._retry_timeout_seconds
+                )
+
+            self._connection = sql.connect(**connect_kwargs)
 
             with self._connection.cursor() as cursor:
                 cursor.execute("SELECT 1")

{table_validator-0.1.1 → table_validator-0.1.3}/table_validator/validators/catalog_validator.py

@@ -88,6 +88,30 @@ class CatalogValidator:
         self.partition_prompt = partition_prompt
         logger.debug("CatalogValidator initialised")
 
+    @staticmethod
+    def _lookup_primary_key(
+        request: "CatalogValidationRequest",
+        schema_name: str,
+        table_name: str,
+    ) -> Optional[List[str]]:
+        """
+        Look up request.primary_keys (keyed by "schema.table" or bare
+        table name, as typed by the user into config.yaml/the wizard) for
+        the given schema_name/table_name.
+
+        schema_name/table_name here come from whatever casing Databricks'
+        information_schema actually returns, matched case-insensitively
+        against the catalog in compare_schemas/compare_tables - which
+        does not necessarily match the exact casing the user configured.
+        An exact-match dict lookup would silently miss a configured key
+        and fall through to the much more expensive ROW_NUMBER() fallback
+        with no indication why, so this normalizes both sides to
+        lowercase before comparing.
+        """
+        lowered = {k.lower(): v for k, v in request.primary_keys.items()}
+        key_lookup = f"{schema_name}.{table_name}".lower()
+        return lowered.get(key_lookup) or lowered.get(table_name.lower())
+
     # ------------------------------------------------------------------
     # Stage 1 + top-level orchestration
     # ------------------------------------------------------------------
@@ -697,10 +721,7 @@ class CatalogValidator:
 
         # Configured PK column missing from either side is also BLOCKING -
         # every later tier depends on being able to resolve a usable key.
-        key_lookup = f"{schema_name}.{table_name}"
-        configured_key = request.primary_keys.get(key_lookup) or request.primary_keys.get(
-            table_name
-        )
+        configured_key = self._lookup_primary_key(request, schema_name, table_name)
         if configured_key:
             common_lower = {c.lower() for c in common_cols}
             if any(k.lower() not in common_lower for k in configured_key):
@@ -1040,8 +1061,7 @@ class CatalogValidator:
             )
             return
 
-        key_lookup = f"{schema_name}.{table_name}"
-        key_columns = request.primary_keys.get(key_lookup) or request.primary_keys.get(table_name)
+        key_columns = self._lookup_primary_key(request, schema_name, table_name)
         candidates = self._partition_candidates(common_cols, key_columns)
 
         context = PartitionPromptContext(
@@ -1184,10 +1204,7 @@ class CatalogValidator:
             result.tier_reached = ValidationTier.ROW_HASH
 
             if not using_row_number_fallback:
-                key_lookup = f"{schema_name}.{table_name}"
-                key_columns = request.primary_keys.get(key_lookup) or request.primary_keys.get(
-                    table_name
-                )
+                key_columns = self._lookup_primary_key(request, schema_name, table_name)
                 mismatched_keys = [m.primary_key for m in mismatches if m.status == "MISMATCH"]
                 if key_columns and mismatched_keys:
                     self._tier5_column_diff(
@@ -1238,10 +1255,7 @@ class CatalogValidator:
         written straight to `result.data`, so multiple bucket calls
         aggregate instead of each overwriting the last.
         """
-        key_lookup = f"{schema_name}.{table_name}"
-        row_hash_key_columns = request.primary_keys.get(key_lookup) or request.primary_keys.get(
-            table_name
-        )
+        row_hash_key_columns = self._lookup_primary_key(request, schema_name, table_name)
 
         using_row_number_fallback = not row_hash_key_columns
         if using_row_number_fallback:
@@ -1470,10 +1484,7 @@ class CatalogValidator:
     ) -> DataValidationResult:
 
         mode = request.data_compare_mode
-        key = f"{schema_name}.{table_name}"
-        key_columns = request.primary_keys.get(key) or request.primary_keys.get(
-            table_name
-        )
+        key_columns = self._lookup_primary_key(request, schema_name, table_name)
 
         logger.info(
             "[compare_data] table=%s.%s | mode=%s | resolved_key_columns=%s",

{table_validator-0.1.1 → table_validator-0.1.3/table_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-validator
-Version: 0.1.1
+Version: 0.1.3
 Summary: CLI tool for validating data migrations between Azure (Blob Storage / SQL Database) and Databricks Delta Lake catalogs
 License: MIT
 Requires-Python: >=3.9
@@ -174,6 +174,14 @@ Everything lives outside the repo, under your home directory:
   by construction, since it's written under your home directory rather
   than the working directory.
 
+Optional: `DATABRICKS_RETRY_TIMEOUT_SECONDS` (a plain environment variable,
+not part of `config.yaml`) raises the CloudFetch HTTP retry timeout above
+its 300-second default. On a slow or unstable network, downloading a
+large row-hash result set can legitimately take longer than that, causing
+a `Retry request would exceed Retry policy max retry duration` failure
+even though the query itself succeeded. Set it higher (e.g. `900` for 15
+minutes) if you hit this on large tables.
+
 A future version will replace manual credential entry with Azure CLI /
 Service Principal auth and Databricks CLI / OAuth login, without changing
 the config file format or any command usage above.

{table_validator-0.1.1 → table_validator-0.1.3}/table_validator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.1.1",
+  "tag": "0.1.3",
   "distance": 0,
-  "node": "gd0c901c9dfc750527e33620c5dea6b15e43e4443",
+  "node": "g8f7a326158b935508f39bc9369245d7817f1dc89",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-27"

{table_validator-0.1.1 → table_validator-0.1.3}/tests/test_catalog_validator.py

@@ -991,6 +991,46 @@ def test_tier5_column_diff_runs_for_mismatched_key_and_names_exact_column():
     assert detail.verified is True
 
 
+def test_primary_key_lookup_is_case_insensitive_against_catalog_metadata():
+    """request.primary_keys is keyed by whatever casing the user typed
+    into config.yaml/the wizard, but schema_name/table_name at lookup
+    time come from Databricks' information_schema (matched
+    case-insensitively against the catalog elsewhere in this file). A
+    primary key configured as "Bronze.Customers" must still be found for
+    the real "bronze.customers" table - an exact-match lookup would
+    silently miss it and fall through to the much slower ROW_NUMBER()
+    fallback instead of using the real configured key."""
+    connector = _mismatched_fingerprint_connector()
+    connector.get_row_hashes.side_effect = lambda catalog, schema, table, cols, pk, bucket_predicate=None: (
+        _hash_df([(1, "aaa"), (2, "bbb")])
+        if catalog == "cat_source"
+        else _hash_df([(1, "aaa"), (2, "zzz")])
+    )
+    connector.get_row_detail_for_keys.return_value = [
+        {
+            "key": {"id": 2},
+            "mismatched_columns": ["name"],
+            "source_values": {"name": "old-value"},
+            "target_values": {"name": "new-value"},
+            "source_row_hash": "bbb",
+            "target_row_hash": "zzz",
+        }
+    ]
+    validator = CatalogValidator(connector)
+
+    result = validator.compare_catalogs(
+        _request(primary_keys={"Bronze.Customers": ["id"]})
+    )
+    table = result.schemas[0].tables[0]
+
+    connector.get_row_hashes_by_row_number.assert_not_called()
+    assert connector.get_row_hashes.call_count == 2  # once per side (source, target)
+    connector.get_row_detail_for_keys.assert_called_once()
+    assert table.data.key_columns == ["id"]
+    assert table.tier_reached == ValidationTier.COLUMN_DIFF
+    assert table.data.sample_changed_detail[0].verified is True
+
+
 def test_tier5_column_diff_runs_for_row_number_fallback_when_mismatched():
     """No primary key configured -> row-number fallback is used for Tier
     4, but a real mismatch must still let Tier 5 attempt best-effort

{table_validator-0.1.1 → table_validator-0.1.3}/tests/test_databricks_connector.py

@@ -8,9 +8,12 @@ so these tests verify SQL-generation logic and result parsing only.
 
 from __future__ import annotations
 
+from unittest.mock import MagicMock
+
 import pandas as pd
 import pytest
 
+import table_validator.connectors.databricks_connector as databricks_connector_module
 from table_validator.connectors.databricks_connector import DatabricksConnector
 
 
@@ -402,3 +405,78 @@ def test_get_row_detail_for_row_numbers_wraps_exceptions(monkeypatch):
             row_numbers=[2],
             value_columns=["id"],
         )
+
+
+# ---------------------------------------------------------------------------
+# connect(): retry timeout override for slow/unstable CloudFetch downloads.
+# ---------------------------------------------------------------------------
+def test_connect_omits_retry_kwarg_by_default(monkeypatch):
+    """Without an explicit retry_timeout_seconds or
+    DATABRICKS_RETRY_TIMEOUT_SECONDS, connect() must not pass
+    _retry_stop_after_attempts_duration at all, preserving
+    databricks-sql-connector's own default (300s)."""
+    monkeypatch.delenv("DATABRICKS_RETRY_TIMEOUT_SECONDS", raising=False)
+    captured = {}
+
+    def fake_connect(**kwargs):
+        captured.update(kwargs)
+        return MagicMock()
+
+    monkeypatch.setattr(databricks_connector_module.sql, "connect", fake_connect)
+    connector = _connector()
+
+    connector.connect()
+
+    assert "_retry_stop_after_attempts_duration" not in captured
+
+
+def test_connect_passes_explicit_retry_timeout_seconds(monkeypatch):
+    monkeypatch.delenv("DATABRICKS_RETRY_TIMEOUT_SECONDS", raising=False)
+    captured = {}
+
+    def fake_connect(**kwargs):
+        captured.update(kwargs)
+        return MagicMock()
+
+    monkeypatch.setattr(databricks_connector_module.sql, "connect", fake_connect)
+    connector = DatabricksConnector(
+        host="h", token="t", http_path="p", retry_timeout_seconds=900.0
+    )
+
+    connector.connect()
+
+    assert captured["_retry_stop_after_attempts_duration"] == 900.0
+
+
+def test_connect_falls_back_to_retry_timeout_env_var(monkeypatch):
+    monkeypatch.setenv("DATABRICKS_RETRY_TIMEOUT_SECONDS", "1200")
+    captured = {}
+
+    def fake_connect(**kwargs):
+        captured.update(kwargs)
+        return MagicMock()
+
+    monkeypatch.setattr(databricks_connector_module.sql, "connect", fake_connect)
+    connector = _connector()
+
+    connector.connect()
+
+    assert captured["_retry_stop_after_attempts_duration"] == 1200.0
+
+
+def test_connect_explicit_arg_takes_priority_over_env_var(monkeypatch):
+    monkeypatch.setenv("DATABRICKS_RETRY_TIMEOUT_SECONDS", "1200")
+    captured = {}
+
+    def fake_connect(**kwargs):
+        captured.update(kwargs)
+        return MagicMock()
+
+    monkeypatch.setattr(databricks_connector_module.sql, "connect", fake_connect)
+    connector = DatabricksConnector(
+        host="h", token="t", http_path="p", retry_timeout_seconds=600.0
+    )
+
+    connector.connect()
+
+    assert captured["_retry_stop_after_attempts_duration"] == 600.0