snowflake-data-validation 1.42.0__tar.gz → 1.42.2__tar.gz

{snowflake_data_validation-1.42.0 → snowflake_data_validation-1.42.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.42.0
+Version: 1.42.2
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -42,7 +42,7 @@ Requires-Dist: snowflake-connector-python>=4.7.1
 Requires-Dist: toml>=0.10.2
 Requires-Dist: typer>=0.15.3
 Requires-Dist: typing-extensions>=4.13.2
-Requires-Dist: urllib3>=2.7.0
+Requires-Dist: urllib3>=2.8.0
 Provides-Extra: all
 Requires-Dist: azure-identity>=1.15.0; extra == 'all'
 Requires-Dist: coverage>=7.6.7; extra == 'all'
@@ -79,7 +79,7 @@ Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgresql'
 Provides-Extra: redshift
 Requires-Dist: redshift-connector>=2.1.14; extra == 'redshift'
 Provides-Extra: scai
-Requires-Dist: snowflake-code-unit-registry>=0.8.78; extra == 'scai'
+Requires-Dist: snowflake-code-unit-registry>=0.8.83; extra == 'scai'
 Provides-Extra: sqlite
 Requires-Dist: sqlean-py>=3.45.0; extra == 'sqlite'
 Provides-Extra: sqlserver
@@ -190,6 +190,16 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.42.2
+
+No user-facing changes in this release.
+
+### v1.42.1
+
+**Improvements**
+
+- Updated Redshift Iceberg `TIMESTAMP`/`TIMESTAMPTZ` mapping to land as `VARCHAR` for zero-copy `ADD_FILES_COPY` compatibility.
+
 ### v1.42.0
 
 **New features**

{snowflake_data_validation-1.42.0 → snowflake_data_validation-1.42.2}/README.md

@@ -99,6 +99,16 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.42.2
+
+No user-facing changes in this release.
+
+### v1.42.1
+
+**Improvements**
+
+- Updated Redshift Iceberg `TIMESTAMP`/`TIMESTAMPTZ` mapping to land as `VARCHAR` for zero-copy `ADD_FILES_COPY` compatibility.
+
 ### v1.42.0
 
 **New features**

{snowflake_data_validation-1.42.0 → snowflake_data_validation-1.42.2}/pyproject.toml

@@ -42,7 +42,7 @@ dependencies = [
   "pydantic-yaml>=1.4.0",
   "typing-extensions>=4.13.2",
   "typer>=0.15.3",
-  "urllib3>=2.7.0",
+  "urllib3>=2.8.0",
 ]
 description = "Snowflake Data Validation"
 dynamic = ['version']
@@ -100,7 +100,7 @@ db2 = [
 # musllinux build, and SDV is published to public PyPI. Inside DMO/DEA the
 # dependency is already declared, so SDV's logger tree is covered there.
 scai = [
-  "snowflake-code-unit-registry>=0.8.78",
+  "snowflake-code-unit-registry>=0.8.83",
 ]
 
 [project.urls]

{snowflake_data_validation-1.42.0 → snowflake_data_validation-1.42.2}/src/snowflake/snowflake_data_validation/utils/iceberg_dv_common.py

@@ -7,8 +7,10 @@ Iceberg compatible L3 normalization key overlays.
 Source/target SQL bridge overlays for Snowflake Iceberg tables under
 ``storageMode=compatible``:
 
-- v2: timestamps → microsecond (``FF6``) canonization (VARCHAR ISO storage, not ``FF9``).
-- v3: timestamps stay typed (no FF6 VARCHAR overlay).
+- v2: typed Snowflake timestamps → microsecond (``FF6``) canonization.
+- v3: typed Snowflake timestamps stay on default ``FF9`` templates (no FF6 overlay).
+- Redshift ``TIMESTAMP`` / ``TIMESTAMPTZ`` land as ``VARCHAR`` on both specs and
+  compare as the stored UNLOAD text.
 - Redshift flattened values → the same text emitted by migration UNLOAD.
 - Complex / semi-structured values stored as ``VARCHAR`` →
   canonical ``TO_JSON(PARSE_JSON(...))`` text.
@@ -53,15 +55,12 @@ _FLATTENED_REDSHIFT_SOURCE_BASES = frozenset(
         "TIME WITHOUT TIME ZONE",
         "TIMETZ",
         "TIME WITH TIME ZONE",
-        "VARBYTE",
-        "BINARY VARYING",
-    }
-)
-
-_NATIVE_REDSHIFT_TIMESTAMPTZ_SOURCE_BASES = frozenset(
-    {
+        "TIMESTAMP",
+        "TIMESTAMP WITHOUT TIME ZONE",
         "TIMESTAMPTZ",
         "TIMESTAMP WITH TIME ZONE",
+        "VARBYTE",
+        "BINARY VARYING",
     }
 )
 
@@ -70,6 +69,8 @@ _REDSHIFT_SOURCE_ICEBERG_KEYS = {
     "TIME WITHOUT TIME ZONE": "TIME_ICEBERG",
     "TIMETZ": "TIMETZ_ICEBERG",
     "TIME WITH TIME ZONE": "TIMETZ_ICEBERG",
+    "TIMESTAMPTZ": "TIMESTAMPTZ_ICEBERG",
+    "TIMESTAMP WITH TIME ZONE": "TIMESTAMPTZ_ICEBERG",
     "VARBYTE": "VARBYTE_ICEBERG",
     "BINARY VARYING": "VARBYTE_ICEBERG",
 }
@@ -160,8 +161,6 @@ def resolve_iceberg_v2_compatible_normalization_key(
         return "VARCHAR"
 
     base = _base_type(key)
-    if is_redshift_source and paired_source_base in _NATIVE_REDSHIFT_TIMESTAMPTZ_SOURCE_BASES and base == "TIMESTAMP_LTZ":
-        return "TIMESTAMP_LTZ_UTC_FF9_OFFSET"
 
     if spec == _ICEBERG_SPEC_V2:
         if key in _TIMESTAMP_FF6_KEYS:

{snowflake_data_validation-1.42.0 → snowflake_data_validation-1.42.2}/src/snowflake/snowflake_data_validation/validation/data_validator_base.py

@@ -123,9 +123,9 @@ def float_family_mapping_matches_target(mapped_value: str | None, norm_target: s
     return mapped_base in _SNOWFLAKE_FLOAT_FAMILY and target_base in _SNOWFLAKE_FLOAT_FAMILY
 
 
-# Iceberg v3 declares the timestamp precision the Parquet footer requires (e.g.
-# TIMESTAMP_LTZ(6) for Redshift UNLOAD INT96). INFORMATION_SCHEMA.DATA_TYPE
-# reports the unparameterized token and keeps the precision in DATETIME_PRECISION.
+# INFORMATION_SCHEMA.DATA_TYPE reports TIMESTAMP_NTZ/LTZ/TZ without the DDL
+# precision parameter (kept in DATETIME_PRECISION). Redshift Iceberg
+# TIMESTAMP/TIMESTAMPTZ are VARCHAR and do not use this match.
 _SNOWFLAKE_CATALOG_TIMESTAMP_TOKENS = frozenset({"TIMESTAMP_NTZ", "TIMESTAMP_LTZ", "TIMESTAMP_TZ"})
 
 

{snowflake_data_validation-1.42.0 → snowflake_data_validation-1.42.2}/tests/unit/test_data_validator.py

@@ -1515,8 +1515,8 @@ def test_basic_validation_with_datatype_mapping(mock_context):
     assert len(failure_rows) == 0
 
 
-def test_basic_validation_iceberg_v3_timestamp_precision_matches_catalog(mock_context):
-    """Iceberg v3 declares TIMESTAMP_LTZ(6); the catalog reports TIMESTAMP_LTZ."""
+def test_basic_validation_iceberg_timestamp_varchar_matches_catalog_text(mock_context):
+    """Iceberg TIMESTAMP/TIMESTAMPTZ land as VARCHAR; the catalog reports TEXT."""
     source_df = pd.DataFrame(
         {
             COLUMN_VALIDATED: ["DTTM2"],
@@ -1526,7 +1526,7 @@ def test_basic_validation_iceberg_v3_timestamp_precision_matches_catalog(mock_co
     target_df = pd.DataFrame(
         {
             COLUMN_VALIDATED: ["DTTM2"],
-            "DATA_TYPE": ["TIMESTAMP_LTZ"],
+            "DATA_TYPE": ["TEXT"],
         }
     )
 
@@ -1538,7 +1538,7 @@ def test_basic_validation_iceberg_v3_timestamp_precision_matches_catalog(mock_co
         target_df=target_df,
         source_df=source_df,
         column_mappings={},
-        datatypes_mappings={"TIMESTAMP WITHOUT TIME ZONE": "TIMESTAMP_LTZ(6)"},
+        datatypes_mappings={"TIMESTAMP WITHOUT TIME ZONE": "VARCHAR"},
     )
 
     failure_rows = result[result[STATUS_KEY] == Result.FAILURE.value]

{snowflake_data_validation-1.42.0 → snowflake_data_validation-1.42.2}/tests/unit/test_iceberg_dv_l3_normalization.py

@@ -212,20 +212,20 @@ class TestResolveIcebergL3NormalizationKey:
         "paired_source_type",
         ["TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"],
     )
-    def test_native_redshift_timestamptz_uses_utc_ff9_ltz_key(
+    def test_redshift_timestamptz_varchar_compares_as_stored_text(
         self,
         iceberg_spec: str,
         paired_source_type: str,
     ) -> None:
         assert (
             resolve_iceberg_v2_compatible_normalization_key(
-                "TIMESTAMP_LTZ(6)",
+                "VARCHAR",
                 paired_source_type=paired_source_type,
                 paired_source_platform=Platform.REDSHIFT,
                 iceberg_spec=iceberg_spec,
                 storage_mode="compatible",
             )
-            == "TIMESTAMP_LTZ_UTC_FF9_OFFSET"
+            == "VARCHAR"
         )
 
     def test_non_redshift_iso_tz_varchar_keeps_ff6_overlay(self) -> None:
@@ -240,6 +240,17 @@ class TestResolveIcebergL3NormalizationKey:
             == "VARCHAR_ISO_TZ_FF6"
         )
 
+    def test_v3_varchar_json_when_paired_complex_source(self) -> None:
+        assert (
+            resolve_iceberg_v2_compatible_normalization_key(
+                "VARCHAR",
+                paired_source_type="JSON",
+                iceberg_spec="v3",
+                storage_mode="compatible",
+            )
+            == "VARCHAR_JSON_ICEBERG"
+        )
+
     def test_plain_varchar_unchanged(self) -> None:
         assert (
             resolve_iceberg_v2_compatible_normalization_key(
@@ -268,6 +279,14 @@ class TestResolveIcebergL3NormalizationKey:
             )
             == "TIMESTAMP_NTZ"
         )
+        assert (
+            resolve_iceberg_v2_compatible_normalization_key(
+                "TIMESTAMP_LTZ",
+                iceberg_spec="v3",
+                storage_mode="compatible",
+            )
+            == "TIMESTAMP_LTZ"
+        )
         assert (
             resolve_iceberg_v2_compatible_normalization_key(
                 "VARCHAR",
@@ -278,6 +297,33 @@ class TestResolveIcebergL3NormalizationKey:
             == "VARCHAR_JSON_ICEBERG"
         )
 
+    def test_redshift_v3_timestamp_source_target_ff9_parity(self) -> None:
+        from pathlib import Path
+
+        import yaml
+
+        import snowflake.snowflake_data_validation as _sdv
+
+        sdv_root = Path(_sdv.__file__).parent
+        redshift_templates = yaml.safe_load(
+            (sdv_root / "redshift" / "extractor" / "templates" / "redshift_datatypes_normalization_templates.yaml").read_text(
+                encoding="utf-8"
+            )
+        )
+        snowflake_templates = yaml.safe_load(
+            (sdv_root / "snowflake" / "extractor" / "templates" / "snowflake_datatypes_normalization_templates.yaml").read_text(
+                encoding="utf-8"
+            )
+        )
+        target_key = resolve_iceberg_v2_compatible_normalization_key(
+            "TIMESTAMP_NTZ",
+            iceberg_spec="v3",
+            storage_mode="compatible",
+        )
+        assert ".US') || '000'" in redshift_templates["TIMESTAMP"]
+        assert "FF9" in snowflake_templates[target_key]
+        assert "FF6" not in snowflake_templates[target_key]
+
     def test_v3_geo_as_varchar(self) -> None:
         assert (
             resolve_iceberg_v2_compatible_normalization_key(
@@ -300,8 +346,8 @@ class TestResolveIcebergL3SourceNormalizationKey:
             ("TIME WITHOUT TIME ZONE", "TIME_ICEBERG"),
             ("TIMETZ", "TIMETZ_ICEBERG"),
             ("TIME WITH TIME ZONE", "TIMETZ_ICEBERG"),
-            ("TIMESTAMPTZ", "TIMESTAMPTZ"),
-            ("TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH TIME ZONE"),
+            ("TIMESTAMPTZ", "TIMESTAMPTZ_ICEBERG"),
+            ("TIMESTAMP WITH TIME ZONE", "TIMESTAMPTZ_ICEBERG"),
             ("TIMESTAMP", "TIMESTAMP"),
             ("VARBYTE", "VARBYTE_ICEBERG"),
             ("BINARY VARYING", "VARBYTE_ICEBERG"),
@@ -487,12 +533,12 @@ class TestIcebergL3RenderedSqlParity:
         "source_type",
         ["TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"],
     )
-    def test_native_timestamptz_target_matches_source_utc_ff9(
+    def test_timestamptz_target_uses_stored_varchar(
         self,
         source_type: str,
     ) -> None:
         context = self._context(
-            [_column("COL_TIMESTAMPTZ", "TIMESTAMP_LTZ(6)")],
+            [_column("COL_TIMESTAMPTZ", "VARCHAR")],
             _SNOWFLAKE_TEMPLATES,
         )
         context.paired_source_column_types = {"COL_TIMESTAMPTZ": source_type}
@@ -501,10 +547,7 @@ class TestIcebergL3RenderedSqlParity:
             context
         )
 
-        assert rendered["COL_TIMESTAMPTZ"] == (
-            """TO_CHAR(CONVERT_TIMEZONE('UTC', "COL_TIMESTAMPTZ"), """
-            """'YYYY-MM-DD"T"HH24:MI:SS.FF9') || '+00:00'"""
-        )
+        assert rendered["COL_TIMESTAMPTZ"] == '"COL_TIMESTAMPTZ"'
 
     def test_flattened_super_target_template_preserves_staged_text(self) -> None:
         rules = yaml.safe_load(_SNOWFLAKE_TEMPLATES.read_text())