snowflake-data-validation 1.44.1__tar.gz → 1.45.0__tar.gz

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.44.1
+Version: 1.45.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -214,6 +214,28 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.45.0
+
+**New features**
+
+- Added support for applying custom L3 normalization per side, including `targetDataType: "*"` to target the mapped type.
+- Added the `includeCountDistinct` option; L2 `COUNT(DISTINCT)` is now off by default and opt-in globally or per table.
+- Added support for additional table and view types in data validation, including materialized views, temporary tables, BigQuery clones and snapshots, and DB2 materialized query tables.
+
+**Improvements**
+
+- Improved row comparison with positional column identity, so duplicate and case-variant column names no longer collapse.
+- Improved Redshift connection resiliency with TCP keepalive defaults.
+- Improved L1 validation to treat native integer bits-versus-digits precision differences as compatible, reported as a warning.
+
+**Bug fixes**
+
+- Fixed Oracle L2 metric rows and L3 cell frames upper-casing column names, which made mixed-case columns collide.
+- Fixed L3 failing rows over a lowercase SQL Server `UNIQUEIDENTIFIER` or an ignored `FLOAT` `targetExpression`.
+- Fixed a possible deadlock in expression casts when validators run on several threads.
+- Fixed Redshift and PostgreSQL `REAL` values being compared against a non-exact widened value.
+- Fixed Iceberg `FULL_INGEST` validation to follow the lossless-only temporal promotion of the load.
+
 ### v1.44.1
 
 **New features**

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/README.md

@@ -123,6 +123,28 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.45.0
+
+**New features**
+
+- Added support for applying custom L3 normalization per side, including `targetDataType: "*"` to target the mapped type.
+- Added the `includeCountDistinct` option; L2 `COUNT(DISTINCT)` is now off by default and opt-in globally or per table.
+- Added support for additional table and view types in data validation, including materialized views, temporary tables, BigQuery clones and snapshots, and DB2 materialized query tables.
+
+**Improvements**
+
+- Improved row comparison with positional column identity, so duplicate and case-variant column names no longer collapse.
+- Improved Redshift connection resiliency with TCP keepalive defaults.
+- Improved L1 validation to treat native integer bits-versus-digits precision differences as compatible, reported as a warning.
+
+**Bug fixes**
+
+- Fixed Oracle L2 metric rows and L3 cell frames upper-casing column names, which made mixed-case columns collide.
+- Fixed L3 failing rows over a lowercase SQL Server `UNIQUEIDENTIFIER` or an ignored `FLOAT` `targetExpression`.
+- Fixed a possible deadlock in expression casts when validators run on several threads.
+- Fixed Redshift and PostgreSQL `REAL` values being compared against a non-exact widened value.
+- Fixed Iceberg `FULL_INGEST` validation to follow the lossless-only temporal promotion of the load.
+
 ### v1.44.1
 
 **New features**

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -52,6 +52,9 @@ NVARCHAR: '"{{ col_name }}"'
 DATE: CONVERT(VARCHAR(10), "{{ col_name }}", 23)
 # Same text as SQL Server TIME: hh:mi:ss.fffffff + '00', cut to {{ time_text_length }}.
 TIME: LEFT(SUBSTRING(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 12, 16) + '00', {{ time_text_length }})
+# Iceberg compatible TIME stored as VARCHAR (CETAS): the extract's own text, TIME(7)
+# widened then '00', nine digits whatever the scale. '+' keeps NULL as NULL.
+TIME_ICEBERG: CAST(CAST("{{ col_name }}" AS TIME(7)) AS VARCHAR(16)) + '00'
 DATETIME: STUFF(CONVERT(VARCHAR(27), CAST(CAST("{{ col_name }}" AS DATETIME2(3)) AS DATETIME2(7)), 121), 11, 1, 'T') + '00'
 DATETIME2: LEFT(STUFF(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 11, 1, 'T'), 26) + '000'
 SMALLDATETIME: CONVERT(VARCHAR(19), "{{ col_name }}", 126) + '.000000000'

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -53,6 +53,10 @@ from snowflake.snowflake_data_validation.sqlserver.query.charset_normalization i
 from snowflake.snowflake_data_validation.utils.constants import (
     Platform,
 )
+from snowflake.snowflake_data_validation.utils.iceberg_dv_common import (
+    resolve_iceberg_compatible_tsql_source_normalization_key,
+)
+from snowflake.snowflake_data_validation.utils.iso_dv_common import get_paired_target_column_types
 from snowflake.snowflake_data_validation.utils.measure_collation import (
     TSQL_ROW_HASH_COLLATION_SUFFIX,
     tsql_comparison_collation_tokens,
@@ -141,8 +145,16 @@ class QueryGeneratorAzureSynapse(TsqlFloatMetricModifierMixin, QueryGeneratorBas
         datatypes_normalization_templates = table_context.templates_loader_manager.datatypes_normalization_templates
         rendered: dict[str, str] = {}
         renderer_columns = self._row_md5_normalization_columns(table_context)
+        paired_target_types = get_paired_target_column_types(table_context)
+        iceberg_spec = table_context.iceberg_spec
+        storage_mode = table_context.storage_mode
         for column in renderer_columns:
-            lookup_key = (column.data_type or "").strip().upper()
+            lookup_key = resolve_iceberg_compatible_tsql_source_normalization_key(
+                (column.data_type or "").strip().upper(),
+                paired_target_type=paired_target_types.get(column.name),
+                iceberg_spec=iceberg_spec,
+                storage_mode=storage_mode,
+            )
             normalization_template = self._lookup_normalization_template(
                 datatypes_normalization_templates,
                 lookup_key,

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_table_metadata_query.sql.j2

@@ -48,7 +48,10 @@ WITH rc_cte AS (
         (
             SELECT CASE table_type
                 WHEN 'BASE TABLE' THEN 'TABLE'
+                WHEN 'CLONE' THEN 'TABLE'
+                WHEN 'SNAPSHOT' THEN 'TABLE'
                 WHEN 'VIEW' THEN 'VIEW'
+                WHEN 'MATERIALIZED VIEW' THEN 'VIEW'
                 ELSE 'UNKNOWN'
             END
             FROM {{ object_database_sql }}.{{ object_schema_sql }}.INFORMATION_SCHEMA.TABLES

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/configuration/model/configuration_model.py

@@ -96,6 +96,7 @@ class ConfigurationModel(BaseModel):
             self._check_max_failed_rows_number(table)
             self._check_chunk_number(table)
             self.set_exclude_metrics(table)
+            self.set_include_count_distinct(table)
             self.set_apply_metric_column_modifier(table)
             self.set_include_lob_columns_in_row_validation(table)
 
@@ -113,6 +114,7 @@ class ConfigurationModel(BaseModel):
             self._check_max_failed_rows_number(view)
             self._check_chunk_number(view)
             self.set_exclude_metrics(view)
+            self.set_include_count_distinct(view)
             self.set_apply_metric_column_modifier(view)
             self.set_include_lob_columns_in_row_validation(view)
 
@@ -198,6 +200,10 @@ class ConfigurationModel(BaseModel):
         if table.exclude_metrics is None:
             table.exclude_metrics = self.validation_configuration.exclude_metrics
 
+    def set_include_count_distinct(self, table: TableConfiguration) -> None:
+        if table.include_count_distinct is None:
+            table.include_count_distinct = self.validation_configuration.include_count_distinct
+
     def set_include_lob_columns_in_row_validation(self, table: TableConfiguration) -> None:
         if table.include_lob_columns_in_row_validation is None:
             table.include_lob_columns_in_row_validation = (

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/configuration/model/table_configuration.py

@@ -32,6 +32,7 @@ VISIBLE_PROPERTIES: set[str] = {
     "chunk_number",
     "column_mappings",
     "exclude_metrics",
+    "include_count_distinct",
     "include_lob_columns_in_row_validation",
     "index_column_list",
     "max_failed_rows_number",
@@ -83,6 +84,7 @@ class TableConfiguration(BaseModel):
     column_mappings: dict[str, str] = {}
     max_failed_rows_number: int | None = None
     exclude_metrics: bool | None = None
+    include_count_distinct: bool | None = None
     apply_metric_column_modifier: bool | None = None
     include_lob_columns_in_row_validation: bool | None = None
     object_type: ObjectType = ObjectType.TABLE

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/configuration/model/validation_configuration.py

@@ -41,6 +41,9 @@ class ValidationConfiguration(BaseModel):
     custom_templates_path: Path | None = None
     max_failed_rows_number: int = MAX_FAILED_ROWS_NUMBER_DEFAULT_VALUE
     exclude_metrics: bool = False
+    # L2 COUNT(DISTINCT) is opt-in (SNOW-4338258): it dominates L2 cost on the source
+    # (one SQL Server batch 26.4 s with it, 1.4 s without) and L2 is the fast metrics level.
+    include_count_distinct: bool = False
     apply_metric_column_modifier: bool = True
     include_lob_columns_in_row_validation: bool = False
 

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_table_metadata_query.sql.j2

@@ -49,13 +49,15 @@ SELECT
     (
         SELECT CASE TYPE
             WHEN 'T' THEN 'TABLE'
+            WHEN 'G' THEN 'TABLE'
             WHEN 'V' THEN 'VIEW'
+            WHEN 'S' THEN 'VIEW'
             ELSE 'UNKNOWN'
         END
         FROM SYSCAT.TABLES
         WHERE TABSCHEMA = '{{ object_schema | escape_sql_literal }}'
           AND TABNAME = '{{ object_name | escape_sql_literal }}'
-          AND TYPE IN ('T', 'V')
+          AND TYPE IN ('T', 'G', 'V', 'S')
     ) AS OBJECT_TYPE,
     rc.ROW_COUNT
 FROM SYSCAT.COLUMNS C

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/executer/base_validation_executor.py

@@ -234,6 +234,7 @@ class BaseValidationExecutor(ABC):
             columns=source_table_column_metadata.columns,
             database_name=configured_object_name(self.context.source_platform, table_configuration.source_database),
             exclude_metrics=table_configuration.exclude_metrics,
+            include_count_distinct=bool(table_configuration.include_count_distinct),
             fully_qualified_name=table_configuration.fully_qualified_name,
             has_where_clause=table_configuration.has_where_clause,
             max_failed_rows_number=table_configuration.max_failed_rows_number,
@@ -279,6 +280,7 @@ class BaseValidationExecutor(ABC):
             columns=target_table_column_metadata.columns,
             database_name=configured_object_name(self.context.target_platform, table_configuration.target_database),
             exclude_metrics=table_configuration.exclude_metrics,
+            include_count_distinct=bool(table_configuration.include_count_distinct),
             fully_qualified_name=table_configuration.target_fully_qualified_name,
             has_where_clause=table_configuration.has_target_where_clause,
             max_failed_rows_number=table_configuration.max_failed_rows_number,

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/executer/source_validation_executor.py

@@ -128,6 +128,7 @@ class SourceValidationExecutor:
             columns=source_table_column_metadata.columns,
             database_name=configured_object_name(self.context.source_platform, table_configuration.source_database),
             exclude_metrics=table_configuration.exclude_metrics,
+            include_count_distinct=bool(table_configuration.include_count_distinct),
             fully_qualified_name=table_configuration.fully_qualified_name,
             has_where_clause=table_configuration.has_where_clause,
             max_failed_rows_number=table_configuration.max_failed_rows_number,

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_table_metadata_query.sql.j2

@@ -43,12 +43,13 @@ SELECT
         SELECT CASE object_type
             WHEN 'TABLE' THEN 'TABLE'
             WHEN 'VIEW' THEN 'VIEW'
+            WHEN 'MATERIALIZED VIEW' THEN 'VIEW'
             ELSE 'UNKNOWN'
         END
         FROM ALL_OBJECTS
         WHERE owner = '{{ object_schema | escape_sql_literal }}'
           AND object_name = '{{ object_name | escape_sql_literal }}'
-          AND object_type IN ('TABLE', 'VIEW')
+          AND object_type IN ('TABLE', 'VIEW', 'MATERIALIZED VIEW')
     ) AS OBJECT_TYPE,
     rc.ROW_COUNT,
     s.STAT_ROW_COUNT

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/public/_internal.py

@@ -73,7 +73,7 @@ from snowflake.snowflake_data_validation.utils.helpers.helper_dataframe import (
     HelperDataFrame,
 )
 from snowflake.snowflake_data_validation.utils.helpers.helper_polars_cast import (
-    build_polars_cast_expression,
+    cast_polars_columns,
 )
 from snowflake.snowflake_data_validation.utils.model.table_column_metadata import (
     TableColumnMetadata,
@@ -582,7 +582,7 @@ def apply_schema_to_polars_dataframe(
     """
     datatypes_mapping = _get_snowflake_to_polars_datatypes_mapping()
 
-    cast_expressions: list[pl.Expr] = []
+    target_types: dict[str, pl.DataType] = {}
     for col_name, col_dtype in column_schema:
         if col_name not in dataframe.columns:
             continue
@@ -592,22 +592,12 @@ def apply_schema_to_polars_dataframe(
             continue
 
         target_type = datatypes_mapping[normalized_type]
-        current_type = dataframe[col_name].dtype
-        if current_type == target_type:
+        if dataframe[col_name].dtype == target_type:
             continue
 
-        cast_expressions.append(
-            build_polars_cast_expression(
-                col_name=col_name,
-                current_type=current_type,
-                target_type=target_type,
-            )
-        )
+        target_types[col_name] = target_type
 
-    if cast_expressions:
-        dataframe = dataframe.with_columns(cast_expressions)
-
-    return dataframe
+    return cast_polars_columns(dataframe, target_types)
 
 
 def process_table_column_metadata_internal(

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/public/validation/metrics_validation.py

@@ -17,6 +17,8 @@
 
 from __future__ import annotations
 
+from collections.abc import Mapping
+
 import pandas as pd
 
 from snowflake.snowflake_data_validation.public.model import (
@@ -89,6 +91,7 @@ def validate_metrics(
     tolerance: float | None = DEFAULT_TOLERANCE,
     relative_tolerance: float | None = None,
     excluded_columns: list[str] | None = None,
+    target_names: Mapping[str, str] | None = None,
 ) -> ValidationResult:
     """Validate metrics by comparing source and target DataFrames.
 
@@ -106,6 +109,10 @@ def validate_metrics(
             When both are set, a pair passes if either check passes (OR).
         excluded_columns: Columns left out of the metrics SELECT (Iceberg v2
             flattened VARCHAR columns); each is reported as one WARNING row.
+        target_names: Each actual source column's actual target name, resolved by
+            the caller (Cloud Data Validation: the column data migration creates).
+            With it the comparison pairs names exactly; without it, through
+            ``column_mappings`` ignoring case, as the standalone CLI does.
 
     Returns:
         ValidationResult: The validation result.
@@ -126,6 +133,7 @@ def validate_metrics(
             column_mappings=column_mappings,
             tolerance=tolerance,
             relative_tolerance=relative_tolerance,
+            target_names=target_names,
         )
         results_dataframe = append_flattened_metrics_exclusions(
             results_dataframe,

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/query/query_generator_base.py

@@ -93,6 +93,7 @@ from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     get_paired_source_column_types,
     get_paired_target_column_types,
 )
+from snowflake.snowflake_data_validation.utils.measure_collation import COUNT_DISTINCT_METRIC
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext, configured_object_name
 from snowflake.snowflake_data_validation.utils.qvci_fallback import (
     resolve_teradata_metadata_is_temporary_table,
@@ -837,6 +838,9 @@ class QueryGeneratorBase(ABC):
         where_clause = table_context.where_clause
         has_where_clause = table_context.has_where_clause
         exclude_metrics = table_context.exclude_metrics
+        # COUNT(DISTINCT) is opt-in (SNOW-4338258). `is True` so test doubles (MagicMock) that never
+        # set the attribute keep the default instead of a truthy Mock.
+        include_count_distinct = getattr(table_context, "include_count_distinct", False) is True
         apply_metric_column_modifier = table_context.apply_metric_column_modifier
 
         select_exprs: list[str] = []
@@ -869,6 +873,8 @@ class QueryGeneratorBase(ABC):
                 metric_name = row[METRIC_COLUMN_KEY]
                 if exclude_metrics and metric_name in METRICS_TO_EXCLUDE:
                     continue
+                if metric_name == COUNT_DISTINCT_METRIC and not include_count_distinct:
+                    continue
                 override = column_overrides.get(metric_name)
                 if override is not None and override.removes_metric:
                     applied_override_names.add(metric_name)
@@ -908,6 +914,8 @@ class QueryGeneratorBase(ABC):
             for metric_name, override in column_overrides.items():
                 if metric_name in applied_override_names or override.removes_metric or not override.metric_query:
                     continue
+                if metric_name == COUNT_DISTINCT_METRIC and not include_count_distinct:
+                    continue
                 metric_expr = self._render_metric_select_expr(
                     col,
                     metric_name,

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/redshift/connector/connector_redshift.py

@@ -15,6 +15,11 @@ import types
 
 from typing import TYPE_CHECKING, Any
 
+from shared.redshift.constants import (
+    TCP_KEEPALIVE_COUNT,
+    TCP_KEEPALIVE_IDLE_SECONDS,
+    TCP_KEEPALIVE_INTERVAL_SECONDS,
+)
 from snowflake.snowflake_data_validation.connector.connector_base import (
     ConnectorBase as BaseConnector,
 )
@@ -46,14 +51,6 @@ that disappears mid-handshake) blocks the caller forever and the retry loop in
 ``connect`` never gets a turn.
 """
 
-# Keepalive probes turn a silently dropped connection into an error. The OS
-# defaults wait about two hours before the first probe; with these, a peer that
-# stops answering is detected in about two minutes, while a live server running
-# a long statement keeps answering the probes and is never cut off.
-TCP_KEEPALIVE_IDLE_SECONDS = 60
-TCP_KEEPALIVE_INTERVAL_SECONDS = 15
-TCP_KEEPALIVE_COUNT = 4
-
 
 @Helper.import_dependencies("redshift")
 def import_redshift():

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/redshift/extractor/templates/redshift_get_columns_metadata.sql.j2

@@ -72,17 +72,24 @@ SELECT
     c.collation_name AS collation_name
 FROM
     SVV_COLUMNS c
+-- The table literals stay in the ON clauses: Redshift does not carry the WHERE
+-- filter on c through the LEFT JOINs, so without them both views read every
+-- constraint in the database. In WHERE they would drop no-PK columns.
 LEFT JOIN
     information_schema.key_column_usage kcu
     ON c.column_name = kcu.column_name
     AND c.table_name = kcu.table_name
     AND c.table_schema = kcu.table_schema
+    AND kcu.table_name = '{{table_name | lower | escape_sql_literal}}'
+    AND kcu.table_schema = '{{schema_name | escape_sql_literal}}'
 LEFT JOIN
     information_schema.table_constraints tco
     ON kcu.constraint_name = tco.constraint_name
     AND kcu.table_schema = tco.table_schema
     AND kcu.table_name = tco.table_name
     AND tco.constraint_type = 'PRIMARY KEY'
+    AND tco.table_name = '{{table_name | lower | escape_sql_literal}}'
+    AND tco.table_schema = '{{schema_name | escape_sql_literal}}'
 WHERE
     c.table_name = '{{table_name | lower | escape_sql_literal}}'
     AND c.table_schema = '{{schema_name | escape_sql_literal}}';

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/redshift/redshift_super_normalization.py

@@ -40,7 +40,6 @@ Python canonicalization as native VARIANT.
 from __future__ import annotations
 
 from collections.abc import Mapping
-from typing import TYPE_CHECKING
 
 from shared.datatypes.common import CommonType
 from shared.datatypes.postgresql import PostgreSQLType
@@ -53,12 +52,7 @@ from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     is_snowflake_varchar_like_type,
     normalize_snowflake_base_type,
 )
-
-
-if TYPE_CHECKING:
-    from snowflake.snowflake_data_validation.utils.model.templates_loader_manager import (
-        TemplatesLoaderManager,
-    )
+from snowflake.snowflake_data_validation.validation.comparison_codec import has_custom_column_normalization
 
 
 # Redshift's own catalog metadata query explicitly UPPER()s data_type, so
@@ -122,23 +116,6 @@ def is_cell_canonicalized_json_pair(
     return normalize_snowflake_base_type(target_snowflake_type) in _REDSHIFT_SUPER_TARGET_BASE_TYPES
 
 
-def has_custom_column_normalization(
-    column_name: str,
-    data_type: str | None,
-    column_normalization_map: Mapping[str, str] | None,
-    templates_loader_manager: TemplatesLoaderManager,
-) -> bool:
-    """Whether a user normalization replaces this column's template.
-
-    Either a per-column rule (*column_normalization_map*) or a workflow
-    override of its data type. Its output need not be JSON, so such a column
-    is compared as rendered rather than canonicalized.
-    """
-    if column_normalization_map and column_name in column_normalization_map:
-        return True
-    return _normalize_redshift_source_type(data_type) in templates_loader_manager.overridden_normalization_types
-
-
 def resolve_snowflake_redshift_super_normalization_key(
     target_snowflake_type: str,
     source_data_type: str | None,

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_datatypes_normalization_templates.yaml

@@ -19,9 +19,6 @@ TIMESTAMP_TZ: REGEXP_REPLACE(TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.
 # Paired PostgreSQL timestamptz: compare UTC instants (Oracle/Teradata use TIMESTAMP_TZ).
 TIMESTAMP_TZ_UTC: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF9') || '+00:00'
 VARCHAR_ISO_TZ: REGEXP_REPLACE(TO_CHAR(TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}")), 'YYYY-MM-DD"T"HH24:MI:SS.FF9TZH:TZM'), 'Z$', '+00:00')
-# Iceberg v2 + compatible target overlays (microsecond / FF6).
-TIMESTAMP_LTZ_FF6: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF6')
-TIMESTAMP_LTZ_UTC_FF6: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF6')
 # Canonical UTC-instant form for every Snowflake TIMESTAMP_LTZ mirror of a
 # UTC-instant TIMESTAMPTZ source (Redshift / PostgreSQL), reached from either
 # the native path (iceberg_spec and storage_mode both None, target
@@ -29,10 +26,6 @@ TIMESTAMP_LTZ_UTC_FF6: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-
 # compatible overlay (TIMESTAMP_LTZ(6)). Source side emits the same
 # FF9-padded +00:00 string.
 TIMESTAMP_LTZ_UTC_FF9_OFFSET: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF9') || '+00:00'
-TIMESTAMP_NTZ_FF6: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF6')
-TIMESTAMP_TZ_FF6: REGEXP_REPLACE(TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF6TZH:TZM'), 'Z$', '+00:00')
-TIMESTAMP_TZ_UTC_FF6: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF6') || '+00:00'
-VARCHAR_ISO_TZ_FF6: REGEXP_REPLACE(TO_CHAR(TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}")), 'YYYY-MM-DD"T"HH24:MI:SS.FF6TZH:TZM'), 'Z$', '+00:00')
 # Complex values flattened to VARCHAR under Iceberg compatible DDL retain the
 # exact Redshift JSON_SERIALIZE text staged by UNLOAD. Compare that migration
 # representation directly: PARSE_JSON + TO_JSON recursively sorts object keys
@@ -82,6 +75,9 @@ PERIOD(TIMESTAMP_TZ): "REPLACE(REPLACE('[' || TRIM(TO_CHAR(PERIOD_BEGIN(\"{{ col
 PERIOD(TIMESTAMP): "'[' || TRIM(TO_CHAR(PERIOD_BEGIN(\"{{ col_name }}\"), 'YYYY-MM-DD HH24:MI:SS.FF6')) || ', ' || TRIM(TO_CHAR(PERIOD_END(\"{{ col_name }}\"), 'YYYY-MM-DD HH24:MI:SS.FF6')) || ')'"
 PERIOD: TRIM(CAST("{{ col_name }}" AS VARCHAR))
 VARCHAR: '"{{ col_name }}"'
+# Text mirror of a SQL Server / Synapse UNIQUEIDENTIFIER: T-SQL renders a GUID in uppercase,
+# so uppercase the mirror, which keeps the case it was loaded with.
+VARCHAR_UNIQUEIDENTIFIER: UPPER("{{ col_name }}")
 # Spatial → WKT for L3 hashing. One _wkt_despace.sql.j2 base chain, parameterized on
 # wkt_input, so native geo (ST_ASWKT) and flattened geo (stored WKT text) can't drift,
 # and Snowflake matches PostGIS / SQL Server / Oracle / Teradata / Redshift / BigQuery.

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_table_metadata_query.sql.j2

@@ -18,7 +18,9 @@ SELECT
     (
         SELECT CASE TABLE_TYPE
             WHEN 'BASE TABLE' THEN 'TABLE'
+            WHEN 'LOCAL TEMPORARY' THEN 'TABLE'
             WHEN 'VIEW' THEN 'VIEW'
+            WHEN 'MATERIALIZED VIEW' THEN 'VIEW'
             ELSE 'UNKNOWN'
         END
         FROM {{ object_database_ref }}.INFORMATION_SCHEMA.TABLES

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/snowflake/query/query_generator_snowflake.py

@@ -70,6 +70,9 @@ from snowflake.snowflake_data_validation.sqlserver.sqlserver_tz_normalization im
 from snowflake.snowflake_data_validation.sqlserver.sqlserver_tz_normalization import (
     resolve_snowflake_timestamp_tz_metrics_key as resolve_sqlserver_timestamp_tz_metrics_key,
 )
+from snowflake.snowflake_data_validation.sqlserver.sqlserver_uniqueidentifier_normalization import (
+    resolve_snowflake_uniqueidentifier_normalization_key,
+)
 from snowflake.snowflake_data_validation.teradata.teradata_iso_normalization import (
     resolve_snowflake_timestamp_tz_normalization_key as resolve_teradata_timestamp_tz_normalization_key,
 )
@@ -95,6 +98,7 @@ from snowflake.snowflake_data_validation.utils.measure_collation import (
     TARGET_CHARACTER_TYPES,
 )
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
+from snowflake.snowflake_data_validation.utils.model.templates_loader_manager import TemplatesLoaderManager
 from snowflake.snowflake_data_validation.utils.period_common import (
     resolve_snowflake_period_normalization_key,
 )
@@ -110,6 +114,7 @@ from snowflake.snowflake_data_validation.utils.vector_normalization import (
 from snowflake.snowflake_data_validation.validation.comparison_codec import (
     ComparisonCodec,
     apply_text_codec_to_renderer_templates,
+    has_custom_column_normalization,
     resolve_column_normalization_template,
 )
 
@@ -128,9 +133,20 @@ _PAIRED_SOURCE_NORMALIZATION_KEY_RESOLVERS: dict[Platform, tuple[_NormalizationK
     ),
     Platform.DB2: (resolve_snowflake_db2_lob_normalization_key,),
     Platform.TERADATA: (resolve_teradata_timestamp_tz_normalization_key,),
-    Platform.SQLSERVER: (resolve_sqlserver_datetimeoffset_ltz_utc_normalization_key,),
+    Platform.SQLSERVER: (
+        resolve_sqlserver_datetimeoffset_ltz_utc_normalization_key,
+        resolve_snowflake_uniqueidentifier_normalization_key,
+    ),
+    Platform.AZURE_SYNAPSE: (resolve_snowflake_uniqueidentifier_normalization_key,),
 }
 
+# Resolvers that only re-render the mirror's own text, with no source-side template
+# paired to their key (as a LOB digest is), so a workflow or custom templates YAML
+# override of the mirror's type replaces them: that override is what the user hashes.
+_TYPE_OVERRIDE_YIELDING_RESOLVERS: frozenset[_NormalizationKeyResolver] = frozenset(
+    {resolve_snowflake_uniqueidentifier_normalization_key},
+)
+
 
 # Per-source-platform resolvers for the key of a Snowflake ``TIMESTAMP_TZ`` mirror's
 # L2 min/max normalization, ``(target_key, paired_source_type) -> target_key``.
@@ -146,10 +162,25 @@ def resolve_paired_source_normalization_key(
     paired_source_platform: Platform | None,
     target_key: str,
     paired_source_type: str | None,
+    templates_loader_manager: TemplatesLoaderManager,
 ) -> str:
-    """Apply the paired source platform's registered normalization-key resolvers."""
+    """Apply the paired source platform's registered normalization-key resolvers.
+
+    A resolver's key keeps the incoming one when the templates lack it (a custom
+    templates YAML copied before the key existed), so that YAML's own template for the
+    type applies rather than ``DEFAULT``, and, for a
+    :data:`_TYPE_OVERRIDE_YIELDING_RESOLVERS` resolver, when the user overrode the
+    incoming type.
+    """
+    templates = templates_loader_manager.datatypes_normalization_templates
+    overridden_types = templates_loader_manager.overridden_normalization_types
     for resolve in _PAIRED_SOURCE_NORMALIZATION_KEY_RESOLVERS.get(paired_source_platform, ()):
-        target_key = resolve(target_key, paired_source_type)
+        resolved = resolve(target_key, paired_source_type)
+        if resolved == target_key or resolved not in templates:
+            continue
+        if resolve in _TYPE_OVERRIDE_YIELDING_RESOLVERS and normalize_snowflake_base_type(target_key) in overridden_types:
+            continue
+        target_key = resolved
     return target_key
 
 
@@ -268,9 +299,17 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
         *,
         paired_source_platform: Platform | None = None,
         paired_source_type: str | None = None,
+        has_custom_normalization: bool = False,
     ) -> str:
+        """Render *template* for *column*; a FLOAT column hashes its binary64 tuple.
+
+        ``has_custom_normalization`` marks *template* as the user's (a column's
+        ``targetExpression``, or a workflow or custom templates YAML override of its
+        type): it is rendered as written, FLOAT included, as the source side renders
+        its own.
+        """
         target_base, _, _ = self._parse_type_precision_scale(getattr(column, "data_type", "") or "")
-        if target_base in SNOWFLAKE_FLOAT_TYPE_NAMES:
+        if target_base in SNOWFLAKE_FLOAT_TYPE_NAMES and not has_custom_normalization:
             return build_snowflake_float_bit_canonical_expr(write_identifier_for_sql(column.name))
         if (
             paired_source_platform == Platform.TERADATA
@@ -321,6 +360,7 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
                 table_context.paired_source_platform,
                 data_type,
                 paired.get(column_name),
+                table_context.templates_loader_manager,
             )
             from snowflake.snowflake_data_validation.redshift.redshift_super_normalization import (
                 SNOWFLAKE_REDSHIFT_SUPER_NORMALIZATION_KEY,
@@ -411,6 +451,12 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
                 column,
                 paired_source_platform=table_context.paired_source_platform,
                 paired_source_type=paired_source_types.get(column.name),
+                has_custom_normalization=has_custom_column_normalization(
+                    column.name,
+                    column.data_type,
+                    column_normalization_map,
+                    table_context.templates_loader_manager,
+                ),
             )
             renderer_templates[column.name] = rendered
         renderer_templates = apply_text_codec_to_renderer_templates(
@@ -706,7 +752,6 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
             SNOWFLAKE_REDSHIFT_SUPER_JSON_ONLY_NORMALIZATION_KEY,
             SNOWFLAKE_REDSHIFT_SUPER_NORMALIZATION_KEY,
             SNOWFLAKE_VARIANT_JSON_NORMALIZATION_KEY,
-            has_custom_column_normalization,
             is_cell_canonicalized_json_pair,
             resolve_snowflake_redshift_super_normalization_key,
         )

{snowflake_data_validation-1.44.1 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/sqlserver/extractor/templates/sqlserver_datatypes_normalization_templates.yaml

@@ -39,6 +39,9 @@ TEXT: CONVERT(VARCHAR(MAX), "{{ col_name }}")
 # maps 'time' to .NET TimeSpan, where 'HH' is the literal "HH". '00' pads to nine
 # digits, then LEFT cuts to the precision both sides share ({{ time_text_length }}).
 TIME: LEFT(SUBSTRING(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 12, 16) + '00', {{ time_text_length }})
+# Iceberg compatible TIME stored as VARCHAR (CETAS): the extract's own text, TIME(7)
+# widened then '00', nine digits whatever the scale. '+' keeps NULL as NULL.
+TIME_ICEBERG: CAST(CAST("{{ col_name }}" AS TIME(7)) AS VARCHAR(16)) + '00'
 TIMESTAMP: UPPER(CONVERT(VARCHAR(MAX), CAST("{{ col_name }}" AS VARBINARY(8)), 2))
 ROWVERSION: UPPER(CONVERT(VARCHAR(MAX), CAST("{{ col_name }}" AS VARBINARY(8)), 2))
 TINYINT: CONVERT(VARCHAR(20), "{{ col_name }}")