snowflake-data-validation 1.42.2__tar.gz → 1.42.3__tar.gz

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.42.2
+Version: 1.42.3
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -190,6 +190,18 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.42.3
+
+**Improvements**
+
+- Improved comparison of clock values stored as `VARCHAR` in Iceberg metrics.
+- Added support for staging Redshift `REAL` NaN and infinity values without aborting the extract.
+
+**Bug fixes**
+
+- Fixed false L2 mismatch on T-SQL integer `STDDEV` columns.
+- Fixed Redshift `TIMETZ` source-vs-target value mismatch in L2 validation.
+
 ### v1.42.2
 
 No user-facing changes in this release.

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/README.md

@@ -99,6 +99,18 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.42.3
+
+**Improvements**
+
+- Improved comparison of clock values stored as `VARCHAR` in Iceberg metrics.
+- Added support for staging Redshift `REAL` NaN and infinity values without aborting the extract.
+
+**Bug fixes**
+
+- Fixed false L2 mismatch on T-SQL integer `STDDEV` columns.
+- Fixed Redshift `TIMETZ` source-vs-target value mismatch in L2 validation.
+
 ### v1.42.2
 
 No user-facing changes in this release.

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml

@@ -63,8 +63,13 @@ INT:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
     metric_column_modifier: "% 2097143"
+  # Exact integer variance (SNOW-4229286): n*SUM(x^2) - SUM(x)^2 in DECIMAL(38, 0), only the
+  # final SQRT in FLOAT. STDEVP over FLOAT cancels catastrophically when values are large
+  # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
+  # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
+  # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
   stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
+    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
@@ -92,8 +97,13 @@ SMALLINT:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
     metric_column_modifier: "% 2097143"
+  # Exact integer variance (SNOW-4229286): n*SUM(x^2) - SUM(x)^2 in DECIMAL(38, 0), only the
+  # final SQRT in FLOAT. STDEVP over FLOAT cancels catastrophically when values are large
+  # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
+  # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
+  # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
   stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
+    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
@@ -121,8 +131,13 @@ TINYINT:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
     metric_column_modifier: "% 2097143"
+  # Exact integer variance (SNOW-4229286): n*SUM(x^2) - SUM(x)^2 in DECIMAL(38, 0), only the
+  # final SQRT in FLOAT. STDEVP over FLOAT cancels catastrophically when values are large
+  # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
+  # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
+  # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
   stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
+    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/src/snowflake/snowflake_data_validation/redshift/extractor/templates/redshift_column_metrics_templates.yaml

@@ -666,13 +666,17 @@ TIME WITHOUT TIME ZONE:
      metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
      metric_return_datatype: BIGINT
 
+# TIMETZ migrates to TIMESTAMP_TZ on a 1970-01-01 sentinel (Snowflake has no
+# offset-aware TIME); render min/max like the L3 template so both sides match.
 TIMETZ:
   min: 
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: TIMETZ
+    metric_normalization: "'1970-01-01T' || TO_CHAR((CURRENT_DATE + {{ metric_query }}) AT TIME ZONE 'UTC', 'HH24:MI:SS.US') || '000+00:00'"
   max: 
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: TIMETZ
+    metric_normalization: "'1970-01-01T' || TO_CHAR((CURRENT_DATE + {{ metric_query }}) AT TIME ZONE 'UTC', 'HH24:MI:SS.US') || '000+00:00'"
   count_null:
      metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
      metric_return_datatype: BIGINT
@@ -681,9 +685,11 @@ TIME WITH TIME ZONE:
   min: 
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: TIMETZ
+    metric_normalization: "'1970-01-01T' || TO_CHAR((CURRENT_DATE + {{ metric_query }}) AT TIME ZONE 'UTC', 'HH24:MI:SS.US') || '000+00:00'"
   max: 
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: TIMETZ
+    metric_normalization: "'1970-01-01T' || TO_CHAR((CURRENT_DATE + {{ metric_query }}) AT TIME ZONE 'UTC', 'HH24:MI:SS.US') || '000+00:00'"
   count_null:
      metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
      metric_return_datatype: BIGINT

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/src/snowflake/snowflake_data_validation/redshift/query/query_generator_redshift.py

@@ -112,9 +112,17 @@ def _ln_power_tme_canonical_expr(col_ref: str) -> str:
         f"ELSE SUBSTRING({mantissa_base}, 1, {decimal_pos} - 1) || "
         f"CASE WHEN {fraction_trimmed} = '' THEN '' ELSE '.' || {fraction_trimmed} END END"
     )
+    # NaN and ±Inf must return before the NUMERIC cast. Redshift raises 8001
+    # (numeric_bound.cpp, "scale float to decimal") if either reaches it.
+    # A stored REAL NaN does not compare equal to ``'NaN'::float4``; its text does.
+    # ±Inf does compare equal to the float8 literals. Snowflake TO_DOUBLE accepts
+    # the texts below.
     return (
         f"CASE WHEN {col_ref} IS NULL THEN NULL "
         f"WHEN {col_ref} = 0 THEN '0E0' "
+        f"WHEN LOWER(CAST({col_ref} AS VARCHAR)) = 'nan' THEN 'nan' "
+        f"WHEN {col_ref} = 'Infinity'::float8 THEN 'inf' "
+        f"WHEN {col_ref} = '-Infinity'::float8 THEN '-inf' "
         f"ELSE UPPER({mantissa_trimmed}) || 'E' || TRIM(CAST({exp_int} AS VARCHAR)) END"
     )
 

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_column_metrics_templates.yaml

@@ -186,6 +186,51 @@ VARCHAR_ISO_TZ:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: NUMBER
 
+# VARCHAR mirror of a source clock. Parse the text, then emit FF6 and drop an
+# all-zero fraction (the literal '.000000' that TO_CHAR FF6 always writes).
+# TIME WITH TIME ZONE uses its own parser: TRY_TO_TIMESTAMP_TZ without a format
+# returns NULL for 'HH:MI:SS±HH:MM' text.
+VARCHAR_TIME:
+  min:
+    metric_query: MIN(TRY_TO_TIME(TRIM("{{ col_name }}")))
+    metric_return_datatype: VARCHAR
+    metric_normalization: REPLACE(TO_CHAR({{ metric_query }}, 'HH24:MI:SS.FF6'), '.000000', '')
+  max:
+    metric_query: MAX(TRY_TO_TIME(TRIM("{{ col_name }}")))
+    metric_return_datatype: VARCHAR
+    metric_normalization: REPLACE(TO_CHAR({{ metric_query }}, 'HH24:MI:SS.FF6'), '.000000', '')
+  count_null:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
+    metric_return_datatype: NUMBER
+
+VARCHAR_TIMESTAMP:
+  min:
+    metric_query: MIN(TRY_TO_TIMESTAMP_NTZ(TRIM("{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF9'))
+    metric_return_datatype: VARCHAR
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
+  max:
+    metric_query: MAX(TRY_TO_TIMESTAMP_NTZ(TRIM("{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF9'))
+    metric_return_datatype: VARCHAR
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
+  count_null:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
+    metric_return_datatype: NUMBER
+
+# Order by UTC time-of-day. A plain MIN of the parsed instant uses today's date
+# and can cross midnight, so the formatted value would keep the wrong offset.
+VARCHAR_TIME_TZ:
+  min:
+    metric_query: MIN_BY(TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}"), 'HH24:MI:SS.FF6TZH:TZM'), TIME(CONVERT_TIMEZONE('UTC', TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}"), 'HH24:MI:SS.FF6TZH:TZM'))))
+    metric_return_datatype: VARCHAR
+    metric_normalization: REPLACE(REGEXP_REPLACE(TO_CHAR({{ metric_query }}, 'HH24:MI:SS.FF6TZH:TZM'), 'Z$', '+00:00'), '.000000', '')
+  max:
+    metric_query: MAX_BY(TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}"), 'HH24:MI:SS.FF6TZH:TZM'), TIME(CONVERT_TIMEZONE('UTC', TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}"), 'HH24:MI:SS.FF6TZH:TZM'))))
+    metric_return_datatype: VARCHAR
+    metric_normalization: REPLACE(REGEXP_REPLACE(TO_CHAR({{ metric_query }}, 'HH24:MI:SS.FF6TZH:TZM'), 'Z$', '+00:00'), '.000000', '')
+  count_null:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
+    metric_return_datatype: NUMBER
+
 # Paired PostgreSQL BYTEA / SQL Server VARBINARY mirrors.
 BINARY:
   min:

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/src/snowflake/snowflake_data_validation/snowflake/query/query_generator_snowflake.py

@@ -15,6 +15,7 @@
 
 
 import logging
+import re
 
 import pandas as pd
 
@@ -50,6 +51,7 @@ from snowflake.snowflake_data_validation.teradata.teradata_iso_normalization imp
 )
 from snowflake.snowflake_data_validation.utils.constants import (
     COL_NAME_NO_QUOTES_PLACEHOLDER,
+    COL_NAME_QUOTES_PLACEHOLDER,
     EXPLICIT_METRIC_NORMALIZATION_COLUMN_KEY,
     METRIC_QUERY_PLACEHOLDER,
     NORMALIZATION_COLUMN_KEY,
@@ -61,6 +63,7 @@ from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     normalize_snowflake_base_type,
     resolve_snowflake_interval_normalization_key,
     resolve_snowflake_iso_tz_varchar_metrics_type,
+    resolve_snowflake_varchar_temporal_metrics_type,
 )
 from snowflake.snowflake_data_validation.utils.measure_collation import (
     COUNT_DISTINCT_METRIC,
@@ -133,6 +136,23 @@ def build_snowflake_float_bit_canonical_expr(col_ref: str) -> str:
     return build_binary64_canonical_expr(col_ref)
 
 
+# Paired T-SQL sources whose L2 stddev is exact integer math (SNOW-4229286). Snowflake must
+# match it: STDDEV_POP over FLOAT drifts too (0.0055 on 3M identical inv_date_sk values).
+_EXACT_STDDEV_SOURCE_PLATFORMS = (Platform.SQLSERVER, Platform.AZURE_SYNAPSE)
+_EXACT_STDDEV_SOURCE_TYPES = frozenset({"INT", "SMALLINT", "TINYINT"})
+_FLOAT_STDDEV_POP_RE = re.compile(r"STDDEV_POP\(CAST\((?P<operand>.+) AS FLOAT\)\)")
+
+
+def _exact_integer_stddev(metric_template: str) -> str | None:
+    """Rewrite ``STDDEV_POP(CAST(x AS FLOAT))`` as exact ``SQRT(n*SUM(x^2) - SUM(x)^2) / n``."""
+    match = _FLOAT_STDDEV_POP_RE.fullmatch(metric_template.strip())
+    if not match:
+        return None
+    col = COL_NAME_QUOTES_PLACEHOLDER
+    x = f"CAST({match.group('operand')} AS NUMBER(38, 0))"
+    return f"SQRT(CAST(COUNT({col}) * SUM({x} * {x}) - SUM({x}) * SUM({x}) AS FLOAT)) / NULLIF(COUNT({col}), 0)"
+
+
 class QueryGeneratorSnowflake(QueryGeneratorBase):
     """Snowflake-specific implementation of query generator."""
 
@@ -417,6 +437,15 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
         )
         if iso_tz_type:
             return iso_tz_type
+        temporal_text_type = resolve_snowflake_varchar_temporal_metrics_type(
+            col.data_type,
+            paired_source,
+            paired_source_platform=table_context.paired_source_platform,
+            iceberg_spec=table_context.iceberg_spec,
+            storage_mode=table_context.storage_mode,
+        )
+        if temporal_text_type:
+            return temporal_text_type
         if target_base in {"TEXT", "STRING"}:
             return "VARCHAR"
         return target_base or col.data_type
@@ -438,6 +467,8 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
         if metric_name == "count_null":
             return metric_template
         paired_source_type = get_paired_source_column_types(table_context).get(col.name)
+        if metric_name == "stddev":
+            return self._paired_stddev_template(metric_template, paired_source_type, table_context)
         if not is_blank_padded_char_type(paired_source_type or ""):
             return metric_template
         if not should_rtrim_snowflake_for_padded_source(col.data_type, paired_source_type, metric_template):
@@ -445,6 +476,19 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
         quoted_col = f'"{COL_NAME_NO_QUOTES_PLACEHOLDER}"'
         return metric_template.replace(quoted_col, wrap_rtrim(quoted_col))
 
+    @staticmethod
+    def _paired_stddev_template(
+        metric_template: str,
+        paired_source_type: str | None,
+        table_context: TableContext,
+    ) -> str:
+        """Use the exact integer stddev the paired T-SQL source computes, else keep the template."""
+        if getattr(table_context, "paired_source_platform", None) not in _EXACT_STDDEV_SOURCE_PLATFORMS:
+            return metric_template
+        if (paired_source_type or "").upper().split("(")[0].strip() not in _EXACT_STDDEV_SOURCE_TYPES:
+            return metric_template
+        return _exact_integer_stddev(metric_template) or metric_template
+
     def _resolve_metric_normalization(
         self,
         col,

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/src/snowflake/snowflake_data_validation/sqlserver/extractor/templates/sqlserver_column_metrics_templates.yaml

@@ -260,10 +260,13 @@ INT:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
     metric_column_modifier: "% 2097143"
-  # Cast STDEVP input to FLOAT to prevent SUM-of-squares overflow
-  # (SNOW-3443973).
-  stddev: 
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
+  # Exact integer variance (SNOW-4229286): n*SUM(x^2) - SUM(x)^2 in DECIMAL(38, 0), only the
+  # final SQRT in FLOAT. STDEVP over FLOAT cancels catastrophically when values are large
+  # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
+  # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
+  # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
+  stddev:
+    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
@@ -479,8 +482,13 @@ SMALLINT:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
     metric_column_modifier: "% 2097143"
-  stddev: 
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
+  # Exact integer variance (SNOW-4229286): n*SUM(x^2) - SUM(x)^2 in DECIMAL(38, 0), only the
+  # final SQRT in FLOAT. STDEVP over FLOAT cancels catastrophically when values are large
+  # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
+  # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
+  # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
+  stddev:
+    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
@@ -572,8 +580,13 @@ TINYINT:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
     metric_column_modifier: "% 2097143"
+  # Exact integer variance (SNOW-4229286): n*SUM(x^2) - SUM(x)^2 in DECIMAL(38, 0), only the
+  # final SQRT in FLOAT. STDEVP over FLOAT cancels catastrophically when values are large
+  # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
+  # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
+  # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
   stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
+    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/src/snowflake/snowflake_data_validation/utils/iso_dv_common.py

@@ -14,6 +14,8 @@ from __future__ import annotations
 
 import re
 
+from enum import StrEnum
+
 import pandas as pd
 
 from snowflake.snowflake_data_validation.utils.constants import (
@@ -23,6 +25,9 @@ from snowflake.snowflake_data_validation.utils.constants import (
     NOT_APPLICABLE_CRITERIA_VALUE,
     Platform,
 )
+from snowflake.snowflake_data_validation.utils.iceberg_dv_common import (
+    is_iceberg_compatible_dv_profile,
+)
 from snowflake.snowflake_data_validation.utils.interval_common import (
     SNOWFLAKE_INTERVAL_DV_FAMILIES,
     normalize_interval_datatype_key,
@@ -39,6 +44,49 @@ _SNOWFLAKE_TIMESTAMP_FAMILY = frozenset({"TIMESTAMP_NTZ", "TIMESTAMP_TZ", "TIMES
 PAIRED_TARGET_COLUMN_TYPES_ATTR = "paired_target_column_types"
 PAIRED_SOURCE_COLUMN_TYPES_ATTR = "paired_source_column_types"
 VARCHAR_ISO_TZ_METRICS_TYPE = "VARCHAR_ISO_TZ"
+VARCHAR_TIME_METRICS_TYPE = "VARCHAR_TIME"
+VARCHAR_TIMESTAMP_METRICS_TYPE = "VARCHAR_TIMESTAMP"
+VARCHAR_TIME_TZ_METRICS_TYPE = "VARCHAR_TIME_TZ"
+
+
+class _IsoClockTextKey(StrEnum):
+    """Lookup key for a source clock whose extract writes the template text.
+
+    ``TIME_TZ`` is synthesized from ``TIME … WITH TIME ZONE``. The other
+    members match the leading catalog token.
+    """
+
+    TIME = "TIME"
+    TIME_TZ = "TIME_TZ"
+    TIMESTAMP = "TIMESTAMP"
+    DATETIME = "DATETIME"
+    DATETIME2 = "DATETIME2"
+    SMALLDATETIME = "SMALLDATETIME"
+
+
+# SQL Server and Azure Synapse emit the same datetime text canon.
+_SQLSERVER_DATETIME_CLOCK_METRICS: dict[_IsoClockTextKey, str] = {
+    _IsoClockTextKey.DATETIME: VARCHAR_TIMESTAMP_METRICS_TYPE,
+    _IsoClockTextKey.DATETIME2: VARCHAR_TIMESTAMP_METRICS_TYPE,
+    _IsoClockTextKey.SMALLDATETIME: VARCHAR_TIMESTAMP_METRICS_TYPE,
+}
+
+# Redshift TIME and PostgreSQL TIMESTAMP are absent: their extract is not that canon.
+_ISO_CLOCK_TEXT_METRICS: dict[Platform, dict[_IsoClockTextKey, str]] = {
+    Platform.TERADATA: {
+        _IsoClockTextKey.TIME: VARCHAR_TIME_METRICS_TYPE,
+        _IsoClockTextKey.TIME_TZ: VARCHAR_TIME_TZ_METRICS_TYPE,
+        _IsoClockTextKey.TIMESTAMP: VARCHAR_TIMESTAMP_METRICS_TYPE,
+    },
+    Platform.REDSHIFT: {
+        _IsoClockTextKey.TIMESTAMP: VARCHAR_TIMESTAMP_METRICS_TYPE,
+    },
+    Platform.ORACLE: {
+        _IsoClockTextKey.TIMESTAMP: VARCHAR_TIMESTAMP_METRICS_TYPE,
+    },
+    Platform.SQLSERVER: _SQLSERVER_DATETIME_CLOCK_METRICS,
+    Platform.AZURE_SYNAPSE: _SQLSERVER_DATETIME_CLOCK_METRICS,
+}
 
 _MAPPED_VARCHAR_LENGTH_RE = re.compile(
     r"^\s*VARCHAR\s*\(\s*(\d+)\s*\)\s*$",
@@ -113,6 +161,53 @@ def resolve_snowflake_iso_tz_varchar_metrics_type(
     return None
 
 
+def _iso_clock_text_key(source_data_type: str) -> _IsoClockTextKey | None:
+    """Return the extract-canon key for a source clock label."""
+    upper = source_data_type.strip().upper()
+    if not upper or upper.startswith("PERIOD") or "WITH LOCAL TIME ZONE" in upper:
+        return None
+    leading = upper.split("(", 1)[0].strip().split()[0]
+    has_time_zone = "WITH TIME ZONE" in upper
+    if leading == _IsoClockTextKey.TIME and has_time_zone:
+        return _IsoClockTextKey.TIME_TZ
+    if leading == _IsoClockTextKey.TIME:
+        return _IsoClockTextKey.TIME
+    if leading == _IsoClockTextKey.TIMESTAMP and not has_time_zone:
+        return _IsoClockTextKey.TIMESTAMP
+    if leading in _SQLSERVER_DATETIME_CLOCK_METRICS:
+        return _IsoClockTextKey(leading)
+    return None
+
+
+def resolve_snowflake_varchar_temporal_metrics_type(
+    target_snowflake_type: str,
+    source_data_type: str | None,
+    *,
+    paired_source_platform: Platform | None = None,
+    iceberg_spec: str | None = None,
+    storage_mode: str | None = None,
+) -> str | None:
+    """Return the VARCHAR clock metric key when this extract writes that canon.
+
+    Iceberg compatible stores some source clocks as VARCHAR. The parser is
+    selected only for a platform whose extract emits the template text.
+    Any other pairing stays on VARCHAR length metrics.
+    """
+    if not source_data_type:
+        return None
+    if not is_iceberg_compatible_dv_profile(iceberg_spec, storage_mode):
+        return None
+    if normalize_snowflake_base_type(target_snowflake_type) not in _VARCHAR_LIKE_SNOWFLAKE_BASE_TYPES:
+        return None
+    clock_key = _iso_clock_text_key(source_data_type)
+    if clock_key is None or paired_source_platform is None:
+        return None
+    platform_metrics = _ISO_CLOCK_TEXT_METRICS.get(paired_source_platform)
+    if platform_metrics is None:
+        return None
+    return platform_metrics.get(clock_key)
+
+
 def is_snowflake_varchar_like_type(data_type: str | None) -> bool:
     """Return True for any Snowflake VARCHAR-family catalog label.
 

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/tests/unit/Redshift/test_redshift_timetz_sentinel_normalization.py

@@ -96,3 +96,41 @@ def test_timetz_l1_enrichment_clears_source_scale() -> None:
     )
 
     assert enriched["NUMERIC_SCALE"].isna().all()
+
+
+_METRICS_YAML = _TEMPLATES / "redshift_column_metrics_templates.yaml"
+
+
+@pytest.fixture(scope="module")
+def metrics_templates() -> dict[str, dict[str, dict[str, str]]]:
+    """Load Redshift L2 column metrics templates from disk."""
+    with _METRICS_YAML.open(encoding="utf-8") as handle:
+        return yaml.safe_load(handle)
+
+
+@pytest.mark.parametrize("metric", ["min", "max"])
+@pytest.mark.parametrize("redshift_type", _TIMETZ_ALIASES)
+def test_timetz_l2_min_max_emit_unix_epoch_sentinel_utc_iso(
+    metrics_templates: dict[str, dict[str, dict[str, str]]],
+    redshift_type: str,
+    metric: str,
+) -> None:
+    """L2 min/max must render the same sentinel TIMESTAMP_TZ text as L3.
+
+    Snowflake reports MIN/MAX of the TIMESTAMP_TZ mirror as
+    ``1970-01-01T12:00:00.000000000+00:00``; a bare ``12:00:00+00:00`` from
+    Redshift would flag equal values as different.
+    """
+    from snowflake.snowflake_data_validation.query.query_generator_base import (
+        QueryGeneratorBase,
+    )
+
+    entry = metrics_templates[redshift_type][metric]
+    normalization = entry.get("metric_normalization")
+    assert normalization, f"{redshift_type}.{metric} has no metric_normalization"
+    aggregate = f'{metric.upper()}("COL_TIMETZ")'
+    rendered = QueryGeneratorBase._wrap_metric_with_normalization(normalization, aggregate)
+    assert rendered == (
+        f"'1970-01-01T' || TO_CHAR((CURRENT_DATE + {aggregate}) AT TIME ZONE 'UTC', "
+        "'HH24:MI:SS.US') || '000+00:00'"
+    )

snowflake_data_validation-1.42.3/tests/unit/sqlserver/test_tsql_integer_stddev_exact.py

@@ -0,0 +1,140 @@
+# Copyright 2026 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+"""T-SQL integer stddev is computed exactly, not by STDEVP over FLOAT (SNOW-4229286).
+
+STDEVP(CAST(col AS FLOAT)) cancels catastrophically when values are large and the spread
+small: dmvf-benchmark #158 saw inventory.inv_date_sk stddev swing 0-5.5 per partition on
+SQL Server while Snowflake reported a steady 3.5 on identical data (MIN/MAX/SUM/AVG all
+matched). 32-bit integer columns now get n*SUM(x^2) - SUM(x)^2 in DECIMAL(38, 0), with only
+the final SQRT in FLOAT, and the Snowflake side of those pairings does the same in NUMBER(38, 0)
+because its STDDEV_POP over FLOAT drifts as well (0.0055 on 3M identical values).
+"""
+
+from __future__ import annotations
+
+import math
+
+from types import SimpleNamespace
+
+from pathlib import Path
+
+import pytest
+import yaml
+
+from jinja2 import Template
+
+from snowflake.snowflake_data_validation.snowflake.query.query_generator_snowflake import (
+    QueryGeneratorSnowflake,
+)
+from snowflake.snowflake_data_validation.utils.constants import Platform
+
+
+TEMPLATES_ROOT = Path(__file__).resolve().parents[3] / "src/snowflake/snowflake_data_validation"
+TEMPLATE_FILES = {
+    "sqlserver": TEMPLATES_ROOT / "sqlserver/extractor/templates/sqlserver_column_metrics_templates.yaml",
+    "azure_synapse": TEMPLATES_ROOT
+    / "azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml",
+}
+INTEGER_TYPES = ("INT", "SMALLINT", "TINYINT")
+MODIFIER = "% 2097143"
+
+
+def _stddev(platform: str, section: str) -> dict:
+    return yaml.safe_load(TEMPLATE_FILES[platform].read_text())[section]["stddev"]
+
+
+@pytest.mark.parametrize("platform", TEMPLATE_FILES)
+@pytest.mark.parametrize("section", INTEGER_TYPES)
+def test_integer_stddev_uses_exact_decimal_variance(platform, section):
+    metric = _stddev(platform, section)
+    query = metric["metric_query"]
+    assert "STDEVP" not in query, f"{platform}.{section} still uses float STDEVP: {query}"
+    assert "AS DECIMAL(38, 0)" in query
+    assert "COUNT_BIG(" in query
+    assert "NULLIF(COUNT_BIG(" in query, "empty partition must yield NULL, not divide by zero"
+    assert metric["metric_column_modifier"] == MODIFIER
+    assert metric["metric_return_datatype"] == "FLOAT"
+
+
+@pytest.mark.parametrize("platform", TEMPLATE_FILES)
+def test_bigint_stddev_stays_on_float(platform):
+    # Raw 64-bit squares overflow DECIMAL(38) once the modifier is turned off.
+    assert _stddev(platform, "BIGINT")["metric_query"].startswith("STDEVP(CAST(")
+
+
+@pytest.mark.parametrize("platform", TEMPLATE_FILES)
+def test_rendered_query_applies_modifier_to_every_operand(platform):
+    rendered = Template(_stddev(platform, "INT")["metric_query"]).render(
+        col_name="INV_DATE_SK", modifier=MODIFIER
+    )
+    assert rendered.count(f'CAST("INV_DATE_SK" {MODIFIER} AS DECIMAL(38, 0))') == 4
+    assert rendered.count('COUNT_BIG("INV_DATE_SK")') == 2
+
+
+def test_formula_matches_population_stddev_where_float_cancels():
+    # Two weekly date keys, as in an inventory partition: exact answer is 3.5.
+    values = [2451000 % 2097143] * 3_000_000 + [2451007 % 2097143] * 3_000_000
+    n = len(values)
+    s = sum(values)
+    ss = sum(v * v for v in values)
+    assert math.sqrt(n * ss - s * s) / n == pytest.approx(3.5, abs=1e-9)
+
+
+SNOWFLAKE_TEMPLATES = (
+    TEMPLATES_ROOT / "snowflake/extractor/templates/snowflake_column_metrics_templates.yaml"
+)
+# Other-pairing cases only need a shape the regex accepts. The rewrite case loads
+# NUMBER.stddev from YAML so a template edit that misses _FLOAT_STDDEV_POP_RE fails here
+# instead of silently keeping float STDDEV_POP.
+SNOWFLAKE_FLOAT_STDDEV = 'STDDEV_POP(CAST("{{ col_name }}" % 2097143 AS FLOAT))'
+
+
+def _snowflake_number_stddev_template() -> str:
+    metric = yaml.safe_load(SNOWFLAKE_TEMPLATES.read_text())["NUMBER"]["stddev"]
+    return metric["metric_query"].replace("{{ modifier }}", metric["metric_column_modifier"])
+
+
+def _snowflake_ctx(platform, source_type):
+    return SimpleNamespace(
+        paired_source_platform=platform,
+        paired_source_column_types={"INV_DATE_SK": source_type},
+    )
+
+
+@pytest.mark.parametrize("platform", [Platform.SQLSERVER, Platform.AZURE_SYNAPSE])
+@pytest.mark.parametrize("source_type", ["int", "SMALLINT", "TINYINT"])
+def test_snowflake_matches_exact_tsql_integer_stddev(platform, source_type):
+    template = _snowflake_number_stddev_template()
+    assert "STDDEV_POP(CAST(" in template
+    query = QueryGeneratorSnowflake._paired_stddev_template(
+        template, source_type, _snowflake_ctx(platform, source_type)
+    )
+    assert "STDDEV_POP" not in query
+    assert query.count('CAST("{{ col_name }}" % 2097143 AS NUMBER(38, 0))') == 4
+    assert query.endswith('/ NULLIF(COUNT("{{ col_name }}"), 0)')
+
+
+def test_paired_stddev_keeps_template_when_shape_misses():
+    raw = 'STDDEV_POP("x")'
+    query = QueryGeneratorSnowflake._paired_stddev_template(
+        raw, "int", _snowflake_ctx(Platform.SQLSERVER, "int")
+    )
+    assert query == raw
+
+
+@pytest.mark.parametrize(
+    ("platform", "source_type"),
+    [
+        (Platform.SQLSERVER, "bigint"),
+        (Platform.SQLSERVER, "decimal(18,2)"),
+        (Platform.ORACLE, "NUMBER"),
+        (Platform.TERADATA, "INTEGER"),
+        (None, None),
+    ],
+)
+def test_snowflake_keeps_float_stddev_for_other_pairings(platform, source_type):
+    query = QueryGeneratorSnowflake._paired_stddev_template(
+        SNOWFLAKE_FLOAT_STDDEV, source_type, _snowflake_ctx(platform, source_type)
+    )
+    assert query == SNOWFLAKE_FLOAT_STDDEV

{snowflake_data_validation-1.42.2 → snowflake_data_validation-1.42.3}/tests/unit/test_float_bit_canonical.py

@@ -92,6 +92,10 @@ def test_redshift_tme_shims_remain_for_data_migration_staging() -> None:
     real_expr = build_redshift_real_tme_shim("v")
     assert float_expr == real_expr
     assert "'0E0'" in real_expr
+    assert "WHEN LOWER(CAST(v AS VARCHAR)) = 'nan' THEN 'nan'" in real_expr
+    assert "WHEN v = 'Infinity'::float8 THEN 'inf'" in real_expr
+    assert "WHEN v = '-Infinity'::float8 THEN '-inf'" in real_expr
+    assert real_expr.index("= 'nan'") < real_expr.index("NUMERIC(38, 15)")
     assert "POWER(10" in real_expr
     assert "'0|0|0'" not in real_expr