snowflake-data-validation 1.45.3__tar.gz → 1.46.0__tar.gz

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.45.3
+Version: 1.46.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -38,7 +38,7 @@ Requires-Dist: pydantic-yaml>=1.4.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: ruamel-yaml>=0.18.0
-Requires-Dist: snowflake-connector-python<5,>=4.7.1
+Requires-Dist: snowflake-connector-python<5,>=4.7.3
 Requires-Dist: toml>=0.10.2
 Requires-Dist: typer>=0.15.3
 Requires-Dist: typing-extensions>=4.13.2
@@ -79,7 +79,7 @@ Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgresql'
 Provides-Extra: redshift
 Requires-Dist: redshift-connector>=2.1.14; extra == 'redshift'
 Provides-Extra: scai
-Requires-Dist: snowflake-code-unit-registry>=0.9.5; extra == 'scai'
+Requires-Dist: snowflake-code-unit-registry>=0.9.7; extra == 'scai'
 Provides-Extra: sqlite
 Requires-Dist: sqlean-py>=3.45.0; extra == 'sqlite'
 Provides-Extra: sqlserver
@@ -230,6 +230,32 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.46.0
+
+**Bug fixes**
+
+- Fixed L2 `DECFLOAT` `MIN` and `MAX` being compared as `FLOAT`, which let differences past about 16 significant digits pass.
+- Fixed high-precision decimals being rounded twice when cast to `Float64`, so identical values failed by one unit in the last place.
+- Fixed SQL Server `FLOAT` and `REAL` row validation using `TRIM()`, which SQL Server 2016 does not support.
+- Fixed result-set comparison failing for column names that contain a double quote.
+- Fixed SQL Server row validation of wide tables failing with "expression services limit has been reached".
+- Fixed L2 validation of an empty source and empty target table not being reported as valid.
+- Fixed direct row validation failing tables whose rows are full duplicates when the row drill-down finds no differences.
+- Fixed SQL Server Iceberg L3 validation reporting mismatches between 9-digit source and 6-digit target fractional seconds.
+- Fixed Teradata object-type mapping: tables with no primary index are now validated as tables, and triggers are no longer mapped as tables.
+- Fixed schema validation failing for SQL Server `image` columns.
+
+### v1.45.4
+
+**Improvements**
+
+- Improved Iceberg validation to read the table format version from the Snowflake catalog once before validation starts, so every validation level uses the actual format version.
+
+**Bug fixes**
+
+- Fixed Redshift Iceberg metrics validation comparing signed-year `DATE` and `TIMESTAMPTZ` values.
+- Fixed Teradata `UNICODE` `CHAR`, `VARCHAR`, and `CLOB` column lengths being reported in bytes, which raised false truncation warnings.
+
 ### v1.45.3
 
 **Bug fixes**

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/README.md

@@ -139,6 +139,32 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.46.0
+
+**Bug fixes**
+
+- Fixed L2 `DECFLOAT` `MIN` and `MAX` being compared as `FLOAT`, which let differences past about 16 significant digits pass.
+- Fixed high-precision decimals being rounded twice when cast to `Float64`, so identical values failed by one unit in the last place.
+- Fixed SQL Server `FLOAT` and `REAL` row validation using `TRIM()`, which SQL Server 2016 does not support.
+- Fixed result-set comparison failing for column names that contain a double quote.
+- Fixed SQL Server row validation of wide tables failing with "expression services limit has been reached".
+- Fixed L2 validation of an empty source and empty target table not being reported as valid.
+- Fixed direct row validation failing tables whose rows are full duplicates when the row drill-down finds no differences.
+- Fixed SQL Server Iceberg L3 validation reporting mismatches between 9-digit source and 6-digit target fractional seconds.
+- Fixed Teradata object-type mapping: tables with no primary index are now validated as tables, and triggers are no longer mapped as tables.
+- Fixed schema validation failing for SQL Server `image` columns.
+
+### v1.45.4
+
+**Improvements**
+
+- Improved Iceberg validation to read the table format version from the Snowflake catalog once before validation starts, so every validation level uses the actual format version.
+
+**Bug fixes**
+
+- Fixed Redshift Iceberg metrics validation comparing signed-year `DATE` and `TIMESTAMPTZ` values.
+- Fixed Teradata `UNICODE` `CHAR`, `VARCHAR`, and `CLOB` column lengths being reported in bytes, which raised false truncation warnings.
+
 ### v1.45.3
 
 **Bug fixes**

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/pyproject.toml

@@ -5,7 +5,7 @@ requires = ["hatchling"]
 [tool.uv]
 constraint-dependencies = [
   "certifi>=2026.10.23,<2026.11",
-  "virtualenv>=21.7.13,<21.8",
+  "virtualenv>=21.14.9,<21.15",
 ]
 
 [project]
@@ -36,7 +36,7 @@ dependencies = [
   "connectorx>=0.3.0",
   "cryptography>=50.0.0",
   "deepdiff>=8.0.0",
-  "snowflake-connector-python>=4.7.1,<5",
+  "snowflake-connector-python>=4.7.3,<5",
   "pandas>=2.2.3",
   "polars>=1.0.0",
   "pyarrow>=14.0.0",
@@ -106,7 +106,7 @@ db2 = [
 # musllinux build, and SDV is published to public PyPI. Inside DMO/DEA the
 # dependency is already declared, so SDV's logger tree is covered there.
 scai = [
-  "snowflake-code-unit-registry>=0.9.5",
+  "snowflake-code-unit-registry>=0.9.7",
 ]
 
 [project.urls]

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -34,9 +34,6 @@ from typing import TYPE_CHECKING
 
 from shared.identifiers.managers import identifier_manager
 from shared.identifiers.sql_identifier import SqlIdentifier
-from snowflake.snowflake_data_validation.query.float_bit_normalization import (
-    build_binary64_canonical_expr,
-)
 from snowflake.snowflake_data_validation.query.numeric_scale_resolution import (
     float_bit_input_expr,
 )
@@ -44,6 +41,9 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
     resolve_table_context_modifiers,
 )
+from snowflake.snowflake_data_validation.query.tsql_float_bits import (
+    build_tsql_float_bit_canonical_expr,
+)
 from snowflake.snowflake_data_validation.query.tsql_float_remainder import (
     TsqlFloatMetricModifierMixin,
 )
@@ -77,7 +77,7 @@ LOGGER = logging.getLogger(__name__)
 
 def build_azure_synapse_float_bit_canonical_expr(col_ref: str) -> str:
     """Return the exact binary64 tuple used for Azure Synapse float validation."""
-    return build_binary64_canonical_expr(col_ref, dialect=Platform.AZURE_SYNAPSE.value)
+    return build_tsql_float_bit_canonical_expr(col_ref)
 
 
 class QueryGeneratorAzureSynapse(TsqlFloatMetricModifierMixin, QueryGeneratorBase):

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/bigquery/query/query_generator_bigquery.py

@@ -157,7 +157,7 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
                 normalized = normalized[:index]
         return normalized.strip()
 
-    def _resolve_metrics_col_type(self, col, table_context: TableContext) -> str:
+    def _resolve_metrics_col_type(self, col: ColumnMetadata, table_context: TableContext) -> str:
         """Key the metrics YAML lookup off the parameterless base type."""
         return self.normalize_bigquery_type(col.data_type)
 

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_column_metrics_templates.yaml

@@ -338,13 +338,18 @@ DOUBLE PRECISION:
     metric_return_datatype: FLOAT
     metric_column_modifier: ' - 2097143 * (CAST("{{ col_name }}" AS DECIMAL(31,0)) / 2097143)'
 
+# min/max are emitted as exact canonical decimal text (``{{ decfloat_shim }}``
+# over the aggregate), matching Snowflake DECFLOAT's: binary64 keeps only ~16
+# of DECFLOAT's 34 digits.
 DECFLOAT:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: FLOAT
+    metric_normalization: "{{ decfloat_shim }}"
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: FLOAT
+    metric_normalization: "{{ decfloat_shim }}"
   count_distinct:
     metric_query: COUNT(DISTINCT "{{ col_name }}")
     metric_return_datatype: INTEGER

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/db2/query/query_generator_db2.py

@@ -28,6 +28,7 @@ below give DB2 its own DB2-valid rendering that still matches Snowflake
 from pathlib import Path
 
 import jinja2
+import pandas as pd
 
 from shared.datatypes.common import CommonType
 from shared.datatypes.db2 import Db2Type
@@ -49,6 +50,8 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
 )
 from snowflake.snowflake_data_validation.query.sql_type_families import SNOWFLAKE_FLOAT_TYPE_NAMES
 from snowflake.snowflake_data_validation.utils.constants import (
+    DECFLOAT_SHIM_PLACEHOLDER,
+    METRIC_QUERY_PLACEHOLDER,
     NULL_HASHING_SENTINEL,
     NULL_HASHING_SENTINEL_KEY,
     ROW_HASH_COLUMN_DELIMITER,
@@ -59,6 +62,7 @@ from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     get_paired_target_column_types,
     normalize_snowflake_base_type,
 )
+from snowflake.snowflake_data_validation.utils.model.column_metadata import ColumnMetadata
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
 
 
@@ -66,6 +70,9 @@ from snowflake.snowflake_data_validation.utils.model.table_context import TableC
 # leading zero for |v| < 1 (matching Snowflake TO_VARCHAR).
 _DB2_INTEGER_NINES = 31
 
+# MIN/MAX of a DECFLOAT mirrored as a Snowflake FLOAT, in place of its exact text.
+DB2_DECFLOAT_AS_FLOAT_METRIC_NORMALIZATION = f"CAST({METRIC_QUERY_PLACEHOLDER} AS DOUBLE)"
+
 
 def build_db2_float_bit_canonical_expr(col_ref: str) -> str:
     """Return the exact binary64 tuple used for DB2 float validation.
@@ -153,21 +160,43 @@ class QueryGeneratorDb2(QueryGeneratorOracle):
         datatypes_normalization_templates: dict[str, str],
         table_context: TableContext,
     ) -> str:
-        """Hash a DECFLOAT mirrored as a Snowflake FLOAT as binary64, as that target does.
-
-        DMO creates DECFLOAT; only a user-mapped target lands on FLOAT, and it
-        holds ~16 digits, so the exact decimal text could never match it.
-        """
-        if self._parse_type_precision_scale(column.data_type or "")[0] == Db2Type.DECFLOAT:
-            target_type = get_paired_target_column_types(table_context).get(column.name)
-            if isinstance(target_type, str) and normalize_snowflake_base_type(target_type) in SNOWFLAKE_FLOAT_TYPE_NAMES:
-                return datatypes_normalization_templates[CommonType.FLOAT]
+        """Hash a DECFLOAT mirrored as a Snowflake FLOAT as binary64, as that target does."""
+        if self._decfloat_lands_in_float(column, table_context):
+            return datatypes_normalization_templates[CommonType.FLOAT]
         return super()._resolve_datatype_normalization_template(
             column,
             datatypes_normalization_templates,
             table_context,
         )
 
+    def _resolve_metric_normalization(
+        self,
+        col: ColumnMetadata,
+        metric_name: str,
+        row: pd.Series,
+        table_context: TableContext,
+    ) -> str | None:
+        """Measure a DECFLOAT mirrored as a Snowflake FLOAT in binary64, as that target does.
+
+        Only the stock exact-text MIN/MAX is swapped, so the DECFLOAT row's custom
+        removals and queries still apply.
+        """
+        normalization = super()._resolve_metric_normalization(col, metric_name, row, table_context)
+        if normalization and DECFLOAT_SHIM_PLACEHOLDER in normalization and self._decfloat_lands_in_float(col, table_context):
+            return DB2_DECFLOAT_AS_FLOAT_METRIC_NORMALIZATION
+        return normalization
+
+    def _decfloat_lands_in_float(self, column: ColumnMetadata, table_context: TableContext) -> bool:
+        """Return True for a DECFLOAT column whose Snowflake mirror is FLOAT.
+
+        DMO creates DECFLOAT; only a user-mapped target lands on FLOAT, and it
+        holds ~16 digits, so the exact decimal text could never match it.
+        """
+        if self._parse_type_precision_scale(column.data_type or "")[0] != Db2Type.DECFLOAT:
+            return False
+        target_type = get_paired_target_column_types(table_context).get(column.name)
+        return isinstance(target_type, str) and normalize_snowflake_base_type(target_type) in SNOWFLAKE_FLOAT_TYPE_NAMES
+
     # ------------------------------------------------------------------
     # text_comparison_mode: apply the per-column text codec (raw/logical)
     #

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/public/reshape/metrics_reshape.py

@@ -45,8 +45,8 @@ def _to_str(value: object) -> object:
     into a plain ``str``.  This guarantees the long-format DataFrame
     contains only strings and ``pd.NA`` — no mixed native types that
     would break ``sort_values`` inside ``normalize_dataframe``.
-    ``MetricsDataValidator`` already applies ``is_numeric`` /
-    ``float()`` for tolerance checks, so string values work correctly.
+    ``MetricsDataValidator`` parses numeric text for its tolerance checks,
+    exactly when both sides are exact, so string values work correctly.
     """
     if value is None:
         return None
@@ -114,8 +114,10 @@ def reshape_wide_metrics_result(
 
     # Single row is guaranteed by the wide-SELECT contract.  If a driver
     # ever returns more rows (e.g. GROUP BY accident) we use the first.
-    # ``tolist()`` yields Python-native types (Timestamp -> datetime, etc.)
-    row_values = wide_df.iloc[0].tolist()
+    # Each cell is read as its own column's value: a row of only numeric
+    # dtypes would otherwise be upcast to float64 as a whole, rounding an
+    # integer MIN/MAX past 2**53 that the comparator measures exactly.
+    row_values = wide_df.iloc[:1].astype(object).iloc[0].tolist()
 
     rows_by_col: dict[str, dict[str, object]] = {
         col_name: {m: pd.NA for m in metric_columns} for col_name in ordered_col_names

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/query/float_bit_normalization.py

@@ -80,23 +80,9 @@ SNOWFLAKE_BINARY64_SQL = Binary64SqlDialect(
     neg_inf_literals=("-inf", "-infinity"),
 )
 
-# SQL Server / Azure Synapse dedicated pool: LOG is natural log; LEAST/GREATEST
-# need SQL Server 2022+. POWER(2.0, n) is DECIMAL arithmetic (2.0 is
-# decimal(2,1)), so 2.0^-4 becomes 0.1. Force FLOAT: POWER(CONVERT(FLOAT, 2), n).
-# LOG(0) raises divide-by-zero even inside CASE; NULLIF the abs input.
-# CAST(... AS VARCHAR) is VARCHAR(30), which is enough for these integers.
-SQLSERVER_BINARY64_SQL = Binary64SqlDialect(
-    log2="LOG(NULLIF({abs}, 0)) / LOG(CONVERT(FLOAT, 2))",
-    power2="POWER(CONVERT(FLOAT, 2), {exp})",
-    integer_type="BIGINT",
-    string_type="VARCHAR(50)",
-    concat="concat",
-    use_least_greatest=False,
-    text_value="LOWER(LTRIM(RTRIM(CONVERT(VARCHAR(50), {value}, 2))))",
-    nan_literals=("nan",),
-    pos_inf_literals=("inf", "infinity"),
-    neg_inf_literals=("-inf", "-infinity"),
-)
+# SQL Server and Azure Synapse read the tuple from the stored bits instead
+# (``tsql_float_bits``): the LOG/POWER search below repeats the column about a
+# hundred times, and T-SQL row hashes compile into one size-limited expression.
 
 # Redshift and PostgreSQL can share Snowflake's spelling
 # (LN / POWER / BIGINT / ||).
@@ -196,9 +182,6 @@ BINARY64_SQL_BY_DIALECT: dict[str, Binary64SqlDialect] = {
     "snowflake": SNOWFLAKE_BINARY64_SQL,
     "redshift": SNOWFLAKE_BINARY64_SQL,
     "postgresql": SNOWFLAKE_BINARY64_SQL,
-    "sqlserver": SQLSERVER_BINARY64_SQL,
-    # Dedicated SQL pool is T-SQL: same LOG/POWER(FLOAT) spelling as SQL Server.
-    "azure_synapse": SQLSERVER_BINARY64_SQL,
     "oracle": ORACLE_BINARY64_SQL,
     "sqlite": SQLITE_BINARY64_SQL,
     "db2": DB2_BINARY64_SQL,

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/query/normalization_render_context.py

@@ -26,11 +26,23 @@ class NormalizationRenderContext:
     has_custom_normalization: bool = False
 
     @classmethod
-    def from_table_context(cls, table_context: TableContext) -> NormalizationRenderContext:
-        """Resolve the per-table inputs; ``for_column`` adds the per-column paired type."""
+    def from_table_context(
+        cls,
+        table_context: TableContext,
+        *,
+        source_timestamp_fraction_digits: int | None = None,
+    ) -> NormalizationRenderContext:
+        """Resolve the per-table inputs; ``for_column`` adds the per-column paired type.
+
+        ``source_timestamp_fraction_digits`` is the paired source's fixed timestamp
+        text width; it replaces the Iceberg spec's digits when given.
+        """
+        fraction_digits = source_timestamp_fraction_digits
+        if fraction_digits is None:
+            fraction_digits = IcebergProfile.from_table_context(table_context).fraction_digits
         return cls(
             paired_source_platform=table_context.paired_source_platform,
-            iceberg_fraction_digits=IcebergProfile.from_table_context(table_context).fraction_digits,
+            iceberg_fraction_digits=fraction_digits,
         )
 
     def for_column(

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/query/query_generator_base.py

@@ -182,12 +182,15 @@ class QueryGeneratorBase(ABC):
             f"DECFLOAT normalization is not supported for platform {self.platform.value!r}",
         )
 
-    def _expand_decfloat_placeholder(self, template: str, column) -> str:
-        """Replace ``{{ decfloat_shim }}`` when present."""
+    def _expand_decfloat_placeholder(self, template: str, column, col_ref: str | None = None) -> str:
+        """Replace ``{{ decfloat_shim }}`` when present, over *col_ref* (default: the column itself)."""
         if DECFLOAT_SHIM_PLACEHOLDER not in template:
             return template
         try:
-            shim = self.build_decfloat_shim(column, self._float_column_reference(column))
+            shim = self.build_decfloat_shim(
+                column,
+                col_ref if col_ref is not None else self._float_column_reference(column),
+            )
         except NotImplementedError as exc:
             raise ValueError(
                 f"{{ decfloat_shim }} is not supported for platform {self.platform.value!r}",
@@ -225,7 +228,7 @@ class QueryGeneratorBase(ABC):
 
         The shipped templates spell the column ``"{{ col_name }}"``; custom ones may
         use a bare ``{{ col_name }}``. Dialects whose writer delimits the name itself
-        (Snowflake, SQL Server, Synapse) render either as one complete identifier.
+        (Snowflake, SQL Server, Synapse, result set) render either as one complete identifier.
         """
         written = self._write_sql_identifier(identifier)
         rendered = template.replace(self._quoted_metric_identifier_placeholder(), written)
@@ -394,8 +397,7 @@ class QueryGeneratorBase(ABC):
         override this. Combined generators leave the base raise in place.
         """
         raise NotImplementedError(
-            f"{self.platform.value} uses a combined schema query; "
-            "call generate_schema_query instead of generate_schema_row_count_query"
+            f"{self.platform.value} uses a combined schema query; call generate_schema_query instead of generate_schema_row_count_query"
         )
 
     def generate_schema_details_query(self, table_context: TableContext) -> str:
@@ -405,8 +407,7 @@ class QueryGeneratorBase(ABC):
         override this. Combined generators leave the base raise in place.
         """
         raise NotImplementedError(
-            f"{self.platform.value} uses a combined schema query; "
-            "call generate_schema_query instead of generate_schema_details_query"
+            f"{self.platform.value} uses a combined schema query; call generate_schema_query instead of generate_schema_details_query"
         )
 
     def normalize_schema_details_result(
@@ -420,10 +421,7 @@ class QueryGeneratorBase(ABC):
         as a combined schema query (without ``ROW_COUNT``; the fetch helper
         attaches that). Combined generators leave the base raise in place.
         """
-        raise NotImplementedError(
-            f"{self.platform.value} uses a combined schema query; "
-            "normalize_schema_details_result is not used"
-        )
+        raise NotImplementedError(f"{self.platform.value} uses a combined schema query; normalize_schema_details_result is not used")
 
     def generate_metrics_query(self, table_context: TableContext) -> GeneratedMetricsQuery:
         """Generate a single wide metrics SELECT for a table.
@@ -645,7 +643,7 @@ class QueryGeneratorBase(ABC):
                 return numeric_norm
         return None
 
-    def _resolve_metrics_col_type(self, col, table_context: TableContext) -> str:
+    def _resolve_metrics_col_type(self, col: ColumnMetadata, table_context: TableContext) -> str:
         """Return the metrics YAML type key for a column (platforms may override)."""
         from snowflake.snowflake_data_validation.utils.vector_common import is_vector_base_type
 
@@ -952,13 +950,13 @@ class QueryGeneratorBase(ABC):
                     if normalization is None and row.get(EXPLICIT_METRIC_NORMALIZATION_COLUMN_KEY):
                         normalization = row[NORMALIZATION_COLUMN_KEY]
                     if pd.notna(normalization) and normalization:
+                        # A DECFLOAT aggregate keeps every digit as canonical text over the aggregate.
+                        normalization = self._expand_decfloat_placeholder(normalization, col, METRIC_QUERY_PLACEHOLDER)
                         if NUMERIC_FORMAT_MASK_PLACEHOLDER in normalization:
                             mask = self._build_numeric_format_mask(self._resolve_numeric_scale(col))
                             normalization = normalization.replace(NUMERIC_FORMAT_MASK_PLACEHOLDER, mask)
                         if ICEBERG_FRACTION_DIGITS_PLACEHOLDER in normalization:
-                            normalization = normalization.replace(
-                                ICEBERG_FRACTION_DIGITS_PLACEHOLDER, str(iceberg_fraction_digits)
-                            )
+                            normalization = normalization.replace(ICEBERG_FRACTION_DIGITS_PLACEHOLDER, str(iceberg_fraction_digits))
                         metric_expr = self._wrap_metric_with_normalization(normalization, metric_expr)
                 alias = f"c{measured_idx}_m{metric_idx}"
                 select_exprs.append(f"{metric_expr} AS {alias}")

snowflake_data_validation-1.46.0/src/snowflake/snowflake_data_validation/query/tsql_float_bits.py

@@ -0,0 +1,66 @@
+# Copyright 2026 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Read a T-SQL FLOAT's IEEE-754 fields from its stored bits.
+
+SQL Server and the Azure Synapse dedicated SQL pool share this spelling of the
+``sign|exponent|significand`` tuple that Snowflake FLOAT emits.
+"""
+
+from __future__ import annotations
+
+from snowflake.snowflake_data_validation.query.float_bit_normalization import (
+    IEEE754_BINARY64_FRACTION_BITS,
+    IEEE754_BINARY64_MIN_EXPONENT,
+)
+
+
+_BINARY64_EXPONENT_BIAS = 1023
+_BINARY64_MAX_BIASED_EXPONENT = 2047
+# Mask for the 11 exponent bits in the first two big-endian bytes (0x7FF0).
+_BINARY64_EXPONENT_MASK_IN_HIGH_WORD = 0x7FF0
+_BINARY64_EXPONENT_SHIFT_IN_HIGH_WORD = 16
+_BINARY64_FRACTION_MASK = (1 << IEEE754_BINARY64_FRACTION_BITS) - 1
+_BINARY64_IMPLICIT_BIT = 1 << IEEE754_BINARY64_FRACTION_BITS
+
+
+def build_tsql_float_bit_canonical_expr(col_ref: str) -> str:
+    """Return the exact binary64 tuple for a SQL Server or Azure Synapse float.
+
+    Emits the same ``sign|exponent|significand`` strings as
+    ``build_binary64_canonical_expr``, but reads the fields from the stored
+    bits: ``CAST(float AS BINARY(8))`` is the big-endian IEEE-754 encoding.
+    The LOG/POWER exponent search the other dialects use repeats the column
+    about a hundred times, and T-SQL compiles every normalized column of a row
+    hash into one expression with a size limit ("An expression services limit
+    has been reached"), which a few dozen FLOAT columns exceed. Integer
+    literals above INT range are DECIMAL in T-SQL, so the 64-bit masks are
+    cast to BIGINT for the ``&`` operator.
+    """
+    # The inner cast keeps a REAL argument from encoding as four bytes.
+    bits = f"CAST(CAST({col_ref} AS FLOAT) AS BINARY(8))"
+    biased_exponent = (
+        f"((CAST(SUBSTRING({bits}, 1, 2) AS INT) & {_BINARY64_EXPONENT_MASK_IN_HIGH_WORD})"
+        f" / {_BINARY64_EXPONENT_SHIFT_IN_HIGH_WORD})"
+    )
+    fraction = f"(CAST({bits} AS BIGINT) & CAST({_BINARY64_FRACTION_MASK} AS BIGINT))"
+    # Subnormals have a zero biased exponent, a fixed -1022 exponent and no implicit bit.
+    exponent = (
+        f"CASE WHEN {biased_exponent} = 0 THEN {IEEE754_BINARY64_MIN_EXPONENT} "
+        f"ELSE {biased_exponent} - {_BINARY64_EXPONENT_BIAS} END"
+    )
+    significand = (
+        f"{fraction} + CASE WHEN {biased_exponent} = 0 THEN CAST(0 AS BIGINT) "
+        f"ELSE CAST({_BINARY64_IMPLICIT_BIT} AS BIGINT) END"
+    )
+    special_value = (
+        f"CASE WHEN {fraction} <> 0 THEN 'NAN' WHEN {col_ref} < 0 THEN '-INF' ELSE '+INF' END"
+    )
+    sign = f"CASE WHEN {col_ref} < 0 THEN '1' ELSE '0' END"
+    return (
+        f"CASE WHEN {col_ref} IS NULL THEN NULL "
+        f"WHEN {col_ref} = 0 THEN '0|0|0' "
+        f"WHEN {biased_exponent} = {_BINARY64_MAX_BIASED_EXPONENT} THEN {special_value} "
+        f"ELSE CONCAT({sign}, '|', CAST({exponent} AS VARCHAR(50)), '|', "
+        f"CAST({significand} AS VARCHAR(50))) END"
+    )

{snowflake_data_validation-1.45.3 → snowflake_data_validation-1.46.0}/src/snowflake/snowflake_data_validation/redshift/extractor/templates/redshift_column_metrics_templates.yaml

@@ -635,6 +635,36 @@ TIMESTAMP_AS_TEXT:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: BIGINT
 
+# Iceberg VARCHAR DATE (SNOW-4381535): MIN/MAX rendered as the UNLOAD footer's
+# signed astronomical-year text. Kept in sync with DMO by test_redshift_consistency.py.
+DATE_AS_TEXT:
+  min:
+    metric_query: MIN("{{ col_name }}")
+    metric_return_datatype: VARCHAR
+    metric_normalization: '(CASE WHEN (CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END) < 0 THEN ''-'' ELSE '''' END || (CASE WHEN ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END)) < 1000 THEN LPAD(ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR, 4, ''0'') ELSE ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR END)) || ''-'' || LPAD(EXTRACT(MONTH FROM {{ metric_query }})::VARCHAR, 2, ''0'') || ''-'' || LPAD(EXTRACT(DAY FROM {{ metric_query }})::VARCHAR, 2, ''0'')'
+  max:
+    metric_query: MAX("{{ col_name }}")
+    metric_return_datatype: VARCHAR
+    metric_normalization: '(CASE WHEN (CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END) < 0 THEN ''-'' ELSE '''' END || (CASE WHEN ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END)) < 1000 THEN LPAD(ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR, 4, ''0'') ELSE ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR END)) || ''-'' || LPAD(EXTRACT(MONTH FROM {{ metric_query }})::VARCHAR, 2, ''0'') || ''-'' || LPAD(EXTRACT(DAY FROM {{ metric_query }})::VARCHAR, 2, ''0'')'
+  count_null:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
+    metric_return_datatype: BIGINT
+
+# Iceberg VARCHAR TIMESTAMPTZ (SNOW-4381535): MIN/MAX of the instant, rendered in
+# UTC with the UNLOAD footer's signed year. Kept in sync with DMO by test_redshift_consistency.py.
+TIMESTAMPTZ_AS_TEXT:
+  min:
+    metric_query: MIN("{{ col_name }}")
+    metric_return_datatype: VARCHAR
+    metric_normalization: '(CASE WHEN (CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END) < 0 THEN ''-'' ELSE '''' END || (CASE WHEN ABS((CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END)) < 1000 THEN LPAD(ABS((CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END))::VARCHAR, 4, ''0'') ELSE ABS((CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END))::VARCHAR END)) || TO_CHAR(({{ metric_query }} AT TIME ZONE ''UTC''), ''-MM-DD"T"HH24:MI:SS.US'') || ''000'' || ''+00:00'''
+  max:
+    metric_query: MAX("{{ col_name }}")
+    metric_return_datatype: VARCHAR
+    metric_normalization: '(CASE WHEN (CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END) < 0 THEN ''-'' ELSE '''' END || (CASE WHEN ABS((CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END)) < 1000 THEN LPAD(ABS((CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END))::VARCHAR, 4, ''0'') ELSE ABS((CASE WHEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT < 0 THEN EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT + 1 ELSE EXTRACT(YEAR FROM ({{ metric_query }} AT TIME ZONE ''UTC''))::INT END))::VARCHAR END)) || TO_CHAR(({{ metric_query }} AT TIME ZONE ''UTC''), ''-MM-DD"T"HH24:MI:SS.US'') || ''000'' || ''+00:00'''
+  count_null:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
+    metric_return_datatype: BIGINT
+
 TIMESTAMPTZ:
   min: 
     metric_query: MIN("{{ col_name }}")