snowflake-data-validation 1.44.0__tar.gz → 1.45.0__tar.gz

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.44.0
+Version: 1.45.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -79,7 +79,7 @@ Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgresql'
 Provides-Extra: redshift
 Requires-Dist: redshift-connector>=2.1.14; extra == 'redshift'
 Provides-Extra: scai
-Requires-Dist: snowflake-code-unit-registry>=0.9.0; extra == 'scai'
+Requires-Dist: snowflake-code-unit-registry>=0.9.2; extra == 'scai'
 Provides-Extra: sqlite
 Requires-Dist: sqlean-py>=3.45.0; extra == 'sqlite'
 Provides-Extra: sqlserver
@@ -214,6 +214,46 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.45.0
+
+**New features**
+
+- Added support for applying custom L3 normalization per side, including `targetDataType: "*"` to target the mapped type.
+- Added the `includeCountDistinct` option; L2 `COUNT(DISTINCT)` is now off by default and opt-in globally or per table.
+- Added support for additional table and view types in data validation, including materialized views, temporary tables, BigQuery clones and snapshots, and DB2 materialized query tables.
+
+**Improvements**
+
+- Improved row comparison with positional column identity, so duplicate and case-variant column names no longer collapse.
+- Improved Redshift connection resiliency with TCP keepalive defaults.
+- Improved L1 validation to treat native integer bits-versus-digits precision differences as compatible, reported as a warning.
+
+**Bug fixes**
+
+- Fixed Oracle L2 metric rows and L3 cell frames upper-casing column names, which made mixed-case columns collide.
+- Fixed L3 failing rows over a lowercase SQL Server `UNIQUEIDENTIFIER` or an ignored `FLOAT` `targetExpression`.
+- Fixed a possible deadlock in expression casts when validators run on several threads.
+- Fixed Redshift and PostgreSQL `REAL` values being compared against a non-exact widened value.
+- Fixed Iceberg `FULL_INGEST` validation to follow the lossless-only temporal promotion of the load.
+
+### v1.44.1
+
+**New features**
+
+- Added support for L3 row and hybrid validation on Iceberg v3 tables in native storage mode, keyed by the stored column type.
+
+**Bug fixes**
+
+- Fixed `PERIOD` validation normalization and signature extraction.
+- Fixed Teradata `XML` columns not being cast to `UNICODE` before L3 hashing.
+- Fixed SQL Server and Synapse `FLOAT`/`REAL` L2 aggregates being cast to `DECIMAL(38,10)` on the source only.
+- Fixed `TIME` and `TIME WITH TIME ZONE` normalization dropping fractional seconds on several platforms.
+- Fixed `DECFLOAT` L3 comparisons diverging past about 16 digits by comparing exact decimal text.
+- Fixed Teradata float aggregate comparisons by comparing on the signed significand.
+- Fixed false row-hash mismatches on Iceberg-compatible geospatial columns.
+- Fixed offset timestamps (SQL Server `DATETIMEOFFSET`, Teradata `TIMESTAMP WITH TIME ZONE`) not being compared as UTC instants under Iceberg `FULL_INGEST`.
+- Fixed Redshift Iceberg validation findings: L3 `TIME`, L2 `count_distinct` overflow, and the L1 precision message.
+
 ### v1.44.0
 
 **New features**

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/README.md

@@ -123,6 +123,46 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.45.0
+
+**New features**
+
+- Added support for applying custom L3 normalization per side, including `targetDataType: "*"` to target the mapped type.
+- Added the `includeCountDistinct` option; L2 `COUNT(DISTINCT)` is now off by default and opt-in globally or per table.
+- Added support for additional table and view types in data validation, including materialized views, temporary tables, BigQuery clones and snapshots, and DB2 materialized query tables.
+
+**Improvements**
+
+- Improved row comparison with positional column identity, so duplicate and case-variant column names no longer collapse.
+- Improved Redshift connection resiliency with TCP keepalive defaults.
+- Improved L1 validation to treat native integer bits-versus-digits precision differences as compatible, reported as a warning.
+
+**Bug fixes**
+
+- Fixed Oracle L2 metric rows and L3 cell frames upper-casing column names, which made mixed-case columns collide.
+- Fixed L3 failing rows over a lowercase SQL Server `UNIQUEIDENTIFIER` or an ignored `FLOAT` `targetExpression`.
+- Fixed a possible deadlock in expression casts when validators run on several threads.
+- Fixed Redshift and PostgreSQL `REAL` values being compared against a non-exact widened value.
+- Fixed Iceberg `FULL_INGEST` validation to follow the lossless-only temporal promotion of the load.
+
+### v1.44.1
+
+**New features**
+
+- Added support for L3 row and hybrid validation on Iceberg v3 tables in native storage mode, keyed by the stored column type.
+
+**Bug fixes**
+
+- Fixed `PERIOD` validation normalization and signature extraction.
+- Fixed Teradata `XML` columns not being cast to `UNICODE` before L3 hashing.
+- Fixed SQL Server and Synapse `FLOAT`/`REAL` L2 aggregates being cast to `DECIMAL(38,10)` on the source only.
+- Fixed `TIME` and `TIME WITH TIME ZONE` normalization dropping fractional seconds on several platforms.
+- Fixed `DECFLOAT` L3 comparisons diverging past about 16 digits by comparing exact decimal text.
+- Fixed Teradata float aggregate comparisons by comparing on the signed significand.
+- Fixed false row-hash mismatches on Iceberg-compatible geospatial columns.
+- Fixed offset timestamps (SQL Server `DATETIMEOFFSET`, Teradata `TIMESTAMP WITH TIME ZONE`) not being compared as UTC instants under Iceberg `FULL_INGEST`.
+- Fixed Redshift Iceberg validation findings: L3 `TIME`, L2 `count_distinct` overflow, and the L1 precision message.
+
 ### v1.44.0
 
 **New features**

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/pyproject.toml

@@ -105,7 +105,7 @@ db2 = [
 # musllinux build, and SDV is published to public PyPI. Inside DMO/DEA the
 # dependency is already declared, so SDV's logger tree is covered there.
 scai = [
-  "snowflake-code-unit-registry>=0.9.0",
+  "snowflake-code-unit-registry>=0.9.2",
 ]
 
 [project.urls]

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml

@@ -258,6 +258,9 @@ SMALLMONEY:
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
+# avg/sum/stddev aggregate the binary64 value, as Snowflake aggregates its FLOAT
+# (SNOW-4314808); REAL widens exactly. T-SQL has no FLOAT %, so the generator
+# renders {{ modifier }} on this operand as Snowflake's float remainder (tsql_float_remainder.py).
 FLOAT:
   min:
     metric_query: MIN("{{ col_name }}")
@@ -275,18 +278,19 @@ FLOAT:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   avg:
-    metric_query: AVG(CAST("{{ col_name }}" AS DECIMAL(38, 10)) {{ modifier }})
+    metric_query: AVG(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
   sum:
-    metric_query: SUM(CAST("{{ col_name }}" AS DECIMAL(38, 10)) {{ modifier }})
+    metric_query: SUM(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
   stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" AS DECIMAL(38, 10)) {{ modifier }})
+    metric_query: STDEVP(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
+# avg/sum/stddev widen to binary64, as FLOAT's do (SNOW-4314808).
 REAL:
   min:
     metric_query: MIN("{{ col_name }}")
@@ -304,15 +308,15 @@ REAL:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   avg:
-    metric_query: AVG(CAST("{{ col_name }}" AS DECIMAL(38, 10)) {{ modifier }})
+    metric_query: AVG(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
   sum:
-    metric_query: SUM(CAST("{{ col_name }}" AS DECIMAL(38, 10)) {{ modifier }})
+    metric_query: SUM(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
     metric_return_datatype: REAL
     metric_column_modifier: "% 2097143"
   stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" AS DECIMAL(38, 10)) {{ modifier }})
+    metric_query: STDEVP(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -50,7 +50,11 @@ NVARCHAR: '"{{ col_name }}"'
 # Use '+' (not CONCAT) so a NULL input propagates to NULL.
 # DATENAME(TZOFFSET) emits '+HH:MM'/'-HH:MM' for every offset, including UTC.
 DATE: CONVERT(VARCHAR(10), "{{ col_name }}", 23)
-TIME: CONVERT(VARCHAR(8), "{{ col_name }}", 108)
+# Same text as SQL Server TIME: hh:mi:ss.fffffff + '00', cut to {{ time_text_length }}.
+TIME: LEFT(SUBSTRING(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 12, 16) + '00', {{ time_text_length }})
+# Iceberg compatible TIME stored as VARCHAR (CETAS): the extract's own text, TIME(7)
+# widened then '00', nine digits whatever the scale. '+' keeps NULL as NULL.
+TIME_ICEBERG: CAST(CAST("{{ col_name }}" AS TIME(7)) AS VARCHAR(16)) + '00'
 DATETIME: STUFF(CONVERT(VARCHAR(27), CAST(CAST("{{ col_name }}" AS DATETIME2(3)) AS DATETIME2(7)), 121), 11, 1, 'T') + '00'
 DATETIME2: LEFT(STUFF(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 11, 1, 'T'), 26) + '000'
 SMALLDATETIME: CONVERT(VARCHAR(19), "{{ col_name }}", 126) + '.000000000'

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -44,12 +44,19 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
     resolve_table_context_modifiers,
 )
+from snowflake.snowflake_data_validation.query.tsql_float_remainder import (
+    TsqlFloatMetricModifierMixin,
+)
 from snowflake.snowflake_data_validation.sqlserver.query.charset_normalization import (
     SqlServerCharacterSetResolver,
 )
 from snowflake.snowflake_data_validation.utils.constants import (
     Platform,
 )
+from snowflake.snowflake_data_validation.utils.iceberg_dv_common import (
+    resolve_iceberg_compatible_tsql_source_normalization_key,
+)
+from snowflake.snowflake_data_validation.utils.iso_dv_common import get_paired_target_column_types
 from snowflake.snowflake_data_validation.utils.measure_collation import (
     TSQL_ROW_HASH_COLLATION_SUFFIX,
     tsql_comparison_collation_tokens,
@@ -72,7 +79,7 @@ def build_azure_synapse_float_bit_canonical_expr(col_ref: str) -> str:
     return build_binary64_canonical_expr(col_ref, dialect=Platform.AZURE_SYNAPSE.value)
 
 
-class QueryGeneratorAzureSynapse(QueryGeneratorBase):
+class QueryGeneratorAzureSynapse(TsqlFloatMetricModifierMixin, QueryGeneratorBase):
     """Azure Synapse Analytics implementation of the query generator.
 
     Uses T-SQL syntax shared with SQL Server.  L3 row hashing is emitted as a
@@ -138,8 +145,16 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
         datatypes_normalization_templates = table_context.templates_loader_manager.datatypes_normalization_templates
         rendered: dict[str, str] = {}
         renderer_columns = self._row_md5_normalization_columns(table_context)
+        paired_target_types = get_paired_target_column_types(table_context)
+        iceberg_spec = table_context.iceberg_spec
+        storage_mode = table_context.storage_mode
         for column in renderer_columns:
-            lookup_key = (column.data_type or "").strip().upper()
+            lookup_key = resolve_iceberg_compatible_tsql_source_normalization_key(
+                (column.data_type or "").strip().upper(),
+                paired_target_type=paired_target_types.get(column.name),
+                iceberg_spec=iceberg_spec,
+                storage_mode=storage_mode,
+            )
             normalization_template = self._lookup_normalization_template(
                 datatypes_normalization_templates,
                 lookup_key,

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_datatypes_normalization_templates.yaml

@@ -47,11 +47,15 @@ FLOAT: '{{ float_bit_shim }}'
 # bounded BIGNUMERIC(p, s) that fits NUMBER hashes identically. A bare
 # BIGNUMERIC is created as DECFLOAT (SnowConvert engine parity); precision
 # above 38 or scale above 37 reports as an L1 WARNING rather than a schema
-# failure, and L2/L3 are not expected to reach parity on the extreme form.
+# failure.
 NUMERIC: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)
 DECIMAL: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)
 BIGNUMERIC: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)
 BIGDECIMAL: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)
+# A BIGNUMERIC / BIGDECIMAL whose mirror is DECFLOAT (the paired target type, or
+# a bare type when unpaired) emits the exact canonical decimal text Snowflake
+# DECFLOAT does. A value past DECFLOAT's 38 digits then reports as a mismatch.
+BIGNUMERIC_DECFLOAT: '{{ decfloat_shim }}'
 
 # Snowflake BOOLEAN normalizes to uppercase 'TRUE'/'FALSE' (see
 # snowflake_datatypes_normalization_templates.yaml); match it so the row hashes agree.
@@ -65,7 +69,8 @@ BYTES: UPPER(TO_HEX(`{{ col_name }}`))
 
 # %E4Y zero-pads the year to 4 digits so year 1 renders "0001" (matches Snowflake YYYY).
 DATE: FORMAT_DATE('%E4Y-%m-%d', `{{ col_name }}`)
-TIME: FORMAT_TIME('%H:%M:%S', `{{ col_name }}`)
+# TIME: microseconds + '000' pad to nine digits, cut to the precision both sides share.
+TIME: LEFT(FORMAT_TIME('%H:%M:%E6S', `{{ col_name }}`) || '000', {{ time_text_length }})
 DATETIME: FORMAT_DATETIME('%E4Y-%m-%dT%H:%M:%E9S', `{{ col_name }}`)
 TIMESTAMP: FORMAT_TIMESTAMP('%E4Y-%m-%dT%H:%M:%E9S%Ez', `{{ col_name }}`, 'UTC')
 

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_table_metadata_query.sql.j2

@@ -48,7 +48,10 @@ WITH rc_cte AS (
         (
             SELECT CASE table_type
                 WHEN 'BASE TABLE' THEN 'TABLE'
+                WHEN 'CLONE' THEN 'TABLE'
+                WHEN 'SNAPSHOT' THEN 'TABLE'
                 WHEN 'VIEW' THEN 'VIEW'
+                WHEN 'MATERIALIZED VIEW' THEN 'VIEW'
                 ELSE 'UNKNOWN'
             END
             FROM {{ object_database_sql }}.{{ object_schema_sql }}.INFORMATION_SCHEMA.TABLES

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/bigquery/query/query_generator_bigquery.py

@@ -17,7 +17,12 @@ from __future__ import annotations
 
 import logging
 
+from shared.datatypes.bigquery import BigQueryType
 from shared.datatypes.common import CommonType
+from shared.datatypes.snowflake import SnowflakeType
+from snowflake.snowflake_data_validation.query.decimal_text_normalization import (
+    build_decimal_text_canonical_expr,
+)
 from snowflake.snowflake_data_validation.query.float_bit_normalization import (
     build_binary64_canonical_expr,
 )
@@ -33,6 +38,10 @@ from snowflake.snowflake_data_validation.utils.constants import (
     METRIC_COLUMN_MODIFIER_PLACEHOLDER,
     Platform,
 )
+from snowflake.snowflake_data_validation.utils.iso_dv_common import (
+    get_paired_target_column_types,
+    normalize_snowflake_base_type,
+)
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
 from snowflake.snowflake_data_validation.validation.comparison_codec import (
     ComparisonCodec,
@@ -43,6 +52,10 @@ from snowflake.snowflake_data_validation.validation.comparison_codec import (
 
 LOGGER = logging.getLogger(__name__)
 
+_WIDE_NUMERIC_BASE_TYPES = frozenset({BigQueryType.BIGNUMERIC, BigQueryType.BIGDECIMAL})
+# Normalization key for a BIGNUMERIC / BIGDECIMAL whose Snowflake mirror is DECFLOAT.
+BIGNUMERIC_DECFLOAT_NORMALIZATION_KEY = "BIGNUMERIC_DECFLOAT"
+
 
 def build_bigquery_float_bit_canonical_expr(col_ref: str) -> str:
     """Return the exact binary64 tuple used for BigQuery float validation."""
@@ -151,6 +164,27 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
         )
         return build_bigquery_float_bit_canonical_expr(value_expr)
 
+    def build_decfloat_shim(self, column, col_ref: str) -> str:
+        """Canonicalize a BIGNUMERIC mirrored as DECFLOAT from its exact ``STRING`` text."""
+        return build_decimal_text_canonical_expr(f"CAST({col_ref} AS STRING)", dialect=self.platform.value)
+
+    def _lands_in_decfloat(self, column, table_context: TableContext) -> bool:
+        """Return True when a BIGNUMERIC / BIGDECIMAL column's Snowflake mirror is DECFLOAT.
+
+        The paired target type decides when it is known: Iceberg and user-chosen
+        targets keep a bare BIGNUMERIC on NUMBER. Without it, a bare type follows
+        the create path, which emits DECFLOAT (``apply_type_parameters``).
+        """
+        if self.normalize_bigquery_type(column.data_type) not in _WIDE_NUMERIC_BASE_TYPES:
+            return False
+        target_type = get_paired_target_column_types(table_context).get(column.name)
+        if isinstance(target_type, str):
+            return normalize_snowflake_base_type(target_type) == SnowflakeType.DECFLOAT
+        if "(" in column.data_type:
+            return False
+        LOGGER.debug(f"No paired target type for {column.data_type} {column.name}; assuming the DECFLOAT mirror the create path emits.")
+        return True
+
     def _apply_metric_column_modifier(
         self,
         metric_template: str,
@@ -200,8 +234,11 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
         self,
         column,
         datatypes_normalization_templates: dict[str, str],
+        table_context: TableContext,
     ) -> str:
         base_type = self.normalize_bigquery_type(column.data_type)
+        if self._lands_in_decfloat(column, table_context):
+            base_type = BIGNUMERIC_DECFLOAT_NORMALIZATION_KEY
         normalization_template = datatypes_normalization_templates.get(base_type)
         if normalization_template is None:
             LOGGER.debug(
@@ -227,6 +264,7 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
             normalization_template = self._resolve_datatype_normalization_template(
                 column,
                 datatypes_normalization_templates,
+                table_context,
             )
             normalization_template = resolve_column_normalization_template(
                 column.name,

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/configuration/model/configuration_model.py

@@ -96,6 +96,7 @@ class ConfigurationModel(BaseModel):
             self._check_max_failed_rows_number(table)
             self._check_chunk_number(table)
             self.set_exclude_metrics(table)
+            self.set_include_count_distinct(table)
             self.set_apply_metric_column_modifier(table)
             self.set_include_lob_columns_in_row_validation(table)
 
@@ -113,6 +114,7 @@ class ConfigurationModel(BaseModel):
             self._check_max_failed_rows_number(view)
             self._check_chunk_number(view)
             self.set_exclude_metrics(view)
+            self.set_include_count_distinct(view)
             self.set_apply_metric_column_modifier(view)
             self.set_include_lob_columns_in_row_validation(view)
 
@@ -198,6 +200,10 @@ class ConfigurationModel(BaseModel):
         if table.exclude_metrics is None:
             table.exclude_metrics = self.validation_configuration.exclude_metrics
 
+    def set_include_count_distinct(self, table: TableConfiguration) -> None:
+        if table.include_count_distinct is None:
+            table.include_count_distinct = self.validation_configuration.include_count_distinct
+
     def set_include_lob_columns_in_row_validation(self, table: TableConfiguration) -> None:
         if table.include_lob_columns_in_row_validation is None:
             table.include_lob_columns_in_row_validation = (

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/configuration/model/table_configuration.py

@@ -32,6 +32,7 @@ VISIBLE_PROPERTIES: set[str] = {
     "chunk_number",
     "column_mappings",
     "exclude_metrics",
+    "include_count_distinct",
     "include_lob_columns_in_row_validation",
     "index_column_list",
     "max_failed_rows_number",
@@ -83,6 +84,7 @@ class TableConfiguration(BaseModel):
     column_mappings: dict[str, str] = {}
     max_failed_rows_number: int | None = None
     exclude_metrics: bool | None = None
+    include_count_distinct: bool | None = None
     apply_metric_column_modifier: bool | None = None
     include_lob_columns_in_row_validation: bool | None = None
     object_type: ObjectType = ObjectType.TABLE

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/configuration/model/validation_configuration.py

@@ -41,6 +41,9 @@ class ValidationConfiguration(BaseModel):
     custom_templates_path: Path | None = None
     max_failed_rows_number: int = MAX_FAILED_ROWS_NUMBER_DEFAULT_VALUE
     exclude_metrics: bool = False
+    # L2 COUNT(DISTINCT) is opt-in (SNOW-4338258): it dominates L2 cost on the source
+    # (one SQL Server batch 26.4 s with it, 1.4 s without) and L2 is the fast metrics level.
+    include_count_distinct: bool = False
     apply_metric_column_modifier: bool = True
     include_lob_columns_in_row_validation: bool = False
 

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_datatypes_normalization_templates.yaml

@@ -15,6 +15,7 @@
 # - {{ numeric_format_mask }} is a pre-built DB2-valid 9…90 mask (no FM prefix)
 #   from the query generator's _build_fm_to_char_mask override.
 # - {{ float_bit_shim }} expands to the IEEE-754 sign|exponent|significand tuple.
+# - {{ decfloat_shim }} expands to DECFLOAT's exact canonical decimal text.
 # - HEX(HASH(val, 0)) computes MD5 in DB2 11.5 (algorithm 0 = MD5) using the built-in
 #   SYSIBM.HASH scalar — no special privilege (unlike SYSPROC.HASH2, which is often absent).
 # - Double-quoted column names handle lowercase/mixed-case identifiers.
@@ -37,7 +38,8 @@ FLOAT: '{{ float_bit_shim }}'
 REAL: '{{ float_bit_shim }}'
 DOUBLE: '{{ float_bit_shim }}'
 DOUBLE PRECISION: '{{ float_bit_shim }}'
-DECFLOAT: '{{ float_bit_shim }}'
+# DECFLOAT(16/34) is decimal: a DOUBLE drops digits past ~16, so compare its exact text.
+DECFLOAT: '{{ decfloat_shim }}'
 
 # ---- Character types (text family) ----
 # CHAR/NCHAR are blank-padded; the Snowflake E2E/SCAI mirror is VARCHAR/TEXT.
@@ -103,9 +105,9 @@ XML: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN OCTET_LENGTH(XMLSERIALIZE
 # DATE: no fractional seconds in DB2 DATE.
 DATE: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD')
 # TIME: no fractional seconds in DB2 TIME. DB2's TO_CHAR / VARCHAR_FORMAT has no
-# TIME signature (SQL0245N, ambiguous routine), so CHAR(col, JIS) renders the
-# hh:mm:ss form Snowflake's TO_CHAR(col, 'HH24:MI:SS') produces.
-TIME: CHAR("{{ col_name }}", JIS)
+# TIME signature (SQL0245N, ambiguous routine), so CHAR(col, JIS) renders hh:mm:ss,
+# padded to nine zero digits and cut like Snowflake's TIME ({{ time_text_length }}).
+TIME: LEFT(CHAR("{{ col_name }}", JIS) || '.000000000', {{ time_text_length }})
 # TIMESTAMP: ISO 8601 with T-separator and 9 fractional digits, Snowflake
 # TIMESTAMP_NTZ's .FF9 canonical form. FF9 pads TIMESTAMP(0-8) with zeros and keeps
 # digits 7-9 of TIMESTAMP(9-12), the most Snowflake stores. DB2's TO_CHAR rejects an

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_table_metadata_query.sql.j2

@@ -49,13 +49,15 @@ SELECT
     (
         SELECT CASE TYPE
             WHEN 'T' THEN 'TABLE'
+            WHEN 'G' THEN 'TABLE'
             WHEN 'V' THEN 'VIEW'
+            WHEN 'S' THEN 'VIEW'
             ELSE 'UNKNOWN'
         END
         FROM SYSCAT.TABLES
         WHERE TABSCHEMA = '{{ object_schema | escape_sql_literal }}'
           AND TABNAME = '{{ object_name | escape_sql_literal }}'
-          AND TYPE IN ('T', 'V')
+          AND TYPE IN ('T', 'G', 'V', 'S')
     ) AS OBJECT_TYPE,
     rc.ROW_COUNT
 FROM SYSCAT.COLUMNS C

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/db2/query/query_generator_db2.py

@@ -21,15 +21,22 @@ below give DB2 its own DB2-valid rendering that still matches Snowflake
   canonical wrapper and uses the base substitution with the DB2 mask.
 * ``build_float_bit_shim`` — IEEE-754 ``sign|exponent|significand`` tuple (same
   contract as Snowflake FLOAT). DB2 ``LN``/``POWER`` run in ``DOUBLE``.
+* ``build_decfloat_shim`` — DECFLOAT's exact ``CHAR`` text, canonicalized like
+  Snowflake DECFLOAT's ``TO_VARCHAR`` (a ``DOUBLE`` keeps ~16 of its 34 digits).
 """
 
 from pathlib import Path
 
 import jinja2
 
+from shared.datatypes.common import CommonType
+from shared.datatypes.db2 import Db2Type
 from snowflake.snowflake_data_validation.oracle.query.query_generator_oracle import (
     QueryGeneratorOracle,
 )
+from snowflake.snowflake_data_validation.query.decimal_text_normalization import (
+    build_decimal_text_canonical_expr,
+)
 from snowflake.snowflake_data_validation.query.float_bit_normalization import (
     build_binary64_canonical_expr,
 )
@@ -40,6 +47,7 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
     resolve_table_context_modifiers,
 )
+from snowflake.snowflake_data_validation.query.sql_type_families import SNOWFLAKE_FLOAT_TYPE_NAMES
 from snowflake.snowflake_data_validation.utils.constants import (
     NULL_HASHING_SENTINEL,
     NULL_HASHING_SENTINEL_KEY,
@@ -47,6 +55,11 @@ from snowflake.snowflake_data_validation.utils.constants import (
     ROW_HASH_COLUMN_DELIMITER_KEY,
     Platform,
 )
+from snowflake.snowflake_data_validation.utils.iso_dv_common import (
+    get_paired_target_column_types,
+    normalize_snowflake_base_type,
+)
+from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
 
 
 # Wide enough for BIGINT (19 digits) and DECIMAL(31); the trailing 0 forces the
@@ -57,8 +70,8 @@ _DB2_INTEGER_NINES = 31
 def build_db2_float_bit_canonical_expr(col_ref: str) -> str:
     """Return the exact binary64 tuple used for DB2 float validation.
 
-    ``LN``/``POWER`` must run in IEEE-754, so values (including ``DECFLOAT``)
-    are cast to ``DOUBLE`` first. A value that is already ``DOUBLE`` (including
+    ``LN``/``POWER`` must run in IEEE-754, so values are cast to ``DOUBLE``
+    first. A value that is already ``DOUBLE`` (including
     float32-widened ``REAL``) is left unchanged.
     """
     ieee_expr = col_ref
@@ -130,6 +143,31 @@ class QueryGeneratorDb2(QueryGeneratorOracle):
         value_expr = float_bit_input_expr(col_ref, data_type, dialect=self.platform.value)
         return build_db2_float_bit_canonical_expr(value_expr)
 
+    def build_decfloat_shim(self, column, col_ref: str) -> str:
+        """Canonicalize DECFLOAT from its exact ``CHAR`` text (``NaN``/``Infinity`` become markers)."""
+        return build_decimal_text_canonical_expr(f"CHAR({col_ref})", dialect=self.platform.value)
+
+    def _resolve_datatype_normalization_template(
+        self,
+        column,
+        datatypes_normalization_templates: dict[str, str],
+        table_context: TableContext,
+    ) -> str:
+        """Hash a DECFLOAT mirrored as a Snowflake FLOAT as binary64, as that target does.
+
+        DMO creates DECFLOAT; only a user-mapped target lands on FLOAT, and it
+        holds ~16 digits, so the exact decimal text could never match it.
+        """
+        if self._parse_type_precision_scale(column.data_type or "")[0] == Db2Type.DECFLOAT:
+            target_type = get_paired_target_column_types(table_context).get(column.name)
+            if isinstance(target_type, str) and normalize_snowflake_base_type(target_type) in SNOWFLAKE_FLOAT_TYPE_NAMES:
+                return datatypes_normalization_templates[CommonType.FLOAT]
+        return super()._resolve_datatype_normalization_template(
+            column,
+            datatypes_normalization_templates,
+            table_context,
+        )
+
     # ------------------------------------------------------------------
     # text_comparison_mode: apply the per-column text codec (raw/logical)
     #

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/executer/base_validation_executor.py

@@ -39,6 +39,7 @@ from snowflake.snowflake_data_validation.utils.constants import (
 from snowflake.snowflake_data_validation.utils.context import Context
 from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     attach_paired_column_types_for_iso_timestamp_dv,
+    attach_paired_time_comparison_precision,
 )
 from snowflake.snowflake_data_validation.utils.model.table_column_metadata import (
     TableColumnMetadata,
@@ -129,6 +130,11 @@ class BaseValidationExecutor(ABC):
                 table_configuration.column_mappings,
             )
         )
+        attach_paired_time_comparison_precision(
+            source_table_context,
+            target_table_context,
+            self.context.source_platform,
+        )
 
         if validations[SCHEMA_VALIDATION_KEY]:
             self.execute_schema_validation(
@@ -228,6 +234,7 @@ class BaseValidationExecutor(ABC):
             columns=source_table_column_metadata.columns,
             database_name=configured_object_name(self.context.source_platform, table_configuration.source_database),
             exclude_metrics=table_configuration.exclude_metrics,
+            include_count_distinct=bool(table_configuration.include_count_distinct),
             fully_qualified_name=table_configuration.fully_qualified_name,
             has_where_clause=table_configuration.has_where_clause,
             max_failed_rows_number=table_configuration.max_failed_rows_number,
@@ -273,6 +280,7 @@ class BaseValidationExecutor(ABC):
             columns=target_table_column_metadata.columns,
             database_name=configured_object_name(self.context.target_platform, table_configuration.target_database),
             exclude_metrics=table_configuration.exclude_metrics,
+            include_count_distinct=bool(table_configuration.include_count_distinct),
             fully_qualified_name=table_configuration.target_fully_qualified_name,
             has_where_clause=table_configuration.has_target_where_clause,
             max_failed_rows_number=table_configuration.max_failed_rows_number,

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/executer/source_validation_executor.py

@@ -128,6 +128,7 @@ class SourceValidationExecutor:
             columns=source_table_column_metadata.columns,
             database_name=configured_object_name(self.context.source_platform, table_configuration.source_database),
             exclude_metrics=table_configuration.exclude_metrics,
+            include_count_distinct=bool(table_configuration.include_count_distinct),
             fully_qualified_name=table_configuration.fully_qualified_name,
             has_where_clause=table_configuration.has_where_clause,
             max_failed_rows_number=table_configuration.max_failed_rows_number,

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_table_metadata_query.sql.j2

@@ -43,12 +43,13 @@ SELECT
         SELECT CASE object_type
             WHEN 'TABLE' THEN 'TABLE'
             WHEN 'VIEW' THEN 'VIEW'
+            WHEN 'MATERIALIZED VIEW' THEN 'VIEW'
             ELSE 'UNKNOWN'
         END
         FROM ALL_OBJECTS
         WHERE owner = '{{ object_schema | escape_sql_literal }}'
           AND object_name = '{{ object_name | escape_sql_literal }}'
-          AND object_type IN ('TABLE', 'VIEW')
+          AND object_type IN ('TABLE', 'VIEW', 'MATERIALIZED VIEW')
     ) AS OBJECT_TYPE,
     rc.ROW_COUNT,
     s.STAT_ROW_COUNT

{snowflake_data_validation-1.44.0 → snowflake_data_validation-1.45.0}/src/snowflake/snowflake_data_validation/postgresql/extractor/templates/postgresql_datatypes_normalization_templates.yaml

@@ -88,9 +88,10 @@ SERIAL: TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC), '{{ numeric_format_mask
 SMALLINT: TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC), '{{ numeric_format_mask }}'))
 SMALLSERIAL: TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC), '{{ numeric_format_mask }}'))
 TEXT: '"{{ col_name }}"'
-TIME: TO_CHAR("{{ col_name }}", 'HH24:MI:SS')
+# TIME: microseconds + '000' pad to nine digits, cut to the precision both sides share.
+TIME: LEFT(TO_CHAR("{{ col_name }}", 'HH24:MI:SS.US') || '000', {{ time_text_length }})
 TIME WITH TIME ZONE: CASE WHEN "{{ col_name }}" IS NULL THEN NULL ELSE '1970-01-01T' || TO_CHAR(CAST("{{ col_name }}" AS TIME), 'HH24:MI:SS.US') || '000' || CASE WHEN EXTRACT(TIMEZONE FROM "{{ col_name }}") < 0 THEN '-' ELSE '+' END || LPAD(CAST(FLOOR(ABS(EXTRACT(TIMEZONE FROM "{{ col_name }}")) / 3600) AS TEXT), 2, '0') || ':' || LPAD(CAST(FLOOR(MOD(ABS(EXTRACT(TIMEZONE FROM "{{ col_name }}")), 3600) / 60) AS TEXT), 2, '0') END
-TIME WITHOUT TIME ZONE: TO_CHAR("{{ col_name }}", 'HH24:MI:SS')
+TIME WITHOUT TIME ZONE: LEFT(TO_CHAR("{{ col_name }}", 'HH24:MI:SS.US') || '000', {{ time_text_length }})
 TIMESTAMP: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000'
 # timestamptz has no stored offset; compare UTC instants as ISO FF9 (+00:00).
 TIMESTAMP WITH TIME ZONE: TO_CHAR(("{{ col_name }}" AT TIME ZONE 'UTC'), 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000+00:00'