snowflake-data-validation 1.45.2__tar.gz → 1.45.3__tar.gz

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.45.2
+Version: 1.45.3
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -79,7 +79,7 @@ Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgresql'
 Provides-Extra: redshift
 Requires-Dist: redshift-connector>=2.1.14; extra == 'redshift'
 Provides-Extra: scai
-Requires-Dist: snowflake-code-unit-registry>=0.9.2; extra == 'scai'
+Requires-Dist: snowflake-code-unit-registry>=0.9.5; extra == 'scai'
 Provides-Extra: sqlite
 Requires-Dist: sqlean-py>=3.45.0; extra == 'sqlite'
 Provides-Extra: sqlserver
@@ -160,8 +160,12 @@ Some differences are reported as `WARNING` and do not fail L1:
 
 A nullable source column on a `NOT NULL` Snowflake column still fails, since Snowflake can reject the source's NULLs.
 
+An integer column mapped to a `NUMBER` with scale 0 is the exception to the precision warning: SQL Server `INT` (precision 10) in `NUMBER(38,0)` holds every value, so its `NUMERIC_PRECISION` row is `SUCCESS`.
+
 A PostgreSQL user-defined `ENUM` column is validated as `ENUM` against a Snowflake `VARCHAR`. `information_schema` reports every enum as `USER-DEFINED`, so the metadata queries read `pg_type.typtype` to tell enums apart, and report the longest label as the column's length. L2 and L3 compare the label text.
 
+Snowflake database, schema and table names may contain a backslash or an apostrophe, for example `NAFMAIN\TUSDB102`. Snowflake reads backslash escapes inside a string literal, so the Snowflake metadata queries (table metadata, column metadata, case-sensitive columns and object-type lookups) write each name through `escape_snowflake_sql_literal` / `quote_snowflake_sql_literal` in `shared/queries/sql_literals.py`, which double backslashes as well as single quotes. Use these helpers, not `escape_sql_literal`, for any name embedded in a Snowflake string literal.
+
 ### Metrics Validation
 Compares statistical metrics for each column:
 - Row count
@@ -226,6 +230,17 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.45.3
+
+**Bug fixes**
+
+- Fixed schema validation reporting SQL Server `INT` columns mapped to `NUMBER(38,0)` as precision differences.
+- Surfaced rounding of scale-less `NUMBER` values as a lossy mapping instead of a hidden match.
+- Fixed schema validation reporting every column as missing when the schema name contains a backslash (for example `DOMAIN\user` schemas).
+- Fixed Teradata `VARCHAR` distinct and empty-value metrics not matching when values have trailing spaces.
+- Fixed Iceberg timestamp comparisons to use `FF6` or `FF9` fractional precision based on the catalog format version.
+- Fixed metrics validation for signed-year `TIMESTAMP` values, including BC and post-9999 dates.
+
 ### v1.45.2
 
 **Bug fixes**

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/README.md

@@ -69,8 +69,12 @@ Some differences are reported as `WARNING` and do not fail L1:
 
 A nullable source column on a `NOT NULL` Snowflake column still fails, since Snowflake can reject the source's NULLs.
 
+An integer column mapped to a `NUMBER` with scale 0 is the exception to the precision warning: SQL Server `INT` (precision 10) in `NUMBER(38,0)` holds every value, so its `NUMERIC_PRECISION` row is `SUCCESS`.
+
 A PostgreSQL user-defined `ENUM` column is validated as `ENUM` against a Snowflake `VARCHAR`. `information_schema` reports every enum as `USER-DEFINED`, so the metadata queries read `pg_type.typtype` to tell enums apart, and report the longest label as the column's length. L2 and L3 compare the label text.
 
+Snowflake database, schema and table names may contain a backslash or an apostrophe, for example `NAFMAIN\TUSDB102`. Snowflake reads backslash escapes inside a string literal, so the Snowflake metadata queries (table metadata, column metadata, case-sensitive columns and object-type lookups) write each name through `escape_snowflake_sql_literal` / `quote_snowflake_sql_literal` in `shared/queries/sql_literals.py`, which double backslashes as well as single quotes. Use these helpers, not `escape_sql_literal`, for any name embedded in a Snowflake string literal.
+
 ### Metrics Validation
 Compares statistical metrics for each column:
 - Row count
@@ -135,6 +139,17 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.45.3
+
+**Bug fixes**
+
+- Fixed schema validation reporting SQL Server `INT` columns mapped to `NUMBER(38,0)` as precision differences.
+- Surfaced rounding of scale-less `NUMBER` values as a lossy mapping instead of a hidden match.
+- Fixed schema validation reporting every column as missing when the schema name contains a backslash (for example `DOMAIN\user` schemas).
+- Fixed Teradata `VARCHAR` distinct and empty-value metrics not matching when values have trailing spaces.
+- Fixed Iceberg timestamp comparisons to use `FF6` or `FF9` fractional precision based on the catalog format version.
+- Fixed metrics validation for signed-year `TIMESTAMP` values, including BC and post-9999 dates.
+
 ### v1.45.2
 
 **Bug fixes**

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/pyproject.toml

@@ -4,6 +4,7 @@ requires = ["hatchling"]
 
 [tool.uv]
 constraint-dependencies = [
+  "certifi>=2026.10.23,<2026.11",
   "virtualenv>=21.7.13,<21.8",
 ]
 
@@ -105,7 +106,7 @@ db2 = [
 # musllinux build, and SDV is published to public PyPI. Inside DMO/DEA the
 # dependency is already declared, so SDV's logger tree is covered there.
 scai = [
-  "snowflake-code-unit-registry>=0.9.2",
+  "snowflake-code-unit-registry>=0.9.5",
 ]
 
 [project.urls]

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/extractor/sql_queries_template_generator.py

@@ -26,7 +26,7 @@ import snowflake.snowflake_data_validation as sdv_pkg
 
 from shared.identifiers.managers import identifier_manager
 from shared.identifiers.sql_identifier import SqlIdentifier
-from shared.queries.sql_literals import escape_sql_literal
+from shared.queries.sql_literals import escape_snowflake_sql_literal, escape_sql_literal
 from shared.query_modifiers.strict import check_query_modifiers_strict
 from shared.query_modifiers.template_context import QueryModifiersTemplateContext
 from snowflake.snowflake_data_validation.utils.constants import (
@@ -132,6 +132,7 @@ class SQLQueriesTemplateGenerator:
         # Identifiers are written in Python (write_sql_name, column_sql_names)
         # and handed to the templates already written; only literals are escaped here.
         self.env.filters["escape_sql_literal"] = escape_sql_literal
+        self.env.filters["escape_snowflake_sql_literal"] = escape_snowflake_sql_literal
         # Single source of truth for the NULL sentinel used by L3 row-MD5 and
         # cell-validation normalization. Exposed to every template so the
         # COALESCE fallback can keep a real SQL NULL distinct from "".
@@ -351,6 +352,39 @@ class SQLQueriesTemplateGenerator:
         except Exception as e:
             raise Exception(f"Error generating template: {e}") from e
 
+    def get_iceberg_format_version(self, database_name: str, schema_name: str, table_name: str, platform: str) -> str:
+        """Generate the query that reads one target table's Iceberg format version.
+
+        Args:
+            database_name (str): Database that contains the table.
+            schema_name (str): Schema that contains the table.
+            table_name (str): Table whose ``ICEBERG_VERSION`` parameter is read.
+            platform (str): Platform identifier (``Platform.SNOWFLAKE.value``).
+
+        Returns:
+            str: The SQL query. The ``value`` column is ``ICEBERG_FORMAT_VERSION_V2``
+            or ``ICEBERG_FORMAT_VERSION_V3``.
+
+        Raises:
+            ValueError: If the template file is not found.
+            Exception: If an error occurs during template rendering.
+
+        """
+        template_file = f"{platform}_get_iceberg_format_version.sql.j2"
+        try:
+            return self._render(
+                template_file,
+                database_sql=write_sql_name(database_name, platform),
+                schema_sql=write_sql_name(schema_name, platform),
+                table_sql=write_sql_name(table_name, platform),
+            )
+        except jinja2.exceptions.TemplateNotFound:
+            raise ValueError(
+                f"Template not found for platform: {platform}. Please create {template_file} in the {self.template_dir} directory."
+            ) from None
+        except Exception as e:
+            raise Exception(f"Error generating template: {e}") from e
+
     def generate_compute_md5_query(
         self,
         platform: str,

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/oracle/query/query_generator_oracle.py

@@ -77,6 +77,7 @@ from snowflake.snowflake_data_validation.utils.constants import (
     ROW_HASH_COLUMN_DELIMITER,
     ROW_HASH_COLUMN_DELIMITER_KEY,
     Platform,
+    renders_full_precision,
 )
 from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     get_paired_target_column_types,
@@ -156,6 +157,28 @@ def build_oracle_numeric_canonical_expr(col_ref: str, format_mask: str) -> str:
     )
 
 
+def build_oracle_full_precision_numeric_expr(col_ref: str, format_mask: str) -> str:
+    """Render a value that fits the mask's scale through the mask, any other at its stored precision.
+
+    For a bare NUMBER the target scale was never checked against the data. A value
+    the target could hold renders exactly like :func:`build_oracle_numeric_canonical_expr`,
+    the text Snowflake prints. A value with more fractional digits renders with
+    ``TM9`` (shortest exact text, scientific only past 64 characters, which no
+    NUMBER(38, s) value reaches), so the target's rounding shows as a cell diff.
+    """
+    scale = _oracle_scale_from_fm_mask(format_mask)
+    full_text = f"TRIM(REGEXP_REPLACE(TRIM(REGEXP_REPLACE(TRIM(TO_CHAR({col_ref}, 'TM9')), '^-\\.', '-0.')), '^\\.', '0.'))"
+    return (
+        f"CASE WHEN {col_ref} IS NULL THEN NULL "
+        f"WHEN {col_ref} = ROUND({col_ref}, {scale}) THEN {build_oracle_numeric_canonical_expr(col_ref, format_mask)} "
+        f"ELSE {full_text} END"
+    )
+
+
+def _renders_full_precision(column: object) -> bool:
+    return renders_full_precision(getattr(column, "properties", None))
+
+
 class QueryGeneratorOracle(QueryGeneratorBase):
     """Oracle-specific implementation of the query generator.
 
@@ -250,7 +273,10 @@ class QueryGeneratorOracle(QueryGeneratorBase):
         """Wrap numeric ``TO_CHAR`` output to restore the FM-suppressed leading zero."""
         if NUMERIC_FORMAT_MASK_PLACEHOLDER in template and "TO_CHAR" in template and self._is_numeric_normalization_type(column.data_type):
             mask = self._build_numeric_format_mask(self._resolve_numeric_scale(column))
-            return build_oracle_numeric_canonical_expr(self._write_sql_identifier(column.name), mask)
+            col_ref = self._write_sql_identifier(column.name)
+            if _renders_full_precision(column):
+                return build_oracle_full_precision_numeric_expr(col_ref, mask)
+            return build_oracle_numeric_canonical_expr(col_ref, mask)
         return super().render_normalization_template(template, column)
 
     def build_float_bit_shim(self, column, col_ref: str) -> str:

snowflake_data_validation-1.45.3/src/snowflake/snowflake_data_validation/query/normalization_render_context.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from typing import TYPE_CHECKING
+
+from snowflake.snowflake_data_validation.utils.constants import Platform
+from snowflake.snowflake_data_validation.utils.iceberg_dv_common import IcebergProfile
+
+
+if TYPE_CHECKING:
+    from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
+
+
+@dataclass(frozen=True)
+class NormalizationRenderContext:
+    """Per-render inputs a normalization template needs beyond the column itself.
+
+    Resolve the per-table inputs once with ``from_table_context``, then specialize
+    per column with ``for_column``. A new render input becomes a field here instead
+    of another parameter on ``render_normalization_template``.
+    """
+
+    paired_source_platform: Platform | None = None
+    paired_source_type: str | None = None
+    iceberg_fraction_digits: int = IcebergProfile().fraction_digits
+    has_custom_normalization: bool = False
+
+    @classmethod
+    def from_table_context(cls, table_context: TableContext) -> NormalizationRenderContext:
+        """Resolve the per-table inputs; ``for_column`` adds the per-column paired type."""
+        return cls(
+            paired_source_platform=table_context.paired_source_platform,
+            iceberg_fraction_digits=IcebergProfile.from_table_context(table_context).fraction_digits,
+        )
+
+    def for_column(
+        self,
+        paired_source_type: str | None,
+        *,
+        has_custom_normalization: bool = False,
+    ) -> NormalizationRenderContext:
+        """Return a copy specialized for one column; the base context is left unchanged.
+
+        Sets the column's paired source type and whether its template is a user
+        override. ``has_custom_normalization`` resets to False unless passed.
+        """
+        return replace(
+            self,
+            paired_source_type=paired_source_type,
+            has_custom_normalization=has_custom_normalization,
+        )

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/query/query_generator_base.py

@@ -58,6 +58,7 @@ from snowflake.snowflake_data_validation.utils.constants import (
     DEFAULT_MAX_COLUMNS_PER_METRICS_BATCH,
     EXPLICIT_METRIC_NORMALIZATION_COLUMN_KEY,
     FLOAT_BIT_SHIM_PLACEHOLDER,
+    ICEBERG_FRACTION_DIGITS_PLACEHOLDER,
     MEASURE_COLLATION_PREFIX_PLACEHOLDER,
     MEASURE_COLLATION_SUFFIX_PLACEHOLDER,
     METRIC_COLUMN_KEY,
@@ -892,6 +893,7 @@ class QueryGeneratorBase(ABC):
         # set the attribute keep the default instead of a truthy Mock.
         include_count_distinct = getattr(table_context, "include_count_distinct", False) is True
         apply_metric_column_modifier = table_context.apply_metric_column_modifier
+        iceberg_fraction_digits = IcebergProfile.from_table_context(table_context).fraction_digits
 
         select_exprs: list[str] = []
         cells: list[tuple[str, str]] = []
@@ -953,6 +955,10 @@ class QueryGeneratorBase(ABC):
                         if NUMERIC_FORMAT_MASK_PLACEHOLDER in normalization:
                             mask = self._build_numeric_format_mask(self._resolve_numeric_scale(col))
                             normalization = normalization.replace(NUMERIC_FORMAT_MASK_PLACEHOLDER, mask)
+                        if ICEBERG_FRACTION_DIGITS_PLACEHOLDER in normalization:
+                            normalization = normalization.replace(
+                                ICEBERG_FRACTION_DIGITS_PLACEHOLDER, str(iceberg_fraction_digits)
+                            )
                         metric_expr = self._wrap_metric_with_normalization(normalization, metric_expr)
                 alias = f"c{measured_idx}_m{metric_idx}"
                 select_exprs.append(f"{metric_expr} AS {alias}")

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/redshift/extractor/templates/redshift_column_metrics_templates.yaml

@@ -619,6 +619,22 @@ TIMESTAMP WITHOUT TIME ZONE:
      metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
      metric_return_datatype: BIGINT
 
+# Iceberg VARCHAR target (SNOW-4353618): MIN/MAX of the instant, rendered as the
+# signed astronomical-year text the UNLOAD footer writes (``TO_CHAR 'YYYY'``
+# drops the era). Kept in sync with DMO by test_redshift_consistency.py.
+TIMESTAMP_AS_TEXT:
+  min:
+    metric_query: MIN("{{ col_name }}")
+    metric_return_datatype: VARCHAR
+    metric_normalization: '(CASE WHEN (CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END) < 0 THEN ''-'' ELSE '''' END || (CASE WHEN ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END)) < 1000 THEN LPAD(ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR, 4, ''0'') ELSE ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR END)) || TO_CHAR({{ metric_query }}, ''-MM-DD"T"HH24:MI:SS.US'') || ''000'''
+  max:
+    metric_query: MAX("{{ col_name }}")
+    metric_return_datatype: VARCHAR
+    metric_normalization: '(CASE WHEN (CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END) < 0 THEN ''-'' ELSE '''' END || (CASE WHEN ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END)) < 1000 THEN LPAD(ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR, 4, ''0'') ELSE ABS((CASE WHEN EXTRACT(YEAR FROM {{ metric_query }})::INT < 0 THEN EXTRACT(YEAR FROM {{ metric_query }})::INT + 1 ELSE EXTRACT(YEAR FROM {{ metric_query }})::INT END))::VARCHAR END)) || TO_CHAR({{ metric_query }}, ''-MM-DD"T"HH24:MI:SS.US'') || ''000'''
+  count_null:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
+    metric_return_datatype: BIGINT
+
 TIMESTAMPTZ:
   min: 
     metric_query: MIN("{{ col_name }}")

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/redshift/query/query_generator_redshift.py

@@ -62,6 +62,7 @@ from snowflake.snowflake_data_validation.utils.constants import (
     SchemaValidationFields,
 )
 from snowflake.snowflake_data_validation.utils.iceberg_dv_common import (
+    TIMESTAMP_AS_TEXT_KEY,
     IcebergProfile,
     resolve_iceberg_v2_compatible_source_normalization_key,
 )
@@ -71,6 +72,7 @@ from snowflake.snowflake_data_validation.utils.interval_common import (
 )
 from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     get_paired_target_column_types,
+    is_snowflake_varchar_like_type,
 )
 from snowflake.snowflake_data_validation.utils.measure_collation import (
     COUNT_DISTINCT_METRIC,
@@ -83,6 +85,7 @@ from snowflake.snowflake_data_validation.utils.model.table_context import TableC
 
 LOGGER = logging.getLogger(__name__)
 _REDSHIFT_NORMALIZED_COLLATION_TOKENS = ("COLLATE(", ", 'case_sensitive')")
+_TIMESTAMP_BASE_TYPES = frozenset({CommonType.TIMESTAMP, CommonType.TIMESTAMP_WITHOUT_TIME_ZONE})
 
 # Redshift ``TO_CHAR`` formats numbers with 64-bit arithmetic. A numeric format
 # mask with N total digit placeholders (integer nines + fractional zeros) admits
@@ -752,6 +755,21 @@ class QueryGeneratorRedshift(QueryGeneratorBase):
             raise ValueError(f"No normalization template found for data type {data_type} and no DEFAULT template available.")
         return normalization_template
 
+    def _resolve_metrics_col_type(self, col, table_context: TableContext) -> str:
+        """Key an Iceberg VARCHAR TIMESTAMP to the signed-year metrics (SNOW-4353618).
+
+        ``TO_CHAR 'YYYY'`` drops the era, so MIN / MAX would not match the text
+        the UNLOAD staged. Native targets keep the shared TIMESTAMP template.
+        """
+        col_type = super()._resolve_metrics_col_type(col, table_context)
+        if not IcebergProfile.from_table_context(table_context).is_active:
+            return col_type
+        if col_type.upper() not in _TIMESTAMP_BASE_TYPES:
+            return col_type
+        if not is_snowflake_varchar_like_type(get_paired_target_column_types(table_context).get(col.name)):
+            return col_type
+        return TIMESTAMP_AS_TEXT_KEY
+
     def _resolve_normalization_templates(
         self,
         table_context: TableContext,

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_column_metrics_templates.yaml

@@ -249,6 +249,22 @@ VARCHAR_TIMESTAMP:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: NUMBER
 
+# Signed-year ISO text (Redshift Iceberg, SNOW-4353618): TRY_TO_TIMESTAMP_NTZ
+# returns NULL for negative and 5+ digit years, so parse from parts like the
+# Iceberg column comment's cast back (DMO iceberg_column_comment.py).
+VARCHAR_TIMESTAMP_SIGNED:
+  min:
+    metric_query: 'MIN(TIMESTAMP_NTZ_FROM_PARTS(IFF(TRY_TO_NUMBER(REGEXP_SUBSTR(TO_VARCHAR(TRY_TO_DATE(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''^(-?[0-9]+-[0-9]{2}-[0-9]{2})'', 1, 1, ''e'', 1)), ''YYYY-MM-DD''), ''^(-?[0-9]+)-'', 1, 1, ''e'', 1), 38, 0) = TRY_TO_NUMBER(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''^(-?[0-9]+)-'', 1, 1, ''e'', 1), 38, 0), TRY_TO_DATE(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''^(-?[0-9]+-[0-9]{2}-[0-9]{2})'', 1, 1, ''e'', 1)), NULL), TRY_TO_TIME(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''T([0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?)'', 1, 1, ''e'', 1))))'
+    metric_return_datatype: VARCHAR
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
+  max:
+    metric_query: 'MAX(TIMESTAMP_NTZ_FROM_PARTS(IFF(TRY_TO_NUMBER(REGEXP_SUBSTR(TO_VARCHAR(TRY_TO_DATE(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''^(-?[0-9]+-[0-9]{2}-[0-9]{2})'', 1, 1, ''e'', 1)), ''YYYY-MM-DD''), ''^(-?[0-9]+)-'', 1, 1, ''e'', 1), 38, 0) = TRY_TO_NUMBER(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''^(-?[0-9]+)-'', 1, 1, ''e'', 1), 38, 0), TRY_TO_DATE(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''^(-?[0-9]+-[0-9]{2}-[0-9]{2})'', 1, 1, ''e'', 1)), NULL), TRY_TO_TIME(REGEXP_SUBSTR(TRIM("{{ col_name }}"), ''T([0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?)'', 1, 1, ''e'', 1))))'
+    metric_return_datatype: VARCHAR
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
+  count_null:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
+    metric_return_datatype: NUMBER
+
 # Order by UTC time-of-day. A plain MIN of the parsed instant uses today's date
 # and can cross midnight, so the formatted value would keep the wrong offset.
 VARCHAR_TIME_TZ:

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_datatypes_normalization_templates.yaml

@@ -11,20 +11,22 @@ NUMBER: TO_CHAR("{{ col_name }}", '{{ numeric_format_mask }}')
 # truncates the same way on every engine; a CAST to TIME(p) would round.
 TIME: LEFT(TO_CHAR("{{ col_name }}", 'HH24:MI:SS.FF9'), {{ time_text_length }})
 TIME WITH TIME ZONE: LEFT(TO_CHAR("{{ col_name }}", 'HH24:MI:SS.FF9'), {{ time_text_length }})
-TIMESTAMP_LTZ: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
+# Typed Snowflake timestamps. The fractional-second digits are the Iceberg Parquet
+# format version (v2 micro FF6 / v3 nano FF9), substituted at render from
+# resolve_iceberg_fraction_digits; non-Iceberg paths default to FF9.
+TIMESTAMP_LTZ: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF{{ fraction_digits }}')
 # Paired Oracle TIMESTAMP WITH LOCAL TIME ZONE: compare the retained instant in UTC.
-TIMESTAMP_LTZ_UTC: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
-TIMESTAMP_NTZ: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
-TIMESTAMP_TZ: REGEXP_REPLACE(TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF9TZH:TZM'), 'Z$', '+00:00')
+TIMESTAMP_LTZ_UTC: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF{{ fraction_digits }}')
+TIMESTAMP_NTZ: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF{{ fraction_digits }}')
+TIMESTAMP_TZ: REGEXP_REPLACE(TO_CHAR("{{ col_name }}", 'YYYY-MM-DD"T"HH24:MI:SS.FF{{ fraction_digits }}TZH:TZM'), 'Z$', '+00:00')
 # Paired PostgreSQL timestamptz: compare UTC instants (Oracle/Teradata use TIMESTAMP_TZ).
-TIMESTAMP_TZ_UTC: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF9') || '+00:00'
-VARCHAR_ISO_TZ: REGEXP_REPLACE(TO_CHAR(TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}")), 'YYYY-MM-DD"T"HH24:MI:SS.FF9TZH:TZM'), 'Z$', '+00:00')
+TIMESTAMP_TZ_UTC: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF{{ fraction_digits }}') || '+00:00'
+VARCHAR_ISO_TZ: REGEXP_REPLACE(TO_CHAR(TRY_TO_TIMESTAMP_TZ(TRIM("{{ col_name }}")), 'YYYY-MM-DD"T"HH24:MI:SS.FF{{ fraction_digits }}TZH:TZM'), 'Z$', '+00:00')
 # Canonical UTC-instant form for every Snowflake TIMESTAMP_LTZ mirror of a
 # UTC-instant TIMESTAMPTZ source (Redshift / PostgreSQL), reached from either
 # the native path (iceberg_spec and storage_mode both None, target
 # TIMESTAMP_LTZ / TIMESTAMP_LTZ(6) / TIMESTAMP_LTZ(9)) or the Iceberg v2
-# compatible overlay (TIMESTAMP_LTZ(6)). Source side emits the same
-# FF9-padded +00:00 string.
+# compatible overlay (TIMESTAMP_LTZ(6)). Always FF9-padded +00:00 on both sides.
 TIMESTAMP_LTZ_UTC_FF9_OFFSET: TO_CHAR(CONVERT_TIMEZONE('UTC', "{{ col_name }}"), 'YYYY-MM-DD"T"HH24:MI:SS.FF9') || '+00:00'
 # Complex values flattened to VARCHAR under Iceberg compatible DDL retain the
 # exact Redshift JSON_SERIALIZE text staged by UNLOAD. Compare that migration

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_get_case_sensitive_columns.sql.j2

@@ -6,11 +6,11 @@ SELECT
     (
         SELECT MAX(T.IS_ICEBERG)
         FROM {{ database_sql }}.INFORMATION_SCHEMA.TABLES T
-        WHERE T.TABLE_NAME = '{{ table_name | escape_sql_literal }}'
-            AND T.TABLE_SCHEMA = '{{ schema_name | escape_sql_literal }}'
+        WHERE T.TABLE_NAME = '{{ table_name | escape_snowflake_sql_literal }}'
+            AND T.TABLE_SCHEMA = '{{ schema_name | escape_snowflake_sql_literal }}'
     ) AS IS_ICEBERG
 FROM
     {{ database_sql }}.INFORMATION_SCHEMA.COLUMNS
 WHERE
-    TABLE_NAME = '{{ table_name | escape_sql_literal }}'
-    AND TABLE_SCHEMA = '{{ schema_name | escape_sql_literal }}';
+    TABLE_NAME = '{{ table_name | escape_snowflake_sql_literal }}'
+    AND TABLE_SCHEMA = '{{ schema_name | escape_snowflake_sql_literal }}';

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_get_columns_metadata.sql.j2

@@ -12,6 +12,6 @@ SELECT
     NUMERIC_SCALE AS SCALE,
     0 AS CALCULATED_COLUMN_SIZE_IN_BYTES
 FROM {{ database_sql }}.INFORMATION_SCHEMA.COLUMNS
-WHERE TABLE_CATALOG = '{{ database_name | escape_sql_literal }}'
-AND TABLE_SCHEMA = '{{ schema_name | escape_sql_literal }}'
-AND TABLE_NAME = '{{ table_name | escape_sql_literal }}';
+WHERE TABLE_CATALOG = '{{ database_name | escape_snowflake_sql_literal }}'
+AND TABLE_SCHEMA = '{{ schema_name | escape_snowflake_sql_literal }}'
+AND TABLE_NAME = '{{ table_name | escape_snowflake_sql_literal }}';

snowflake_data_validation-1.45.3/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_get_iceberg_format_version.sql.j2

@@ -0,0 +1,2 @@
+{# INFORMATION_SCHEMA.TABLES has IS_ICEBERG only. ICEBERG_VERSION is this table's format version. #}
+SHOW PARAMETERS LIKE 'ICEBERG_VERSION' IN TABLE {{ database_sql }}.{{ schema_sql }}.{{ table_sql }}

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/snowflake/extractor/templates/snowflake_table_metadata_query.sql.j2

@@ -24,16 +24,16 @@ SELECT
             ELSE 'UNKNOWN'
         END
         FROM {{ object_database_ref }}.INFORMATION_SCHEMA.TABLES
-        WHERE TABLE_NAME = '{{ object_name }}'
-          AND TABLE_SCHEMA = '{{ object_schema }}'
-          AND TABLE_CATALOG = '{{ object_database }}'
+        WHERE TABLE_NAME = '{{ object_name | escape_snowflake_sql_literal }}'
+          AND TABLE_SCHEMA = '{{ object_schema | escape_snowflake_sql_literal }}'
+          AND TABLE_CATALOG = '{{ object_database | escape_snowflake_sql_literal }}'
     ) AS OBJECT_TYPE,
     rc.ROW_COUNT
 FROM {{ object_database_ref }}.INFORMATION_SCHEMA.COLUMNS C
 CROSS JOIN rc_cte rc
-WHERE C.TABLE_NAME = '{{ object_name }}'
-AND C.TABLE_SCHEMA = '{{ object_schema }}'
-AND C.TABLE_CATALOG = '{{ object_database }}'
+WHERE C.TABLE_NAME = '{{ object_name | escape_snowflake_sql_literal }}'
+AND C.TABLE_SCHEMA = '{{ object_schema | escape_snowflake_sql_literal }}'
+AND C.TABLE_CATALOG = '{{ object_database | escape_snowflake_sql_literal }}'
 {% if columns %}
     {% if use_as_exclude_list %}
     AND C.COLUMN_NAME NOT IN (
@@ -41,7 +41,7 @@ AND C.TABLE_CATALOG = '{{ object_database }}'
     AND C.COLUMN_NAME IN (
     {% endif %}
         {% for column in columns %}
-            '{{ column }}'{% if not loop.last %}, {% endif %}
+            '{{ column | escape_snowflake_sql_literal }}'{% if not loop.last %}, {% endif %}
         {% endfor %}
     )
 {% endif %}

{snowflake_data_validation-1.45.2 → snowflake_data_validation-1.45.3}/src/snowflake/snowflake_data_validation/snowflake/query/query_generator_snowflake.py

@@ -53,6 +53,9 @@ from snowflake.snowflake_data_validation.query.float_aggregate_reduction import
 from snowflake.snowflake_data_validation.query.float_bit_normalization import (
     build_binary64_canonical_expr,
 )
+from snowflake.snowflake_data_validation.query.normalization_render_context import (
+    NormalizationRenderContext,
+)
 from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
     resolve_table_context_modifiers,
@@ -80,6 +83,7 @@ from snowflake.snowflake_data_validation.utils.constants import (
     COL_NAME_NO_QUOTES_PLACEHOLDER,
     COL_NAME_QUOTES_PLACEHOLDER,
     EXPLICIT_METRIC_NORMALIZATION_COLUMN_KEY,
+    ICEBERG_FRACTION_DIGITS_PLACEHOLDER,
     METRIC_QUERY_PLACEHOLDER,
     NORMALIZATION_COLUMN_KEY,
     NUMERIC_FORMAT_MASK_PLACEHOLDER,
@@ -100,6 +104,7 @@ from snowflake.snowflake_data_validation.utils.measure_collation import (
     SNOWFLAKE_NORMALIZED_COLLATION_TOKENS,
     TARGET_CHARACTER_TYPES,
 )
+from snowflake.snowflake_data_validation.utils.model.column_metadata import ColumnMetadata
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
 from snowflake.snowflake_data_validation.utils.model.templates_loader_manager import TemplatesLoaderManager
 from snowflake.snowflake_data_validation.utils.period_common import (
@@ -108,6 +113,7 @@ from snowflake.snowflake_data_validation.utils.period_common import (
 from snowflake.snowflake_data_validation.utils.teradata_string_rtrim import (
     is_blank_padded_char_type,
     should_rtrim_snowflake_for_padded_source,
+    snowflake_rtrims_teradata_variable_string_comparison,
     wrap_rtrim,
 )
 from snowflake.snowflake_data_validation.utils.vector_common import SnowflakeVectorElement
@@ -299,23 +305,21 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
         self,
         template: str,
         column,
-        *,
-        paired_source_platform: Platform | None = None,
-        paired_source_type: str | None = None,
-        has_custom_normalization: bool = False,
+        context: NormalizationRenderContext | None = None,
     ) -> str:
         """Render *template* for *column*; a FLOAT column hashes its binary64 tuple.
 
-        ``has_custom_normalization`` marks *template* as the user's (a column's
+        ``context.has_custom_normalization`` marks *template* as the user's (a column's
         ``targetExpression``, or a workflow or custom templates YAML override of its
         type): it is rendered as written, FLOAT included, as the source side renders
         its own.
         """
+        context = context or NormalizationRenderContext()
         target_base, _, _ = self._parse_type_precision_scale(getattr(column, "data_type", "") or "")
-        if target_base in SNOWFLAKE_FLOAT_TYPE_NAMES and not has_custom_normalization:
+        if target_base in SNOWFLAKE_FLOAT_TYPE_NAMES and not context.has_custom_normalization:
             return build_snowflake_float_bit_canonical_expr(write_identifier_for_sql(column.name))
         if (
-            paired_source_platform == Platform.TERADATA
+            context.paired_source_platform == Platform.TERADATA
             and NUMERIC_FORMAT_MASK_PLACEHOLDER in template
             and "TO_CHAR" in template
             and self._is_numeric_normalization_type(column.data_type)
@@ -324,9 +328,10 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
             col_ref = write_identifier_for_sql(column.name)
             return build_snowflake_fm_numeric_canonical_expr(col_ref, mask)
         rendered = super().render_normalization_template(template, column)
+        rendered = rendered.replace(ICEBERG_FRACTION_DIGITS_PLACEHOLDER, str(context.iceberg_fraction_digits))
         if should_rtrim_snowflake_for_padded_source(
             getattr(column, "data_type", "") or "",
-            paired_source_type,
+            context.paired_source_type,
             template,
         ):
             return wrap_rtrim(rendered)
@@ -421,6 +426,7 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
     ) -> dict[str, str]:
         datatypes_normalization_templates = table_context.templates_loader_manager.datatypes_normalization_templates
         paired_source_types = get_paired_source_column_types(table_context)
+        base_context = NormalizationRenderContext.from_table_context(table_context)
         renderer_templates: dict[str, str] = {}
         renderer_columns = self._row_md5_normalization_columns(table_context)
         for column in renderer_columns:
@@ -449,13 +455,14 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
             rendered = self.render_normalization_template(
                 normalization_template,
                 column,
-                paired_source_platform=table_context.paired_source_platform,
-                paired_source_type=paired_source_types.get(column.name),
-                has_custom_normalization=has_custom_column_normalization(
-                    column.name,
-                    column.data_type,
-                    column_normalization_map,
-                    table_context.templates_loader_manager,
+                base_context.for_column(
+                    paired_source_types.get(column.name),
+                    has_custom_normalization=has_custom_column_normalization(
+                        column.name,
+                        column.data_type,
+                        column_normalization_map,
+                        table_context.templates_loader_manager,
+                    ),
                 ),
             )
             renderer_templates[column.name] = rendered
@@ -554,9 +561,13 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
         Fixed-width CHAR pads to its declared width and the migrated Snowflake
         VARCHAR/TEXT often keeps or drops that padding depending on the load
         path. Lengths and distinct counts only agree once both sides drop
-        trailing ASCII spaces. SQL Server text columns RTRIM their length and empty
-        metrics too, to match ``LEN`` (SNOW-4283493). Other VARCHAR pairings compare
-        strings as stored.
+        trailing ASCII spaces. Teradata VARCHAR and LONG VARCHAR are not padded,
+        but their comparison ignores trailing spaces, so ``count_distinct`` and
+        ``count_empty`` RTRIM on Snowflake. A column override or a custom metrics
+        YAML is left as written. Length metrics stay as stored.
+        SQL Server text columns RTRIM their length and empty metrics too, to
+        match ``LEN`` (SNOW-4283493). Other VARCHAR pairings compare strings
+        as stored.
 
         Teradata float avg/sum/stddev use the signed significand instead of
         ``% 2097143``, which does not match Teradata ``MOD`` on extreme values.
@@ -594,7 +605,7 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
             return metric_template
         return reduced
 
-    def _is_user_metric(self, col, metric_name: str, table_context: TableContext) -> bool:
+    def _is_user_metric(self, col: ColumnMetadata, metric_name: str, table_context: TableContext) -> bool:
         """Return whether a column override, a workflow override or a custom YAML defines this metric."""
         override = table_context.metric_overrides_for_column(col.name).get(metric_name)
         if override is not None and override.metric_query:
@@ -602,9 +613,9 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
         col_type = self._resolve_metrics_col_type(col, table_context).upper()
         return (col_type, metric_name.lower()) in table_context.templates_loader_manager.overridden_metrics
 
-    @staticmethod
     def _rtrims_paired_string_metric(
-        col,
+        self,
+        col: ColumnMetadata,
         metric_name: str,
         paired_source_type: str | None,
         metric_template: str,
@@ -618,6 +629,14 @@ class QueryGeneratorSnowflake(QueryGeneratorBase):
             metric_name,
         ):
             return True
+        if snowflake_rtrims_teradata_variable_string_comparison(
+            table_context.paired_source_platform,
+            paired_source_type,
+            col.data_type,
+            metric_name,
+            metric_template,
+        ) and not self._is_user_metric(col, metric_name, table_context):
+            return True
         if not is_blank_padded_char_type(paired_source_type or ""):
             return False
         return should_rtrim_snowflake_for_padded_source(col.data_type, paired_source_type, metric_template)