snowflake-data-validation 1.35.2__tar.gz → 1.36.0__tar.gz

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.35.2
+Version: 1.36.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -188,6 +188,23 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.36.0
+
+**New features**
+
+- Added Oracle `BFILE` locator migration support.
+
+**Improvements**
+
+- Improved SQL Server L1 validation by binding source catalogs and qualifying object references.
+
+**Bug fixes**
+
+- Fixed Redshift `SUPER` columns being read as `NULL` by the validator.
+- Fixed parameterized numeric type matching against bare normalization keys.
+- Fixed Snowflake `TO_CHAR` mask overflow by clamping to the 38-digit budget.
+- Fixed mixed-interval column binding in Jinja templates before `pyodbc` evaluation.
+
 ### v1.35.2
 
 **New features**

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/README.md

@@ -99,6 +99,23 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.36.0
+
+**New features**
+
+- Added Oracle `BFILE` locator migration support.
+
+**Improvements**
+
+- Improved SQL Server L1 validation by binding source catalogs and qualifying object references.
+
+**Bug fixes**
+
+- Fixed Redshift `SUPER` columns being read as `NULL` by the validator.
+- Fixed parameterized numeric type matching against bare normalization keys.
+- Fixed Snowflake `TO_CHAR` mask overflow by clamping to the 38-digit budget.
+- Fixed mixed-interval column binding in Jinja templates before `pyodbc` evaluation.
+
 ### v1.35.2
 
 **New features**

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml

@@ -326,7 +326,7 @@ CHAR:
     metric_query: SUM(CASE WHEN DATALENGTH(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -350,7 +350,7 @@ VARCHAR:
     metric_query: SUM(CASE WHEN DATALENGTH("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -370,7 +370,7 @@ NCHAR:
     metric_query: SUM(CASE WHEN RTRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -390,7 +390,7 @@ NVARCHAR:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST(N'|' + CAST("{{ col_name }}" AS NVARCHAR(MAX)) + N'|' AS NVARCHAR(MAX)){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -437,11 +437,11 @@ DATETIME2:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: DATETIME2
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000'
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: DATETIME2
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000'
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
@@ -463,11 +463,11 @@ DATETIMEOFFSET:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: DATETIMEOFFSET
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00' + FORMAT({{ metric_query }}, 'zzz')
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT({{ metric_query }}, 'zzz')
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: DATETIMEOFFSET
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00' + FORMAT({{ metric_query }}, 'zzz')
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT({{ metric_query }}, 'zzz')
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -31,22 +31,29 @@ NCHAR: RTRIM("{{ col_name }}")
 NVARCHAR: '"{{ col_name }}"'
 
 # ---- Date / Time types ----
-# Append '00' to FORMAT('.fffffff') so the output reaches FF9 (Snowflake TIMESTAMP_NTZ format).
-# Use '+' (not CONCAT) so a NULL input propagates to NULL — CONCAT would coerce NULL to ''.
+# DATETIME keeps 7 fractional digits + '00' pad to FF9.
+# DATETIME2 / DATETIMEOFFSET: Snowflake TIMESTAMP_* stores microseconds. FORMAT
+# '.fffffff' keeps the 7th digit (e.g. .999999900) while Snowflake TO_CHAR FF9
+# emits .999999000 for the truncated mirror. Truncate to 6 fractional digits
+# then pad '000' so hashes match ODBC/microsecond storage. LEFT keeps NULL.
+# Use '+' (not CONCAT) so a NULL input propagates to NULL.
 # FORMAT(..., 'zzz') reliably emits '+HH:MM'/'-HH:MM' for every offset, including UTC.
 DATE: FORMAT("{{ col_name }}", 'yyyy-MM-dd')
 TIME: CONVERT(VARCHAR(8), "{{ col_name }}", 108)
 DATETIME: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
-DATETIME2: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
+DATETIME2: LEFT(FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000'
 SMALLDATETIME: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.000000000')
-DATETIMEOFFSET: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00' + FORMAT("{{ col_name }}", 'zzz')
+DATETIMEOFFSET: LEFT(FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT("{{ col_name }}", 'zzz')
 
 # ---- Binary types ----
 BINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
 VARBINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
 
 # ---- Other types ----
-UNIQUEIDENTIFIER: LOWER(CONVERT(VARCHAR(36), "{{ col_name }}"))
+# Azure Synapse dedicated SQL pool always normalizes UNIQUEIDENTIFIER hex digits
+# to uppercase regardless of input casing. Use UPPER so the canonical string
+# matches the DV cases and Snowflake's VARCHAR-based representation.
+UNIQUEIDENTIFIER: UPPER(CONVERT(VARCHAR(36), "{{ col_name }}"))
 SYSNAME: CONVERT(VARCHAR(128), "{{ col_name }}")
 
 # ---- Default fallback ----

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_get_columns_metadata.sql.j2

@@ -31,6 +31,7 @@ SELECT
         END AS character_length,
     CASE WHEN c.precision = 0 THEN NULL ELSE c.precision END AS precision,
     CASE WHEN c.precision = 0 THEN NULL ELSE c.scale END AS scale,
+    c.collation_name AS collation_name,
     CASE
         WHEN t.name IN ('decimal', 'numeric')
             THEN

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_table_metadata_query.sql.j2

@@ -15,12 +15,16 @@ WITH rc_cte AS (
         END AS CHARACTER_MAXIMUM_LENGTH,
         C.NUMERIC_PRECISION AS NUMERIC_PRECISION,
         C.NUMERIC_SCALE AS NUMERIC_SCALE,
+        C.COLLATION_NAME AS COLLATION_NAME,
         C.IS_NULLABLE AS IS_NULLABLE,
         rc.ROW_COUNT
-   FROM INFORMATION_SCHEMA.COLUMNS C
+   FROM {% if object_database_ref %}{{ object_database_ref }}.{% endif %}INFORMATION_SCHEMA.COLUMNS C
 CROSS JOIN rc_cte rc
 WHERE C.TABLE_NAME = '{{ object_name | escape_sql_literal }}'
 AND C.TABLE_SCHEMA = '{{ object_schema | escape_sql_literal }}'
+{% if object_database %}
+AND C.TABLE_CATALOG = '{{ object_database | escape_sql_literal }}'
+{% endif %}
 {% if columns %}
     AND C.COLUMN_NAME IN (
     {% for column in columns %}

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -67,6 +67,16 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
         """Initialize the Azure Synapse query generator."""
         super().__init__(platform=Platform.AZURE_SYNAPSE)
 
+    def _measure_collation_tokens(self, column, metric_name: str) -> tuple[str, str]:
+        character_types = {"CHAR", "NCHAR", "VARCHAR", "NVARCHAR"}
+        if (
+            metric_name == "count_distinct"
+            and self._base_data_type(column) in character_types
+            and self._has_source_collation_metadata(column)
+        ):
+            return "", " COLLATE Latin1_General_BIN2"
+        return "", ""
+
     def _float_column_reference(self, column) -> str:
         return f'"{column.name}"'
 
@@ -96,7 +106,10 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
         rendered: dict[str, str] = {}
         for column in table_context.columns_to_validate:
             lookup_key = (column.data_type or "").strip().upper()
-            normalization_template = datatypes_normalization_templates.get(lookup_key)
+            normalization_template = self._lookup_normalization_template(
+                datatypes_normalization_templates,
+                lookup_key,
+            )
             if normalization_template is None:
                 LOGGER.debug(
                     "No normalization template for data type %s, using DEFAULT.",

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/executer/base_validation_executor.py

@@ -37,6 +37,9 @@ from snowflake.snowflake_data_validation.utils.context import Context
 from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     attach_paired_column_types_for_iso_timestamp_dv,
 )
+from snowflake.snowflake_data_validation.utils.measure_collation import (
+    mark_target_columns_for_normalized_measure_collation,
+)
 from snowflake.snowflake_data_validation.utils.model.table_column_metadata import (
     TableColumnMetadata,
 )
@@ -122,6 +125,12 @@ class BaseValidationExecutor(ABC):
             )
 
         if validations[METRICS_VALIDATION_KEY]:
+            target_table_context.columns = mark_target_columns_for_normalized_measure_collation(
+                source_table_context.platform,
+                source_table_context.columns,
+                target_table_context.columns,
+                table_configuration.column_mappings,
+            )
             self.execute_metrics_validation(
                 source_table_context=source_table_context,
                 target_table_context=target_table_context,

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/extractor/sql_queries_template_generator.py

@@ -146,11 +146,16 @@ class SQLQueriesTemplateGenerator:
             query_modifiers=query_modifiers or QueryModifiersTemplateContext.empty(),
             object_type=object_type_str,
             is_view=object_type_str == "VIEW",
+            object_database_ref="",
         )
         if platform == Platform.SNOWFLAKE.value:
             from snowflake.snowflake_data_validation.utils.helpers.helper_database import HelperDatabase
 
             render_kwargs["object_database_ref"] = HelperDatabase.snowflake_identifier_ref(database_name)
+        elif platform in (Platform.SQLSERVER.value, Platform.AZURE_SYNAPSE.value) and database_name:
+            from snowflake.snowflake_data_validation.utils.helpers.helper_database import HelperDatabase
+
+            render_kwargs["object_database_ref"] = HelperDatabase.sqlserver_identifier_ref(database_name)
         try:
             return self._render(
                 template_file,
@@ -494,8 +499,15 @@ class SQLQueriesTemplateGenerator:
         where_clause: str,
         order_by_columns: list[str] | None,
         query_modifiers: QueryModifiersTemplateContext | None = None,
+        extra_from_joins: list[str] | None = None,
     ) -> str:
-        """Render a SELECT that applies the same per-type normalization as L3 MD5."""
+        """Render a SELECT that applies the same per-type normalization as L3 MD5.
+
+        ``extra_from_joins`` (Redshift SUPER sorted-object joins) is an optional
+        list of ``LEFT JOIN ...`` fragments appended to the inner FROM clause;
+        templates that don't reference it are unaffected (Jinja Undefined is
+        falsy in ``{% if %}``).
+        """
         template_file = f"{platform}_cell_validation_data_select.sql.j2"
         try:
             return self._render(
@@ -507,6 +519,7 @@ class SQLQueriesTemplateGenerator:
                 where_clause=where_clause,
                 order_by_columns=order_by_columns or [],
                 query_modifiers=query_modifiers or QueryModifiersTemplateContext.empty(),
+                extra_from_joins=extra_from_joins or [],
             )
         except jinja2.exceptions.TemplateNotFound:
             raise ValueError(f"Template not found: {template_file}. Please create it in the {self.template_dir} directory.") from None

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_datatypes_normalization_templates.yaml

@@ -51,9 +51,9 @@ RAW: RAWTOHEX({{ col_name }})
 LONG RAW: RAWTOHEX({{ col_name }})
 # MD5 the whole BLOB with DBMS_CRYPTO.HASH (see CLOB above). CASE-guarded for NULL.
 BLOB: CASE WHEN {{ col_name }} IS NULL THEN NULL ELSE RAWTOHEX(DBMS_CRYPTO.HASH({{ col_name }}, 2)) END
-# BFILE is an external file locator; DBMS_CRYPTO cannot hash it, so it stays on a
-# bounded DBMS_LOB read.
-BFILE: RAWTOHEX(DBMS_LOB.SUBSTR({{ col_name }}, 4000, 1))
+# Validate the same DIRECTORY/filename locator text produced by migration.
+# QueryGeneratorOracle adds the statement-local helper that implements this call.
+BFILE: DMVF_BFILE_TO_PATH({{ col_name }})
 
 # ---- Row address types ----
 # ROWID and UROWID are stored as opaque row-address strings; NVL is sufficient.

snowflake_data_validation-1.36.0/src/snowflake/snowflake_data_validation/oracle/oracle_bfile.py

@@ -0,0 +1,25 @@
+"""Shared Oracle ``BFILE`` locator extraction SQL."""
+
+ORACLE_BFILE_TO_PATH_FUNCTION_NAME = "DMVF_BFILE_TO_PATH"
+ORACLE_BFILE_EXTRACTION = f"{ORACLE_BFILE_TO_PATH_FUNCTION_NAME}({{col}})"
+ORACLE_BFILE_NORMALIZATION = (
+    f"{ORACLE_BFILE_TO_PATH_FUNCTION_NAME}({{{{ col_name }}}})"
+)
+ORACLE_BFILE_WITH_FUNCTION = f"""WITH FUNCTION {ORACLE_BFILE_TO_PATH_FUNCTION_NAME}(value BFILE)
+RETURN VARCHAR2 IS
+    directory_name VARCHAR2(128);
+    file_name VARCHAR2(4000);
+BEGIN
+    IF value IS NULL THEN
+        RETURN NULL;
+    END IF;
+    DBMS_LOB.FILEGETNAME(value, directory_name, file_name);
+    RETURN directory_name || '/' || file_name;
+END;
+"""
+
+
+def prepend_oracle_bfile_function(sql: str) -> str:
+    """Add the statement-local helper when SQL contains a BFILE path call."""
+    marker = f"{ORACLE_BFILE_TO_PATH_FUNCTION_NAME}("
+    return ORACLE_BFILE_WITH_FUNCTION + sql if marker in sql else sql

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/oracle/query/query_generator_oracle.py

@@ -37,6 +37,9 @@ import pandas as pd
 from snowflake.snowflake_data_validation.oracle.extractor.metadata_extractor_oracle import (
     normalize_oracle_type,
 )
+from snowflake.snowflake_data_validation.oracle.oracle_bfile import (
+    prepend_oracle_bfile_function,
+)
 from snowflake.snowflake_data_validation.oracle.oracle_iso_normalization import (
     resolve_oracle_timestamp_metric_normalization,
     resolve_oracle_timestamp_normalization_template,
@@ -253,7 +256,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
         if order_by:
             order_by_columns = [col.name for col in table_context.index_column_collection]
 
-        return table_context.sql_generator.generate_row_md5_select_query(
+        sql = table_context.sql_generator.generate_row_md5_select_query(
             platform=self.platform.value,
             index_column_collection=table_context.index_column_collection,
             column_names_separate_by_comma=table_context.join_column_names_for_row_hash(),
@@ -265,6 +268,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
             column_groups=table_context.column_groups,
             order_by_columns=order_by_columns,
         )
+        return prepend_oracle_bfile_function(sql)
 
     def generate_cell_validation_data_query(
         self,
@@ -290,7 +294,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
             column_normalization_map,
         )
 
-        return table_context.sql_generator.generate_cell_validation_data_select_query(
+        sql = table_context.sql_generator.generate_cell_validation_data_select_query(
             platform=table_context.platform.value,
             column_collection=table_context.columns_to_validate,
             datatypes_normalization_renderer_templates=datatypes_normalization_renderer_templates,
@@ -300,6 +304,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
             order_by_columns=order_by_columns,
             query_modifiers=resolve_table_context_modifiers(table_context),
         )
+        return prepend_oracle_bfile_function(sql)
 
     def _generate_compute_md5_chunk_query(
         self,

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/postgresql/query/query_generator_postgresql.py

@@ -36,6 +36,7 @@ from snowflake.snowflake_data_validation.query.vector_element_kind import (
 from snowflake.snowflake_data_validation.utils.constants import (
     NULL_HASHING_SENTINEL,
     NULL_HASHING_SENTINEL_KEY,
+    NUMERIC_FORMAT_MASK_PLACEHOLDER,
     ROW_HASH_COLUMN_DELIMITER,
     ROW_HASH_COLUMN_DELIMITER_KEY,
     Platform,
@@ -69,6 +70,27 @@ def build_postgresql_vector_bit_shim(col_ref: str, element) -> str:
     return f"CASE WHEN {col_ref} IS NULL THEN NULL ELSE ({inner}) END"
 
 
+def build_postgresql_numeric_canonical_expr(col_ref: str, format_mask: str, scale: int) -> str:
+    """Restore FM leading zeros PostgreSQL omits for |v| < 1 to match Snowflake ``TO_VARCHAR``.
+
+    ``TO_CHAR`` with an ``FM`` mask renders ``|v| < 1`` without the integer zero
+    (``.0001``), while Snowflake ``TO_VARCHAR`` keeps it (``0.0001``). Prefix the missing
+    zero (handling the negative ``-.0001`` case) so row hashes and L3 cell comparisons stay
+    aligned. Mirrors Oracle / Teradata / Snowflake / Redshift, each of which restores the
+    leading zero in its own wrapper.
+
+    Exact zeros short-circuit because a mask-rendered zero carries no integer digit at all.
+    """
+    zero_literal = f"0.{'0' * scale}" if scale > 0 else "0"
+    inner = f"TRIM(TO_CHAR(CAST({col_ref} AS NUMERIC), '{format_mask}'))"
+    return (
+        f"CASE WHEN {col_ref} IS NULL THEN NULL "
+        f"WHEN {col_ref} = 0 THEN '{zero_literal}' "
+        f"ELSE TRIM(REGEXP_REPLACE("
+        f"TRIM(REGEXP_REPLACE({inner}, '^-\\.', '-0.')), '^\\.', '0.')) END"
+    )
+
+
 class QueryGeneratorPostgresql(QueryGeneratorBase):
     """Query generator for PostgreSQL (chunked MD5 pipeline aligned with Redshift)."""
 
@@ -97,6 +119,14 @@ class QueryGeneratorPostgresql(QueryGeneratorBase):
         element = vector_element_kind(getattr(column, "data_type", "") or "")
         return build_postgresql_vector_bit_shim(col_ref, element)
 
+    def render_normalization_template(self, template: str, column) -> str:
+        """Wrap numeric ``TO_CHAR`` output to restore the FM-suppressed leading zero."""
+        if NUMERIC_FORMAT_MASK_PLACEHOLDER in template and "TO_CHAR" in template and self._is_numeric_normalization_type(column.data_type):
+            scale = self._resolve_numeric_scale(column)
+            mask = self._build_numeric_format_mask(scale)
+            return build_postgresql_numeric_canonical_expr(f'"{column.name}"', mask, scale)
+        return super().render_normalization_template(template, column)
+
     def _render_normalization_template(self, template: str, column) -> str:
         return super()._render_normalization_template(template, column)
 

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/public/validation/cell_validation.py

@@ -37,6 +37,7 @@ def validate_cell(
     table_context: TableContext,
     index_columns: list[str] | None = None,
     target_index_columns: list[str] | None = None,
+    semi_structured_columns: frozenset[str] | None = None,
 ) -> tuple[ValidationResult, list[DuplicateKeyInfo]]:
     """Validate individual cell values between source and target.
 
@@ -58,6 +59,9 @@ def validate_cell(
             When ``None``, rows are matched positionally.
         target_index_columns: Target-side key column names.
             Defaults to *index_columns* when ``None``.
+        semi_structured_columns: Source-side column names needing recursive
+            JSON canonicalization (e.g. a Redshift SUPER column paired with
+            a Snowflake VARIANT mirror) instead of a plain string compare.
 
     Returns:
         A tuple of ``(ValidationResult, duplicate_key_infos)`` where
@@ -78,6 +82,7 @@ def validate_cell(
             target_index_columns=target_index_columns,
             platform=table_context.platform.value,
             object_id=table_context.id,
+            semi_structured_columns=semi_structured_columns,
         )
 
         return (

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/public/validation/schema_validation.py

@@ -47,6 +47,7 @@ _SCHEMA_MIGRATION_MAPPING_ENRICH_PLATFORMS: frozenset[Platform] = frozenset(
         Platform.ORACLE,
         Platform.SQLSERVER,
         Platform.REDSHIFT,
+        Platform.AZURE_SYNAPSE,
     }
 )
 

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/query/numeric_scale_resolution.py

@@ -46,6 +46,7 @@ _MONEY_SCALE_BY_TYPE = {
 
 # Snowflake NUMBER(p, s): max 38 total digits, max 37 fractional digits.
 SNOWFLAKE_MAX_NUMBER_SCALE = 37
+SNOWFLAKE_MAX_NUMBER_PRECISION = 38
 
 
 def parse_scale_value(raw: object) -> int | None:

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/query/query_generator_base.py

@@ -34,22 +34,32 @@ from snowflake.snowflake_data_validation.query.charset_normalization import (
     DefaultCharacterSetResolver,
 )
 from snowflake.snowflake_data_validation.query.numeric_scale_resolution import (
+    SNOWFLAKE_MAX_NUMBER_PRECISION,
     clamp_scale_for_platform,
 )
 from snowflake.snowflake_data_validation.utils.constants import (
+    APPLY_NORMALIZED_MEASURE_COLLATION_PROPERTY,
     COL_NAME_NO_QUOTES_PLACEHOLDER,
+    COLLATION_NAME_PROPERTY,
     COLUMN_MODIFIER_COLUMN_KEY,
     DEFAULT_MAX_COLUMNS_PER_METRICS_BATCH,
     EXPLICIT_METRIC_NORMALIZATION_COLUMN_KEY,
     FLOAT_BIT_SHIM_PLACEHOLDER,
+    MEASURE_COLLATION_PREFIX_PLACEHOLDER,
+    MEASURE_COLLATION_SUFFIX_PLACEHOLDER,
     METRIC_COLUMN_KEY,
     METRIC_COLUMN_MODIFIER_PLACEHOLDER,
     METRIC_QUERY_PLACEHOLDER,
     METRICS_TO_EXCLUDE,
     NORMALIZATION_COLUMN_KEY,
+    NOT_APPLICABLE_CRITERIA_VALUE,
     NUMERIC_FORMAT_MASK_PLACEHOLDER,
     NUMERIC_SCALE_PROPERTY,
     SCALE_KEY,
+    SEMI_STRUCTURED_EXPORT_SIZE_LIMIT_BYTES,
+    SEMI_STRUCTURED_SIZE_EXCLUDED_SENTINEL,
+    SEMI_STRUCTURED_SIZE_EXCLUDED_SENTINEL_PLACEHOLDER,
+    SEMI_STRUCTURED_SIZE_LIMIT_PLACEHOLDER,
     TEMPLATE_COLUMN_KEY,
     TYPE_COLUMN_KEY,
     VECTOR_BIT_SHIM_PLACEHOLDER,
@@ -180,6 +190,20 @@ class QueryGeneratorBase(ABC):
         if NUMERIC_FORMAT_MASK_PLACEHOLDER in rendered:
             mask = self._build_numeric_format_mask(self._resolve_numeric_scale(column))
             rendered = rendered.replace(NUMERIC_FORMAT_MASK_PLACEHOLDER, mask)
+        if SEMI_STRUCTURED_SIZE_LIMIT_PLACEHOLDER in rendered:
+            # Static constants (no per-column computation, unlike the numeric
+            # mask above) -- shared by the Redshift SUPER source and Snowflake
+            # VARIANT target size guards so oversized semi-structured values
+            # are excluded from comparison symmetrically on both sides
+            # (SEMI_STRUCTURED_EXPORT_SIZE_LIMIT_BYTES in utils/constants.py).
+            rendered = rendered.replace(
+                SEMI_STRUCTURED_SIZE_LIMIT_PLACEHOLDER,
+                str(SEMI_STRUCTURED_EXPORT_SIZE_LIMIT_BYTES),
+            )
+            rendered = rendered.replace(
+                SEMI_STRUCTURED_SIZE_EXCLUDED_SENTINEL_PLACEHOLDER,
+                SEMI_STRUCTURED_SIZE_EXCLUDED_SENTINEL,
+            )
         rendered = self._expand_vector_bit_placeholder(rendered, column)
         if FLOAT_BIT_SHIM_PLACEHOLDER not in rendered:
             return rendered
@@ -339,17 +363,24 @@ class QueryGeneratorBase(ABC):
         return scale
 
     @staticmethod
-    def _build_fm_to_char_mask(scale: int, *, integer_nines: int = 38) -> str:
+    def _build_fm_to_char_mask(scale: int, *, integer_nines: int = SNOWFLAKE_MAX_NUMBER_PRECISION) -> str:
         """Build a Snowflake/Oracle-style ``FM999…9.00…0`` mask for *scale*.
 
         ``FM`` suppresses integer digits written as ``9``, so ``|v| < 1`` renders
         without a leading zero (``.0001``). Each platform restores the leading
         zero in its own ``build_<platform>_numeric_canonical_expr`` wrapper to
-        match Snowflake ``TO_VARCHAR``'s ``0.0001``. Use 38 integer positions so
-        maximum-precision Oracle/Snowflake NUMBER values do not render as
-        overflow markers (``###``).
+        match Snowflake ``TO_VARCHAR``'s ``0.0001``. Start from 38 integer
+        positions so maximum-precision Oracle/Snowflake NUMBER values do not
+        render as overflow markers (``###``).
+
+        Snowflake counts integer and fractional placeholders against a single
+        38-digit model, rejecting anything wider with ``too many digits in
+        numeric format``. Shrink the integer run by *scale* to stay inside that
+        budget, keeping at least one nine.
         """
         scale = max(0, scale)
+        budget = SNOWFLAKE_MAX_NUMBER_PRECISION - scale
+        integer_nines = max(1, min(integer_nines, budget))
         int_part = "9" * integer_nines
         if scale == 0:
             return f"FM{int_part}"
@@ -399,6 +430,27 @@ class QueryGeneratorBase(ABC):
             return base if base else text
         return text
 
+    @classmethod
+    def _lookup_normalization_template(
+        cls,
+        templates: dict[str, str],
+        data_type: str,
+    ) -> str | None:
+        """Return the normalization template for *data_type*, or ``None``.
+
+        Tries the key verbatim first so platforms keeping parameterized keys
+        (``PERIOD(DATE)``) still win, then the ``(p[, s])``-stripped key so
+        scale-enriched types such as ``NUMBER(20, 10)`` reach their bare
+        ``NUMBER`` template instead of silently falling back to ``DEFAULT``.
+        """
+        template = templates.get(data_type)
+        if template is not None:
+            return template
+        stripped = cls._normalization_lookup_key(data_type)
+        if stripped != data_type:
+            template = templates.get(stripped)
+        return template
+
     def _metric_normalization_for_numeric(self, column) -> str | None:
         """Return a ``TO_CHAR``/``FORMAT`` wrapper for numeric min/max metrics."""
         if not self._is_numeric_normalization_type(column.data_type):
@@ -607,6 +659,17 @@ class QueryGeneratorBase(ABC):
                     table_context,
                 )
 
+                collation_prefix, collation_suffix = self._measure_collation_tokens(
+                    col,
+                    metric_name,
+                )
+                metric_template = metric_template.replace(
+                    MEASURE_COLLATION_PREFIX_PLACEHOLDER,
+                    collation_prefix,
+                ).replace(
+                    MEASURE_COLLATION_SUFFIX_PLACEHOLDER,
+                    collation_suffix,
+                )
                 metric_expr = metric_template.replace(COL_NAME_NO_QUOTES_PLACEHOLDER, col.name)
                 metric_expr = self._expand_vector_bit_placeholder(metric_expr, col)
                 if metric_name in ("min", "max") and row.get(EXPLICIT_METRIC_NORMALIZATION_COLUMN_KEY):
@@ -647,6 +710,27 @@ class QueryGeneratorBase(ABC):
         shape = MetricsResultShape(cells=cells, all_metrics=all_metrics)
         return GeneratedMetricsQuery(query_string=sql, result_shape=shape)
 
+    def _measure_collation_tokens(self, column, metric_name: str) -> tuple[str, str]:
+        """Return dialect tokens for explicit placeholders in character metric templates."""
+        return "", ""
+
+    @staticmethod
+    def _has_source_collation_metadata(column) -> bool:
+        """Return whether L1 supplied an effective collation for this column."""
+        value = getattr(column, "properties", {}).get(COLLATION_NAME_PROPERTY)
+        return isinstance(value, str) and bool(value.strip()) and value.strip().upper() != NOT_APPLICABLE_CRITERIA_VALUE
+
+    @staticmethod
+    def _uses_normalized_measure_collation(column) -> bool:
+        """Return whether orchestration marked this target measure for normalization."""
+        value = getattr(column, "properties", {}).get(APPLY_NORMALIZED_MEASURE_COLLATION_PROPERTY)
+        return value is True or (isinstance(value, str) and value.strip().casefold() == "true")
+
+    @staticmethod
+    def _base_data_type(column) -> str:
+        """Return an uppercase type name without precision or length."""
+        return str(getattr(column, "data_type", "") or "").strip().upper().split("(", 1)[0].strip()
+
     def _resolve_normalization_templates(self, table_context: TableContext) -> dict[str, str]:
         """Build a ``{column_name: rendered_expression}`` mapping for L2/L3/cell."""
         datatypes_normalization_templates = table_context.templates_loader_manager.datatypes_normalization_templates
@@ -908,7 +992,10 @@ class QueryGeneratorBase(ABC):
         datatypes_normalization_renderer_templates: dict[str, str] = {}
         for column in table_context.columns_to_validate:
             data_type = self._resolve_datatype_normalization_key(column.data_type)
-            normalization_template = datatypes_normalization_templates.get(data_type)
+            normalization_template = self._lookup_normalization_template(
+                datatypes_normalization_templates,
+                data_type,
+            )
             if normalization_template is None:
                 normalization_template = datatypes_normalization_templates.get("DEFAULT")
                 if normalization_template is None:

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/redshift/extractor/templates/redshift_cell_validation_data_select.sql.j2

@@ -9,6 +9,7 @@
 {% endfor %}
 FROM (
     SELECT{{ qm.render_select_modifier(query_modifiers, 't') }} * {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 't') }}
+    {% if extra_from_joins %}{% for j in extra_from_joins %} {{ j }}{% endfor %}{% endif %}
     {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
     {% if order_by_columns %}
     ORDER BY {% for c in order_by_columns %} "{{ c }}"{% if not loop.last %}, {% endif %}{% endfor %}

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.36.0}/src/snowflake/snowflake_data_validation/redshift/extractor/templates/redshift_chunk_row_concatenated_insert_template.sql.j2

@@ -35,6 +35,7 @@ SELECT
             {% endif %}
          {% endfor %}
      FROM {{ fully_qualified_name }}
+     {% if extra_from_joins %}{% for j in extra_from_joins %} {{ j }}{% endfor %}{% endif %}
      {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
      ORDER BY {% for index_column in index_column_collection %} "{{ index_column.name }}_IDX" {% if not loop.last %},{% endif %} {% endfor %}
      LIMIT {{ fetch }}