snowflake-data-validation 1.40.0__tar.gz → 1.41.0__tar.gz

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/.gitignore

@@ -233,6 +233,9 @@ requirements-dev.lock
 # E2E test session config (contains credentials)
 e2e-tests/config.yaml
 
+# E2E Logs
+e2e-tests/e2e-logs/
+
 # 1Password integration test config (contains account/user identifiers)
 shared/tests/secret_managers/integration/connections.toml
 

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.40.0
+Version: 1.41.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -188,6 +188,31 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.41.0
+
+**Improvements**
+
+- Normalized SQL Server and Synapse text comparisons to use binary collation ordering.
+- Normalized Oracle text comparisons with `COLLATE BINARY` and `ORA-00904` fallback.
+- Added DB2 collation metadata capture for L1 reporting.
+- Normalized BigQuery text comparisons with deterministic binary semantics.
+- Removed legacy dual-cursor row-hash compare path from hybrid L3.
+- Improved `FLOAT` and encoded text display in L3 cell validation results.
+
+**Bug fixes**
+
+- Fixed Oracle `TIMESTAMP WITH LOCAL TIME ZONE` extraction failing with `ORA-01821`.
+- Fixed rows with `NULL` validation key being reported as `NOT_FOUND` on both sides.
+- Fixed digit-only hybrid cell keys not being quoted for character columns.
+- Fixed Teradata `PERIOD(DATE)` comparison using session-sensitive cast instead of deterministic `TO_CHAR`.
+
+### v1.40.1
+
+**Bug fixes**
+
+- Fixed L1 metadata reporting for unbounded string types on SQLite and BigQuery to use correct source capacity values.
+- Fixed L1 character-length capacity comparison to report source-larger-than-target as a warning instead of a failure.
+
 ### v1.40.0
 
 **New features**

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/README.md

@@ -99,6 +99,31 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.41.0
+
+**Improvements**
+
+- Normalized SQL Server and Synapse text comparisons to use binary collation ordering.
+- Normalized Oracle text comparisons with `COLLATE BINARY` and `ORA-00904` fallback.
+- Added DB2 collation metadata capture for L1 reporting.
+- Normalized BigQuery text comparisons with deterministic binary semantics.
+- Removed legacy dual-cursor row-hash compare path from hybrid L3.
+- Improved `FLOAT` and encoded text display in L3 cell validation results.
+
+**Bug fixes**
+
+- Fixed Oracle `TIMESTAMP WITH LOCAL TIME ZONE` extraction failing with `ORA-01821`.
+- Fixed rows with `NULL` validation key being reported as `NOT_FOUND` on both sides.
+- Fixed digit-only hybrid cell keys not being quoted for character columns.
+- Fixed Teradata `PERIOD(DATE)` comparison using session-sensitive cast instead of deterministic `TO_CHAR`.
+
+### v1.40.1
+
+**Bug fixes**
+
+- Fixed L1 metadata reporting for unbounded string types on SQLite and BigQuery to use correct source capacity values.
+- Fixed L1 character-length capacity comparison to report source-larger-than-target as a warning instead of a failure.
+
 ### v1.40.0
 
 **New features**

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_cell_validation_data_select.sql.j2

@@ -10,5 +10,5 @@
 {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 't') }}
 {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
 {% if order_by_columns %}
-ORDER BY {% for c in order_by_columns %} "{{ c }}"{% if not loop.last %}, {% endif %}{% endfor %}
+ORDER BY {% for c in order_by_columns %} {{ order_by_collation_expressions.get(c, '"' ~ c ~ '"') }}{% if not loop.last %}, {% endif %}{% endfor %}
 {% endif %}

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_row_md5_select.sql.j2

@@ -1,32 +1,32 @@
 {% import 'query_modifiers.sql.j2' as qm with context %}
 {{ qm.locking_prefix(query_modifiers) }}SELECT
     {% for index_column in index_column_collection %}
-        "{{ index_column.name }}_IDX" AS "{{ index_column.name }}",
+        {{ (index_column.name ~ '_IDX') | tsql_quote_identifier }} AS {{ index_column.name | tsql_quote_identifier }},
     {% endfor %}
     UPPER(CONVERT(VARCHAR(32), HASHBYTES('MD5',
     {% if use_column_groups %}
         {% for group in column_groups %}
             {% if not loop.first %} + {% endif %}
             UPPER(CONVERT(VARCHAR(32), HASHBYTES('MD5', CAST(CONCAT(
-                {% for col in group %}"{{ col.name }}"{% if not loop.last %}, '{{ row_hash_column_delimiter }}', {% endif %}{% endfor %}
+                {% for col in group %}{{ col.name | tsql_quote_identifier }}{{ row_hash_collation_suffix }}{% if not loop.last %}, '{{ row_hash_column_delimiter }}'{{ row_hash_collation_suffix }}, {% endif %}{% endfor %}
             ) AS VARCHAR(MAX))), 2))
         {% endfor %}
     {% elif column_collection|length > 1 %}
-        CAST(CONCAT({{ column_names_separate_by_comma }}) AS VARCHAR(MAX))
+        CAST(CONCAT({% for column in column_collection %}{{ column.name | tsql_quote_identifier }}{{ row_hash_collation_suffix }}{% if not loop.last %}, '{{ row_hash_column_delimiter }}'{{ row_hash_collation_suffix }}, {% endif %}{% endfor %}) AS VARCHAR(MAX))
     {% else %}
-        CAST({{ column_names_separate_by_comma }} AS VARCHAR(MAX))
+        CAST({{ column_collection[0].name | tsql_quote_identifier }} AS VARCHAR(MAX))
     {% endif %}
     ), 2)) AS ROW_MD5
 FROM
     (SELECT{{ qm.render_select_modifier(query_modifiers, 'src') }}
         {% for index_column in index_column_collection %}
-            {{ datatypes_normalization_renderer_templates[index_column.name] }} AS "{{ index_column.name }}_IDX",
+            {{ datatypes_normalization_renderer_templates[index_column.name] }} AS {{ (index_column.name ~ '_IDX') | tsql_quote_identifier }},
         {% endfor %}
         {% for column in column_collection %}
             {% if column.nullable %}
-                coalesce({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS "{{ column.name }}"{% if not loop.last %},{% endif %}
+                coalesce({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS {{ column.name | tsql_quote_identifier }}{% if not loop.last %},{% endif %}
             {% else %}
-                {{ datatypes_normalization_renderer_templates[column.name] }} AS "{{ column.name }}"{% if not loop.last %},{% endif %}
+                {{ datatypes_normalization_renderer_templates[column.name] }} AS {{ column.name | tsql_quote_identifier }}{% if not loop.last %},{% endif %}
             {% endif %}
         {% endfor %}
     {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 'src') }}
@@ -34,5 +34,5 @@ FROM
     ) AS RW
 {% if order_by_columns %}
 {# T-SQL does not support the NULLS FIRST keyword; under ASC it already sorts NULLs first, matching NULL_KEY (sentinel sorts first) in the streaming merge-join. #}
-ORDER BY {% for c in order_by_columns %}"{{ c }}_IDX"{% if not loop.last %}, {% endif %}{% endfor %}
+ORDER BY {% for c in order_by_columns %}{{ order_by_collation_expressions.get(c, (c ~ '_IDX') | tsql_quote_identifier) }}{% if not loop.last %}, {% endif %}{% endfor %}
 {% endif %}

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -42,11 +42,14 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
     resolve_table_context_modifiers,
 )
+from snowflake.snowflake_data_validation.sqlserver.query.charset_normalization import (
+    SqlServerCharacterSetResolver,
+)
 from snowflake.snowflake_data_validation.utils.constants import (
     Platform,
 )
 from snowflake.snowflake_data_validation.utils.measure_collation import (
-    SOURCE_CHARACTER_TYPES,
+    TSQL_NORMALIZED_COLLATION_SUFFIX,
     tsql_measure_collation_tokens,
 )
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
@@ -77,17 +80,30 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
 
     def __init__(self) -> None:
         """Initialize the Azure Synapse query generator."""
-        super().__init__(platform=Platform.AZURE_SYNAPSE)
+        super().__init__(
+            platform=Platform.AZURE_SYNAPSE,
+            character_set_resolver=SqlServerCharacterSetResolver(),
+        )
 
     def _measure_collation_tokens(self, column, metric_name: str) -> tuple[str, str]:
         return tsql_measure_collation_tokens(
+            Platform.AZURE_SYNAPSE,
             column,
             metric_name,
-            character_types=SOURCE_CHARACTER_TYPES[Platform.AZURE_SYNAPSE],
+        )
+
+    def _comparison_collation_tokens(self, column) -> tuple[str, str]:
+        return tsql_measure_collation_tokens(
+            Platform.AZURE_SYNAPSE,
+            column,
+            "count_distinct",
         )
 
     def _float_column_reference(self, column) -> str:
-        return f'"{column.name}"'
+        return f'"{self._normalization_column_name(column)}"'
+
+    def _normalization_column_name(self, column) -> str:
+        return str(column.name).replace('"', '""')
 
     def build_float_bit_shim(self, column, col_ref: str) -> str:
         value_expr = float_bit_input_expr(
@@ -115,7 +131,8 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
 
         datatypes_normalization_templates = table_context.templates_loader_manager.datatypes_normalization_templates
         rendered: dict[str, str] = {}
-        for column in self._row_md5_normalization_columns(table_context):
+        renderer_columns = self._row_md5_normalization_columns(table_context)
+        for column in renderer_columns:
             lookup_key = (column.data_type or "").strip().upper()
             normalization_template = self._lookup_normalization_template(
                 datatypes_normalization_templates,
@@ -135,12 +152,27 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
                 column_normalization_map,
             )
             rendered[column.name] = self._render_normalization_template(normalization_template, column)
-        return apply_text_codec_to_renderer_templates(
+        rendered = apply_text_codec_to_renderer_templates(
             self.platform,
             rendered,
             column_codec_map,
-            table_context.columns_to_validate,
+            renderer_columns,
         )
+        return {
+            column.name: (
+                rendered[column.name]
+                if self._codec_skips_comparison_collation(column_codec_map, column)
+                else self.wrap_compare_expr(
+                    self.character_set_resolver.wrap_for_comparison(
+                        rendered[column.name],
+                        column,
+                        is_case_sensitive=table_context.is_case_sensitive,
+                    ),
+                    column,
+                )
+            )
+            for column in renderer_columns
+        }
 
     def generate_row_md5_select_query(
         self,
@@ -197,6 +229,11 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
             where_clause=table_context.where_clause,
             column_groups=table_context.column_groups,
             order_by_columns=order_by_columns,
+            order_by_collation_expressions=self._order_by_collation_expressions(
+                table_context.index_column_collection,
+                alias_suffix="_IDX",
+            ),
+            row_hash_collation_suffix=TSQL_NORMALIZED_COLLATION_SUFFIX,
             query_modifiers=resolve_table_context_modifiers(table_context),
         )
 
@@ -227,6 +264,7 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
             has_where_clause=table_context.has_where_clause,
             where_clause=table_context.where_clause,
             order_by_columns=order_by_columns,
+            order_by_collation_expressions=self._order_by_collation_expressions(table_context.columns_to_validate),
             query_modifiers=resolve_table_context_modifiers(table_context),
         )
 

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_cell_validation_data_select.sql.j2

@@ -11,6 +11,6 @@ FROM (
     SELECT{{ qm.render_select_modifier(query_modifiers, 't') }} * {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 't') }}
     {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
     {% if order_by_columns %}
-    ORDER BY {% for c in order_by_columns %} `{{ c }}`{% if not loop.last %}, {% endif %}{% endfor %}
+    ORDER BY {% for c in order_by_columns %} {{ order_by_collation_expressions.get(c, '`' ~ c ~ '`') }}{% if not loop.last %}, {% endif %}{% endfor %}
     {% endif %}
 ) __ordered

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_column_metrics_templates.yaml

@@ -279,7 +279,7 @@ STRING:
     metric_query: COUNTIF(`{{ col_name }}` = '')
     metric_return_datatype: INT64
   count_distinct:
-    metric_query: COUNT(DISTINCT `{{ col_name }}`)
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}`{{ col_name }}`{{ measure_collation_suffix }})
     metric_return_datatype: INT64
   count_null:
     metric_query: COUNTIF(`{{ col_name }}` IS NULL)

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_get_columns_metadata.sql.j2

@@ -39,6 +39,7 @@ SELECT
             THEN 0
         ELSE NULL
     END AS scale,
+    c.collation_name AS collation_name,
     -1 AS calculated_column_size_in_bytes
 FROM `{{ database_name }}`.`{{ schema_name }}`.INFORMATION_SCHEMA.COLUMNS c
 WHERE c.table_name = '{{ table_name | escape_sql_literal }}';

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_row_md5_select.sql.j2

@@ -35,5 +35,5 @@ FROM
     {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
     ) AS RW
 {% if order_by_columns %}
-ORDER BY {% for c in order_by_columns %}`{{ c }}_IDX` NULLS FIRST{% if not loop.last %}, {% endif %}{% endfor %}
+ORDER BY {% for c in order_by_columns %}{{ order_by_collation_expressions.get(c, '`' ~ c ~ '_IDX`') }} NULLS FIRST{% if not loop.last %}, {% endif %}{% endfor %}
 {% endif %}

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_table_metadata_query.sql.j2

@@ -9,11 +9,12 @@ WITH rc_cte AS (
         -- Strip parameters (NUMERIC(10,2), STRING(100), ARRAY<INT64>...) to the base
         -- type so the L1 mapping keys off it; precision/scale below use raw c.data_type.
         TRIM(UPPER(REGEXP_EXTRACT(c.data_type, r'^[^(<]+'))) AS DATA_TYPE,
-        -- STRING length lives in the type string; unbounded -> VARCHAR max. BYTES ->
-        -- BINARY, which reports no length, so emit NULL to match.
+        -- STRING length lives in the type string. Bare STRING has no parameter;
+        -- BigQuery's per-value cap is 10 MiB (10485760 bytes), not Snowflake's
+        -- VARCHAR max. BYTES -> BINARY, which reports no length, so emit NULL.
         CASE
             WHEN STARTS_WITH(UPPER(c.data_type), 'STRING')
-                THEN COALESCE(CAST(REGEXP_EXTRACT(c.data_type, r'\((\d+)\)') AS INT64), 16777216)
+                THEN COALESCE(CAST(REGEXP_EXTRACT(c.data_type, r'\((\d+)\)') AS INT64), 10485760)
             ELSE NULL
         END AS CHARACTER_MAXIMUM_LENGTH,
         -- INFORMATION_SCHEMA omits precision/scale; parse them from the type string.
@@ -43,6 +44,7 @@ WITH rc_cte AS (
             ELSE NULL
         END AS NUMERIC_SCALE,
         c.is_nullable AS IS_NULLABLE,
+        c.collation_name AS COLLATION_NAME,
         rc.ROW_COUNT
    FROM `{{ object_database }}`.`{{ object_schema }}`.INFORMATION_SCHEMA.COLUMNS c
 CROSS JOIN rc_cte rc

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/bigquery/query/query_generator_bigquery.py

@@ -76,6 +76,33 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
         """Initialize the BigQuery query generator."""
         super().__init__(platform=Platform.BIGQUERY)
 
+    def _measure_collation_tokens(self, column, metric_name: str) -> tuple[str, str]:
+        """Force binary DISTINCT semantics only for metadata-gated STRING columns."""
+        if metric_name == "count_distinct" and self._base_data_type(column) == "STRING" and self._has_source_collation_metadata(column):
+            return "COLLATE(", ", '')"
+        return "", ""
+
+    def _comparison_collation_tokens(self, column) -> tuple[str, str]:
+        """Force binary compare/sort/hash semantics for metadata-gated STRING columns."""
+        if self._base_data_type(column) == "STRING" and self._has_source_collation_metadata(column):
+            return "COLLATE(", ", '')"
+        return "", ""
+
+    def _order_by_collation_expressions(
+        self,
+        columns,
+        *,
+        alias_suffix: str = "",
+    ) -> dict[str, str]:
+        """Return BigQuery-backticked ORDER BY expressions that need binary collation."""
+        expressions: dict[str, str] = {}
+        for column in columns:
+            quoted = f"`{column.name}{alias_suffix}`"
+            wrapped = self.wrap_compare_expr(quoted, column)
+            if wrapped != quoted:
+                expressions[column.name] = wrapped
+        return expressions
+
     @staticmethod
     def normalize_bigquery_type(raw_type: str | None) -> str | None:
         """Reduce a BigQuery declared type to its parameterless base type.
@@ -189,8 +216,9 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
         column_normalization_map: dict[str, str] | None = None,
     ) -> dict[str, str]:
         datatypes_normalization_templates = table_context.templates_loader_manager.datatypes_normalization_templates
+        renderer_columns = self._row_md5_normalization_columns(table_context)
         renderer_templates: dict[str, str] = {}
-        for column in self._row_md5_normalization_columns(table_context):
+        for column in renderer_columns:
             normalization_template = self._resolve_datatype_normalization_template(
                 column,
                 datatypes_normalization_templates,
@@ -202,9 +230,22 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
             )
             # Resolve ``{{ col_name }}`` and, for numeric types, the scale-aware
             # ``{{ numeric_format_mask }}`` (BigQuery emits a printf ``%.Nf``).
-            rendered = self._render_normalization_template(normalization_template, column)
-            renderer_templates[column.name] = rendered
-        return apply_text_codec_to_renderer_templates(self.platform, renderer_templates, column_codec_map)
+            renderer_templates[column.name] = self._render_normalization_template(
+                normalization_template,
+                column,
+            )
+        renderer_templates = apply_text_codec_to_renderer_templates(
+            self.platform,
+            renderer_templates,
+            column_codec_map,
+            renderer_columns,
+        )
+        # L3 MD5 / cell fingerprints must be UTF-8 code points. COLLATE() on
+        # those expressions is a compare/sort annotation; CONCAT/MD5 treat a
+        # collated STRING as collation keys and false-diff Snowflake utf8.
+        # Index-only columns are hashed in row-MD5 even when they are not in
+        # columns_to_validate; keep their renderer entries.
+        return {column.name: renderer_templates[column.name] for column in renderer_columns}
 
     def _build_row_md5_select(
         self,
@@ -236,6 +277,10 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
             where_clause=table_context.where_clause,
             column_groups=table_context.column_groups,
             order_by_columns=order_by_columns,
+            order_by_collation_expressions=self._order_by_collation_expressions(
+                table_context.index_column_collection,
+                alias_suffix="_IDX",
+            ),
             query_modifiers=resolve_table_context_modifiers(table_context),
         )
 
@@ -261,6 +306,9 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
             has_where_clause=table_context.has_where_clause,
             where_clause=table_context.where_clause,
             order_by_columns=order_by_columns,
+            order_by_collation_expressions=self._order_by_collation_expressions(
+                [column for column in table_context.columns_to_validate if order_by_columns and column.name in order_by_columns]
+            ),
             query_modifiers=resolve_table_context_modifiers(table_context),
         )
 

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_get_columns_metadata.sql.j2

@@ -14,6 +14,9 @@ SELECT
     C.LENGTH                                    AS character_length,
     C.LENGTH                                    AS precision,
     C.SCALE                                     AS scale,
+    -- Per-column collation of a string column; NULL for every non-string column
+    -- and for string columns defined without one.
+    C.COLLATIONNAME                             AS collation_name,
     CASE
         WHEN UPPER(C.TYPENAME) IN ('CHARACTER', 'CHAR', 'VARCHAR',
                                    'NCHAR', 'NVARCHAR')

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_table_metadata_query.sql.j2

@@ -37,6 +37,9 @@ SELECT
         WHEN UPPER(C.TYPENAME) IN ('SMALLINT', 'INTEGER', 'INT', 'BIGINT')  THEN 0
         ELSE NULL
     END                     AS NUMERIC_SCALE,
+    -- Per-column collation of a string column; NULL for every non-string column
+    -- and for string columns defined without one.
+    C.COLLATIONNAME         AS COLLATION_NAME,
     CASE C.NULLS
         WHEN 'Y' THEN 'YES'
         ELSE 'NO'

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/db2/query/query_generator_db2.py

@@ -83,6 +83,21 @@ class QueryGeneratorDb2(QueryGeneratorOracle):
         self.jinja_env.globals[NULL_HASHING_SENTINEL_KEY] = NULL_HASHING_SENTINEL
         self.jinja_env.globals[ROW_HASH_COLUMN_DELIMITER_KEY] = ROW_HASH_COLUMN_DELIMITER
 
+    def _measure_collation_tokens(self, column, metric_name: str) -> tuple[str, str]:
+        """
+        Keep DB2 LUW measures unwrapped: no fixed, name-free binary equivalent exists.
+
+        L1 does capture ``SYSCAT.COLUMNS.COLLATIONNAME``, but DB2 offers no way to
+        spend it here: ``COLLATE USING IDENTITY`` is DDL (CREATE/ALTER TABLE), not
+        an expression modifier, and ``COLLATION_KEY_BIT`` takes a named collation --
+        which would put the source collation name into generated SQL.
+        """
+        return "", ""
+
+    def _comparison_collation_tokens(self, column) -> tuple[str, str]:
+        """Keep DB2 LUW compare/sort/hash expressions unwrapped (see :meth:`_measure_collation_tokens`)."""
+        return "", ""
+
     @staticmethod
     def _build_fm_to_char_mask(scale: int, *, integer_nines: int = _DB2_INTEGER_NINES) -> str:
         """Return a DB2-valid ``9…90[.0…0]`` mask (Oracle ``FM`` is rejected by DB2).

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/executer/executor_factory.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections.abc import Callable
 from dataclasses import dataclass
 
 from snowflake.snowflake_data_validation.azure_synapse.extractor.metadata_extractor_azure_synapse import (
@@ -139,6 +140,18 @@ from snowflake.snowflake_data_validation.validation.validation_database_manager
 )
 
 
+def _create_oracle_query_generator(connector: ConnectorBase) -> QueryGeneratorOracle:
+    """Create an Oracle query generator with server-derived capabilities."""
+    connection = connector.connection
+    version = getattr(connection, "version", None) if connection is not None else None
+    if not isinstance(version, str) and connection is not None:
+        getinfo = getattr(connection, "getinfo", None)
+        if callable(getinfo):
+            odbc_version = getinfo(18)  # SQL_DBMS_VER
+            version = odbc_version if isinstance(odbc_version, str) else None
+    return QueryGeneratorOracle(oracle_version=version)
+
+
 @dataclass
 class PlatformConfig:
     """Configuration for platform-specific components."""
@@ -146,6 +159,13 @@ class PlatformConfig:
     query_generator: type[QueryGeneratorBase]
     metadata_extractor: type[MetadataExtractorBase]
     script_writer: type[ScriptWriterBase]
+    query_generator_factory: Callable[[ConnectorBase], QueryGeneratorBase] | None = None
+
+    def create_query_generator(self, connector: ConnectorBase) -> QueryGeneratorBase:
+        """Create the platform query generator through its optional extension hook."""
+        if self.query_generator_factory is not None:
+            return self.query_generator_factory(connector)
+        return self.query_generator()
 
 
 class ExecutorFactory:
@@ -193,6 +213,7 @@ class ExecutorFactory:
                 query_generator=QueryGeneratorOracle,
                 metadata_extractor=MetadataExtractorOracle,
                 script_writer=ScriptWriterOracle,
+                query_generator_factory=_create_oracle_query_generator,
             ),
             Platform.POSTGRESQL: PlatformConfig(
                 query_generator=QueryGeneratorPostgresql,
@@ -295,7 +316,7 @@ class ExecutorFactory:
             raise ValueError(f"Unsupported platform: {platform}")
 
         config = self._platform_configs[platform]
-        query_generator = config.query_generator()
+        query_generator = config.create_query_generator(connector)
 
         # Propagate SQL Server version capabilities to the query generator
         if platform == Platform.SQLSERVER and isinstance(connector, ConnectorSqlServer):

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/extractor/sql_queries_template_generator.py

@@ -44,6 +44,12 @@ from snowflake.snowflake_data_validation.utils.qvci_fallback import (
 )
 
 
+def tsql_quote_identifier(identifier: object) -> str:
+    """Quote a T-SQL identifier, escaping embedded double quotes."""
+    escaped_identifier = str(identifier).replace('"', '""')
+    return f'"{escaped_identifier}"'
+
+
 class SQLQueriesTemplateGenerator:
     """
     SQL queries template generator using Jinja2 templates.
@@ -76,6 +82,7 @@ class SQLQueriesTemplateGenerator:
             )
         )
         self.env.filters["escape_sql_literal"] = escape_sql_literal
+        self.env.filters["tsql_quote_identifier"] = tsql_quote_identifier
         # Single source of truth for the NULL sentinel used by L3 row-MD5 and
         # cell-validation normalization. Exposed to every template so the
         # COALESCE fallback can keep a real SQL NULL distinct from "".
@@ -652,6 +659,7 @@ class SQLQueriesTemplateGenerator:
         order_by_collation_expressions: dict[str, str] | None = None,
         query_modifiers: QueryModifiersTemplateContext | None = None,
         hash_md5_function: str | None = None,
+        row_hash_collation_suffix: str = "",
     ) -> str:
         """Render an inline SELECT that returns (index_cols, ROW_MD5) directly.
 
@@ -672,6 +680,7 @@ class SQLQueriesTemplateGenerator:
                 use_column_groups=column_groups is not None,
                 order_by_columns=order_by_columns or [],
                 order_by_collation_expressions=order_by_collation_expressions or {},
+                row_hash_collation_suffix=row_hash_collation_suffix,
             )
             if column_groups is not None:
                 render_kwargs["column_groups"] = column_groups

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/oracle/extractor/metadata_extractor_oracle.py

@@ -273,8 +273,8 @@ class MetadataExtractorOracle(MetadataExtractorBase):
     def compute_md5(self, table_context: TableContext, other_table_name: str) -> None:
         """Not used in the Oracle CDV pipeline.
 
-        Oracle L3 row validation uses ``generate_row_md5_select_query`` (inline
-        SELECT, no temp table) via ``RowValidationHandler`` in the DEA.  The
+        Oracle L3 row hashing uses ``generate_row_md5_select_query`` (inline
+        SELECT, no temp table) on the signature-extract path.  The
         chunked CTAS pipeline (``compute_md5`` → temp table →
         ``extract_md5_rows_chunk``) is not executed for Oracle.
 

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_cell_validation_data_select.sql.j2

@@ -10,5 +10,5 @@
 {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 't') }}
 {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
 {% if order_by_columns %}
-ORDER BY {% for c in order_by_columns %} "{{ c }}"{% if not loop.last %}, {% endif %}{% endfor %}
+ORDER BY {% for c in order_by_columns %} {{ order_by_collation_expressions.get(c, '"' ~ c ~ '"') }}{% if not loop.last %}, {% endif %}{% endfor %}
 {% endif %}

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_column_metrics_templates.yaml

@@ -315,7 +315,7 @@ CHAR:
     metric_query: AVG(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: FLOAT
   count_distinct:
-    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -336,7 +336,7 @@ NCHAR:
     metric_query: AVG(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: FLOAT
   count_distinct:
-    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -356,7 +356,7 @@ VARCHAR2:
     metric_query: AVG(LENGTH("{{ col_name }}"))
     metric_return_datatype: FLOAT
   count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}"{{ col_name }}"{{ measure_collation_suffix }})
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -379,7 +379,7 @@ NVARCHAR2:
     metric_query: AVG(LENGTH("{{ col_name }}"))
     metric_return_datatype: FLOAT
   count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}"{{ col_name }}"{{ measure_collation_suffix }})
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -400,7 +400,7 @@ VARCHAR:
     metric_query: AVG(LENGTH("{{ col_name }}"))
     metric_return_datatype: FLOAT
   count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}"{{ col_name }}"{{ measure_collation_suffix }})
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_datatypes_normalization_templates.yaml

@@ -77,7 +77,8 @@ SDO_GEOMETRY: CASE WHEN {{ col_name }} IS NULL THEN NULL ELSE TRIM(REGEXP_REPLAC
 # Format masks aligned with Snowflake TIMESTAMP_NTZ / TIMESTAMP_TZ normalization:
 #   DATE / TIMESTAMP → ISO 8601 with 9-digit fractional seconds (FF9)
 #   TIMESTAMP WITH TIME ZONE → ISO with per-row offset (TZH:TZM)
-#   TIMESTAMP WITH LOCAL TIME ZONE → session wall-clock ISO (no UTC extract)
+#   TIMESTAMP WITH LOCAL TIME ZONE → generic session wall clock; Oracle/Snowflake
+#     paired DV overrides this template with UTC-instant normalization.
 DATE: TO_CHAR(CAST({{ col_name }} AS TIMESTAMP), 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
 TIMESTAMP: TO_CHAR({{ col_name }}, 'YYYY-MM-DD"T"HH24:MI:SS.FF9')
 TIMESTAMP WITH TIME ZONE: REGEXP_REPLACE(TO_CHAR({{ col_name }}, 'YYYY-MM-DD"T"HH24:MI:SS.FF9TZH:TZM'), 'Z$', '+00:00')

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_get_columns_metadata.sql.j2

@@ -27,6 +27,9 @@ SELECT
     C.CHAR_LENGTH                               AS character_length,
     C.DATA_PRECISION                            AS precision,
     C.DATA_SCALE                                AS scale,
+{% if supports_oracle_collation %}
+    C.COLLATION                                 AS collation_name,
+{% endif %}
     CASE
         WHEN C.DATA_TYPE IN ('CHAR', 'NCHAR', 'VARCHAR2', 'NVARCHAR2')
             THEN C.CHAR_LENGTH

{snowflake_data_validation-1.40.0 → snowflake_data_validation-1.41.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_row_md5_select.sql.j2

@@ -37,5 +37,5 @@ FROM
     {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
     ) RW
 {% if order_by_columns %}
-ORDER BY {% for c in order_by_columns %}"{{ c }}_IDX" NULLS FIRST{% if not loop.last %}, {% endif %}{% endfor %}
+ORDER BY {% for c in order_by_columns %}{{ order_by_collation_expressions.get(c, '"' ~ c ~ '_IDX"') }} NULLS FIRST{% if not loop.last %}, {% endif %}{% endfor %}
 {% endif %}