snowflake-data-validation 1.23.1__tar.gz → 1.24.2__tar.gz

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowflake-data-validation
-Version: 1.23.1
+Version: 1.24.2
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -129,6 +129,8 @@ pip install "snowflake-data-validation[sqlite]"
 
 > **SQLite prerequisite:** This extra installs [sqlean.py](https://github.com/nalgeon/sqlean), a drop-in `sqlite3` replacement that bundles the [sqlean-crypto](https://github.com/nalgeon/sqlean/blob/main/docs/crypto.md) extension. The `crypto_md5` function provided by this extension is required for row-level MD5 computation during SQLite data validation. Wheels are available for **Linux and macOS** only; Windows is not supported by `sqlean.py`.
 
+> **Teradata L3 prerequisite:** Hybrid row validation on Teradata requires a **`HASH_MD5` UDF** installed in the **same database** as the Teradata connection default (the database the worker uses when it connects). Generated SQL calls `HASH_MD5(...)` without qualification; pointing at a UDF in another database is not supported. See [teradata-dv-prerequisites.md](../docs/data-migration-orchestrator/teradata-dv-prerequisites.md) in this repo.
+
 For development and testing:
 
 ```bash
@@ -216,6 +218,42 @@ For complete command reference, configuration options, and examples, see the [Da
 
 ## Changelog
 
+### v1.24.2
+
+**New features**
+
+- Added character set normalization foundation for cross-platform encoding validation.
+
+**Improvements**
+
+- Improved float canonical rendering to full precision for L2/L3 validation.
+- Improved numeric canonical rendering with scale-aware handling for T-SQL and Snowflake.
+- Added query modifier support for SQL Server and Synapse.
+
+**Bug fixes**
+
+- Fixed Teradata L3 row-hash computation.
+
+### v1.24.1
+
+**New features**
+
+- Added charset normalization foundation for cross-platform character encoding validation.
+- Added scale-aware numeric canonical rendering for T-SQL and Snowflake.
+- Added query modifier support for SQL Server and Synapse.
+
+**Improvements**
+
+- Improved float canonical rendering to full precision for L2/L3 validation.
+
+**Bug fixes**
+
+- Fixed Teradata L3 row-hash computation.
+
+### v1.24.0
+
+No user-facing changes in this release.
+
 ### v1.23.1
 
 **New features**

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/README.md

@@ -43,6 +43,8 @@ pip install "snowflake-data-validation[sqlite]"
 
 > **SQLite prerequisite:** This extra installs [sqlean.py](https://github.com/nalgeon/sqlean), a drop-in `sqlite3` replacement that bundles the [sqlean-crypto](https://github.com/nalgeon/sqlean/blob/main/docs/crypto.md) extension. The `crypto_md5` function provided by this extension is required for row-level MD5 computation during SQLite data validation. Wheels are available for **Linux and macOS** only; Windows is not supported by `sqlean.py`.
 
+> **Teradata L3 prerequisite:** Hybrid row validation on Teradata requires a **`HASH_MD5` UDF** installed in the **same database** as the Teradata connection default (the database the worker uses when it connects). Generated SQL calls `HASH_MD5(...)` without qualification; pointing at a UDF in another database is not supported. See [teradata-dv-prerequisites.md](../docs/data-migration-orchestrator/teradata-dv-prerequisites.md) in this repo.
+
 For development and testing:
 
 ```bash
@@ -130,6 +132,42 @@ For complete command reference, configuration options, and examples, see the [Da
 
 ## Changelog
 
+### v1.24.2
+
+**New features**
+
+- Added character set normalization foundation for cross-platform encoding validation.
+
+**Improvements**
+
+- Improved float canonical rendering to full precision for L2/L3 validation.
+- Improved numeric canonical rendering with scale-aware handling for T-SQL and Snowflake.
+- Added query modifier support for SQL Server and Synapse.
+
+**Bug fixes**
+
+- Fixed Teradata L3 row-hash computation.
+
+### v1.24.1
+
+**New features**
+
+- Added charset normalization foundation for cross-platform character encoding validation.
+- Added scale-aware numeric canonical rendering for T-SQL and Snowflake.
+- Added query modifier support for SQL Server and Synapse.
+
+**Improvements**
+
+- Improved float canonical rendering to full precision for L2/L3 validation.
+
+**Bug fixes**
+
+- Fixed Teradata L3 row-hash computation.
+
+### v1.24.0
+
+No user-facing changes in this release.
+
 ### v1.23.1
 
 **New features**

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -10,19 +10,19 @@
 # CASE WHEN (IIF is unsupported on Synapse Dedicated SQL Pool).
 
 # ---- Numeric types ----
-BIGINT: FORMAT("{{ col_name }}", '0.0000')
-INT: FORMAT("{{ col_name }}", '0.0000')
-SMALLINT: FORMAT("{{ col_name }}", '0.0000')
-TINYINT: FORMAT("{{ col_name }}", '0.0000')
-DECIMAL: FORMAT("{{ col_name }}", '0.0000')
-NUMERIC: FORMAT("{{ col_name }}", '0.0000')
-MONEY: FORMAT(ROUND(CAST("{{ col_name }}" AS DECIMAL(38, 4)), 0), '0.0000')
-SMALLMONEY: FORMAT(ROUND(CAST("{{ col_name }}" AS DECIMAL(38, 4)), 0), '0.0000')
-FLOAT: FORMAT("{{ col_name }}", '0.0000')
-REAL: FORMAT("{{ col_name }}", '0.0000')
+BIGINT: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+INT: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+SMALLINT: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+TINYINT: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+DECIMAL: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+NUMERIC: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+MONEY: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+SMALLMONEY: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
+FLOAT: '{{ float_tme_shim }}'
+REAL: '{{ float_tme_shim }}'
 
 # ---- Boolean type ----
-BIT: CASE WHEN "{{ col_name }}" = 1 THEN 'true' ELSE 'false' END
+BIT: CASE WHEN "{{ col_name }}" = 1 THEN 'TRUE' WHEN "{{ col_name }}" = 0 THEN 'FALSE' END
 
 # ---- Character types ----
 # TRIM aligns fixed-width CHAR/NCHAR padding and trailing VARCHAR spaces with Snowflake.
@@ -43,8 +43,8 @@ SMALLDATETIME: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.000000000')
 DATETIMEOFFSET: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00' + FORMAT("{{ col_name }}", 'zzz')
 
 # ---- Binary types ----
-BINARY: CONVERT(VARCHAR(MAX), "{{ col_name }}", 2)
-VARBINARY: CONVERT(VARCHAR(MAX), "{{ col_name }}", 2)
+BINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
+VARBINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
 
 # ---- Other types ----
 UNIQUEIDENTIFIER: LOWER(CONVERT(VARCHAR(36), "{{ col_name }}"))

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -34,8 +34,10 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
     resolve_table_context_modifiers,
 )
+from snowflake.snowflake_data_validation.sqlserver.query.query_generator_sqlserver import (
+    build_sqlserver_float_tme_shim,
+)
 from snowflake.snowflake_data_validation.utils.constants import (
-    COL_NAME_NO_QUOTES_PLACEHOLDER,
     Platform,
 )
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
@@ -57,6 +59,12 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
         """Initialize the Azure Synapse query generator."""
         super().__init__(platform=Platform.AZURE_SYNAPSE)
 
+    def _float_column_reference(self, column) -> str:
+        return f'"{column.name}"'
+
+    def build_float_tme_shim(self, column, col_ref: str) -> str:
+        return build_sqlserver_float_tme_shim(col_ref)
+
     def _resolve_normalization_templates(self, table_context: TableContext) -> dict[str, str]:
         """Build a ``{column_name: rendered_expression}`` mapping.
 
@@ -76,7 +84,7 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
                 normalization_template = datatypes_normalization_templates.get("DEFAULT")
                 if normalization_template is None:
                     raise ValueError(f"No normalization template found for data type {column.data_type} and no DEFAULT template available.")
-            rendered[column.name] = normalization_template.replace(COL_NAME_NO_QUOTES_PLACEHOLDER, column.name)
+            rendered[column.name] = self._render_normalization_template(normalization_template, column)
         return rendered
 
     def generate_row_md5_select_query(

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/configuration/model/table_configuration.py

@@ -118,27 +118,19 @@ class TableConfiguration(BaseModel):
     def _load_source_decomposed_fully_qualified_name(self) -> None:
         self.fully_qualified_name = self.fully_qualified_name.replace('\\"', '"')
 
-        decomposed_tuple = Helper.get_decomposed_fully_qualified_name(
-            self.fully_qualified_name
-        )
+        decomposed_tuple = Helper.get_decomposed_fully_qualified_name(self.fully_qualified_name)
 
         self.source_database = decomposed_tuple[0]
         self.source_schema = decomposed_tuple[1]
         self.source_table = decomposed_tuple[2]
 
-    def _load_target_fully_qualified_name(
-        self, target_platform: Platform | None = None
-    ) -> None:
-        target_capability = TargetPlatformFactory.create(
-            target_platform or Platform.SNOWFLAKE
-        )
+    def _load_target_fully_qualified_name(self, target_platform: Platform | None = None) -> None:
+        target_capability = TargetPlatformFactory.create(target_platform or Platform.SNOWFLAKE)
         normalize = target_capability.normalize_target_identifier
         self.target_database = normalize(self.target_database or self.source_database)
         self.target_schema = normalize(self.target_schema or self.source_schema)
         self.target_name = normalize(self.target_name or self.source_table)
-        self.target_fully_qualified_name = (
-            f"{self.target_database}.{self.target_schema}.{self.target_name}"
-        )
+        self.target_fully_qualified_name = f"{self.target_database}.{self.target_schema}.{self.target_name}"
 
     def _set_has_where_clause(self) -> None:
         self.has_where_clause = self.where_clause != ""
@@ -147,9 +139,18 @@ class TableConfiguration(BaseModel):
         self.has_target_where_clause = self.target_where_clause != ""
 
     def _check_target_where_clause(self) -> None:
+        # Source and target are filtered independently; a single-sided filter compares
+        # different row subsets and almost always reports mismatches. Warn either way.
         if self.has_where_clause and not self.has_target_where_clause:
             LOGGER.warning(
-                "Target where clause was not set for table %s. This may lead to unexpected results.",
+                "Source where clause is set but target where clause is missing for table %s. "
+                "Set both so the same rows are compared on source and target.",
+                self.target_fully_qualified_name,
+            )
+        elif self.has_target_where_clause and not self.has_where_clause:
+            LOGGER.warning(
+                "Target where clause is set but source where clause is missing for table %s. "
+                "Set both so the same rows are compared on source and target.",
                 self.target_fully_qualified_name,
             )
 
@@ -159,9 +160,7 @@ class TableConfiguration(BaseModel):
 
         local_target_index_column_list = []
         for target_column_name in self.target_index_column_list:
-            new_column_name = self.column_mappings.get(
-                target_column_name, target_column_name
-            )
+            new_column_name = self.column_mappings.get(target_column_name, target_column_name)
             local_target_index_column_list.append(new_column_name)
 
         self.target_index_column_list = local_target_index_column_list
@@ -189,8 +188,7 @@ class TableConfiguration(BaseModel):
             return
         elif self.max_failed_rows_number < 1:
             raise ValueError(
-                f"Invalid value for max failed rows number in table f{self.fully_qualified_name}. "
-                "Value must be greater than or equal to 1."
+                f"Invalid value for max failed rows number in table f{self.fully_qualified_name}. Value must be greater than or equal to 1."
             )
 
     def _normalize_where_clause(self) -> None:
@@ -198,18 +196,13 @@ class TableConfiguration(BaseModel):
             self.where_clause = HelperDatabase.remove_escape_quotes(self.where_clause)
 
         if self.has_target_where_clause:
-            self.target_where_clause = HelperDatabase.remove_escape_quotes(
-                self.target_where_clause
-            )
+            self.target_where_clause = HelperDatabase.remove_escape_quotes(self.target_where_clause)
 
     def _set_is_temporary_table(self) -> None:
         self.is_temporary_table = self.object_type != ObjectType.TABLE
 
     def _normalize_column_selection_list(self) -> None:
-        self.column_selection_list = [
-            HelperDatabase.remove_escape_quotes(col)
-            for col in self.column_selection_list
-        ]
+        self.column_selection_list = [HelperDatabase.remove_escape_quotes(col) for col in self.column_selection_list]
 
     def _set_internal_identifiers(
         self,

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/extractor/sql_queries_template_generator.py

@@ -641,6 +641,7 @@ class SQLQueriesTemplateGenerator:
         fully_qualified_temporary_table_name: str,
         fully_qualified_view_name: str,
         platform: str,
+        query_modifiers=None,
     ) -> str:
         """
         Generate the SQL query to materialize a view as a temporary table.
@@ -649,6 +650,7 @@ class SQLQueriesTemplateGenerator:
             fully_qualified_temporary_table_name (str): The fully qualified name of the temporary table.
             fully_qualified_view_name (str): The fully qualified name of the view.
             platform (str): The platform identifier (e.g., Platform.SNOWFLAKE.value, Platform.SQLSERVER.value)
+            query_modifiers: Resolved query modifiers for the source view scan.
 
         Returns:
             str: SQL query string to materialize the view as a temporary table.
@@ -660,6 +662,7 @@ class SQLQueriesTemplateGenerator:
                 template_file,
                 fully_qualified_temporary_table_name=fully_qualified_temporary_table_name,
                 fully_qualified_view_name=fully_qualified_view_name,
+                query_modifiers=query_modifiers,
             )
         except jinja2.exceptions.TemplateNotFound:
             raise ValueError(f"Template not found: {template_file}. Please create it in the {self.template_dir} directory.") from None

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_datatypes_normalization_templates.yaml

@@ -14,16 +14,16 @@
 # ---- Numeric types ----
 # TRIM() strips the leading spaces that TO_CHAR produces, matching Snowflake's FM format output.
 NUMBER: TRIM(TO_CHAR({{ col_name }}, '999999999999999999999999999990.0000'))
-FLOAT: TRIM(TO_CHAR(CAST({{ col_name }} AS NUMBER(28,4)), 'FM999999999999999999999999999990.0000'))
-BINARY_FLOAT: TRIM(TO_CHAR(CAST({{ col_name }} AS NUMBER(28,4)), 'FM999999999999999999999999999990.0000'))
-BINARY_DOUBLE: TRIM(TO_CHAR(CAST({{ col_name }} AS NUMBER(28,4)), 'FM999999999999999999999999999990.0000'))
+FLOAT: '{{ float_tme_shim }}'
+BINARY_FLOAT: '{{ float_tme_shim }}'
+BINARY_DOUBLE: '{{ float_tme_shim }}'
 INTEGER: TRIM(TO_CHAR({{ col_name }}, '999999999999999999999999999990.0000'))
 SMALLINT: TRIM(TO_CHAR({{ col_name }}, '999999999999999999999999999990.0000'))
 # ANSI synonyms — Oracle stores these as NUMBER/INTEGER/FLOAT(63) internally
 DECIMAL: TRIM(TO_CHAR({{ col_name }}, '999999999999999999999999999990.0000'))
 NUMERIC: TRIM(TO_CHAR({{ col_name }}, '999999999999999999999999999990.0000'))
 INT: TRIM(TO_CHAR({{ col_name }}, '999999999999999999999999999990.0000'))
-REAL: TRIM(TO_CHAR(CAST({{ col_name }} AS NUMBER(28,4)), 'FM999999999999999999999999999990.0000'))
+REAL: '{{ float_tme_shim }}'
 
 # ---- Character types ----
 # NCHAR and NVARCHAR2 must be explicitly cast to VARCHAR2 before concatenation.
@@ -43,12 +43,12 @@ NCLOB: NVL(RTRIM(CAST({{ col_name }} AS VARCHAR2(4000))), '')
 LONG: NVL(RTRIM({{ col_name }}), '')
 
 # ---- Binary / Large Object types ----
-RAW: "'0x' || RAWTOHEX({{ col_name }})"
-LONG RAW: "'0x' || RAWTOHEX({{ col_name }})"
-BLOB: "'0x' || RAWTOHEX(DBMS_LOB.SUBSTR({{ col_name }}, 4000, 1))"
+RAW: RAWTOHEX({{ col_name }})
+LONG RAW: RAWTOHEX({{ col_name }})
+BLOB: RAWTOHEX(DBMS_LOB.SUBSTR({{ col_name }}, 4000, 1))
 # BFILE is an external file locator; reads the first 4000 bytes via DBMS_LOB.
 # Requires the file to be accessible from the DB server.
-BFILE: "'0x' || RAWTOHEX(DBMS_LOB.SUBSTR({{ col_name }}, 4000, 1))"
+BFILE: RAWTOHEX(DBMS_LOB.SUBSTR({{ col_name }}, 4000, 1))
 
 # ---- Row address types ----
 # ROWID and UROWID are stored as opaque row-address strings; TRIM + NVL is sufficient.

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/oracle/query/query_generator_oracle.py

@@ -48,7 +48,6 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
     resolve_table_context_modifiers,
 )
 from snowflake.snowflake_data_validation.utils.constants import (
-    COL_NAME_NO_QUOTES_PLACEHOLDER,
     NULL_HASHING_SENTINEL,
     NULL_HASHING_SENTINEL_KEY,
     Platform,
@@ -62,6 +61,27 @@ from snowflake.snowflake_data_validation.utils.model.table_context import TableC
 LOGGER = logging.getLogger(__name__)
 
 
+def build_oracle_float_tme_shim(col_ref: str) -> str:
+    """Return inline Oracle SQL that renders *col_ref* in Snowflake ``TME`` shape."""
+    base = f"LOWER(TRIM(TO_CHAR({col_ref}, 'TME')))"
+    exponent_pos = f"NULLIF(INSTR({base}, 'e'), 0)"
+    mantissa = f"SUBSTR({base}, 1, {exponent_pos} - 1)"
+    exponent_int = f"CAST(SUBSTR({base}, {exponent_pos} + 1, 50) AS NUMBER)"
+    decimal_pos = f"NULLIF(INSTR({mantissa}, '.'), 0)"
+    fraction = f"SUBSTR({mantissa}, {decimal_pos} + 1, 50)"
+    fraction_trimmed = f"RTRIM({fraction}, '0')"
+    mantissa_trimmed = (
+        f"CASE WHEN {decimal_pos} IS NULL THEN {mantissa} "
+        f"ELSE SUBSTR({mantissa}, 1, {decimal_pos} - 1) || "
+        f"CASE WHEN {fraction_trimmed} IS NULL OR {fraction_trimmed} = '' THEN '' ELSE '.' || {fraction_trimmed} END END"
+    )
+    return (
+        f"CASE WHEN {col_ref} IS NULL THEN NULL "
+        f"WHEN {exponent_pos} IS NULL THEN UPPER({base}) "
+        f"ELSE ({mantissa_trimmed}) || 'E' || CAST({exponent_int} AS VARCHAR2(11)) END"
+    )
+
+
 class QueryGeneratorOracle(QueryGeneratorBase):
     """Oracle-specific implementation of the query generator.
 
@@ -78,6 +98,9 @@ class QueryGeneratorOracle(QueryGeneratorBase):
         self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(templates_dir)))
         self.jinja_env.globals[NULL_HASHING_SENTINEL_KEY] = NULL_HASHING_SENTINEL
 
+    def build_float_tme_shim(self, column, col_ref: str) -> str:
+        return build_oracle_float_tme_shim(col_ref)
+
     def generate_extract_md5_rows_chunk_query(self, chunk_id: str, table_context: TableContext) -> str:
         """Not used in the Oracle CDV pipeline.
 
@@ -130,7 +153,10 @@ class QueryGeneratorOracle(QueryGeneratorBase):
                 datatypes_normalization_templates,
                 table_context,
             )
-            renderer_templates[column.name] = normalization_template.replace(COL_NAME_NO_QUOTES_PLACEHOLDER, column.name)
+            renderer_templates[column.name] = self.render_normalization_template(
+                normalization_template,
+                column,
+            )
         return renderer_templates
 
     def _build_row_md5_select(

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/orchestration/view_preprocessor.py

@@ -1,3 +1,4 @@
+from shared.query_modifiers.template_context import QueryModifiersTemplateContext
 from snowflake.snowflake_data_validation.configuration.model.view_configuration import (
     ViewConfiguration,
 )
@@ -49,6 +50,7 @@ class ViewPreProcessor:
             fully_qualified_temporary_table_name=view_configuration.internal_fully_qualified_name,
             platform=context.source_platform,
             sql_generator=context.sql_generator,
+            query_modifiers=QueryModifiersTemplateContext.empty(),
         )
 
         # Execute the materialization (create temporary table)

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/postgresql/extractor/templates/postgresql_datatypes_normalization_templates.yaml

@@ -3,7 +3,7 @@
 # align with ST_ASWKT on the Snowflake side. Native point uses subscripts;
 # PostGIS geometry/geography use ST_AsText when present.
 # CAST(... AS ...) is used instead of :: casts so YAML parses unambiguously.
-# BOOLEAN / BOOL: emit lowercase true/false to match Snowflake TO_CHAR(boolean).
+# BOOLEAN / BOOL: emit uppercase true/false to match Snowflake.
 # NUMERIC / DECIMAL / approximate floats: align canonical text with Snowflake ``TO_CHAR`` on the
 # mirror (Snowflake normalization templates unchanged here — Postgres adjusts to match).
 # DECIMAL / NUMERIC columns round-trip through NUMERIC(38, 12) before TO_CHAR so Postgres output
@@ -29,12 +29,12 @@ BIGINT: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST("{{ col_name }}"
 BIGSERIAL: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST("{{ col_name }}" AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END
 BIT: TRIM(CAST("{{ col_name }}" AS TEXT))
 BIT VARYING: TRIM(CAST("{{ col_name }}" AS TEXT))
-BOOL: TRIM(CASE WHEN "{{ col_name }}" IS TRUE THEN 'true' WHEN "{{ col_name }}" IS FALSE THEN 'false' END)
-BOOLEAN: TRIM(CASE WHEN "{{ col_name }}" IS TRUE THEN 'true' WHEN "{{ col_name }}" IS FALSE THEN 'false' END)
+BOOL: TRIM(CASE WHEN "{{ col_name }}" IS TRUE THEN 'TRUE' WHEN "{{ col_name }}" IS FALSE THEN 'FALSE' END)
+BOOLEAN: TRIM(CASE WHEN "{{ col_name }}" IS TRUE THEN 'TRUE' WHEN "{{ col_name }}" IS FALSE THEN 'FALSE' END)
 BOX: TRIM(CAST("{{ col_name }}" AS TEXT))
 BPCHAR: RTRIM("{{ col_name }}")
-# BYTEA: prefixed uppercase hex must match Snowflake BINARY ('0x' || HEX_ENCODE).
-BYTEA: "'0x' || UPPER(ENCODE(\"{{ col_name }}\", 'hex'))"
+# BYTEA: uppercase hex without prefix (matches Snowflake HEX_ENCODE(...)).
+BYTEA: UPPER(ENCODE("{{ col_name }}", 'hex'))
 CHAR: RTRIM("{{ col_name }}")
 CHARACTER: RTRIM("{{ col_name }}")
 CHARACTER VARYING: RTRIM("{{ col_name }}")
@@ -45,9 +45,9 @@ DATERANGE: TRIM(CAST("{{ col_name }}" AS TEXT))
 DECIMAL: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST("{{ col_name }}" AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END
 DEFAULT: RTRIM(CAST("{{ col_name }}" AS TEXT))
 DOMAIN: RTRIM(CAST("{{ col_name }}" AS TEXT))
-DOUBLE PRECISION: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST(CASE WHEN CAST("{{ col_name }}" AS DOUBLE PRECISION) IS NOT DISTINCT FROM 0::DOUBLE PRECISION THEN 0::DOUBLE PRECISION ELSE CAST("{{ col_name }}" AS DOUBLE PRECISION) END AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST(CASE WHEN CAST("{{ col_name }}" AS DOUBLE PRECISION) IS NOT DISTINCT FROM 0::DOUBLE PRECISION THEN 0::DOUBLE PRECISION ELSE CAST("{{ col_name }}" AS DOUBLE PRECISION) END AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END
+DOUBLE PRECISION: '{{ float_tme_shim }}'
 ENUM: RTRIM(CAST("{{ col_name }}" AS TEXT))
-FLOAT: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST(CASE WHEN CAST("{{ col_name }}" AS DOUBLE PRECISION) IS NOT DISTINCT FROM 0::DOUBLE PRECISION THEN 0::DOUBLE PRECISION ELSE CAST("{{ col_name }}" AS DOUBLE PRECISION) END AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST(CASE WHEN CAST("{{ col_name }}" AS DOUBLE PRECISION) IS NOT DISTINCT FROM 0::DOUBLE PRECISION THEN 0::DOUBLE PRECISION ELSE CAST("{{ col_name }}" AS DOUBLE PRECISION) END AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END
+FLOAT: '{{ float_tme_shim }}'
 GEOGRAPHY: TRIM(ST_AsText("{{ col_name }}"))
 GEOMETRY: TRIM(ST_AsText("{{ col_name }}"))
 INET: TRIM(CAST("{{ col_name }}" AS TEXT))
@@ -69,7 +69,7 @@ OID: TRIM(CAST("{{ col_name }}" AS TEXT))
 PATH: TRIM(CAST("{{ col_name }}" AS TEXT))
 POINT: TRIM(FORMAT('POINT(%s %s)', ("{{ col_name }}")[0], ("{{ col_name }}")[1]))
 POLYGON: TRIM(CAST("{{ col_name }}" AS TEXT))
-REAL: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST(CAST(CASE WHEN CAST("{{ col_name }}" AS REAL) IS NOT DISTINCT FROM 0::REAL THEN 0::REAL ELSE CAST("{{ col_name }}" AS REAL) END AS DOUBLE PRECISION) AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST(CAST(CASE WHEN CAST("{{ col_name }}" AS REAL) IS NOT DISTINCT FROM 0::REAL THEN 0::REAL ELSE CAST("{{ col_name }}" AS REAL) END AS DOUBLE PRECISION) AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END
+REAL: '{{ float_tme_shim }}'
 SERIAL: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST("{{ col_name }}" AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END
 SMALLINT: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST("{{ col_name }}" AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END
 SMALLSERIAL: CASE WHEN "{{ col_name }}" IS NULL THEN NULL WHEN CAST("{{ col_name }}" AS NUMERIC(38, 12)) = 0 THEN '0.0000' ELSE TRIM(REGEXP_REPLACE(TRIM(TO_CHAR(CAST("{{ col_name }}" AS NUMERIC(38, 12)), 'FM9999999999999999999999999999.0000')), '^\\.', '0.')) END

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/postgresql/query/query_generator_postgresql.py

@@ -25,7 +25,6 @@ from snowflake.snowflake_data_validation.query.query_generator_base import (
     resolve_table_context_modifiers,
 )
 from snowflake.snowflake_data_validation.utils.constants import (
-    COL_NAME_NO_QUOTES_PLACEHOLDER,
     NULL_HASHING_SENTINEL,
     NULL_HASHING_SENTINEL_KEY,
     Platform,
@@ -36,6 +35,27 @@ from snowflake.snowflake_data_validation.utils.model.table_context import TableC
 LOGGER = logging.getLogger(__name__)
 
 
+def build_postgresql_float_tme_shim(col_ref: str) -> str:
+    """Return inline PostgreSQL SQL that renders *col_ref* in Snowflake ``TME`` shape."""
+    base = f"LOWER(TRIM(CAST({col_ref} AS TEXT)))"
+    exponent_pos = f"NULLIF(POSITION('e' IN {base}), 0)"
+    mantissa_src = f"CASE WHEN {exponent_pos} IS NULL THEN {base} ELSE SUBSTRING({base} FROM 1 FOR {exponent_pos} - 1) END"
+    exponent_int = f"CAST(TRIM(SUBSTRING({base} FROM {exponent_pos} + 1)) AS BIGINT)"
+    decimal_pos = f"NULLIF(POSITION('.' IN {mantissa_src}), 0)"
+    fraction = f"SUBSTRING({mantissa_src} FROM {decimal_pos} + 1)"
+    fraction_trimmed = f"RTRIM({fraction}, '0')"
+    mantissa_trimmed = (
+        f"CASE WHEN {decimal_pos} IS NULL THEN {mantissa_src} "
+        f"ELSE SUBSTRING({mantissa_src} FROM 1 FOR {decimal_pos} - 1) || "
+        f"CASE WHEN {fraction_trimmed} = '' THEN '' ELSE '.' || {fraction_trimmed} END END"
+    )
+    return (
+        f"CASE WHEN {col_ref} IS NULL THEN NULL "
+        f"WHEN {exponent_pos} IS NULL THEN UPPER({mantissa_trimmed}) || 'E0' "
+        f"ELSE UPPER({mantissa_trimmed}) || 'E' || TRIM(CAST({exponent_int} AS TEXT)) END"
+    )
+
+
 class QueryGeneratorPostgresql(QueryGeneratorBase):
     """Query generator for PostgreSQL (chunked MD5 pipeline aligned with Redshift)."""
 
@@ -47,6 +67,15 @@ class QueryGeneratorPostgresql(QueryGeneratorBase):
         self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(templates_dir)))
         self.jinja_env.globals[NULL_HASHING_SENTINEL_KEY] = NULL_HASHING_SENTINEL
 
+    def _float_column_reference(self, column) -> str:
+        return f'"{column.name}"'
+
+    def build_float_tme_shim(self, column, col_ref: str) -> str:
+        return build_postgresql_float_tme_shim(col_ref)
+
+    def _render_normalization_template(self, template: str, column) -> str:
+        return super()._render_normalization_template(template, column)
+
     def generate_cell_validation_data_query(
         self,
         table_context: TableContext,
@@ -58,8 +87,10 @@ class QueryGeneratorPostgresql(QueryGeneratorBase):
         for column in table_context.columns_to_validate:
             data_type = "VARCHAR" if column.data_type == "TEXT" else column.data_type
             normalization_template = datatypes_normalization_templates[data_type]
-            normalization_template_rendered = normalization_template.replace(COL_NAME_NO_QUOTES_PLACEHOLDER, column.name)
-            datatypes_normalization_renderer_templates[column.name] = normalization_template_rendered
+            datatypes_normalization_renderer_templates[column.name] = self._render_normalization_template(
+                normalization_template,
+                column,
+            )
 
         return table_context.sql_generator.generate_cell_validation_data_select_query(
             platform=table_context.platform.value,
@@ -101,8 +132,10 @@ class QueryGeneratorPostgresql(QueryGeneratorBase):
                 normalization_template = datatypes_normalization_templates.get("DEFAULT")
                 if normalization_template is None:
                     raise ValueError(f"No normalization template found for data type {column.data_type} and no DEFAULT template available.")
-            normalization_template_rendered = normalization_template.replace(COL_NAME_NO_QUOTES_PLACEHOLDER, column.name)
-            datatypes_normalization_renderer_templates[column.name] = normalization_template_rendered
+            datatypes_normalization_renderer_templates[column.name] = self._render_normalization_template(
+                normalization_template,
+                column,
+            )
 
         column_groups = table_context.column_groups
 

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/public/query_generation/metadata_queries.py

@@ -17,6 +17,7 @@
 
 from __future__ import annotations
 
+from shared.query_modifiers.template_context import QueryModifiersTemplateContext
 from snowflake.snowflake_data_validation.configuration.model.table_configuration import (
     TableConfiguration,
 )
@@ -41,6 +42,7 @@ def generate_view_materialization_query(
     fully_qualified_temporary_table_name: str,
     platform: Platform,
     sql_generator: SQLQueriesTemplateGenerator,
+    query_modifiers: QueryModifiersTemplateContext | None = None,
 ) -> GeneratedQuery:
     """Generate a query to materialize a view as a temporary table.
 
@@ -55,6 +57,7 @@ def generate_view_materialization_query(
             temporary table to create (e.g., "database.schema.temp_table_name").
         platform: The database platform (SNOWFLAKE, SQLSERVER, etc.).
         sql_generator: The SQLQueriesTemplateGenerator instance to use for query generation.
+        query_modifiers: Resolved source-side SQL modifiers for the view scan.
 
     Returns:
         GeneratedQuery: The generated DDL statement to create the temporary table.
@@ -70,6 +73,7 @@ def generate_view_materialization_query(
             fully_qualified_view_name=fully_qualified_view_name,
             platform=platform,
             sql_generator=sql_generator,
+            query_modifiers=query_modifiers,
         )
         return GeneratedQuery(query_string=query_string)
 
@@ -199,9 +203,7 @@ def generate_case_sensitive_columns_query(
 
     """
     try:
-        database_name = HelperDatabase.normalize_identifier(
-            identifier=target_database_name
-        )
+        database_name = HelperDatabase.normalize_identifier(identifier=target_database_name)
 
         schema_name = HelperDatabase.normalize_identifier(identifier=target_schema_name)
 

{snowflake_data_validation-1.23.1 → snowflake_data_validation-1.24.2}/src/snowflake/snowflake_data_validation/query/__init__.py

@@ -15,8 +15,16 @@
 
 """Query generation package for database-specific query generators."""
 
+from snowflake.snowflake_data_validation.query.charset_normalization import (
+    CharacterSetResolver,
+    DefaultCharacterSetResolver,
+)
 from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
 )
 
-__all__ = ["QueryGeneratorBase"]
+__all__ = [
+    "CharacterSetResolver",
+    "DefaultCharacterSetResolver",
+    "QueryGeneratorBase",
+]