snowflake-data-validation 1.35.1__tar.gz → 1.35.2__tar.gz

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.35.1
+Version: 1.35.2
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -188,6 +188,18 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.35.2
+
+**New features**
+
+- Added L3 signature comparison with Snowflake pushdown.
+- Added as-stored string comparison, with space-only `RTRIM` for padded `CHAR` types.
+
+**Bug fixes**
+
+- Fixed Teradata `FLOAT` comparison to use IEEE-754 bit identity.
+- Unified float comparison to IEEE-754 bit identity and removed decimal `TME` from data validation.
+
 ### v1.35.1
 
 **New features**

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/README.md

@@ -99,6 +99,18 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.35.2
+
+**New features**
+
+- Added L3 signature comparison with Snowflake pushdown.
+- Added as-stored string comparison, with space-only `RTRIM` for padded `CHAR` types.
+
+**Bug fixes**
+
+- Fixed Teradata `FLOAT` comparison to use IEEE-754 bit identity.
+- Unified float comparison to IEEE-754 bit identity and removed decimal `TME` from data validation.
+
 ### v1.35.1
 
 **New features**

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml

@@ -317,22 +317,22 @@ BIT:
 
 CHAR:
   min:
-    metric_query: MIN(DATALENGTH("{{ col_name }}"))
+    metric_query: MIN(DATALENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: INT
   max:
-    metric_query: MAX(DATALENGTH("{{ col_name }}"))
+    metric_query: MAX(DATALENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: INT
   count_empty:
-    metric_query: SUM(CASE WHEN DATALENGTH("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN DATALENGTH(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT TRIM("{{ col_name }}"))
+    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   avg:
-    metric_query: AVG(CAST(DATALENGTH("{{ col_name }}") AS FLOAT))
+    metric_query: AVG(CAST(DATALENGTH(RTRIM("{{ col_name }}")) AS FLOAT))
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
@@ -358,19 +358,19 @@ VARCHAR:
 
 NCHAR:
   min:
-    metric_query: MIN(LEN("{{ col_name }}"))
+    metric_query: MIN(LEN(RTRIM("{{ col_name }}")))
     metric_return_datatype: INT
   max:
-    metric_query: MAX(LEN("{{ col_name }}"))
+    metric_query: MAX(LEN(RTRIM("{{ col_name }}")))
     metric_return_datatype: INT
   avg:
-    metric_query: AVG(CAST(LEN("{{ col_name }}") AS FLOAT))
+    metric_query: AVG(CAST(LEN(RTRIM("{{ col_name }}")) AS FLOAT))
     metric_return_datatype: FLOAT
   count_empty:
-    metric_query: SUM(CASE WHEN "{{ col_name }}" = '' THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN RTRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
+    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -18,19 +18,17 @@ DECIMAL: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
 NUMERIC: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
 MONEY: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
 SMALLMONEY: FORMAT("{{ col_name }}", '{{ numeric_format_mask }}')
-FLOAT: '{{ float_tme_shim }}'
-REAL: '{{ float_tme_shim }}'
+FLOAT: '{{ float_bit_shim }}'
+REAL: '{{ float_bit_shim }}'
 
 # ---- Boolean type ----
 BIT: CASE WHEN "{{ col_name }}" = 1 THEN 'TRUE' WHEN "{{ col_name }}" = 0 THEN 'FALSE' END
 
 # ---- Character types ----
-# RTRIM aligns fixed-width CHAR/NCHAR padding and trailing VARCHAR spaces with the
-# Snowflake VARCHAR mirror. Use RTRIM (not TRIM) so leading spaces stay significant.
 CHAR: RTRIM("{{ col_name }}")
-VARCHAR: RTRIM("{{ col_name }}")
+VARCHAR: '"{{ col_name }}"'
 NCHAR: RTRIM("{{ col_name }}")
-NVARCHAR: RTRIM("{{ col_name }}")
+NVARCHAR: '"{{ col_name }}"'
 
 # ---- Date / Time types ----
 # Append '00' to FORMAT('.fffffff') so the output reaches FF9 (Snowflake TIMESTAMP_NTZ format).
@@ -49,7 +47,7 @@ VARBINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
 
 # ---- Other types ----
 UNIQUEIDENTIFIER: LOWER(CONVERT(VARCHAR(36), "{{ col_name }}"))
-SYSNAME: RTRIM(CONVERT(VARCHAR(128), "{{ col_name }}"))
+SYSNAME: CONVERT(VARCHAR(128), "{{ col_name }}")
 
 # ---- Default fallback ----
 DEFAULT: CONVERT(VARCHAR(MAX), "{{ col_name }}")

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -34,7 +34,7 @@ from snowflake.snowflake_data_validation.query.float_bit_normalization import (
     build_binary64_canonical_expr,
 )
 from snowflake.snowflake_data_validation.query.numeric_scale_resolution import (
-    float_tme_input_expr,
+    float_bit_input_expr,
 )
 from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
@@ -70,8 +70,8 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
     def _float_column_reference(self, column) -> str:
         return f'"{column.name}"'
 
-    def build_float_tme_shim(self, column, col_ref: str) -> str:
-        value_expr = float_tme_input_expr(
+    def build_float_bit_shim(self, column, col_ref: str) -> str:
+        value_expr = float_bit_input_expr(
             col_ref,
             getattr(column, "data_type", "") or "",
             dialect=self.platform.value,

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_cell_validation_data_select.sql.j2

@@ -2,9 +2,9 @@
 {{ qm.locking_prefix(query_modifiers) }}SELECT
 {% for column in column_collection %}
     {% if column.nullable %}
-        RTRIM(COALESCE({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}')) AS `{{ column.name }}`{% if not loop.last %},{% endif %}
+        COALESCE({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS `{{ column.name }}`{% if not loop.last %},{% endif %}
     {% else %}
-        RTRIM({{ datatypes_normalization_renderer_templates[column.name] }}) AS `{{ column.name }}`{% if not loop.last %},{% endif %}
+        {{ datatypes_normalization_renderer_templates[column.name] }} AS `{{ column.name }}`{% if not loop.last %},{% endif %}
     {% endif %}
 {% endfor %}
 FROM (

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_datatypes_normalization_templates.yaml

@@ -37,10 +37,9 @@ INT64: FORMAT('{{ numeric_format_mask }}', CAST(`{{ col_name }}` AS NUMERIC))
 INTEGER: FORMAT('{{ numeric_format_mask }}', CAST(`{{ col_name }}` AS NUMERIC))
 
 # FLOAT64 maps to Snowflake FLOAT. L3 compares IEEE-754 bit identity
-# (sign|exponent|significand), not TME decimal text. See
-# QueryGeneratorBigQuery.build_float_tme_shim.
-FLOAT64: '{{ float_tme_shim }}'
-FLOAT: '{{ float_tme_shim }}'
+# (sign|exponent|significand). See QueryGeneratorBigQuery.build_float_bit_shim.
+FLOAT64: '{{ float_bit_shim }}'
+FLOAT: '{{ float_bit_shim }}'
 
 # NUMERIC/BIGNUMERIC map to Snowflake NUMBER, which renders scale-aware via
 # TO_CHAR(col, 'FM...0.00..0'). BigQuery mirrors that scale with the resolved
@@ -57,7 +56,7 @@ BIGDECIMAL: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)
 BOOL: CASE WHEN `{{ col_name }}` THEN 'TRUE' WHEN NOT `{{ col_name }}` THEN 'FALSE' END
 BOOLEAN: CASE WHEN `{{ col_name }}` THEN 'TRUE' WHEN NOT `{{ col_name }}` THEN 'FALSE' END
 
-STRING: RTRIM(`{{ col_name }}`)
+STRING: '`{{ col_name }}`'
 # Snowflake BINARY normalizes via HEX_ENCODE (uppercase hex, no 0x prefix); match
 # it exactly so the row hashes agree.
 BYTES: UPPER(TO_HEX(`{{ col_name }}`))

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_row_md5_select.sql.j2

@@ -8,15 +8,15 @@
         CONCAT(
         {% for group in column_groups %}
             {% if not loop.first %}, {% endif %}
-            UPPER(TO_HEX(MD5(RTRIM(CONCAT(
+            UPPER(TO_HEX(MD5(CONCAT(
                 {% for col in group %}`{{ col.name }}`{% if not loop.last %}, '{{ row_hash_column_delimiter }}', {% endif %}{% endfor %}
-            )))))
+            ))))
         {% endfor %}
         )
     {% elif column_collection|length > 1 %}
-        RTRIM(CONCAT({% for col in column_collection %}`{{ col.name }}`{% if not loop.last %}, '{{ row_hash_column_delimiter }}', {% endif %}{% endfor %}))
+        CONCAT({% for col in column_collection %}`{{ col.name }}`{% if not loop.last %}, '{{ row_hash_column_delimiter }}', {% endif %}{% endfor %})
     {% else %}
-        TRIM({% for col in column_collection %}`{{ col.name }}`{% endfor %})
+        {% for col in column_collection %}`{{ col.name }}`{% endfor %}
     {% endif %}
     ))) AS ROW_MD5
 FROM
@@ -26,9 +26,9 @@ FROM
         {% endfor %}
         {% for column in column_collection %}
             {% if column.nullable %}
-                TRIM(COALESCE({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}')) AS `{{ column.name }}`{% if not loop.last %},{% endif %}
+                COALESCE({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS `{{ column.name }}`{% if not loop.last %},{% endif %}
             {% else %}
-                TRIM({{ datatypes_normalization_renderer_templates[column.name] }}) AS `{{ column.name }}`{% if not loop.last %},{% endif %}
+                {{ datatypes_normalization_renderer_templates[column.name] }} AS `{{ column.name }}`{% if not loop.last %},{% endif %}
             {% endif %}
         {% endfor %}
     {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 'src') }}

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/bigquery/query/query_generator_bigquery.py

@@ -21,7 +21,7 @@ from snowflake.snowflake_data_validation.query.float_bit_normalization import (
     build_binary64_canonical_expr,
 )
 from snowflake.snowflake_data_validation.query.numeric_scale_resolution import (
-    float_tme_input_expr,
+    float_bit_input_expr,
 )
 from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
@@ -43,23 +43,6 @@ from snowflake.snowflake_data_validation.validation.comparison_codec import (
 LOGGER = logging.getLogger(__name__)
 
 
-def build_bigquery_float_tme_shim(col_ref: str) -> str:
-    """Return inline BigQuery SQL that renders *col_ref* in Snowflake ``TME`` shape.
-
-    Kept for migration-staging / decimal TME compatibility. L3 float validation
-    uses :func:`build_bigquery_float_bit_canonical_expr` instead.
-    """
-    base = f"LOWER(FORMAT('%.15e', {col_ref}))"
-    mantissa = f"SPLIT({base}, 'e')[OFFSET(0)]"
-    exponent = f"CAST(SPLIT({base}, 'e')[OFFSET(1)] AS INT64)"
-    mantissa_trimmed = f"RTRIM(RTRIM({mantissa}, '0'), '.')"
-    return (
-        f"CASE WHEN {col_ref} IS NULL THEN NULL "
-        f"WHEN {col_ref} = 0 THEN '0E0' "
-        f"ELSE CONCAT({mantissa_trimmed}, 'E', CAST({exponent} AS STRING)) END"
-    )
-
-
 def build_bigquery_float_bit_canonical_expr(col_ref: str) -> str:
     """Return the exact binary64 tuple used for BigQuery float validation."""
     return build_binary64_canonical_expr(col_ref, dialect=Platform.BIGQUERY.value)
@@ -124,12 +107,12 @@ class QueryGeneratorBigQuery(QueryGeneratorBase):
         return self.normalize_bigquery_type(col.data_type)
 
     def _float_column_reference(self, column) -> str:
-        """Backtick-quote the column for the float TME shim (BigQuery identifiers)."""
+        """Backtick-quote the column for the float bit shim (BigQuery identifiers)."""
         return f"`{column.name}`"
 
-    def build_float_tme_shim(self, column, col_ref: str) -> str:
+    def build_float_bit_shim(self, column, col_ref: str) -> str:
         """Render *col_ref* as the shared IEEE-754 ``sign|exponent|significand`` tuple."""
-        value_expr = float_tme_input_expr(
+        value_expr = float_bit_input_expr(
             col_ref,
             getattr(column, "data_type", "") or "",
             dialect=self.platform.value,

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_column_metrics_templates.yaml

@@ -382,16 +382,22 @@ DECFLOAT:
 #    INTEGER (integer division truncates), so an un-cast AVG yields e.g. 5 for
 #    lengths averaging 5.1, while Snowflake's AVG returns 5.1 -> another spurious L2
 #    mismatch. CAST(... AS FLOAT) forces DB2 to average in floating point (gotcha W).
+#
+# 3. CHAR/NCHAR RTRIM (gotcha Z): DB2 CHAR is blank-padded; the Snowflake mirror is
+#    TEXT. Length metrics RTRIM the pad so they match VARCHAR/TEXT as stored.
+#    RTRIM (not TRIM) keeps leading spaces significant, matching the L3 hash and
+#    the RTRIM the Snowflake generator applies to the paired mirror column.
+#    VARCHAR itself is not trimmed.
 
 CHARACTER:
   min:
-    metric_query: MIN(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32))
+    metric_query: MIN(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32))
     metric_return_datatype: INTEGER
   max:
-    metric_query: MAX(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32))
+    metric_query: MAX(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32))
     metric_return_datatype: INTEGER
   avg:
-    metric_query: AVG(CAST(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32) AS FLOAT))
+    metric_query: AVG(CAST(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32) AS FLOAT))
     metric_return_datatype: FLOAT
   count_distinct:
     metric_query: COUNT(DISTINCT "{{ col_name }}")
@@ -400,18 +406,18 @@ CHARACTER:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
   count_empty:
-    metric_query: SUM(CASE WHEN TRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN RTRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
 
 CHAR:
   min:
-    metric_query: MIN(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32))
+    metric_query: MIN(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32))
     metric_return_datatype: INTEGER
   max:
-    metric_query: MAX(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32))
+    metric_query: MAX(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32))
     metric_return_datatype: INTEGER
   avg:
-    metric_query: AVG(CAST(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32) AS FLOAT))
+    metric_query: AVG(CAST(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32) AS FLOAT))
     metric_return_datatype: FLOAT
   count_distinct:
     metric_query: COUNT(DISTINCT "{{ col_name }}")
@@ -420,7 +426,7 @@ CHAR:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
   count_empty:
-    metric_query: SUM(CASE WHEN TRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN RTRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
 
 VARCHAR:
@@ -443,16 +449,16 @@ VARCHAR:
     metric_query: SUM(CASE WHEN CHARACTER_LENGTH("{{ col_name }}", CODEUNITS32) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
 
-# National character (Unicode) fixed-length: same metrics as CHAR (LENGTH(TRIM)).
+# National character (Unicode) fixed-length: same metrics as CHAR (LENGTH(RTRIM)).
 NCHAR:
   min:
-    metric_query: MIN(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32))
+    metric_query: MIN(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32))
     metric_return_datatype: INTEGER
   max:
-    metric_query: MAX(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32))
+    metric_query: MAX(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32))
     metric_return_datatype: INTEGER
   avg:
-    metric_query: AVG(CAST(CHARACTER_LENGTH(TRIM("{{ col_name }}"), CODEUNITS32) AS FLOAT))
+    metric_query: AVG(CAST(CHARACTER_LENGTH(RTRIM("{{ col_name }}"), CODEUNITS32) AS FLOAT))
     metric_return_datatype: FLOAT
   count_distinct:
     metric_query: COUNT(DISTINCT "{{ col_name }}")
@@ -461,7 +467,7 @@ NCHAR:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
   count_empty:
-    metric_query: SUM(CASE WHEN TRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN RTRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
 
 # National character (Unicode) varying: same metrics as VARCHAR (LENGTH).

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_datatypes_normalization_templates.yaml

@@ -14,7 +14,7 @@
 # - TO_CHAR() is available but REJECTS the Oracle FM fill-mode prefix (SQL20476N).
 # - {{ numeric_format_mask }} is a pre-built DB2-valid 9…90 mask (no FM prefix)
 #   from the query generator's _build_fm_to_char_mask override.
-# - {{ float_tme_shim }} expands to the IEEE-754 sign|exponent|significand tuple.
+# - {{ float_bit_shim }} expands to the IEEE-754 sign|exponent|significand tuple.
 # - HEX(HASH(val, 0)) computes MD5 in DB2 11.5 (algorithm 0 = MD5) using the built-in
 #   SYSIBM.HASH scalar — no special privilege (unlike SYSPROC.HASH2, which is often absent).
 # - Double-quoted column names handle lowercase/mixed-case identifiers.
@@ -33,27 +33,29 @@ INT: TRIM(TO_CHAR("{{ col_name }}", '{{ numeric_format_mask }}'))
 BIGINT: TRIM(TO_CHAR("{{ col_name }}", '{{ numeric_format_mask }}'))
 SMALLINT: TRIM(TO_CHAR("{{ col_name }}", '{{ numeric_format_mask }}'))
 # Float types use the IEEE-754 binary64 tuple (same contract as Snowflake FLOAT).
-FLOAT: '{{ float_tme_shim }}'
-REAL: '{{ float_tme_shim }}'
-DOUBLE: '{{ float_tme_shim }}'
-DOUBLE PRECISION: '{{ float_tme_shim }}'
-DECFLOAT: '{{ float_tme_shim }}'
+FLOAT: '{{ float_bit_shim }}'
+REAL: '{{ float_bit_shim }}'
+DOUBLE: '{{ float_bit_shim }}'
+DOUBLE PRECISION: '{{ float_bit_shim }}'
+DECFLOAT: '{{ float_bit_shim }}'
 
 # ---- Character types (text family) ----
-# RTRIM strips trailing pad characters; NULL passes through for the sentinel wrap.
+# CHAR/NCHAR are blank-padded; the Snowflake E2E/SCAI mirror is VARCHAR/TEXT.
+# RTRIM trailing pads so L3 matches as-stored VARCHAR. VARCHAR/NVARCHAR stay
+# identity (compared as stored). NULL passes through for the sentinel wrap.
 CHARACTER: RTRIM("{{ col_name }}")
 CHAR: RTRIM("{{ col_name }}")
-VARCHAR: RTRIM("{{ col_name }}")
+VARCHAR: '"{{ col_name }}"'
 # CLOB / DBCLOB → Snowflake VARCHAR. MD5 hash via HEX(HASH(.., 0)) (built-in SYSIBM.HASH,
 # no special privilege), CASE-guarded for NULL. HASH rejects LOB arguments (SQL0171N), so the
 # LOB is cast to VARCHAR(32672) — DB2's max VARCHAR — before hashing. Values beyond 32672 bytes
 # are truncated for the hash; test fixtures stay within that bound.
 CLOB: CASE WHEN "{{ col_name }}" IS NULL THEN NULL ELSE UPPER(HEX(HASH(CAST("{{ col_name }}" AS VARCHAR(32672)), 0))) END
 DBCLOB: CASE WHEN "{{ col_name }}" IS NULL THEN NULL ELSE UPPER(HEX(HASH(CAST("{{ col_name }}" AS VARCHAR(32672)), 0))) END
-# National character (Unicode) strings: NCHAR/NVARCHAR behave like CHAR/VARCHAR
-# (RTRIM trailing pad, NULL passthrough); NCLOB is a LOB, hashed via HEX(HASH(.., 0)) like DBCLOB.
+# National character: NCHAR is blank-padded like CHAR; NVARCHAR is as-stored.
+# NCLOB is a LOB, hashed via HEX(HASH(.., 0)) like DBCLOB.
 NCHAR: RTRIM("{{ col_name }}")
-NVARCHAR: RTRIM("{{ col_name }}")
+NVARCHAR: '"{{ col_name }}"'
 NCLOB: CASE WHEN "{{ col_name }}" IS NULL THEN NULL ELSE UPPER(HEX(HASH(CAST("{{ col_name }}" AS VARCHAR(32672)), 0))) END
 
 # ---- Boolean ----
@@ -84,4 +86,4 @@ TIME: TO_CHAR("{{ col_name }}", 'HH24:MI:SS')
 TIMESTAMP: TO_CHAR("{{ col_name }}", 'YYYY-MM-DD') || 'T' || TO_CHAR("{{ col_name }}", 'HH24:MI:SS.FF6') || '000'
 
 # ---- Default fallback ----
-DEFAULT: TRIM(CAST("{{ col_name }}" AS VARCHAR(1000)))
+DEFAULT: CAST("{{ col_name }}" AS VARCHAR(1000))

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_row_md5_select.sql.j2

@@ -1,5 +1,5 @@
 {% import 'query_modifiers.sql.j2' as qm with context %}
--- DB2 L3 row hashing. Uses the Oracle-compatible SQL surface (||, NVL, RTRIM)
+-- DB2 L3 row hashing. Uses the Oracle-compatible SQL surface (||, NVL)
 -- with DB2's built-in MD5 primitive HEX(HASH(expr, 0)) in place of Oracle's
 -- STANDARD_HASH/RAWTOHEX. SYSIBM.HASH(expr, 0) returns the 16-byte MD5 digest
 -- (algorithm 0 = MD5, no special privilege required, unlike SYSPROC.HASH2);
@@ -12,19 +12,18 @@ SELECT
     {% if use_column_groups %}
         {% for group in column_groups %}
             {% if not loop.first %} || {% endif %}
-            UPPER(HEX(HASH(RTRIM(
+            UPPER(HEX(HASH(
                 {% for col in group %}
                     NVL("{{ col.name }}", '{{ null_hashing_sentinel }}')
                     {% if not loop.last %} || '{{ row_hash_column_delimiter }}' || {% endif %}
                 {% endfor %}
-            ), 0)))
+            , 0)))
         {% endfor %}
     {% else %}
-        RTRIM(
         {% for column in column_collection %}
             NVL("{{ column.name }}", '{{ null_hashing_sentinel }}')
             {% if not loop.last %} || '{{ row_hash_column_delimiter }}' || {% endif %}
-        {% endfor %})
+        {% endfor %}
     {% endif %}
     , 0))) AS ROW_MD5
 FROM
@@ -34,9 +33,9 @@ FROM
         {% endfor %}
         {% for column in column_collection %}
             {% if column.nullable %}
-                RTRIM(NVL({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}')) AS "{{ column.name }}"{% if not loop.last %},{% endif %}
+                NVL({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS "{{ column.name }}"{% if not loop.last %},{% endif %}
             {% else %}
-                RTRIM({{ datatypes_normalization_renderer_templates[column.name] }}) AS "{{ column.name }}"{% if not loop.last %},{% endif %}
+                {{ datatypes_normalization_renderer_templates[column.name] }} AS "{{ column.name }}"{% if not loop.last %},{% endif %}
             {% endif %}
         {% endfor %}
     {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 'src') }}

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/db2/query/query_generator_db2.py

@@ -19,7 +19,7 @@ below give DB2 its own DB2-valid rendering that still matches Snowflake
   Oracle ``REGEXP_REPLACE`` leading-zero wrapper is needed.
 * ``render_normalization_template`` — bypasses Oracle's ``FM``+``REGEXP`` numeric
   canonical wrapper and uses the base substitution with the DB2 mask.
-* ``build_float_tme_shim`` — IEEE-754 ``sign|exponent|significand`` tuple (same
+* ``build_float_bit_shim`` — IEEE-754 ``sign|exponent|significand`` tuple (same
   contract as Snowflake FLOAT). DB2 ``LN``/``POWER`` run in ``DOUBLE``.
 """
 
@@ -34,7 +34,7 @@ from snowflake.snowflake_data_validation.query.float_bit_normalization import (
     build_binary64_canonical_expr,
 )
 from snowflake.snowflake_data_validation.query.numeric_scale_resolution import (
-    float_tme_input_expr,
+    float_bit_input_expr,
 )
 from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
@@ -49,9 +49,6 @@ from snowflake.snowflake_data_validation.utils.constants import (
 )
 
 
-# Mantissa is normalized to [1, 10); one integer 9 plus 15 fractional zeros gives
-# the same 15-decimal rounding Oracle's ROUND(mantissa, 15) + TM9 produces.
-_DB2_MANTISSA_MASK = "9.000000000000000"
 # Wide enough for BIGINT (19 digits) and DECIMAL(31); the trailing 0 forces the
 # leading zero for |v| < 1 (matching Snowflake TO_VARCHAR).
 _DB2_INTEGER_NINES = 31
@@ -70,42 +67,6 @@ def build_db2_float_bit_canonical_expr(col_ref: str) -> str:
     return build_binary64_canonical_expr(ieee_expr, dialect=Platform.DB2.value)
 
 
-def build_db2_float_tme_shim(col_ref: str) -> str:
-    """DB2 analogue of ``build_oracle_float_tme_shim`` (DB2-valid casts/masks).
-
-    Identical LN/POWER canonical form, but the exponent is cast ``AS INTEGER``
-    (DB2 has no ``NUMBER`` type) and the mantissa is rendered with a plain
-    ``9.0…0`` mask (DB2 ``TO_CHAR`` rejects Oracle's ``TM9``). The trailing-zero
-    trimming below still collapses ``1.000000000000000`` → ``1`` and drops the
-    dangling decimal point, matching Snowflake ``TME`` (e.g. ``100`` → ``1E2``).
-    """
-    abs_ref = f"ABS({col_ref})"
-    log10 = f"(LN({abs_ref}) / LN(10))"
-    exp0 = f"CAST(FLOOR({log10}) AS INTEGER)"
-    mant0 = f"({col_ref} / POWER(10, {exp0}))"
-    exp_int = f"CASE WHEN ABS({mant0}) >= 10 THEN {exp0} + 1 WHEN ABS({mant0}) < 1 THEN {exp0} - 1 ELSE {exp0} END"
-    mantissa = f"({col_ref} / POWER(10, {exp_int}))"
-    mantissa_rounded = f"ROUND({mantissa}, 15)"
-    mantissa_base = f"TRIM(TO_CHAR({mantissa_rounded}, '{_DB2_MANTISSA_MASK}'))"
-    decimal_pos = f"NULLIF(INSTR({mantissa_base}, '.'), 0)"
-    # 2-arg SUBSTR (to end of string): DB2's 3-arg SUBSTR blank-pads the result to
-    # the requested length, which would leave trailing spaces that defeat the
-    # RTRIM('0') trailing-zero trim (Oracle's 3-arg SUBSTR returns only the
-    # available chars). The fraction is always the mask's 15 decimals.
-    fraction = f"SUBSTR({mantissa_base}, {decimal_pos} + 1)"
-    fraction_trimmed = f"RTRIM({fraction}, '0')"
-    mantissa_trimmed = (
-        f"CASE WHEN {decimal_pos} IS NULL THEN {mantissa_base} "
-        f"ELSE SUBSTR({mantissa_base}, 1, {decimal_pos} - 1) || "
-        f"CASE WHEN {fraction_trimmed} IS NULL OR {fraction_trimmed} = '' THEN '' ELSE '.' || {fraction_trimmed} END END"
-    )
-    return (
-        f"CASE WHEN {col_ref} IS NULL THEN NULL "
-        f"WHEN {col_ref} = 0 THEN '0E0' "
-        f"ELSE UPPER({mantissa_trimmed}) || 'E' || TRIM(TO_CHAR({exp_int})) END"
-    )
-
-
 class QueryGeneratorDb2(QueryGeneratorOracle):
     """DB2 LUW query generator (see module docstring for the DB2-vs-Oracle rendering rationale)."""
 
@@ -141,10 +102,10 @@ class QueryGeneratorDb2(QueryGeneratorOracle):
         """Use base substitution: DB2's mask needs no Oracle ``FM``+``REGEXP`` wrapper."""
         return QueryGeneratorBase.render_normalization_template(self, template, column)
 
-    def build_float_tme_shim(self, column, col_ref: str) -> str:
+    def build_float_bit_shim(self, column, col_ref: str) -> str:
         """Render *col_ref* as the shared IEEE-754 ``sign|exponent|significand`` tuple."""
         data_type = getattr(column, "data_type", "") or ""
-        value_expr = float_tme_input_expr(col_ref, data_type, dialect=self.platform.value)
+        value_expr = float_bit_input_expr(col_ref, data_type, dialect=self.platform.value)
         return build_db2_float_bit_canonical_expr(value_expr)
 
     # ------------------------------------------------------------------

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/extractor/sql_queries_template_generator.py

@@ -42,9 +42,6 @@ from snowflake.snowflake_data_validation.utils.model.column_metadata import Colu
 from snowflake.snowflake_data_validation.utils.qvci_fallback import (
     object_type_render_value,
 )
-from snowflake.snowflake_data_validation.utils.redshift_tab_safe_rtrim import (
-    build_redshift_tab_safe_rtrim,
-)
 
 
 class SQLQueriesTemplateGenerator:
@@ -79,7 +76,6 @@ class SQLQueriesTemplateGenerator:
             )
         )
         self.env.filters["escape_sql_literal"] = escape_sql_literal
-        self.env.globals["redshift_tab_safe_rtrim"] = build_redshift_tab_safe_rtrim
         # Single source of truth for the NULL sentinel used by L3 row-MD5 and
         # cell-validation normalization. Exposed to every template so the
         # COALESCE fallback can keep a real SQL NULL distinct from "".

{snowflake_data_validation-1.35.1 → snowflake_data_validation-1.35.2}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_column_metrics_templates.yaml

@@ -306,44 +306,43 @@ REAL:
 
 CHAR:
   min:
-    metric_query: MIN(LENGTH(TRIM("{{ col_name }}")))
+    metric_query: MIN(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: INTEGER
   max:
-    metric_query: MAX(LENGTH(TRIM("{{ col_name }}")))
+    metric_query: MAX(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: INTEGER
   avg:
-    metric_query: AVG(LENGTH(TRIM("{{ col_name }}")))
+    metric_query: AVG(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: FLOAT
   count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
+    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
-  # Oracle TRIM of an all-whitespace string returns NULL (empty string = NULL
-  # in Oracle), so use IS NULL rather than = '' to detect blank-only values.
+  # Oracle coerces '' to NULL; LENGTH = 0 matches the VARCHAR2 empty metric.
   count_empty:
-    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NOT NULL AND TRIM("{{ col_name }}") IS NULL THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN LENGTH(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
 
 NCHAR:
   min:
-    metric_query: MIN(LENGTH(TRIM("{{ col_name }}")))
+    metric_query: MIN(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: INTEGER
   max:
-    metric_query: MAX(LENGTH(TRIM("{{ col_name }}")))
+    metric_query: MAX(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: INTEGER
   avg:
-    metric_query: AVG(LENGTH(TRIM("{{ col_name }}")))
+    metric_query: AVG(LENGTH(RTRIM("{{ col_name }}")))
     metric_return_datatype: FLOAT
   count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
+    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
   count_empty:
-    metric_query: SUM(CASE WHEN "{{ col_name }}" IS NOT NULL AND TRIM("{{ col_name }}") IS NULL THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN LENGTH(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
 
 VARCHAR2:
@@ -591,7 +590,7 @@ BOOL:
 
 VECTOR:
   count_distinct:
-    metric_query: 'COUNT(DISTINCT ({{ vector_tme_shim }}))'
+    metric_query: 'COUNT(DISTINCT ({{ vector_bit_shim }}))'
     metric_return_datatype: INTEGER
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)