snowflake-data-validation 1.35.2__tar.gz → 1.37.0__tar.gz

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.35.2
+Version: 1.37.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -28,7 +28,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Requires-Dist: beartype>=0.18.0
 Requires-Dist: connectorx>=0.3.0
-Requires-Dist: cryptography<50,>=3.4.0
+Requires-Dist: cryptography>=50.0.0
 Requires-Dist: deepdiff>=8.0.0
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: pandas>=2.2.3
@@ -38,11 +38,11 @@ Requires-Dist: pydantic-yaml>=1.4.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: ruamel-yaml>=0.18.0
-Requires-Dist: snowflake-connector-python>=4.0.0
+Requires-Dist: snowflake-connector-python>=4.7.1
 Requires-Dist: toml>=0.10.2
 Requires-Dist: typer>=0.15.3
 Requires-Dist: typing-extensions>=4.13.2
-Requires-Dist: urllib3>=2.6.0
+Requires-Dist: urllib3>=2.7.0
 Provides-Extra: all
 Requires-Dist: azure-identity>=1.15.0; extra == 'all'
 Requires-Dist: coverage>=7.6.7; extra == 'all'
@@ -55,7 +55,7 @@ Requires-Dist: pyodbc>=5.2.0; extra == 'all'
 Requires-Dist: pytest-cov>=6.0.0; extra == 'all'
 Requires-Dist: pytest-mock>=3.14.1; extra == 'all'
 Requires-Dist: pytest>=8.3.3; extra == 'all'
-Requires-Dist: redshift-connector>=2.1.8; extra == 'all'
+Requires-Dist: redshift-connector>=2.1.14; extra == 'all'
 Requires-Dist: setuptools>=70.0.0; extra == 'all'
 Requires-Dist: sqlean-py>=3.45.0; extra == 'all'
 Requires-Dist: teradatasql>=17.20.0.0; extra == 'all'
@@ -77,7 +77,7 @@ Requires-Dist: twine==6.0.1; extra == 'development'
 Provides-Extra: postgresql
 Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgresql'
 Provides-Extra: redshift
-Requires-Dist: redshift-connector>=2.1.8; extra == 'redshift'
+Requires-Dist: redshift-connector>=2.1.14; extra == 'redshift'
 Provides-Extra: sqlite
 Requires-Dist: sqlean-py>=3.45.0; extra == 'sqlite'
 Provides-Extra: sqlserver
@@ -188,6 +188,29 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.37.0
+
+**New features**
+
+- Added cloud-direct L3 signature copy-extract execution for Teradata `WRITE_NOS`, Oracle `DBMS_CLOUD`, Synapse `CETAS`, and BigQuery `EXPORT DATA`.
+
+### v1.36.0
+
+**New features**
+
+- Added Oracle `BFILE` locator migration support.
+
+**Improvements**
+
+- Improved SQL Server L1 validation by binding source catalogs and qualifying object references.
+
+**Bug fixes**
+
+- Fixed Redshift `SUPER` columns being read as `NULL` by the validator.
+- Fixed parameterized numeric type matching against bare normalization keys.
+- Fixed Snowflake `TO_CHAR` mask overflow by clamping to the 38-digit budget.
+- Fixed mixed-interval column binding in Jinja templates before `pyodbc` evaluation.
+
 ### v1.35.2
 
 **New features**

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/README.md

@@ -99,6 +99,29 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.37.0
+
+**New features**
+
+- Added cloud-direct L3 signature copy-extract execution for Teradata `WRITE_NOS`, Oracle `DBMS_CLOUD`, Synapse `CETAS`, and BigQuery `EXPORT DATA`.
+
+### v1.36.0
+
+**New features**
+
+- Added Oracle `BFILE` locator migration support.
+
+**Improvements**
+
+- Improved SQL Server L1 validation by binding source catalogs and qualifying object references.
+
+**Bug fixes**
+
+- Fixed Redshift `SUPER` columns being read as `NULL` by the validator.
+- Fixed parameterized numeric type matching against bare normalization keys.
+- Fixed Snowflake `TO_CHAR` mask overflow by clamping to the 38-digit budget.
+- Fixed mixed-interval column binding in Jinja templates before `pyodbc` evaluation.
+
 ### v1.35.2
 
 **New features**

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/pyproject.toml

@@ -28,11 +28,9 @@ classifiers = [
 dependencies = [
   "beartype>=0.18.0",
   "connectorx>=0.3.0",
-  # Cap <50: cryptography 50 drops symbols still required by pyOpenSSL<26
-  # (resolved with snowflake-connector-python 4.0.x); no pyOpenSSL supports 50 yet.
-  "cryptography>=3.4.0,<50",
+  "cryptography>=50.0.0",
   "deepdiff>=8.0.0",
-  "snowflake-connector-python>=4.0.0",
+  "snowflake-connector-python>=4.7.1",
   "pandas>=2.2.3",
   "polars>=1.0.0",
   "pyarrow>=14.0.0",
@@ -44,7 +42,7 @@ dependencies = [
   "pydantic-yaml>=1.4.0",
   "typing-extensions>=4.13.2",
   "typer>=0.15.3",
-  "urllib3>=2.6.0",
+  "urllib3>=2.7.0",
 ]
 description = "Snowflake Data Validation"
 dynamic = ['version']
@@ -86,7 +84,7 @@ sqlite = [
   "sqlean.py>=3.45.0",
 ]
 redshift = [
-  "redshift-connector>=2.1.8",
+  "redshift-connector>=2.1.14",
 ]
 postgresql = [
   "psycopg[binary]>=3.2.0",

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml

@@ -326,7 +326,7 @@ CHAR:
     metric_query: SUM(CASE WHEN DATALENGTH(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -350,7 +350,7 @@ VARCHAR:
     metric_query: SUM(CASE WHEN DATALENGTH("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -370,7 +370,7 @@ NCHAR:
     metric_query: SUM(CASE WHEN RTRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT RTRIM("{{ col_name }}"))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -390,7 +390,7 @@ NVARCHAR:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = '' THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
-    metric_query: COUNT(DISTINCT CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)))
+    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST(N'|' + CAST("{{ col_name }}" AS NVARCHAR(MAX)) + N'|' AS NVARCHAR(MAX)){{ measure_collation_suffix }})
     metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
@@ -437,11 +437,11 @@ DATETIME2:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: DATETIME2
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000'
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: DATETIME2
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000'
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
@@ -463,11 +463,11 @@ DATETIMEOFFSET:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: DATETIMEOFFSET
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00' + FORMAT({{ metric_query }}, 'zzz')
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT({{ metric_query }}, 'zzz')
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: DATETIMEOFFSET
-    metric_normalization: FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00' + FORMAT({{ metric_query }}, 'zzz')
+    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT({{ metric_query }}, 'zzz')
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -31,22 +31,29 @@ NCHAR: RTRIM("{{ col_name }}")
 NVARCHAR: '"{{ col_name }}"'
 
 # ---- Date / Time types ----
-# Append '00' to FORMAT('.fffffff') so the output reaches FF9 (Snowflake TIMESTAMP_NTZ format).
-# Use '+' (not CONCAT) so a NULL input propagates to NULL — CONCAT would coerce NULL to ''.
+# DATETIME keeps 7 fractional digits + '00' pad to FF9.
+# DATETIME2 / DATETIMEOFFSET: Snowflake TIMESTAMP_* stores microseconds. FORMAT
+# '.fffffff' keeps the 7th digit (e.g. .999999900) while Snowflake TO_CHAR FF9
+# emits .999999000 for the truncated mirror. Truncate to 6 fractional digits
+# then pad '000' so hashes match ODBC/microsecond storage. LEFT keeps NULL.
+# Use '+' (not CONCAT) so a NULL input propagates to NULL.
 # FORMAT(..., 'zzz') reliably emits '+HH:MM'/'-HH:MM' for every offset, including UTC.
 DATE: FORMAT("{{ col_name }}", 'yyyy-MM-dd')
 TIME: CONVERT(VARCHAR(8), "{{ col_name }}", 108)
 DATETIME: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
-DATETIME2: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00'
+DATETIME2: LEFT(FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000'
 SMALLDATETIME: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.000000000')
-DATETIMEOFFSET: FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff') + '00' + FORMAT("{{ col_name }}", 'zzz')
+DATETIMEOFFSET: LEFT(FORMAT("{{ col_name }}", 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT("{{ col_name }}", 'zzz')
 
 # ---- Binary types ----
 BINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
 VARBINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
 
 # ---- Other types ----
-UNIQUEIDENTIFIER: LOWER(CONVERT(VARCHAR(36), "{{ col_name }}"))
+# Azure Synapse dedicated SQL pool always normalizes UNIQUEIDENTIFIER hex digits
+# to uppercase regardless of input casing. Use UPPER so the canonical string
+# matches the DV cases and Snowflake's VARCHAR-based representation.
+UNIQUEIDENTIFIER: UPPER(CONVERT(VARCHAR(36), "{{ col_name }}"))
 SYSNAME: CONVERT(VARCHAR(128), "{{ col_name }}")
 
 # ---- Default fallback ----

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_get_columns_metadata.sql.j2

@@ -31,6 +31,7 @@ SELECT
         END AS character_length,
     CASE WHEN c.precision = 0 THEN NULL ELSE c.precision END AS precision,
     CASE WHEN c.precision = 0 THEN NULL ELSE c.scale END AS scale,
+    c.collation_name AS collation_name,
     CASE
         WHEN t.name IN ('decimal', 'numeric')
             THEN

snowflake_data_validation-1.37.0/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_row_count_query.sql.j2

@@ -0,0 +1,2 @@
+{% import 'query_modifiers.sql.j2' as qm with context %}
+{{ qm.locking_prefix(query_modifiers) }}SELECT COUNT_BIG(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }}{% if has_where_clause %} WHERE {{ where_clause }}{% endif %}

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_table_metadata_query.sql.j2

@@ -1,7 +1,7 @@
 {% import 'query_modifiers.sql.j2' as qm with context %}
 {{ qm.locking_prefix(query_modifiers) }}
 WITH rc_cte AS (
-        SELECT COUNT(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }} {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
+        SELECT COUNT_BIG(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }} {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
 )
     SELECT
         C.COLUMN_NAME AS COLUMN_VALIDATED,
@@ -15,12 +15,16 @@ WITH rc_cte AS (
         END AS CHARACTER_MAXIMUM_LENGTH,
         C.NUMERIC_PRECISION AS NUMERIC_PRECISION,
         C.NUMERIC_SCALE AS NUMERIC_SCALE,
+        C.COLLATION_NAME AS COLLATION_NAME,
         C.IS_NULLABLE AS IS_NULLABLE,
         rc.ROW_COUNT
-   FROM INFORMATION_SCHEMA.COLUMNS C
+   FROM {% if object_database_ref %}{{ object_database_ref }}.{% endif %}INFORMATION_SCHEMA.COLUMNS C
 CROSS JOIN rc_cte rc
 WHERE C.TABLE_NAME = '{{ object_name | escape_sql_literal }}'
 AND C.TABLE_SCHEMA = '{{ object_schema | escape_sql_literal }}'
+{% if object_database %}
+AND C.TABLE_CATALOG = '{{ object_database | escape_sql_literal }}'
+{% endif %}
 {% if columns %}
     AND C.COLUMN_NAME IN (
     {% for column in columns %}

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -67,6 +67,16 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
         """Initialize the Azure Synapse query generator."""
         super().__init__(platform=Platform.AZURE_SYNAPSE)
 
+    def _measure_collation_tokens(self, column, metric_name: str) -> tuple[str, str]:
+        character_types = {"CHAR", "NCHAR", "VARCHAR", "NVARCHAR"}
+        if (
+            metric_name == "count_distinct"
+            and self._base_data_type(column) in character_types
+            and self._has_source_collation_metadata(column)
+        ):
+            return "", " COLLATE Latin1_General_BIN2"
+        return "", ""
+
     def _float_column_reference(self, column) -> str:
         return f'"{column.name}"'
 
@@ -96,7 +106,10 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
         rendered: dict[str, str] = {}
         for column in table_context.columns_to_validate:
             lookup_key = (column.data_type or "").strip().upper()
-            normalization_template = datatypes_normalization_templates.get(lookup_key)
+            normalization_template = self._lookup_normalization_template(
+                datatypes_normalization_templates,
+                lookup_key,
+            )
             if normalization_template is None:
                 LOGGER.debug(
                     "No normalization template for data type %s, using DEFAULT.",

snowflake_data_validation-1.37.0/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_row_count_query.sql.j2

@@ -0,0 +1,2 @@
+{% import 'query_modifiers.sql.j2' as qm with context %}
+{{ qm.locking_prefix(query_modifiers) }}SELECT COUNT_BIG(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }}{% if has_where_clause %} WHERE {{ where_clause }}{% endif %}

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/db2/extractor/templates/db2_table_metadata_query.sql.j2

@@ -5,7 +5,7 @@
 -- carries per-column type, length, scale, nullability and PK ordinal (KEYSEQ).
 -- COLNO is 0-based, so ORDINAL_POSITION = COLNO + 1.
 WITH rc_cte AS (
-    SELECT COUNT(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }}{% if has_where_clause %} WHERE {{ where_clause }}{% endif %}
+    SELECT COUNT_BIG(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }}{% if has_where_clause %} WHERE {{ where_clause }}{% endif %}
 )
 SELECT
     C.COLNAME               AS COLUMN_VALIDATED,

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/executer/base_validation_executor.py

@@ -37,6 +37,9 @@ from snowflake.snowflake_data_validation.utils.context import Context
 from snowflake.snowflake_data_validation.utils.iso_dv_common import (
     attach_paired_column_types_for_iso_timestamp_dv,
 )
+from snowflake.snowflake_data_validation.utils.measure_collation import (
+    mark_target_columns_for_normalized_measure_collation,
+)
 from snowflake.snowflake_data_validation.utils.model.table_column_metadata import (
     TableColumnMetadata,
 )
@@ -122,6 +125,12 @@ class BaseValidationExecutor(ABC):
             )
 
         if validations[METRICS_VALIDATION_KEY]:
+            target_table_context.columns = mark_target_columns_for_normalized_measure_collation(
+                source_table_context.platform,
+                source_table_context.columns,
+                target_table_context.columns,
+                table_configuration.column_mappings,
+            )
             self.execute_metrics_validation(
                 source_table_context=source_table_context,
                 target_table_context=target_table_context,

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/extractor/sql_queries_template_generator.py

@@ -146,11 +146,16 @@ class SQLQueriesTemplateGenerator:
             query_modifiers=query_modifiers or QueryModifiersTemplateContext.empty(),
             object_type=object_type_str,
             is_view=object_type_str == "VIEW",
+            object_database_ref="",
         )
         if platform == Platform.SNOWFLAKE.value:
             from snowflake.snowflake_data_validation.utils.helpers.helper_database import HelperDatabase
 
             render_kwargs["object_database_ref"] = HelperDatabase.snowflake_identifier_ref(database_name)
+        elif platform in (Platform.SQLSERVER.value, Platform.AZURE_SYNAPSE.value) and database_name:
+            from snowflake.snowflake_data_validation.utils.helpers.helper_database import HelperDatabase
+
+            render_kwargs["object_database_ref"] = HelperDatabase.sqlserver_identifier_ref(database_name)
         try:
             return self._render(
                 template_file,
@@ -494,8 +499,15 @@ class SQLQueriesTemplateGenerator:
         where_clause: str,
         order_by_columns: list[str] | None,
         query_modifiers: QueryModifiersTemplateContext | None = None,
+        extra_from_joins: list[str] | None = None,
     ) -> str:
-        """Render a SELECT that applies the same per-type normalization as L3 MD5."""
+        """Render a SELECT that applies the same per-type normalization as L3 MD5.
+
+        ``extra_from_joins`` (Redshift SUPER sorted-object joins) is an optional
+        list of ``LEFT JOIN ...`` fragments appended to the inner FROM clause;
+        templates that don't reference it are unaffected (Jinja Undefined is
+        falsy in ``{% if %}``).
+        """
         template_file = f"{platform}_cell_validation_data_select.sql.j2"
         try:
             return self._render(
@@ -507,6 +519,7 @@ class SQLQueriesTemplateGenerator:
                 where_clause=where_clause,
                 order_by_columns=order_by_columns or [],
                 query_modifiers=query_modifiers or QueryModifiersTemplateContext.empty(),
+                extra_from_joins=extra_from_joins or [],
             )
         except jinja2.exceptions.TemplateNotFound:
             raise ValueError(f"Template not found: {template_file}. Please create it in the {self.template_dir} directory.") from None

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/oracle/extractor/templates/oracle_datatypes_normalization_templates.yaml

@@ -51,9 +51,9 @@ RAW: RAWTOHEX({{ col_name }})
 LONG RAW: RAWTOHEX({{ col_name }})
 # MD5 the whole BLOB with DBMS_CRYPTO.HASH (see CLOB above). CASE-guarded for NULL.
 BLOB: CASE WHEN {{ col_name }} IS NULL THEN NULL ELSE RAWTOHEX(DBMS_CRYPTO.HASH({{ col_name }}, 2)) END
-# BFILE is an external file locator; DBMS_CRYPTO cannot hash it, so it stays on a
-# bounded DBMS_LOB read.
-BFILE: RAWTOHEX(DBMS_LOB.SUBSTR({{ col_name }}, 4000, 1))
+# Validate the same DIRECTORY/filename locator text produced by migration.
+# QueryGeneratorOracle adds the statement-local helper that implements this call.
+BFILE: DMVF_BFILE_TO_PATH({{ col_name }})
 
 # ---- Row address types ----
 # ROWID and UROWID are stored as opaque row-address strings; NVL is sufficient.

snowflake_data_validation-1.37.0/src/snowflake/snowflake_data_validation/oracle/oracle_bfile.py

@@ -0,0 +1,25 @@
+"""Shared Oracle ``BFILE`` locator extraction SQL."""
+
+ORACLE_BFILE_TO_PATH_FUNCTION_NAME = "DMVF_BFILE_TO_PATH"
+ORACLE_BFILE_EXTRACTION = f"{ORACLE_BFILE_TO_PATH_FUNCTION_NAME}({{col}})"
+ORACLE_BFILE_NORMALIZATION = (
+    f"{ORACLE_BFILE_TO_PATH_FUNCTION_NAME}({{{{ col_name }}}})"
+)
+ORACLE_BFILE_WITH_FUNCTION = f"""WITH FUNCTION {ORACLE_BFILE_TO_PATH_FUNCTION_NAME}(value BFILE)
+RETURN VARCHAR2 IS
+    directory_name VARCHAR2(128);
+    file_name VARCHAR2(4000);
+BEGIN
+    IF value IS NULL THEN
+        RETURN NULL;
+    END IF;
+    DBMS_LOB.FILEGETNAME(value, directory_name, file_name);
+    RETURN directory_name || '/' || file_name;
+END;
+"""
+
+
+def prepend_oracle_bfile_function(sql: str) -> str:
+    """Add the statement-local helper when SQL contains a BFILE path call."""
+    marker = f"{ORACLE_BFILE_TO_PATH_FUNCTION_NAME}("
+    return ORACLE_BFILE_WITH_FUNCTION + sql if marker in sql else sql

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/oracle/query/query_generator_oracle.py

@@ -37,6 +37,9 @@ import pandas as pd
 from snowflake.snowflake_data_validation.oracle.extractor.metadata_extractor_oracle import (
     normalize_oracle_type,
 )
+from snowflake.snowflake_data_validation.oracle.oracle_bfile import (
+    prepend_oracle_bfile_function,
+)
 from snowflake.snowflake_data_validation.oracle.oracle_iso_normalization import (
     resolve_oracle_timestamp_metric_normalization,
     resolve_oracle_timestamp_normalization_template,
@@ -253,7 +256,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
         if order_by:
             order_by_columns = [col.name for col in table_context.index_column_collection]
 
-        return table_context.sql_generator.generate_row_md5_select_query(
+        sql = table_context.sql_generator.generate_row_md5_select_query(
             platform=self.platform.value,
             index_column_collection=table_context.index_column_collection,
             column_names_separate_by_comma=table_context.join_column_names_for_row_hash(),
@@ -265,6 +268,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
             column_groups=table_context.column_groups,
             order_by_columns=order_by_columns,
         )
+        return prepend_oracle_bfile_function(sql)
 
     def generate_cell_validation_data_query(
         self,
@@ -290,7 +294,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
             column_normalization_map,
         )
 
-        return table_context.sql_generator.generate_cell_validation_data_select_query(
+        sql = table_context.sql_generator.generate_cell_validation_data_select_query(
             platform=table_context.platform.value,
             column_collection=table_context.columns_to_validate,
             datatypes_normalization_renderer_templates=datatypes_normalization_renderer_templates,
@@ -300,6 +304,7 @@ class QueryGeneratorOracle(QueryGeneratorBase):
             order_by_columns=order_by_columns,
             query_modifiers=resolve_table_context_modifiers(table_context),
         )
+        return prepend_oracle_bfile_function(sql)
 
     def _generate_compute_md5_chunk_query(
         self,

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/postgresql/extractor/templates/postgresql_column_metrics_templates.yaml

@@ -528,6 +528,10 @@ BOOL:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
 
+# min/max must declare metric_normalization so L2 wraps with the same ISO
+# templates as postgresql_datatypes_normalization_templates.yaml (canonical
+# T-separator + 9 fractional digits). Without it, MIN/MAX return native
+# timestamps and Python str() fails string compare against Snowflake FF9.
 DATE:
   min:
     metric_query: MIN("{{ col_name }}")
@@ -543,9 +547,11 @@ TIMESTAMP:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: TIMESTAMP
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000'
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: TIMESTAMP
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000'
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
@@ -554,9 +560,11 @@ TIMESTAMP WITHOUT TIME ZONE:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: TIMESTAMP
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000'
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: TIMESTAMP
+    metric_normalization: TO_CHAR({{ metric_query }}, 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000'
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
@@ -565,9 +573,11 @@ TIMESTAMPTZ:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: TIMESTAMP WITH TIME ZONE
+    metric_normalization: TO_CHAR({{ metric_query }} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000+00:00'
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: TIMESTAMP WITH TIME ZONE
+    metric_normalization: TO_CHAR({{ metric_query }} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000+00:00'
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER
@@ -576,9 +586,11 @@ TIMESTAMP WITH TIME ZONE:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: TIMESTAMP WITH TIME ZONE
+    metric_normalization: TO_CHAR({{ metric_query }} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000+00:00'
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: TIMESTAMP WITH TIME ZONE
+    metric_normalization: TO_CHAR({{ metric_query }} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US') || '000+00:00'
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INTEGER

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/postgresql/query/query_generator_postgresql.py

@@ -36,6 +36,7 @@ from snowflake.snowflake_data_validation.query.vector_element_kind import (
 from snowflake.snowflake_data_validation.utils.constants import (
     NULL_HASHING_SENTINEL,
     NULL_HASHING_SENTINEL_KEY,
+    NUMERIC_FORMAT_MASK_PLACEHOLDER,
     ROW_HASH_COLUMN_DELIMITER,
     ROW_HASH_COLUMN_DELIMITER_KEY,
     Platform,
@@ -69,6 +70,27 @@ def build_postgresql_vector_bit_shim(col_ref: str, element) -> str:
     return f"CASE WHEN {col_ref} IS NULL THEN NULL ELSE ({inner}) END"
 
 
+def build_postgresql_numeric_canonical_expr(col_ref: str, format_mask: str, scale: int) -> str:
+    """Restore FM leading zeros PostgreSQL omits for |v| < 1 to match Snowflake ``TO_VARCHAR``.
+
+    ``TO_CHAR`` with an ``FM`` mask renders ``|v| < 1`` without the integer zero
+    (``.0001``), while Snowflake ``TO_VARCHAR`` keeps it (``0.0001``). Prefix the missing
+    zero (handling the negative ``-.0001`` case) so row hashes and L3 cell comparisons stay
+    aligned. Mirrors Oracle / Teradata / Snowflake / Redshift, each of which restores the
+    leading zero in its own wrapper.
+
+    Exact zeros short-circuit because a mask-rendered zero carries no integer digit at all.
+    """
+    zero_literal = f"0.{'0' * scale}" if scale > 0 else "0"
+    inner = f"TRIM(TO_CHAR(CAST({col_ref} AS NUMERIC), '{format_mask}'))"
+    return (
+        f"CASE WHEN {col_ref} IS NULL THEN NULL "
+        f"WHEN {col_ref} = 0 THEN '{zero_literal}' "
+        f"ELSE TRIM(REGEXP_REPLACE("
+        f"TRIM(REGEXP_REPLACE({inner}, '^-\\.', '-0.')), '^\\.', '0.')) END"
+    )
+
+
 class QueryGeneratorPostgresql(QueryGeneratorBase):
     """Query generator for PostgreSQL (chunked MD5 pipeline aligned with Redshift)."""
 
@@ -97,6 +119,14 @@ class QueryGeneratorPostgresql(QueryGeneratorBase):
         element = vector_element_kind(getattr(column, "data_type", "") or "")
         return build_postgresql_vector_bit_shim(col_ref, element)
 
+    def render_normalization_template(self, template: str, column) -> str:
+        """Wrap numeric ``TO_CHAR`` output to restore the FM-suppressed leading zero."""
+        if NUMERIC_FORMAT_MASK_PLACEHOLDER in template and "TO_CHAR" in template and self._is_numeric_normalization_type(column.data_type):
+            scale = self._resolve_numeric_scale(column)
+            mask = self._build_numeric_format_mask(scale)
+            return build_postgresql_numeric_canonical_expr(f'"{column.name}"', mask, scale)
+        return super().render_normalization_template(template, column)
+
     def _render_normalization_template(self, template: str, column) -> str:
         return super()._render_normalization_template(template, column)
 

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/public/validation/cell_validation.py

@@ -37,6 +37,7 @@ def validate_cell(
     table_context: TableContext,
     index_columns: list[str] | None = None,
     target_index_columns: list[str] | None = None,
+    semi_structured_columns: frozenset[str] | None = None,
 ) -> tuple[ValidationResult, list[DuplicateKeyInfo]]:
     """Validate individual cell values between source and target.
 
@@ -58,6 +59,9 @@ def validate_cell(
             When ``None``, rows are matched positionally.
         target_index_columns: Target-side key column names.
             Defaults to *index_columns* when ``None``.
+        semi_structured_columns: Source-side column names needing recursive
+            JSON canonicalization (e.g. a Redshift SUPER column paired with
+            a Snowflake VARIANT mirror) instead of a plain string compare.
 
     Returns:
         A tuple of ``(ValidationResult, duplicate_key_infos)`` where
@@ -78,6 +82,7 @@ def validate_cell(
             target_index_columns=target_index_columns,
             platform=table_context.platform.value,
             object_id=table_context.id,
+            semi_structured_columns=semi_structured_columns,
         )
 
         return (

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/public/validation/schema_validation.py

@@ -47,6 +47,7 @@ _SCHEMA_MIGRATION_MAPPING_ENRICH_PLATFORMS: frozenset[Platform] = frozenset(
         Platform.ORACLE,
         Platform.SQLSERVER,
         Platform.REDSHIFT,
+        Platform.AZURE_SYNAPSE,
     }
 )
 

{snowflake_data_validation-1.35.2 → snowflake_data_validation-1.37.0}/src/snowflake/snowflake_data_validation/query/numeric_scale_resolution.py

@@ -46,6 +46,7 @@ _MONEY_SCALE_BY_TYPE = {
 
 # Snowflake NUMBER(p, s): max 38 total digits, max 37 fractional digits.
 SNOWFLAKE_MAX_NUMBER_SCALE = 37
+SNOWFLAKE_MAX_NUMBER_PRECISION = 38
 
 
 def parse_scale_value(raw: object) -> int | None: