snowflake-data-validation 1.47.0__tar.gz → 1.48.0__tar.gz

{snowflake_data_validation-1.47.0 → snowflake_data_validation-1.48.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.47.0
+Version: 1.48.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -79,7 +79,7 @@ Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgresql'
 Provides-Extra: redshift
 Requires-Dist: redshift-connector>=2.1.14; extra == 'redshift'
 Provides-Extra: scai
-Requires-Dist: snowflake-code-unit-registry>=0.9.15; extra == 'scai'
+Requires-Dist: snowflake-code-unit-registry>=0.9.19; extra == 'scai'
 Provides-Extra: sqlite
 Requires-Dist: sqlean-py>=3.45.0; extra == 'sqlite'
 Provides-Extra: sqlserver
@@ -164,7 +164,7 @@ An integer column mapped to a `NUMBER` with scale 0 is the exception to the prec
 
 A PostgreSQL user-defined `ENUM` column is validated as `ENUM` against a Snowflake `VARCHAR`. `information_schema` reports every enum as `USER-DEFINED`, so the metadata queries read `pg_type.typtype` to tell enums apart, and report the longest label as the column's length. L2 and L3 compare the label text.
 
-Snowflake database, schema and table names may contain a backslash or an apostrophe, for example `NAFMAIN\TUSDB102`. Snowflake reads backslash escapes inside a string literal, so the Snowflake metadata queries (table metadata, column metadata, case-sensitive columns and object-type lookups) write each name through `escape_snowflake_sql_literal` / `quote_snowflake_sql_literal` in `shared/queries/sql_literals.py`, which double backslashes as well as single quotes. Use these helpers, not `escape_sql_literal`, for any name embedded in a Snowflake string literal.
+Snowflake database, schema and table names may contain a backslash or an apostrophe, for example `NAFMAIN\TUSDB102`. Snowflake reads backslash escapes inside a string literal, so the Snowflake metadata queries (table metadata, column metadata, case-sensitive columns, object-type lookups, and the column list and table list that column partitioning and the configuration generator read through `utils/catalog_literals.py`) write each name through `escape_snowflake_sql_literal` / `quote_snowflake_sql_literal` in `shared/queries/sql_literals.py`, which double backslashes as well as single quotes. Use these helpers, not `escape_sql_literal`, for any name embedded in a Snowflake string literal.
 
 ### Metrics Validation
 Compares statistical metrics for each column:
@@ -266,10 +266,37 @@ empty result becomes an empty DataFrame with no columns.
 
 For configuration options and examples, see the [official documentation](https://docs.snowflake.com/en/migrations/snowconvert-docs/data-validation-cli/CLI_QUICK_REFERENCE).
 
+Async validation reads extracted schema and metrics files laid out as described in [Async validation files](../docs/snowflake-data-validation/async-validation-files.md).
+
 ---
 
 ## Changelog
 
+### v1.48.0
+
+**New features**
+
+- Added a row checksum gate that validates a large matching table with one aggregate query per side before extracting row signatures.
+- Added support for Snowflake `IPADDRESS` as an opt-in target for PostgreSQL `INET`.
+- Added L1, L2, and L3 validation parity for Oracle Iceberg tables.
+- Added validation of `INTERVAL` and `NUMBER(*)` target encodings.
+- Added a validation config setting that names one `HASH_MD5` UDF for every Teradata source database.
+- Added the adoption row-hash check for partially migrated Snowflake targets.
+
+**Improvements**
+
+- Improved L2 metrics by dropping the `FLOAT` sum, adding exact sign counts, and aligning the text and binary metric sets.
+- Improved Oracle `SDO_GEOMETRY` validation by comparing the whole WKT by MD5 and keeping the SRID as EWKT on `GEOMETRY` targets.
+
+**Bug fixes**
+
+- Fixed Snowflake catalog and `SHOW` lookups not doubling backslashes in name literals.
+- Fixed Azure Synapse `VARCHAR` time values not being compared as the text CETAS stored.
+- Fixed `etl-validate` row validation by pairing each target column with its source type.
+- Fixed Oracle dates and timestamps before year 1 losing their sign.
+- Fixed Oracle integer spatial coordinates not being compared as Snowflake writes them.
+- Fixed Teradata L3 false mismatches on non-ASCII values in `LATIN` columns.
+
 ### v1.47.0
 
 **New features**

{snowflake_data_validation-1.47.0 → snowflake_data_validation-1.48.0}/README.md

@@ -73,7 +73,7 @@ An integer column mapped to a `NUMBER` with scale 0 is the exception to the prec
 
 A PostgreSQL user-defined `ENUM` column is validated as `ENUM` against a Snowflake `VARCHAR`. `information_schema` reports every enum as `USER-DEFINED`, so the metadata queries read `pg_type.typtype` to tell enums apart, and report the longest label as the column's length. L2 and L3 compare the label text.
 
-Snowflake database, schema and table names may contain a backslash or an apostrophe, for example `NAFMAIN\TUSDB102`. Snowflake reads backslash escapes inside a string literal, so the Snowflake metadata queries (table metadata, column metadata, case-sensitive columns and object-type lookups) write each name through `escape_snowflake_sql_literal` / `quote_snowflake_sql_literal` in `shared/queries/sql_literals.py`, which double backslashes as well as single quotes. Use these helpers, not `escape_sql_literal`, for any name embedded in a Snowflake string literal.
+Snowflake database, schema and table names may contain a backslash or an apostrophe, for example `NAFMAIN\TUSDB102`. Snowflake reads backslash escapes inside a string literal, so the Snowflake metadata queries (table metadata, column metadata, case-sensitive columns, object-type lookups, and the column list and table list that column partitioning and the configuration generator read through `utils/catalog_literals.py`) write each name through `escape_snowflake_sql_literal` / `quote_snowflake_sql_literal` in `shared/queries/sql_literals.py`, which double backslashes as well as single quotes. Use these helpers, not `escape_sql_literal`, for any name embedded in a Snowflake string literal.
 
 ### Metrics Validation
 Compares statistical metrics for each column:
@@ -175,10 +175,37 @@ empty result becomes an empty DataFrame with no columns.
 
 For configuration options and examples, see the [official documentation](https://docs.snowflake.com/en/migrations/snowconvert-docs/data-validation-cli/CLI_QUICK_REFERENCE).
 
+Async validation reads extracted schema and metrics files laid out as described in [Async validation files](../docs/snowflake-data-validation/async-validation-files.md).
+
 ---
 
 ## Changelog
 
+### v1.48.0
+
+**New features**
+
+- Added a row checksum gate that validates a large matching table with one aggregate query per side before extracting row signatures.
+- Added support for Snowflake `IPADDRESS` as an opt-in target for PostgreSQL `INET`.
+- Added L1, L2, and L3 validation parity for Oracle Iceberg tables.
+- Added validation of `INTERVAL` and `NUMBER(*)` target encodings.
+- Added a validation config setting that names one `HASH_MD5` UDF for every Teradata source database.
+- Added the adoption row-hash check for partially migrated Snowflake targets.
+
+**Improvements**
+
+- Improved L2 metrics by dropping the `FLOAT` sum, adding exact sign counts, and aligning the text and binary metric sets.
+- Improved Oracle `SDO_GEOMETRY` validation by comparing the whole WKT by MD5 and keeping the SRID as EWKT on `GEOMETRY` targets.
+
+**Bug fixes**
+
+- Fixed Snowflake catalog and `SHOW` lookups not doubling backslashes in name literals.
+- Fixed Azure Synapse `VARCHAR` time values not being compared as the text CETAS stored.
+- Fixed `etl-validate` row validation by pairing each target column with its source type.
+- Fixed Oracle dates and timestamps before year 1 losing their sign.
+- Fixed Oracle integer spatial coordinates not being compared as Snowflake writes them.
+- Fixed Teradata L3 false mismatches on non-ASCII values in `LATIN` columns.
+
 ### v1.47.0
 
 **New features**

{snowflake_data_validation-1.47.0 → snowflake_data_validation-1.48.0}/pyproject.toml

@@ -106,7 +106,7 @@ db2 = [
 # musllinux build, and SDV is published to public PyPI. Inside DMO/DEA the
 # dependency is already declared, so SDV's logger tree is covered there.
 scai = [
-  "snowflake-code-unit-registry>=0.9.15",
+  "snowflake-code-unit-registry>=0.9.19",
 ]
 
 [project.urls]

{snowflake_data_validation-1.47.0 → snowflake_data_validation-1.48.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml

@@ -18,27 +18,16 @@ BIGINT:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: BIGINT
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM("{{ col_name }}" {{ modifier }})
     metric_return_datatype: BIGINT
     metric_column_modifier: "% 2097143"
-  stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 INT:
   min:
@@ -47,19 +36,12 @@ INT:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: INT
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
@@ -69,10 +51,6 @@ INT:
   # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
   # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
   # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
-  stddev:
-    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 SMALLINT:
   min:
@@ -81,19 +59,12 @@ SMALLINT:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: SMALLINT
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
@@ -103,10 +74,6 @@ SMALLINT:
   # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
   # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
   # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
-  stddev:
-    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 TINYINT:
   min:
@@ -115,19 +82,12 @@ TINYINT:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: TINYINT
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM(CAST("{{ col_name }}" {{ modifier }} AS BIGINT))
     metric_return_datatype: BIGINT
@@ -137,10 +97,6 @@ TINYINT:
   # and spread small (inventory.inv_date_sk: 0-5.5 vs Snowflake 3.5 on identical data).
   # 32-bit values keep n*SUM(x^2) inside DECIMAL(38) even with the modifier off; BIGINT
   # stays on FLOAT STDEVP because raw 64-bit squares would overflow it.
-  stddev:
-    metric_query: SQRT(CAST(COUNT_BIG("{{ col_name }}") * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0)) * CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) - SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) * SUM(CAST("{{ col_name }}" {{ modifier }} AS DECIMAL(38, 0))) AS FLOAT)) / NULLIF(COUNT_BIG("{{ col_name }}"), 0)
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 DECIMAL:
   min:
@@ -149,27 +105,16 @@ DECIMAL:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: DECIMAL
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM("{{ col_name }}" {{ modifier }})
     metric_return_datatype: DECIMAL
     metric_column_modifier: "% 2097143"
-  stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 NUMERIC:
   min:
@@ -178,27 +123,16 @@ NUMERIC:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: NUMERIC
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM("{{ col_name }}" {{ modifier }})
     metric_return_datatype: NUMERIC
     metric_column_modifier: "% 2097143"
-  stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 MONEY:
   min:
@@ -207,27 +141,16 @@ MONEY:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: MONEY
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM("{{ col_name }}" {{ modifier }})
     metric_return_datatype: MONEY
     metric_column_modifier: "% 2097143"
-  stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 SMALLMONEY:
   min:
@@ -236,31 +159,17 @@ SMALLMONEY:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: SMALLMONEY
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
   sum:
     metric_query: SUM("{{ col_name }}" {{ modifier }})
     metric_return_datatype: SMALLMONEY
     metric_column_modifier: "% 2097143"
-  stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" {{ modifier }} AS FLOAT))
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
-# avg/sum/stddev aggregate the binary64 value, as Snowflake aggregates its FLOAT
-# (SNOW-4314808); REAL widens exactly. T-SQL has no FLOAT %, so the generator
-# renders {{ modifier }} on this operand as Snowflake's float remainder (tsql_float_remainder.py).
 FLOAT:
   min:
     metric_query: MIN("{{ col_name }}")
@@ -268,29 +177,18 @@ FLOAT:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: FLOAT
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
+  count_positive:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" > 0 AND "{{ col_name }}" = "{{ col_name }}" THEN 1 ELSE 0 END)
+    metric_return_datatype: INT
+  count_negative:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" < 0 AND "{{ col_name }}" = "{{ col_name }}" THEN 1 ELSE 0 END)
+    metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
-  sum:
-    metric_query: SUM(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
-  stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
-
-# avg/sum/stddev widen to binary64, as FLOAT's do (SNOW-4314808).
 REAL:
   min:
     metric_query: MIN("{{ col_name }}")
@@ -298,29 +196,18 @@ REAL:
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: REAL
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
+  count_positive:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" > 0 AND "{{ col_name }}" = "{{ col_name }}" THEN 1 ELSE 0 END)
+    metric_return_datatype: INT
+  count_negative:
+    metric_query: SUM(CASE WHEN "{{ col_name }}" < 0 AND "{{ col_name }}" = "{{ col_name }}" THEN 1 ELSE 0 END)
+    metric_return_datatype: INT
   count_zero:
     metric_query: SUM(CASE WHEN "{{ col_name }}" = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
-  sum:
-    metric_query: SUM(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
-    metric_return_datatype: REAL
-    metric_column_modifier: "% 2097143"
-  stddev:
-    metric_query: STDEVP(CAST("{{ col_name }}" AS FLOAT) {{ modifier }})
-    metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
-
-# ---- Boolean type ----
 
 BIT:
   count_true:
@@ -346,40 +233,33 @@ CHAR:
   count_empty:
     metric_query: SUM(CASE WHEN LEN(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  count_distinct:
-    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST(LEN(RTRIM("{{ col_name }}")) AS FLOAT))
+  sum:
+    metric_query: SUM(CAST(LEN(RTRIM("{{ col_name }}")) AS FLOAT))
     metric_return_datatype: FLOAT
-    metric_column_modifier: "% 2097143"
 
 VARCHAR:
   # Characters, trailing spaces trimmed (SNOW-4292391, as SQL Server's SNOW-4283493).
   # DATALENGTH counted bytes, so 'É' under a UTF-8 collation was 2 against Snowflake's 1.
   # LEN ignores trailing spaces and the Snowflake side RTRIMs, so 'abc  ' is 3 and '   '
-  # is empty on both sides. count_distinct keeps trailing spaces significant.
+  # is empty on both sides.
   min:
     metric_query: MIN(LEN("{{ col_name }}"))
     metric_return_datatype: INT
   max:
     metric_query: MAX(LEN("{{ col_name }}"))
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST(LEN("{{ col_name }}") AS FLOAT))
-    metric_return_datatype: FLOAT
   count_empty:
     metric_query: SUM(CASE WHEN LEN("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  count_distinct:
-    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)){{ measure_collation_suffix }})
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
+  sum:
+    metric_query: SUM(CAST(LEN("{{ col_name }}") AS FLOAT))
+    metric_return_datatype: FLOAT
 
 NCHAR:
   min:
@@ -388,18 +268,15 @@ NCHAR:
   max:
     metric_query: MAX(LEN(RTRIM("{{ col_name }}")))
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST(LEN(RTRIM("{{ col_name }}")) AS FLOAT))
-    metric_return_datatype: FLOAT
   count_empty:
     metric_query: SUM(CASE WHEN LEN(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  count_distinct:
-    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
+  sum:
+    metric_query: SUM(CAST(LEN(RTRIM("{{ col_name }}")) AS FLOAT))
+    metric_return_datatype: FLOAT
 
 NVARCHAR:
   # As VARCHAR above (SNOW-4292391). count_empty uses the same LEN, so '   ' is empty
@@ -411,18 +288,15 @@ NVARCHAR:
   max:
     metric_query: MAX(LEN("{{ col_name }}"))
     metric_return_datatype: INT
-  avg:
-    metric_query: AVG(CAST(LEN("{{ col_name }}") AS FLOAT))
-    metric_return_datatype: FLOAT
   count_empty:
     metric_query: SUM(CASE WHEN LEN("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
-  count_distinct:
-    metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST(N'|' + CAST("{{ col_name }}" AS NVARCHAR(MAX)) + N'|' AS NVARCHAR(MAX)){{ measure_collation_suffix }})
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
+  sum:
+    metric_query: SUM(CAST(LEN("{{ col_name }}") AS FLOAT))
+    metric_return_datatype: FLOAT
 
 # ---- Date / Time types ----
 
@@ -517,9 +391,6 @@ VARBINARY:
 # ---- Other types ----
 
 UNIQUEIDENTIFIER:
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
@@ -527,9 +398,6 @@ UNIQUEIDENTIFIER:
 # ---- Default fallback ----
 
 DEFAULT:
-  count_distinct:
-    metric_query: COUNT(DISTINCT "{{ col_name }}")
-    metric_return_datatype: INT
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT

{snowflake_data_validation-1.47.0 → snowflake_data_validation-1.48.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -55,6 +55,9 @@ TIME: LEFT(SUBSTRING(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7))
 # Iceberg compatible TIME stored as VARCHAR (CETAS): the extract's own text, TIME(7)
 # widened then '00', nine digits whatever the scale. '+' keeps NULL as NULL.
 TIME_ICEBERG: CAST(CAST("{{ col_name }}" AS TIME(7)) AS VARCHAR(16)) + '00'
+# CETAS text. Seven digits, not the six-digit DATETIME2 / DATETIMEOFFSET lines below.
+DATETIME2_ICEBERG: STUFF(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 11, 1, 'T') + '00'
+DATETIMEOFFSET_ICEBERG: STUFF(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 11, 1, 'T') + '00' + DATENAME(TZOFFSET, "{{ col_name }}")
 DATETIME: STUFF(CONVERT(VARCHAR(27), CAST(CAST("{{ col_name }}" AS DATETIME2(3)) AS DATETIME2(7)), 121), 11, 1, 'T') + '00'
 DATETIME2: LEFT(STUFF(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 11, 1, 'T'), 26) + '000'
 SMALLDATETIME: CONVERT(VARCHAR(19), "{{ col_name }}", 126) + '.000000000'

{snowflake_data_validation-1.47.0 → snowflake_data_validation-1.48.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_table_metadata_query.sql.j2

@@ -1,7 +1,7 @@
 {% import 'query_modifiers.sql.j2' as qm with context %}
 {{ qm.locking_prefix(query_modifiers) }}
 WITH rc_cte AS (
-        SELECT COUNT_BIG(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }} {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
+        {% if omit_row_count %}SELECT CAST(0 AS BIGINT) AS ROW_COUNT{% else %}SELECT COUNT_BIG(*) AS ROW_COUNT {{ qm.render_from_clause(query_modifiers, fully_qualified_name) }} {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}{% endif %}
 )
     SELECT
         C.COLUMN_NAME AS COLUMN_VALIDATED,

{snowflake_data_validation-1.47.0 → snowflake_data_validation-1.48.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -40,6 +40,7 @@ from snowflake.snowflake_data_validation.query.numeric_scale_resolution import (
 from snowflake.snowflake_data_validation.query.query_generator_base import (
     QueryGeneratorBase,
     resolve_table_context_modifiers,
+    tsql_unsigned_hex_slice_sql,
 )
 from snowflake.snowflake_data_validation.query.tsql_float_bits import (
     build_tsql_float_bit_canonical_expr,
@@ -96,6 +97,10 @@ class QueryGeneratorAzureSynapse(TsqlFloatMetricModifierMixin, QueryGeneratorBas
             character_set_resolver=SqlServerCharacterSetResolver(),
         )
 
+    def unsigned_hex_slice_sql(self, column: str, start: int, length: int = 8) -> str | None:
+        """Read hex characters of *column* as an unsigned integer."""
+        return tsql_unsigned_hex_slice_sql(column, start, length)
+
     def _measure_collation_tokens(self, column, metric_name: str) -> tuple[str, str]:
         return tsql_measure_collation_tokens(
             Platform.AZURE_SYNAPSE,