snowflake-data-validation 1.43.0__tar.gz → 1.44.0__tar.gz

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/.gitignore

@@ -232,6 +232,7 @@ requirements-dev.lock
 
 # E2E test session config (contains credentials)
 e2e-tests/config.yaml
+e2e-tests/dev-1tb-iceberg-worker.toml
 
 # E2E Logs
 e2e-tests/e2e-logs/

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: snowflake-data-validation
-Version: 1.43.0
+Version: 1.44.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/
@@ -38,7 +38,7 @@ Requires-Dist: pydantic-yaml>=1.4.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: ruamel-yaml>=0.18.0
-Requires-Dist: snowflake-connector-python>=4.7.1
+Requires-Dist: snowflake-connector-python<5,>=4.7.1
 Requires-Dist: toml>=0.10.2
 Requires-Dist: typer>=0.15.3
 Requires-Dist: typing-extensions>=4.13.2
@@ -79,7 +79,7 @@ Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgresql'
 Provides-Extra: redshift
 Requires-Dist: redshift-connector>=2.1.14; extra == 'redshift'
 Provides-Extra: scai
-Requires-Dist: snowflake-code-unit-registry>=0.8.83; extra == 'scai'
+Requires-Dist: snowflake-code-unit-registry>=0.9.0; extra == 'scai'
 Provides-Extra: sqlite
 Requires-Dist: sqlean-py>=3.45.0; extra == 'sqlite'
 Provides-Extra: sqlserver
@@ -162,6 +162,8 @@ Compares statistical metrics for each column:
 - Null count
 - Distinct count
 
+For some types these metrics do not depend on the values themselves (for SQL Server: `VECTOR`, `XML`, `IMAGE`, `SQL_VARIANT`, `JSON`, `GEOGRAPHY`, `GEOMETRY`, `UNIQUEIDENTIFIER`), so a changed value can pass L2. Use L3 row validation to compare those columns' values. The per-type list is in `docs/data-migration-orchestrator/sqlserver-data-type-coverage.md` in this repo.
+
 ### Views
 
 For **Redshift** and **Teradata** sources, views are validated on the view itself (no `CREATE TABLE AS SELECT` materialization in local sync mode). Other platforms may still materialize views to temporary tables before L1/L2/L3. **Teradata** views are read via `DBC.ColumnsQV` (Teradata's Query View Column Information catalog), so view L1 covers the full field set — column existence, datatype, precision, scale, length, nullability, and ordinal position — matching the table path. When `DBC.ColumnsQV` is unavailable (QVCI disabled → Teradata error 9719, missing `SELECT` grant, any other failure of that catalog query, or a degenerate result with null `DATA_TYPE`), the extractor falls back to `HELP COLUMN` with `basic_validation=True` and emits one WARNING schema row per view naming the trigger and the remediation. If HELP fails or source schema metadata still has no usable `DATA_TYPE` (including a view validated as a table whose `ColumnsV` types are null), L1 fails that object and does not continue to L2/L3. **Redshift** uses full schema metadata like tables.
@@ -182,6 +184,28 @@ Performs row-by-row comparison:
 
 ---
 
+## 🐍 Python version inside SnowConvert AI
+
+The `scai` CLI also ships this package vendored and runs it in the interpreter CSnakes downloads,
+not the Python you `pip install` into. That interpreter is pinned by
+`ServiceCollectionExtensions.RedistributablePython` in
+`Snowflake.SnowConvertDesktop/Snowflake.SnowConvert.Jobs/Extensions/PythonEnvironmentExtenstion.cs`
+(today `RedistributablePythonVersion.Python3_12`, which CSnakes.Runtime 1.2.1 resolves to CPython 3.12.9).
+
+The `[CI][SCAI] CSnakes interpreter-pin audit` workflow (`scai-csnakes-pin-audit.yml`) fails when that
+version does not meet the `requires-python` of this package or of the other projects vendored beside it.
+To update the pin:
+
+1. Change `RedistributablePython` to another `RedistributablePythonVersion` member, or bump
+   `CSnakes.Runtime` to the same version in both `Snowflake.SnowConvert.Jobs.csproj` and
+   `Snowflake.SnowConvert.Assessment.csproj`.
+2. Run `python .github/scripts/snowconvert-ai/csnakes_pin_audit.py` (needs `dotnet` and `packaging`).
+   It prints the resolved version and every audited floor.
+
+Raising `requires-python` here above the pinned version fails the audit until the pin moves too.
+
+---
+
 ## 📚 Documentation
 
 For configuration options and examples, see the [official documentation](https://docs.snowflake.com/en/migrations/snowconvert-docs/data-validation-cli/CLI_QUICK_REFERENCE).
@@ -190,6 +214,46 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.44.0
+
+**New features**
+
+- Added Teradata `PERIOD` type dual-path validation helpers.
+- Added per-column L2 metric overrides for data validation.
+- Added column-scoped custom validation metrics model.
+- Added Iceberg `FULL_INGEST` native temporal type migration support.
+
+**Improvements**
+
+- Improved trace-id correlation across SCAI and DMVF logs.
+- Improved L2 metrics on Iceberg profiles by excluding flattened `VARCHAR` columns.
+- Improved `TIME` type extraction to use ISO text format across all native data migration paths.
+- Improved handling of opaque and complex types to use NULL and distinct-count checks instead of value comparison.
+
+**Bug fixes**
+
+- Fixed ETL compare failure when decoding the bundled datatype-normalization YAML.
+- Fixed Oracle `XMLTYPE` hybrid L3 validation failure with `ORA-28817` in `DBMS_CRYPTO.HASH`.
+- Fixed bare BigQuery `BIGNUMERIC` alignment with the engine `DECFLOAT` mapping.
+- Fixed Oracle metadata templates incorrectly upper-casing quoted table and column names.
+- Fixed `DATETIMEOFFSET` false mismatches caused by offset not being normalized to UTC.
+- Fixed SQL Server `XML` and `IMAGE` columns failing the entire L2 metrics batch.
+- Fixed SQL Server `TEXT` and `NTEXT` `count_distinct` producing invalid T-SQL and missing `BIN2` collation.
+- Fixed PostgreSQL `BIT`, `BIT VARYING`, and `VARBIT` `count_distinct` failing the L2 metrics batch.
+- Fixed DB2 `XML`, `LONG VARCHAR`, and `LONG VARGRAPHIC` falling to default `COUNT(DISTINCT)` and failing the L2 metrics batch.
+- Fixed DB2 `BOOLEAN` metric set mismatch with Snowflake `BOOLEAN` metrics.
+- Fixed SQL Server row validation failure on tables with an `IMAGE` column.
+- Fixed string-length metrics producing false failures on SQL Server to Snowflake validation.
+- Fixed SQL identifier quoting to use dialect-correct syntax across all platforms.
+- Fixed row validation incorrectly reporting `DATA_MATCH` when the target table contains extra duplicate rows.
+
+### v1.43.1
+
+**Bug fixes**
+
+- Fixed Teradata `WRITE_NOS` extract retry failing with Error 9134 (object already exists) after a session drop by clearing the landing prefix before each attempt.
+- Fixed false `MISMATCH` results for PostgreSQL `JSON`/`JSONB` vs Snowflake `VARIANT` on key order, number spelling, and whitespace inside strings.
+
 ### v1.43.0
 
 **New features**

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/README.md

@@ -71,6 +71,8 @@ Compares statistical metrics for each column:
 - Null count
 - Distinct count
 
+For some types these metrics do not depend on the values themselves (for SQL Server: `VECTOR`, `XML`, `IMAGE`, `SQL_VARIANT`, `JSON`, `GEOGRAPHY`, `GEOMETRY`, `UNIQUEIDENTIFIER`), so a changed value can pass L2. Use L3 row validation to compare those columns' values. The per-type list is in `docs/data-migration-orchestrator/sqlserver-data-type-coverage.md` in this repo.
+
 ### Views
 
 For **Redshift** and **Teradata** sources, views are validated on the view itself (no `CREATE TABLE AS SELECT` materialization in local sync mode). Other platforms may still materialize views to temporary tables before L1/L2/L3. **Teradata** views are read via `DBC.ColumnsQV` (Teradata's Query View Column Information catalog), so view L1 covers the full field set — column existence, datatype, precision, scale, length, nullability, and ordinal position — matching the table path. When `DBC.ColumnsQV` is unavailable (QVCI disabled → Teradata error 9719, missing `SELECT` grant, any other failure of that catalog query, or a degenerate result with null `DATA_TYPE`), the extractor falls back to `HELP COLUMN` with `basic_validation=True` and emits one WARNING schema row per view naming the trigger and the remediation. If HELP fails or source schema metadata still has no usable `DATA_TYPE` (including a view validated as a table whose `ColumnsV` types are null), L1 fails that object and does not continue to L2/L3. **Redshift** uses full schema metadata like tables.
@@ -91,6 +93,28 @@ Performs row-by-row comparison:
 
 ---
 
+## 🐍 Python version inside SnowConvert AI
+
+The `scai` CLI also ships this package vendored and runs it in the interpreter CSnakes downloads,
+not the Python you `pip install` into. That interpreter is pinned by
+`ServiceCollectionExtensions.RedistributablePython` in
+`Snowflake.SnowConvertDesktop/Snowflake.SnowConvert.Jobs/Extensions/PythonEnvironmentExtenstion.cs`
+(today `RedistributablePythonVersion.Python3_12`, which CSnakes.Runtime 1.2.1 resolves to CPython 3.12.9).
+
+The `[CI][SCAI] CSnakes interpreter-pin audit` workflow (`scai-csnakes-pin-audit.yml`) fails when that
+version does not meet the `requires-python` of this package or of the other projects vendored beside it.
+To update the pin:
+
+1. Change `RedistributablePython` to another `RedistributablePythonVersion` member, or bump
+   `CSnakes.Runtime` to the same version in both `Snowflake.SnowConvert.Jobs.csproj` and
+   `Snowflake.SnowConvert.Assessment.csproj`.
+2. Run `python .github/scripts/snowconvert-ai/csnakes_pin_audit.py` (needs `dotnet` and `packaging`).
+   It prints the resolved version and every audited floor.
+
+Raising `requires-python` here above the pinned version fails the audit until the pin moves too.
+
+---
+
 ## 📚 Documentation
 
 For configuration options and examples, see the [official documentation](https://docs.snowflake.com/en/migrations/snowconvert-docs/data-validation-cli/CLI_QUICK_REFERENCE).
@@ -99,6 +123,46 @@ For configuration options and examples, see the [official documentation](https:/
 
 ## Changelog
 
+### v1.44.0
+
+**New features**
+
+- Added Teradata `PERIOD` type dual-path validation helpers.
+- Added per-column L2 metric overrides for data validation.
+- Added column-scoped custom validation metrics model.
+- Added Iceberg `FULL_INGEST` native temporal type migration support.
+
+**Improvements**
+
+- Improved trace-id correlation across SCAI and DMVF logs.
+- Improved L2 metrics on Iceberg profiles by excluding flattened `VARCHAR` columns.
+- Improved `TIME` type extraction to use ISO text format across all native data migration paths.
+- Improved handling of opaque and complex types to use NULL and distinct-count checks instead of value comparison.
+
+**Bug fixes**
+
+- Fixed ETL compare failure when decoding the bundled datatype-normalization YAML.
+- Fixed Oracle `XMLTYPE` hybrid L3 validation failure with `ORA-28817` in `DBMS_CRYPTO.HASH`.
+- Fixed bare BigQuery `BIGNUMERIC` alignment with the engine `DECFLOAT` mapping.
+- Fixed Oracle metadata templates incorrectly upper-casing quoted table and column names.
+- Fixed `DATETIMEOFFSET` false mismatches caused by offset not being normalized to UTC.
+- Fixed SQL Server `XML` and `IMAGE` columns failing the entire L2 metrics batch.
+- Fixed SQL Server `TEXT` and `NTEXT` `count_distinct` producing invalid T-SQL and missing `BIN2` collation.
+- Fixed PostgreSQL `BIT`, `BIT VARYING`, and `VARBIT` `count_distinct` failing the L2 metrics batch.
+- Fixed DB2 `XML`, `LONG VARCHAR`, and `LONG VARGRAPHIC` falling to default `COUNT(DISTINCT)` and failing the L2 metrics batch.
+- Fixed DB2 `BOOLEAN` metric set mismatch with Snowflake `BOOLEAN` metrics.
+- Fixed SQL Server row validation failure on tables with an `IMAGE` column.
+- Fixed string-length metrics producing false failures on SQL Server to Snowflake validation.
+- Fixed SQL identifier quoting to use dialect-correct syntax across all platforms.
+- Fixed row validation incorrectly reporting `DATA_MATCH` when the target table contains extra duplicate rows.
+
+### v1.43.1
+
+**Bug fixes**
+
+- Fixed Teradata `WRITE_NOS` extract retry failing with Error 9134 (object already exists) after a session drop by clearing the landing prefix before each attempt.
+- Fixed false `MISMATCH` results for PostgreSQL `JSON`/`JSONB` vs Snowflake `VARIANT` on key order, number spelling, and whitespace inside strings.
+
 ### v1.43.0
 
 **New features**

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
   "connectorx>=0.3.0",
   "cryptography>=50.0.0",
   "deepdiff>=8.0.0",
-  "snowflake-connector-python>=4.7.1",
+  "snowflake-connector-python>=4.7.1,<5",
   "pandas>=2.2.3",
   "polars>=1.0.0",
   "pyarrow>=14.0.0",
@@ -105,7 +105,7 @@ db2 = [
 # musllinux build, and SDV is published to public PyPI. Inside DMO/DEA the
 # dependency is already declared, so SDV's logger tree is covered there.
 scai = [
-  "snowflake-code-unit-registry>=0.8.83",
+  "snowflake-code-unit-registry>=0.9.0",
 ]
 
 [project.urls]
@@ -201,7 +201,7 @@ check = [
   'python -m pip install --upgrade pip -q && pip list',
   "python -c \"import pathlib, shutil, site; roots=[pathlib.Path(p) for p in site.getsitepackages()]; stale=[r/'shared' for r in roots if (r/'shared').is_dir()]; [shutil.rmtree(p, ignore_errors=True) for p in stale]; print('removed', stale)\"",
   'python -m pip install -U -q -e ../shared',
-  'pytest -vvv --junitxml=tests/outcome/test-results.xml --cov=. --cov-branch --cov-config=tests/.coveragerc --cov-report=xml:tests/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.xml --cov-report=term --cov-report=html:tests/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.html --cov-report=json:tests/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.json tests --ignore=tests/integration',
+  'pytest -vvv --junitxml=tests/outcome/test-results.xml --cov=. --cov-branch --cov-config=tests/.coveragerc --cov-report=xml:tests/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.xml --cov-report=term --cov-report=html:tests/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.html --cov-report=json:tests/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.json {args:tests --ignore=tests/integration}',
 ]
 
 coverage = [

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_cell_validation_data_select.sql.j2

@@ -2,13 +2,13 @@
 {{ qm.locking_prefix(query_modifiers) }}SELECT
 {% for column in column_collection %}
     {% if column.nullable %}
-        coalesce({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS "{{ column.name }}"{% if not loop.last %},{% endif %}
+        coalesce({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS {{ column_sql_names[column.name] }}{% if not loop.last %},{% endif %}
     {% else %}
-        {{ datatypes_normalization_renderer_templates[column.name] }} AS "{{ column.name }}"{% if not loop.last %},{% endif %}
+        {{ datatypes_normalization_renderer_templates[column.name] }} AS {{ column_sql_names[column.name] }}{% if not loop.last %},{% endif %}
     {% endif %}
 {% endfor %}
 {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 't') }}
 {% if has_where_clause %} WHERE {{ where_clause }} {% endif %}
 {% if order_by_columns %}
-ORDER BY {% for c in order_by_columns %} {{ order_by_collation_expressions.get(c, '"' ~ c ~ '"') }}{% if not loop.last %}, {% endif %}{% endfor %}
+ORDER BY {% for c in order_by_columns %} {{ order_by_collation_expressions.get(c, column_sql_names[c]) }}{% if not loop.last %}, {% endif %}{% endfor %}
 {% endif %}

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_column_metrics_templates.yaml

@@ -4,8 +4,9 @@
 # Azure Synapse uses T-SQL syntax (subset of SQL Server):
 #   - CASE WHEN for conditional expressions (IIF unsupported in Dedicated SQL Pool)
 #   - STDEVP for population standard deviation
-#   - DATALENGTH for byte-length of variable-length columns
-#   - LEN for character-length of unicode columns
+#   - LEN for the character length of every text column. DATALENGTH counts bytes, which
+#     drifts from Snowflake LENGTH under a UTF-8 collation. LEN ignores trailing spaces;
+#     the Snowflake generator pairs it with LENGTH(RTRIM()) (SNOW-4292391)
 #   - % 2097143 overflow guard for numeric aggregations (SNOW-3443973)
 
 # ---- Numeric types ----
@@ -331,14 +332,15 @@ BIT:
 # ---- Character types ----
 
 CHAR:
+  # Characters, not bytes, as VARCHAR below (SNOW-4292391). RTRIM drops CHAR padding.
   min:
-    metric_query: MIN(DATALENGTH(RTRIM("{{ col_name }}")))
+    metric_query: MIN(LEN(RTRIM("{{ col_name }}")))
     metric_return_datatype: INT
   max:
-    metric_query: MAX(DATALENGTH(RTRIM("{{ col_name }}")))
+    metric_query: MAX(LEN(RTRIM("{{ col_name }}")))
     metric_return_datatype: INT
   count_empty:
-    metric_query: SUM(CASE WHEN DATALENGTH(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN LEN(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
     metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
@@ -347,22 +349,26 @@ CHAR:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   avg:
-    metric_query: AVG(CAST(DATALENGTH(RTRIM("{{ col_name }}")) AS FLOAT))
+    metric_query: AVG(CAST(LEN(RTRIM("{{ col_name }}")) AS FLOAT))
     metric_return_datatype: FLOAT
     metric_column_modifier: "% 2097143"
 
 VARCHAR:
+  # Characters, trailing spaces trimmed (SNOW-4292391, as SQL Server's SNOW-4283493).
+  # DATALENGTH counted bytes, so 'É' under a UTF-8 collation was 2 against Snowflake's 1.
+  # LEN ignores trailing spaces and the Snowflake side RTRIMs, so 'abc  ' is 3 and '   '
+  # is empty on both sides. count_distinct keeps trailing spaces significant.
   min:
-    metric_query: MIN(DATALENGTH("{{ col_name }}"))
+    metric_query: MIN(LEN("{{ col_name }}"))
     metric_return_datatype: INT
   max:
-    metric_query: MAX(DATALENGTH("{{ col_name }}"))
+    metric_query: MAX(LEN("{{ col_name }}"))
     metric_return_datatype: INT
   avg:
-    metric_query: AVG(CAST(DATALENGTH("{{ col_name }}") AS FLOAT))
+    metric_query: AVG(CAST(LEN("{{ col_name }}") AS FLOAT))
     metric_return_datatype: FLOAT
   count_empty:
-    metric_query: SUM(CASE WHEN DATALENGTH("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN LEN("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
     metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST('|' + "{{ col_name }}" + '|' AS VARCHAR(MAX)){{ measure_collation_suffix }})
@@ -382,7 +388,7 @@ NCHAR:
     metric_query: AVG(CAST(LEN(RTRIM("{{ col_name }}")) AS FLOAT))
     metric_return_datatype: FLOAT
   count_empty:
-    metric_query: SUM(CASE WHEN RTRIM("{{ col_name }}") = '' THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN LEN(RTRIM("{{ col_name }}")) = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
     metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}RTRIM("{{ col_name }}"){{ measure_collation_suffix }})
@@ -392,6 +398,9 @@ NCHAR:
     metric_return_datatype: INT
 
 NVARCHAR:
+  # As VARCHAR above (SNOW-4292391). count_empty uses the same LEN, so '   ' is empty
+  # here and in Snowflake's LENGTH(RTRIM()) = 0; comparing to an empty literal matched
+  # it on this side only.
   min:
     metric_query: MIN(LEN("{{ col_name }}"))
     metric_return_datatype: INT
@@ -402,7 +411,7 @@ NVARCHAR:
     metric_query: AVG(CAST(LEN("{{ col_name }}") AS FLOAT))
     metric_return_datatype: FLOAT
   count_empty:
-    metric_query: SUM(CASE WHEN "{{ col_name }}" = '' THEN 1 ELSE 0 END)
+    metric_query: SUM(CASE WHEN LEN("{{ col_name }}") = 0 THEN 1 ELSE 0 END)
     metric_return_datatype: INT
   count_distinct:
     metric_query: COUNT(DISTINCT {{ measure_collation_prefix }}CAST(N'|' + CAST("{{ col_name }}" AS NVARCHAR(MAX)) + N'|' AS NVARCHAR(MAX)){{ measure_collation_suffix }})
@@ -474,15 +483,17 @@ SMALLDATETIME:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT
 
+# MIN/MAX in UTC: the row they return keeps its own offset, so the same instant
+# would differ by offset. Snowflake TIMESTAMP_TZ pairs with TIMESTAMP_TZ_UTC.
 DATETIMEOFFSET:
   min:
     metric_query: MIN("{{ col_name }}")
     metric_return_datatype: DATETIMEOFFSET
-    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT({{ metric_query }}, 'zzz')
+    metric_normalization: LEFT(FORMAT(SWITCHOFFSET({{ metric_query }}, '+00:00'), 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000+00:00'
   max:
     metric_query: MAX("{{ col_name }}")
     metric_return_datatype: DATETIMEOFFSET
-    metric_normalization: LEFT(FORMAT({{ metric_query }}, 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000' + FORMAT({{ metric_query }}, 'zzz')
+    metric_normalization: LEFT(FORMAT(SWITCHOFFSET({{ metric_query }}, '+00:00'), 'yyyy-MM-dd''T''HH:mm:ss.fffffff'), 26) + '000+00:00'
   count_null:
     metric_query: SUM(CASE WHEN "{{ col_name }}" IS NULL THEN 1 ELSE 0 END)
     metric_return_datatype: INT

snowflake_data_validation-1.44.0/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_datatypes_normalization_templates.yaml

@@ -0,0 +1,71 @@
+# Numerics and dates use native CONVERT/CAST, not FORMAT(): FORMAT runs through the .NET
+# CLR and is the dominant per-row cost of the row hash. Each expression emits the same text
+# FORMAT did. Style 121 always prints 7 fraction digits (126 drops a zero fraction); STUFF
+# swaps its space for 'T'. DATETIME goes through DATETIME2(3) so its 1/300 s ticks print as
+# the millisecond FORMAT showed. DECIMAL/NUMERIC convert at the column's own scale, then pad the
+# text to the comparison scale; widening to DECIMAL(38, comparison_scale) can overflow valid values.
+# The padding counts the digits CONVERT printed instead of trusting the column's type string:
+# DMO hands a bare DECIMAL the cross-side scale (DECIMAL(18,6) -> NUMBER(20,10) arrives as
+# DECIMAL(18, 10)), which would pad nothing. LEFT('.', s) adds the point to a scale-0 value
+# only when s > 0; ISNULL keeps an unpaddable value as printed, and a NULL column stays NULL.
+# Money widens only to its native precision because CONVERT(VARCHAR, money) keeps 2 dp.
+# Azure Synapse datatype normalization templates.
+# Each entry maps a Synapse type (uppercase) to a SQL expression that normalizes
+# the column value to a canonical string for hashing.
+# Use {{ col_name }} as the column placeholder; the query generator will substitute
+# the actual quoted column name at render time.
+#
+# These templates MUST produce identical string representations to the Snowflake
+# normalization templates so that MD5 hashes match during row validation.
+# Mirrors sqlserver_datatypes_normalization_templates.yaml with IIF replaced by
+# CASE WHEN (IIF is unsupported on Synapse Dedicated SQL Pool).
+
+# ---- Numeric types ----
+BIGINT: CONVERT(VARCHAR(20), "{{ col_name }}")
+INT: CONVERT(VARCHAR(20), "{{ col_name }}")
+SMALLINT: CONVERT(VARCHAR(20), "{{ col_name }}")
+TINYINT: CONVERT(VARCHAR(20), "{{ col_name }}")
+DECIMAL: CONVERT(VARCHAR(64), "{{ col_name }}") + ISNULL(CASE WHEN CHARINDEX('.', CONVERT(VARCHAR(64), "{{ col_name }}")) = 0 THEN LEFT('.', {{ numeric_scale }}) + REPLICATE('0', {{ numeric_scale }}) ELSE REPLICATE('0', {{ numeric_scale }} + CHARINDEX('.', CONVERT(VARCHAR(64), "{{ col_name }}")) - LEN(CONVERT(VARCHAR(64), "{{ col_name }}"))) END, '')
+NUMERIC: CONVERT(VARCHAR(64), "{{ col_name }}") + ISNULL(CASE WHEN CHARINDEX('.', CONVERT(VARCHAR(64), "{{ col_name }}")) = 0 THEN LEFT('.', {{ numeric_scale }}) + REPLICATE('0', {{ numeric_scale }}) ELSE REPLICATE('0', {{ numeric_scale }} + CHARINDEX('.', CONVERT(VARCHAR(64), "{{ col_name }}")) - LEN(CONVERT(VARCHAR(64), "{{ col_name }}"))) END, '')
+MONEY: CONVERT(VARCHAR(64), CAST("{{ col_name }}" AS DECIMAL(19, 4))) + '{{ numeric_scale_padding }}'
+SMALLMONEY: CONVERT(VARCHAR(64), CAST("{{ col_name }}" AS DECIMAL(10, 4))) + '{{ numeric_scale_padding }}'
+FLOAT: '{{ float_bit_shim }}'
+REAL: '{{ float_bit_shim }}'
+
+# ---- Boolean type ----
+BIT: CASE WHEN "{{ col_name }}" = 1 THEN 'TRUE' WHEN "{{ col_name }}" = 0 THEN 'FALSE' END
+
+# ---- Character types ----
+CHAR: RTRIM("{{ col_name }}")
+VARCHAR: '"{{ col_name }}"'
+NCHAR: RTRIM("{{ col_name }}")
+NVARCHAR: '"{{ col_name }}"'
+
+# ---- Date / Time types ----
+# DATETIME keeps 7 fractional digits + '00' pad to FF9.
+# DATETIME2 / DATETIMEOFFSET: Snowflake TIMESTAMP_* stores microseconds. FORMAT
+# '.fffffff' keeps the 7th digit (e.g. .999999900) while Snowflake TO_CHAR FF9
+# emits .999999000 for the truncated mirror. Truncate to 6 fractional digits
+# then pad '000' so hashes match ODBC/microsecond storage. LEFT keeps NULL.
+# Use '+' (not CONCAT) so a NULL input propagates to NULL.
+# DATENAME(TZOFFSET) emits '+HH:MM'/'-HH:MM' for every offset, including UTC.
+DATE: CONVERT(VARCHAR(10), "{{ col_name }}", 23)
+TIME: CONVERT(VARCHAR(8), "{{ col_name }}", 108)
+DATETIME: STUFF(CONVERT(VARCHAR(27), CAST(CAST("{{ col_name }}" AS DATETIME2(3)) AS DATETIME2(7)), 121), 11, 1, 'T') + '00'
+DATETIME2: LEFT(STUFF(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 11, 1, 'T'), 26) + '000'
+SMALLDATETIME: CONVERT(VARCHAR(19), "{{ col_name }}", 126) + '.000000000'
+DATETIMEOFFSET: LEFT(STUFF(CONVERT(VARCHAR(27), CAST("{{ col_name }}" AS DATETIME2(7)), 121), 11, 1, 'T'), 26) + '000' + DATENAME(TZOFFSET, "{{ col_name }}")
+
+# ---- Binary types ----
+BINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
+VARBINARY: UPPER(CONVERT(VARCHAR(MAX), "{{ col_name }}", 2))
+
+# ---- Other types ----
+# Azure Synapse dedicated SQL pool always normalizes UNIQUEIDENTIFIER hex digits
+# to uppercase regardless of input casing. Use UPPER so the canonical string
+# matches the DV cases and Snowflake's VARCHAR-based representation.
+UNIQUEIDENTIFIER: UPPER(CONVERT(VARCHAR(36), "{{ col_name }}"))
+SYSNAME: CONVERT(VARCHAR(128), "{{ col_name }}")
+
+# ---- Default fallback ----
+DEFAULT: CONVERT(VARCHAR(MAX), "{{ col_name }}")

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_row_md5_select.sql.j2

@@ -1,32 +1,32 @@
 {% import 'query_modifiers.sql.j2' as qm with context %}
 {{ qm.locking_prefix(query_modifiers) }}SELECT
     {% for index_column in index_column_collection %}
-        {{ (index_column.name ~ '_IDX') | tsql_quote_identifier }} AS {{ index_column.name | tsql_quote_identifier }},
+        {{ index_sql_aliases[index_column.name] }} AS {{ column_sql_names[index_column.name] }},
     {% endfor %}
     UPPER(CONVERT(VARCHAR(32), HASHBYTES('MD5',
     {% if use_column_groups %}
         {% for group in column_groups %}
             {% if not loop.first %} + {% endif %}
             UPPER(CONVERT(VARCHAR(32), HASHBYTES('MD5', CAST(CONCAT(CAST('' AS VARCHAR(MAX)){{ row_hash_collation_suffix }},
-                {% for col in group %}{{ col.name | tsql_quote_identifier }}{{ row_hash_collation_suffix }}{% if not loop.last %}, '{{ row_hash_column_delimiter }}'{{ row_hash_collation_suffix }}, {% endif %}{% endfor %}
+                {% for col in group %}{{ column_sql_names[col.name] }}{{ row_hash_collation_suffix }}{% if not loop.last %}, '{{ row_hash_column_delimiter }}'{{ row_hash_collation_suffix }}, {% endif %}{% endfor %}
             ) AS VARCHAR(MAX))), 2))
         {% endfor %}
     {% elif column_collection|length > 1 %}
-        CAST(CONCAT(CAST('' AS VARCHAR(MAX)){{ row_hash_collation_suffix }}, {% for column in column_collection %}{{ column.name | tsql_quote_identifier }}{{ row_hash_collation_suffix }}{% if not loop.last %}, '{{ row_hash_column_delimiter }}'{{ row_hash_collation_suffix }}, {% endif %}{% endfor %}) AS VARCHAR(MAX))
+        CAST(CONCAT(CAST('' AS VARCHAR(MAX)){{ row_hash_collation_suffix }}, {% for column in column_collection %}{{ column_sql_names[column.name] }}{{ row_hash_collation_suffix }}{% if not loop.last %}, '{{ row_hash_column_delimiter }}'{{ row_hash_collation_suffix }}, {% endif %}{% endfor %}) AS VARCHAR(MAX))
     {% else %}
-        CAST({{ column_collection[0].name | tsql_quote_identifier }}{{ row_hash_collation_suffix }} AS VARCHAR(MAX))
+        CAST({{ column_sql_names[column_collection[0].name] }}{{ row_hash_collation_suffix }} AS VARCHAR(MAX))
     {% endif %}
     ), 2)) AS ROW_MD5
 FROM
     (SELECT{{ qm.render_select_modifier(query_modifiers, 'src') }}
         {% for index_column in index_column_collection %}
-            {{ datatypes_normalization_renderer_templates[index_column.name] }} AS {{ (index_column.name ~ '_IDX') | tsql_quote_identifier }},
+            {{ datatypes_normalization_renderer_templates[index_column.name] }} AS {{ index_sql_aliases[index_column.name] }},
         {% endfor %}
         {% for column in column_collection %}
             {% if column.nullable %}
-                coalesce({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS {{ column.name | tsql_quote_identifier }}{% if not loop.last %},{% endif %}
+                coalesce({{ datatypes_normalization_renderer_templates[column.name] }}, '{{ null_hashing_sentinel }}') AS {{ column_sql_names[column.name] }}{% if not loop.last %},{% endif %}
             {% else %}
-                {{ datatypes_normalization_renderer_templates[column.name] }} AS {{ column.name | tsql_quote_identifier }}{% if not loop.last %},{% endif %}
+                {{ datatypes_normalization_renderer_templates[column.name] }} AS {{ column_sql_names[column.name] }}{% if not loop.last %},{% endif %}
             {% endif %}
         {% endfor %}
     {{ qm.render_from_clause(query_modifiers, fully_qualified_name, 'src') }}
@@ -34,5 +34,5 @@ FROM
     ) AS RW
 {% if order_by_columns %}
 {# T-SQL does not support the NULLS FIRST keyword; under ASC it already sorts NULLs first, matching NULL_KEY (sentinel sorts first) in the streaming merge-join. #}
-ORDER BY {% for c in order_by_columns %}{{ order_by_collation_expressions.get(c, (c ~ '_IDX') | tsql_quote_identifier) }}{% if not loop.last %}, {% endif %}{% endfor %}
+ORDER BY {% for c in order_by_columns %}{{ order_by_collation_expressions.get(c, index_sql_aliases[c]) }}{% if not loop.last %}, {% endif %}{% endfor %}
 {% endif %}

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/azure_synapse/extractor/templates/azure_synapse_table_metadata_query.sql.j2

@@ -17,6 +17,19 @@ WITH rc_cte AS (
         C.NUMERIC_SCALE AS NUMERIC_SCALE,
         C.COLLATION_NAME AS COLLATION_NAME,
         C.IS_NULLABLE AS IS_NULLABLE,
+        (
+            SELECT CASE TABLE_TYPE
+                WHEN 'BASE TABLE' THEN 'TABLE'
+                WHEN 'VIEW' THEN 'VIEW'
+                ELSE 'UNKNOWN'
+            END
+            FROM {% if object_database_ref %}{{ object_database_ref }}.{% endif %}INFORMATION_SCHEMA.TABLES
+            WHERE TABLE_NAME = '{{ object_name | escape_sql_literal }}'
+              AND TABLE_SCHEMA = '{{ object_schema | escape_sql_literal }}'
+              {% if object_database %}
+              AND TABLE_CATALOG = '{{ object_database | escape_sql_literal }}'
+              {% endif %}
+        ) AS OBJECT_TYPE,
         rc.ROW_COUNT
    FROM {% if object_database_ref %}{{ object_database_ref }}.{% endif %}INFORMATION_SCHEMA.COLUMNS C
 CROSS JOIN rc_cte rc

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/azure_synapse/query/query_generator_azure_synapse.py

@@ -32,6 +32,8 @@ import logging
 
 from typing import TYPE_CHECKING
 
+from shared.identifiers.managers import identifier_manager
+from shared.identifiers.sql_identifier import SqlIdentifier
 from snowflake.snowflake_data_validation.query.float_bit_normalization import (
     build_binary64_canonical_expr,
 )
@@ -50,6 +52,7 @@ from snowflake.snowflake_data_validation.utils.constants import (
 )
 from snowflake.snowflake_data_validation.utils.measure_collation import (
     TSQL_ROW_HASH_COLLATION_SUFFIX,
+    tsql_comparison_collation_tokens,
     tsql_measure_collation_tokens,
 )
 from snowflake.snowflake_data_validation.utils.model.table_context import TableContext
@@ -93,17 +96,20 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
         )
 
     def _comparison_collation_tokens(self, column) -> tuple[str, str]:
-        return tsql_measure_collation_tokens(
-            Platform.AZURE_SYNAPSE,
-            column,
-            "count_distinct",
-        )
+        return tsql_comparison_collation_tokens(Platform.AZURE_SYNAPSE, column)
+
+    def _write_sql_identifier(self, identifier: str) -> str:
+        """Write a column name as Azure Synapse's identifier manager does: bracketed, valid whatever QUOTED_IDENTIFIER is."""
+        return identifier_manager(self.platform).write_sql_name(SqlIdentifier(identifier))
 
     def _float_column_reference(self, column) -> str:
         return self._write_sql_identifier(column.name)
 
-    def _normalization_column_name(self, column) -> str:
-        return str(column.name).replace('"', '""')
+    def _render_normalization_template_identifier(self, template: str, identifier: str) -> str:
+        return self._render_written_identifier_template(template, identifier)
+
+    def _render_metric_template_identifier(self, metric_template: str, identifier: str) -> str:
+        return self._render_written_identifier_template(metric_template, identifier)
 
     def build_float_bit_shim(self, column, col_ref: str) -> str:
         value_expr = float_bit_input_expr(
@@ -235,6 +241,7 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
             ),
             row_hash_collation_suffix=TSQL_ROW_HASH_COLLATION_SUFFIX,
             query_modifiers=resolve_table_context_modifiers(table_context),
+            write_identifier=self._write_sql_identifier,
         )
 
     def generate_cell_validation_data_query(
@@ -266,6 +273,7 @@ class QueryGeneratorAzureSynapse(QueryGeneratorBase):
             order_by_columns=order_by_columns,
             order_by_collation_expressions=self._order_by_collation_expressions(table_context.columns_to_validate),
             query_modifiers=resolve_table_context_modifiers(table_context),
+            write_identifier=self._write_sql_identifier,
         )
 
     def generate_extract_md5_rows_chunk_query(self, chunk_id: str, table_context: TableContext) -> str:

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_datatypes_normalization_templates.yaml

@@ -44,8 +44,10 @@ FLOAT: '{{ float_bit_shim }}'
 # NUMERIC/BIGNUMERIC map to Snowflake NUMBER, which renders scale-aware via
 # TO_CHAR(col, 'FM...0.00..0'). BigQuery mirrors that scale with the resolved
 # printf mask ``{{ numeric_format_mask }}`` (e.g. NUMBER(38,9) -> '%.9f'), so a
-# bounded BIGNUMERIC(p, s) that fits NUMBER hashes identically. (A bare
-# BIGNUMERIC (76, 38) cannot map to NUMBER and fails L1 before L2/L3.)
+# bounded BIGNUMERIC(p, s) that fits NUMBER hashes identically. A bare
+# BIGNUMERIC is created as DECFLOAT (SnowConvert engine parity); precision
+# above 38 or scale above 37 reports as an L1 WARNING rather than a schema
+# failure, and L2/L3 are not expected to reach parity on the extreme form.
 NUMERIC: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)
 DECIMAL: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)
 BIGNUMERIC: FORMAT('{{ numeric_format_mask }}', `{{ col_name }}`)

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_get_columns_metadata.sql.j2

@@ -41,5 +41,5 @@ SELECT
     END AS scale,
     c.collation_name AS collation_name,
     -1 AS calculated_column_size_in_bytes
-FROM `{{ database_name }}`.`{{ schema_name }}`.INFORMATION_SCHEMA.COLUMNS c
+FROM {{ database_sql }}.{{ schema_sql }}.INFORMATION_SCHEMA.COLUMNS c
 WHERE c.table_name = '{{ table_name | escape_sql_literal }}';

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/bigquery/extractor/templates/bigquery_table_metadata_query.sql.j2

@@ -45,8 +45,17 @@ WITH rc_cte AS (
         END AS NUMERIC_SCALE,
         c.is_nullable AS IS_NULLABLE,
         c.collation_name AS COLLATION_NAME,
+        (
+            SELECT CASE table_type
+                WHEN 'BASE TABLE' THEN 'TABLE'
+                WHEN 'VIEW' THEN 'VIEW'
+                ELSE 'UNKNOWN'
+            END
+            FROM {{ object_database_sql }}.{{ object_schema_sql }}.INFORMATION_SCHEMA.TABLES
+            WHERE table_name = '{{ object_name | escape_sql_literal }}'
+        ) AS OBJECT_TYPE,
         rc.ROW_COUNT
-   FROM `{{ object_database }}`.`{{ object_schema }}`.INFORMATION_SCHEMA.COLUMNS c
+   FROM {{ object_database_sql }}.{{ object_schema_sql }}.INFORMATION_SCHEMA.COLUMNS c
 CROSS JOIN rc_cte rc
 WHERE c.table_name = '{{ object_name | escape_sql_literal }}'
 {% if columns %}

{snowflake_data_validation-1.43.0 → snowflake_data_validation-1.44.0}/src/snowflake/snowflake_data_validation/configuration/configuration_loader.py

@@ -6,6 +6,7 @@ from snowflake.snowflake_data_validation.configuration.model.configuration_model
     ConfigurationModel,
 )
 from snowflake.snowflake_data_validation.configuration.singleton import Singleton
+from snowflake.snowflake_data_validation.utils.helpers.helper_io import HelperIO
 
 
 class ConfigurationLoader(metaclass=Singleton):
@@ -38,18 +39,17 @@ class ConfigurationLoader(metaclass=Singleton):
             raise ValueError("The configuration file path cannot be None value")
 
         if not (file_path.suffix == ".yaml" or file_path.suffix == ".yml"):
-            raise Exception(
-                "The configuration file must have a .yaml or .yml extension"
-            )
+            raise Exception("The configuration file must have a .yaml or .yml extension")
 
         if not file_path.exists():
             raise FileNotFoundError(f"Configuration file not found in {file_path}")
 
+        # Outside the try: the encoding error is already actionable and must not
+        # be flattened into the generic "error occurred" wrapper below.
+        file_content = HelperIO.read_text_file(file_path)
+
         try:
-            file_content = file_path.read_text()
-            self.configuration_model = parse_yaml_raw_as(
-                ConfigurationModel, file_content
-            )
+            self.configuration_model = parse_yaml_raw_as(ConfigurationModel, file_content)
 
         except Exception as exception:
             error_msg = "An error occurred while loading the configuration file:"