snowflake-data-validation 1.3.7__tar.gz → 1.4.0__tar.gz

{snowflake_data_validation-1.3.7 → snowflake_data_validation-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowflake-data-validation
-Version: 1.3.7
+Version: 1.4.0
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/

{snowflake_data_validation-1.3.7 → snowflake_data_validation-1.4.0}/src/snowflake/snowflake_data_validation/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.3.7"
+__version__ = "1.4.0"

{snowflake_data_validation-1.3.7 → snowflake_data_validation-1.4.0}/src/snowflake/snowflake_data_validation/public/DATA_VALIDATION_API_OVERVIEW.md

@@ -110,14 +110,30 @@ Functions to validate snapshots by comparing saved schema/metrics files from two
 
 | Function | Purpose |
 |----------|---------|
-| `validate_schema_snapshot()` | Validates schema snapshots by loading and comparing metadata from two directories (local or Snowflake stage) |
-| `validate_metrics_snapshot()` | Validates metrics snapshots by loading and comparing metrics data from two directories (local or Snowflake stage) |
+| `validate_schema_snapshot_table()` | Validates schema snapshots by loading and comparing metadata from two directories (local or Snowflake stage) |
+| `validate_metrics_snapshot_table()` | Validates metrics snapshots by loading and comparing metrics data from two directories (local or Snowflake stage) |
 
 Both snapshot validation functions support:
 - Local directories or Snowflake stages as input
 - Column name mappings between source and target
 - Datatype mappings for cross-platform comparisons
 
+### Step 3.3.1: Result Set Snapshot Validation
+
+Functions to validate snapshots specifically for result set comparisons.
+
+| Function | Purpose |
+|----------|---------|
+| `validate_schema_snapshot_result_set()` | Validates schema snapshots for result sets by comparing metadata from two directories |
+| `validate_metrics_snapshot_result_set()` | Validates metrics snapshots for result sets with configurable tolerance |
+| `validate_cell_snapshot_result_set()` | Validates cell snapshots for result sets by performing cell-level comparison |
+
+These result set snapshot validation functions support:
+- Local directories or Snowflake stages as input
+- Column name mappings between source and target (schema and metrics)
+- Datatype mappings for schema comparisons
+- Configurable tolerance for metrics comparisons
+
 ---
 
 ## Snapshot Generation

{snowflake_data_validation-1.3.7 → snowflake_data_validation-1.4.0}/src/snowflake/snowflake_data_validation/public/__init__.py

@@ -75,10 +75,13 @@ from snowflake.snowflake_data_validation.public.data_validation_api import (
     run_source_validation,
     run_validation,
     validate_cell,
+    validate_cell_snapshot_result_set,
     validate_metrics,
-    validate_metrics_snapshot,
+    validate_metrics_snapshot_result_set,
+    validate_metrics_snapshot_table,
     validate_schema,
-    validate_schema_snapshot,
+    validate_schema_snapshot_table,
+    validate_schema_snapshot_result_set,
     write_metrics_query_to_script,
     write_schema_query_to_script,
     write_view_materialization_query_to_script,
@@ -205,10 +208,13 @@ __all__ = [
     "run_source_validation",
     "run_validation",
     "validate_cell",
+    "validate_cell_snapshot_result_set",
     "validate_metrics",
-    "validate_metrics_snapshot",
+    "validate_metrics_snapshot_result_set",
+    "validate_metrics_snapshot_table",
     "validate_schema",
-    "validate_schema_snapshot",
+    "validate_schema_snapshot_table",
+    "validate_schema_snapshot_result_set",
     "write_metrics_query_to_script",
     "write_schema_query_to_script",
     "write_view_materialization_query_to_script",

{snowflake_data_validation-1.3.7 → snowflake_data_validation-1.4.0}/src/snowflake/snowflake_data_validation/public/_internal.py

@@ -30,6 +30,7 @@ from collections.abc import Callable
 from pathlib import Path
 
 import pandas as pd
+import polars as pl
 
 from snowflake.snowflake_data_validation.configuration.model.table_configuration import (
     TableConfiguration,
@@ -55,7 +56,8 @@ from snowflake.snowflake_data_validation.utils.helpers.helper_dataframe import (
 )
 from snowflake.snowflake_data_validation.utils.snapshot_validation_statements import (
     CREATE_SNOWFLAKE_FILE_FORMAT_STATEMENT,
-    SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT,
+    SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT_RESULT_SET,
+    SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT_TABLE,
 )
 
 
@@ -272,6 +274,8 @@ def load_snapshot_dataframe(
     file_name: str,
     validation_level: ValidationLevel,
     snowflake_stage_connector: ConnectorSnowflake,
+    load_as_polars_df: bool = False,
+    is_result_set: bool = False,
 ) -> pd.DataFrame:
     is_stage = is_snowflake_stage(directory)
     if is_stage:
@@ -280,12 +284,15 @@ def load_snapshot_dataframe(
             file_name=file_name,
             validation_level=validation_level,
             snowflake_stage_connector=snowflake_stage_connector,
+            load_as_polars_df=load_as_polars_df,
+            is_result_set=is_result_set,
         )
         return dataframe
     else:
         dataframe = load_snapshot_dataframe_from_local_file(
             directory=directory,
             file_name=file_name,
+            load_as_polars_df=load_as_polars_df,
         )
         return dataframe
 
@@ -293,7 +300,8 @@ def load_snapshot_dataframe(
 def load_snapshot_dataframe_from_local_file(
     directory: str,
     file_name: str,
-) -> pd.DataFrame:
+    load_as_polars_df: bool = False,
+) -> pd.DataFrame | pl.DataFrame:
     directory_path = Path(directory)
     if not directory_path.is_dir():
         raise ValueError(f"Directory does not exist: {directory}")
@@ -303,7 +311,9 @@ def load_snapshot_dataframe_from_local_file(
     if not file_path.is_file():
         raise ValueError(f"Snapshot file does not exist: {file_path}")
 
-    dataframe = pd.read_parquet(file_path)
+    dataframe = (
+        pl.read_parquet(file_path) if load_as_polars_df else pd.read_parquet(file_path)
+    )
     return dataframe
 
 
@@ -312,22 +322,27 @@ def load_snapshot_dataframe_from_stage(
     file_name: str,
     validation_level: ValidationLevel,
     snowflake_stage_connector: ConnectorSnowflake,
-) -> pd.DataFrame:
+    load_as_polars_df: bool = False,
+    is_result_set: bool = False,
+) -> pd.DataFrame | pl.DataFrame:
     normalized_stage = HelperDatabase.normalize_snowflake_stage(stage)
     stage_file = normalized_stage + file_name
 
     if not file_exists_in_stage(stage_file, snowflake_stage_connector):
         raise ValueError(f"Snapshot file does not exist in stage: {stage_file}")
 
-    select_parquet_data_format = SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT[
-        validation_level
-    ]
+    select_parquet_data_format = (
+        SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT_TABLE[validation_level]
+        if not is_result_set
+        else SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT_RESULT_SET[validation_level]
+    )
+
     select_parquet_data_statement = select_parquet_data_format.format(
         stage_file=stage_file
     )
     snowflake_stage_connector.execute_statement(CREATE_SNOWFLAKE_FILE_FORMAT_STATEMENT)
     result = snowflake_stage_connector.execute_query(select_parquet_data_statement)
-    dataframe = HelperDataFrame().process_snowflake_query_result_to_dataframe(result)
+    dataframe = pl.DataFrame(result) if load_as_polars_df else pd.DataFrame(result)
     return dataframe