snowflake-data-validation 1.3.4__tar.gz → 1.3.6__tar.gz

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/.gitignore

@@ -203,5 +203,8 @@ data-exchange-agent/src/mocked_api/*
 requirements.lock
 requirements-dev.lock
 
+# E2E test session config (contains credentials)
+e2e-tests/config.yaml
+
 # Temporary files
 temp/

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowflake-data-validation
-Version: 1.3.4
+Version: 1.3.6
 Summary: Snowflake Data Validation
 Project-URL: Bug Tracker, https://github.com/snowflake-eng/migrations-data-validation/issues
 Project-URL: Source code, https://github.com/snowflake-eng/migrations-data-validation/

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/src/snowflake/snowflake_data_validation/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.3.4"
+__version__ = "1.3.6"

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/src/snowflake/snowflake_data_validation/configuration/model/configuration_model.py

@@ -84,6 +84,8 @@ class ConfigurationModel(BaseModel):
     def check_tables(self) -> None:
         table: TableConfiguration
         for table in self.tables:
+            if self.source_platform.lower() == Platform.TERADATA.value:
+                self._add_database(object=table)
             self._apply_mappings_to_table(table)
             self._check_max_failed_rows_number(table)
             self._check_chunk_number(table)
@@ -93,6 +95,8 @@ class ConfigurationModel(BaseModel):
     def check_views(self) -> None:
         view: ViewConfiguration
         for view in self.views:
+            if self.source_platform.lower() == Platform.TERADATA.value:
+                self._add_database(object=view)
             self._apply_mappings_to_view(view)
             self._check_max_failed_rows_number(view)
             self._check_chunk_number(view)
@@ -173,3 +177,7 @@ class ConfigurationModel(BaseModel):
             table.apply_metric_column_modifier = (
                 self.validation_configuration.apply_metric_column_modifier
             )
+
+    def _add_database(self, object: TableConfiguration) -> None:
+        object.target_database = self.target_database
+        object._load_target_fully_qualified_name()

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/src/snowflake/snowflake_data_validation/configuration/model/table_configuration.py

@@ -119,15 +119,9 @@ class TableConfiguration(BaseModel):
             self.fully_qualified_name
         )
 
-        if len(decomposed_tuple) == 2:
-            self.source_database = None
-            self.source_schema = decomposed_tuple[0]
-            self.source_table = decomposed_tuple[1]
-
-        else:
-            self.source_database = decomposed_tuple[0]
-            self.source_schema = decomposed_tuple[1]
-            self.source_table = decomposed_tuple[2]
+        self.source_database = decomposed_tuple[0]
+        self.source_schema = decomposed_tuple[1]
+        self.source_table = decomposed_tuple[2]
 
     def _load_target_fully_qualified_name(self) -> None:
         self.target_database = HelperDatabase.normalize_to_snowflake_identifier(

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/src/snowflake/snowflake_data_validation/executer/sync_validation_executor.py

@@ -401,11 +401,11 @@ class SyncValidationExecutor(BaseValidationExecutor):
                 "Differences found in MD5 comparison between table %s and table %s:\n%s",
                 source_table_context.fully_qualified_name,
                 target_table_context.fully_qualified_name,
-                comparison_result.differences_dataframe,
+                comparison_result.results_dataframe,
             )
 
         self._validate_md5_rows_chunk(
-            compared_df=comparison_result.differences_dataframe,
+            compared_df=comparison_result.results_dataframe,
             source_table_context=source_table_context,
             target_table_context=target_table_context,
         )
@@ -659,15 +659,15 @@ class SyncValidationExecutor(BaseValidationExecutor):
                 current_chunk_id,
                 source_table_context.fully_qualified_name,
                 target_table_context.fully_qualified_name,
-                rows_comparison_result.differences_dataframe,
+                rows_comparison_result.results_dataframe,
             )
 
             rows_compared_df = pd.concat(
-                [rows_compared_df, rows_comparison_result.differences_dataframe]
+                [rows_compared_df, rows_comparison_result.results_dataframe]
             )
 
             if (
-                len(rows_comparison_result.differences_dataframe)
+                len(rows_comparison_result.results_dataframe)
                 >= source_table_context.max_failed_rows_number
             ):
                 LOGGER.warning(
@@ -683,7 +683,7 @@ class SyncValidationExecutor(BaseValidationExecutor):
                         f"Row validation between "
                         f"{source_table_context.fully_qualified_name} and "
                         f"{target_table_context.fully_qualified_name} was stopped because "
-                        f"the number of failed rows ({len(rows_comparison_result.differences_dataframe)}) "
+                        f"the number of failed rows ({len(rows_comparison_result.results_dataframe)}) "
                         f"reached the configured threshold "
                         f"({source_table_context.max_failed_rows_number})."
                     ),

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/src/snowflake/snowflake_data_validation/public/DATA_VALIDATION_API_OVERVIEW.md

@@ -6,10 +6,11 @@ The `data_validation_api.py` module provides a comprehensive set of functions fo
 
 The API supports multiple platforms via the `Platform` enum:
 
-- **Snowflake** (target)
+- **Snowflake** (source and target)
 - **SQL Server** (source)
 - **Redshift** (source)
 - **Teradata** (source)
+- **Result Set** (in-memory comparison)
 
 ---
 
@@ -83,6 +84,37 @@ Functions to execute generated queries and retrieve results.
 
 The cell validation provides a lightweight alternative to MD5-based row validation. It uses Polars for efficient vectorized comparison and returns detailed information about each differing cell including column name, row index, source value, and target value.
 
+### Step 3.2.1: Result Set Comparison
+
+| Function | Purpose |
+|----------|---------|
+| `result_set_cell_level_validation()` | Compares two `ResultSet` objects and determines if they are equivalent |
+
+This function compares the dataframes of two result sets, checking for equivalence in terms of data values (ignoring order and duplicates).
+
+### Step 3.3: Snapshot Validation
+
+Functions to validate snapshots by comparing saved schema/metrics files from two tables.
+
+| Function | Purpose |
+|----------|---------|
+| `validate_schema_snapshot()` | Validates schema snapshots by loading and comparing metadata from two directories (local or Snowflake stage) |
+| `validate_metrics_snapshot()` | Validates metrics snapshots by loading and comparing metrics data from two directories (local or Snowflake stage) |
+
+Both snapshot validation functions support:
+- Local directories or Snowflake stages as input
+- Column name mappings between source and target
+- Datatype mappings for cross-platform comparisons
+
+---
+
+## Snapshot Generation
+
+| Function | Purpose |
+|----------|---------|
+| `generate_snapshot()` | Generates validation snapshots to a local directory or Snowflake stage |
+
+
 ---
 
 ## Utility Functions
@@ -96,6 +128,50 @@ The cell validation provides a lightweight alternative to MD5-based row validati
 
 ---
 
+## Configuration Functions
+
+| Function | Purpose |
+|----------|---------|
+| `get_configuration_file()` | Exports configuration template files (YAML and optionally Jinja2 query templates) to a specified directory |
+
+### get_configuration_file
+
+Exports the YAML configuration templates and optionally the Jinja2 query templates for a specific platform to a directory. These templates can be customized for validation.
+
+**Parameters:**
+- `templates_directory: str` - Directory path where templates will be written (must exist and be writable)
+- `include_query_templates: bool` - If True, includes Jinja2 (.j2) query template files
+- `platform: Platform` - The source platform (SNOWFLAKE, SQLSERVER, TERADATA, REDSHIFT, RESULT_SET)
+
+**Example:**
+```python
+from snowflake.snowflake_data_validation.public import (
+    get_configuration_file,
+    Platform,
+)
+
+get_configuration_file(
+    templates_directory="/path/to/templates",
+    include_query_templates=True,
+    platform=Platform.SQLSERVER,
+)
+```
+
+---
+
+## Orchestration Functions
+
+High-level functions to run complete validation workflows.
+
+| Function | Purpose | Parameters |
+|----------|---------|------------|
+| `run_validation()` | Run synchronous data validation based on a configuration file | `platform: Platform`, `data_validation_config_file: str` |
+| `run_async_generation()` | Generate validation scripts for later execution | `platform: Platform`, `data_validation_config_file: str` |
+| `run_async_validation()` | Run asynchronous data validation | `platform: Platform`, `data_validation_config_file: str` |
+| `run_source_validation()` | Execute validation queries on source only and save results as Parquet files | `platform: Platform`, `data_validation_config_file: str` |
+
+---
+
 ## Async Mode Support
 
 The API supports **asynchronous execution** where queries are written to files for later execution.
@@ -122,7 +198,7 @@ The API supports **asynchronous execution** where queries are written to files f
 
 - **`GeneratedQuery`** - Wraps a generated SQL query string
 - **`QueryExecutionResult`** - Contains a pandas or Polars DataFrame with query results
-- **`ValidationResult`** - Contains `is_valid` flag and `differences_dataframe`
+- **`ValidationResult`** - Contains `is_valid` flag and `results_dataframe`
 - **`TableContext`** - Table metadata including platform, FQN, and configuration
 - **`TableConfiguration`** - Configuration for table validation (mappings, selections)
 - **`TableColumnMetadata`** - Structured column metadata information
@@ -145,9 +221,9 @@ The API supports **asynchronous execution** where queries are written to files f
 
 | Enum | Values |
 |------|--------|
-| `Platform` | SNOWFLAKE, SQLSERVER, REDSHIFT, TERADATA |
+| `Platform` | SNOWFLAKE, SQLSERVER, REDSHIFT, TERADATA, RESULT_SET |
 | `Origin` | SOURCE, TARGET |
-| `ObjectType` | TABLE, VIEW |
+| `ObjectType` | TABLE, VIEW, RESULT_SET |
 | `ExecutionMode` | SYNC, ASYNC |
 
 ---

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/src/snowflake/snowflake_data_validation/public/__init__.py

@@ -42,9 +42,6 @@ from snowflake.snowflake_data_validation.public.data_validation_api import (
     build_validation_file_path,
     compare_md5_chunks,
     compare_md5_rows_chunk,
-    create_source_table_context_from_configuration,
-    create_table_context,
-    create_target_table_context_from_configuration,
     execute_query,
     execute_query_from_table_configuration,
     execute_statement,
@@ -59,14 +56,24 @@ from snowflake.snowflake_data_validation.public.data_validation_api import (
     generate_metrics_query,
     generate_row_count_query,
     generate_schema_query,
+    generate_snapshot,
     generate_view_materialization_query,
+    get_configuration_file,
     load_validation_result_from_file,
     load_validation_results,
     process_table_column_metadata,
     process_target_table_column_metadata_from_source_table,
+    result_set_cell_level_validation,
+    result_set_metrics_level_validation,
+    run_async_generation,
+    run_async_validation,
+    run_source_validation,
+    run_validation,
     validate_cell,
     validate_metrics,
+    validate_metrics_snapshot,
     validate_schema,
+    validate_schema_snapshot,
     write_metrics_query_to_script,
     write_schema_query_to_script,
     write_view_materialization_query_to_script,
@@ -80,6 +87,7 @@ from snowflake.snowflake_data_validation.public.model import (
     GeneratedQuery,
     QueryExecutionResult,
     ValidationResult,
+    ResultSet,
 )
 
 # =============================================================================
@@ -90,6 +98,7 @@ from snowflake.snowflake_data_validation.public.enums import (
     ObjectType,
     Origin,
     Platform,
+    ValidationLevel,
 )
 
 # =============================================================================
@@ -158,9 +167,6 @@ __all__ = [
     "build_validation_file_path",
     "compare_md5_chunks",
     "compare_md5_rows_chunk",
-    "create_source_table_context_from_configuration",
-    "create_table_context",
-    "create_target_table_context_from_configuration",
     "execute_query",
     "execute_query_from_table_configuration",
     "execute_statement",
@@ -175,14 +181,24 @@ __all__ = [
     "generate_metrics_query",
     "generate_row_count_query",
     "generate_schema_query",
+    "generate_snapshot",
     "generate_view_materialization_query",
+    "get_configuration_file",
     "load_validation_result_from_file",
     "load_validation_results",
     "process_table_column_metadata",
     "process_target_table_column_metadata_from_source_table",
+    "result_set_cell_level_validation",
+    "result_set_metrics_level_validation",
+    "run_async_generation",
+    "run_async_validation",
+    "run_source_validation",
+    "run_validation",
     "validate_cell",
     "validate_metrics",
+    "validate_metrics_snapshot",
     "validate_schema",
+    "validate_schema_snapshot",
     "write_metrics_query_to_script",
     "write_schema_query_to_script",
     "write_view_materialization_query_to_script",
@@ -202,6 +218,7 @@ __all__ = [
     "TableContext",
     "ColumnMetadata",
     "TableColumnMetadata",
+    "ResultSet",
     "ConnectorBase",
     "ConnectorRedshift",
     "ConnectorSnowflake",
@@ -215,4 +232,5 @@ __all__ = [
     "TeradataConnectorFactory",
     "SQLQueriesTemplateGenerator",
     "TemplatesLoaderManager",
+    "ValidationLevel",
 ]

{snowflake_data_validation-1.3.4 → snowflake_data_validation-1.3.6}/src/snowflake/snowflake_data_validation/public/_internal.py

@@ -27,6 +27,7 @@ from __future__ import annotations
 import os
 
 from collections.abc import Callable
+from pathlib import Path
 
 import pandas as pd
 
@@ -39,7 +40,23 @@ from snowflake.snowflake_data_validation.public.model import (
     GeneratedQuery,
     QueryExecutionResult,
 )
-from snowflake.snowflake_data_validation.utils.constants import Platform
+from snowflake.snowflake_data_validation.snowflake.connector.connector_snowflake import (
+    ConnectorSnowflake,
+)
+from snowflake.snowflake_data_validation.utils.constants import (
+    Platform,
+    ValidationLevel,
+)
+from snowflake.snowflake_data_validation.utils.helpers.helper_database import (
+    HelperDatabase,
+)
+from snowflake.snowflake_data_validation.utils.helpers.helper_dataframe import (
+    HelperDataFrame,
+)
+from snowflake.snowflake_data_validation.utils.snapshot_validation_statements import (
+    CREATE_SNOWFLAKE_FILE_FORMAT_STATEMENT,
+    SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT,
+)
 
 
 def execute_query_internal(
@@ -226,3 +243,98 @@ def get_target_cols(
             n_col = case_sensitive_dict.get(col.upper())
         res.append(n_col)
     return res
+
+
+def is_snowflake_stage(path: str) -> bool:
+    """Check if the path is a Snowflake stage reference."""
+    return path.strip().startswith("@")
+
+
+def upload_to_stage(connector: ConnectorBase, local_file: str, stage: str) -> str:
+    """Upload a local file to a Snowflake stage using PUT command.
+
+    Args:
+        connector: The Snowflake connector instance
+        local_file: Path to the local file to upload
+        stage: The Snowflake stage path (e.g., @my_stage/output/)
+
+    """
+    put_command = (
+        f"PUT 'file://{local_file}' '{stage}' AUTO_COMPRESS=FALSE OVERWRITE=TRUE"
+    )
+    connector.execute_statement(put_command)
+    file_output = f"{stage}{os.path.basename(local_file)}"
+    return file_output
+
+
+def load_snapshot_dataframe(
+    directory: str,
+    file_name: str,
+    validation_level: ValidationLevel,
+    snowflake_stage_connector: ConnectorSnowflake,
+) -> pd.DataFrame:
+    is_stage = is_snowflake_stage(directory)
+    if is_stage:
+        dataframe = load_snapshot_dataframe_from_stage(
+            stage=directory,
+            file_name=file_name,
+            validation_level=validation_level,
+            snowflake_stage_connector=snowflake_stage_connector,
+        )
+        return dataframe
+    else:
+        dataframe = load_snapshot_dataframe_from_local_file(
+            directory=directory,
+            file_name=file_name,
+        )
+        return dataframe
+
+
+def load_snapshot_dataframe_from_local_file(
+    directory: str,
+    file_name: str,
+) -> pd.DataFrame:
+    directory_path = Path(directory)
+    if not directory_path.is_dir():
+        raise ValueError(f"Directory does not exist: {directory}")
+
+    file_path = directory_path / file_name
+
+    if not file_path.is_file():
+        raise ValueError(f"Snapshot file does not exist: {file_path}")
+
+    dataframe = pd.read_parquet(file_path)
+    return dataframe
+
+
+def load_snapshot_dataframe_from_stage(
+    stage: str,
+    file_name: str,
+    validation_level: ValidationLevel,
+    snowflake_stage_connector: ConnectorSnowflake,
+) -> pd.DataFrame:
+    normalized_stage = HelperDatabase.normalize_snowflake_stage(stage)
+    stage_file = normalized_stage + file_name
+
+    if not file_exists_in_stage(stage_file, snowflake_stage_connector):
+        raise ValueError(f"Snapshot file does not exist in stage: {stage_file}")
+
+    select_parquet_data_format = SNAPSHOT_PARQUET_DATA_STATEMENT_FORMAT_SELECT[
+        validation_level
+    ]
+    select_parquet_data_statement = select_parquet_data_format.format(
+        stage_file=stage_file
+    )
+    snowflake_stage_connector.execute_statement(CREATE_SNOWFLAKE_FILE_FORMAT_STATEMENT)
+    result = snowflake_stage_connector.execute_query(select_parquet_data_statement)
+    dataframe = HelperDataFrame().process_snowflake_query_result_to_dataframe(result)
+    return dataframe
+
+
+def file_exists_in_stage(
+    stage_file: str, snowflake_stage_connector: ConnectorSnowflake
+) -> bool:
+    list_statement = f"LIST {stage_file}"
+    result = snowflake_stage_connector.execute_query(list_statement)
+    dataframe = HelperDataFrame().process_snowflake_query_result_to_dataframe(result)
+    return not dataframe.empty