snowflake-data-validation 1.0.1__py3-none-any.whl

snowflake/snowflake_data_validation/__init__.py

@@ -0,0 +1,112 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Snowflake Data Validation
+=========================
+
+This package provides comprehensive data validation functionality for Snowflake,
+enabling robust data quality checks and migration validation between different
+database systems.
+
+Features
+--------
+- Multi-level data validation (Level 1: table metadata, Level 2: column metadata)
+- Support for multiple source systems (SQL Server, Snowflake)
+- CLI interface for validation operations
+- Configurable validation processes
+- Progress reporting and detailed validation reports
+
+Main Components
+---------------
+ComparisonOrchestrator
+    Main class for orchestrating data comparisons between source and target systems
+
+CLI Interface
+    Command-line interface available via `main_cli` module for:
+    - Setting up database connections
+    - Running validation operations
+    - Managing configuration files
+
+Supported Dialects
+------------------
+- Snowflake to Snowflake validation
+- SQL Server to Snowflake migration validation
+
+Example Usage
+-------------
+Programmatic API:
+>>> from snowflake.snowflake_data_validation import ComparisonOrchestrator
+>>> # Set up extractors, context, and configuration
+>>> orchestrator = ComparisonOrchestrator(
+...     source_extractor=source_extractor,
+...     target_extractor=target_extractor,
+...     context=context
+... )
+>>> orchestrator.run_sync_comparison()
+
+Command Line Interface:
+
+.. code-block:: bash
+
+    # Set up SQL Server connection
+    python -m snowflake.snowflake_data_validation sqlserver source-connection \
+        --host localhost --port 1433 --username user --password pass --database mydb
+
+    # Run validation
+    python -m snowflake.snowflake_data_validation sqlserver run-validation \
+        --data-validation-config-file config.json
+
+For more detailed examples and usage, please refer to the documentation.
+"""
+
+from snowflake.snowflake_data_validation.comparison_orchestrator import (
+    ComparisonOrchestrator,
+)
+from snowflake.snowflake_data_validation.__version__ import __version__
+
+# Import submodules and make them available at the package level
+# This helps Sphinx properly document the submodules
+import snowflake.snowflake_data_validation.validation as validation
+import snowflake.snowflake_data_validation.extractor as extractor
+import snowflake.snowflake_data_validation.connector as connector
+import snowflake.snowflake_data_validation.utils as utils
+import snowflake.snowflake_data_validation.snowflake as snowflake
+import snowflake.snowflake_data_validation.sqlserver as sqlserver
+import snowflake.snowflake_data_validation.redshift as redshift
+import logging
+
+
+# Add a NullHandler to prevent logging messages from being output to
+# sys.stderr if no logging configuration is provided.
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+# Define the public API
+__all__ = [
+    "__version__",
+    "ComparisonOrchestrator",
+    # Submodules
+    "validation",
+    "extractor",
+    "connector",
+    "utils",
+    "snowflake",
+    "sqlserver",
+    "redshift"
+]
+
+
+# Version information
+__version_info__ = tuple(int(i) for i in __version__.split(".") if i.isdigit())

snowflake/snowflake_data_validation/__main__.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Main entry point for the Snowflake Data Validation CLI when run as a module.
+
+This allows the package to be run with: python -m snowflake.snowflake_data_validation
+"""
+
+import sys
+
+import typer
+
+from snowflake.snowflake_data_validation.main_cli import data_validation_app
+
+
+def main():
+    """Provide main entry point with error handling."""
+    try:
+        data_validation_app()
+    except KeyboardInterrupt:
+        typer.secho("\nOperation cancelled by user", fg=typer.colors.YELLOW, err=True)
+        sys.exit(1)
+    except ImportError as e:
+        typer.secho(f"Import error: {e}", fg=typer.colors.RED, err=True)
+        typer.secho(
+            "Please ensure all dependencies are installed correctly.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        sys.exit(1)
+    except Exception as e:
+        typer.secho(f"Unexpected error: {e}", fg=typer.colors.RED, err=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

snowflake/snowflake_data_validation/__version__.py

@@ -0,0 +1,16 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "1.0.1"

snowflake/snowflake_data_validation/comparison_orchestrator.py

@@ -0,0 +1,250 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from typing import Optional, Union
+
+from snowflake.snowflake_data_validation.configuration.model.table_configuration import (
+    TableConfiguration,
+)
+from snowflake.snowflake_data_validation.configuration.model.validation_configuration import (
+    ValidationConfiguration,
+)
+from snowflake.snowflake_data_validation.executer import (
+    ExecutorFactory,
+)
+from snowflake.snowflake_data_validation.orchestration.parallel_execution_engine import (
+    ParallelExecutionEngine,
+)
+from snowflake.snowflake_data_validation.orchestration.table_metadata_processor import (
+    TableMetadataProcessor,
+)
+from snowflake.snowflake_data_validation.orchestration.validation_progress_reporter import (
+    ValidationProgressReporter,
+)
+from snowflake.snowflake_data_validation.utils.arguments_manager_base import (
+    ValidationEnvironmentObject,
+)
+from snowflake.snowflake_data_validation.utils.connection_pool import (
+    ConnectionPoolManager,
+)
+from snowflake.snowflake_data_validation.utils.constants import (
+    TEMPORARY_DEFAULT_OPTIMAL_LIMIT,
+    ExecutionMode,
+)
+from snowflake.snowflake_data_validation.utils.context import Context
+from snowflake.snowflake_data_validation.utils.logging_utils import log
+from snowflake.snowflake_data_validation.utils.telemetry import (
+    report_telemetry,
+)
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ComparisonOrchestrator:
+
+    """Orchestrator for validation operations that creates appropriate components based on command type."""
+
+    @log
+    def __init__(
+        self,
+        connection_pool_manager: ConnectionPoolManager,
+        context: Context,
+        max_threads: int,
+    ):
+        """Initialize the orchestrator.
+
+        Args:
+            connection_pool_manager: Connection pool manager for threaded processing
+            context: Validation context containing configuration and runtime info
+            max_threads: Maximum number of threads for parallel table processing
+
+        """
+        LOGGER.debug("Initializing ComparisonOrchestrator")
+
+        self.connection_pool_manager = connection_pool_manager
+        self.context = context
+        self.executor_factory = ExecutorFactory()
+        self.max_threads = max_threads
+
+        # Initialize orchestration modules
+        self.metadata_processor = TableMetadataProcessor()
+        self.execution_engine = ParallelExecutionEngine(max_threads=self.max_threads)
+        self.progress_reporter = ValidationProgressReporter()
+
+        LOGGER.debug(
+            "ComparisonOrchestrator initialized with max_threads=%d", self.max_threads
+        )
+
+    @classmethod
+    @log(log_args=False)
+    def from_validation_environment(
+        cls,
+        validation_env: ValidationEnvironmentObject,
+    ) -> "ComparisonOrchestrator":
+        """Create a ComparisonOrchestrator from a ValidationEnvironmentObject.
+
+        Args:
+            validation_env: ValidationEnvironmentObject instance containing all required components
+
+        Returns:
+            ComparisonOrchestrator: Configured orchestrator ready to run validation
+
+        """
+        LOGGER.debug("Creating ComparisonOrchestrator from validation environment")
+
+        config = validation_env.context.configuration
+        num_tables: int = len(config.tables)
+        max_threads_config: Union[str, int] = config.max_threads
+
+        # Hardcoded optimal pool size for now, can be adjusted later on based on changes in SnowConvert AI
+        optimal_max_threads: int = TEMPORARY_DEFAULT_OPTIMAL_LIMIT
+        # CpuOptimizer.get_optimal_thread_count(num_tables=num_tables, max_threads=max_threads_config)
+        optimal_pool_size: int = TEMPORARY_DEFAULT_OPTIMAL_LIMIT
+        # optimal_max_threads
+
+        LOGGER.debug(
+            "Using CPUOptimizer: num_tables=%d, config_max_threads=%s, optimal_max_threads=%d, optimal_pool_size=%d",
+            num_tables,
+            max_threads_config,
+            optimal_max_threads,
+            optimal_pool_size,
+        )
+
+        connection_pool_manager: ConnectionPoolManager = (
+            validation_env.create_connection_pool_manager(pool_size=optimal_pool_size)
+        )
+
+        return cls(
+            connection_pool_manager=connection_pool_manager,
+            context=validation_env.context,
+            max_threads=optimal_max_threads,
+        )
+
+    @log
+    @report_telemetry()
+    def run_sync_comparison(self) -> None:
+        """Run the complete synchronous validation comparison process using parallel execution.
+
+        Uses the sync validation executor with metadata extractors for real-time validation.
+        Each table is processed in a separate thread.
+        """
+        LOGGER.info("Starting synchronous validation comparison")
+        self._execute_parallel_validation(ExecutionMode.SYNC_VALIDATION)
+        self.progress_reporter.flush_validation_reports(self.context)
+        LOGGER.info("Synchronous validation comparison completed")
+
+    @log
+    @report_telemetry()
+    def run_async_generation(self) -> None:
+        """Generate validation scripts for all tables in parallel.
+
+        Uses script printers to write SQL queries to files.
+        Each table is processed in a separate thread for maximum throughput.
+        """
+        LOGGER.info("Starting async script generation")
+        self._execute_parallel_validation(
+            ExecutionMode.ASYNC_GENERATION, use_script_printer=True
+        )
+        self.progress_reporter.flush_validation_reports(self.context)
+        LOGGER.info("Async script generation completed")
+
+    @log
+    @report_telemetry()
+    def run_async_comparison(self) -> None:
+        """Run the asynchronous validation comparison process using parallel execution.
+
+        Uses the async validation executor with metadata extractors for deferred validation.
+        Each table is processed in a separate thread for maximum throughput.
+        """
+        LOGGER.info("Starting asynchronous validation comparison")
+        self._execute_parallel_validation(ExecutionMode.ASYNC_VALIDATION)
+        self.progress_reporter.flush_validation_reports(self.context)
+        LOGGER.info("Asynchronous validation comparison completed")
+
+    @log(log_args=False)
+    def _get_validation_configuration(
+        self,
+        table: TableConfiguration,
+        default_configuration: Optional[ValidationConfiguration],
+    ) -> ValidationConfiguration:
+        """Get the validation configuration for a table.
+
+        Args:
+            table: Table configuration object
+            default_configuration: Default validation configuration
+
+        Returns:
+            ValidationConfiguration: The configuration to use for this table
+
+        """
+        if table.validation_configuration:
+            LOGGER.debug(
+                "Using table-specific validation configuration for %s",
+                table.fully_qualified_name,
+            )
+            return table.validation_configuration
+        elif default_configuration:
+            LOGGER.debug(
+                "Using default validation configuration for %s",
+                table.fully_qualified_name,
+            )
+            return default_configuration
+        else:
+            LOGGER.debug(
+                "Using empty validation configuration for %s",
+                table.fully_qualified_name,
+            )
+            return ValidationConfiguration()
+
+    @log
+    def _execute_parallel_validation(
+        self, execution_mode: ExecutionMode, use_script_printer: bool = False
+    ) -> None:
+        """Execute validation using parallel threads with connection pool.
+
+        Args:
+            execution_mode: The execution mode to use
+            use_script_printer: Whether to use script printers instead of metadata extractors
+
+        """
+        self.connection_pool_manager.initialize_pool()
+
+        LOGGER.debug(
+            "Executing parallel validation with execution_mode=%s, use_script_printer=%s",
+            execution_mode,
+            use_script_printer,
+        )
+
+        self.execution_engine.execute_parallel_validation(
+            tables=self.context.configuration.tables,
+            default_configuration=self.context.configuration.validation_configuration,
+            execution_mode=execution_mode,
+            context=self.context,
+            connection_pool=self.connection_pool_manager,
+            use_script_printer=use_script_printer,
+            progress_callback=lambda table_name, columns: self.progress_reporter.report_progress_for_table(
+                table_name=table_name,
+                column_selection_list=columns,
+                context=self.context,
+            ),
+            validation_config_callback=lambda table, default_config: (
+                self._get_validation_configuration(
+                    table=table, default_configuration=default_config
+                )
+            ),
+        )

snowflake/snowflake_data_validation/configuration/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is intentionally left blank.

snowflake/snowflake_data_validation/configuration/configuration_loader.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+
+from pydantic_yaml import parse_yaml_raw_as
+
+from snowflake.snowflake_data_validation.configuration.model.configuration_model import (
+    ConfigurationModel,
+)
+from snowflake.snowflake_data_validation.configuration.singleton import Singleton
+
+
+class ConfigurationLoader(metaclass=Singleton):
+    """
+    ConfigurationLoader class.
+
+    This is a singleton class that reads the configuration.yaml file
+    and provides an interface to get the configuration settings model.
+
+    Args:
+        metaclass (Singleton, optional): Defaults to Singleton.
+
+    """
+
+    def __init__(self, file_path: Path) -> None:
+        """
+        Initialize the ConfigurationLoader with a configuration file path.
+
+        Args:
+            file_path (Path): The path to the configuration file to load.
+
+        """
+        self.configuration_model: ConfigurationModel = ConfigurationModel(
+            source_platform="",
+            target_platform="",
+            output_directory_path="",
+        )
+
+        if file_path is None:
+            raise ValueError("The configuration file path cannot be None value")
+
+        if not (file_path.suffix == ".yaml" or file_path.suffix == ".yml"):
+            raise Exception(
+                "The configuration file must have a .yaml or .yml extension"
+            )
+
+        if not file_path.exists():
+            raise FileNotFoundError(f"Configuration file not found in {file_path}")
+
+        try:
+            file_content = file_path.read_text()
+            self.configuration_model = parse_yaml_raw_as(
+                ConfigurationModel, file_content
+            )
+
+        except Exception as exception:
+            error_msg = "An error occurred while loading the configuration file:"
+            raise Exception(f"{error_msg}\n{exception}") from None
+
+    def get_configuration_model(self) -> ConfigurationModel:
+        """
+        Get the configuration model.
+
+        Returns:
+            ConfigurationModel: The configuration model instance.
+
+        """
+        return self.configuration_model

snowflake/snowflake_data_validation/configuration/model/configuration_model.py

@@ -0,0 +1,122 @@
+from pydantic import BaseModel, field_validator, model_validator
+from typing_extensions import Self
+
+from snowflake.snowflake_data_validation.configuration.model.connection_types import (
+    Connection,
+)
+from snowflake.snowflake_data_validation.configuration.model.logging_configuration import (
+    LoggingConfiguration,
+)
+from snowflake.snowflake_data_validation.configuration.model.table_configuration import (
+    TableConfiguration,
+)
+from snowflake.snowflake_data_validation.configuration.model.validation_configuration import (
+    ValidationConfiguration,
+)
+from snowflake.snowflake_data_validation.utils.constants import (
+    DEFAULT_THREAD_COUNT_OPTION,
+    VALIDATION_CONFIGURATION_DEFAULT_VALUE,
+)
+
+
+class ConfigurationModel(BaseModel):
+    """Configuration model.
+
+    Args:
+        pydantic.BaseModel (pydantic.BaseModel): pydantic BaseModel
+
+    """
+
+    source_platform: str
+    target_platform: str
+    output_directory_path: str
+    max_threads: str | int = (
+        "auto"  # "auto" for auto-detection or number for specific thread count
+    )
+    source_connection: Connection | None = None
+    target_connection: Connection | None = None
+    source_validation_files_path: str | None = None
+    target_validation_files_path: str | None = None
+    target_database: str | None = None
+    validation_configuration: ValidationConfiguration = ValidationConfiguration(
+        **VALIDATION_CONFIGURATION_DEFAULT_VALUE
+    )
+    comparison_configuration: dict[str, str | float | int] | None = None
+    database_mappings: dict[str, str] = {}
+    schema_mappings: dict[str, str] = {}
+    tables: list[TableConfiguration] = []
+    logging_configuration: LoggingConfiguration | None = None
+
+    @field_validator("max_threads")
+    @classmethod
+    def validate_max_threads(cls, value: str | int) -> str | int:
+        """Validate max_threads field accepts only 'auto' or positive integers."""
+        if isinstance(value, str):
+            if value.lower() != DEFAULT_THREAD_COUNT_OPTION:
+                raise ValueError(
+                    f"String value for max_threads must be '{DEFAULT_THREAD_COUNT_OPTION}'"
+                )
+            return value.lower()
+        elif isinstance(value, int):
+            if value < 1:
+                raise ValueError(
+                    "Numeric value for max_threads must be a positive integer"
+                )
+            return value
+        else:
+            raise ValueError(
+                f"max_threads must be either '{DEFAULT_THREAD_COUNT_OPTION}' or a positive integer"
+            )
+
+    @model_validator(mode="after")
+    def load(self) -> Self:
+        self.check_tables()
+        return self
+
+    def check_tables(self) -> None:
+        table: TableConfiguration
+        for table in self.tables:
+            self._load_target_fully_qualified_name(table)
+            self._check_max_failed_rows_number(table)
+            self._check_chunk_number(table)
+            self.set_exclude_metrics(table)
+            self.set_apply_metric_column_modifier(table)
+
+    def _load_target_fully_qualified_name(self, table: TableConfiguration) -> None:
+        # If target_database is set in config, use it for Teradata sources
+        if self.target_database is not None and table.source_database is None:
+            table.target_database = self.target_database
+        # Otherwise use database mappings if available
+        elif (
+            table.source_database is not None
+            and self.database_mappings.get(table.source_database) is not None
+        ):
+            table.target_database = self.database_mappings[table.source_database]
+
+        if (
+            table.source_schema is not None
+            and self.schema_mappings.get(table.source_schema) is not None
+        ):
+            table.target_schema = self.schema_mappings[table.source_schema]
+
+        table._load_target_fully_qualified_name()
+
+    def _check_max_failed_rows_number(self, table: TableConfiguration) -> None:
+        if table.max_failed_rows_number is None:
+            table.max_failed_rows_number = (
+                self.validation_configuration.max_failed_rows_number
+            )
+
+    def _check_chunk_number(self, table: TableConfiguration) -> None:
+        if table.chunk_number is None:
+            table.chunk_number = 0
+
+    def set_exclude_metrics(self, table: TableConfiguration) -> None:
+        if table.exclude_metrics is None:
+            table.exclude_metrics = self.validation_configuration.exclude_metrics
+
+    def set_apply_metric_column_modifier(self, table: TableConfiguration) -> None:
+        if table.apply_metric_column_modifier is None:
+            table.apply_metric_column_modifier = (
+                self.validation_configuration.apply_metric_column_modifier
+            )

snowflake/snowflake_data_validation/configuration/model/connection_types.py

@@ -0,0 +1,46 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Connection type definitions for YAML configuration.
+
+This file imports connection models from the connections module and creates
+the union types used by the main configuration model for both source and
+target connections.
+"""
+
+from typing import Union
+
+# Import connection models from the dedicated connections module
+from snowflake.snowflake_data_validation.configuration.model.connections import (
+    RedshiftCredentialsConnection,
+    SnowflakeCredentialsConnection,
+    SnowflakeDefaultConnection,
+    SnowflakeNamedConnection,
+    SqlServerCredentialsConnection,
+    TeradataCredentialsConnection,
+)
+
+
+# Union type for configuration (YAML and IPC)
+# Note: This is used for both source_connection and target_connection fields
+Connection = Union[
+    # Snowflake connections (all modes)
+    SnowflakeNamedConnection,
+    SnowflakeDefaultConnection,
+    SnowflakeCredentialsConnection,
+    # SQL Server connections
+    SqlServerCredentialsConnection,
+    # Teradata connections
+    TeradataCredentialsConnection,
+    # Redshift connections
+    RedshiftCredentialsConnection,
+]