snowflake-data-validation 1.0.1__py3-none-any.whl

snowflake/snowflake_data_validation/configuration/model/connections/__init__.py

@@ -0,0 +1,47 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Connection models for different database dialects."""
+
+# Import SQL Server connection models
+from snowflake.snowflake_data_validation.sqlserver.model.sqlserver_credentials_connection import (
+    SqlServerCredentialsConnection,
+)
+
+from snowflake.snowflake_data_validation.redshift.model.redshift_credentials_connection import (
+    RedshiftCredentialsConnection,
+)
+
+# Import Teradata connection models
+from snowflake.snowflake_data_validation.teradata.model.teradata_credentials_connection import (
+    TeradataCredentialsConnection,
+)
+
+# Import Snowflake connection models
+from snowflake.snowflake_data_validation.snowflake.model.snowflake_default_connection import (
+    SnowflakeDefaultConnection,
+)
+from snowflake.snowflake_data_validation.snowflake.model.snowflake_named_connection import (
+    SnowflakeNamedConnection,
+)
+from snowflake.snowflake_data_validation.snowflake.model.snowflake_credentials_connection import (
+    SnowflakeCredentialsConnection,
+)
+
+__all__ = [
+    "SqlServerCredentialsConnection",
+    "TeradataCredentialsConnection",
+    "SnowflakeDefaultConnection",
+    "SnowflakeNamedConnection",
+    "SnowflakeCredentialsConnection",
+    "RedshiftCredentialsConnection",
+]

snowflake/snowflake_data_validation/configuration/model/logging_configuration.py

@@ -0,0 +1,63 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Logging configuration model for Snowflake Data Validation."""
+
+import logging
+
+from pydantic import BaseModel, field_validator
+
+
+class LoggingConfiguration(BaseModel):
+    """Logging configuration model.
+
+    Args:
+        level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        console_level: Console logging level (defaults to level if not specified)
+        file_level: File logging level (defaults to level if not specified)
+
+    """
+
+    level: str = "INFO"
+    console_level: str | None = None
+    file_level: str | None = None
+
+    @field_validator("level", "console_level", "file_level")
+    @classmethod
+    def validate_log_level(cls, value: str | None) -> str | None:
+        """Validate logging level values."""
+        if value is None:
+            return value
+
+        valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+        if value.upper() not in valid_levels:
+            raise ValueError(
+                f"Invalid logging level: {value}. "
+                f"Valid levels are: {', '.join(valid_levels)}"
+            )
+        return value.upper()
+
+    def get_console_level(self) -> str:
+        """Get the console logging level."""
+        return self.console_level or self.level
+
+    def get_file_level(self) -> str:
+        """Get the file logging level."""
+        return self.file_level or self.level
+
+    def get_console_level_int(self) -> int:
+        """Get the console logging level as integer."""
+        return getattr(logging, self.get_console_level())
+
+    def get_file_level_int(self) -> int:
+        """Get the file logging level as integer."""
+        return getattr(logging, self.get_file_level())

snowflake/snowflake_data_validation/configuration/model/table_configuration.py

@@ -0,0 +1,210 @@
+import logging
+
+from typing import ClassVar
+
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+from snowflake.snowflake_data_validation.configuration.model.validation_configuration import (
+    ValidationConfiguration,
+)
+from snowflake.snowflake_data_validation.utils.constants import NEWLINE
+from snowflake.snowflake_data_validation.utils.helper import Helper
+from snowflake.snowflake_data_validation.utils.helpers.helper_database import (
+    HelperDatabase,
+)
+
+
+LOGGER = logging.getLogger(__name__)
+
+MANDATORY_PROPERTIES: set[str] = {
+    "target_name",
+    "use_column_selection_as_exclude_list",
+    "column_selection_list",
+}
+
+VISIBLE_PROPERTIES: set[str] = {
+    "apply_metric_column_modifier",
+    "chunk_number",
+    "column_mappings",
+    "exclude_metrics",
+    "index_column_list",
+    "is_case_sensitive",
+    "max_failed_rows_number",
+    "target_database",
+    "target_index_column_list",
+    "target_schema",
+    "target_where_clause",
+    "where_clause",
+}
+
+
+class TableConfiguration(BaseModel):
+    """Table configuration model.
+
+    Args:
+        pydantic.BaseModel (pydantic.BaseModel): pydantic BaseModel
+
+    """
+
+    _id_counter: ClassVar[int] = 0
+    id: int | None = None
+
+    fully_qualified_name: str
+    use_column_selection_as_exclude_list: bool
+    column_selection_list: list[str]
+    validation_configuration: ValidationConfiguration | None = None
+
+    source_database: str | None = None
+    source_schema: str | None = None
+    source_table: str | None = None
+    target_name: str | None = None
+
+    target_fully_qualified_name: str = ""
+    where_clause: str = ""
+    target_where_clause: str = ""
+    has_where_clause: bool = False
+    has_target_where_clause: bool = False
+    target_database: str | None = None
+    target_schema: str | None = None
+    index_column_list: list[str] = []
+    target_index_column_list: list[str] = []
+    is_case_sensitive: bool = False
+    chunk_number: int | None = None
+    column_mappings: dict[str, str] = {}
+    max_failed_rows_number: int | None = None
+    exclude_metrics: bool | None = None
+    apply_metric_column_modifier: bool | None = None
+
+    @model_validator(mode="after")
+    def load(self) -> Self:
+        self._assign_id()
+        self._load_source_decomposed_fully_qualified_name()
+        self._load_target_fully_qualified_name()
+        self._set_has_where_clause()
+        self._set_has_target_where_clause()
+        self._normalize_where_clause()
+        self._set_target_index_column_list()
+        self._check_chunk_number()
+        self._check_target_where_clause()
+        self._set_column_mappings()
+        self._check_max_failed_rows_number()
+        return self
+
+    def _assign_id(self) -> None:
+        if self.id is None:
+            TableConfiguration._id_counter += 1
+            self.id = TableConfiguration._id_counter
+
+    def _load_source_decomposed_fully_qualified_name(self) -> None:
+        self.fully_qualified_name = self.fully_qualified_name.replace('\\"', '"')
+
+        decomposed_tuple = Helper.get_decomposed_fully_qualified_name(
+            self.fully_qualified_name
+        )
+
+        if len(decomposed_tuple) == 2:
+            self.source_database = None
+            self.source_schema = decomposed_tuple[0]
+            self.source_table = decomposed_tuple[1]
+
+        else:
+            self.source_database = decomposed_tuple[0]
+            self.source_schema = decomposed_tuple[1]
+            self.source_table = decomposed_tuple[2]
+
+    def _load_target_fully_qualified_name(self) -> None:
+        if self.target_database is None:
+            self.target_database = self.source_database
+
+        if self.target_schema is None:
+            self.target_schema = self.source_schema
+
+        if self.target_name is None:
+            self.target_name = self.source_table
+        else:
+            self.target_name = HelperDatabase.normalize_escape_quotes(self.target_name)
+
+        self.target_fully_qualified_name = (
+            f"{self.target_database}.{self.target_schema}.{self.target_name}"
+        )
+
+    def _set_has_where_clause(self) -> None:
+        self.has_where_clause = self.where_clause != ""
+
+    def _set_has_target_where_clause(self) -> None:
+        self.has_target_where_clause = self.target_where_clause != ""
+
+    def _check_target_where_clause(self) -> None:
+        if self.has_where_clause and not self.has_target_where_clause:
+            LOGGER.warning(
+                "Target where clause was not set for table %s. This may lead to unexpected results.",
+                self.target_fully_qualified_name,
+            )
+
+    def _set_target_index_column_list(self) -> None:
+        if self.target_index_column_list == []:
+            self.target_index_column_list = self.index_column_list.copy()
+
+        local_target_index_column_list = []
+        for target_column_name in self.target_index_column_list:
+            new_column_name = self.column_mappings.get(
+                target_column_name, target_column_name
+            )
+            local_target_index_column_list.append(new_column_name)
+
+        self.target_index_column_list = local_target_index_column_list
+
+    def _check_chunk_number(self) -> None:
+        if self.chunk_number is None:
+            return
+        elif self.chunk_number < 1:
+            raise ValueError("Chunk number must be greater than or equal to 1.")
+
+    def _set_column_mappings(self) -> None:
+        """Normalize the keys and values of the column mappings dictionary to uppercase."""
+        local_column_mapping = {}
+        for key in self.column_mappings.keys():
+            if self.is_case_sensitive:
+                local_column_mapping[key] = self.column_mappings[key]
+            else:
+                local_column_mapping[key.upper()] = self.column_mappings[key].upper()
+        self.column_mappings = local_column_mapping
+
+    def _check_max_failed_rows_number(self) -> None:
+        # Accept None, because it will be overwritten by ConfigurationModel
+        # based on the value of ValidationConfiguration model.
+        if self.max_failed_rows_number is None:
+            return
+        elif self.max_failed_rows_number < 1:
+            raise ValueError(
+                f"Invalid value for max failed rows number in table f{self.fully_qualified_name}. "
+                "Value must be greater than or equal to 1."
+            )
+
+    def _normalize_where_clause(self) -> None:
+        if self.has_where_clause:
+            self.where_clause = HelperDatabase.remove_escape_quotes(self.where_clause)
+
+        if self.has_target_where_clause:
+            self.target_where_clause = HelperDatabase.remove_escape_quotes(
+                self.target_where_clause
+            )
+
+    def __str__(self) -> str:
+        """Return a formatted string representation of the table configuration."""
+        properties = []
+
+        # Add mandatory properties
+        for attr in MANDATORY_PROPERTIES:
+            properties.append(f"    {attr}: {getattr(self, attr)}")
+
+        # Add visible properties with non-empty values
+        for attr in VISIBLE_PROPERTIES:
+            value = getattr(self, attr)
+            if value not in (None, [], {}, "", 0):
+                properties.append(f"    {attr}: {value}")
+
+        properties.sort()
+        properties_str = NEWLINE.join(properties)
+        return f"  - fully_qualified_name: {self.fully_qualified_name}{NEWLINE}{properties_str}"

snowflake/snowflake_data_validation/configuration/model/validation_configuration.py

@@ -0,0 +1,59 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+from pydantic import BaseModel, model_validator
+
+from snowflake.snowflake_data_validation.utils.constants import (
+    MAX_FAILED_ROWS_NUMBER_DEFAULT_VALUE,
+)
+
+
+class ValidationConfiguration(BaseModel):
+    """Class representing the validation levels to be applied to a table."""
+
+    schema_validation: bool | None = False
+    metrics_validation: bool | None = False
+    row_validation: bool | None = False
+    # Custom templates path for validation scripts.
+    custom_templates_path: Path | None = None
+    max_failed_rows_number: int = MAX_FAILED_ROWS_NUMBER_DEFAULT_VALUE
+    exclude_metrics: bool = False
+    apply_metric_column_modifier: bool = False
+
+    @model_validator(mode="after")
+    def validate_configuration(self) -> "ValidationConfiguration":
+        """Validate the configuration after initialization.
+
+        This method checks if at least one validation type is enabled.
+
+        Raises:
+            ValueError: If no validation type is enabled.
+
+        """
+        if not self.model_dump(exclude_none=True):
+            raise ValueError(
+                "At least one validation type must be enabled in case of adding the property."
+            )
+        self._check_max_failed_rows_number()
+        return self
+
+    def _check_max_failed_rows_number(self) -> None:
+        if self.max_failed_rows_number < 1:
+            raise ValueError(
+                "Invalid value for max failed rows number in validation configuration. "
+                "Value must be greater than or equal to 1."
+            )

snowflake/snowflake_data_validation/configuration/singleton.py

@@ -0,0 +1,16 @@
+class Singleton(type):
+
+    """Singleton metaclass.
+
+    This metaclass ensures that only one instance of a class exists.
+    Any class using this metaclass will return the same instance
+    every time it is instantiated, effectively implementing the
+    Singleton design pattern.
+    """
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super().__call__(*args, **kwargs)
+        return cls._instances[cls]

snowflake/snowflake_data_validation/connector/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Description: This file is used to expose the ConnectorBase class to the outside world.
+# The ConnectorBase class is used to define the interface for the Snowflake connector.
+# It is used to define the methods that the connector must implement.
+# The methods defined in the ConnectorBase class are connect
+# execute_query, and close.
+# The connect method is used to establish a connection to the Snowflake database.
+
+__all__ = ["ConnectorBase", "NullConnector", "ConnectorFactoryBase"]
+
+from snowflake.snowflake_data_validation.connector.connector_base import (
+    ConnectorBase,
+    NullConnector,
+)
+from snowflake.snowflake_data_validation.connector.connector_factory_base import (
+    ConnectorFactoryBase,
+)

snowflake/snowflake_data_validation/connector/connector_base.py

@@ -0,0 +1,183 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import time
+
+from abc import ABC, abstractmethod
+
+from snowflake.snowflake_data_validation.utils.constants import (
+    DEFAULT_CONNECTION_MODE,
+)
+
+
+class ConnectorBase(ABC):
+    """Abstract base class for database connectors."""
+
+    def __init__(self, output_path: str | None = None):
+        """
+        Initialize the database connector base class.
+
+        Args:
+            output_path (Optional[str], optional): Path for output files. Defaults to None.
+
+        """
+        self.connection: object | None = None
+        self.output_path = output_path
+        self.logger = logging.getLogger(__name__)
+
+    @abstractmethod
+    def connect(
+        self, mode: str = DEFAULT_CONNECTION_MODE, connection_name: str = ""
+    ) -> None:
+        """Establish a connection to the database."""
+        pass
+
+    def connect_with_retry(
+        self,
+        connect_func,
+        max_attempts: int = 3,
+        delay_seconds: float = 1.0,
+        delay_multiplier: float = 2.0,
+        *args,
+        **kwargs,
+    ) -> None:
+        """
+        Establish a connection with retry logic.
+
+        Args:
+            connect_func: The connection function to call
+            max_attempts: Maximum number of connection attempts
+            delay_seconds: Initial delay between retries in seconds
+            delay_multiplier: Multiplier for exponential backoff
+            *args: Arguments to pass to connect_func
+            **kwargs: Keyword arguments to pass to connect_func
+
+        Raises:
+            ConnectionError: If all connection attempts fail
+
+        """
+        last_exception = None
+        current_delay = delay_seconds
+
+        for attempt in range(1, max_attempts + 1):
+            try:
+                self.logger.info(f"Connection attempt {attempt}/{max_attempts}")
+                connect_func(*args, **kwargs)
+                self.logger.info("Connection established successfully")
+                return
+
+            except Exception as e:
+                last_exception = e
+                self.logger.warning(
+                    f"Connection attempt {attempt}/{max_attempts} failed: {str(e)}"
+                )
+                if attempt < max_attempts:
+                    self.logger.info(f"Retrying in {current_delay:.1f} seconds...")
+                    time.sleep(current_delay)
+                    current_delay *= delay_multiplier
+                else:
+                    self.logger.error(f"All {max_attempts} connection attempts failed")
+        raise ConnectionError(
+            f"Failed to establish connection after {max_attempts} attempts. "
+            f"Last error: {last_exception}"
+        ) from last_exception
+
+    def _verify_connection(self) -> None:
+        """
+        Verify the connection by executing a simple test query.
+
+        Default implementation that can be overridden by child classes.
+        Child classes should implement their own verification logic.
+
+        Raises:
+            ConnectionError: If connection verification fails
+
+        """
+        if self.connection is None:
+            raise ConnectionError("Connection is None")
+
+        # Default implementation - child classes should override this
+        self.logger.debug("Using default connection verification (no-op)")
+
+    def _attempt_connection_and_verify(self, connect_func, *args, **kwargs) -> None:
+        """
+        Attempt connection and verify it.
+
+        Args:
+            connect_func: The connection function to call
+            *args: Arguments to pass to connect_func
+            **kwargs: Keyword arguments to pass to connect_func
+
+        Raises:
+            ConnectionError: If connection or verification fails
+
+        """
+        try:
+            connect_func(*args, **kwargs)
+            self._verify_connection()
+
+        except Exception:
+            if self.connection is not None:
+                try:
+                    close_method = getattr(self.connection, "close", None)
+                    if close_method is not None:
+                        close_method()
+                except Exception:
+                    pass  # Ignore errors during cleanup
+                self.connection = None
+            raise
+
+    @abstractmethod
+    def execute_query(self, query: str) -> list[tuple]:
+        """Execute a query on the database."""
+        pass
+
+    @abstractmethod
+    def execute_statement(self, statement: str) -> None:
+        """Execute a statement on the database."""
+        pass
+
+    @abstractmethod
+    def execute_query_no_return(self, query: str) -> None:
+        """Execute a query on the database without returning results."""
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """Close the connection to the database."""
+        pass
+
+
+class NullConnector(ConnectorBase):
+    """A connector that does nothing. Used as a null object for async validation scenarios."""
+
+    def connect(
+        self, mode: str = DEFAULT_CONNECTION_MODE, connection_name: str = ""
+    ) -> None:
+        pass
+
+    def execute_query(self, query: str) -> list[tuple]:
+        return []
+
+    def execute_query_no_return(self, query: str) -> None:
+        pass
+
+    def execute_statement(self, statement: str) -> None:
+        pass
+
+    def close(self) -> None:
+        pass