snowflake-data-validation 1.0.1__py3-none-any.whl

snowflake/snowflake_data_validation/utils/helpers/helper_templates.py

@@ -0,0 +1,294 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+from snowflake.snowflake_data_validation.utils.constants import (
+    COL_NAME_QUOTES_PLACEHOLDER,
+    COLUMN_MODIFIER_COLUMN_KEY,
+    EDITABLE_YAML_FILE_FORMAT_ERROR,
+    METRIC_COLUMN_KEY,
+    METRIC_METRIC_COLUMN_MODIFIER_KEY,
+    METRIC_NORMALIZATION_KEY,
+    METRIC_QUERY_KEY,
+    METRIC_QUERY_PLACEHOLDER,
+    METRIC_RETURN_DATATYPE_KEY,
+    NORMALIZATION_COLUMN_KEY,
+    TEMPLATE_COLUMN_KEY,
+    TYPE_COLUMN_KEY,
+)
+
+
+class HelperTemplates:
+    """Helper class for loading and processing templates from YAML files."""
+
+    @staticmethod
+    def load_metrics_templates_from_yaml(
+        yaml_path: Path, datatypes_normalization_templates: dict[str, str]
+    ) -> pd.DataFrame:
+        """Load metrics templates from a YAML file into a pandas DataFrame.
+
+        Args:
+            yaml_path (Path): The file path to the YAML file.
+            datatypes_normalization_templates (dict[str, str]): A dictionary containing
+            normalization templates for data types.
+
+        Raises:
+            FileNotFoundError: If the specified YAML file does not exist.
+            KeyError: If a required key is missing in the YAML data.
+            RuntimeError: If there is an error in the format of the YAML file.
+
+        Returns:
+            pd.DataFrame: A DataFrame containing the data from the YAML file.
+
+        """
+        if not yaml_path.exists():
+            raise FileNotFoundError(f"Template file not found at: {yaml_path}")
+
+        try:
+            file_content = yaml_path.read_text()
+            yaml_data = yaml.safe_load(file_content)
+            yaml_data_reformatted = HelperTemplates._reformat_metrics_yaml_data(
+                yaml_data=yaml_data,
+                datatypes_normalization_templates=datatypes_normalization_templates,
+            )
+            df = pd.DataFrame.from_dict(yaml_data_reformatted)
+
+        except KeyError as e:
+            error_message = (
+                f"Missing {e.args[0]} datatype in datatypes normalization file."
+            )
+            raise RuntimeError(error_message) from e
+
+        except Exception as e:
+            error_message = EDITABLE_YAML_FILE_FORMAT_ERROR.format(
+                file_name=yaml_path.name
+            )
+            raise RuntimeError(error_message) from e
+
+        return df
+
+    @staticmethod
+    def load_datatypes_templates_from_yaml(
+        yaml_path: Path, platform: str
+    ) -> pd.DataFrame:
+        """Load datatypes templates from a YAML file into a pandas DataFrame.
+
+        Args:
+            yaml_path (Path): The file path to the YAML file.
+            platform (str): The platform identifier to use.
+
+        Raises:
+            FileNotFoundError: If the specified YAML file does not exist.
+            RuntimeError: If there is an error in the format of the YAML file.
+
+        Returns:
+            pd.DataFrame: A DataFrame containing the data from the YAML file.
+
+        """
+        if not yaml_path.exists():
+            raise FileNotFoundError(f"Template file not found at: {yaml_path}")
+
+        try:
+            file_content = yaml_path.read_text()
+            yaml_data = yaml.safe_load(file_content)
+            yaml_data_reformatted = HelperTemplates._reformat_datatypes_yaml_data(
+                yaml_data, platform
+            )
+            df = pd.DataFrame.from_dict(yaml_data_reformatted)
+
+        except Exception as e:
+            error_message = EDITABLE_YAML_FILE_FORMAT_ERROR.format(
+                file_name=yaml_path.name
+            )
+            raise RuntimeError(error_message) from e
+
+        return df
+
+    @staticmethod
+    def load_datatypes_normalization_templates_from_yaml(
+        yaml_path: Path,
+    ) -> dict[str, str]:
+        """Load datatypes normalization templates from a YAML file into a pandas DataFrame.
+
+        Args:
+            yaml_path (Path): The file path to the YAML file.
+
+        Raises:
+            FileNotFoundError: If the specified YAML file does not exist.
+            RuntimeError: If there is an error in the format of the YAML file.
+
+        Returns:
+            dict[str, str]: A dictionary containing the data from the YAML file.
+
+        """
+        if not yaml_path.exists():
+            raise FileNotFoundError(f"Template file not found at: {yaml_path}")
+
+        try:
+            file_content = yaml_path.read_text()
+            yaml_data = yaml.safe_load(file_content)
+            yaml_data_reformatted = {
+                key.upper(): yaml_data[key].replace('\\"', '"')
+                for key in yaml_data.keys()
+            }
+
+        except Exception as e:
+            error_message = EDITABLE_YAML_FILE_FORMAT_ERROR.format(
+                file_name=yaml_path.name
+            )
+            raise RuntimeError(error_message) from e
+
+        return yaml_data_reformatted
+
+    @staticmethod
+    def _reformat_datatypes_yaml_data(
+        yaml_data: dict, platform: str
+    ) -> dict[str, list[str]]:
+        """Reformat YAML data to ensure it is in a consistent format.
+
+        Args:
+            yaml_data (dict): The original YAML data.
+            platform (str): The platform identifier to use.
+
+        Raises:
+            ValueError: If the YAML data does not contain the expected structure.
+
+        Returns:
+            dict: The reformatted YAML data.
+
+        """
+        source_platform_data_types_collection = list(yaml_data.keys())
+        platform_data_types_dict_collection = {
+            str.lower(platform): source_platform_data_types_collection
+        }
+        temporal_platform_data_types_dict_collection = {}
+        for source_data_type in source_platform_data_types_collection:
+            for target_data_type in yaml_data[source_data_type]:
+                if (
+                    temporal_platform_data_types_dict_collection.get(target_data_type)
+                    is None
+                ):
+                    temporal_platform_data_types_dict_collection[target_data_type] = []
+                current_data_type = yaml_data[source_data_type][target_data_type]
+                temporal_platform_data_types_dict_collection[target_data_type].append(
+                    current_data_type
+                )
+
+        platform_data_types_dict_collection.update(
+            temporal_platform_data_types_dict_collection
+        )
+        return platform_data_types_dict_collection
+
+    @staticmethod
+    def _reformat_metrics_yaml_data(
+        yaml_data: dict, datatypes_normalization_templates: dict[str, str]
+    ) -> dict[str, list[str]]:
+        """Reformat YAML data to ensure it is in a consistent format.
+
+        Args:
+            yaml_data (dict): The original YAML data.
+            datatypes_normalization_templates (dict[str, str]): A dictionary containing
+            normalization templates for data types.
+
+        Raises:
+            ValueError: If the YAML data does not contain the expected structure.
+
+        Returns:
+            dict: The reformatted YAML data.
+
+        """
+        type_column = []
+        metric_column = []
+        metric_template_column = []
+        metric_normalization_template_column = []
+        column_modifier_column = []
+
+        data_type_collection = list(yaml_data.keys())
+        for data_type in data_type_collection:
+            metric_name_collection = list(yaml_data[data_type].keys())
+            template_collection = list(yaml_data[data_type].values())
+            for metric_name, templates in zip(
+                metric_name_collection, template_collection, strict=False
+            ):
+                type_column.append(data_type.upper())
+                metric_column.append(metric_name)
+                metric_template_column.append(templates[METRIC_QUERY_KEY])
+
+                if templates.get(METRIC_NORMALIZATION_KEY, None) is not None:
+                    metric_normalization_template_column.append(
+                        templates[METRIC_NORMALIZATION_KEY]
+                    )
+                else:
+                    metric_return_datatype = templates[METRIC_RETURN_DATATYPE_KEY]
+                    normalization_template = datatypes_normalization_templates[
+                        metric_return_datatype.upper()
+                    ]
+                    normalization_template_normalized = normalization_template.replace(
+                        COL_NAME_QUOTES_PLACEHOLDER, METRIC_QUERY_PLACEHOLDER
+                    )
+                    metric_normalization_template_column.append(
+                        normalization_template_normalized
+                    )
+
+                column_modifier_column.append(
+                    templates.get(METRIC_METRIC_COLUMN_MODIFIER_KEY, None)
+                )
+
+        type_metric_dict_collection = {
+            TYPE_COLUMN_KEY: type_column,
+            METRIC_COLUMN_KEY: metric_column,
+            TEMPLATE_COLUMN_KEY: metric_template_column,
+            NORMALIZATION_COLUMN_KEY: metric_normalization_template_column,
+            COLUMN_MODIFIER_COLUMN_KEY: column_modifier_column,
+        }
+
+        return type_metric_dict_collection
+
+    @staticmethod
+    def load_datatypes_mapping_templates_from_yaml(yaml_path: Path) -> dict[str, str]:
+        """Load datatypes mapping templates from a YAML file into a dictionary.
+
+        Args:
+            yaml_path (Path): The file path to the YAML file.
+
+        Raises:
+            FileNotFoundError: If the specified YAML file does not exist.
+            RuntimeError: If there is an error in the format of the YAML file.
+
+        Returns:
+            dict[str, str]: A dictionary containing the data from the YAML file.
+
+        """
+        if not yaml_path.exists():
+            raise FileNotFoundError(f"Template file not found at: {yaml_path}")
+
+        try:
+            file_content = yaml_path.read_text()
+            if len(file_content) == 0:
+                return {}
+
+            yaml_data = yaml.safe_load(file_content)
+
+        except Exception as e:
+            error_message = EDITABLE_YAML_FILE_FORMAT_ERROR.format(
+                file_name=yaml_path.name
+            )
+            raise RuntimeError(error_message) from e
+
+        return yaml_data

snowflake/snowflake_data_validation/utils/logging_config.py

@@ -0,0 +1,197 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Logging configuration for Snowflake Data Validation CLI."""
+
+import logging.config
+import os
+import shutil
+
+from datetime import datetime
+
+
+class LoggingManager:
+    """Manages logging configuration and log file relocation."""
+
+    # Logger names that need file handler management
+    _MANAGED_LOGGER_NAMES = ["", "snowflake.snowflake_data_validation"]
+
+    def __init__(self):
+        """Initialize the logging manager."""
+        self._log_timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        self._current_log_file: str | None = None
+
+    def setup_logging(
+        self,
+        log_level: str = "INFO",
+        console_level: str | None = None,
+        file_level: str | None = None,
+    ) -> str:
+        """Set up logging configuration.
+
+        Args:
+            log_level: Default logging level for both console and file
+            console_level: Console-specific logging level (overrides log_level)
+            file_level: File-specific logging level (overrides log_level)
+
+        Returns:
+            str: Path to the initial log file
+
+        """
+        # Create initial log file in current directory
+        log_filename = f"data_validation_{self._log_timestamp}.log"
+        self._current_log_file = log_filename
+
+        # Determine actual log levels
+        actual_console_level = console_level or log_level
+        actual_file_level = file_level or log_level
+
+        logging.config.dictConfig(
+            {
+                "version": 1,
+                "disable_existing_loggers": False,
+                "formatters": {
+                    "standard": {
+                        "format": "{asctime} - {name} - {levelname} - {message}",
+                        "style": "{",
+                        "datefmt": "%Y-%m-%d %H:%M:%S",
+                    },
+                },
+                "handlers": {
+                    "console": {
+                        "class": "logging.StreamHandler",
+                        "formatter": "standard",
+                        "level": actual_console_level,
+                    },
+                    "file": {
+                        "class": "logging.FileHandler",
+                        "formatter": "standard",
+                        "filename": log_filename,
+                        "level": actual_file_level,
+                        "encoding": "utf-8",
+                    },
+                },
+                "loggers": {
+                    "snowflake.snowflake_data_validation": {
+                        "handlers": ["file"],
+                        "level": actual_file_level,
+                        "propagate": False,
+                    },
+                },
+                "root": {
+                    "handlers": ["console", "file"],
+                    "level": min(actual_console_level, actual_file_level),
+                },
+            }
+        )
+
+        return log_filename
+
+    def relocate_log_file(self, output_directory_path: str) -> bool:
+        """Move the log file to the output directory.
+
+        Args:
+            output_directory_path: Target directory for the log file
+
+        Returns:
+            bool: True if successful, False otherwise
+
+        """
+        if not self._current_log_file or not output_directory_path:
+            return False
+
+        try:
+            os.makedirs(output_directory_path, exist_ok=True)
+            new_log_path = os.path.join(
+                output_directory_path, f"data_validation_{self._log_timestamp}.log"
+            )
+            if os.path.exists(self._current_log_file):
+                self._close_file_handlers()
+
+                shutil.move(self._current_log_file, new_log_path)
+                self._current_log_file = new_log_path
+
+                self._update_file_handler(new_log_path)
+
+                logging.getLogger(__name__).info(
+                    f"Log file relocated to: {new_log_path}"
+                )
+                return True
+
+        except Exception as e:
+            logging.getLogger(__name__).error(
+                f"Failed to relocate log file: {e}", exc_info=True
+            )
+
+        return False
+
+    def get_current_log_file(self) -> str:
+        """Get the current log file path."""
+        return self._current_log_file or ""
+
+    def _close_file_handlers(self):
+        """Close all file handlers to release file locks (important for Windows)."""
+        for logger_name in self._MANAGED_LOGGER_NAMES:
+            logger = logging.getLogger(logger_name)
+            for handler in logger.handlers[:]:
+                if isinstance(handler, logging.FileHandler):
+                    handler.flush()
+                    handler.close()
+
+    def _update_file_handler(self, new_log_path: str):
+        """Update file handlers to use the new log file path."""
+        for logger_name in self._MANAGED_LOGGER_NAMES:
+            logger = logging.getLogger(logger_name)
+            # Iterate over a shallow copy of the logger's handlers list.
+            # This allows us to safely modify (remove) handlers from the original list during iteration.
+            for handler in logger.handlers[:]:
+                if isinstance(handler, logging.FileHandler):
+                    new_handler = logging.FileHandler(new_log_path, encoding="utf-8")
+                    new_handler.setLevel(handler.level)
+                    new_handler.setFormatter(handler.formatter)
+
+                    # Replace old handler
+                    logger.removeHandler(handler)
+                    logger.addHandler(new_handler)
+
+
+# Singleton instance
+_logging_manager = LoggingManager()
+
+
+def setup_logging(
+    log_level: str = "INFO",
+    console_level: str | None = None,
+    file_level: str | None = None,
+) -> str:
+    """Set up logging configuration.
+
+    Args:
+        log_level: Default logging level for both console and file
+        console_level: Console-specific logging level (overrides log_level)
+        file_level: File-specific logging level (overrides log_level)
+
+    Returns:
+        str: Path to the initial log file
+
+    """
+    return _logging_manager.setup_logging(log_level, console_level, file_level)
+
+
+def relocate_log_file(output_directory_path: str) -> bool:
+    """Relocate the log file to the output directory."""
+    return _logging_manager.relocate_log_file(output_directory_path)
+
+
+def get_current_log_file() -> str:
+    """Get the current log file path."""
+    return _logging_manager.get_current_log_file()

snowflake/snowflake_data_validation/utils/logging_utils.py

@@ -0,0 +1,68 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from collections.abc import Callable
+from functools import wraps
+from typing import TypeVar
+
+from typing_extensions import ParamSpec
+
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def log(
+    _func: Callable[P, R] | None = None,
+    *,
+    logger: logging.Logger | None = None,
+    log_args: bool = True,
+) -> Callable[[Callable[P, R]], Callable[P, R]]:
+    """Log the function call and any exceptions that occur.
+
+    Args:
+        _func: The function to log.
+        logger: The logger to use for logging. If not provided, a logger will be created using the
+                function's module name.
+        log_args: Whether to log the arguments passed to the function.
+
+    Returns:
+        A decorator that logs the function call and any exceptions that occur.
+
+    """
+
+    def decorator(func: Callable[P, R]) -> Callable[P, R]:
+        @wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+            _logger = logging.getLogger(func.__module__) if logger is None else logger
+            if log_args:
+                args_repr = [repr(a) for a in args]
+                kwargs_repr = [f"{k}={v!r}" for k, v in kwargs.items()]
+                formatted_args = ", ".join([*args_repr, *kwargs_repr])
+                _logger.debug("%s called with args %s", func.__name__, formatted_args)
+            try:
+                return func(*args, **kwargs)
+            except Exception:
+                _logger.exception("An error occurred in %s", func.__name__)
+                raise
+
+        return wrapper
+
+    # Handle the case where the decorator is used without parentheses
+    if _func is None:
+        return decorator
+    return decorator(_func)

snowflake/snowflake_data_validation/utils/model/chunk.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Chunk:
+
+    """Represents a chunk of data to be processed.
+
+    Attributes:
+        fetch (int): The number of records to fetch in this chunk.
+        offset (int): The starting point from which to fetch the records.
+
+    """
+
+    fetch: int
+    offset: int

snowflake/snowflake_data_validation/utils/model/column_metadata.py

@@ -0,0 +1,45 @@
+class ColumnMetadata:
+
+    """Represents metadata for a column in a Snowflake table."""
+
+    def __init__(
+        self,
+        name: str,
+        data_type: str,
+        nullable: bool,
+        is_primary_key: bool,
+        calculated_column_size_in_bytes: int,
+        properties: dict[str, any],
+    ):
+        """Initialize the ColumnMetadata object.
+
+        Args:
+            name (str): The name of the column.
+            data_type (str): The data type of the column.
+            nullable (bool): Indicates if the column can contain NULL values.
+            is_primary_key (bool): Indicates if the column is part of the primary key.
+            calculated_column_size_in_bytes (int): The calculated size of the column in bytes.
+            properties (dict[str, any]): Additional properties of the column.
+
+        """
+        self.name: str = name
+        self.data_type: str = data_type
+        self.nullable: bool = nullable
+        self.is_primary_key: bool = is_primary_key
+        self.calculated_column_size_in_bytes: int = calculated_column_size_in_bytes
+        self.properties: dict[str, any] = properties
+
+    def copy(self) -> "ColumnMetadata":
+        """Create a copy of the ColumnMetadata instance."""
+        return ColumnMetadata(
+            self.name,
+            self.data_type,
+            self.nullable,
+            self.is_primary_key,
+            self.calculated_column_size_in_bytes,
+            self.properties.copy(),
+        )
+
+    def to_upper_name(self) -> None:
+        """Convert the column name to uppercase."""
+        self.name = self.name.upper()