snowpark-checkpoints-validators 0.2.1__tar.gz → 0.3.1__tar.gz

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-validators
-Version: 0.2.1
+Version: 0.3.1
 Summary: Migration tools for Snowpark
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/
@@ -27,9 +27,11 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: <3.12,>=3.9
 Requires-Dist: pandera[io]==0.20.4
+Requires-Dist: pydantic>=2.0
 Requires-Dist: snowflake-connector-python[pandas]
 Requires-Dist: snowflake-snowpark-python>=1.23.0
 Provides-Extra: development
+Requires-Dist: certifi==2025.1.31; extra == 'development'
 Requires-Dist: coverage>=7.6.7; extra == 'development'
 Requires-Dist: deepdiff==8.1.1; extra == 'development'
 Requires-Dist: deepdiff>=8.0.0; extra == 'development'
@@ -320,4 +322,37 @@ sp_dataframe = session.create_dataframe(df)
 preprocessed_dataframe = preprocessor(sp_dataframe)
 ```
 
+### Skip validation
+The method `xvalidate_dataframe_checkpoint` can be used to avoid the validation of the checkpoint defined.
+The result of the checkpoint will be `SKIP` and a message will be logged in the terminal. The method contains the same signature of `validate_dataframe_checkpoint`.
+
+#### Usage Example
+
+```python
+from snowflake.snowpark import Session
+from snowflake.snowpark_checkpoints.utils.constant import (
+    CheckpointMode,
+)
+from snowflake.snowpark_checkpoints.checkpoint import validate_dataframe_checkpoint
+from snowflake.snowpark_checkpoints.spark_migration import SamplingStrategy
+from snowflake.snowpark_checkpoints.job_context import SnowparkJobContext
+from pyspark.sql import SparkSession
+
+session = Session.builder.getOrCreate()
+job_context = SnowparkJobContext(
+    session, SparkSession.builder.getOrCreate(), "job_context", True
+)
+df = session.read.format("csv").load("data.csv")
+
+xvalidate_dataframe_checkpoint(
+    df,
+    "schema_checkpoint",
+    job_context=job_context,
+    mode=CheckpointMode.SCHEMA,
+    sample_frac=0.1,
+    sampling_strategy=SamplingStrategy.RANDOM_SAMPLE
+)
+```
+
+
 ------

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/README.md

@@ -273,4 +273,37 @@ sp_dataframe = session.create_dataframe(df)
 preprocessed_dataframe = preprocessor(sp_dataframe)
 ```
 
+### Skip validation
+The method `xvalidate_dataframe_checkpoint` can be used to avoid the validation of the checkpoint defined.
+The result of the checkpoint will be `SKIP` and a message will be logged in the terminal. The method contains the same signature of `validate_dataframe_checkpoint`.
+
+#### Usage Example
+
+```python
+from snowflake.snowpark import Session
+from snowflake.snowpark_checkpoints.utils.constant import (
+    CheckpointMode,
+)
+from snowflake.snowpark_checkpoints.checkpoint import validate_dataframe_checkpoint
+from snowflake.snowpark_checkpoints.spark_migration import SamplingStrategy
+from snowflake.snowpark_checkpoints.job_context import SnowparkJobContext
+from pyspark.sql import SparkSession
+
+session = Session.builder.getOrCreate()
+job_context = SnowparkJobContext(
+    session, SparkSession.builder.getOrCreate(), "job_context", True
+)
+df = session.read.format("csv").load("data.csv")
+
+xvalidate_dataframe_checkpoint(
+    df,
+    "schema_checkpoint",
+    job_context=job_context,
+    mode=CheckpointMode.SCHEMA,
+    sample_frac=0.1,
+    sampling_strategy=SamplingStrategy.RANDOM_SAMPLE
+)
+```
+
+
 ------

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/pyproject.toml

@@ -29,6 +29,7 @@ dependencies = [
   "snowflake-snowpark-python>=1.23.0",
   "snowflake-connector-python[pandas]",
   "pandera[io]==0.20.4",
+  "pydantic>=2.0"
 ]
 description = "Migration tools for Snowpark"
 dynamic = ['version']
@@ -61,6 +62,7 @@ development = [
   "pyarrow>=18.0.0",
   "deepdiff>=8.0.0",
   "pyspark>=3.5.0",
+  "certifi==2025.1.31",
 ]
 
 [project.urls]

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/src/snowflake/snowpark_checkpoints/__init__.py

@@ -39,6 +39,7 @@ __all__ = [
     "check_dataframe_schema",
     "check_output_schema",
     "check_input_schema",
+    "xvalidate_dataframe_checkpoint",
     "validate_dataframe_checkpoint",
     "CheckpointMode",
 ]

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/src/snowflake/snowpark_checkpoints/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.2.1"
+__version__ = "0.3.1"

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/src/snowflake/snowpark_checkpoints/checkpoint.py

@@ -33,6 +33,7 @@ from snowflake.snowpark_checkpoints.snowpark_sampler import (
 from snowflake.snowpark_checkpoints.utils.constants import (
     FAIL_STATUS,
     PASS_STATUS,
+    SKIP_STATUS,
     CheckpointMode,
 )
 from snowflake.snowpark_checkpoints.utils.extra_config import is_checkpoint_enabled
@@ -92,11 +93,10 @@ def validate_dataframe_checkpoint(
     checkpoint_name = _replace_special_characters(checkpoint_name)
 
     if not is_checkpoint_enabled(checkpoint_name):
-        LOGGER.warning(
-            "Checkpoint '%s' is disabled. Skipping DataFrame checkpoint validation.",
-            checkpoint_name,
+        raise Exception(
+            f"Checkpoint '{checkpoint_name}' is disabled. Please enable it in the checkpoints.json file.",
+            "In case you want to skip it, use the xvalidate_dataframe_checkpoint method instead.",
         )
-        return None
 
     LOGGER.info(
         "Starting DataFrame checkpoint validation for checkpoint '%s'", checkpoint_name
@@ -132,6 +132,48 @@ def validate_dataframe_checkpoint(
     )
 
 
+@log
+def xvalidate_dataframe_checkpoint(
+    df: SnowparkDataFrame,
+    checkpoint_name: str,
+    job_context: Optional[SnowparkJobContext] = None,
+    mode: Optional[CheckpointMode] = CheckpointMode.SCHEMA,
+    custom_checks: Optional[dict[Any, Any]] = None,
+    skip_checks: Optional[dict[Any, Any]] = None,
+    sample_frac: Optional[float] = 1.0,
+    sample_number: Optional[int] = None,
+    sampling_strategy: Optional[SamplingStrategy] = SamplingStrategy.RANDOM_SAMPLE,
+    output_path: Optional[str] = None,
+) -> Union[tuple[bool, PandasDataFrame], None]:
+    """Skips the validation of a Snowpark DataFrame against a specified checkpoint.
+
+    Args:
+        df (SnowparkDataFrame): The DataFrame to validate.
+        checkpoint_name (str): The name of the checkpoint to validate against.
+        job_context (SnowparkJobContext, optional): The job context for the validation. Required for PARQUET mode.
+        mode (CheckpointMode): The mode of validation (e.g., SCHEMA, PARQUET). Defaults to SCHEMA.
+        custom_checks (Optional[dict[Any, Any]], optional): Custom checks to apply during validation.
+        skip_checks (Optional[dict[Any, Any]], optional): Checks to skip during validation.
+        sample_frac (Optional[float], optional): Fraction of the DataFrame to sample for validation. Defaults to 0.1.
+        sample_number (Optional[int], optional): Number of rows to sample for validation.
+        sampling_strategy (Optional[SamplingStrategy], optional): Strategy to use for sampling.
+            Defaults to RANDOM_SAMPLE.
+        output_path (Optional[str], optional): The output path for the validation results.
+
+    Raises:
+        ValueError: If an invalid validation mode is provided or if job_context is None for PARQUET mode.
+
+    """
+    checkpoint_name = _replace_special_characters(checkpoint_name)
+
+    LOGGER.warning(
+        "Checkpoint '%s' is disabled. Skipping DataFrame checkpoint validation.",
+        checkpoint_name,
+    )
+
+    _update_validation_result(checkpoint_name, SKIP_STATUS, output_path)
+
+
 def _check_dataframe_schema_file(
     df: SnowparkDataFrame,
     checkpoint_name: str,

snowpark_checkpoints_validators-0.3.1/src/snowflake/snowpark_checkpoints/io_utils/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = ["EnvStrategy", "IOFileManager", "IODefaultStrategy"]
+
+from snowflake.snowpark_checkpoints.io_utils.io_env_strategy import (
+    EnvStrategy,
+)
+from snowflake.snowpark_checkpoints.io_utils.io_default_strategy import (
+    IODefaultStrategy,
+)
+from snowflake.snowpark_checkpoints.io_utils.io_file_manager import (
+    IOFileManager,
+)

snowpark_checkpoints_validators-0.3.1/src/snowflake/snowpark_checkpoints/io_utils/io_default_strategy.py

@@ -0,0 +1,57 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+
+from pathlib import Path
+from typing import Optional
+
+from snowflake.snowpark_checkpoints.io_utils import EnvStrategy
+
+
+class IODefaultStrategy(EnvStrategy):
+    def mkdir(self, path: str, exist_ok: bool = False) -> None:
+        os.makedirs(path, exist_ok=exist_ok)
+
+    def folder_exists(self, path: str) -> bool:
+        return os.path.isdir(path)
+
+    def file_exists(self, path: str) -> bool:
+        return os.path.isfile(path)
+
+    def write(self, file_path: str, file_content: str, overwrite: bool = True) -> None:
+        mode = "w" if overwrite else "x"
+        with open(file_path, mode) as file:
+            file.write(file_content)
+
+    def read(
+        self, file_path: str, mode: str = "r", encoding: Optional[str] = None
+    ) -> str:
+        with open(file_path, mode=mode, encoding=encoding) as file:
+            return file.read()
+
+    def read_bytes(self, file_path: str) -> bytes:
+        with open(file_path, mode="rb") as f:
+            return f.read()
+
+    def ls(self, path: str, recursive: bool = False) -> list[str]:
+        return glob.glob(path, recursive=recursive)
+
+    def getcwd(self) -> str:
+        return os.getcwd()
+
+    def telemetry_path_files(self, path: str) -> Path:
+        return Path(path)

snowpark_checkpoints_validators-0.3.1/src/snowflake/snowpark_checkpoints/io_utils/io_env_strategy.py

@@ -0,0 +1,133 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Optional
+
+
+class EnvStrategy(ABC):
+
+    """An abstract base class that defines methods for file and directory operations.
+
+    Subclasses should implement these methods to provide environment-specific behavior.
+    """
+
+    @abstractmethod
+    def mkdir(self, path: str, exist_ok: bool = False) -> None:
+        """Create a directory.
+
+        Args:
+            path: The name of the directory to create.
+            exist_ok: If False, an error is raised if the directory already exists.
+
+        """
+
+    @abstractmethod
+    def folder_exists(self, path: str) -> bool:
+        """Check if a folder exists.
+
+        Args:
+            path: The path to the folder.
+
+        Returns:
+            bool: True if the folder exists, False otherwise.
+
+        """
+
+    @abstractmethod
+    def file_exists(self, path: str) -> bool:
+        """Check if a file exists.
+
+        Args:
+            path: The path to the file.
+
+        Returns:
+            bool: True if the file exists, False otherwise.
+
+        """
+
+    @abstractmethod
+    def write(self, file_path: str, file_content: str, overwrite: bool = True) -> None:
+        """Write content to a file.
+
+        Args:
+            file_path: The name of the file to write to.
+            file_content: The content to write to the file.
+            overwrite: If True, overwrite the file if it exists.
+
+        """
+
+    @abstractmethod
+    def read(
+        self, file_path: str, mode: str = "r", encoding: Optional[str] = None
+    ) -> str:
+        """Read content from a file.
+
+        Args:
+            file_path: The path to the file to read from.
+            mode: The mode in which to open the file.
+            encoding: The encoding to use for reading the file.
+
+        Returns:
+            str: The content of the file.
+
+        """
+
+    @abstractmethod
+    def read_bytes(self, file_path: str) -> bytes:
+        """Read binary content from a file.
+
+        Args:
+            file_path: The path to the file to read from.
+
+        Returns:
+            bytes: The binary content of the file.
+
+        """
+
+    @abstractmethod
+    def ls(self, path: str, recursive: bool = False) -> list[str]:
+        """List the contents of a directory.
+
+        Args:
+            path: The path to the directory.
+            recursive: If True, list the contents recursively.
+
+        Returns:
+            list[str]: A list of the contents of the directory.
+
+        """
+
+    @abstractmethod
+    def getcwd(self) -> str:
+        """Get the current working directory.
+
+        Returns:
+            str: The current working directory.
+
+        """
+
+    @abstractmethod
+    def telemetry_path_files(self, path: str) -> Path:
+        """Get the path to the telemetry files.
+
+        Args:
+            path: The path to the telemetry directory.
+
+        Returns:
+            Path: The path object representing the telemetry files.
+
+        """

snowpark_checkpoints_validators-0.3.1/src/snowflake/snowpark_checkpoints/io_utils/io_file_manager.py

@@ -0,0 +1,76 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from typing import Optional
+
+from snowflake.snowpark_checkpoints.io_utils import (
+    EnvStrategy,
+    IODefaultStrategy,
+)
+from snowflake.snowpark_checkpoints.singleton import Singleton
+
+
+class IOFileManager(metaclass=Singleton):
+    def __init__(self, strategy: Optional[EnvStrategy] = None):
+        self.strategy = strategy or IODefaultStrategy()
+
+    def mkdir(self, path: str, exist_ok: bool = False) -> None:
+        return self.strategy.mkdir(path, exist_ok)
+
+    def folder_exists(self, path: str) -> bool:
+        return self.strategy.folder_exists(path)
+
+    def file_exists(self, path: str) -> bool:
+        return self.strategy.file_exists(path)
+
+    def write(self, file_path: str, file_content: str, overwrite: bool = True) -> None:
+        return self.strategy.write(file_path, file_content, overwrite)
+
+    def read(
+        self, file_path: str, mode: str = "r", encoding: Optional[str] = None
+    ) -> str:
+        return self.strategy.read(file_path, mode, encoding)
+
+    def read_bytes(self, file_path: str) -> bytes:
+        return self.strategy.read_bytes(file_path)
+
+    def ls(self, path: str, recursive: bool = False) -> list[str]:
+        return self.strategy.ls(path, recursive)
+
+    def getcwd(self) -> str:
+        return self.strategy.getcwd()
+
+    def telemetry_path_files(self, path: str) -> Path:
+        return self.strategy.telemetry_path_files(path)
+
+    def set_strategy(self, strategy: EnvStrategy):
+        """Set the strategy for file and directory operations.
+
+        Args:
+            strategy (EnvStrategy): The strategy to use for file and directory operations.
+
+        """
+        self.strategy = strategy
+
+
+def get_io_file_manager():
+    """Get the singleton instance of IOFileManager.
+
+    Returns:
+        IOFileManager: The singleton instance of IOFileManager.
+
+    """
+    return IOFileManager()

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/src/snowflake/snowpark_checkpoints/utils/constants.py

@@ -39,6 +39,7 @@ STACK_POSITION_CHECKPOINT: Final[int] = 6
 # Validation status
 PASS_STATUS: Final[str] = "PASS"
 FAIL_STATUS: Final[str] = "FAIL"
+SKIP_STATUS: Final[str] = "SKIP"
 
 # Validation result keys
 DEFAULT_KEY: Final[str] = "default"

{snowpark_checkpoints_validators-0.2.1 → snowpark_checkpoints_validators-0.3.1}/src/snowflake/snowpark_checkpoints/utils/extra_config.py

@@ -18,6 +18,7 @@ import os
 
 from typing import Optional
 
+from snowflake.snowpark_checkpoints.io_utils.io_file_manager import get_io_file_manager
 from snowflake.snowpark_checkpoints.utils.constants import (
     SNOWFLAKE_CHECKPOINT_CONTRACT_FILE_PATH_ENV_VAR,
 )
@@ -28,7 +29,48 @@ LOGGER = logging.getLogger(__name__)
 
 # noinspection DuplicatedCode
 def _get_checkpoint_contract_file_path() -> str:
-    return os.environ.get(SNOWFLAKE_CHECKPOINT_CONTRACT_FILE_PATH_ENV_VAR, os.getcwd())
+    return os.environ.get(
+        SNOWFLAKE_CHECKPOINT_CONTRACT_FILE_PATH_ENV_VAR, get_io_file_manager().getcwd()
+    )
+
+
+def _set_conf_io_strategy() -> None:
+    try:
+        from snowflake.snowpark_checkpoints.io_utils.io_default_strategy import (
+            IODefaultStrategy,
+        )
+        from snowflake.snowpark_checkpoints_configuration.io_utils.io_file_manager import (
+            EnvStrategy as ConfEnvStrategy,
+        )
+        from snowflake.snowpark_checkpoints_configuration.io_utils.io_file_manager import (
+            get_io_file_manager as get_conf_io_file_manager,
+        )
+
+        is_default_strategy = isinstance(
+            get_io_file_manager().strategy, IODefaultStrategy
+        )
+
+        if is_default_strategy:
+            return
+
+        class CustomConfEnvStrategy(ConfEnvStrategy):
+            def file_exists(self, path: str) -> bool:
+                return get_io_file_manager().file_exists(path)
+
+            def read(
+                self, file_path: str, mode: str = "r", encoding: Optional[str] = None
+            ) -> Optional[str]:
+                return get_io_file_manager().read(file_path, mode, encoding)
+
+            def getcwd(self) -> str:
+                return get_io_file_manager().getcwd()
+
+        get_conf_io_file_manager().set_strategy(CustomConfEnvStrategy())
+
+    except ImportError:
+        LOGGER.debug(
+            "snowpark-checkpoints-configuration is not installed. Cannot get a checkpoint metadata instance."
+        )
 
 
 # noinspection DuplicatedCode
@@ -39,6 +81,7 @@ def _get_metadata():
         )
 
         path = _get_checkpoint_contract_file_path()
+        _set_conf_io_strategy()
         LOGGER.debug("Loading checkpoint metadata from '%s'", path)
         metadata = CheckpointMetadata(path)
         return True, metadata