snowpark-checkpoints-validators 0.3.0__tar.gz → 0.3.2__tar.gz

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-validators
-Version: 0.3.0
+Version: 0.3.2
 Summary: Migration tools for Snowpark
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/
@@ -322,4 +322,37 @@ sp_dataframe = session.create_dataframe(df)
 preprocessed_dataframe = preprocessor(sp_dataframe)
 ```
 
+### Skip validation
+The method `xvalidate_dataframe_checkpoint` can be used to avoid the validation of the checkpoint defined.
+The result of the checkpoint will be `SKIP` and a message will be logged in the terminal. The method contains the same signature of `validate_dataframe_checkpoint`.
+
+#### Usage Example
+
+```python
+from snowflake.snowpark import Session
+from snowflake.snowpark_checkpoints.utils.constant import (
+    CheckpointMode,
+)
+from snowflake.snowpark_checkpoints.checkpoint import validate_dataframe_checkpoint
+from snowflake.snowpark_checkpoints.spark_migration import SamplingStrategy
+from snowflake.snowpark_checkpoints.job_context import SnowparkJobContext
+from pyspark.sql import SparkSession
+
+session = Session.builder.getOrCreate()
+job_context = SnowparkJobContext(
+    session, SparkSession.builder.getOrCreate(), "job_context", True
+)
+df = session.read.format("csv").load("data.csv")
+
+xvalidate_dataframe_checkpoint(
+    df,
+    "schema_checkpoint",
+    job_context=job_context,
+    mode=CheckpointMode.SCHEMA,
+    sample_frac=0.1,
+    sampling_strategy=SamplingStrategy.RANDOM_SAMPLE
+)
+```
+
+
 ------

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/README.md

@@ -273,4 +273,37 @@ sp_dataframe = session.create_dataframe(df)
 preprocessed_dataframe = preprocessor(sp_dataframe)
 ```
 
+### Skip validation
+The method `xvalidate_dataframe_checkpoint` can be used to avoid the validation of the checkpoint defined.
+The result of the checkpoint will be `SKIP` and a message will be logged in the terminal. The method contains the same signature of `validate_dataframe_checkpoint`.
+
+#### Usage Example
+
+```python
+from snowflake.snowpark import Session
+from snowflake.snowpark_checkpoints.utils.constant import (
+    CheckpointMode,
+)
+from snowflake.snowpark_checkpoints.checkpoint import validate_dataframe_checkpoint
+from snowflake.snowpark_checkpoints.spark_migration import SamplingStrategy
+from snowflake.snowpark_checkpoints.job_context import SnowparkJobContext
+from pyspark.sql import SparkSession
+
+session = Session.builder.getOrCreate()
+job_context = SnowparkJobContext(
+    session, SparkSession.builder.getOrCreate(), "job_context", True
+)
+df = session.read.format("csv").load("data.csv")
+
+xvalidate_dataframe_checkpoint(
+    df,
+    "schema_checkpoint",
+    job_context=job_context,
+    mode=CheckpointMode.SCHEMA,
+    sample_frac=0.1,
+    sampling_strategy=SamplingStrategy.RANDOM_SAMPLE
+)
+```
+
+
 ------

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/src/snowflake/snowpark_checkpoints/__init__.py

@@ -27,6 +27,7 @@ from snowflake.snowpark_checkpoints.checkpoint import (
     check_input_schema,
     check_output_schema,
     validate_dataframe_checkpoint,
+    xvalidate_dataframe_checkpoint,
 )
 from snowflake.snowpark_checkpoints.job_context import SnowparkJobContext
 from snowflake.snowpark_checkpoints.spark_migration import check_with_spark
@@ -39,6 +40,7 @@ __all__ = [
     "check_dataframe_schema",
     "check_output_schema",
     "check_input_schema",
+    "xvalidate_dataframe_checkpoint",
     "validate_dataframe_checkpoint",
     "CheckpointMode",
 ]

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/src/snowflake/snowpark_checkpoints/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/src/snowflake/snowpark_checkpoints/checkpoint.py

@@ -33,6 +33,7 @@ from snowflake.snowpark_checkpoints.snowpark_sampler import (
 from snowflake.snowpark_checkpoints.utils.constants import (
     FAIL_STATUS,
     PASS_STATUS,
+    SKIP_STATUS,
     CheckpointMode,
 )
 from snowflake.snowpark_checkpoints.utils.extra_config import is_checkpoint_enabled
@@ -92,11 +93,10 @@ def validate_dataframe_checkpoint(
     checkpoint_name = _replace_special_characters(checkpoint_name)
 
     if not is_checkpoint_enabled(checkpoint_name):
-        LOGGER.warning(
-            "Checkpoint '%s' is disabled. Skipping DataFrame checkpoint validation.",
-            checkpoint_name,
+        raise Exception(
+            f"Checkpoint '{checkpoint_name}' is disabled. Please enable it in the checkpoints.json file.",
+            "In case you want to skip it, use the xvalidate_dataframe_checkpoint method instead.",
         )
-        return None
 
     LOGGER.info(
         "Starting DataFrame checkpoint validation for checkpoint '%s'", checkpoint_name
@@ -132,6 +132,48 @@ def validate_dataframe_checkpoint(
     )
 
 
+@log
+def xvalidate_dataframe_checkpoint(
+    df: SnowparkDataFrame,
+    checkpoint_name: str,
+    job_context: Optional[SnowparkJobContext] = None,
+    mode: Optional[CheckpointMode] = CheckpointMode.SCHEMA,
+    custom_checks: Optional[dict[Any, Any]] = None,
+    skip_checks: Optional[dict[Any, Any]] = None,
+    sample_frac: Optional[float] = 1.0,
+    sample_number: Optional[int] = None,
+    sampling_strategy: Optional[SamplingStrategy] = SamplingStrategy.RANDOM_SAMPLE,
+    output_path: Optional[str] = None,
+) -> Union[tuple[bool, PandasDataFrame], None]:
+    """Skips the validation of a Snowpark DataFrame against a specified checkpoint.
+
+    Args:
+        df (SnowparkDataFrame): The DataFrame to validate.
+        checkpoint_name (str): The name of the checkpoint to validate against.
+        job_context (SnowparkJobContext, optional): The job context for the validation. Required for PARQUET mode.
+        mode (CheckpointMode): The mode of validation (e.g., SCHEMA, PARQUET). Defaults to SCHEMA.
+        custom_checks (Optional[dict[Any, Any]], optional): Custom checks to apply during validation.
+        skip_checks (Optional[dict[Any, Any]], optional): Checks to skip during validation.
+        sample_frac (Optional[float], optional): Fraction of the DataFrame to sample for validation. Defaults to 0.1.
+        sample_number (Optional[int], optional): Number of rows to sample for validation.
+        sampling_strategy (Optional[SamplingStrategy], optional): Strategy to use for sampling.
+            Defaults to RANDOM_SAMPLE.
+        output_path (Optional[str], optional): The output path for the validation results.
+
+    Raises:
+        ValueError: If an invalid validation mode is provided or if job_context is None for PARQUET mode.
+
+    """
+    checkpoint_name = _replace_special_characters(checkpoint_name)
+
+    LOGGER.warning(
+        "Checkpoint '%s' is disabled. Skipping DataFrame checkpoint validation.",
+        checkpoint_name,
+    )
+
+    _update_validation_result(checkpoint_name, SKIP_STATUS, output_path)
+
+
 def _check_dataframe_schema_file(
     df: SnowparkDataFrame,
     checkpoint_name: str,

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/src/snowflake/snowpark_checkpoints/utils/constants.py

@@ -39,6 +39,7 @@ STACK_POSITION_CHECKPOINT: Final[int] = 6
 # Validation status
 PASS_STATUS: Final[str] = "PASS"
 FAIL_STATUS: Final[str] = "FAIL"
+SKIP_STATUS: Final[str] = "SKIP"
 
 # Validation result keys
 DEFAULT_KEY: Final[str] = "default"

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/test/integ/test_parquet.py

@@ -408,15 +408,17 @@ def test_validate_dataframe_checkpoint_disabled_checkpoint(
 ):
     mock_is_checkpoint_enabled.return_value = False
     caplog.set_level(level=logging.WARNING, logger=LOGGER_NAME)
-
+    expected_exception_error_msg = "Checkpoint 'test_checkpoint' is disabled. Please enable it in the checkpoints.json file."
+    expected_fix_suggestion_msg = "In case you want to skip it, use the xvalidate_dataframe_checkpoint method instead."
     df = MagicMock()
     checkpoint_name = "test_checkpoint"
-    result = validate_dataframe_checkpoint(df=df, checkpoint_name=checkpoint_name)
-
-    mock_is_checkpoint_enabled.assert_called_once_with(checkpoint_name)
-    assert result is None
-    assert "disabled" in caplog.text
-    assert checkpoint_name in caplog.text
+    try:
+        validate_dataframe_checkpoint(df=df, checkpoint_name=checkpoint_name)
+    except Exception as error:
+        error_msg = error.args[0]
+        fix_suggestion_msg = error.args[1]
+        assert error_msg == expected_exception_error_msg
+        assert fix_suggestion_msg == expected_fix_suggestion_msg
 
 
 def test_io_strategy(

snowpark_checkpoints_validators-0.3.2/test/unit/test_checkpoints.py

@@ -0,0 +1,78 @@
+# Copyright 2025 Snowflake Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import get_type_hints
+from unittest.mock import patch
+
+from snowflake.snowpark_checkpoints.checkpoint import (
+    validate_dataframe_checkpoint,
+    xvalidate_dataframe_checkpoint,
+)
+from snowflake.snowpark_checkpoints.utils.constants import (
+    SKIP_STATUS,
+)
+from unittest.mock import MagicMock
+from snowflake.snowpark_checkpoints.validation_result_metadata import (
+    ValidationResultsMetadata,
+)
+
+
+def test_xvalidate_dataframe_checkpoint():
+    checkpoint_name = "my_checkpoint"
+    module_name = "snowflake.snowpark_checkpoints.validation_result_metadata"
+    validation_results_metadata = ValidationResultsMetadata("some/dummy/path")
+    with (
+        patch(
+            f"{module_name}.ValidationResultsMetadata",
+            return_value=validation_results_metadata,
+        )
+    ):
+        pyspark_df = MagicMock()
+        checkpoint_name = "my_checkpoint"
+
+        xvalidate_dataframe_checkpoint(pyspark_df, checkpoint_name)
+
+        validation_results = validation_results_metadata.validation_results
+        validation_results = validation_results.model_dump()
+        my_checkpoint_result = validation_results["results"][0]
+
+        assert my_checkpoint_result["checkpoint_name"] == checkpoint_name
+        assert my_checkpoint_result["result"] == SKIP_STATUS
+
+
+def test_skip_validator_parameters_commutability():
+    validation_hints = get_type_hints(validate_dataframe_checkpoint)
+    x_validation_hints = get_type_hints(xvalidate_dataframe_checkpoint)
+
+    validate_params = {
+        name: hint for name, hint in validation_hints.items() if name != "return"
+    }
+    x_validate_params = {
+        name: hint for name, hint in x_validation_hints.items() if name != "return"
+    }
+    assert (
+        validate_params == x_validate_params
+    ), "The parameters of validate_dataframe_checkpoint and xvalidate_dataframe_checkpoint must be the same."
+
+
+def test_skip_validator_return_type_commutability():
+    validation_hints = get_type_hints(validate_dataframe_checkpoint)
+    x_validation_hints = get_type_hints(xvalidate_dataframe_checkpoint)
+
+    validate_return = validation_hints.get("return")
+    x_validate_return = x_validation_hints.get("return")
+    assert (
+        validate_return == x_validate_return
+    ), "The return type of validate_dataframe_checkpoint and xvalidate_dataframe_checkpoint must be the same."

{snowpark_checkpoints_validators-0.3.0 → snowpark_checkpoints_validators-0.3.2}/test/unit/test_utils_checks.py

@@ -32,13 +32,10 @@ from snowflake.snowpark_checkpoints.utils.constants import (
     FLOAT_TYPE,
     MEAN_KEY,
     NAME_KEY,
-    NULL_COUNT_KEY,
     OVERWRITE_MODE,
     PASS_STATUS,
-    ROWS_COUNT_KEY,
     SNOWPARK_CHECKPOINTS_OUTPUT_DIRECTORY_NAME,
     TYPE_KEY,
-    SKIP_ALL,
     MARGIN_ERROR_KEY,
 )
 from pandera import Column, Check, DataFrameSchema
@@ -477,3 +474,13 @@ def test_replace_special_characters_valid(name, expected):
 def test_replace_special_characters_invalid(name):
     with raises(ValueError):
         _replace_special_characters(name)
+
+
+def test_invalid_xvalidate_import():
+    try:
+        from snowflake.snowpark_checkpoints import xvalidate_dataframe_checkpoint
+    except ImportError as e:
+        assert (
+            str(e)
+            == "xvalidate_dataframe_checkpoint is not available in the specified package."
+        )