snowpark-checkpoints-validators 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

snowflake/snowpark_checkpoints/__init__.py

@@ -39,6 +39,7 @@ __all__ = [
     "check_dataframe_schema",
     "check_output_schema",
     "check_input_schema",
+    "xvalidate_dataframe_checkpoint",
     "validate_dataframe_checkpoint",
     "CheckpointMode",
 ]

snowflake/snowpark_checkpoints/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

snowflake/snowpark_checkpoints/checkpoint.py

@@ -33,6 +33,7 @@ from snowflake.snowpark_checkpoints.snowpark_sampler import (
 from snowflake.snowpark_checkpoints.utils.constants import (
     FAIL_STATUS,
     PASS_STATUS,
+    SKIP_STATUS,
     CheckpointMode,
 )
 from snowflake.snowpark_checkpoints.utils.extra_config import is_checkpoint_enabled
@@ -92,11 +93,10 @@ def validate_dataframe_checkpoint(
     checkpoint_name = _replace_special_characters(checkpoint_name)
 
     if not is_checkpoint_enabled(checkpoint_name):
-        LOGGER.warning(
-            "Checkpoint '%s' is disabled. Skipping DataFrame checkpoint validation.",
-            checkpoint_name,
+        raise Exception(
+            f"Checkpoint '{checkpoint_name}' is disabled. Please enable it in the checkpoints.json file.",
+            "In case you want to skip it, use the xvalidate_dataframe_checkpoint method instead.",
         )
-        return None
 
     LOGGER.info(
         "Starting DataFrame checkpoint validation for checkpoint '%s'", checkpoint_name
@@ -132,6 +132,48 @@ def validate_dataframe_checkpoint(
     )
 
 
+@log
+def xvalidate_dataframe_checkpoint(
+    df: SnowparkDataFrame,
+    checkpoint_name: str,
+    job_context: Optional[SnowparkJobContext] = None,
+    mode: Optional[CheckpointMode] = CheckpointMode.SCHEMA,
+    custom_checks: Optional[dict[Any, Any]] = None,
+    skip_checks: Optional[dict[Any, Any]] = None,
+    sample_frac: Optional[float] = 1.0,
+    sample_number: Optional[int] = None,
+    sampling_strategy: Optional[SamplingStrategy] = SamplingStrategy.RANDOM_SAMPLE,
+    output_path: Optional[str] = None,
+) -> Union[tuple[bool, PandasDataFrame], None]:
+    """Skips the validation of a Snowpark DataFrame against a specified checkpoint.
+
+    Args:
+        df (SnowparkDataFrame): The DataFrame to validate.
+        checkpoint_name (str): The name of the checkpoint to validate against.
+        job_context (SnowparkJobContext, optional): The job context for the validation. Required for PARQUET mode.
+        mode (CheckpointMode): The mode of validation (e.g., SCHEMA, PARQUET). Defaults to SCHEMA.
+        custom_checks (Optional[dict[Any, Any]], optional): Custom checks to apply during validation.
+        skip_checks (Optional[dict[Any, Any]], optional): Checks to skip during validation.
+        sample_frac (Optional[float], optional): Fraction of the DataFrame to sample for validation. Defaults to 0.1.
+        sample_number (Optional[int], optional): Number of rows to sample for validation.
+        sampling_strategy (Optional[SamplingStrategy], optional): Strategy to use for sampling.
+            Defaults to RANDOM_SAMPLE.
+        output_path (Optional[str], optional): The output path for the validation results.
+
+    Raises:
+        ValueError: If an invalid validation mode is provided or if job_context is None for PARQUET mode.
+
+    """
+    checkpoint_name = _replace_special_characters(checkpoint_name)
+
+    LOGGER.warning(
+        "Checkpoint '%s' is disabled. Skipping DataFrame checkpoint validation.",
+        checkpoint_name,
+    )
+
+    _update_validation_result(checkpoint_name, SKIP_STATUS, output_path)
+
+
 def _check_dataframe_schema_file(
     df: SnowparkDataFrame,
     checkpoint_name: str,

snowflake/snowpark_checkpoints/utils/constants.py

@@ -39,6 +39,7 @@ STACK_POSITION_CHECKPOINT: Final[int] = 6
 # Validation status
 PASS_STATUS: Final[str] = "PASS"
 FAIL_STATUS: Final[str] = "FAIL"
+SKIP_STATUS: Final[str] = "SKIP"
 
 # Validation result keys
 DEFAULT_KEY: Final[str] = "default"

{snowpark_checkpoints_validators-0.3.0.dist-info → snowpark_checkpoints_validators-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-validators
-Version: 0.3.0
+Version: 0.3.1
 Summary: Migration tools for Snowpark
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/
@@ -322,4 +322,37 @@ sp_dataframe = session.create_dataframe(df)
 preprocessed_dataframe = preprocessor(sp_dataframe)
 ```
 
+### Skip validation
+The method `xvalidate_dataframe_checkpoint` can be used to avoid the validation of the checkpoint defined.
+The result of the checkpoint will be `SKIP` and a message will be logged in the terminal. The method contains the same signature of `validate_dataframe_checkpoint`.
+
+#### Usage Example
+
+```python
+from snowflake.snowpark import Session
+from snowflake.snowpark_checkpoints.utils.constant import (
+    CheckpointMode,
+)
+from snowflake.snowpark_checkpoints.checkpoint import validate_dataframe_checkpoint
+from snowflake.snowpark_checkpoints.spark_migration import SamplingStrategy
+from snowflake.snowpark_checkpoints.job_context import SnowparkJobContext
+from pyspark.sql import SparkSession
+
+session = Session.builder.getOrCreate()
+job_context = SnowparkJobContext(
+    session, SparkSession.builder.getOrCreate(), "job_context", True
+)
+df = session.read.format("csv").load("data.csv")
+
+xvalidate_dataframe_checkpoint(
+    df,
+    "schema_checkpoint",
+    job_context=job_context,
+    mode=CheckpointMode.SCHEMA,
+    sample_frac=0.1,
+    sampling_strategy=SamplingStrategy.RANDOM_SAMPLE
+)
+```
+
+
 ------

{snowpark_checkpoints_validators-0.3.0.dist-info → snowpark_checkpoints_validators-0.3.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-snowflake/snowpark_checkpoints/__init__.py,sha256=p7fzH3f8foD5nhNJHZ00JT3ODTXJGGkWTd3xRKx-8aQ,1435
-snowflake/snowpark_checkpoints/__version__.py,sha256=kbbDnlkY7JOLNHvfWYkCO_mOBOV9GniMGdxYoQpLhyg,632
-snowflake/snowpark_checkpoints/checkpoint.py,sha256=i-iDRYbGvQHy9ipW7UxHVhJhQ9BXNSO-bsCcHyg3oLA,22056
+snowflake/snowpark_checkpoints/__init__.py,sha256=V2HtQkoek-2twos_Qit-ZRS9FrSsbQ58nVf0uuGnyyk,1473
+snowflake/snowpark_checkpoints/__version__.py,sha256=uSRs7fRupFeQ-z3PtU_6qh6ry8YBaSAnEIAvLhJKUR8,632
+snowflake/snowpark_checkpoints/checkpoint.py,sha256=pU-HdpoS4SYzJU0qEaFzS5QBUE8K55Sn8K27zJe9_xM,24187
 snowflake/snowpark_checkpoints/errors.py,sha256=9KjzRf8bjDZTTNL4LeySJAwuucDOyz0Ka7EFBKWFpyg,1821
 snowflake/snowpark_checkpoints/job_context.py,sha256=RMK0g0HrbDVrOAvai4PgsGvsAn_GIo9aFmh-tWlyieY,4183
 snowflake/snowpark_checkpoints/singleton.py,sha256=7AgIHQBXVRvPBBCkmBplzkdrrm-xVWf_N8svzA2vF8E,836
@@ -13,14 +13,14 @@ snowflake/snowpark_checkpoints/io_utils/io_default_strategy.py,sha256=VMfdqj4uDg
 snowflake/snowpark_checkpoints/io_utils/io_env_strategy.py,sha256=ltG_rxm0CkJFXpskOf__ByZw-C6B9LtycqlyB9EmaJI,3569
 snowflake/snowpark_checkpoints/io_utils/io_file_manager.py,sha256=YHrxRBzTlhIUrSFrsoWkRY_Qa-TXgDWglr00T98Tc5g,2485
 snowflake/snowpark_checkpoints/utils/__init__.py,sha256=I4srmZ8G1q9DU6Suo1S91aVfNvETyisKH95uvLAvEJ0,609
-snowflake/snowpark_checkpoints/utils/constants.py,sha256=pgFttLDQ6fTa6obSdvivWBYClS21ap41YVDNGAS4sxY,4146
+snowflake/snowpark_checkpoints/utils/constants.py,sha256=M3vLdvKiVOhHMo0oPu4P42Wn_v6UDqmK6wHOGuoG6sY,4179
 snowflake/snowpark_checkpoints/utils/extra_config.py,sha256=xOYaG6MfsUCAHI0C_7qWF_m96xcLIZWwrgxY4UlpaZI,4325
 snowflake/snowpark_checkpoints/utils/logging_utils.py,sha256=yyi6X5DqKeTg0HRhvsH6ymYp2P0wbnyKIzI2RzrQS7k,2278
 snowflake/snowpark_checkpoints/utils/pandera_check_manager.py,sha256=tQIozLO-2kM8WZ-gGKfRwmXBx1cDPaIZB0qIcArp8xA,16100
 snowflake/snowpark_checkpoints/utils/supported_types.py,sha256=GrMX2tHdSFnK7LlPbZx20UufD6Br6TNVRkkBwIxdPy0,1433
 snowflake/snowpark_checkpoints/utils/telemetry.py,sha256=GfuyIaI3QG4a4_qWwyJHvWRM0GENunNexuEJ6IgscF4,32684
 snowflake/snowpark_checkpoints/utils/utils_checks.py,sha256=oQ1c4n-uAA2kFIpWIRPWhbCW8e-wwOIL8qDqLvr5Fok,14398
-snowpark_checkpoints_validators-0.3.0.dist-info/METADATA,sha256=RbOlEHK5kumiBPP2S7-7k7zxzzLYag7Yb6TtQeOYbV0,11557
-snowpark_checkpoints_validators-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-snowpark_checkpoints_validators-0.3.0.dist-info/licenses/LICENSE,sha256=pmjhbh6uVhV5MBXOlou_UZgFP7CYVQITkCCdvfcS5lY,11340
-snowpark_checkpoints_validators-0.3.0.dist-info/RECORD,,
+snowpark_checkpoints_validators-0.3.1.dist-info/METADATA,sha256=mtI8xnknt0g9McBdjcbNqOHRCDlJs2GtB1A1zcuH_00,12676
+snowpark_checkpoints_validators-0.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+snowpark_checkpoints_validators-0.3.1.dist-info/licenses/LICENSE,sha256=pmjhbh6uVhV5MBXOlou_UZgFP7CYVQITkCCdvfcS5lY,11340
+snowpark_checkpoints_validators-0.3.1.dist-info/RECORD,,