snowpark-checkpoints-validators 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

snowflake/snowpark_checkpoints/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.1.2"
+__version__ = "0.1.4"

snowflake/snowpark_checkpoints/checkpoint.py

@@ -14,11 +14,11 @@
 # limitations under the License.
 
 # Wrapper around pandera which logs to snowflake
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, cast
 
 from pandas import DataFrame as PandasDataFrame
-from pandera import Check, DataFrameSchema
-from pandera_report import DataFrameValidator
+from pandera import Check, DataFrameModel, DataFrameSchema
+from pandera.errors import SchemaError, SchemaErrors
 
 from snowflake.snowpark import DataFrame as SnowparkDataFrame
 from snowflake.snowpark_checkpoints.errors import SchemaValidationError
@@ -259,12 +259,7 @@ def _check_dataframe_schema(
     pandera_schema_upper, sample_df = _process_sampling(
         df, pandera_schema, job_context, sample_frac, sample_number, sampling_strategy
     )
-
-    # Raises SchemaError on validation issues
-    validator = DataFrameValidator()
-    is_valid, validation_result = validator.validate(
-        pandera_schema_upper, sample_df, validity_flag=True
-    )
+    is_valid, validation_result = _validate(pandera_schema_upper, sample_df)
     if is_valid:
         if job_context is not None:
             job_context._mark_pass(checkpoint_name)
@@ -342,10 +337,8 @@ def check_output_schema(
             sampler.process_args([snowpark_results])
             pandas_sample_args = sampler.get_sampled_pandas_args()
 
-            # Raises SchemaError on validation issues
-            validator = DataFrameValidator()
-            is_valid, validation_result = validator.validate(
-                pandera_schema, pandas_sample_args[0], validity_flag=True
+            is_valid, validation_result = _validate(
+                pandera_schema, pandas_sample_args[0]
             )
             logger = CheckpointLogger().get_logger()
             logger.info(
@@ -440,11 +433,9 @@ def check_input_schema(
             for arg in pandas_sample_args:
                 if isinstance(arg, PandasDataFrame):
 
-                    validator = DataFrameValidator()
-                    is_valid, validation_result = validator.validate(
+                    is_valid, validation_result = _validate(
                         pandera_schema,
                         arg,
-                        validity_flag=True,
                     )
 
                     logger = CheckpointLogger().get_logger()
@@ -480,3 +471,19 @@ def check_input_schema(
         return wrapper
 
     return check_input_with_decorator
+
+
+def _validate(
+    schema: Union[type[DataFrameModel], DataFrameSchema],
+    df: PandasDataFrame,
+    lazy: bool = True,
+) -> tuple[bool, PandasDataFrame]:
+    if not isinstance(schema, DataFrameSchema):
+        schema = schema.to_schema()
+    is_valid = True
+    try:
+        df = schema.validate(df, lazy=lazy)
+    except (SchemaErrors, SchemaError) as schema_errors:
+        df = cast(PandasDataFrame, schema_errors.failure_cases)
+        is_valid = False
+    return is_valid, df

snowflake/snowpark_checkpoints/utils/telemetry.py

@@ -330,11 +330,11 @@ def _generate_event(
     if sc_version is not None:
         metadata["snowpark_checkpoints_version"] = sc_version
     message = {
-        "type": event_type,
+        "event_type": event_type,
+        "type": "snowpark-checkpoints",
         "event_name": event_name,
         "driver_type": "PythonConnector",
         "driver_version": SNOWFLAKE_CONNECTOR_VERSION,
-        "source": "snowpark-checkpoints",
         "metadata": metadata,
         "data": json.dumps(parameters_info or {}),
     }

{snowpark_checkpoints_validators-0.1.2.dist-info → snowpark_checkpoints_validators-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-validators
-Version: 0.1.2
+Version: 0.1.4
 Summary: Migration tools for Snowpark
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/
@@ -26,21 +26,23 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: <3.12,>=3.9
-Requires-Dist: pandera-report==0.1.2
 Requires-Dist: pandera[io]==0.20.4
-Requires-Dist: pyspark
 Requires-Dist: snowflake-connector-python[pandas]
-Requires-Dist: snowflake-snowpark-python==1.26.0
+Requires-Dist: snowflake-snowpark-python>=1.23.0
 Provides-Extra: development
 Requires-Dist: coverage>=7.6.7; extra == 'development'
+Requires-Dist: deepdiff==8.1.1; extra == 'development'
 Requires-Dist: deepdiff>=8.0.0; extra == 'development'
 Requires-Dist: hatchling==1.25.0; extra == 'development'
 Requires-Dist: pre-commit>=4.0.1; extra == 'development'
 Requires-Dist: pyarrow>=18.0.0; extra == 'development'
+Requires-Dist: pyspark>=3.5.0; extra == 'development'
 Requires-Dist: pytest-cov>=6.0.0; extra == 'development'
 Requires-Dist: pytest>=8.3.3; extra == 'development'
 Requires-Dist: setuptools>=70.0.0; extra == 'development'
 Requires-Dist: twine==5.1.1; extra == 'development'
+Provides-Extra: pyspark
+Requires-Dist: pyspark>=3.5.0; extra == 'pyspark'
 Description-Content-Type: text/markdown
 
 # snowpark-checkpoints-validators
@@ -51,6 +53,17 @@ Description-Content-Type: text/markdown
 
 **snowpark-checkpoints-validators** is a package designed to validate Snowpark DataFrames against predefined schemas and checkpoints. This package ensures data integrity and consistency by performing schema and data validation checks at various stages of a Snowpark pipeline.
 
+---
+## Install the library
+```bash
+pip install snowpark-checkpoints-validators
+```
+This package requires PySpark to be installed in the same environment. If you do not have it, you can install PySpark alongside Snowpark Checkpoints by running the following command:
+```bash
+pip install "snowpark-checkpoints-validators[pyspark]"
+```
+---
+
 ## Features
 
 - Validate Snowpark DataFrames against predefined Pandera schemas.

{snowpark_checkpoints_validators-0.1.2.dist-info → snowpark_checkpoints_validators-0.1.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
 snowflake/snowpark_checkpoints/__init__.py,sha256=1_xzSopIHWpw1i3gQqWLN0wCfWWEefjr4cl1vl0xSdY,1211
-snowflake/snowpark_checkpoints/__version__.py,sha256=qNTBwMUtsLu0okWXwrUvl9AohG1pXd4kalMC8v10gHM,632
-snowflake/snowpark_checkpoints/checkpoint.py,sha256=-y1iWdGxYGuTWdngOEXdA59MT33PCiM7cP1s3jJs9jE,18997
+snowflake/snowpark_checkpoints/__version__.py,sha256=XLR7FIaqd--6dSDY841wZ2c8LM8IAx8L5fVTjJIHRkQ,632
+snowflake/snowpark_checkpoints/checkpoint.py,sha256=VceodgS3muZLH5uWhqmuMsvS7cTBAOWOWFCRKt2VsNg,19150
 snowflake/snowpark_checkpoints/errors.py,sha256=9KjzRf8bjDZTTNL4LeySJAwuucDOyz0Ka7EFBKWFpyg,1821
 snowflake/snowpark_checkpoints/job_context.py,sha256=7LdJ682lC8hCJOYUn-AVXq_Llv18R9oGdK2F-amYR_o,2990
 snowflake/snowpark_checkpoints/singleton.py,sha256=7AgIHQBXVRvPBBCkmBplzkdrrm-xVWf_N8svzA2vF8E,836
@@ -14,9 +14,9 @@ snowflake/snowpark_checkpoints/utils/constants.py,sha256=pgFttLDQ6fTa6obSdvivWBY
 snowflake/snowpark_checkpoints/utils/extra_config.py,sha256=pmGLYT7cu9WMKzQwcEPkgk1DMnnT1fREm45p19e79hk,2567
 snowflake/snowpark_checkpoints/utils/pandera_check_manager.py,sha256=ddTwXauuZdowIRwPMT61GWYCG4XGKOFkVyfZO49bc-8,14516
 snowflake/snowpark_checkpoints/utils/supported_types.py,sha256=GrMX2tHdSFnK7LlPbZx20UufD6Br6TNVRkkBwIxdPy0,1433
-snowflake/snowpark_checkpoints/utils/telemetry.py,sha256=zUQw9kfOgqLJYQuIW7bl2fRJjX3oEKyKjDlYQPFPHtA,31366
+snowflake/snowpark_checkpoints/utils/telemetry.py,sha256=_WOVo19BxcF6cpQDplID6BEOvgJfHTGK1JZI1-OI4uc,31370
 snowflake/snowpark_checkpoints/utils/utils_checks.py,sha256=o9HOBrDuTxSIgzZQHfsa9pMzzXRUsRAISI7L6OURouo,13528
-snowpark_checkpoints_validators-0.1.2.dist-info/METADATA,sha256=bw7FnPXUQUPZINvLZQwKJUdnRu7LeiuzG1vYp2T85Kc,11009
-snowpark_checkpoints_validators-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-snowpark_checkpoints_validators-0.1.2.dist-info/licenses/LICENSE,sha256=pmjhbh6uVhV5MBXOlou_UZgFP7CYVQITkCCdvfcS5lY,11340
-snowpark_checkpoints_validators-0.1.2.dist-info/RECORD,,
+snowpark_checkpoints_validators-0.1.4.dist-info/METADATA,sha256=L00oXPSPvCUBEJuYcAPPt08BPyKvbd3ZgPLpr3DgvqQ,11470
+snowpark_checkpoints_validators-0.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+snowpark_checkpoints_validators-0.1.4.dist-info/licenses/LICENSE,sha256=pmjhbh6uVhV5MBXOlou_UZgFP7CYVQITkCCdvfcS5lY,11340
+snowpark_checkpoints_validators-0.1.4.dist-info/RECORD,,