pointblank 0.13.4__py3-none-any.whl → 0.15.0__py3-none-any.whl

pointblank/data/validations/README.md

@@ -0,0 +1,108 @@
+# Validation Serialization Test Infrastructure
+
+This directory contains test files and utilities for ensuring serialization compatibility of pointblank validation objects across versions.
+
+## Overview
+
+The serialization functionality in pointblank allows validation objects to be saved to disk and reloaded later. To ensure this works correctly across different versions and with various types of preprocessing functions, we maintain a collection of reference validation files for regression testing.
+
+## Files
+
+### Preprocessing Functions (`preprocessing_functions.py`)
+
+Contains preprocessing functions used in validation examples:
+
+- `double_column_a()` - Simple column transformation
+- `add_computed_column()` - Creates computed columns
+- `filter_by_d_gt_100()` - Filtering operations
+- `narwhals_median_transform()` - Cross-backend compatible functions using narwhals
+- `complex_preprocessing()` - Complex multi-step transformations
+- `pandas_compatible_transform()` - Functions that work with both pandas and polars
+
+### Test File Generator (`generate_test_files.py`)
+
+Script that creates reference validation objects with various preprocessing functions:
+
+- Creates test datasets
+- Defines validation objects with different preprocessing scenarios
+- Saves both pickle (`.pkl`) and JSON (`.json`) files
+- Each validation object is interrogated to populate results
+
+### Test Cases (`tests/test_serialization_compat.py`)
+
+Comprehensive tests for serialization functionality located in the main tests directory:
+
+- **Roundtrip testing**: Pickle and unpickle validation objects
+- **Preprocessing preservation**: Verify functions are correctly serialized
+- **Cross-backend compatibility**: Test narwhals functions work after deserialization
+- **Complex workflows**: Multi-step validation with different preprocessing functions
+
+### Generated Files
+
+The following validation files are generated for regression testing:
+
+#### Basic Validation Examples
+
+- `no_preprocessing.pkl/.json` - Control case without preprocessing
+- `simple_preprocessing.pkl/.json` - Basic single-function preprocessing
+
+#### Advanced Validation Examples
+
+- `complex_preprocessing.pkl/.json` - Multi-step transformations
+- `multiple_steps.pkl/.json` - Different preprocessing per validation step
+- `narwhals_function.pkl/.json` - Cross-backend compatible functions
+- `pandas_compatible.pkl/.json` - Functions that work with multiple backends
+
+## Usage
+
+### Running Tests
+
+```bash
+# Run all serialization compatibility tests
+python -m pytest tests/test_serialization_compat.py -v
+
+# Generate new test files (if functions change)
+cd pointblank/data/validations
+python generate_test_files.py
+```
+
+### Adding New Test Cases
+
+1. Add new preprocessing functions to `preprocessing_functions.py`
+2. Update `generate_test_files.py` to create validations using the new functions
+3. Add corresponding test cases in `tests/test_serialization_compat.py`
+4. Regenerate test files: `python generate_test_files.py`
+
+## Version Compatibility
+
+These reference files serve as regression tests to ensure:
+
+- New versions can load validation files created with previous versions
+- Preprocessing functions are correctly preserved across serialization
+- Cross-backend compatibility is maintained
+- Complex workflows continue to work after deserialization
+
+The pickle files are the authoritative test cases, while JSON files provide human-readable versions for debugging.
+
+## Best Practices
+
+### For Preprocessing Functions
+
+- Always use proper function definitions (not lambdas) for serializable functions
+- Import required libraries inside functions for self-contained serialization
+- Use narwhals for cross-backend compatibility when possible
+- Test functions work with both polars and pandas DataFrames
+
+### For Test Coverage
+
+- Include examples of each type of preprocessing function
+- Test both simple and complex multi-step workflows
+- Verify roundtrip serialization (pickle → unpickle → pickle again)
+- Check that deserialized functions produce expected results
+
+### For Maintenance
+
+- Regenerate test files when adding new preprocessing function types
+- Keep test functions focused and well-documented
+- Update tests when validation object structure changes
+- Document any breaking changes that affect serialization compatibility

pointblank/data/validations/complex_preprocessing.json

@@ -0,0 +1,54 @@
+[
+    {
+        "i": 1,
+        "i_o": 1,
+        "assertion_type": "col_vals_gt",
+        "column": "a_doubled",
+        "values": 0,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": "def complex_preprocessing(df):\n    \"\"\"Complex preprocessing combining multiple operations.\"\"\"\n    return (\n        df.filter(pl.col(\"a\") > 1)\n        .with_columns((pl.col(\"a\") * 2).alias(\"a_doubled\"), (pl.col(\"d\") / 10).alias(\"d_scaled\"))\n        .filter(pl.col(\"d_scaled\") > 10)\n    )",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 7,
+        "n_passed": 7,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.706+00:00",
+        "proc_duration_s": 0.00161
+    },
+    {
+        "i": 2,
+        "i_o": 2,
+        "assertion_type": "col_vals_gt",
+        "column": "d_scaled",
+        "values": 15,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": "def complex_preprocessing(df):\n    \"\"\"Complex preprocessing combining multiple operations.\"\"\"\n    return (\n        df.filter(pl.col(\"a\") > 1)\n        .with_columns((pl.col(\"a\") * 2).alias(\"a_doubled\"), (pl.col(\"d\") / 10).alias(\"d_scaled\"))\n        .filter(pl.col(\"d_scaled\") > 10)\n    )",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": false,
+        "n": 7,
+        "n_passed": 5,
+        "n_failed": 2,
+        "f_passed": 0.7142857142857143,
+        "f_failed": 0.2857142857142857,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.708+00:00",
+        "proc_duration_s": 0.001607
+    }
+]

pointblank/data/validations/generate_test_files.py

@@ -0,0 +1,127 @@
+"""
+Generate reference validation files for serialization regression testing.
+
+This script creates validation objects with various preprocessing functions
+and stores them as pickled files in the validations directory. These files
+serve as regression tests to ensure serialization compatibility across versions.
+"""
+
+import pickle
+
+# Add the parent directory to Python path to import pointblank
+import sys
+from pathlib import Path
+
+import polars as pl
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from preprocessing_functions import (
+    add_computed_column,
+    complex_preprocessing,
+    double_column_a,
+    filter_by_d_gt_100,
+    narwhals_median_transform,
+    pandas_compatible_transform,
+)
+
+import pointblank as pb
+
+
+def create_test_data():
+    """Create a test dataset for validation examples."""
+    return pl.DataFrame(
+        {
+            "a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+            "b": [10, 20, 30, 40, 50, 60, 70, 80, 90, 100],
+            "c": ["x", "y", "x", "y", "x", "y", "x", "y", "x", "y"],
+            "d": [50, 75, 100, 125, 150, 175, 200, 225, 250, 275],
+        }
+    )
+
+
+def create_validation_examples():
+    """Create various validation objects for testing serialization."""
+    data = create_test_data()
+    validations = {}
+
+    # Basic validation with simple preprocessing
+    validations["simple_preprocessing"] = (
+        pb.Validate(data, tbl_name="test_data")
+        .col_vals_gt("a", value=0, pre=double_column_a)
+        .col_vals_in_set("c", set=["x", "y"])
+    )
+
+    # Validation with complex preprocessing
+    validations["complex_preprocessing"] = (
+        pb.Validate(data, tbl_name="test_data")
+        .col_vals_gt("a_doubled", value=0, pre=complex_preprocessing)
+        .col_vals_gt("d_scaled", value=15, pre=complex_preprocessing)
+    )
+
+    # Validation with narwhals function
+    validations["narwhals_function"] = pb.Validate(data, tbl_name="test_data").col_vals_gt(
+        "a", value=5, pre=narwhals_median_transform
+    )
+
+    # Validation with multiple preprocessing steps
+    validations["multiple_steps"] = (
+        pb.Validate(data, tbl_name="test_data")
+        .col_vals_gt("a", value=2, pre=double_column_a)
+        .col_vals_in_set("c", set=["x", "y"], pre=filter_by_d_gt_100)
+        .col_vals_gt("sum_ab", value=100, pre=add_computed_column)
+    )
+
+    # Validation with pandas-compatible function
+    validations["pandas_compatible"] = pb.Validate(data, tbl_name="test_data").col_vals_gt(
+        "a_plus_b", value=10, pre=pandas_compatible_transform
+    )
+
+    # Basic validation without preprocessing (control case)
+    validations["no_preprocessing"] = (
+        pb.Validate(data, tbl_name="test_data")
+        .col_vals_gt("a", value=0)
+        .col_vals_lt("d", value=300)
+        .col_vals_in_set("c", set=["x", "y"])
+    )
+
+    return validations
+
+
+def save_validation_files(validations, output_dir):
+    """Save validation objects as pickled files."""
+    output_path = Path(output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    for name, validation in validations.items():
+        # Interrogate to populate results
+        validation.interrogate()
+
+        # Save the validation object
+        file_path = output_path / f"{name}.pkl"
+        with open(file_path, "wb") as f:
+            pickle.dump(validation, f)
+
+        print(f"Saved {name} validation to {file_path}")
+
+        # Also save as JSON for human readability
+        json_path = output_path / f"{name}.json"
+        try:
+            json_report = validation.get_json_report()
+            with open(json_path, "w") as f:
+                f.write(json_report)
+            print(f"Saved {name} validation JSON to {json_path}")
+        except Exception as e:
+            print(f"Could not save JSON for {name}: {e}")
+
+
+if __name__ == "__main__":
+    # Create validation examples
+    validations = create_validation_examples()
+
+    # Save to the validations directory
+    output_dir = Path(__file__).parent
+    save_validation_files(validations, output_dir)
+
+    print(f"\nCreated {len(validations)} test validation files in {output_dir}")
+    print("These files can be used for regression testing serialization compatibility.")

pointblank/data/validations/multiple_steps.json

@@ -0,0 +1,83 @@
+[
+    {
+        "i": 1,
+        "i_o": 1,
+        "assertion_type": "col_vals_gt",
+        "column": "a",
+        "values": 2,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": "def double_column_a(df):\n    \"\"\"Double the values in column 'a'.\"\"\"\n    return df.with_columns(pl.col(\"a\") * 2)",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": false,
+        "n": 10,
+        "n_passed": 9,
+        "n_failed": 1,
+        "f_passed": 0.9,
+        "f_failed": 0.1,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.712+00:00",
+        "proc_duration_s": 0.00152
+    },
+    {
+        "i": 2,
+        "i_o": 2,
+        "assertion_type": "col_vals_in_set",
+        "column": "c",
+        "values": [
+            "x",
+            "y"
+        ],
+        "inclusive": null,
+        "na_pass": null,
+        "pre": "def filter_by_d_gt_100(df):\n    \"\"\"Filter rows where column 'd' is greater than 100.\"\"\"\n    return df.filter(pl.col(\"d\") > 100)",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 7,
+        "n_passed": 7,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.713+00:00",
+        "proc_duration_s": 0.001
+    },
+    {
+        "i": 3,
+        "i_o": 3,
+        "assertion_type": "col_vals_gt",
+        "column": "sum_ab",
+        "values": 100,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": "def add_computed_column(df):\n    \"\"\"Add a computed column based on existing columns.\"\"\"\n    return df.with_columns((pl.col(\"a\") + pl.col(\"b\")).alias(\"sum_ab\"))",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": false,
+        "n": 10,
+        "n_passed": 1,
+        "n_failed": 9,
+        "f_passed": 0.1,
+        "f_failed": 0.9,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.714+00:00",
+        "proc_duration_s": 0.001464
+    }
+]

pointblank/data/validations/narwhals_function.json

@@ -0,0 +1,28 @@
+[
+    {
+        "i": 1,
+        "i_o": 1,
+        "assertion_type": "col_vals_gt",
+        "column": "a",
+        "values": 5,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": "def narwhals_median_transform(df):\n    \"\"\"Use narwhals to compute median - cross-backend compatible.\"\"\"\n    return nw.from_native(df).select(nw.median(\"a\"), nw.median(\"d\"))",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 1,
+        "n_passed": 1,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.710+00:00",
+        "proc_duration_s": 0.001455
+    }
+]

pointblank/data/validations/no_preprocessing.json

@@ -0,0 +1,83 @@
+[
+    {
+        "i": 1,
+        "i_o": 1,
+        "assertion_type": "col_vals_gt",
+        "column": "a",
+        "values": 0,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": null,
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 10,
+        "n_passed": 10,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.718+00:00",
+        "proc_duration_s": 0.001148
+    },
+    {
+        "i": 2,
+        "i_o": 2,
+        "assertion_type": "col_vals_lt",
+        "column": "d",
+        "values": 300,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": null,
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 10,
+        "n_passed": 10,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.719+00:00",
+        "proc_duration_s": 0.001181
+    },
+    {
+        "i": 3,
+        "i_o": 3,
+        "assertion_type": "col_vals_in_set",
+        "column": "c",
+        "values": [
+            "x",
+            "y"
+        ],
+        "inclusive": null,
+        "na_pass": null,
+        "pre": null,
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 10,
+        "n_passed": 10,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.720+00:00",
+        "proc_duration_s": 0.000892
+    }
+]

pointblank/data/validations/pandas_compatible.json

@@ -0,0 +1,28 @@
+[
+    {
+        "i": 1,
+        "i_o": 1,
+        "assertion_type": "col_vals_gt",
+        "column": "a_plus_b",
+        "values": 10,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": "def pandas_compatible_transform(df):\n    \"\"\"Transform that works with pandas DataFrames.\"\"\"\n    if hasattr(df, \"assign\"):  # pandas\n        return df.assign(a_plus_b=df[\"a\"] + df.get(\"b\", 0))\n    else:  # polars or other\n        return df.with_columns((pl.col(\"a\") + pl.col(\"b\")).alias(\"a_plus_b\"))",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 10,
+        "n_passed": 10,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.717+00:00",
+        "proc_duration_s": 0.001428
+    }
+]

pointblank/data/validations/preprocessing_functions.py

@@ -0,0 +1,46 @@
+"""
+Test preprocessing functions for validation serialization examples.
+
+These functions are used to create validation objects that can be serialized
+and stored as reference files for regression testing.
+"""
+
+import narwhals as nw
+import polars as pl
+
+
+def double_column_a(df):
+    """Double the values in column 'a'."""
+    return df.with_columns(pl.col("a") * 2)
+
+
+def add_computed_column(df):
+    """Add a computed column based on existing columns."""
+    return df.with_columns((pl.col("a") + pl.col("b")).alias("sum_ab"))
+
+
+def filter_by_d_gt_100(df):
+    """Filter rows where column 'd' is greater than 100."""
+    return df.filter(pl.col("d") > 100)
+
+
+def narwhals_median_transform(df):
+    """Use narwhals to compute median - cross-backend compatible."""
+    return nw.from_native(df).select(nw.median("a"), nw.median("d"))
+
+
+def complex_preprocessing(df):
+    """Complex preprocessing combining multiple operations."""
+    return (
+        df.filter(pl.col("a") > 1)
+        .with_columns((pl.col("a") * 2).alias("a_doubled"), (pl.col("d") / 10).alias("d_scaled"))
+        .filter(pl.col("d_scaled") > 10)
+    )
+
+
+def pandas_compatible_transform(df):
+    """Transform that works with pandas DataFrames."""
+    if hasattr(df, "assign"):  # pandas
+        return df.assign(a_plus_b=df["a"] + df.get("b", 0))
+    else:  # polars or other
+        return df.with_columns((pl.col("a") + pl.col("b")).alias("a_plus_b"))

pointblank/data/validations/simple_preprocessing.json

@@ -0,0 +1,57 @@
+[
+    {
+        "i": 1,
+        "i_o": 1,
+        "assertion_type": "col_vals_gt",
+        "column": "a",
+        "values": 0,
+        "inclusive": null,
+        "na_pass": false,
+        "pre": "def double_column_a(df):\n    \"\"\"Double the values in column 'a'.\"\"\"\n    return df.with_columns(pl.col(\"a\") * 2)",
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 10,
+        "n_passed": 10,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.702+00:00",
+        "proc_duration_s": 0.00387
+    },
+    {
+        "i": 2,
+        "i_o": 2,
+        "assertion_type": "col_vals_in_set",
+        "column": "c",
+        "values": [
+            "x",
+            "y"
+        ],
+        "inclusive": null,
+        "na_pass": null,
+        "pre": null,
+        "segments": null,
+        "thresholds": "Thresholds(warning=None, error=None, critical=None)",
+        "label": null,
+        "brief": null,
+        "active": true,
+        "all_passed": true,
+        "n": 10,
+        "n_passed": 10,
+        "n_failed": 0,
+        "f_passed": 1.0,
+        "f_failed": 0.0,
+        "warning": null,
+        "error": null,
+        "critical": null,
+        "time_processed": "2025-10-02T04:16:44.703+00:00",
+        "proc_duration_s": 0.000983
+    }
+]

pointblank/datascan.py

@@ -143,17 +143,17 @@ class DataScan:
             for conv_method in valid_conversion_methods:
                 try:
                     valid_native = getattr(ibis_native, conv_method)()
-                except (NotImplementedError, ImportError, ModuleNotFoundError):
-                    continue
+                except (NotImplementedError, ImportError, ModuleNotFoundError):  # pragma: no cover
+                    continue  # pragma: no cover
                 break
-            else:
+            else:  # pragma: no cover
                 msg = (
                     "To use `ibis` as input, you must have one of arrow, pandas, polars or numpy "
                     "available in the process. Until `ibis` is fully supported by Narwhals, this is "
                     "necessary. Additionally, the data must be collected in order to calculate some "
                     "structural statistics, which may be performance detrimental."
                 )
-                raise ImportError(msg)
+                raise ImportError(msg)  # pragma: no cover
             as_native = nw.from_native(valid_native)
 
         self.nw_data: Frame = nw.from_native(as_native)