patito 0.5.1__py3-none-any.whl → 0.6.2__py3-none-any.whl

patito/validators.py

@@ -1,31 +1,35 @@
 """Module for validating datastructures with respect to model specifications."""
+
 from __future__ import annotations
 
-import sys
-from typing import TYPE_CHECKING, Type, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Optional,
+    Sequence,
+    Type,
+    Union,
+    _UnionGenericAlias,
+    cast,
+)
 
 import polars as pl
-from typing_extensions import get_args, get_origin
+from pydantic.aliases import AliasGenerator
+from typing_extensions import get_args
 
+from patito._pydantic.dtypes import is_optional
 from patito.exceptions import (
     ColumnDTypeError,
+    DataFrameValidationError,
     ErrorWrapper,
     MissingColumnsError,
     MissingValuesError,
     RowValueError,
-    SuperflousColumnsError,
-    ValidationError,
+    SuperfluousColumnsError,
 )
 
-if sys.version_info >= (3, 10):  # pragma: no cover
-    from types import UnionType  # pyright: ignore
-
-    UNION_TYPES = (Union, UnionType)
-else:
-    UNION_TYPES = (Union,)  # pragma: no cover
-
 try:
-    import pandas as pd
+    import pandas as pd  # type: ignore
 
     _PANDAS_AVAILABLE = True
 except ImportError:
@@ -38,7 +42,7 @@ if TYPE_CHECKING:
 VALID_POLARS_TYPES = {
     "enum": {pl.Categorical},
     "boolean": {pl.Boolean},
-    "string": {pl.Utf8, pl.Datetime, pl.Date},
+    "string": {pl.String, pl.Datetime, pl.Date},
     "number": {pl.Float32, pl.Float64},
     "integer": {
         pl.Int8,
@@ -53,24 +57,8 @@ VALID_POLARS_TYPES = {
 }
 
 
-def _is_optional(type_annotation: Type) -> bool:
-    """
-    Return True if the given type annotation is an Optional annotation.
-
-    Args:
-        type_annotation: The type annotation to be checked.
-
-    Returns:
-        True if the outermost type is Optional.
-    """
-    return (get_origin(type_annotation) in UNION_TYPES) and (
-        type(None) in get_args(type_annotation)
-    )
-
-
-def _dewrap_optional(type_annotation: Type) -> Type:
-    """
-    Return the inner, wrapped type of an Optional.
+def _dewrap_optional(type_annotation: Type[Any] | Any) -> Type:
+    """Return the inner, wrapped type of an Optional.
 
     Is a no-op for non-Optional types.
 
@@ -79,6 +67,7 @@ def _dewrap_optional(type_annotation: Type) -> Type:
 
     Returns:
         The input type, but with the outermost Optional removed.
+
     """
     return (
         next(  # pragma: no cover
@@ -86,54 +75,94 @@ def _dewrap_optional(type_annotation: Type) -> Type:
             for valid_type in get_args(type_annotation)
             if valid_type is not type(None)  # noqa: E721
         )
-        if _is_optional(type_annotation)
+        if is_optional(type_annotation)
         else type_annotation
     )
 
 
+def _transform_df(dataframe: pl.DataFrame, schema: type[Model]) -> pl.DataFrame:
+    """Transform any properties of the dataframe according to the model.
+
+    Currently only supports using AliasGenerator to transform column names to match a model.
+
+    Args:
+        dataframe: Polars DataFrame to be validated.
+        schema: Patito model which specifies how the dataframe should be structured.
+
+    """
+    # Check if an alias generator is present in model_config
+    if alias_gen := schema.model_config.get("alias_generator"):
+        if isinstance(alias_gen, AliasGenerator):
+            alias_func = alias_gen.validation_alias or alias_gen.alias
+            assert (
+                alias_func is not None
+            ), "An AliasGenerator must contain a transforming function"
+        else:  # alias_gen is a function
+            alias_func = alias_gen
+
+        new_cols: list[str] = [
+            alias_func(field_name) for field_name in dataframe.columns
+        ]  # type: ignore
+        dataframe.columns = new_cols
+    return dataframe
+
+
 def _find_errors(  # noqa: C901
     dataframe: pl.DataFrame,
     schema: Type[Model],
+    columns: Optional[Sequence[str]] = None,
+    allow_missing_columns: bool = False,
+    allow_superfluous_columns: bool = False,
 ) -> list[ErrorWrapper]:
-    """
-    Validate the given dataframe.
+    """Validate the given dataframe.
 
     Args:
         dataframe: Polars DataFrame to be validated.
         schema: Patito model which specifies how the dataframe should be structured.
+        columns: If specified, only validate the given columns. Missing columns will
+            check if any specified columns are missing from the inputted dataframe,
+            and superfluous columns will check if any columns not specified in the
+            schema are present in the columns list.
+        allow_missing_columns: If True, missing columns will not be considered an error.
+        allow_superfluous_columns: If True, additional columns will not be considered an error.
 
     Returns:
         A list of patito.exception.ErrorWrapper instances. The specific validation
         error can be retrieved from the "exc" attribute on each error wrapper instance.
 
         MissingColumnsError: If there are any missing columns.
-        SuperflousColumnsError: If there are additional, non-specified columns.
+        SuperfluousColumnsError: If there are additional, non-specified columns.
         MissingValuesError: If there are nulls in a non-optional column.
         ColumnDTypeError: If any column has the wrong dtype.
         NotImplementedError: If validation has not been implement for the given
             type.
+
     """
     errors: list[ErrorWrapper] = []
-    # Check if any columns are missing
-    for missig_column in set(schema.columns) - set(dataframe.columns):
-        errors.append(
-            ErrorWrapper(
-                MissingColumnsError("Missing column"),
-                loc=missig_column,
+    schema_subset = columns or schema.columns
+    column_subset = columns or dataframe.columns
+    if not allow_missing_columns:
+        # Check if any columns are missing
+        for missing_column in set(schema_subset) - set(dataframe.columns):
+            errors.append(
+                ErrorWrapper(
+                    MissingColumnsError("Missing column"),
+                    loc=missing_column,
+                )
             )
-        )
 
-    # Check if any additional columns are included
-    for superflous_column in set(dataframe.columns) - set(schema.columns):
-        errors.append(
-            ErrorWrapper(
-                SuperflousColumnsError("Superflous column"),
-                loc=superflous_column,
+    if not allow_superfluous_columns:
+        # Check if any additional columns are included
+        for superfluous_column in set(column_subset) - set(schema.columns):
+            errors.append(
+                ErrorWrapper(
+                    SuperfluousColumnsError("Superfluous column"),
+                    loc=superfluous_column,
+                )
             )
-        )
 
     # Check if any non-optional columns have null values
-    for column in schema.non_nullable_columns.intersection(dataframe.columns):
+    for column in schema.non_nullable_columns.intersection(column_subset):
         num_missing_values = dataframe.get_column(name=column).null_count()
         if num_missing_values:
             errors.append(
@@ -147,10 +176,12 @@ def _find_errors(  # noqa: C901
             )
 
     for column, dtype in schema.dtypes.items():
+        if column not in column_subset:
+            continue
         if not isinstance(dtype, pl.List):
             continue
 
-        annotation = schema.__annotations__[column]  # type: ignore[unreachable]
+        annotation = schema.model_fields[column].annotation  # type: ignore[unreachable]
 
         # Retrieve the annotation of the list itself,
         # dewrapping any potential Optional[...]
@@ -158,7 +189,7 @@ def _find_errors(  # noqa: C901
 
         # Check if the list items themselves should be considered nullable
         item_type = get_args(list_type)[0]
-        if _is_optional(item_type):
+        if is_optional(item_type):
             continue
 
         num_missing_values = (
@@ -189,7 +220,8 @@ def _find_errors(  # noqa: C901
     valid_dtypes = schema.valid_dtypes
     dataframe_datatypes = dict(zip(dataframe.columns, dataframe.dtypes))
     for column_name, column_properties in schema._schema_properties().items():
-        if column_name not in dataframe.columns:
+        column_info = schema.column_infos[column_name]
+        if column_name not in dataframe.columns or column_name not in column_subset:
             continue
 
         polars_type = dataframe_datatypes[column_name]
@@ -204,23 +236,16 @@ def _find_errors(  # noqa: C901
             )
 
         # Test for when only specific values are accepted
-        if "enum" in column_properties:
-            permissible_values = set(column_properties["enum"])
-            if column_name in schema.nullable_columns:
-                permissible_values.add(None)
-            actual_values = set(dataframe[column_name].unique())
-            impermissible_values = actual_values - permissible_values
-            if impermissible_values:
-                errors.append(
-                    ErrorWrapper(
-                        RowValueError(
-                            f"Rows with invalid values: {impermissible_values}."
-                        ),
-                        loc=column_name,
-                    )
-                )
+        e = _find_enum_errors(
+            df=dataframe,
+            column_name=column_name,
+            props=column_properties,
+            schema=schema,
+        )
+        if e is not None:
+            errors.append(e)
 
-        if column_properties.get("unique", False):
+        if column_info.unique:
             # Coalescing to 0 in the case of dataframe of height 0
             num_duplicated = dataframe[column_name].is_duplicated().sum() or 0
             if num_duplicated > 0:
@@ -231,47 +256,117 @@ def _find_errors(  # noqa: C901
                     )
                 )
 
+        # Intercept struct columns, and process errors separately
+        if schema.dtypes[column_name] == pl.Struct:
+            nested_schema = schema.model_fields[column_name].annotation
+
+            # Additional unpack required if structs column is optional
+            if type(nested_schema) == _UnionGenericAlias:
+                nested_schema = nested_schema.__args__[0]
+
+                # We need to filter out any null rows as the submodel won't know
+                # that all of a row's columns may be null
+                dataframe = dataframe.filter(pl.col(column_name).is_not_null())
+                if dataframe.is_empty():
+                    continue
+
+            struct_errors = _find_errors(
+                dataframe=dataframe.select(column_name).unnest(column_name),
+                schema=nested_schema,
+            )
+
+            # Format nested errors
+            for error in struct_errors:
+                error._loc = f"{column_name}.{error._loc}"
+
+            errors.extend(struct_errors)
+
+            # No need to do any more checks
+            continue
+
+        # Intercept list of structs columns, and process errors separately
+        elif schema.dtypes[column_name] == pl.List(pl.Struct):
+            nested_schema = schema.model_fields[column_name].annotation.__args__[0]
+
+            # Additional unpack required if structs column is optional
+            if type(nested_schema) == _UnionGenericAlias:
+                nested_schema = nested_schema.__args__[0]
+
+                # We need to filter out any null rows as the submodel won't know
+                # that all of a row's columns may be null
+                dataframe = dataframe.filter(pl.col(column_name).is_not_null())
+                if dataframe.is_empty():
+                    continue
+
+            list_struct_errors = _find_errors(
+                dataframe=dataframe.select(column_name)
+                .explode(column_name)
+                .unnest(column_name),
+                schema=nested_schema,
+            )
+
+            # Format nested errors
+            for error in list_struct_errors:
+                error._loc = f"{column_name}.{error._loc}"
+
+            errors.extend(list_struct_errors)
+
+            # No need to do any more checks
+            continue
+
         # Check for bounded value fields
         col = pl.col(column_name)
         filters = {
-            "maximum": lambda v: col <= v,
-            "exclusiveMaximum": lambda v: col < v,
-            "minimum": lambda v: col >= v,
-            "exclusiveMinimum": lambda v: col > v,
-            "multipleOf": lambda v: (col == 0) | ((col % v) == 0),
-            "const": lambda v: col == v,
-            "pattern": lambda v: col.str.contains(v),
-            "minLength": lambda v: col.str.lengths() >= v,
-            "maxLength": lambda v: col.str.lengths() <= v,
+            "maximum": lambda v, col=col: col <= v,
+            "exclusiveMaximum": lambda v, col=col: col < v,
+            "minimum": lambda v, col=col: col >= v,
+            "exclusiveMinimum": lambda v, col=col: col > v,
+            "multipleOf": lambda v, col=col: (col == 0) | ((col % v) == 0),
+            "const": lambda v, col=col: col == v,
+            "pattern": lambda v, col=col: col.str.contains(v),
+            "minLength": lambda v, col=col: col.str.len_chars() >= v,
+            "maxLength": lambda v, col=col: col.str.len_chars() <= v,
         }
-        checks = [
+        if "anyOf" in column_properties:
+            checks = [
+                check(x[key])
+                for key, check in filters.items()
+                for x in column_properties["anyOf"]
+                if key in x
+            ]
+        else:
+            checks = []
+        checks += [
             check(column_properties[key])
             for key, check in filters.items()
             if key in column_properties
         ]
         if checks:
-            lazy_df = dataframe.lazy()
+            n_invalid_rows = 0
             for check in checks:
-                lazy_df = lazy_df.filter(check)
-            valid_rows = lazy_df.collect()
-            invalid_rows = dataframe.height - valid_rows.height
-            if invalid_rows > 0:
+                lazy_df = dataframe.lazy()
+                lazy_df = lazy_df.filter(
+                    ~check
+                )  # get failing rows (nulls will evaluate to null on boolean check, we only want failures (false)))
+                invalid_rows = lazy_df.collect()
+                n_invalid_rows += invalid_rows.height
+            if n_invalid_rows > 0:
                 errors.append(
                     ErrorWrapper(
                         RowValueError(
-                            f"{invalid_rows} row{'' if invalid_rows == 1 else 's'} "
+                            f"{n_invalid_rows} row{'' if n_invalid_rows == 1 else 's'} "
                             "with out of bound values."
                         ),
                         loc=column_name,
                     )
                 )
 
-        if "constraints" in column_properties:
-            custom_constraints = column_properties["constraints"]
+        if column_info.constraints is not None:
+            custom_constraints = column_info.constraints
             if isinstance(custom_constraints, pl.Expr):
                 custom_constraints = [custom_constraints]
-            constraints = pl.all_horizontal(
-                [constraint.is_not() for constraint in custom_constraints]
+            constraints = pl.any_horizontal(
+                [constraint.not_() for constraint in custom_constraints]
             )
             if "_" in constraints.meta.root_names():
                 # An underscore is an alias for the current field
@@ -295,24 +390,62 @@ def _find_errors(  # noqa: C901
     return errors
 
 
+def _find_enum_errors(
+    df: pl.DataFrame, column_name: str, props: dict[str, Any], schema: Type[Model]
+) -> ErrorWrapper | None:
+    if "enum" not in props:
+        if "items" in props and "enum" in props["items"]:
+            return _find_enum_errors(df, column_name, props["items"], schema)
+        return None
+    permissible_values = set(props["enum"])
+    if column_name in schema.nullable_columns:
+        permissible_values.add(None)
+    if isinstance(df[column_name].dtype, pl.List):
+        actual_values = set(df[column_name].explode().unique())
+    else:
+        actual_values = set(df[column_name].unique())
+    impermissible_values = actual_values - permissible_values
+    if impermissible_values:
+        return ErrorWrapper(
+            RowValueError(f"Rows with invalid values: {impermissible_values}."),
+            loc=column_name,
+        )
+    return None
+
+
 def validate(
-    dataframe: Union["pd.DataFrame", pl.DataFrame], schema: Type[Model]
+    dataframe: Union["pd.DataFrame", pl.DataFrame],
+    schema: Type[Model],
+    columns: Optional[Sequence[str]] = None,
+    allow_missing_columns: bool = False,
+    allow_superfluous_columns: bool = False,
 ) -> None:
-    """
-    Validate the given dataframe.
+    """Validate the given dataframe.
 
     Args:
         dataframe: Polars DataFrame to be validated.
         schema: Patito model which specifies how the dataframe should be structured.
+        columns: Optional list of columns to validate. If not provided, all columns
+            of the dataframe will be validated.
+        allow_missing_columns: If True, missing columns will not be considered an error.
+        allow_superfluous_columns: If True, additional columns will not be considered an error.
 
     Raises:
-        ValidationError: If the given dataframe does not match the given schema.
+        DataFrameValidationError: If the given dataframe does not match the given schema.
+
     """
     if _PANDAS_AVAILABLE and isinstance(dataframe, pd.DataFrame):
         polars_dataframe = pl.from_pandas(dataframe)
     else:
         polars_dataframe = cast(pl.DataFrame, dataframe)
 
-    errors = _find_errors(dataframe=polars_dataframe, schema=schema)
+    polars_dataframe = _transform_df(polars_dataframe, schema)
+    errors = _find_errors(
+        dataframe=polars_dataframe,
+        schema=schema,
+        columns=columns,
+        allow_missing_columns=allow_missing_columns,
+        allow_superfluous_columns=allow_superfluous_columns,
+    )
     if errors:
-        raise ValidationError(errors=errors, model=schema)
+        raise DataFrameValidationError(errors=errors, model=schema)

{patito-0.5.1.dist-info → patito-0.6.2.dist-info}/METADATA

@@ -1,29 +1,26 @@
 Metadata-Version: 2.1
 Name: patito
-Version: 0.5.1
+Version: 0.6.2
 Summary: A dataframe modelling library built on top of polars and pydantic.
-Home-page: https://github.com/kolonialno/patito
+Home-page: https://github.com/JakobGM/patito
 License: MIT
 Keywords: validation,dataframe
 Author: Jakob Gerhard Martinussen
 Author-email: jakobgm@gmail.com
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.9
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: caching
 Provides-Extra: docs
-Provides-Extra: duckdb
 Provides-Extra: pandas
 Requires-Dist: Sphinx (<7) ; extra == "docs"
-Requires-Dist: duckdb (>=0.6.0) ; (python_version >= "3.8" and python_version < "4.0") and (extra == "duckdb")
-Requires-Dist: pandas ; (python_version >= "3.8" and python_version < "4.0") and (extra == "pandas")
-Requires-Dist: polars (>=0.18.7)
-Requires-Dist: pyarrow (>=5.0.0) ; (python_version >= "3.8" and python_version < "4.0") and (extra == "caching" or extra == "duckdb")
-Requires-Dist: pydantic (>=1.7.0,<2.0.0)
+Requires-Dist: pandas ; extra == "pandas"
+Requires-Dist: polars (>=0.20.1)
+Requires-Dist: pyarrow (>=5.0.0) ; extra == "caching"
+Requires-Dist: pydantic (>=2.4.1)
 Requires-Dist: sphinx-autobuild ; extra == "docs"
 Requires-Dist: sphinx-autodoc-typehints ; extra == "docs"
 Requires-Dist: sphinx-rtd-theme ; extra == "docs"
@@ -31,10 +28,10 @@ Requires-Dist: sphinx-toolbox ; extra == "docs"
 Requires-Dist: sphinxcontrib-mermaid ; extra == "docs"
 Requires-Dist: typing-extensions
 Project-URL: Documentation, https://patito.readthedocs.io
-Project-URL: Repository, https://github.com/kolonialno/patito
+Project-URL: Repository, https://github.com/JakobGM/patito
 Description-Content-Type: text/markdown
 
-# <center><img height="30px" src="https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/samsung/78/duck_1f986.png"> Patito<center>
+# <center><img height="30px" src="https://em-content.zobj.net/thumbs/120/samsung/78/duck_1f986.png"> Patito<center>
 
 <p align="center">
     <em>
@@ -66,7 +63,6 @@ These schema can be used for:
 🧪 Easy generation of valid mock data frames for tests.\
 🐍 Retrieve and represent singular rows in an object-oriented manner.\
 🧠 Provide a single source of truth for the core data models in your code base. \
-🦆 Integration with DuckDB for running flexible SQL queries.
 
 Patito has first-class support for [polars]("https://github.com/pola-rs/polars"), a _"blazingly fast DataFrames library written in Rust"_.
 
@@ -76,16 +72,6 @@ Patito has first-class support for [polars]("https://github.com/pola-rs/polars")
 pip install patito
 ```
 
-#### DuckDB Integration
-
-Patito can also integrate with [DuckDB](https://duckdb.org/).
-In order to enable this integration you must explicitly specify it during installation:
-
-```sh
-pip install 'patito[duckdb]'
-```
-
-
 ## Documentation
 
 The full documentation of Patio can be found [here](https://patito.readthedocs.io).
@@ -96,7 +82,7 @@ Patito allows you to specify the type of each column in your dataframe by creati
 
 ```py
 # models.py
-from typing import Literal, Optional
+from typing import Literal
 
 import patito as pt
 
@@ -121,7 +107,7 @@ df = pl.DataFrame(
 )
 try:
     Product.validate(df)
-except pt.ValidationError as exc:
+except pt.exceptions.DataFrameValidationError as exc:
     print(exc)
 # 3 validation errors for Product
 # is_for_sale
@@ -167,7 +153,7 @@ def num_products_for_sale(products: pl.DataFrame) -> int:
     return products.filter(pl.col("is_for_sale")).height
 ```
 
-The following test would fail with a `patito.ValidationError`:
+The following test would fail with a `patito.exceptions.DataFrameValidationError`:
 
 ```py
 def test_num_products_for_sale():

patito-0.6.2.dist-info/RECORD

@@ -0,0 +1,17 @@
+patito/__init__.py,sha256=4qD13kfoa85_kyTCChm3xQcKKzIy3G8AZQp8T_bjcmo,844
+patito/_docs.py,sha256=9mfttyylWpqaOZv8xfDMEwCHHaY7GQwfyI7CDg7tWe8,162
+patito/_pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+patito/_pydantic/column_info.py,sha256=zy3z0gCdQZhNA3_eQ9mEf3Di-gOR8Gt4vmS3v2iULkc,3536
+patito/_pydantic/dtypes/__init__.py,sha256=2vTvL4N4yMN0cbv2CSoa1OFwCswx6FhuQdsYhMaz_dU,578
+patito/_pydantic/dtypes/dtypes.py,sha256=nHtgyI0LsvA_hEIELUxtS4JnDwutqem7iT6nxMDLJxc,9510
+patito/_pydantic/dtypes/utils.py,sha256=6g2mVVSYCs0LSqiPlc4D2Wm3X2gm8sKJnXZYcthfabY,7017
+patito/_pydantic/repr.py,sha256=l9WLjwJ85nJwZCxLIwHih7UuMVVgz17W5th_UD7XZAM,4341
+patito/_pydantic/schema.py,sha256=1XLByZ1jJVP7PUNTkoSPDo0D_hy8QncNLjXKV2N0XDE,3622
+patito/exceptions.py,sha256=VfkkpLblu2Go4QnfWwew7g1NJ_gmynv28p-eGH84tLs,6060
+patito/polars.py,sha256=iAnMFfVyJfSdHantESrIdaX6tZDWj71jyWBery325ac,35333
+patito/pydantic.py,sha256=1gyPfo8-68sdy26yC8c7CQhk_9Mmr0KRsyuS4g54Ddw,48685
+patito/validators.py,sha256=d7lu3MBqaaLLvBVMd5BgarLYpGYHMeJEuTSGAoYqDf0,16231
+patito-0.6.2.dist-info/LICENSE,sha256=3bc4YyuF0e5nd59E3CsR8QM1Ua7pqKfC9DD1LVBVMs4,1139
+patito-0.6.2.dist-info/METADATA,sha256=vUijDkEO0zT5uxED3sN3fTvgsFEIojFYFcENJ1u9_cA,13947
+patito-0.6.2.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+patito-0.6.2.dist-info/RECORD,,