patito 0.7.0__tar.gz → 0.8.2__tar.gz

{patito-0.7.0 → patito-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: patito
-Version: 0.7.0
+Version: 0.8.2
 Summary: A dataframe modelling library built on top of polars and pydantic.
 Home-page: https://github.com/JakobGM/patito
 License: MIT
@@ -13,12 +13,13 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: caching
 Provides-Extra: docs
 Provides-Extra: pandas
 Requires-Dist: Sphinx (<7) ; extra == "docs"
 Requires-Dist: pandas ; extra == "pandas"
-Requires-Dist: polars (>=1.0.0)
+Requires-Dist: polars (>=1.10.0)
 Requires-Dist: pyarrow (>=5.0.0) ; extra == "caching"
 Requires-Dist: pydantic (>=2.7.0)
 Requires-Dist: sphinx-autobuild ; extra == "docs"
@@ -74,7 +75,7 @@ pip install patito
 
 ## Documentation
 
-The full documentation of Patio can be found [here](https://patito.readthedocs.io).
+The full documentation of Patito can be found [here](https://patito.readthedocs.io).
 
 ## 👮 Data validation
 

{patito-0.7.0 → patito-0.8.2}/README.md

@@ -41,7 +41,7 @@ pip install patito
 
 ## Documentation
 
-The full documentation of Patio can be found [here](https://patito.readthedocs.io).
+The full documentation of Patito can be found [here](https://patito.readthedocs.io).
 
 ## 👮 Data validation
 

{patito-0.7.0 → patito-0.8.2}/pyproject.toml

@@ -1,8 +1,8 @@
 [tool.poetry]
 name = "patito"
-version = "0.7.0"
+version = "0.8.2"
 description = "A dataframe modelling library built on top of polars and pydantic."
-authors = ["Jakob Gerhard Martinussen <jakobgm@gmail.com>"]
+authors = ["Jakob Gerhard Martinussen <jakobgm@gmail.com>", "Thomas Aarholt <thomasaarholt@gmail.com>"]
 license = "MIT"
 readme = "README.md"
 homepage = "https://github.com/JakobGM/patito"
@@ -13,7 +13,7 @@ keywords = ["validation", "dataframe"]
 [tool.poetry.dependencies]
 python = ">=3.9"
 pydantic = ">=2.7.0"
-polars = ">=1.0.0"
+polars = ">=1.10.0"
 # Required for typing.get_args backports in python3.9 and 3.10
 typing-extensions = "*"
 pandas = {version = "*", optional = true}
@@ -41,13 +41,8 @@ docs = [
 
 [tool.poetry.group.dev.dependencies]
 ruff = ">=0.2.1"
+pre-commit = "^3.8.0"
 coverage = {version = "*", extras = ["toml"]}
-flake8 = "3.9.2"
-flake8-annotations = { version = "*", python = ">=3.9,<4.0" }
-flake8-bandit = "*"
-flake8-black = "*"
-flake8-bugbear = "*"
-flake8-isort = "*"
 pyright = ">=1.1.239"
 pytest = ">=7.1.2"
 pytest-cov = ">=3.0.0"
@@ -94,11 +89,19 @@ exclude_lines = [
 fail_under = 99.64
 show_missing = true
 
-[tool.isort]
-profile = "black"
-
 [tool.pyright]
-exclude = ["noxfile.py"]
+typeCheckingMode = "basic"
+venvPath =  "."
+venv =  ".venv"
+pythonVersion = "3.9"
+
+exclude = [
+  ".venv",
+  "noxfile.py",
+  "**/node_modules",
+  "**/__pycache__",
+  "**/.*"
+]
 
 [tool.mypy]
 warn_unused_configs = true
@@ -114,6 +117,9 @@ allow_redefinition = true
 show_error_codes = true
 exclude = [
   "noxfile.py",
+  "**/node_modules",
+  "**/__pycache__",
+  "**/.*"
 ]
 
 
@@ -122,10 +128,12 @@ module = ["tests.test_validators"]
 warn_unused_ignores = false
 
 [tool.ruff]
+target-version = "py39"
 extend-exclude= ["tests/__init__.py"]
 
 [tool.ruff.lint]
-select = ["E4", "E7", "E9", "F", "I", "B", "D"]
+select = ["E4", "E7", "E9", "F", "I", "B", "D", "UP"]
+ignore = ["UP007"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"

patito-0.8.2/src/patito/_pydantic/column_info.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import io
+import json
+from typing import Annotated, Optional, Union
+
+import polars as pl
+from polars.datatypes import *  # noqa: F403 # type: ignore
+from polars.datatypes import DataType, DataTypeClass
+from polars.exceptions import ComputeError
+from pydantic import BaseModel, BeforeValidator, field_serializer
+
+
+def dtype_deserializer(dtype: str | DataTypeClass | DataType | None):
+    """Deserialize a dtype from json."""
+    if isinstance(dtype, DataTypeClass) or isinstance(dtype, DataType):
+        return dtype
+    else:
+        if dtype == "null" or dtype is None:
+            return None
+        else:
+            return eval(dtype)
+
+
+def expr_deserializer(
+    expr: str | pl.Expr | list[pl.Expr] | None,
+) -> pl.Expr | list[pl.Expr] | None:
+    """Deserialize a polars expression or list thereof from json.
+
+    This is applied both during deserialization and validation.
+    """
+    if expr is None:
+        return None
+    elif isinstance(expr, pl.Expr):
+        return expr
+    elif isinstance(expr, list):
+        return expr
+    elif isinstance(expr, str):
+        if expr == "null":
+            return None
+        # can be either a list of expr or expr
+        elif expr[0] == "[":
+            return [
+                pl.Expr.deserialize(io.StringIO(e), format="json")
+                for e in json.loads(expr)
+            ]
+        else:
+            return pl.Expr.deserialize(io.StringIO(expr), format="json")
+    else:
+        raise ValueError(f"{expr} can not be deserialized.")
+
+
+def expr_or_col_name_deserializer(expr: str | pl.Expr | None) -> pl.Expr | str | None:
+    """Deserialize a polars expression or column name from json.
+
+    This is applied both during deserialization and validation.
+    """
+    if expr is None:
+        return None
+    elif isinstance(expr, pl.Expr):
+        return expr
+    elif isinstance(expr, list):
+        return expr
+    elif isinstance(expr, str):
+        # Default behaviour
+        if expr == "null":
+            return None
+        else:
+            try:
+                return pl.Expr.deserialize(io.StringIO(expr), format="json")
+            except ComputeError:
+                try:
+                    # Column name is being deserialized
+                    return json.loads(expr)
+                except json.JSONDecodeError:
+                    # Column name has been passed literally
+                    # to ColumnInfo(derived_from="foo")
+                    return expr
+    else:
+        raise ValueError(f"{expr} can not be deserialized.")
+
+
+class ColumnInfo(BaseModel, arbitrary_types_allowed=True):
+    """patito-side model for storing column metadata.
+
+    Args:
+        allow_missing (bool): Column may be missing.
+        constraints (Union[polars.Expression, List[polars.Expression]): A single
+            constraint or list of constraints, expressed as a polars expression objects.
+            All rows must satisfy the given constraint. You can refer to the given column
+            with ``pt.field``, which will automatically be replaced with
+            ``polars.col(<field_name>)`` before evaluation.
+        derived_from (Union[str, polars.Expr]): used to mark fields that are meant to be derived from other fields. Users can specify a polars expression that will be called to derive the column value when `pt.DataFrame.derive` is called.
+        dtype (polars.datatype.DataType): The given dataframe column must have the given
+            polars dtype, for instance ``polars.UInt64`` or ``pl.Float32``.
+        unique (bool): All row values must be unique.
+
+    """
+
+    allow_missing: Optional[bool] = None
+    dtype: Annotated[
+        Optional[Union[DataTypeClass, DataType]],
+        BeforeValidator(dtype_deserializer),
+    ] = None
+    constraints: Annotated[
+        Optional[Union[pl.Expr, list[pl.Expr]]],
+        BeforeValidator(expr_deserializer),
+    ] = None
+    derived_from: Annotated[
+        Optional[Union[str, pl.Expr]],
+        BeforeValidator(expr_or_col_name_deserializer),
+    ] = None
+    unique: Optional[bool] = None
+
+    def __repr__(self) -> str:
+        """Print only Field attributes whose values are not default (mainly None)."""
+        not_default_field = {
+            field: getattr(self, field)
+            for field in self.model_fields
+            if getattr(self, field) is not self.model_fields[field].default
+        }
+
+        string = ""
+        for field, value in not_default_field.items():
+            string += f"{field}={value}, "
+        if string:
+            # remove trailing comma and space
+            string = string[:-2]
+        return f"ColumnInfo({string})"
+
+    @field_serializer("constraints", "derived_from")
+    def expr_serializer(self, expr: None | pl.Expr | list[pl.Expr]):
+        """Converts polars expr to json."""
+        if expr is None:
+            return "null"
+        elif isinstance(expr, str):
+            return json.dumps(expr)
+        elif isinstance(expr, list):
+            return json.dumps([e.meta.serialize(format="json") for e in expr])
+        else:
+            return expr.meta.serialize(format="json")
+
+    @field_serializer("dtype")
+    def dtype_serializer(self, dtype: DataTypeClass | DataType | None) -> str:
+        """Converts polars dtype to json."""
+        if dtype is None:
+            return "null"
+        else:
+            return str(dtype)

{patito-0.7.0 → patito-0.8.2}/src/patito/_pydantic/dtypes/dtypes.py

@@ -1,21 +1,22 @@
 from __future__ import annotations
 
+from collections.abc import Mapping
 from functools import cache, reduce
-from operator import and_
-from typing import TYPE_CHECKING, Any, Dict, FrozenSet, Mapping, Optional, Type
+from operator import or_
+from typing import TYPE_CHECKING, Any
 
 import polars as pl
 from polars.datatypes import DataType, DataTypeClass
 from polars.datatypes.group import DataTypeGroup
 from pydantic import TypeAdapter
 
+from patito._pydantic.column_info import ColumnInfo
 from patito._pydantic.dtypes.utils import (
     PT_BASE_SUPPORTED_DTYPES,
     PydanticBaseType,
     _pyd_type_to_default_dtype,
     _pyd_type_to_valid_dtypes,
     _without_optional,
-    dtype_from_string,
 )
 from patito._pydantic.repr import display_as_type
 
@@ -25,8 +26,8 @@ if TYPE_CHECKING:
 
 @cache
 def valid_dtypes_for_model(
-    cls: Type[ModelType],
-) -> Mapping[str, FrozenSet[DataTypeClass]]:
+    cls: type[ModelType],
+) -> Mapping[str, frozenset[DataTypeClass]]:
     return {
         column: (
             DtypeResolver(cls.model_fields[column].annotation).valid_polars_dtypes()
@@ -39,7 +40,7 @@ def valid_dtypes_for_model(
 
 @cache
 def default_dtypes_for_model(
-    cls: Type[ModelType],
+    cls: type[ModelType],
 ) -> dict[str, DataType]:
     default_dtypes: dict[str, DataType] = {}
     for column in cls.columns:
@@ -57,7 +58,7 @@ def default_dtypes_for_model(
 def validate_polars_dtype(
     annotation: type[Any] | None,
     dtype: DataType | DataTypeClass | None,
-    column: Optional[str] = None,
+    column: str | None = None,
 ) -> None:
     """Check that the polars dtype is valid for the given annotation. Raises ValueError if not.
 
@@ -84,7 +85,7 @@ def validate_polars_dtype(
 
 
 def validate_annotation(
-    annotation: type[Any] | Any | None, column: Optional[str] = None
+    annotation: type[Any] | Any | None, column: str | None = None
 ) -> None:
     """Check that the provided annotation has polars/patito support (we can resolve it to a default dtype). Raises ValueError if not.
 
@@ -114,7 +115,8 @@ def validate_annotation(
 class DtypeResolver:
     def __init__(self, annotation: Any | None):
         self.annotation = annotation
-        self.schema = TypeAdapter(annotation).json_schema()
+        # mode='serialization' allows nested models with structs, see #86
+        self.schema = TypeAdapter(annotation).json_schema(mode="serialization")
         self.defs = self.schema.get("$defs", {})
 
     def valid_polars_dtypes(self) -> DataTypeGroup:
@@ -129,7 +131,7 @@ class DtypeResolver:
 
     def _valid_polars_dtypes_for_schema(
         self,
-        schema: Dict,
+        schema: dict,
     ) -> DataTypeGroup:
         valid_type_sets = []
         if "anyOf" in schema:
@@ -142,11 +144,11 @@ class DtypeResolver:
             valid_type_sets.append(
                 self._pydantic_subschema_to_valid_polars_types(schema)
             )
-        return reduce(and_, valid_type_sets) if valid_type_sets else DataTypeGroup([])
+        return reduce(or_, valid_type_sets) if valid_type_sets else DataTypeGroup([])
 
     def _pydantic_subschema_to_valid_polars_types(
         self,
-        props: Dict,
+        props: dict,
     ) -> DataTypeGroup:
         if "type" not in props:
             if "enum" in props:
@@ -158,6 +160,7 @@ class DtypeResolver:
                     self.defs[props["$ref"].split("/")[-1]]
                 )
             return DataTypeGroup([])
+
         pyd_type = props.get("type")
         if pyd_type == "array":
             if "items" not in props:
@@ -168,28 +171,27 @@ class DtypeResolver:
             return DataTypeGroup(
                 [pl.List(dtype) for dtype in item_dtypes], match_base_type=False
             )
+
         elif pyd_type == "object":
             if "properties" not in props:
                 return DataTypeGroup([])
             object_props = props["properties"]
+            struct_fields: list[pl.Field] = []
+            for name, sub_props in object_props.items():
+                dtype = self._default_polars_dtype_for_schema(sub_props)
+                assert dtype is not None
+                struct_fields.append(pl.Field(name, dtype))
             return DataTypeGroup(
-                [
-                    pl.Struct(
-                        [
-                            pl.Field(
-                                name, self._default_polars_dtype_for_schema(sub_props)
-                            )
-                            for name, sub_props in object_props.items()
-                        ]
-                    )
-                ],
+                [pl.Struct(struct_fields)],
                 match_base_type=False,
             )  # for structs, return only the default dtype set to avoid combinatoric issues
         return _pyd_type_to_valid_dtypes(
             PydanticBaseType(pyd_type), props.get("format"), props.get("enum")
         )
 
-    def _default_polars_dtype_for_schema(self, schema: Dict) -> DataType | None:
+    def _default_polars_dtype_for_schema(
+        self, schema: dict[str, Any]
+    ) -> DataType | None:
         if "anyOf" in schema:
             if len(schema["anyOf"]) == 2:  # look for optionals first
                 schema = _without_optional(schema)
@@ -205,13 +207,14 @@ class DtypeResolver:
 
     def _pydantic_subschema_to_default_dtype(
         self,
-        props: Dict,
+        props: dict[str, Any],
     ) -> DataType | None:
         if "column_info" in props:  # user has specified in patito model
-            if props["column_info"]["dtype"] is not None:
-                dtype = dtype_from_string(props["column_info"]["dtype"])
-                dtype = dtype() if isinstance(dtype, DataTypeClass) else dtype
+            ci = ColumnInfo.model_validate_json(props["column_info"])
+            if ci.dtype is not None:
+                dtype = ci.dtype() if isinstance(ci.dtype, DataTypeClass) else ci.dtype
                 return dtype
+
         if "type" not in props:
             if "enum" in props:
                 raise TypeError("Mixed type enums not supported by patito.")
@@ -222,10 +225,12 @@ class DtypeResolver:
                     self.defs[props["$ref"].split("/")[-1]]
                 )
             return None
+
         pyd_type = props.get("type")
         if pyd_type == "numeric":
             pyd_type = "number"
-        if pyd_type == "array":
+
+        elif pyd_type == "array":
             if "items" not in props:
                 raise NotImplementedError(
                     "Unexpected error processing pydantic schema. Please file an issue."
@@ -235,18 +240,21 @@ class DtypeResolver:
             if inner_default_type is None:
                 return None
             return pl.List(inner_default_type)
-        elif pyd_type == "object":
+
+        elif pyd_type == "object":  # these are structs
             if "properties" not in props:
                 raise NotImplementedError(
                     "dictionaries not currently supported by patito"
                 )
-            object_props = props["properties"]
-            return pl.Struct(
-                [
-                    pl.Field(name, self._default_polars_dtype_for_schema(sub_props))
-                    for name, sub_props in object_props.items()
-                ]
-            )
+            object_props: dict[str, dict[str, str]] = props["properties"]
+            struct_fields: list[pl.Field] = []
+
+            for name, sub_props in object_props.items():
+                dtype = self._default_polars_dtype_for_schema(sub_props)
+                assert dtype is not None
+                struct_fields.append(pl.Field(name, dtype))
+            return pl.Struct(struct_fields)
+
         return _pyd_type_to_default_dtype(
             PydanticBaseType(pyd_type), props.get("format"), props.get("enum")
         )

{patito-0.7.0 → patito-0.8.2}/src/patito/_pydantic/dtypes/utils.py

@@ -1,15 +1,11 @@
 from __future__ import annotations
 
 import sys
+from collections.abc import Sequence
 from enum import Enum
 from typing import (
     Any,
-    Dict,
-    List,
-    Optional,
-    Sequence,
     Union,
-    cast,
     get_args,
     get_origin,
 )
@@ -23,9 +19,6 @@ from polars.datatypes.group import (
     INTEGER_DTYPES,
     DataTypeGroup,
 )
-from polars.polars import (
-    dtype_str_repr,  # TODO: this is a rust function, can we implement our own string parser for Time/Duration/Datetime?
-)
 
 PYTHON_TO_PYDANTIC_TYPES = {
     str: "string",
@@ -90,38 +83,48 @@ def is_optional(type_annotation: type[Any] | Any | None) -> bool:
     )
 
 
+def unwrap_optional(type_annotation: type[Any] | Any) -> type:
+    """Return the inner, wrapped type of an Optional.
+
+    Is a no-op for non-Optional types.
+
+    Args:
+        type_annotation: The type annotation to be dewrapped.
+
+    Returns:
+        The input type, but with the outermost Optional removed.
+
+    """
+    return (
+        next(  # pragma: no cover
+            valid_type
+            for valid_type in get_args(type_annotation)
+            if valid_type is not type(None)  # noqa: E721
+        )
+        if is_optional(type_annotation)
+        else type_annotation
+    )
+
+
 def parse_composite_dtype(dtype: DataTypeClass | DataType) -> str:
     """For serialization, converts polars dtype to string representation."""
-    if dtype.is_nested():
-        if dtype == pl.Struct or isinstance(dtype, pl.Struct):
-            raise NotImplementedError("Structs not yet supported by patito")
-        if not isinstance(dtype, pl.List) or isinstance(dtype, pl.Array):
-            raise NotImplementedError(
-                f"Unsupported nested dtype: {dtype} of type {type(dtype)}"
-            )
-        if dtype.inner is None:
-            return convert.DataTypeMappings.DTYPE_TO_FFINAME[dtype.base_type()]
-        return f"{convert.DataTypeMappings.DTYPE_TO_FFINAME[dtype.base_type()]}[{parse_composite_dtype(dtype.inner)}]"
-    elif dtype.is_temporal():
-        return cast(str, dtype_str_repr(dtype))
-    else:
-        return convert.DataTypeMappings.DTYPE_TO_FFINAME[dtype]
+    return str(dtype)
 
 
-def dtype_from_string(v: str) -> Optional[Union[DataTypeClass, DataType]]:
+def dtype_from_string(v: str) -> DataTypeClass | DataType | None:
     """For deserialization."""
     # TODO test all dtypes
     return convert.dtype_short_repr_to_dtype(v)
 
 
 def _pyd_type_to_valid_dtypes(
-    pyd_type: PydanticBaseType, string_format: Optional[str], enum: List[str] | None
+    pyd_type: PydanticBaseType, string_format: str | None, enum: list[str] | None
 ) -> DataTypeGroup:
     if enum is not None:
         _validate_enum_values(pyd_type, enum)
         return DataTypeGroup([pl.Enum(enum), pl.String], match_base_type=False)
     if pyd_type.value == "integer":
-        return DataTypeGroup(INTEGER_DTYPES | FLOAT_DTYPES)
+        return DataTypeGroup(INTEGER_DTYPES)
     elif pyd_type.value == "number":
         return (
             FLOAT_DTYPES
@@ -142,7 +145,7 @@ def _pyd_type_to_valid_dtypes(
 
 
 def _pyd_type_to_default_dtype(
-    pyd_type: PydanticBaseType, string_format: Optional[str], enum: List[str] | None
+    pyd_type: PydanticBaseType, string_format: str | None, enum: list[str] | None
 ) -> DataTypeClass | DataType:
     if enum is not None:
         _validate_enum_values(pyd_type, enum)
@@ -208,7 +211,7 @@ def _pyd_string_format_to_default_dtype(
         raise NotImplementedError
 
 
-def _without_optional(schema: Dict) -> Dict:
+def _without_optional(schema: dict) -> dict:
     if "anyOf" in schema:
         for sub_props in schema["anyOf"]:
             if "type" in sub_props and sub_props["type"] == "null":

{patito-0.7.0 → patito-0.8.2}/src/patito/_pydantic/repr.py

@@ -1,26 +1,23 @@
 import sys
 import types
 import typing
+from collections.abc import Generator, Iterable, Sequence
 from typing import (
     Any,
     Callable,
-    Generator,
-    Iterable,
     Literal,
     Optional,
-    Sequence,
-    Tuple,
-    Type,
     Union,
     get_args,
     get_origin,
 )
+from typing import GenericAlias as TypingGenericAlias  # type: ignore
 
 if typing.TYPE_CHECKING:
-    Loc = Tuple[Union[int, str], ...]
-    ReprArgs = Sequence[Tuple[Optional[str], Any]]
+    Loc = tuple[Union[int, str], ...]
+    ReprArgs = Sequence[tuple[Optional[str], Any]]
     RichReprResult = Iterable[
-        Union[Any, Tuple[Any], Tuple[str, Any], Tuple[str, Any, Any]]
+        Union[Any, tuple[Any], tuple[str, Any], tuple[str, Any, Any]]
     ]
 
 try:
@@ -30,15 +27,10 @@ except ImportError:
 
 typing_base = _TypingBase
 
-if sys.version_info < (3, 9):
-    # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on)
-    TypingGenericAlias = ()
-else:
-    from typing import GenericAlias as TypingGenericAlias  # type: ignore
 
 if sys.version_info < (3, 10):
 
-    def origin_is_union(tp: Optional[Type[Any]]) -> bool:
+    def origin_is_union(tp: Optional[type[Any]]) -> bool:
         return tp is typing.Union
 
     WithArgsTypes = (TypingGenericAlias,)
@@ -58,7 +50,7 @@ class Representation:
     of objects.
     """
 
-    __slots__: Tuple[str, ...] = tuple()
+    __slots__: tuple[str, ...] = tuple()
 
     def __repr_args__(self) -> "ReprArgs":
         """Returns the attributes to show in __str__, __repr__, and __pretty__ this is generally overridden.