patito 0.6.2__tar.gz → 0.8.0__tar.gz

{patito-0.6.2 → patito-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: patito
-Version: 0.6.2
+Version: 0.8.0
 Summary: A dataframe modelling library built on top of polars and pydantic.
 Home-page: https://github.com/JakobGM/patito
 License: MIT
@@ -13,14 +13,16 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: caching
 Provides-Extra: docs
 Provides-Extra: pandas
 Requires-Dist: Sphinx (<7) ; extra == "docs"
 Requires-Dist: pandas ; extra == "pandas"
-Requires-Dist: polars (>=0.20.1)
+Requires-Dist: polars (>=1.10.0)
+Requires-Dist: pre-commit (>=3.8.0,<4.0.0)
 Requires-Dist: pyarrow (>=5.0.0) ; extra == "caching"
-Requires-Dist: pydantic (>=2.4.1)
+Requires-Dist: pydantic (>=2.7.0)
 Requires-Dist: sphinx-autobuild ; extra == "docs"
 Requires-Dist: sphinx-autodoc-typehints ; extra == "docs"
 Requires-Dist: sphinx-rtd-theme ; extra == "docs"
@@ -74,7 +76,7 @@ pip install patito
 
 ## Documentation
 
-The full documentation of Patio can be found [here](https://patito.readthedocs.io).
+The full documentation of Patito can be found [here](https://patito.readthedocs.io).
 
 ## 👮 Data validation
 

{patito-0.6.2 → patito-0.8.0}/README.md

@@ -41,7 +41,7 @@ pip install patito
 
 ## Documentation
 
-The full documentation of Patio can be found [here](https://patito.readthedocs.io).
+The full documentation of Patito can be found [here](https://patito.readthedocs.io).
 
 ## 👮 Data validation
 

{patito-0.6.2 → patito-0.8.0}/pyproject.toml

@@ -1,8 +1,8 @@
 [tool.poetry]
 name = "patito"
-version = "0.6.2"
+version = "0.8.0"
 description = "A dataframe modelling library built on top of polars and pydantic."
-authors = ["Jakob Gerhard Martinussen <jakobgm@gmail.com>"]
+authors = ["Jakob Gerhard Martinussen <jakobgm@gmail.com>", "Thomas Aarholt <thomasaarholt@gmail.com>"]
 license = "MIT"
 readme = "README.md"
 homepage = "https://github.com/JakobGM/patito"
@@ -12,8 +12,8 @@ keywords = ["validation", "dataframe"]
 
 [tool.poetry.dependencies]
 python = ">=3.9"
-pydantic = ">=2.4.1"
-polars = ">=0.20.1"
+pydantic = ">=2.7.0"
+polars = ">=1.10.0"
 # Required for typing.get_args backports in python3.9 and 3.10
 typing-extensions = "*"
 pandas = {version = "*", optional = true}
@@ -25,6 +25,7 @@ sphinx-autobuild = {version = "*", optional = true}
 sphinx-autodoc-typehints = {version = "*", optional = true}
 sphinx-toolbox = {version = "*", optional = true}
 sphinxcontrib-mermaid = {version = "*", optional = true}
+pre-commit = "^3.8.0"
 
 [tool.poetry.extras]
 # The pyarrow.parquet module is required for writing parquet caches to disk
@@ -42,12 +43,6 @@ docs = [
 [tool.poetry.group.dev.dependencies]
 ruff = ">=0.2.1"
 coverage = {version = "*", extras = ["toml"]}
-flake8 = "3.9.2"
-flake8-annotations = { version = "*", python = ">=3.9,<4.0" }
-flake8-bandit = "*"
-flake8-black = "*"
-flake8-bugbear = "*"
-flake8-isort = "*"
 pyright = ">=1.1.239"
 pytest = ">=7.1.2"
 pytest-cov = ">=3.0.0"
@@ -59,6 +54,7 @@ types-setuptools = ">=57.4.14"
 pandas-stubs = ">=1.2.0"
 codecov = "^2.1.12"
 blackdoc = "*"
+ipykernel = "^6.29.4"
 
 
 [tool.poetry.group.docs.dependencies]
@@ -93,11 +89,19 @@ exclude_lines = [
 fail_under = 99.64
 show_missing = true
 
-[tool.isort]
-profile = "black"
-
 [tool.pyright]
-exclude = ["noxfile.py"]
+typeCheckingMode = "basic"
+venvPath =  "."
+venv =  ".venv"
+pythonVersion = "3.9"
+
+exclude = [
+  ".venv",
+  "noxfile.py",
+  "**/node_modules",
+  "**/__pycache__",
+  "**/.*"
+]
 
 [tool.mypy]
 warn_unused_configs = true
@@ -113,6 +117,9 @@ allow_redefinition = true
 show_error_codes = true
 exclude = [
   "noxfile.py",
+  "**/node_modules",
+  "**/__pycache__",
+  "**/.*"
 ]
 
 
@@ -121,10 +128,11 @@ module = ["tests.test_validators"]
 warn_unused_ignores = false
 
 [tool.ruff]
+target-version = "py39"
 extend-exclude= ["tests/__init__.py"]
 
 [tool.ruff.lint]
-select = ["E4", "E7", "E9", "F", "I", "B", "D"]
+select = ["E4", "E7", "E9", "F", "I", "B", "D", "UP"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"

patito-0.8.0/src/patito/_pydantic/column_info.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import io
+import json
+from typing import Annotated, Optional, Union
+
+import polars as pl
+from polars.datatypes import *  # noqa: F403 # type: ignore
+from polars.datatypes import DataType, DataTypeClass
+from polars.exceptions import ComputeError
+from pydantic import BaseModel, BeforeValidator, field_serializer
+
+
+def dtype_deserializer(dtype: str | DataTypeClass | DataType | None):
+    """Deserialize a dtype from json."""
+    if isinstance(dtype, DataTypeClass) or isinstance(dtype, DataType):
+        return dtype
+    else:
+        if dtype == "null" or dtype is None:
+            return None
+        else:
+            return eval(dtype)
+
+
+def expr_deserializer(
+    expr: str | pl.Expr | list[pl.Expr] | None,
+) -> pl.Expr | list[pl.Expr] | None:
+    """Deserialize a polars expression or list thereof from json.
+
+    This is applied both during deserialization and validation.
+    """
+    if expr is None:
+        return None
+    elif isinstance(expr, pl.Expr):
+        return expr
+    elif isinstance(expr, list):
+        return expr
+    elif isinstance(expr, str):
+        if expr == "null":
+            return None
+        # can be either a list of expr or expr
+        elif expr[0] == "[":
+            return [
+                pl.Expr.deserialize(io.StringIO(e), format="json")
+                for e in json.loads(expr)
+            ]
+        else:
+            return pl.Expr.deserialize(io.StringIO(expr), format="json")
+    else:
+        raise ValueError(f"{expr} can not be deserialized.")
+
+
+def expr_or_col_name_deserializer(expr: str | pl.Expr | None) -> pl.Expr | str | None:
+    """Deserialize a polars expression or column name from json.
+
+    This is applied both during deserialization and validation.
+    """
+    if expr is None:
+        return None
+    elif isinstance(expr, pl.Expr):
+        return expr
+    elif isinstance(expr, list):
+        return expr
+    elif isinstance(expr, str):
+        # Default behaviour
+        if expr == "null":
+            return None
+        else:
+            try:
+                return pl.Expr.deserialize(io.StringIO(expr), format="json")
+            except ComputeError:
+                try:
+                    # Column name is being deserialized
+                    return json.loads(expr)
+                except json.JSONDecodeError:
+                    # Column name has been passed literally
+                    # to ColumnInfo(derived_from="foo")
+                    return expr
+    else:
+        raise ValueError(f"{expr} can not be deserialized.")
+
+
+class ColumnInfo(BaseModel, arbitrary_types_allowed=True):
+    """patito-side model for storing column metadata.
+
+    Args:
+        allow_missing (bool): Column may be missing.
+        constraints (Union[polars.Expression, List[polars.Expression]): A single
+            constraint or list of constraints, expressed as a polars expression objects.
+            All rows must satisfy the given constraint. You can refer to the given column
+            with ``pt.field``, which will automatically be replaced with
+            ``polars.col(<field_name>)`` before evaluation.
+        derived_from (Union[str, polars.Expr]): used to mark fields that are meant to be derived from other fields. Users can specify a polars expression that will be called to derive the column value when `pt.DataFrame.derive` is called.
+        dtype (polars.datatype.DataType): The given dataframe column must have the given
+            polars dtype, for instance ``polars.UInt64`` or ``pl.Float32``.
+        unique (bool): All row values must be unique.
+
+    """
+
+    allow_missing: Optional[bool] = None  # noqa: UP007
+    dtype: Annotated[
+        Optional[Union[DataTypeClass, DataType]],  # noqa: UP007
+        BeforeValidator(dtype_deserializer),
+    ] = None
+    constraints: Annotated[
+        Optional[Union[pl.Expr, list[pl.Expr]]],  # noqa: UP007
+        BeforeValidator(expr_deserializer),
+    ] = None
+    derived_from: Annotated[
+        Optional[Union[str, pl.Expr]],  # noqa: UP007
+        BeforeValidator(expr_or_col_name_deserializer),
+    ] = None
+    unique: Optional[bool] = None  # noqa : UP007
+
+    def __repr__(self) -> str:
+        """Print only Field attributes whose values are not default (mainly None)."""
+        not_default_field = {
+            field: getattr(self, field)
+            for field in self.model_fields
+            if getattr(self, field) is not self.model_fields[field].default
+        }
+
+        string = ""
+        for field, value in not_default_field.items():
+            string += f"{field}={value}, "
+        if string:
+            # remove trailing comma and space
+            string = string[:-2]
+        return f"ColumnInfo({string})"
+
+    @field_serializer("constraints", "derived_from")
+    def expr_serializer(self, expr: None | pl.Expr | list[pl.Expr]):
+        """Converts polars expr to json."""
+        if expr is None:
+            return "null"
+        elif isinstance(expr, str):
+            return json.dumps(expr)
+        elif isinstance(expr, list):
+            return json.dumps([e.meta.serialize(format="json") for e in expr])
+        else:
+            return expr.meta.serialize(format="json")
+
+    @field_serializer("dtype")
+    def dtype_serializer(self, dtype: DataTypeClass | DataType | None) -> str:
+        """Converts polars dtype to json."""
+        if dtype is None:
+            return "null"
+        else:
+            return str(dtype)

{patito-0.6.2 → patito-0.8.0}/src/patito/_pydantic/dtypes/dtypes.py

@@ -1,20 +1,22 @@
 from __future__ import annotations
 
+from collections.abc import Mapping
 from functools import cache, reduce
 from operator import and_
-from typing import TYPE_CHECKING, Any, Dict, FrozenSet, Mapping, Optional, Type
+from typing import TYPE_CHECKING, Any
 
 import polars as pl
-from polars.datatypes import DataType, DataTypeClass, DataTypeGroup
+from polars.datatypes import DataType, DataTypeClass
+from polars.datatypes.group import DataTypeGroup
 from pydantic import TypeAdapter
 
+from patito._pydantic.column_info import ColumnInfo
 from patito._pydantic.dtypes.utils import (
     PT_BASE_SUPPORTED_DTYPES,
     PydanticBaseType,
     _pyd_type_to_default_dtype,
     _pyd_type_to_valid_dtypes,
     _without_optional,
-    dtype_from_string,
 )
 from patito._pydantic.repr import display_as_type
 
@@ -24,8 +26,8 @@ if TYPE_CHECKING:
 
 @cache
 def valid_dtypes_for_model(
-    cls: Type[ModelType],
-) -> Mapping[str, FrozenSet[DataTypeClass]]:
+    cls: type[ModelType],
+) -> Mapping[str, frozenset[DataTypeClass]]:
     return {
         column: (
             DtypeResolver(cls.model_fields[column].annotation).valid_polars_dtypes()
@@ -38,7 +40,7 @@ def valid_dtypes_for_model(
 
 @cache
 def default_dtypes_for_model(
-    cls: Type[ModelType],
+    cls: type[ModelType],
 ) -> dict[str, DataType]:
     default_dtypes: dict[str, DataType] = {}
     for column in cls.columns:
@@ -56,7 +58,7 @@ def default_dtypes_for_model(
 def validate_polars_dtype(
     annotation: type[Any] | None,
     dtype: DataType | DataTypeClass | None,
-    column: Optional[str] = None,
+    column: str | None = None,
 ) -> None:
     """Check that the polars dtype is valid for the given annotation. Raises ValueError if not.
 
@@ -83,7 +85,7 @@ def validate_polars_dtype(
 
 
 def validate_annotation(
-    annotation: type[Any] | Any | None, column: Optional[str] = None
+    annotation: type[Any] | Any | None, column: str | None = None
 ) -> None:
     """Check that the provided annotation has polars/patito support (we can resolve it to a default dtype). Raises ValueError if not.
 
@@ -128,7 +130,7 @@ class DtypeResolver:
 
     def _valid_polars_dtypes_for_schema(
         self,
-        schema: Dict,
+        schema: dict,
     ) -> DataTypeGroup:
         valid_type_sets = []
         if "anyOf" in schema:
@@ -145,7 +147,7 @@ class DtypeResolver:
 
     def _pydantic_subschema_to_valid_polars_types(
         self,
-        props: Dict,
+        props: dict,
     ) -> DataTypeGroup:
         if "type" not in props:
             if "enum" in props:
@@ -188,7 +190,7 @@ class DtypeResolver:
             PydanticBaseType(pyd_type), props.get("format"), props.get("enum")
         )
 
-    def _default_polars_dtype_for_schema(self, schema: Dict) -> DataType | None:
+    def _default_polars_dtype_for_schema(self, schema: dict) -> DataType | None:
         if "anyOf" in schema:
             if len(schema["anyOf"]) == 2:  # look for optionals first
                 schema = _without_optional(schema)
@@ -204,12 +206,12 @@ class DtypeResolver:
 
     def _pydantic_subschema_to_default_dtype(
         self,
-        props: Dict,
+        props: dict,
     ) -> DataType | None:
         if "column_info" in props:  # user has specified in patito model
-            if props["column_info"]["dtype"] is not None:
-                dtype = dtype_from_string(props["column_info"]["dtype"])
-                dtype = dtype() if isinstance(dtype, DataTypeClass) else dtype
+            ci = ColumnInfo.model_validate_json(props["column_info"])
+            if ci.dtype is not None:
+                dtype = ci.dtype() if isinstance(ci.dtype, DataTypeClass) else ci.dtype
                 return dtype
         if "type" not in props:
             if "enum" in props:
@@ -222,6 +224,8 @@ class DtypeResolver:
                 )
             return None
         pyd_type = props.get("type")
+        if pyd_type == "numeric":
+            pyd_type = "number"
         if pyd_type == "array":
             if "items" not in props:
                 raise NotImplementedError(

{patito-0.6.2 → patito-0.8.0}/src/patito/_pydantic/dtypes/utils.py

@@ -1,29 +1,23 @@
 from __future__ import annotations
 
 import sys
+from collections.abc import Sequence
 from enum import Enum
 from typing import (
     Any,
-    Dict,
-    List,
-    Optional,
-    Sequence,
     Union,
-    cast,
     get_args,
     get_origin,
 )
 
 import polars as pl
-from polars.datatypes import DataType, DataTypeClass, DataTypeGroup, convert
-from polars.datatypes.constants import (
+from polars.datatypes import DataType, DataTypeClass, convert
+from polars.datatypes.group import (
     DATETIME_DTYPES,
     DURATION_DTYPES,
     FLOAT_DTYPES,
     INTEGER_DTYPES,
-)
-from polars.polars import (
-    dtype_str_repr,  # TODO: this is a rust function, can we implement our own string parser for Time/Duration/Datetime?
+    DataTypeGroup,
 )
 
 PYTHON_TO_PYDANTIC_TYPES = {
@@ -89,32 +83,42 @@ def is_optional(type_annotation: type[Any] | Any | None) -> bool:
     )
 
 
+def unwrap_optional(type_annotation: type[Any] | Any) -> type:
+    """Return the inner, wrapped type of an Optional.
+
+    Is a no-op for non-Optional types.
+
+    Args:
+        type_annotation: The type annotation to be dewrapped.
+
+    Returns:
+        The input type, but with the outermost Optional removed.
+
+    """
+    return (
+        next(  # pragma: no cover
+            valid_type
+            for valid_type in get_args(type_annotation)
+            if valid_type is not type(None)  # noqa: E721
+        )
+        if is_optional(type_annotation)
+        else type_annotation
+    )
+
+
 def parse_composite_dtype(dtype: DataTypeClass | DataType) -> str:
     """For serialization, converts polars dtype to string representation."""
-    if dtype in pl.NESTED_DTYPES:
-        if dtype == pl.Struct or isinstance(dtype, pl.Struct):
-            raise NotImplementedError("Structs not yet supported by patito")
-        if not isinstance(dtype, pl.List) or isinstance(dtype, pl.Array):
-            raise NotImplementedError(
-                f"Unsupported nested dtype: {dtype} of type {type(dtype)}"
-            )
-        if dtype.inner is None:
-            return convert.DataTypeMappings.DTYPE_TO_FFINAME[dtype.base_type()]
-        return f"{convert.DataTypeMappings.DTYPE_TO_FFINAME[dtype.base_type()]}[{parse_composite_dtype(dtype.inner)}]"
-    elif dtype in pl.TEMPORAL_DTYPES:
-        return cast(str, dtype_str_repr(dtype))
-    else:
-        return convert.DataTypeMappings.DTYPE_TO_FFINAME[dtype]
+    return str(dtype)
 
 
-def dtype_from_string(v: str) -> Optional[Union[DataTypeClass, DataType]]:
+def dtype_from_string(v: str) -> DataTypeClass | DataType | None:
     """For deserialization."""
     # TODO test all dtypes
     return convert.dtype_short_repr_to_dtype(v)
 
 
 def _pyd_type_to_valid_dtypes(
-    pyd_type: PydanticBaseType, string_format: Optional[str], enum: List[str] | None
+    pyd_type: PydanticBaseType, string_format: str | None, enum: list[str] | None
 ) -> DataTypeGroup:
     if enum is not None:
         _validate_enum_values(pyd_type, enum)
@@ -141,7 +145,7 @@ def _pyd_type_to_valid_dtypes(
 
 
 def _pyd_type_to_default_dtype(
-    pyd_type: PydanticBaseType, string_format: Optional[str], enum: List[str] | None
+    pyd_type: PydanticBaseType, string_format: str | None, enum: list[str] | None
 ) -> DataTypeClass | DataType:
     if enum is not None:
         _validate_enum_values(pyd_type, enum)
@@ -207,7 +211,7 @@ def _pyd_string_format_to_default_dtype(
         raise NotImplementedError
 
 
-def _without_optional(schema: Dict) -> Dict:
+def _without_optional(schema: dict) -> dict:
     if "anyOf" in schema:
         for sub_props in schema["anyOf"]:
             if "type" in sub_props and sub_props["type"] == "null":

{patito-0.6.2 → patito-0.8.0}/src/patito/_pydantic/repr.py

@@ -1,26 +1,23 @@
 import sys
 import types
 import typing
+from collections.abc import Generator, Iterable, Sequence
 from typing import (
     Any,
     Callable,
-    Generator,
-    Iterable,
     Literal,
     Optional,
-    Sequence,
-    Tuple,
-    Type,
     Union,
     get_args,
     get_origin,
 )
+from typing import GenericAlias as TypingGenericAlias  # type: ignore
 
 if typing.TYPE_CHECKING:
-    Loc = Tuple[Union[int, str], ...]
-    ReprArgs = Sequence[Tuple[Optional[str], Any]]
+    Loc = tuple[Union[int, str], ...]
+    ReprArgs = Sequence[tuple[Optional[str], Any]]
     RichReprResult = Iterable[
-        Union[Any, Tuple[Any], Tuple[str, Any], Tuple[str, Any, Any]]
+        Union[Any, tuple[Any], tuple[str, Any], tuple[str, Any, Any]]
     ]
 
 try:
@@ -30,15 +27,10 @@ except ImportError:
 
 typing_base = _TypingBase
 
-if sys.version_info < (3, 9):
-    # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on)
-    TypingGenericAlias = ()
-else:
-    from typing import GenericAlias as TypingGenericAlias  # type: ignore
 
 if sys.version_info < (3, 10):
 
-    def origin_is_union(tp: Optional[Type[Any]]) -> bool:
+    def origin_is_union(tp: Optional[type[Any]]) -> bool:
         return tp is typing.Union
 
     WithArgsTypes = (TypingGenericAlias,)
@@ -58,7 +50,7 @@ class Representation:
     of objects.
     """
 
-    __slots__: Tuple[str, ...] = tuple()
+    __slots__: tuple[str, ...] = tuple()
 
     def __repr_args__(self) -> "ReprArgs":
         """Returns the attributes to show in __str__, __repr__, and __pretty__ this is generally overridden.

{patito-0.6.2 → patito-0.8.0}/src/patito/_pydantic/schema.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+from collections.abc import Mapping
 from functools import cache
-from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Type, cast, get_args
+from typing import TYPE_CHECKING, Any, get_args
 
 from pydantic.fields import FieldInfo
 
@@ -13,7 +14,7 @@ if TYPE_CHECKING:
 
 
 @cache
-def schema_for_model(cls: Type[ModelType]) -> Dict[str, Dict[str, Any]]:
+def schema_for_model(cls: type[ModelType]) -> dict[str, dict[str, Any]]:
     """Return schema properties where definition references have been resolved.
 
     Returns:
@@ -46,27 +47,27 @@ def schema_for_model(cls: Type[ModelType]) -> Dict[str, Dict[str, Any]]:
 
 
 @cache
-def column_infos_for_model(cls: Type[ModelType]) -> Mapping[str, ColumnInfo]:
+def column_infos_for_model(cls: type[ModelType]) -> Mapping[str, ColumnInfo]:
     fields = cls.model_fields
 
     def get_column_info(field: FieldInfo) -> ColumnInfo:
         if field.json_schema_extra is None:
-            return cast(ColumnInfo, cls.column_info_class())
+            return ColumnInfo()
         elif callable(field.json_schema_extra):
             raise NotImplementedError(
                 "Callable json_schema_extra not supported by patito."
             )
-        return cast(ColumnInfo, field.json_schema_extra["column_info"])
+        return ColumnInfo.model_validate_json(field.json_schema_extra["column_info"])
 
     return {k: get_column_info(v) for k, v in fields.items()}
 
 
 def _append_field_info_to_props(
-    field_info: Dict[str, Any],
+    field_info: dict[str, Any],
     field_name: str,
-    model_schema: Dict[str, Any],
-    required: Optional[bool] = None,
-) -> Dict[str, Any]:
+    model_schema: dict[str, Any],
+    required: bool | None = None,
+) -> dict[str, Any]:
     if "$ref" in field_info:  # TODO onto runtime append
         definition = model_schema["$defs"][field_info["$ref"]]
         if "enum" in definition and "type" not in definition: