dataframely 1.5.0__tar.gz → 1.6.0__tar.gz

{dataframely-1.5.0 → dataframely-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 1.5.0
+Version: 1.6.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -28,7 +28,7 @@ Project-URL: Repository, https://github.com/quantco/dataframely
 [![conda-forge](https://img.shields.io/conda/vn/conda-forge/dataframely?logoColor=white&logo=conda-forge&style=flat-square)](https://prefix.dev/channels/conda-forge/packages/dataframely)
 [![pypi-version](https://img.shields.io/pypi/v/dataframely.svg?logo=pypi&logoColor=white&style=flat-square)](https://pypi.org/project/dataframely)
 [![python-version](https://img.shields.io/pypi/pyversions/dataframely?logoColor=white&logo=python&style=flat-square)](https://pypi.org/project/dataframely)
-[![codecov](https://codecov.io/gh/Quantco/dataframely/graph/badge.svg)](https://codecov.io/gh/Quantco/dataframely)
+[![codecov](https://codecov.io/gh/Quantco/dataframely/graph/badge.svg?token=QOvhS7Zri2)](https://codecov.io/gh/Quantco/dataframely)
 
 </div>
 

{dataframely-1.5.0 → dataframely-1.6.0}/README.md

@@ -11,7 +11,7 @@
 [![conda-forge](https://img.shields.io/conda/vn/conda-forge/dataframely?logoColor=white&logo=conda-forge&style=flat-square)](https://prefix.dev/channels/conda-forge/packages/dataframely)
 [![pypi-version](https://img.shields.io/pypi/v/dataframely.svg?logo=pypi&logoColor=white&style=flat-square)](https://pypi.org/project/dataframely)
 [![python-version](https://img.shields.io/pypi/pyversions/dataframely?logoColor=white&logo=python&style=flat-square)](https://pypi.org/project/dataframely)
-[![codecov](https://codecov.io/gh/Quantco/dataframely/graph/badge.svg)](https://codecov.io/gh/Quantco/dataframely)
+[![codecov](https://codecov.io/gh/Quantco/dataframely/graph/badge.svg?token=QOvhS7Zri2)](https://codecov.io/gh/Quantco/dataframely)
 
 </div>
 

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/__init__.py

@@ -51,7 +51,7 @@ from .functional import (
     filter_relationship_one_to_at_least_one,
     filter_relationship_one_to_one,
 )
-from .schema import Schema
+from .schema import Schema, deserialize_schema
 
 __all__ = [
     "random",
@@ -67,6 +67,7 @@ __all__ = [
     "filter_relationship_one_to_at_least_one",
     "filter_relationship_one_to_one",
     "Schema",
+    "deserialize_schema",
     "Any",
     "Bool",
     "Column",

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/_base_collection.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import textwrap
 import typing
 from abc import ABCMeta
 from collections.abc import Iterable
@@ -245,6 +246,32 @@ class CollectionMeta(ABCMeta):
             # Some other unknown annotation
             raise AnnotationImplementationError(attr, type_annotation)
 
+    def __repr__(cls) -> str:
+        parts = [f'[Collection "{cls.__class__.__name__}"]']
+        parts.append(textwrap.indent("Members:", prefix=" " * 2))
+        for name, member in cls.members().items():  # type: ignore
+            parts.append(
+                textwrap.indent(
+                    f'- "{name}": {member.schema.__name__}'
+                    f"(optional={member.is_optional}, "
+                    f"ignored_in_filters={member.ignored_in_filters}, "
+                    f"inline_for_sampling={member.inline_for_sampling})",
+                    prefix=" " * 4,
+                )
+            )
+        if filters := cls._filters():  # type: ignore
+            parts.append(textwrap.indent("Filters:", prefix=" " * 2))
+            for name, member in filters.items():
+                parts.append(textwrap.indent(f'- "{name}":', prefix=" " * 4))
+                parts.append(
+                    textwrap.indent(
+                        f"{member.logic(cls.create_empty()).explain()}",  # type: ignore
+                        prefix=" " * 8,
+                    )
+                )
+        parts.append("")  # Add line break at the end
+        return "\n".join(parts)
+
 
 class BaseCollection(metaclass=CollectionMeta):
     """Internal utility abstraction to reference collections without introducing

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/_base_schema.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import textwrap
 from abc import ABCMeta
 from copy import copy
 from dataclasses import dataclass, field
@@ -10,9 +11,9 @@ from typing import Any, Self
 
 import polars as pl
 
-from ._rule import GroupRule, Rule, with_evaluation_rules
+from ._rule import GroupRule, Rule
 from .columns import Column
-from .exc import ImplementationError, RuleImplementationError
+from .exc import ImplementationError
 
 _COLUMN_ATTR = "__dataframely_columns__"
 _RULE_ATTR = "__dataframely_rules__"
@@ -111,25 +112,15 @@ class SchemaMeta(ABCMeta):
                         f"which are not in the schema: {missing_list}."
                     )
 
-        # 3) Assuming that non-custom rules are implemented correctly, we check that all
-        # custom rules are _also_ implemented correctly by evaluating rules on an
-        # empty data frame and checking for the evaluated dtypes.
-        if len(result.rules) > 0:
-            lf_empty = pl.LazyFrame(
-                schema={col_name: col.dtype for col_name, col in result.columns.items()}
-            )
-            # NOTE: For some reason, `polars` does not yield correct dtypes when calling
-            #  `collect_schema()`
-            schema = with_evaluation_rules(lf_empty, result.rules).collect().schema
-            for rule_name, rule in result.rules.items():
-                dtype = schema[rule_name]
-                if not isinstance(dtype, pl.Boolean):
-                    raise RuleImplementationError(
-                        rule_name, dtype, isinstance(rule, GroupRule)
-                    )
-
         return super().__new__(mcs, name, bases, namespace, *args, **kwargs)
 
+    def __getattribute__(cls, name: str) -> Any:
+        val = super().__getattribute__(name)
+        # Dynamically set the name of the column if it is a `Column` instance.
+        if isinstance(val, Column):
+            val._name = val.alias or name
+        return val
+
     @staticmethod
     def _get_metadata_recursively(kls: type[object]) -> Metadata:
         result = Metadata()
@@ -145,9 +136,7 @@ class SchemaMeta(ABCMeta):
             k: v for k, v in source.items() if not k.startswith("__")
         }.items():
             if isinstance(value, Column):
-                if not value.alias:
-                    value.alias = attr
-                result.columns[value.alias] = value
+                result.columns[value.alias or attr] = value
             if isinstance(value, Rule):
                 # We must ensure that custom rules do not clash with internal rules.
                 if attr == "primary_key":
@@ -157,6 +146,18 @@ class SchemaMeta(ABCMeta):
                 result.rules[attr] = value
         return result
 
+    def __repr__(cls) -> str:
+        parts = [f'[Schema "{cls.__name__}"]']
+        parts.append(textwrap.indent("Columns:", prefix=" " * 2))
+        for name, col in cls.columns().items():
+            parts.append(textwrap.indent(f'- "{name}": {col!r}', prefix=" " * 4))
+        if validation_rules := cls._schema_validation_rules():
+            parts.append(textwrap.indent("Rules:", prefix=" " * 2))
+            for name, rule in validation_rules.items():
+                parts.append(textwrap.indent(f'- "{name}": {rule!r}', prefix=" " * 4))
+        parts.append("")  # Add line break at the end
+        return "\n".join(parts)
+
 
 class BaseSchema(metaclass=SchemaMeta):
     """Internal utility abstraction to reference schemas without introducing cyclical

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/_deprecation.py

@@ -37,3 +37,14 @@ def warn_nullable_default_change() -> None:
         FutureWarning,
         stacklevel=4,
     )
+
+
+@skip_if(env="DATAFRAMELY_NO_FUTURE_WARNINGS")
+def warn_no_nullable_primary_keys() -> None:
+    warnings.warn(
+        "Nullable primary keys are not supported. "
+        "Setting `nullable=True` on a primary key column is ignored "
+        "and will raise an error in a future release.",
+        FutureWarning,
+        stacklevel=4,
+    )

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/_filter.py

@@ -34,6 +34,12 @@ def filter() -> Callable[[Callable[[C], pl.LazyFrame]], Filter[C]]:
     Attention:
         Make sure to provide unique combinations of the primary keys or the filters
         might introduce duplicate rows.
+
+    Attention:
+        The filter logic should return a lazy frame with a static computational graph.
+        Other implementations using arbitrary python logic works for filtering and
+        validation, but may lead to wrong results in Collection comparisons
+        and (de-)serialization.
     """
 
     def decorator(validation_fn: Callable[[C], pl.LazyFrame]) -> Filter[C]:

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/_rule.py

@@ -1,8 +1,11 @@
 # Copyright (c) QuantCo 2025-2025
 # SPDX-License-Identifier: BSD-3-Clause
 
+from __future__ import annotations
+
 from collections import defaultdict
 from collections.abc import Callable
+from typing import Any, Self
 
 import polars as pl
 
@@ -12,17 +15,68 @@ ValidationFunction = Callable[[], pl.Expr]
 class Rule:
     """Internal class representing validation rules."""
 
-    def __init__(self, expr: pl.Expr) -> None:
-        self.expr = expr
+    def __init__(self, expr: pl.Expr | ValidationFunction) -> None:
+        self._expr = expr
+
+    @property
+    def expr(self) -> pl.Expr:
+        """Get the expression of the rule."""
+        if callable(self._expr):
+            return self._expr()
+        return self._expr
+
+    def matches(self, other: Rule) -> bool:
+        """Check whether this rule semantically matches another rule.
+
+        Args:
+            other: The rule to compare with.
+
+        Returns:
+            Whether the rules are semantically equal.
+        """
+        return self.expr.meta.eq(other.expr)
+
+    def as_dict(self) -> dict[str, Any]:
+        """Turn the rule into a dictionary."""
+        return {"rule_type": self.__class__.__name__, "expr": self.expr}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Self:
+        """Read the rule from a dictionary.
+
+        Args:
+            data: The dictionary that was created via :meth:`asdict`.
+        """
+        return cls(data["expr"])
+
+    def __repr__(self) -> str:
+        return str(self.expr)
 
 
 class GroupRule(Rule):
     """Rule that is evaluated on a group of columns."""
 
-    def __init__(self, expr: pl.Expr, group_columns: list[str]) -> None:
+    def __init__(
+        self, expr: pl.Expr | ValidationFunction, group_columns: list[str]
+    ) -> None:
         super().__init__(expr)
         self.group_columns = group_columns
 
+    def matches(self, other: Rule) -> bool:
+        if not isinstance(other, GroupRule):
+            return False
+        return super().matches(other) and self.group_columns == other.group_columns
+
+    def as_dict(self) -> dict[str, Any]:
+        return {**super().as_dict(), "group_columns": self.group_columns}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Self:
+        return cls(data["expr"], group_columns=data["group_columns"])
+
+    def __repr__(self) -> str:
+        return f"{super().__repr__()} grouped by {self.group_columns}"
+
 
 def rule(*, group_by: list[str] | None = None) -> Callable[[ValidationFunction], Rule]:
     """Mark a function as a rule to evaluate during validation.
@@ -52,12 +106,18 @@ def rule(*, group_by: list[str] | None = None) -> Callable[[ValidationFunction],
         rules. By default, any rule that evaluates to ``null`` because one of the
         columns used in the rule is ``null`` is interpreted as ``true``, i.e. the row
         is assumed to be valid.
+
+    Attention:
+        The rule logic should return a static result.
+        Other implementations using arbitrary python logic works for filtering and
+        validation, but may lead to wrong results in Schema comparisons
+        and (de-)serialization.
     """
 
     def decorator(validation_fn: ValidationFunction) -> Rule:
         if group_by is not None:
-            return GroupRule(expr=validation_fn(), group_columns=group_by)
-        return Rule(expr=validation_fn())
+            return GroupRule(expr=validation_fn, group_columns=group_by)
+        return Rule(expr=validation_fn)
 
     return decorator
 
@@ -130,3 +190,30 @@ def _with_group_rules(lf: pl.LazyFrame, rules: dict[str, GroupRule]) -> pl.LazyF
             frame, on=list(group_columns), how="left", nulls_equal=True
         )
     return result
+
+
+# ------------------------------------------------------------------------------------ #
+#                                        FACTORY                                       #
+# ------------------------------------------------------------------------------------ #
+
+_TYPE_MAPPING: dict[str, type[Rule]] = {
+    Rule.__name__: Rule,
+    GroupRule.__name__: GroupRule,
+}
+
+
+def rule_from_dict(data: dict[str, Any]) -> Rule:
+    """Dynamically read a rule object from a dictionary.
+
+    Args:
+        data: The dictionary obtained by calling :meth:`~Rule.asdict` on a rule object.
+            The dictionary must contain a key ``"rule_type"`` that indicates which rule
+            type to instantiate.
+
+    Returns:
+        The rule object as read from ``data``.
+    """
+    name = data["rule_type"]
+    if name not in _TYPE_MAPPING:
+        raise ValueError(f"Unknown rule type: {name}")
+    return _TYPE_MAPPING[name].from_dict(data)

dataframely-1.6.0/dataframely/_serialization.py

@@ -0,0 +1,88 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+import datetime as dt
+import decimal
+from io import BytesIO
+from json import JSONDecoder, JSONEncoder
+from typing import Any, cast
+
+import polars as pl
+
+
+class SchemaJSONEncoder(JSONEncoder):
+    """Custom JSON encoder to properly serialize all types serialized by schemas."""
+
+    def encode(self, obj: Any) -> str:
+        def hint_tuples(item: Any) -> Any:
+            if isinstance(item, tuple):
+                return {"__type__": "tuple", "value": list(item)}
+            if isinstance(item, list):
+                return [hint_tuples(i) for i in item]
+            if isinstance(item, dict):
+                return {k: hint_tuples(v) for k, v in item.items()}
+            return item
+
+        return super().encode(hint_tuples(obj))
+
+    def default(self, obj: Any) -> Any:
+        match obj:
+            case pl.Expr():
+                return {
+                    "__type__": "expression",
+                    "value": obj.meta.serialize(format="json"),
+                }
+            case decimal.Decimal():
+                return {"__type__": "decimal", "value": str(obj)}
+            case dt.datetime():
+                return {"__type__": "datetime", "value": obj.isoformat()}
+            case dt.date():
+                return {"__type__": "date", "value": obj.isoformat()}
+            case dt.time():
+                return {"__type__": "time", "value": obj.isoformat()}
+            case dt.timedelta():
+                return {"__type__": "timedelta", "value": obj.total_seconds()}
+            case dt.tzinfo():
+                offset = obj.utcoffset(dt.datetime.now())
+                return {
+                    "__type__": "tzinfo",
+                    "value": offset.total_seconds() if offset is not None else None,
+                }
+            case _:
+                return super().default(obj)
+
+
+class SchemaJSONDecoder(JSONDecoder):
+    """Custom JSON decoder to properly deserialize all types serialized by schemas."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, dct: dict[str, Any]) -> Any:
+        if "__type__" not in dct:
+            return dct
+
+        match dct["__type__"]:
+            case "tuple":
+                return tuple(dct["value"])
+            case "expression":
+                data = BytesIO(cast(str, dct["value"]).encode("utf-8"))
+                return pl.Expr.deserialize(data, format="json")
+            case "decimal":
+                return decimal.Decimal(dct["value"])
+            case "datetime":
+                return dt.datetime.fromisoformat(dct["value"])
+            case "date":
+                return dt.date.fromisoformat(dct["value"])
+            case "time":
+                return dt.time.fromisoformat(dct["value"])
+            case "timedelta":
+                return dt.timedelta(seconds=float(dct["value"]))
+            case "tzinfo":
+                return (
+                    dt.timezone(dt.timedelta(seconds=float(dct["value"])))
+                    if dct["value"] is not None
+                    else dt.timezone(dt.timedelta(0))
+                )
+            case _:
+                raise TypeError(f"Unknown type '{dct['__type__']}' in JSON data.")

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/collection.py

@@ -4,6 +4,7 @@
 import warnings
 from abc import ABC
 from collections.abc import Mapping, Sequence
+from dataclasses import asdict
 from pathlib import Path
 from typing import Any, Self, cast
 
@@ -225,6 +226,65 @@ class Collection(BaseCollection, ABC):
         # 3) Eventually, we initialize the final collection and return
         return cls.validate(members)
 
+    @classmethod
+    def matches(cls, other: type["Collection"]) -> bool:
+        """Check whether this collection semantically matches another.
+
+        Args:
+            other: The collection to compare with.
+
+        Returns:
+            Whether the two collections are semantically equal.
+
+        Attention:
+            For custom filters, reliable comparison results are only guaranteed
+            if the filter always returns a static polars expression.
+            Otherwise, this function may falsely indicate a match.
+        """
+
+        def _members_match() -> bool:
+            members_lhs = cls.members()
+            members_rhs = other.members()
+
+            # Member names must match
+            if members_lhs.keys() != members_rhs.keys():
+                return False
+
+            # Member attributes must match
+            for name in members_lhs:
+                lhs = asdict(members_lhs[name])
+                rhs = asdict(members_rhs[name])
+                for attr in lhs.keys() | rhs.keys():
+                    if attr == "schema":
+                        if not lhs[attr].matches(rhs[attr]):
+                            return False
+                    else:
+                        if lhs[attr] != rhs[attr]:
+                            return False
+            return True
+
+        def _filters_match() -> bool:
+            filters_lhs = cls._filters()
+            filters_rhs = other._filters()
+
+            # Filter names must match
+            if filters_lhs.keys() != filters_rhs.keys():
+                return False
+
+            # Computational graph of filter logic must match
+            # Evaluate on empty dataframes
+            empty_left = cls.create_empty()
+            empty_right = other.create_empty()
+
+            for name in filters_lhs:
+                lhs = filters_lhs[name].logic(empty_left)
+                rhs = filters_rhs[name].logic(empty_right)
+                if lhs.serialize(format="json") != rhs.serialize(format="json"):
+                    return False
+            return True
+
+        return _members_match() and _filters_match()
+
     @classmethod
     def _preprocess_sample(
         cls, sample: dict[str, Any], index: int, generator: Generator

{dataframely-1.5.0 → dataframely-1.6.0}/dataframely/columns/__init__.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 from ._base import Column
+from ._registry import column_from_dict
 from .any import Any
 from .array import Array
 from .bool import Bool
@@ -17,6 +18,7 @@ from .struct import Struct
 
 __all__ = [
     "Column",
+    "column_from_dict",
     "Any",
     "Array",
     "Bool",