dataframely 1.8.2__tar.gz → 1.10.0__tar.gz

{dataframely-1.8.2 → dataframely-1.10.0}/.github/workflows/ci.yml

@@ -41,6 +41,14 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
         environment: [py310, py311, py312, py313]
+        with_optionals: [false]
+        include:
+          - os: ubuntu-latest
+            environment: py313-optionals
+            with_optionals: true
+          - os: windows-latest
+            environment: py313-optionals
+            with_optionals: true
     steps:
       - name: Checkout branch
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -51,7 +59,7 @@ jobs:
       - name: Install repository
         run: pixi run -e ${{ matrix.environment }} postinstall
       - name: Run pytest
-        run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes
+        run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes ${{ matrix.with_optionals && '-m with_optionals' || '-m "not with_optionals"'}} --cov=dataframely --cov-report=xml
       - name: Upload codecov
         uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
         with:

{dataframely-1.8.2 → dataframely-1.10.0}/.gitignore

@@ -388,3 +388,6 @@ $RECYCLE.BIN/
 
 # interactive testing
 /*.ipynb
+
+# pytest-benchmark
+.benchmarks/

{dataframely-1.8.2 → dataframely-1.10.0}/Cargo.toml

@@ -2,6 +2,7 @@
 edition = "2021"
 name = "dataframely"
 version = "0.1.0"
+readme = "README.md"
 
 [lib]
 crate-type = ["cdylib"]

{dataframely-1.8.2 → dataframely-1.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 1.8.2
+Version: 1.10.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -9,6 +9,12 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: numpy
 Requires-Dist: polars>=1.32
 Requires-Dist: typing-extensions ; python_full_version < '3.11'
+Requires-Dist: deltalake ; extra == 'deltalake'
+Requires-Dist: sqlalchemy ; extra == 'sqlalchemy'
+Requires-Dist: pyarrow ; extra == 'pyarrow'
+Provides-Extra: deltalake
+Provides-Extra: sqlalchemy
+Provides-Extra: pyarrow
 License-File: LICENSE
 Summary: A declarative, polars-native data frame validation library
 Author-email: Andreas Albert <andreas.albert@quantco.com>, Daniel Elsner <daniel.elsner@quantco.com>, Oliver Borchert <oliver.borchert@quantco.com>

{dataframely-1.8.2 → dataframely-1.10.0}/dataframely/_base_collection.py

@@ -66,8 +66,8 @@ class CollectionMember:
 # --------------------------------------- UTILS -------------------------------------- #
 
 
-def _common_primary_keys(columns: Iterable[type[Schema]]) -> set[str]:
-    return set.intersection(*[set(schema.primary_keys()) for schema in columns])
+def _common_primary_keys(schemas: Iterable[type[Schema]]) -> set[str]:
+    return set.intersection(*[set(schema.primary_keys()) for schema in schemas])
 
 
 # ------------------------------------------------------------------------------------ #
@@ -137,7 +137,7 @@ class CollectionMeta(ABCMeta):
                     for member in result.members.values()
                 ),
                 *(
-                    set(member.schema._validation_rules())
+                    set(member.schema._validation_rules(with_cast=True))
                     for member in result.members.values()
                 ),
             )

{dataframely-1.8.2 → dataframely-1.10.0}/dataframely/_base_schema.py

@@ -12,7 +12,7 @@ from typing import Any
 
 import polars as pl
 
-from ._rule import GroupRule, Rule
+from ._rule import DtypeCastRule, GroupRule, Rule
 from .columns import Column
 from .exc import ImplementationError
 
@@ -24,11 +24,13 @@ else:
 _COLUMN_ATTR = "__dataframely_columns__"
 _RULE_ATTR = "__dataframely_rules__"
 
+ORIGINAL_COLUMN_PREFIX = "__DATAFRAMELY_ORIGINAL__"
+
 # --------------------------------------- UTILS -------------------------------------- #
 
 
 def _build_rules(
-    custom: dict[str, Rule], columns: dict[str, Column]
+    custom: dict[str, Rule], columns: dict[str, Column], *, with_cast: bool
 ) -> dict[str, Rule]:
     # NOTE: Copy here to prevent in-place modification of the custom rules
     rules: dict[str, Rule] = copy(custom)
@@ -46,6 +48,21 @@ def _build_rules(
     }
     rules.update(column_rules)
 
+    # Add casting rules if requested. Here, we can simply check whether the nullability
+    # property of a column changes due to lenient dtype casting. Whenever casting fails,
+    # the value is set to `null`, mismatching the previous nullability.
+    # NOTE: This check assumes that both the original and cast column are present in the
+    #  data frame.
+    if with_cast:
+        casting_rules = {
+            f"{col_name}|dtype": DtypeCastRule(
+                pl.col(col_name).is_null()
+                == pl.col(f"{ORIGINAL_COLUMN_PREFIX}{col_name}").is_null()
+            )
+            for col_name in columns
+        }
+        rules.update(casting_rules)
+
     return rules
 
 
@@ -93,9 +110,7 @@ class SchemaMeta(ABCMeta):
         # we assume that users cast dtypes, i.e. additional rules for dtype casting
         # are also checked.
         all_column_names = set(result.columns)
-        all_rule_names = set(_build_rules(result.rules, result.columns).keys()) | set(
-            f"{col}|dtype" for col in result.columns
-        )
+        all_rule_names = set(_build_rules(result.rules, result.columns, with_cast=True))
         common_names = all_column_names & all_rule_names
         if len(common_names) > 0:
             common_list = ", ".join(sorted(f"'{col}'" for col in common_names))
@@ -189,8 +204,10 @@ class BaseSchema(metaclass=SchemaMeta):
         return _primary_keys(cls.columns())
 
     @classmethod
-    def _validation_rules(cls) -> dict[str, Rule]:
-        return _build_rules(cls._schema_validation_rules(), cls.columns())
+    def _validation_rules(cls, *, with_cast: bool) -> dict[str, Rule]:
+        return _build_rules(
+            cls._schema_validation_rules(), cls.columns(), with_cast=with_cast
+        )
 
     @classmethod
     def _schema_validation_rules(cls) -> dict[str, Rule]:

{dataframely-1.8.2 → dataframely-1.10.0}/dataframely/_compat.py

@@ -13,11 +13,24 @@ class _DummyModule:  # pragma: no cover
         raise ValueError(f"Module '{self.module}' is not installed.")
 
 
+# ------------------------------------ DELTALAKE ------------------------------------- #
+
+try:
+    import deltalake
+    from deltalake import DeltaTable
+except ImportError:  # pragma: no cover
+    deltalake = _DummyModule("deltalake")  # type: ignore
+
+    class DeltaTable:  # type: ignore # noqa: N801
+        pass
 # ------------------------------------ SQLALCHEMY ------------------------------------ #
 
 try:
     import sqlalchemy as sa
     import sqlalchemy.dialects.mssql as sa_mssql
+    from sqlalchemy import Dialect
+    from sqlalchemy.dialects.mssql.pyodbc import MSDialect_pyodbc
+    from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
     from sqlalchemy.sql.type_api import TypeEngine as sa_TypeEngine
 except ImportError:  # pragma: no cover
     sa = _DummyModule("sqlalchemy")  # type: ignore
@@ -26,7 +39,14 @@ except ImportError:  # pragma: no cover
     class sa_TypeEngine:  # type: ignore # noqa: N801
         pass
 
+    class MSDialect_pyodbc:  # type: ignore # noqa: N801
+        pass
+
+    class PGDialect_psycopg2:  # type: ignore # noqa: N801
+        pass
 
+    class Dialect:  # type: ignore # noqa: N801
+        pass
 # -------------------------------------- PYARROW ------------------------------------- #
 
 try:
@@ -36,4 +56,12 @@ except ImportError:  # pragma: no cover
 
 # ------------------------------------------------------------------------------------ #
 
-__all__ = ["sa", "sa_mssql", "sa_TypeEngine", "pa"]
+__all__ = [
+    "deltalake",
+    "sa",
+    "sa_mssql",
+    "sa_TypeEngine",
+    "pa",
+    "MSDialect_pyodbc",
+    "PGDialect_psycopg2",
+]

{dataframely-1.8.2 → dataframely-1.10.0}/dataframely/_polars.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import datetime as dt
-from collections.abc import Iterable
 from typing import TypeVar
 
 import polars as pl
@@ -14,27 +13,6 @@ FrameType = TypeVar("FrameType", pl.DataFrame, pl.LazyFrame)
 EPOCH_DATETIME = dt.datetime(1970, 1, 1)
 SECONDS_PER_DAY = 86400
 
-# --------------------------------------- JOINS -------------------------------------- #
-
-
-def join_all_inner(dfs: Iterable[FrameType], on: str | list[str]) -> FrameType:
-    it = iter(dfs)
-    result = next(it)
-    while (df := next(it, None)) is not None:
-        result = result.join(df, on=on)
-    return result
-
-
-def join_all_outer(dfs: Iterable[FrameType], on: str | list[str]) -> FrameType:
-    it = iter(dfs)
-    result = next(it)
-    while (df := next(it, None)) is not None:
-        result = result.join(df, on=on, how="full", coalesce=True)
-    return result
-
-
-# ------------------------------------- DATETIMES ------------------------------------ #
-
 
 def date_matches_resolution(t: dt.date, resolution: str) -> bool:
     return pl.Series([t], dtype=pl.Date).dt.truncate(resolution).item() == t

{dataframely-1.8.2 → dataframely-1.10.0}/dataframely/_rule.py

@@ -59,6 +59,14 @@ class Rule:
         return str(self.expr)
 
 
+class DtypeCastRule(Rule):
+    """Rule that evaluates whether casting a column to another dtype is successful.
+
+    The only purpose of this rule is to provide a runtime type to distinguish it from
+    other rules.
+    """
+
+
 class GroupRule(Rule):
     """Rule that is evaluated on a group of columns."""
 
@@ -147,7 +155,8 @@ def with_evaluation_rules(lf: pl.LazyFrame, rules: dict[str, Rule]) -> pl.LazyFr
         while ``False`` indicates an issue.
     """
     # Rules must be distinguished into two types of rules:
-    #  1. Simple rules can simply be selected on the data frame
+    #  1. Simple rules can simply be selected on the data frame (this includes rules
+    #     that check whether dtype casts succeeded)
     #  2. "Group" rules require a `group_by` and a subsequent join
     simple_exprs = {
         name: rule.expr
@@ -160,7 +169,7 @@ def with_evaluation_rules(lf: pl.LazyFrame, rules: dict[str, Rule]) -> pl.LazyFr
 
     # Before we can select all of the simple expressions, we need to turn the
     # group rules into something to use in a `select` statement as well.
-    return (
+    result = (
         # NOTE: A value of `null` always validates successfully as nullability should
         #  already be checked via dedicated rules.
         _with_group_rules(lf, group_rules).with_columns(
@@ -168,6 +177,27 @@ def with_evaluation_rules(lf: pl.LazyFrame, rules: dict[str, Rule]) -> pl.LazyFr
         )
     )
 
+    # If there is at least one rule that checks for successful dtype casting, we need
+    # to take an extra step: rules other than the "dtype rules" might not be reliable
+    # if casting failed, i.e. if any of the "dtype rules" evaluated to `False`. For
+    # this reason, we set all other rule evaluations to `null` in the case of dtype
+    # casting failure.
+    dtype_rule_names = [
+        name for name, rule in rules.items() if isinstance(rule, DtypeCastRule)
+    ]
+    if len(dtype_rule_names) > 0:
+        non_dtype_rule_names = [
+            name for name, rule in rules.items() if not isinstance(rule, DtypeCastRule)
+        ]
+        all_dtype_casts_valid = pl.all_horizontal(dtype_rule_names)
+        return result.with_columns(
+            pl.when(all_dtype_casts_valid)
+            .then(pl.col(non_dtype_rule_names))
+            .otherwise(pl.lit(None, dtype=pl.Boolean))
+        )
+
+    return result
+
 
 def _with_group_rules(lf: pl.LazyFrame, rules: dict[str, GroupRule]) -> pl.LazyFrame:
     # First, we partition the rules by group columns. This will minimize the number
@@ -184,16 +214,12 @@ def _with_group_rules(lf: pl.LazyFrame, rules: dict[str, GroupRule]) -> pl.LazyF
         # We group by the group columns and apply all expressions
         group_evaluations[group_columns] = lf.group_by(group_columns).agg(**group_rules)
 
-    # Eventually, we apply the rule evaluations onto the input data frame. For this,
-    # we're using left-joins. This has two effects:
-    #  - We're essentially "broadcasting" the results within each group across rows
-    #    in the same group.
-    #  - While an inner-join would be semantically more accurate, the left-join
-    #    preserves the order of the left data frame.
+    # Eventually, we apply the rule evaluations onto the input data frame. For this, we
+    # "broadcast" the results within each group across rows in the same group.
     result = lf
     for group_columns, frame in group_evaluations.items():
         result = result.join(
-            frame, on=list(group_columns), how="left", nulls_equal=True
+            frame, on=list(group_columns), nulls_equal=True, maintain_order="left"
         )
     return result
 

{dataframely-1.8.2 → dataframely-1.10.0}/dataframely/_serialization.py

@@ -10,8 +10,6 @@ from typing import Any, cast
 
 import polars as pl
 
-SCHEMA_METADATA_KEY = "dataframely_schema"
-COLLECTION_METADATA_KEY = "dataframely_collection"
 SERIALIZATION_FORMAT_VERSION = "1"
 
 

dataframely-1.10.0/dataframely/_storage/__init__.py

@@ -0,0 +1,8 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+from ._base import StorageBackend
+
+__all__ = [
+    "StorageBackend",
+]

dataframely-1.10.0/dataframely/_storage/_base.py

@@ -0,0 +1,204 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from typing import Any
+
+import polars as pl
+
+SerializedSchema = str
+SerializedCollection = str
+SerializedRules = str
+
+
+class StorageBackend(ABC):
+    """Base class for storage backends.
+
+    A storage backend encapsulates a way of serializing and deserializing dataframlely
+    data-/lazyframes and collections. This base class provides a unified interface for
+    all such use cases.
+
+    The interface is designed to operate on data provided as polars frames, and metadata
+    provided as serialized strings. This design is meant to limit the coupling between
+    the Schema/Collection classes and specifics of how data and metadata is stored.
+    """
+
+    # ----------------------------------- Schemas -------------------------------------
+    @abstractmethod
+    def sink_frame(
+        self, lf: pl.LazyFrame, serialized_schema: SerializedSchema, **kwargs: Any
+    ) -> None:
+        """Stream the contents of a dataframe, and its metadata to the storage backend.
+
+        Args:
+            lf: A frame containing the data to be stored.
+            serialized_schema: String-serialized schema information.
+            kwargs: Additional keyword arguments to pass to the underlying storage
+                implementation.
+        """
+
+    @abstractmethod
+    def write_frame(
+        self, df: pl.DataFrame, serialized_schema: SerializedSchema, **kwargs: Any
+    ) -> None:
+        """Write the contents of a dataframe, and its metadata to the storage backend.
+
+        Args:
+            df: A dataframe containing the data to be stored.
+            frame: String-serialized schema information.
+            kwargs: Additional keyword arguments to pass to the underlying storage
+                implementation.
+        """
+
+    @abstractmethod
+    def scan_frame(self, **kwargs: Any) -> tuple[pl.LazyFrame, SerializedSchema | None]:
+        """Lazily read frame data and metadata from the storage backend.
+
+        Args:
+            kwargs: Keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords
+                are available.
+        Returns:
+            A tuple of the lazy frame data and metadata if available.
+        """
+
+    @abstractmethod
+    def read_frame(self, **kwargs: Any) -> tuple[pl.DataFrame, SerializedSchema | None]:
+        """Eagerly read frame data and metadata from the storage backend.
+
+        Args:
+            kwargs: Keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords
+                are available.
+        Returns:
+            A tuple of the lazy frame data and metadata if available.
+        """
+
+    # ------------------------------ Collections ---------------------------------------
+    @abstractmethod
+    def sink_collection(
+        self,
+        dfs: dict[str, pl.LazyFrame],
+        serialized_collection: SerializedCollection,
+        serialized_schemas: dict[str, str],
+        **kwargs: Any,
+    ) -> None:
+        """Stream the members of this collection into the storage backend.
+
+        Args:
+            dfs: Dictionary containing the data to be stored.
+            serialized_collection: String-serialized information about the origin Collection.
+            serialized_schemas: String-serialized information about the individual Schemas
+                for each of the member frames. This information is also logically included
+                in the collection metadata, but it is passed separately here to ensure that
+                each member can also be read back as an individual frame.
+        """
+
+    @abstractmethod
+    def write_collection(
+        self,
+        dfs: dict[str, pl.LazyFrame],
+        serialized_collection: SerializedCollection,
+        serialized_schemas: dict[str, str],
+        **kwargs: Any,
+    ) -> None:
+        """Write the members of this collection into the storage backend.
+
+        Args:
+            dfs: Dictionary containing the data to be stored.
+            serialized_collection: String-serialized information about the origin Collection.
+            serialized_schemas: String-serialized information about the individual Schemas
+                for each of the member frames. This information is also logically included
+                in the collection metadata, but it is passed separately here to ensure that
+                each member can also be read back as an individual frame.
+        """
+
+    @abstractmethod
+    def scan_collection(
+        self, members: Iterable[str], **kwargs: Any
+    ) -> tuple[dict[str, pl.LazyFrame], list[SerializedCollection | None]]:
+        """Lazily read  all collection members from the storage backend.
+
+        Args:
+            members: Collection member names to read.
+            kwargs: Additional keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords are available.
+        Returns:
+            A tuple of the collection data and metadata if available.
+            Depending on the storage implementation, multiple copies of the metadata
+            may be available, which are returned as a list.
+            It is up to the caller to decide how to handle the presence/absence/consistency
+            of the returned values.
+        """
+
+    @abstractmethod
+    def read_collection(
+        self, members: Iterable[str], **kwargs: Any
+    ) -> tuple[dict[str, pl.LazyFrame], list[SerializedCollection | None]]:
+        """Lazily read  all collection members from the storage backend.
+
+        Args:
+            members: Collection member names to read.
+            kwargs: Additional keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords are available.
+        Returns:
+            A tuple of the collection data and metadata if available.
+            Depending on the storage implementation, multiple copies of the metadata
+            may be available, which are returned as a list.
+            It is up to the caller to decide how to handle the presence/absence/consistency
+            of the returned values.
+        """
+
+    # ------------------------------ Failure Info --------------------------------------
+    @abstractmethod
+    def sink_failure_info(
+        self,
+        lf: pl.LazyFrame,
+        serialized_rules: SerializedRules,
+        serialized_schema: SerializedSchema,
+        **kwargs: Any,
+    ) -> None:
+        """Stream the failure info to the storage backend.
+
+        Args:
+            lf: LazyFrame backing the failure info.
+            serialized_rules: JSON-serialized list of rule column names
+                used for validation.
+            serialized_schema: String-serialized schema information.
+        """
+
+    @abstractmethod
+    def write_failure_info(
+        self,
+        df: pl.DataFrame,
+        serialized_rules: SerializedRules,
+        serialized_schema: SerializedSchema,
+        **kwargs: Any,
+    ) -> None:
+        """Write the failure info to the storage backend.
+
+        Args:
+            df: DataFrame backing the failure info.
+            serialized_rules: JSON-serialized list of rule column names
+                used for validation.
+            serialized_schema: String-serialized schema information.
+        """
+
+    @abstractmethod
+    def scan_failure_info(
+        self, **kwargs: Any
+    ) -> tuple[pl.LazyFrame, SerializedRules, SerializedSchema]:
+        """Lazily read the failure info from the storage backend."""
+
+    def read_failure_info(
+        self, **kwargs: Any
+    ) -> tuple[pl.DataFrame, SerializedRules, SerializedSchema]:
+        """Read the failure info from the storage backend."""
+
+        lf, rule_metadata, schema_metadata = self.scan_failure_info(**kwargs)
+        return (
+            lf.collect(),
+            rule_metadata,
+            schema_metadata,
+        )

dataframely-1.10.0/dataframely/_storage/_exc.py

@@ -0,0 +1,10 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+
+def assert_failure_info_metadata(metadata: str | None) -> str:
+    if metadata:
+        return metadata
+    raise ValueError(
+        "The required FailureInfo metadata was not found in the storage backend."
+    )

dataframely-1.10.0/dataframely/_storage/constants.py

@@ -0,0 +1,6 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+SCHEMA_METADATA_KEY = "dataframely_schema"
+COLLECTION_METADATA_KEY = "dataframely_collection"
+RULE_METADATA_KEY = "dataframely_rule_columns"