dataframely 1.9.0__tar.gz → 1.10.0__tar.gz

{dataframely-1.9.0 → dataframely-1.10.0}/.github/workflows/ci.yml

@@ -41,6 +41,14 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
         environment: [py310, py311, py312, py313]
+        with_optionals: [false]
+        include:
+          - os: ubuntu-latest
+            environment: py313-optionals
+            with_optionals: true
+          - os: windows-latest
+            environment: py313-optionals
+            with_optionals: true
     steps:
       - name: Checkout branch
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -51,7 +59,7 @@ jobs:
       - name: Install repository
         run: pixi run -e ${{ matrix.environment }} postinstall
       - name: Run pytest
-        run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes
+        run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes ${{ matrix.with_optionals && '-m with_optionals' || '-m "not with_optionals"'}} --cov=dataframely --cov-report=xml
       - name: Upload codecov
         uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
         with:

{dataframely-1.9.0 → dataframely-1.10.0}/Cargo.toml

@@ -2,6 +2,7 @@
 edition = "2021"
 name = "dataframely"
 version = "0.1.0"
+readme = "README.md"
 
 [lib]
 crate-type = ["cdylib"]

{dataframely-1.9.0 → dataframely-1.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 1.9.0
+Version: 1.10.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -9,6 +9,12 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: numpy
 Requires-Dist: polars>=1.32
 Requires-Dist: typing-extensions ; python_full_version < '3.11'
+Requires-Dist: deltalake ; extra == 'deltalake'
+Requires-Dist: sqlalchemy ; extra == 'sqlalchemy'
+Requires-Dist: pyarrow ; extra == 'pyarrow'
+Provides-Extra: deltalake
+Provides-Extra: sqlalchemy
+Provides-Extra: pyarrow
 License-File: LICENSE
 Summary: A declarative, polars-native data frame validation library
 Author-email: Andreas Albert <andreas.albert@quantco.com>, Daniel Elsner <daniel.elsner@quantco.com>, Oliver Borchert <oliver.borchert@quantco.com>

{dataframely-1.9.0 → dataframely-1.10.0}/dataframely/_compat.py

@@ -13,11 +13,24 @@ class _DummyModule:  # pragma: no cover
         raise ValueError(f"Module '{self.module}' is not installed.")
 
 
+# ------------------------------------ DELTALAKE ------------------------------------- #
+
+try:
+    import deltalake
+    from deltalake import DeltaTable
+except ImportError:  # pragma: no cover
+    deltalake = _DummyModule("deltalake")  # type: ignore
+
+    class DeltaTable:  # type: ignore # noqa: N801
+        pass
 # ------------------------------------ SQLALCHEMY ------------------------------------ #
 
 try:
     import sqlalchemy as sa
     import sqlalchemy.dialects.mssql as sa_mssql
+    from sqlalchemy import Dialect
+    from sqlalchemy.dialects.mssql.pyodbc import MSDialect_pyodbc
+    from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
     from sqlalchemy.sql.type_api import TypeEngine as sa_TypeEngine
 except ImportError:  # pragma: no cover
     sa = _DummyModule("sqlalchemy")  # type: ignore
@@ -26,7 +39,14 @@ except ImportError:  # pragma: no cover
     class sa_TypeEngine:  # type: ignore # noqa: N801
         pass
 
+    class MSDialect_pyodbc:  # type: ignore # noqa: N801
+        pass
+
+    class PGDialect_psycopg2:  # type: ignore # noqa: N801
+        pass
 
+    class Dialect:  # type: ignore # noqa: N801
+        pass
 # -------------------------------------- PYARROW ------------------------------------- #
 
 try:
@@ -36,4 +56,12 @@ except ImportError:  # pragma: no cover
 
 # ------------------------------------------------------------------------------------ #
 
-__all__ = ["sa", "sa_mssql", "sa_TypeEngine", "pa"]
+__all__ = [
+    "deltalake",
+    "sa",
+    "sa_mssql",
+    "sa_TypeEngine",
+    "pa",
+    "MSDialect_pyodbc",
+    "PGDialect_psycopg2",
+]

{dataframely-1.9.0 → dataframely-1.10.0}/dataframely/_serialization.py

@@ -10,8 +10,6 @@ from typing import Any, cast
 
 import polars as pl
 
-SCHEMA_METADATA_KEY = "dataframely_schema"
-COLLECTION_METADATA_KEY = "dataframely_collection"
 SERIALIZATION_FORMAT_VERSION = "1"
 
 

dataframely-1.10.0/dataframely/_storage/__init__.py

@@ -0,0 +1,8 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+from ._base import StorageBackend
+
+__all__ = [
+    "StorageBackend",
+]

dataframely-1.10.0/dataframely/_storage/_base.py

@@ -0,0 +1,204 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from typing import Any
+
+import polars as pl
+
+SerializedSchema = str
+SerializedCollection = str
+SerializedRules = str
+
+
+class StorageBackend(ABC):
+    """Base class for storage backends.
+
+    A storage backend encapsulates a way of serializing and deserializing dataframlely
+    data-/lazyframes and collections. This base class provides a unified interface for
+    all such use cases.
+
+    The interface is designed to operate on data provided as polars frames, and metadata
+    provided as serialized strings. This design is meant to limit the coupling between
+    the Schema/Collection classes and specifics of how data and metadata is stored.
+    """
+
+    # ----------------------------------- Schemas -------------------------------------
+    @abstractmethod
+    def sink_frame(
+        self, lf: pl.LazyFrame, serialized_schema: SerializedSchema, **kwargs: Any
+    ) -> None:
+        """Stream the contents of a dataframe, and its metadata to the storage backend.
+
+        Args:
+            lf: A frame containing the data to be stored.
+            serialized_schema: String-serialized schema information.
+            kwargs: Additional keyword arguments to pass to the underlying storage
+                implementation.
+        """
+
+    @abstractmethod
+    def write_frame(
+        self, df: pl.DataFrame, serialized_schema: SerializedSchema, **kwargs: Any
+    ) -> None:
+        """Write the contents of a dataframe, and its metadata to the storage backend.
+
+        Args:
+            df: A dataframe containing the data to be stored.
+            frame: String-serialized schema information.
+            kwargs: Additional keyword arguments to pass to the underlying storage
+                implementation.
+        """
+
+    @abstractmethod
+    def scan_frame(self, **kwargs: Any) -> tuple[pl.LazyFrame, SerializedSchema | None]:
+        """Lazily read frame data and metadata from the storage backend.
+
+        Args:
+            kwargs: Keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords
+                are available.
+        Returns:
+            A tuple of the lazy frame data and metadata if available.
+        """
+
+    @abstractmethod
+    def read_frame(self, **kwargs: Any) -> tuple[pl.DataFrame, SerializedSchema | None]:
+        """Eagerly read frame data and metadata from the storage backend.
+
+        Args:
+            kwargs: Keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords
+                are available.
+        Returns:
+            A tuple of the lazy frame data and metadata if available.
+        """
+
+    # ------------------------------ Collections ---------------------------------------
+    @abstractmethod
+    def sink_collection(
+        self,
+        dfs: dict[str, pl.LazyFrame],
+        serialized_collection: SerializedCollection,
+        serialized_schemas: dict[str, str],
+        **kwargs: Any,
+    ) -> None:
+        """Stream the members of this collection into the storage backend.
+
+        Args:
+            dfs: Dictionary containing the data to be stored.
+            serialized_collection: String-serialized information about the origin Collection.
+            serialized_schemas: String-serialized information about the individual Schemas
+                for each of the member frames. This information is also logically included
+                in the collection metadata, but it is passed separately here to ensure that
+                each member can also be read back as an individual frame.
+        """
+
+    @abstractmethod
+    def write_collection(
+        self,
+        dfs: dict[str, pl.LazyFrame],
+        serialized_collection: SerializedCollection,
+        serialized_schemas: dict[str, str],
+        **kwargs: Any,
+    ) -> None:
+        """Write the members of this collection into the storage backend.
+
+        Args:
+            dfs: Dictionary containing the data to be stored.
+            serialized_collection: String-serialized information about the origin Collection.
+            serialized_schemas: String-serialized information about the individual Schemas
+                for each of the member frames. This information is also logically included
+                in the collection metadata, but it is passed separately here to ensure that
+                each member can also be read back as an individual frame.
+        """
+
+    @abstractmethod
+    def scan_collection(
+        self, members: Iterable[str], **kwargs: Any
+    ) -> tuple[dict[str, pl.LazyFrame], list[SerializedCollection | None]]:
+        """Lazily read  all collection members from the storage backend.
+
+        Args:
+            members: Collection member names to read.
+            kwargs: Additional keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords are available.
+        Returns:
+            A tuple of the collection data and metadata if available.
+            Depending on the storage implementation, multiple copies of the metadata
+            may be available, which are returned as a list.
+            It is up to the caller to decide how to handle the presence/absence/consistency
+            of the returned values.
+        """
+
+    @abstractmethod
+    def read_collection(
+        self, members: Iterable[str], **kwargs: Any
+    ) -> tuple[dict[str, pl.LazyFrame], list[SerializedCollection | None]]:
+        """Lazily read  all collection members from the storage backend.
+
+        Args:
+            members: Collection member names to read.
+            kwargs: Additional keyword arguments to pass to the underlying storage.
+                Refer to the individual implementation to see which keywords are available.
+        Returns:
+            A tuple of the collection data and metadata if available.
+            Depending on the storage implementation, multiple copies of the metadata
+            may be available, which are returned as a list.
+            It is up to the caller to decide how to handle the presence/absence/consistency
+            of the returned values.
+        """
+
+    # ------------------------------ Failure Info --------------------------------------
+    @abstractmethod
+    def sink_failure_info(
+        self,
+        lf: pl.LazyFrame,
+        serialized_rules: SerializedRules,
+        serialized_schema: SerializedSchema,
+        **kwargs: Any,
+    ) -> None:
+        """Stream the failure info to the storage backend.
+
+        Args:
+            lf: LazyFrame backing the failure info.
+            serialized_rules: JSON-serialized list of rule column names
+                used for validation.
+            serialized_schema: String-serialized schema information.
+        """
+
+    @abstractmethod
+    def write_failure_info(
+        self,
+        df: pl.DataFrame,
+        serialized_rules: SerializedRules,
+        serialized_schema: SerializedSchema,
+        **kwargs: Any,
+    ) -> None:
+        """Write the failure info to the storage backend.
+
+        Args:
+            df: DataFrame backing the failure info.
+            serialized_rules: JSON-serialized list of rule column names
+                used for validation.
+            serialized_schema: String-serialized schema information.
+        """
+
+    @abstractmethod
+    def scan_failure_info(
+        self, **kwargs: Any
+    ) -> tuple[pl.LazyFrame, SerializedRules, SerializedSchema]:
+        """Lazily read the failure info from the storage backend."""
+
+    def read_failure_info(
+        self, **kwargs: Any
+    ) -> tuple[pl.DataFrame, SerializedRules, SerializedSchema]:
+        """Read the failure info from the storage backend."""
+
+        lf, rule_metadata, schema_metadata = self.scan_failure_info(**kwargs)
+        return (
+            lf.collect(),
+            rule_metadata,
+            schema_metadata,
+        )

dataframely-1.10.0/dataframely/_storage/_exc.py

@@ -0,0 +1,10 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+
+def assert_failure_info_metadata(metadata: str | None) -> str:
+    if metadata:
+        return metadata
+    raise ValueError(
+        "The required FailureInfo metadata was not found in the storage backend."
+    )

dataframely-1.10.0/dataframely/_storage/constants.py

@@ -0,0 +1,6 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+SCHEMA_METADATA_KEY = "dataframely_schema"
+COLLECTION_METADATA_KEY = "dataframely_collection"
+RULE_METADATA_KEY = "dataframely_rule_columns"

dataframely-1.10.0/dataframely/_storage/delta.py

@@ -0,0 +1,200 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+from dataframely._compat import deltalake
+
+from ._base import (
+    SerializedCollection,
+    SerializedRules,
+    SerializedSchema,
+    StorageBackend,
+)
+from ._exc import assert_failure_info_metadata
+from .constants import COLLECTION_METADATA_KEY, RULE_METADATA_KEY, SCHEMA_METADATA_KEY
+
+
+class DeltaStorageBackend(StorageBackend):
+    def sink_frame(
+        self, lf: pl.LazyFrame, serialized_schema: SerializedSchema, **kwargs: Any
+    ) -> None:
+        _raise_on_lazy_write()
+
+    def write_frame(
+        self, df: pl.DataFrame, serialized_schema: SerializedSchema, **kwargs: Any
+    ) -> None:
+        target = kwargs.pop("target")
+        metadata = kwargs.pop("metadata", {})
+        delta_write_options = kwargs.pop("delta_write_options", {})
+
+        # Delta lake does not allow partitioning if there is only one column
+        # We dynamically remove this setting here to allow users to still specify it
+        # on the collection level without having to worry about each individual member
+        if len(df.columns) < 2:
+            delta_write_options.pop("partition_by", None)
+
+        df.write_delta(
+            target,
+            delta_write_options=(
+                delta_write_options
+                | {
+                    "commit_properties": deltalake.CommitProperties(
+                        custom_metadata=metadata
+                        | {SCHEMA_METADATA_KEY: serialized_schema}
+                    ),
+                }
+            ),
+            **kwargs,
+        )
+
+    def scan_frame(self, **kwargs: Any) -> tuple[pl.LazyFrame, SerializedSchema | None]:
+        table = _to_delta_table(kwargs.pop("source"))
+        serialized_schema = _read_serialized_schema(table)
+        df = pl.scan_delta(table, **kwargs)
+        return df, serialized_schema
+
+    def read_frame(self, **kwargs: Any) -> tuple[pl.DataFrame, SerializedSchema | None]:
+        table = _to_delta_table(kwargs.pop("source"))
+        serialized_schema = _read_serialized_schema(table)
+        df = pl.read_delta(table, **kwargs)
+        return df, serialized_schema
+
+    # ------------------------------ Collections ---------------------------------------
+    def sink_collection(
+        self,
+        dfs: dict[str, pl.LazyFrame],
+        serialized_collection: SerializedCollection,
+        serialized_schemas: dict[str, str],
+        **kwargs: Any,
+    ) -> None:
+        _raise_on_lazy_write()
+
+    def write_collection(
+        self,
+        dfs: dict[str, pl.LazyFrame],
+        serialized_collection: SerializedCollection,
+        serialized_schemas: dict[str, str],
+        **kwargs: Any,
+    ) -> None:
+        uri = Path(kwargs.pop("target"))
+
+        # The collection schema is serialized as part of the member parquet metadata
+        kwargs["metadata"] = kwargs.get("metadata", {}) | {
+            COLLECTION_METADATA_KEY: serialized_collection
+        }
+
+        for key, lf in dfs.items():
+            self.write_frame(
+                lf.collect(),
+                serialized_schema=serialized_schemas[key],
+                target=uri / key,
+                **kwargs,
+            )
+
+    def scan_collection(
+        self, members: Iterable[str], **kwargs: Any
+    ) -> tuple[dict[str, pl.LazyFrame], list[SerializedCollection | None]]:
+        uri = Path(kwargs.pop("source"))
+
+        data = {}
+        collection_types = []
+        for key in members:
+            member_uri = uri / key
+            if not deltalake.DeltaTable.is_deltatable(str(member_uri)):
+                continue
+            table = _to_delta_table(member_uri)
+            data[key] = pl.scan_delta(table, **kwargs)
+            collection_types.append(_read_serialized_collection(table))
+
+        return data, collection_types
+
+    def read_collection(
+        self, members: Iterable[str], **kwargs: Any
+    ) -> tuple[dict[str, pl.LazyFrame], list[SerializedCollection | None]]:
+        lazy, collection_types = self.scan_collection(members, **kwargs)
+        eager = {name: lf.collect().lazy() for name, lf in lazy.items()}
+        return eager, collection_types
+
+    # ------------------------------ Failure Info --------------------------------------
+    def sink_failure_info(
+        self,
+        lf: pl.LazyFrame,
+        serialized_rules: SerializedRules,
+        serialized_schema: SerializedSchema,
+        **kwargs: Any,
+    ) -> None:
+        _raise_on_lazy_write()
+
+    def write_failure_info(
+        self,
+        df: pl.DataFrame,
+        serialized_rules: SerializedRules,
+        serialized_schema: SerializedSchema,
+        **kwargs: Any,
+    ) -> None:
+        self.write_frame(
+            df,
+            serialized_schema,
+            metadata={
+                RULE_METADATA_KEY: serialized_rules,
+            },
+            **kwargs,
+        )
+
+    def scan_failure_info(
+        self, **kwargs: Any
+    ) -> tuple[pl.LazyFrame, SerializedRules, SerializedSchema]:
+        """Lazily read the failure info from the storage backend."""
+        table = _to_delta_table(kwargs.pop("source"))
+
+        # Metadata
+        serialized_rules = assert_failure_info_metadata(_read_serialized_rules(table))
+        serialized_schema = assert_failure_info_metadata(_read_serialized_schema(table))
+
+        # Data
+        lf = pl.scan_delta(table, **kwargs)
+
+        return lf, serialized_rules, serialized_schema
+
+
+def _raise_on_lazy_write() -> None:
+    raise NotImplementedError("Lazy writes are not currently supported for deltalake.")
+
+
+def _read_serialized_schema(table: deltalake.DeltaTable) -> SerializedSchema | None:
+    [last_commit] = table.history(limit=1)
+    return last_commit.get(SCHEMA_METADATA_KEY, None)
+
+
+def _read_serialized_collection(
+    table: deltalake.DeltaTable,
+) -> SerializedCollection | None:
+    [last_commit] = table.history(limit=1)
+    return last_commit.get(COLLECTION_METADATA_KEY, None)
+
+
+def _read_serialized_rules(
+    table: deltalake.DeltaTable,
+) -> SerializedRules | None:
+    [last_commit] = table.history(limit=1)
+    return last_commit.get(RULE_METADATA_KEY, None)
+
+
+def _to_delta_table(
+    table: Path | str | deltalake.DeltaTable,
+) -> deltalake.DeltaTable:
+    from deltalake import DeltaTable
+
+    match table:
+        case DeltaTable():
+            return table
+        case str() | Path():
+            return DeltaTable(table)
+        case _:
+            raise TypeError(f"Unsupported type {table!r}")