dataframely 2.5.1__tar.gz → 2.7.0__tar.gz

{dataframely-2.5.1 → dataframely-2.7.0}/.github/workflows/build.yml

@@ -13,11 +13,11 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3
+        uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4
         with:
           environments: build
       - name: Set version
@@ -48,16 +48,16 @@ jobs:
           - target-platform: win-64
             os: windows-latest
     steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3
+        uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4
         with:
           environments: build
       - name: Set version
         run: pixi run -e build set-version
-      - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: "3.10"
       - name: Build wheel

{dataframely-2.5.1 → dataframely-2.7.0}/.github/workflows/chore.yml

@@ -58,7 +58,7 @@ jobs:
       pull-requests: write
     steps:
       - name: ${{ github.event_name == 'pull_request_target' && 'Assign labels' || 'Update release draft' }}
-        uses: release-drafter/release-drafter@b1476f6e6eb133afa41ed8589daba6dc69b4d3f5 # v6.1.0
+        uses: release-drafter/release-drafter@6db134d15f3909ccc9eefd369f02bd1e9cffdf97 # v6.2.0
         with:
           disable-releaser: ${{ github.event_name == 'pull_request_target' }}
           disable-autolabeler: ${{ github.event_name == 'push' }}

{dataframely-2.5.1 → dataframely-2.7.0}/.github/workflows/ci.yml

@@ -19,14 +19,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           # needed for 'pre-commit-mirrors-insert-license'
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3
+        uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4
         with:
-          environments: default lint
+          environments: default lint polars-minimal
       - name: Install Rust
         run: rustup show
       - name: Cache Rust dependencies
@@ -45,6 +45,7 @@ jobs:
         environment: [py310, py311, py312, py313, py314]
         with_optionals: [false]
         include:
+          # Test with optional dependencies across OSes
           - os: ubuntu-latest
             environment: py314-optionals
             with_optionals: true
@@ -54,11 +55,15 @@ jobs:
           - os: macos-latest
             environment: py314-optionals
             with_optionals: true
+          # Test polars backward compatibility, only one OS
+          - os: ubuntu-latest
+            environment: polars-minimal
+            with_optionals: false
     steps:
       - name: Checkout branch
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3
+        uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4
         with:
           environments: ${{ matrix.environment }}
           # FIXME: Remove when `s3_server` fixture does not start a process anymore

{dataframely-2.5.1 → dataframely-2.7.0}/.github/workflows/copilot-setup-steps.yml

@@ -13,9 +13,9 @@ jobs:
       id-token: write
     steps:
       - name: Checkout branch
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3
+        uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4
         with:
           environments: default
       - name: Install Rust

{dataframely-2.5.1 → dataframely-2.7.0}/.github/workflows/nightly.yml

@@ -23,9 +23,9 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout branch
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7 # v0.9.3
+        uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4
         with:
           environments: nightly
       - name: Install polars nightly

{dataframely-2.5.1 → dataframely-2.7.0}/.github/workflows/scorecard.yml

@@ -35,7 +35,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
@@ -74,6 +74,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v3.29.5
+        uses: github/codeql-action/upload-sarif@b20883b0cd1f46c72ae0ba6d1090936928f9fa30 # v3.29.5
         with:
           sarif_file: results.sarif

{dataframely-2.5.1 → dataframely-2.7.0}/.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       # workaround for https://github.com/prefix-dev/pixi/issues/1482
       - id: pixi-install
         name: pixi-install
-        entry: pixi install -e default -e lint
+        entry: pixi install -e default -e lint -e polars-minimal
         language: system
         always_run: true
         require_serial: true
@@ -48,6 +48,13 @@ repos:
         language: system
         types: [python]
         require_serial: true
+      # mypy with oldest supported polars version
+      - id: mypy
+        name: mypy-polars-minimal
+        entry: pixi run -e default-polars-minimal  mypy --cache-dir .mypy_cache_polars_minimal
+        language: system
+        types: [python]
+        require_serial: true
       # cargo-fmt
       - id: cargo-fmt
         name: cargo-fmt

{dataframely-2.5.1 → dataframely-2.7.0}/Cargo.lock

@@ -152,9 +152,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.10.1"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 dependencies = [
  "serde",
 ]

{dataframely-2.5.1 → dataframely-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 2.5.1
+Version: 2.7.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -9,7 +9,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: fsspec>=2025.9
 Requires-Dist: numpy
-Requires-Dist: polars>=1.36
+Requires-Dist: polars>=1.35
 Requires-Dist: typing-extensions ; python_full_version < '3.11'
 Requires-Dist: deltalake ; extra == 'deltalake'
 Requires-Dist: pyarrow ; extra == 'pyarrow'

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/_compat.py

@@ -1,9 +1,10 @@
 # Copyright (c) QuantCo 2025-2026
 # SPDX-License-Identifier: BSD-3-Clause
 
-
 from typing import Any
 
+import polars as pl
+
 
 class _DummyModule:  # pragma: no cover
     def __init__(self, module: str) -> None:
@@ -58,7 +59,6 @@ try:
 except ImportError:
     pa = _DummyModule("pyarrow")
 
-
 # -------------------------------------- PYDANTIC ------------------------------------ #
 
 try:
@@ -71,6 +71,24 @@ try:
 except ImportError:
     pydantic_core_schema = _DummyModule("pydantic_core_schema")  # type: ignore
 
+# --------------------------------------- POLARS ------------------------------------- #
+
+_polars_version_tuple = tuple(
+    int(part) if part.isdigit() else part for part in pl.__version__.split(".")
+)
+if _polars_version_tuple < (1, 36):
+    from polars._typing import (  # type: ignore[attr-defined,unused-ignore]
+        PartitioningScheme as PartitionSchemeOrSinkDirectory,
+    )
+elif _polars_version_tuple < (1, 38):  # pragma: no cover
+    from polars.io.partition import (  # type: ignore[no-redef,attr-defined,unused-ignore]
+        _SinkDirectory as PartitionSchemeOrSinkDirectory,
+    )
+else:
+    from polars.io.partition import (  # type: ignore[no-redef,attr-defined,unused-ignore]
+        PartitionBy as PartitionSchemeOrSinkDirectory,
+    )
+
 # ------------------------------------------------------------------------------------ #
 
 __all__ = [
@@ -78,6 +96,7 @@ __all__ = [
     "DeltaTable",
     "Dialect",
     "MSDialect_pyodbc",
+    "PartitionSchemeOrSinkDirectory",
     "pa",
     "PGDialect_psycopg2",
     "pydantic_core_schema",

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/_storage/__init__.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 from ._base import StorageBackend
+from ._fsspec import get_file_prefix
 
-__all__ = [
-    "StorageBackend",
-]
+__all__ = ["StorageBackend", "get_file_prefix"]

dataframely-2.7.0/dataframely/_storage/_fsspec.py

@@ -0,0 +1,18 @@
+# Copyright (c) QuantCo 2025-2026
+# SPDX-License-Identifier: BSD-3-Clause
+
+from fsspec import AbstractFileSystem
+
+
+def get_file_prefix(fs: AbstractFileSystem) -> str:
+    match fs.protocol:
+        case "file":
+            return ""
+        case str():
+            return f"{fs.protocol}://"
+        case ["file", *_]:
+            return ""
+        case [str(proto), *_]:
+            return f"{proto}://"
+        case _:
+            raise ValueError(f"Unexpected fs.protocol: {fs.protocol}")

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/_storage/parquet.py

@@ -7,6 +7,8 @@ from typing import Any
 import polars as pl
 from fsspec import AbstractFileSystem, url_to_fs
 
+from dataframely._storage import get_file_prefix
+
 from ._base import (
     SerializedCollection,
     SerializedRules,
@@ -79,7 +81,9 @@ class ParquetStorageBackend(StorageBackend):
         fs: AbstractFileSystem = url_to_fs(path)[0]
         for key, lf in dfs.items():
             destination = (
-                fs.sep.join([path, key])
+                # Enforce that the path ends with a separator. Otherwise
+                # polars misbehaves on Windows.
+                fs.sep.join([path, key]) + fs.sep
                 if "partition_by" in kwargs
                 else fs.sep.join([path, f"{key}.parquet"])
             )
@@ -107,7 +111,9 @@ class ParquetStorageBackend(StorageBackend):
         fs: AbstractFileSystem = url_to_fs(path)[0]
         for key, lf in dfs.items():
             destination = (
-                fs.sep.join([path, key])
+                # Enforce that the path ends with a separator. Otherwise
+                # polars misbehaves on Windows.
+                fs.sep.join([path, key]) + fs.sep
                 if "partition_by" in kwargs
                 else fs.sep.join([path, f"{key}.parquet"])
             )
@@ -155,15 +161,7 @@ class ParquetStorageBackend(StorageBackend):
                 if is_file:
                     collection_types.append(_read_serialized_collection(source_path))
                 else:
-                    prefix = (
-                        ""
-                        if fs.protocol == "file"
-                        else (
-                            f"{fs.protocol}://"
-                            if isinstance(fs.protocol, str)
-                            else f"{fs.protocol[0]}://"
-                        )
-                    )
+                    prefix = get_file_prefix(fs)
                     for file in fs.glob(fs.sep.join([source_path, "**", "*.parquet"])):
                         collection_types.append(
                             _read_serialized_collection(f"{prefix}{file}")

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/collection/collection.py

@@ -864,9 +864,8 @@ class Collection(BaseCollection, ABC):
         members which are not provided in the current collection.
 
         Args:
-            directory: The directory the Parquet files should be written to.
-                If the directory does not exist, it is created automatically,
-                including all of its parents.
+            directory: The directory where the Parquet files should be written to.
+                The `mkdir` kwarg controls whether the directory is created if needed.
             kwargs: Additional keyword arguments passed to :meth:`polars.DataFrame.write_parquet`.
                 `metadata` may only be provided if it is a dictionary.
 

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/columns/decimal.py

@@ -29,10 +29,10 @@ class Decimal(OrdinalMixin[decimal.Decimal], Column):
         *,
         nullable: bool = False,
         primary_key: bool = False,
-        min: decimal.Decimal | None = None,
-        min_exclusive: decimal.Decimal | None = None,
-        max: decimal.Decimal | None = None,
-        max_exclusive: decimal.Decimal | None = None,
+        min: decimal.Decimal | int | None = None,
+        min_exclusive: decimal.Decimal | int | None = None,
+        max: decimal.Decimal | int | None = None,
+        max_exclusive: decimal.Decimal | int | None = None,
         check: Check | None = None,
         alias: str | None = None,
         metadata: dict[str, Any] | None = None,
@@ -70,6 +70,15 @@ class Decimal(OrdinalMixin[decimal.Decimal], Column):
                 names, the specified alias is the only valid name.
             metadata: A dictionary of metadata to attach to the column.
         """
+        if isinstance(min, int):
+            min = decimal.Decimal(min)
+        if isinstance(min_exclusive, int):
+            min_exclusive = decimal.Decimal(min_exclusive)
+        if isinstance(max, int):
+            max = decimal.Decimal(max)
+        if isinstance(max_exclusive, int):
+            max_exclusive = decimal.Decimal(max_exclusive)
+
         if min is not None:
             _validate(min, precision, scale, "min")
         if min_exclusive is not None:

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/filter_result.py

@@ -10,11 +10,11 @@ from pathlib import Path
 from typing import IO, TYPE_CHECKING, Any, Generic, TypeVar
 
 import polars as pl
-from polars.io.partition import _SinkDirectory as SinkDirectory
 
 from dataframely._base_schema import BaseSchema
 from dataframely._compat import deltalake
 
+from ._compat import PartitionSchemeOrSinkDirectory
 from ._storage import StorageBackend
 from ._storage.delta import DeltaStorageBackend
 from ._storage.parquet import ParquetStorageBackend
@@ -32,6 +32,7 @@ UNKNOWN_SCHEMA_NAME = "__DATAFRAMELY_UNKNOWN__"
 
 S = TypeVar("S", bound=BaseSchema)
 
+
 # ----------------------------------- FILTER RESULT ---------------------------------- #
 
 
@@ -73,7 +74,8 @@ class LazyFilterResult(NamedTuple, Generic[S]):
             [self.result.lazy(), self.failure._lf], **kwargs
         )
         return FilterResult(
-            result=result_df,  # type: ignore
+            # Whether the type ignore is necessary depends on the polars version.
+            result=result_df,  # type: ignore[arg-type,unused-ignore]
             failure=FailureInfo(
                 failure_df.lazy(), self.failure._rule_columns, self.failure.schema
             ),
@@ -152,7 +154,8 @@ class FailureInfo(Generic[S]):
         Args:
             file: The file path or writable file-like object to which to write the
                 parquet file. This should be a path to a directory if writing a
-                partitioned dataset.
+                partitioned dataset. The `mkdir` kwarg controls whether the directory
+                is created if needed.
             kwargs: Additional keyword arguments passed directly to
                 :meth:`polars.write_parquet`. ``metadata`` may only be provided if it
                 is a dictionary.
@@ -164,7 +167,9 @@ class FailureInfo(Generic[S]):
         self._write(ParquetStorageBackend(), file=file, **kwargs)
 
     def sink_parquet(
-        self, file: str | Path | IO[bytes] | SinkDirectory, **kwargs: Any
+        self,
+        file: str | Path | IO[bytes] | PartitionSchemeOrSinkDirectory,
+        **kwargs: Any,
     ) -> None:
         """Stream the failure info to a single parquet file.
 

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/schema.py

@@ -15,12 +15,11 @@ from typing import IO, Any, Literal, overload
 import polars as pl
 import polars.exceptions as plexc
 from polars._typing import FileSource
-from polars.io.partition import _SinkDirectory as SinkDirectory
 
 from dataframely._compat import deltalake
 
 from ._base_schema import ORIGINAL_COLUMN_PREFIX, BaseSchema
-from ._compat import pa, sa
+from ._compat import PartitionSchemeOrSinkDirectory, pa, sa
 from ._match_to_schema import match_to_schema
 from ._native import format_rule_failures
 from ._plugin import all_rules, all_rules_horizontal, all_rules_required
@@ -55,9 +54,9 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
-
 _COLUMN_VALID = "__DATAFRAMELY_VALID__"
 
+
 # ------------------------------------------------------------------------------------ #
 #                                   SCHEMA DEFINITION                                  #
 # ------------------------------------------------------------------------------------ #
@@ -554,8 +553,9 @@ class Schema(BaseSchema, ABC):
 
         Returns:
             The input eager or lazy frame, wrapped in a generic version of the
-            input's data frame type to reflect schema adherence. This operation is
-            guaranteed to maintain input ordering of rows.
+            input's data frame type to reflect schema adherence. Columns not defined
+            in the schema are removed from the output. This operation is guaranteed
+            to maintain input ordering of rows.
 
         Raises:
             SchemaError: If `eager=True` and the input data frame misses columns or
@@ -705,7 +705,8 @@ class Schema(BaseSchema, ABC):
         Returns:
             A tuple of the validated rows in the input data frame (potentially
             empty) and a simple dataclass carrying information about the rows of the
-            data frame which could not be validated successfully. Just like in polars'
+            data frame which could not be validated successfully. Columns not defined
+            in the schema are removed from the output. Just like in polars'
             native :meth:`~polars.DataFrame.filter`, the order of rows in the returned
             data frame is maintained.
 
@@ -885,7 +886,8 @@ class Schema(BaseSchema, ABC):
             df: The data frame to write to the parquet file.
             file: The file path or writable file-like object to which to write the
                 parquet file. This should be a path to a directory if writing a
-                partitioned dataset.
+                partitioned dataset. The `mkdir` kwarg controls whether the directory
+                is created if needed.
             kwargs: Additional keyword arguments passed directly to
                 :meth:`polars.write_parquet`. `metadata` may only be provided if it
                 is a dictionary.
@@ -901,7 +903,7 @@ class Schema(BaseSchema, ABC):
         cls,
         lf: LazyFrame[Self],
         /,
-        file: str | Path | IO[bytes] | SinkDirectory,
+        file: str | Path | IO[bytes] | PartitionSchemeOrSinkDirectory,
         **kwargs: Any,
     ) -> None:
         """Stream a typed lazy frame with this schema to a parquet file.

{dataframely-2.5.1 → dataframely-2.7.0}/dataframely/testing/storage.py

@@ -10,6 +10,7 @@ from fsspec import AbstractFileSystem, url_to_fs
 import dataframely as dy
 from dataframely import FailureInfo, Validation
 from dataframely._compat import deltalake
+from dataframely._storage import get_file_prefix
 from dataframely._storage.delta import _to_delta_table
 
 # ----------------------------------- Schema -------------------------------------------
@@ -190,19 +191,7 @@ class CollectionStorageTester(ABC):
         metadata."""
 
     def _prefix_path(self, path: str, fs: AbstractFileSystem) -> str:
-        return f"{self._get_prefix(fs)}{path}"
-
-    @staticmethod
-    def _get_prefix(fs: AbstractFileSystem) -> str:
-        return (
-            ""
-            if fs.protocol == "file"
-            else (
-                f"{fs.protocol}://"
-                if isinstance(fs.protocol, str)
-                else f"{fs.protocol[0]}://"
-            )
-        )
+        return f"{get_file_prefix(fs)}{path}"
 
 
 class ParquetCollectionStorageTester(CollectionStorageTester):

{dataframely-2.5.1 → dataframely-2.7.0}/docs/guides/faq.md

@@ -33,3 +33,12 @@ class UserSchema(dy.Schema):
 ## How do I fix the ruff error `First argument of a method should be named self`?
 
 See our documentation on [group rules](./quickstart.md#group-rules).
+
+## What versions of `polars` does `dataframely` support?
+
+Our CI automatically tests `dataframely` for a minimal supported version of `polars`, which is currently `1.35.*`,
+and the latest stable version.
+We aim to extend support for new `polars` versions as they are released.
+
+If your `polars` version is not in the range of supported versions, `dataframely` may still work, but you may
+encounter unexpected issues.