dataframely 2.3.0__tar.gz → 2.4.0__tar.gz

{dataframely-2.3.0 → dataframely-2.4.0}/.github/copilot-instructions.md

@@ -206,6 +206,29 @@ validated_df: dy.DataFrame[MySchema] = MySchema.validate(df, cast=True)
 4. **Documentation**: Update docstrings
 5. **API changes**: Ensure backward compatibility or document migration path
 
+### Pull request titles (required)
+
+Pull request titles must follow the Conventional Commits format: `<type>[!]: <Subject>`
+
+Allowed `type` values:
+
+- `feat`: A new feature
+- `fix`: A bug fix
+- `docs`: Documentation only changes
+- `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
+- `refactor`: A code change that neither fixes a bug nor adds a feature
+- `perf`: A code change that improves performance
+- `test`: Adding missing tests or correcting existing tests
+- `build`: Changes that affect the build system or external dependencies
+- `ci`: Changes to our CI configuration files and scripts
+- `chore`: Other changes that don't modify src or test files
+- `revert`: Reverts a previous commit
+
+Additional rules:
+
+- Use `!` only for **breaking changes**
+- `Subject` must start with an **uppercase** letter and must **not** end with `.` or a trailing space
+
 ## Performance Considerations
 
 - Validation uses native polars expressions for performance

{dataframely-2.3.0 → dataframely-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 2.3.0
+Version: 2.4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -9,7 +9,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: fsspec>=2025.9
 Requires-Dist: numpy
-Requires-Dist: polars>=1.34
+Requires-Dist: polars>=1.36
 Requires-Dist: typing-extensions ; python_full_version < '3.11'
 Requires-Dist: deltalake ; extra == 'deltalake'
 Requires-Dist: pyarrow ; extra == 'pyarrow'

{dataframely-2.3.0 → dataframely-2.4.0}/dataframely/_base_schema.py

@@ -162,6 +162,19 @@ class SchemaMeta(ABCMeta):
                     f"Did you forget to add parentheses?"
                 )
 
+            # Check for pl.DataType instance or type (e.g., pl.String() or pl.String instead of dy.String())
+            if isinstance(value, pl.DataType) or (
+                isinstance(value, type) and issubclass(value, pl.DataType)
+            ):
+                value_type = "instance" if isinstance(value, pl.DataType) else "type"
+                example = (
+                    "pl.String()" if isinstance(value, pl.DataType) else "pl.String"
+                )
+                raise TypeError(
+                    f"Schema member '{attr}' is a polars DataType {value_type}. "
+                    f"Use dataframely column types (e.g., dy.String()) instead of polars types (e.g., {example})."
+                )
+
         return cls
 
     if not TYPE_CHECKING:

{dataframely-2.3.0 → dataframely-2.4.0}/dataframely/columns/decimal.py

@@ -98,7 +98,11 @@ class Decimal(OrdinalMixin[decimal.Decimal], Column):
         return pl.Decimal(self.precision, self.scale)
 
     def validate_dtype(self, dtype: PolarsDataType) -> bool:
-        return dtype.is_decimal()
+        return (
+            isinstance(dtype, pl.Decimal)
+            and dtype.scale == self.scale
+            and (self.precision is None or dtype.precision == self.precision)
+        )
 
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         if self.scale and not self.precision:

{dataframely-2.3.0 → dataframely-2.4.0}/dataframely/columns/enum.py

@@ -87,9 +87,9 @@ class Enum(Column):
 
     @property
     def pyarrow_dtype(self) -> pa.DataType:
-        if len(self.categories) <= 2**8 - 2:
+        if len(self.categories) <= 2**8 - 1:
             dtype = pa.uint8()
-        elif len(self.categories) <= 2**16 - 2:
+        elif len(self.categories) <= 2**16 - 1:
             dtype = pa.uint16()
         else:
             dtype = pa.uint32()

{dataframely-2.3.0 → dataframely-2.4.0}/dataframely/columns/list.py

@@ -131,9 +131,12 @@ class List(Column):
     def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
         # First, sample the number of items per list element
         # NOTE: We default to 32 for the upper bound as we need some kind of reasonable
-        #  upper bound if none is set.
+        #  upper bound if none is set. If min_length is greater than 32, we use
+        #  min_length as the default upper bound instead.
+        min_len = self.min_length or 0
+        default_max = max(32, min_len)
         element_lengths = generator.sample_int(
-            n, min=self.min_length or 0, max=(self.max_length or 32) + 1
+            n, min=min_len, max=(self.max_length or default_max) + 1
         )
 
         # Then, we can sample the inner elements in a flat series

{dataframely-2.3.0 → dataframely-2.4.0}/dataframely/columns/string.py

@@ -14,6 +14,8 @@ from dataframely.random import Generator
 from ._base import Check, Column
 from ._registry import register
 
+DEFAULT_SAMPLING_REGEX = r"[0-9a-zA-Z]"
+
 
 @register
 class String(Column):
@@ -126,9 +128,9 @@ class String(Column):
             str_max = f"{self.max_length}" if self.max_length is not None else ""
             # NOTE: We generate single-byte unicode characters here as validation uses
             #  `len_bytes()`. Potentially we need to be more accurate at some point...
-            regex = f"[\x01-\x7a]{{{str_min},{str_max}}}"
+            regex = f"{DEFAULT_SAMPLING_REGEX}{{{str_min},{str_max}}}"
         else:
-            regex = r"[\x01-\x7a]*"
+            regex = rf"{DEFAULT_SAMPLING_REGEX}*"
 
         return generator.sample_string(
             n,

{dataframely-2.3.0 → dataframely-2.4.0}/dataframely/filter_result.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from typing import IO, TYPE_CHECKING, Any, Generic, TypeVar
 
 import polars as pl
-from polars._typing import PartitioningScheme
+from polars.io.partition import _SinkDirectory as SinkDirectory
 
 from dataframely._base_schema import BaseSchema
 from dataframely._compat import deltalake
@@ -164,7 +164,7 @@ class FailureInfo(Generic[S]):
         self._write(ParquetStorageBackend(), file=file, **kwargs)
 
     def sink_parquet(
-        self, file: str | Path | IO[bytes] | PartitioningScheme, **kwargs: Any
+        self, file: str | Path | IO[bytes] | SinkDirectory, **kwargs: Any
     ) -> None:
         """Stream the failure info to a single parquet file.
 

{dataframely-2.3.0 → dataframely-2.4.0}/dataframely/schema.py

@@ -7,14 +7,15 @@ import json
 import sys
 import warnings
 from abc import ABC
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
 from json import JSONDecodeError
 from pathlib import Path
 from typing import IO, Any, Literal, overload
 
 import polars as pl
 import polars.exceptions as plexc
-from polars._typing import FileSource, PartitioningScheme
+from polars._typing import FileSource
+from polars.io.partition import _SinkDirectory as SinkDirectory
 
 from dataframely._compat import deltalake
 
@@ -176,7 +177,7 @@ class Schema(BaseSchema, ABC):
         num_rows: int | None = None,
         *,
         overrides: (
-            Mapping[str, Iterable[Any]] | Sequence[Mapping[str, Any]] | None
+            Mapping[str, Sequence[Any] | Any] | Sequence[Mapping[str, Any]] | None
         ) = None,
         generator: Generator | None = None,
     ) -> DataFrame[Self]:
@@ -233,26 +234,22 @@ class Schema(BaseSchema, ABC):
         g = generator or Generator()
 
         # Precondition: valid overrides. We put them into a data frame to remember which
-        # values have been used in the algorithm below.
-        if overrides:
+        # values have been used in the algorithm below. When the user passes a sequence
+        # of mappings, they do not require to have the same keys. Hence, we have to
+        # remember that the data frame has "holes".
+        missing_override_indices: dict[str, pl.Series] = {}
+        if overrides is not None:
             override_keys = (
-                set(overrides) if isinstance(overrides, Mapping) else set(overrides[0])
+                set(overrides)
+                if isinstance(overrides, Mapping)
+                else (
+                    set.union(*[set(o.keys()) for o in overrides])
+                    if len(overrides) > 0
+                    else set()
+                )
             )
-            if isinstance(overrides, Sequence):
-                # Check that overrides entries are consistent. Not necessary for mapping
-                # overrides as polars checks the series lists upon data frame construction.
-                inconsistent_override_keys = [
-                    index
-                    for index, current in enumerate(overrides)
-                    if set(current) != override_keys
-                ]
-                if len(inconsistent_override_keys) > 0:
-                    raise ValueError(
-                        "The `overrides` entries at the following indices "
-                        "do not provide the same keys as the first entry: "
-                        f"{inconsistent_override_keys}."
-                    )
 
+            # Check that all override keys refer to valid columns
             column_names = set(cls.column_names())
             if not override_keys.issubset(column_names):
                 raise ValueError(
@@ -260,6 +257,19 @@ class Schema(BaseSchema, ABC):
                     "which are not in the schema."
                 )
 
+            # Remember the "holes" of the inputs if overrides are provided as a sequence
+            if isinstance(overrides, Sequence):
+                for key in override_keys:
+                    indices = [
+                        i for i, override in enumerate(overrides) if key not in override
+                    ]
+                    if len(indices) > 0:
+                        missing_override_indices[key] = pl.Series(indices)
+
+            # NOTE: Even if the user-provided overrides have "holes", we can still just
+            #  create the data frame. Polars will fill the missing values with nulls, we
+            #  will replace them later during sampling. If we were to already replace
+            #  them here, we would not be able to resample these values.
             values = pl.DataFrame(
                 overrides,
                 schema={
@@ -322,6 +332,7 @@ class Schema(BaseSchema, ABC):
             used_values=values.slice(0, 0),
             remaining_values=values,
             override_expressions=override_expressions,
+            missing_value_indices=missing_override_indices,
         )
 
         sampling_rounds = 1
@@ -359,6 +370,7 @@ class Schema(BaseSchema, ABC):
                 used_values=used_values,
                 remaining_values=remaining_values,
                 override_expressions=override_expressions,
+                missing_value_indices=missing_override_indices,
             )
             sampling_rounds += 1
 
@@ -387,6 +399,7 @@ class Schema(BaseSchema, ABC):
         used_values: pl.DataFrame,
         remaining_values: pl.DataFrame,
         override_expressions: list[pl.Expr],
+        missing_value_indices: dict[str, pl.Series],
     ) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
         """Private method to sample a data frame with the schema including subsequent
         filtering.
@@ -405,6 +418,33 @@ class Schema(BaseSchema, ABC):
             }
         )
 
+        # If we have missing value indices, we need to sample new values for the
+        # indices that overlap with indices in the remaining values and replace them
+        # in the sampled data frame.
+        for name, indices in missing_value_indices.items():
+            remapped_indices = (
+                indices.to_frame("idx")
+                .join(
+                    remaining_values.select("__row_index__").with_row_index(
+                        "__row_index_loop__"
+                    ),
+                    left_on="idx",
+                    right_on="__row_index__",
+                )
+                .select("__row_index_loop__")
+                .to_series()
+            )
+            if (num := len(remapped_indices)) > 0:
+                sampled_values = cls.columns()[name].sample(generator, num)
+                sampled = sampled.with_columns(
+                    sampled[name]
+                    # NOTE: We need to sort here as `scatter` requires sorted indices.
+                    #  Due to concatenations in `remaining_values`, the indices can go
+                    #  out of order.
+                    .scatter(remapped_indices.sort(), sampled_values)
+                    .alias(name)
+                )
+
         combined_dataframe = pl.concat([previous_result, sampled])
         # Pre-process columns before filtering.
         combined_dataframe = combined_dataframe.with_columns(override_expressions)
@@ -861,7 +901,7 @@ class Schema(BaseSchema, ABC):
         cls,
         lf: LazyFrame[Self],
         /,
-        file: str | Path | IO[bytes] | PartitioningScheme,
+        file: str | Path | IO[bytes] | SinkDirectory,
         **kwargs: Any,
     ) -> None:
         """Stream a typed lazy frame with this schema to a parquet file.