dataframely 2.3.1__tar.gz → 2.4.0__tar.gz

{dataframely-2.3.1 → dataframely-2.4.0}/.github/copilot-instructions.md

@@ -206,6 +206,29 @@ validated_df: dy.DataFrame[MySchema] = MySchema.validate(df, cast=True)
 4. **Documentation**: Update docstrings
 5. **API changes**: Ensure backward compatibility or document migration path
 
+### Pull request titles (required)
+
+Pull request titles must follow the Conventional Commits format: `<type>[!]: <Subject>`
+
+Allowed `type` values:
+
+- `feat`: A new feature
+- `fix`: A bug fix
+- `docs`: Documentation only changes
+- `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
+- `refactor`: A code change that neither fixes a bug nor adds a feature
+- `perf`: A code change that improves performance
+- `test`: Adding missing tests or correcting existing tests
+- `build`: Changes that affect the build system or external dependencies
+- `ci`: Changes to our CI configuration files and scripts
+- `chore`: Other changes that don't modify src or test files
+- `revert`: Reverts a previous commit
+
+Additional rules:
+
+- Use `!` only for **breaking changes**
+- `Subject` must start with an **uppercase** letter and must **not** end with `.` or a trailing space
+
 ## Performance Considerations
 
 - Validation uses native polars expressions for performance

{dataframely-2.3.1 → dataframely-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 2.3.1
+Version: 2.4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11

{dataframely-2.3.1 → dataframely-2.4.0}/dataframely/_base_schema.py

@@ -162,6 +162,19 @@ class SchemaMeta(ABCMeta):
                     f"Did you forget to add parentheses?"
                 )
 
+            # Check for pl.DataType instance or type (e.g., pl.String() or pl.String instead of dy.String())
+            if isinstance(value, pl.DataType) or (
+                isinstance(value, type) and issubclass(value, pl.DataType)
+            ):
+                value_type = "instance" if isinstance(value, pl.DataType) else "type"
+                example = (
+                    "pl.String()" if isinstance(value, pl.DataType) else "pl.String"
+                )
+                raise TypeError(
+                    f"Schema member '{attr}' is a polars DataType {value_type}. "
+                    f"Use dataframely column types (e.g., dy.String()) instead of polars types (e.g., {example})."
+                )
+
         return cls
 
     if not TYPE_CHECKING:

{dataframely-2.3.1 → dataframely-2.4.0}/dataframely/columns/decimal.py

@@ -98,7 +98,11 @@ class Decimal(OrdinalMixin[decimal.Decimal], Column):
         return pl.Decimal(self.precision, self.scale)
 
     def validate_dtype(self, dtype: PolarsDataType) -> bool:
-        return dtype.is_decimal()
+        return (
+            isinstance(dtype, pl.Decimal)
+            and dtype.scale == self.scale
+            and (self.precision is None or dtype.precision == self.precision)
+        )
 
     def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
         if self.scale and not self.precision:

{dataframely-2.3.1 → dataframely-2.4.0}/dataframely/columns/list.py

@@ -131,9 +131,12 @@ class List(Column):
     def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
         # First, sample the number of items per list element
         # NOTE: We default to 32 for the upper bound as we need some kind of reasonable
-        #  upper bound if none is set.
+        #  upper bound if none is set. If min_length is greater than 32, we use
+        #  min_length as the default upper bound instead.
+        min_len = self.min_length or 0
+        default_max = max(32, min_len)
         element_lengths = generator.sample_int(
-            n, min=self.min_length or 0, max=(self.max_length or 32) + 1
+            n, min=min_len, max=(self.max_length or default_max) + 1
         )
 
         # Then, we can sample the inner elements in a flat series

{dataframely-2.3.1 → dataframely-2.4.0}/dataframely/columns/string.py

@@ -14,6 +14,8 @@ from dataframely.random import Generator
 from ._base import Check, Column
 from ._registry import register
 
+DEFAULT_SAMPLING_REGEX = r"[0-9a-zA-Z]"
+
 
 @register
 class String(Column):
@@ -126,9 +128,9 @@ class String(Column):
             str_max = f"{self.max_length}" if self.max_length is not None else ""
             # NOTE: We generate single-byte unicode characters here as validation uses
             #  `len_bytes()`. Potentially we need to be more accurate at some point...
-            regex = f"[\x01-\x7a]{{{str_min},{str_max}}}"
+            regex = f"{DEFAULT_SAMPLING_REGEX}{{{str_min},{str_max}}}"
         else:
-            regex = r"[\x01-\x7a]*"
+            regex = rf"{DEFAULT_SAMPLING_REGEX}*"
 
         return generator.sample_string(
             n,

{dataframely-2.3.1 → dataframely-2.4.0}/dataframely/schema.py

@@ -7,7 +7,7 @@ import json
 import sys
 import warnings
 from abc import ABC
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
 from json import JSONDecodeError
 from pathlib import Path
 from typing import IO, Any, Literal, overload
@@ -177,7 +177,7 @@ class Schema(BaseSchema, ABC):
         num_rows: int | None = None,
         *,
         overrides: (
-            Mapping[str, Iterable[Any]] | Sequence[Mapping[str, Any]] | None
+            Mapping[str, Sequence[Any] | Any] | Sequence[Mapping[str, Any]] | None
         ) = None,
         generator: Generator | None = None,
     ) -> DataFrame[Self]:
@@ -234,26 +234,22 @@ class Schema(BaseSchema, ABC):
         g = generator or Generator()
 
         # Precondition: valid overrides. We put them into a data frame to remember which
-        # values have been used in the algorithm below.
-        if overrides:
+        # values have been used in the algorithm below. When the user passes a sequence
+        # of mappings, they do not require to have the same keys. Hence, we have to
+        # remember that the data frame has "holes".
+        missing_override_indices: dict[str, pl.Series] = {}
+        if overrides is not None:
             override_keys = (
-                set(overrides) if isinstance(overrides, Mapping) else set(overrides[0])
+                set(overrides)
+                if isinstance(overrides, Mapping)
+                else (
+                    set.union(*[set(o.keys()) for o in overrides])
+                    if len(overrides) > 0
+                    else set()
+                )
             )
-            if isinstance(overrides, Sequence):
-                # Check that overrides entries are consistent. Not necessary for mapping
-                # overrides as polars checks the series lists upon data frame construction.
-                inconsistent_override_keys = [
-                    index
-                    for index, current in enumerate(overrides)
-                    if set(current) != override_keys
-                ]
-                if len(inconsistent_override_keys) > 0:
-                    raise ValueError(
-                        "The `overrides` entries at the following indices "
-                        "do not provide the same keys as the first entry: "
-                        f"{inconsistent_override_keys}."
-                    )
 
+            # Check that all override keys refer to valid columns
             column_names = set(cls.column_names())
             if not override_keys.issubset(column_names):
                 raise ValueError(
@@ -261,6 +257,19 @@ class Schema(BaseSchema, ABC):
                     "which are not in the schema."
                 )
 
+            # Remember the "holes" of the inputs if overrides are provided as a sequence
+            if isinstance(overrides, Sequence):
+                for key in override_keys:
+                    indices = [
+                        i for i, override in enumerate(overrides) if key not in override
+                    ]
+                    if len(indices) > 0:
+                        missing_override_indices[key] = pl.Series(indices)
+
+            # NOTE: Even if the user-provided overrides have "holes", we can still just
+            #  create the data frame. Polars will fill the missing values with nulls, we
+            #  will replace them later during sampling. If we were to already replace
+            #  them here, we would not be able to resample these values.
             values = pl.DataFrame(
                 overrides,
                 schema={
@@ -323,6 +332,7 @@ class Schema(BaseSchema, ABC):
             used_values=values.slice(0, 0),
             remaining_values=values,
             override_expressions=override_expressions,
+            missing_value_indices=missing_override_indices,
         )
 
         sampling_rounds = 1
@@ -360,6 +370,7 @@ class Schema(BaseSchema, ABC):
                 used_values=used_values,
                 remaining_values=remaining_values,
                 override_expressions=override_expressions,
+                missing_value_indices=missing_override_indices,
             )
             sampling_rounds += 1
 
@@ -388,6 +399,7 @@ class Schema(BaseSchema, ABC):
         used_values: pl.DataFrame,
         remaining_values: pl.DataFrame,
         override_expressions: list[pl.Expr],
+        missing_value_indices: dict[str, pl.Series],
     ) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
         """Private method to sample a data frame with the schema including subsequent
         filtering.
@@ -406,6 +418,33 @@ class Schema(BaseSchema, ABC):
             }
         )
 
+        # If we have missing value indices, we need to sample new values for the
+        # indices that overlap with indices in the remaining values and replace them
+        # in the sampled data frame.
+        for name, indices in missing_value_indices.items():
+            remapped_indices = (
+                indices.to_frame("idx")
+                .join(
+                    remaining_values.select("__row_index__").with_row_index(
+                        "__row_index_loop__"
+                    ),
+                    left_on="idx",
+                    right_on="__row_index__",
+                )
+                .select("__row_index_loop__")
+                .to_series()
+            )
+            if (num := len(remapped_indices)) > 0:
+                sampled_values = cls.columns()[name].sample(generator, num)
+                sampled = sampled.with_columns(
+                    sampled[name]
+                    # NOTE: We need to sort here as `scatter` requires sorted indices.
+                    #  Due to concatenations in `remaining_values`, the indices can go
+                    #  out of order.
+                    .scatter(remapped_indices.sort(), sampled_values)
+                    .alias(name)
+                )
+
         combined_dataframe = pl.concat([previous_result, sampled])
         # Pre-process columns before filtering.
         combined_dataframe = combined_dataframe.with_columns(override_expressions)

{dataframely-2.3.1 → dataframely-2.4.0}/pyproject.toml

@@ -27,7 +27,7 @@ description = "A declarative, polars-native data frame validation library"
 name = "dataframely"
 readme = "README.md"
 requires-python = ">=3.10"
-version = "2.3.1"
+version = "2.4.0"
 
 [project.optional-dependencies]
 deltalake = ["deltalake"]

{dataframely-2.3.1 → dataframely-2.4.0}/tests/collection/test_sample.py

@@ -189,3 +189,16 @@ def test_duplicate_column_inlined_for_sampling() -> None:
                 ],
             },
         )
+
+
+def test_sample_override_sequence_with_missing_keys() -> None:
+    collection = MyCollection.sample(
+        overrides=[{"first": {"a": 1}, "second": [{"c": 2}, {}, {"b": 5}]}]
+    )
+    assert collection.first.collect().height == 1
+    assert collection.second is not None
+
+    second = collection.second.collect()
+    assert second.height == 3
+    assert second["c"].item(0) == 2
+    assert second["b"].item(2) == 5

{dataframely-2.3.1 → dataframely-2.4.0}/tests/column_types/test_decimal.py

@@ -66,7 +66,7 @@ def test_invalid_args(kwargs: dict[str, Any]) -> None:
 
 
 @pytest.mark.parametrize(
-    "dtype", [pl.Decimal, pl.Decimal(12), pl.Decimal(None, 8), pl.Decimal(6, 2)]
+    "dtype", [pl.Decimal, pl.Decimal(12), pl.Decimal(None, 0), pl.Decimal(6, 0)]
 )
 def test_any_decimal_dtype_passes(dtype: DataTypeClass) -> None:
     df = pl.DataFrame(schema={"a": dtype})
@@ -171,3 +171,58 @@ def test_validate_range(
     actual = evaluate_rules(lf, rules_from_exprs(column.validation_rules(pl.col("a"))))
     expected = pl.LazyFrame(valid)
     assert_frame_equal(actual, expected)
+
+
+@pytest.mark.parametrize(
+    ("schema_precision", "schema_scale", "dtype", "should_pass"),
+    [
+        # Exact match should pass
+        (38, 10, pl.Decimal(38, 10), True),
+        # Wrong scale should fail
+        (38, 10, pl.Decimal(38, 3), False),
+        # Wrong precision should fail
+        (10, 2, pl.Decimal(38, 2), False),
+        # Both wrong should fail
+        (10, 2, pl.Decimal(38, 5), False),
+        # precision=None should accept any precision with matching scale
+        (None, 5, pl.Decimal(10, 5), True),
+        (None, 5, pl.Decimal(20, 5), True),
+        # precision=None with wrong scale should fail
+        (None, 5, pl.Decimal(10, 3), False),
+    ],
+)
+def test_precision_scale_validation(
+    schema_precision: int | None,
+    schema_scale: int,
+    dtype: DataTypeClass,
+    should_pass: bool,
+) -> None:
+    class TestSchema(dy.Schema):
+        a = dy.Decimal(precision=schema_precision, scale=schema_scale)
+
+    df = pl.DataFrame(schema={"a": dtype})
+    assert TestSchema.is_valid(df) == should_pass
+
+
+@pytest.mark.parametrize(
+    ("schema_precision", "schema_scale", "input_dtype"),
+    [
+        (38, 10, pl.Decimal(38, 3)),
+        (10, 2, pl.Decimal(38, 2)),
+        (10, 5, pl.Decimal(20, 3)),
+    ],
+)
+def test_precision_scale_casting(
+    schema_precision: int,
+    schema_scale: int,
+    input_dtype: pl.DataType,
+) -> None:
+    class TestSchema(dy.Schema):
+        a = dy.Decimal(precision=schema_precision, scale=schema_scale)
+
+    df_input = pl.DataFrame({"a": [decimal.Decimal("12.34")]}).with_columns(
+        pl.col("a").cast(input_dtype)
+    )
+    df_validated = TestSchema.validate(df_input, cast=True)
+    assert df_validated.schema["a"].precision == schema_precision  # type: ignore[attr-defined]
+    assert df_validated.schema["a"].scale == schema_scale  # type: ignore[attr-defined]

{dataframely-2.3.1 → dataframely-2.4.0}/tests/column_types/test_list.py

@@ -1,6 +1,8 @@
 # Copyright (c) QuantCo 2025-2025
 # SPDX-License-Identifier: BSD-3-Clause
 
+from typing import cast
+
 import polars as pl
 import pytest
 
@@ -179,3 +181,14 @@ def test_inner_primary_key_struct(
     _, failure = schema.filter(df)
     assert failure.counts() == {"a|primary_key": failure_count}
     assert validation_mask(df, failure).to_list() == mask
+
+
+@pytest.mark.parametrize("min_length", [0, 10, 33, 100])
+def test_list_sampling_with_min_length(min_length: int) -> None:
+    """Test that sampling works correctly when min_length > 32."""
+    schema = create_schema("test", {"a": dy.List(dy.Int64(), min_length=min_length)})
+    df = schema.sample(num_rows=10)
+    assert len(df) == 10
+    # Verify all lists have at least min_length elements
+    min_list_len = cast(int, df["a"].list.len().min())
+    assert min_list_len >= min_length

{dataframely-2.3.1 → dataframely-2.4.0}/tests/columns/test_str.py

@@ -1,10 +1,13 @@
 # Copyright (c) QuantCo 2025-2025
 # SPDX-License-Identifier: BSD-3-Clause
+import re
 
 import pytest
 
 import dataframely as dy
 from dataframely.columns import Column
+from dataframely.columns.string import DEFAULT_SAMPLING_REGEX
+from dataframely.random import Generator
 from dataframely.testing import ALL_COLUMN_TYPES
 
 
@@ -32,3 +35,17 @@ def test_string_representation_array() -> None:
 def test_string_representation_struct() -> None:
     column = dy.Struct({"a": dy.String()})
     assert str(column) == dy.Struct.__name__.lower()
+
+
+@pytest.mark.parametrize("min_length", [None, 5, 10])
+@pytest.mark.parametrize("max_length", [None, 20])
+def test_string_sampling_without_regex(
+    min_length: int | None, max_length: int | None
+) -> None:
+    # Check that if no regex is provided, the sampled strings only use
+    # characters from the DEFAULT_SAMPLING_REGEX.
+    column = dy.String(min_length=min_length, max_length=max_length)
+    generator = Generator(seed=42)
+    sample = column.sample(generator=generator, n=1000)
+
+    assert all(re.match(f"{DEFAULT_SAMPLING_REGEX}*", value) for value in sample)

{dataframely-2.3.1 → dataframely-2.4.0}/tests/schema/test_base.py

@@ -125,3 +125,19 @@ def test_user_error_column_type_not_instance() -> None:
         class MySchemaWithColumnTypeNotInstance(dy.Schema):
             a = dy.Int32(nullable=False, primary_key=True)
             b = dy.Float64  # User error: Forgot parentheses!
+
+
+def test_user_error_polars_datatype_instance() -> None:
+    with pytest.raises(TypeError, match="polars DataType instance"):
+
+        class MySchemaWithPolarsDataTypeInstance(dy.Schema):
+            a = dy.Int32(nullable=False)
+            b = pl.String()  # User error: Used pl.String() instead of dy.String()
+
+
+def test_user_error_polars_datatype_type() -> None:
+    with pytest.raises(TypeError, match="polars DataType type"):
+
+        class MySchemaWithPolarsDataTypeType(dy.Schema):
+            a = dy.Int32(nullable=False)
+            b = pl.String  # User error: Used pl.String instead of dy.String()

{dataframely-2.3.1 → dataframely-2.4.0}/tests/schema/test_sample.py

@@ -9,6 +9,7 @@ from polars.testing import assert_frame_equal
 
 import dataframely as dy
 from dataframely.random import Generator
+from dataframely.testing import create_schema
 
 
 class MySimpleSchema(dy.Schema):
@@ -214,23 +215,6 @@ def test_sample_raises_superfluous_column_override() -> None:
         SchemaWithIrrelevantColumnPreProcessing.sample(100)
 
 
-def test_sample_with_inconsistent_overrides_keys_raises() -> None:
-    with pytest.raises(
-        ValueError,
-        match=(
-            r"The `overrides` entries at the following indices do not provide "
-            r"the same keys as the first entry: \[1, 2\]."
-        ),
-    ):
-        MySimpleSchema.sample(
-            overrides=[
-                {"a": 1, "b": "one"},
-                {"a": 2},
-                {"b": 2},
-            ]
-        )
-
-
 @pytest.mark.parametrize(
     "overrides,failed_column,failed_rule,failed_rows",
     [
@@ -252,3 +236,28 @@ def test_sample_invalid_override_values_raises(
     ):
         with dy.Config(max_sampling_iterations=100):  # speed up the test
             MyAdvancedSchema.sample(overrides=overrides)
+
+
+def test_sample_empty_override_sequence() -> None:
+    df = MySimpleSchema.sample(overrides=[])
+    assert len(df) == 0
+
+
+def test_sample_override_sequence_with_missing_keys() -> None:
+    df = MySimpleSchema.sample(overrides=[{"a": 1}, {"b": "two"}])
+    assert df.item(0, 0) == 1
+    assert df.item(1, 1) == "two"
+    assert len(df) == 2
+
+
+def test_sample_override_sequence_with_missing_keys_and_resampling() -> None:
+    schema = create_schema("test", {"a": dy.UInt8(primary_key=True), "b": dy.String()})
+    generator = Generator(seed=42)
+    df = schema.sample(
+        overrides=[{"a": i} for i in range(250)] + [{"b": "two"}, {"b": "three"}],
+        generator=generator,
+    )
+    assert len(df) == 252
+    assert all(df.item(i, 0) == i for i in range(250))
+    assert df.item(250, 1) == "two"
+    assert df.item(251, 1) == "three"