dataframely 3.0.0__tar.gz → 3.1.0__tar.gz

{dataframely-3.0.0 → dataframely-3.1.0}/.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0
+        uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1
         with:
           environments: build
       - name: Set version
@@ -52,7 +52,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0
+        uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1
         with:
           environments: build
       - name: Set version
@@ -87,7 +87,6 @@ jobs:
           command: build
           args: --out dist --release -i python3.10
           manylinux: auto
-          sccache: ${{ github.event_name != 'release' }}
           # NOTE: We also need to set up cargo-auditable inside the docker container
           # where the linux build is performed.
           docker-options: ${{ case(github.event_name == 'release', format('-e CARGO={0}', env.CARGO), '') }}
@@ -115,4 +114,4 @@ jobs:
           path: dist
           merge-multiple: true
       - name: Publish package on PyPi
-        uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

{dataframely-3.0.0 → dataframely-3.1.0}/.github/workflows/chore.yml

@@ -54,6 +54,6 @@ jobs:
       pull-requests: write
     steps:
       - name: Assign labels
-        uses: release-drafter/release-drafter/autolabeler@eada3c96a64734dd381cfbda23511034e328ddb0 # v7.6.0
+        uses: release-drafter/release-drafter/autolabeler@34d80673e067bdc0c24568d3af899c216adcfaa9 # v7.7.0
         with:
           token: ${{ github.token }}

{dataframely-3.0.0 → dataframely-3.1.0}/.github/workflows/ci.yml

@@ -24,13 +24,13 @@ jobs:
           # needed for 'pre-commit-mirrors-insert-license'
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0
+        uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1
         with:
-          environments: default lint polars-minimal
+          environments: default default-py310
       - name: Install Rust
         run: rustup show
       - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
       - name: Run linting
         run: pixi run lint
         env:
@@ -65,7 +65,7 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0
+        uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1
         with:
           environments: ${{ matrix.environment }}
           # FIXME: Remove when `s3_server` fixture does not start a process anymore
@@ -73,7 +73,7 @@ jobs:
       - name: Install Rust
         run: rustup show
       - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
       - name: Install repository
         run: pixi run -e ${{ matrix.environment }} postinstall
       - name: Run pytest

{dataframely-3.0.0 → dataframely-3.1.0}/.github/workflows/copilot-setup-steps.yml

@@ -15,12 +15,12 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0
+        uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1
         with:
           environments: default
       - name: Install Rust
         run: rustup show
       - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
       - name: Install repository
         run: pixi run postinstall

{dataframely-3.0.0 → dataframely-3.1.0}/.github/workflows/nightly.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0
+        uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1
         with:
           environments: nightly
       - name: Install polars nightly

{dataframely-3.0.0 → dataframely-3.1.0}/.github/workflows/release-drafter.yml

@@ -16,6 +16,6 @@ jobs:
       pull-requests: read
     steps:
       - name: Update release draft
-        uses: release-drafter/release-drafter@eada3c96a64734dd381cfbda23511034e328ddb0 # v7.6.0
+        uses: release-drafter/release-drafter@34d80673e067bdc0c24568d3af899c216adcfaa9 # v7.7.0
         with:
           token: ${{ github.token }}

{dataframely-3.0.0 → dataframely-3.1.0}/.github/workflows/scorecard.yml

@@ -40,7 +40,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+        uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
         with:
           results_file: results.sarif
           results_format: sarif
@@ -74,6 +74,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
+        uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8
         with:
           sarif_file: results.sarif

{dataframely-3.0.0 → dataframely-3.1.0}/.lefthook.yaml

@@ -21,7 +21,7 @@ pre-commit:
             run: pixi {run} ruff format --force-exclude
           - name: mypy
             glob: "*.py"
-            run: pixi {run} mypy {staged_files}
+            run: pixi {run} -e default-py310 mypy {staged_files}
           - name: prettier
             glob: "*.{md,yml,yaml}"
             run: pixi {run} prettier --write --no-error-on-unmatched-pattern --list-different --ignore-unknown {staged_files}

{dataframely-3.0.0 → dataframely-3.1.0}/Cargo.lock

@@ -2278,9 +2278,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.29.0"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
 dependencies = [
  "libc",
  "once_cell",
@@ -2292,18 +2292,18 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.29.0"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
 dependencies = [
  "target-lexicon",
 ]
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.29.0"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -2311,9 +2311,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.29.0"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -2323,9 +2323,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.29.0"
+version = "0.29.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -3215,18 +3215,18 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "thiserror"
-version = "2.0.19"
+version = "2.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
+checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.19"
+version = "2.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
+checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
 dependencies = [
  "proc-macro2",
  "quote",

{dataframely-3.0.0 → dataframely-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 3.0.0
+Version: 3.1.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -69,6 +69,7 @@ pip install dataframely
 import dataframely as dy
 import polars as pl
 
+
 class HouseSchema(dy.Schema):
     zip_code = dy.String(nullable=False, min_length=3)
     num_bedrooms = dy.UInt8(nullable=False)
@@ -88,15 +89,16 @@ class HouseSchema(dy.Schema):
 ### Validating data against schema
 
 ```python
-
 import polars as pl
 
-df = pl.DataFrame({
-    "zip_code": ["01234", "01234", "1", "213", "123", "213"],
-    "num_bedrooms": [2, 2, 1, None, None, 2],
-    "num_bathrooms": [1, 2, 1, 1, 0, 8],
-    "price": [100_000, 110_000, 50_000, 80_000, 60_000, 160_000]
-})
+df = pl.DataFrame(
+    {
+        "zip_code": ["01234", "01234", "1", "213", "123", "213"],
+        "num_bedrooms": [2, 2, 1, None, None, 2],
+        "num_bathrooms": [1, 2, 1, 1, 0, 8],
+        "price": [100_000, 110_000, 50_000, 80_000, 60_000, 160_000],
+    }
+)
 
 # Validate the data and cast columns to expected types
 validated_df: dy.DataFrame[HouseSchema] = HouseSchema.validate(df, cast=True)

{dataframely-3.0.0 → dataframely-3.1.0}/README.md

@@ -45,6 +45,7 @@ pip install dataframely
 import dataframely as dy
 import polars as pl
 
+
 class HouseSchema(dy.Schema):
     zip_code = dy.String(nullable=False, min_length=3)
     num_bedrooms = dy.UInt8(nullable=False)
@@ -64,15 +65,16 @@ class HouseSchema(dy.Schema):
 ### Validating data against schema
 
 ```python
-
 import polars as pl
 
-df = pl.DataFrame({
-    "zip_code": ["01234", "01234", "1", "213", "123", "213"],
-    "num_bedrooms": [2, 2, 1, None, None, 2],
-    "num_bathrooms": [1, 2, 1, 1, 0, 8],
-    "price": [100_000, 110_000, 50_000, 80_000, 60_000, 160_000]
-})
+df = pl.DataFrame(
+    {
+        "zip_code": ["01234", "01234", "1", "213", "123", "213"],
+        "num_bedrooms": [2, 2, 1, None, None, 2],
+        "num_bathrooms": [1, 2, 1, 1, 0, 8],
+        "price": [100_000, 110_000, 50_000, 80_000, 60_000, 160_000],
+    }
+)
 
 # Validate the data and cast columns to expected types
 validated_df: dy.DataFrame[HouseSchema] = HouseSchema.validate(df, cast=True)

{dataframely-3.0.0 → dataframely-3.1.0}/dataframely/_base_schema.py

@@ -190,8 +190,7 @@ class SchemaMeta(ABCMeta):
             # Dynamically set the name of the column if it is a `Column` instance.
             # Also, we "register" the name of the schema that set the name.
             if isinstance(val, Column):
-                val._schema = f"{cls.__module__}:{cls.__name__}"
-                val._name = val.alias or name
+                val._bind(f"{cls.__module__}:{cls.__name__}", val.alias or name)
             return val
 
     @staticmethod
@@ -294,8 +293,7 @@ class BaseSchema(metaclass=SchemaMeta):
         columns: dict[str, Column] = getattr(cls, _COLUMN_ATTR)
         for name in columns.keys():
             # Dynamically set the name and source schema of the columns.
-            columns[name]._schema = f"{cls.__module__}:{cls.__name__}"
-            columns[name]._name = name
+            columns[name]._bind(f"{cls.__module__}:{cls.__name__}", name)
         return columns
 
     @classmethod

{dataframely-3.0.0 → dataframely-3.1.0}/dataframely/columns/_base.py

@@ -276,6 +276,11 @@ class Column(ABC):
 
     # ------------------------------------ HELPER ------------------------------------ #
 
+    def _bind(self, schema: str, name: str) -> None:
+        """Set the schema and column path used by schema-scoped data types."""
+        self._schema = schema
+        self._name = name
+
     @property
     def name(self) -> str:
         """Get the name of the column in a schema."""

{dataframely-3.0.0 → dataframely-3.1.0}/dataframely/columns/array.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 import math
 import warnings
+from copy import copy
 from typing import Any, cast
 
 import polars as pl
@@ -75,6 +76,11 @@ class Array(Column):
         self.inner = inner
         self.shape = shape if isinstance(shape, tuple) else (shape,)
 
+    def _bind(self, schema: str, name: str) -> None:
+        super()._bind(schema, name)
+        self.inner = copy(self.inner)
+        self.inner._bind(schema, f"{name}.inner")
+
     @property
     def dtype(self) -> pl.DataType:
         return pl.Array(self.inner.dtype, self.shape)

{dataframely-3.0.0 → dataframely-3.1.0}/dataframely/columns/categorical.py

@@ -39,6 +39,9 @@ class Categorical(Column):
                 a data type is provided, name and namespace are synthesized from the
                 enclosing schema and column name, automatically creating a column-
                 scoped categories dictionary.
+                List and array elements append `.inner` to the containing column's
+                name; struct fields append their field names with dots (for example,
+                `items.inner.kind` for a field in a list of structs).
             nullable: Whether this column may contain null values.
                 Explicitly set `nullable=True` if you want your column to be nullable.
                 In a future release, `nullable=False` will be the default if `nullable`

{dataframely-3.0.0 → dataframely-3.1.0}/dataframely/columns/list.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+from copy import copy
 from itertools import chain
 from typing import Any, cast
 
@@ -85,6 +86,11 @@ class List(Column):
         self.min_length = min_length
         self.max_length = max_length
 
+    def _bind(self, schema: str, name: str) -> None:
+        super()._bind(schema, name)
+        self.inner = copy(self.inner)
+        self.inner._bind(schema, f"{name}.inner")
+
     @property
     def dtype(self) -> pl.DataType:
         return pl.List(self.inner.dtype)

{dataframely-3.0.0 → dataframely-3.1.0}/dataframely/columns/struct.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+from copy import copy
 from typing import Any, cast
 
 import polars as pl
@@ -76,6 +77,12 @@ class Struct(Column):
         )
         self.inner = inner
 
+    def _bind(self, schema: str, name: str) -> None:
+        super()._bind(schema, name)
+        self.inner = {field: copy(col) for field, col in self.inner.items()}
+        for field, col in self.inner.items():
+            col._bind(schema, f"{name}.{field}")
+
     @property
     def dtype(self) -> pl.DataType:
         return pl.Struct({name: col.dtype for name, col in self.inner.items()})

{dataframely-3.0.0 → dataframely-3.1.0}/dataframely/filter_result.py

@@ -128,6 +128,9 @@ class FailureInfo:
         filters in addition to member-level rules, or when calling :meth:`Schema.filter`
         with `cast=True` and dtype-casting fails for a value.
         """
+        if len(self._rule_columns) == 0:
+            return self.invalid()
+
         return self._df.select(
             pl.exclude(self._rule_columns),
             pl.col(*self._rule_columns).replace_strict(
@@ -166,24 +169,41 @@ class FailureInfo:
 
     # ---------------------------------- PERSISTENCE --------------------------------- #
 
-    def write_parquet(self, file: str | Path | IO[bytes], **kwargs: Any) -> None:
+    def write_parquet(
+        self,
+        file: str | Path | IO[bytes],
+        *,
+        only_failing_rules: bool = False,
+        **kwargs: Any,
+    ) -> None:
         """Write the failure info to a single parquet file.
 
         Writes the invalid rows along with additional boolean rule columns indicating
         which validation rules failed. Unlike :meth:`invalid`, this includes columns
-        for each rule, where ``False`` indicates the rule failed for that row.
+        for each rule by default, where ``False`` indicates the rule failed for that row.
 
         Args:
             file: The file path or writable file-like object to which to write the
                 parquet file.
+            only_failing_rules: Whether to write only rule columns containing at least
+                one validation failure.
             kwargs: Additional keyword arguments passed directly to
                 :meth:`polars.write_parquet`. `metadata` may only be provided if it
                 is a dictionary.
         """
         metadata = kwargs.pop("metadata", {}) or {}
-        self._df.write_parquet(
+        df = self._df
+        rule_columns = self._rule_columns
+        if only_failing_rules:
+            counts = _compute_counts(df, self._rule_columns)
+            rule_columns = [column for column in self._rule_columns if column in counts]
+            df = df.drop(
+                column for column in self._rule_columns if column not in counts
+            )
+
+        df.write_parquet(
             file,
-            metadata={**metadata, "rule_columns": json.dumps(self._rule_columns)},
+            metadata={**metadata, "rule_columns": json.dumps(rule_columns)},
             **kwargs,
         )
 

{dataframely-3.0.0 → dataframely-3.1.0}/docs/guides/coding-agents.md

@@ -9,13 +9,13 @@ Coding agents like [Claude Code](https://code.claude.com/), [Codex](https://open
 When writing data processing logic, `dataframely` helps to fulfill these criteria.
 
 To help your coding agent write idiomatic `dataframely` code, we provide a `dataframely`
-[skill](https://raw.githubusercontent.com/Quantco/dataframely/refs/heads/main/SKILL.md) following the
+[skill](https://raw.githubusercontent.com/Quantco/dataframely/refs/heads/main/skills/SKILL.md) following the
 [`agentskills.io` spec](https://agentskills.io/specification). You can install it by placing it where your agent can
 find it. For example, if you are using Claude Code:
 
 ```bash
 mkdir -p .claude/skills/dataframely/
-curl -o .claude/skills/dataframely/SKILL.md https://raw.githubusercontent.com/Quantco/dataframely/refs/heads/main/skills/SKILL.md
+curl -fLo .claude/skills/dataframely/SKILL.md https://raw.githubusercontent.com/Quantco/dataframely/refs/heads/main/skills/SKILL.md
 ```
 
 or if you are using [skills.sh](https://skills.sh/) to manage your skills:
@@ -36,15 +36,15 @@ right time.
 For example:
 
 ```python
-def preprocess(raw: dy.LazyFrame[MyRawSchema]) -> dy.DataFrame[MyPreprocessedSchema]:
-    ...
+def preprocess(
+    raw: dy.LazyFrame[MyRawSchema],
+) -> dy.DataFrame[MyPreprocessedSchema]: ...
 ```
 
 gives a coding agent much more information than the schema-less alternative:
 
 ```python
-def load_data(raw: pl.LazyFrame) -> pl.DataFrame:
-    ...
+def load_data(raw: pl.LazyFrame) -> pl.DataFrame: ...
 ```
 
 This convention also makes your code more readable and maintainable for human developers.

{dataframely-3.0.0 → dataframely-3.1.0}/docs/guides/features/column-metadata.md

@@ -8,10 +8,12 @@ For instance, one may use the `metadata` parameter to mark a column as pseudonym
 class UserSchema(dy.Schema):
     id = dy.String(primary_key=True)
     # Mark last name column as pseudonymized and (non-docstring) comment on it.
-    last_name = dy.String(metadata={
-        "pseudonymized": True,
-        "comment": "Pseudonymized using cryptographic hash function"
-    })
+    last_name = dy.String(
+        metadata={
+            "pseudonymized": True,
+            "comment": "Pseudonymized using cryptographic hash function",
+        }
+    )
     # Add information about database column type.
     address = dy.String(metadata={"database-type": "VARCHAR(MAX)"})
 ```

{dataframely-3.0.0 → dataframely-3.1.0}/docs/guides/features/data-generation.md

@@ -17,6 +17,7 @@ class InvoiceSchema(dy.Schema):
     discharge_date = dy.Date(nullable=False)
     amount = dy.Decimal(nullable=False)
 
+
 # Get data frame with correct type hint.
 df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.create_empty()
 ```
@@ -34,6 +35,7 @@ class InvoiceSchema(dy.Schema):
     discharge_date = dy.Date(nullable=False)
     amount = dy.Decimal(nullable=False)
 
+
 df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample(num_rows=100)
 ```
 
@@ -54,6 +56,7 @@ class InvoiceSchema(dy.Schema):
     def discharge_after_admission(cls) -> pl.Expr:
         return InvoiceSchema.discharge_date.col >= InvoiceSchema.admission_date.col
 
+
 # `@dy.rule`s will be respected as well for data generation.
 df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample(num_rows=100)
 ```
@@ -79,25 +82,30 @@ The column-wise specification specifies an iterable of values for each specified
 from datetime import date
 
 # Override values for specific columns.
-df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample(overrides={
-    # Use either <schema>.<column>.name or just the column name as a string.
-    InvoiceSchema.invoice_id.name: ["1234567890", "2345678901", "3456789012"],
-    # Dataframely will automatically infer the number of rows based on the longest given
-    # sequence of values and broadcast all other columns to that shape.
-    "admission_date": date(2025, 1, 1),
-})
+df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample(
+    overrides={
+        # Use either <schema>.<column>.name or just the column name as a string.
+        InvoiceSchema.invoice_id.name: ["1234567890", "2345678901", "3456789012"],
+        # Dataframely will automatically infer the number of rows based on the longest given
+        # sequence of values and broadcast all other columns to that shape.
+        "admission_date": date(2025, 1, 1),
+    }
+)
 ```
 
 The row-wise specification implements an iterable of mappings for the rows that should be sampled. It is particularly helpful if you want to make it easy to understand how values will be combined in specific rows (e.g., when each row represents one object).
 
 ```python
 from datetime import date
+
 # Override values for specific columns.
-df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample(overrides=[
-    {"invoice_id": "1234567890", "admission_date": date(2025, 1, 1)},
-    {"invoice_id": "2345678901", "admission_date": date(2025, 1, 1)},
-    {"invoice_id": "3456789012", "admission_date": date(2025, 1, 1)},
-])
+df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample(
+    overrides=[
+        {"invoice_id": "1234567890", "admission_date": date(2025, 1, 1)},
+        {"invoice_id": "2345678901", "admission_date": date(2025, 1, 1)},
+        {"invoice_id": "3456789012", "admission_date": date(2025, 1, 1)},
+    ]
+)
 ```
 
 ### Providing custom column overrides
@@ -107,6 +115,8 @@ Complex validation rules (such as dependencies between columns or ordering crite
 ```python
 import polars as pl
 import dataframely as dy
+
+
 class OrderedSchema(dy.Schema):
     """A schema that requires `iter` to be ordered with respect to `a` and `b`."""
 
@@ -116,7 +126,9 @@ class OrderedSchema(dy.Schema):
 
     @dy.rule()
     def iter_order_correct(cls) -> pl.Expr:
-        return pl.col("iter").rank(method="ordinal") == pl.struct(pl.col("a"), pl.col("b")).rank(method="ordinal")
+        return pl.col("iter").rank(method="ordinal") == pl.struct(
+            pl.col("a"), pl.col("b")
+        ).rank(method="ordinal")
 
     @classmethod
     def _sampling_overrides(cls) -> dict[str, pl.Expr]:
@@ -124,6 +136,7 @@ class OrderedSchema(dy.Schema):
             "iter": pl.struct(pl.col("a"), pl.col("b")).rank(method="ordinal"),
         }
 
+
 result = OrderedSchema.sample(100)
 ```
 
@@ -137,10 +150,12 @@ class DiagnosisSchema(dy.Schema):
     invoice_id = dy.String(primary_key=True)
     code = dy.String(nullable=False, regex=r"[A-Z][0-9]{2,4}")
 
+
 class HospitalInvoiceData(dy.Collection):
     invoice: dy.LazyFrame[InvoiceSchema]
     diagnosis: dy.LazyFrame[DiagnosisSchema]
 
+
 invoice_data: HospitalInvoiceData = HospitalInvoiceData.sample(num_rows=10)
 ```
 
@@ -153,6 +168,7 @@ class DiagnosisSchema(dy.Schema):
     invoice_id = dy.String(primary_key=True)
     code = dy.String(primary_key=True, regex=r"[A-Z][0-9]{2,4}")
 
+
 class HospitalInvoiceData(dy.Collection):
     invoice: dy.LazyFrame[InvoiceSchema]
     diagnosis: dy.LazyFrame[DiagnosisSchema]
@@ -190,7 +206,9 @@ class HospitalInvoiceData(dy.Collection):
 
     @classmethod
     @override
-    def _preprocess_sample(cls, sample: dict[str, Any], index: int, generator: Generator):
+    def _preprocess_sample(
+        cls, sample: dict[str, Any], index: int, generator: Generator
+    ):
         # Set common primary key.
         if "invoice_id" not in sample:
             sample["invoice_id"] = str(index)

{dataframely-3.0.0 → dataframely-3.1.0}/docs/guides/features/serialization.md

@@ -65,3 +65,14 @@ failure.write_parquet("failures.parquet")
 # ...and read it back.
 failure = dy.FailureInfo.read_parquet("failures.parquet")
 ```
+
+By default, all rule-output columns are persisted. To create a narrower parquet file
+for debugging, keep only rule columns that contain at least one validation failure:
+
+```python
+failure.write_parquet("failures.parquet", only_failing_rules=True)
+```
+
+All data columns present in the failure information are still written. This reduced
+representation is intentionally lossy: rule columns containing only successful or
+unknown outcomes are omitted and unavailable after reading the file.