dataframely 1.7.6__tar.gz → 1.8.0__tar.gz

{dataframely-1.7.6 → dataframely-1.8.0}/.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@14c8aabd75893f83f4ab30c03e7cf853c8208961 # v0.8.10
+        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
         with:
           environments: build
       - name: Set version
@@ -52,7 +52,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@14c8aabd75893f83f4ab30c03e7cf853c8208961 # v0.8.10
+        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
         with:
           environments: build
       - name: Set version

{dataframely-1.7.6 → dataframely-1.8.0}/.github/workflows/chore.yml

@@ -28,7 +28,7 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
       - name: Post comment about invalid PR title
         if: failure()
-        uses: marocchino/sticky-pull-request-comment@d2ad0de260ae8b0235ce059e63f2949ba9e05943 # v2.9.3
+        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
         with:
           header: conventional-commit-pr-title
           message: |
@@ -45,7 +45,7 @@ jobs:
             </details>
       - name: Delete comment about invalid PR title
         if: success()
-        uses: marocchino/sticky-pull-request-comment@d2ad0de260ae8b0235ce059e63f2949ba9e05943 # v2.9.3
+        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
         with:
           header: conventional-commit-pr-title
           delete: true

{dataframely-1.7.6 → dataframely-1.8.0}/.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
           # needed for 'pre-commit-mirrors-insert-license'
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@14c8aabd75893f83f4ab30c03e7cf853c8208961 # v0.8.10
+        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
         with:
           environments: default lint
       - name: Install repository
@@ -45,7 +45,7 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@14c8aabd75893f83f4ab30c03e7cf853c8208961 # v0.8.10
+        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
         with:
           environments: ${{ matrix.environment }}
       - name: Install repository

{dataframely-1.7.6 → dataframely-1.8.0}/.github/workflows/nightly.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Checkout branch
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@14c8aabd75893f83f4ab30c03e7cf853c8208961 # v0.8.10
+        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
         with:
           environments: nightly
       - name: Install polars nightly

{dataframely-1.7.6 → dataframely-1.8.0}/.github/workflows/scorecard.yml

@@ -74,6 +74,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
+        uses: github/codeql-action/upload-sarif@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
         with:
           sarif_file: results.sarif

{dataframely-1.7.6 → dataframely-1.8.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 1.7.6
+Version: 1.8.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: numpy
-Requires-Dist: polars>=1.30
+Requires-Dist: polars>=1.32
 Requires-Dist: typing-extensions ; python_full_version < '3.11'
 License-File: LICENSE
 Summary: A declarative, polars-native data frame validation library

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/__init__.py

@@ -15,11 +15,16 @@ from ._base_collection import CollectionMember
 from ._filter import filter
 from ._rule import rule
 from ._typing import DataFrame, LazyFrame, Validation
-from .collection import Collection, deserialize_collection
+from .collection import (
+    Collection,
+    deserialize_collection,
+    read_parquet_metadata_collection,
+)
 from .columns import (
     Any,
     Array,
     Bool,
+    Categorical,
     Column,
     Date,
     Datetime,
@@ -70,8 +75,10 @@ __all__ = [
     "Schema",
     "deserialize_schema",
     "read_parquet_metadata_schema",
+    "read_parquet_metadata_collection",
     "Any",
     "Bool",
+    "Categorical",
     "Column",
     "Date",
     "Datetime",

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/_serialization.py

@@ -11,6 +11,7 @@ from typing import Any, cast
 import polars as pl
 
 SCHEMA_METADATA_KEY = "dataframely_schema"
+COLLECTION_METADATA_KEY = "dataframely_collection"
 SERIALIZATION_FORMAT_VERSION = "1"
 
 
@@ -45,7 +46,7 @@ class SchemaJSONEncoder(JSONEncoder):
             case pl.Expr():
                 return {
                     "__type__": "expression",
-                    "value": obj.meta.serialize(format="json"),
+                    "value": base64.b64encode(obj.meta.serialize()).decode("utf-8"),
                 }
             case pl.LazyFrame():
                 return {
@@ -86,8 +87,14 @@ class SchemaJSONDecoder(JSONDecoder):
             case "tuple":
                 return tuple(dct["value"])
             case "expression":
-                data = BytesIO(cast(str, dct["value"]).encode("utf-8"))
-                return pl.Expr.deserialize(data, format="json")
+                value_str = cast(str, dct["value"]).encode("utf-8")
+                if value_str.startswith(b"{"):
+                    # NOTE: This branch is for backwards-compatibility only
+                    data = BytesIO(value_str)
+                    return pl.Expr.deserialize(data, format="json")
+                else:
+                    data = BytesIO(base64.b64decode(value_str))
+                    return pl.Expr.deserialize(data)
             case "lazyframe":
                 data = BytesIO(
                     base64.b64decode(cast(str, dct["value"]).encode("utf-8"))

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/collection.py

@@ -1,14 +1,16 @@
 # Copyright (c) QuantCo 2025-2025
 # SPDX-License-Identifier: BSD-3-Clause
+from __future__ import annotations
 
 import json
 import sys
 import warnings
 from abc import ABC
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import asdict
+from json import JSONDecodeError
 from pathlib import Path
-from typing import Annotated, Any, cast
+from typing import IO, Annotated, Any, cast
 
 import polars as pl
 import polars.exceptions as plexc
@@ -17,6 +19,7 @@ from ._base_collection import BaseCollection, CollectionMember
 from ._filter import Filter
 from ._polars import FrameType, join_all_inner, join_all_outer
 from ._serialization import (
+    COLLECTION_METADATA_KEY,
     SERIALIZATION_FORMAT_VERSION,
     SchemaJSONDecoder,
     SchemaJSONEncoder,
@@ -147,9 +150,16 @@ class Collection(BaseCollection, ABC):
             A collection where all members (including optional ones) have been sampled
             according to the input parameters.
 
+        Attention:
+            In case the collection has members with a common primary key, the
+            `_preprocess_sample` method must return distinct primary key values for each
+            sample. The default implementation does this on a best-effort basis but may
+            cause primary key violations. Hence, it is recommended to override this
+            method and ensure that all primary key columns are set.
+
         Raises:
             ValueError: If the :meth:`_preprocess_sample` method does not return all
-                common primary keys for all samples.
+                common primary key columns for all samples.
             ValidationError: If the sampled members violate any of the collection
                 filters. If the collection does not have filters, this error is never
                 raised. To prevent validation errors, overwrite the
@@ -247,7 +257,7 @@ class Collection(BaseCollection, ABC):
         return cls.validate(members)
 
     @classmethod
-    def matches(cls, other: type["Collection"]) -> bool:
+    def matches(cls, other: type[Collection]) -> bool:
         """Check whether this collection semantically matches another.
 
         Args:
@@ -317,9 +327,7 @@ class Collection(BaseCollection, ABC):
         of fuzzy sampling rounds when sampling individual members.
 
         Args:
-            sample: The sample to preprocess. By default, this is a simple dictionary.
-                Subclasses may decide, however, to introduce a :class:`TypedDict` to
-                ease the creation of samples.
+            sample: The sample to preprocess.
             index: The index of the sample in the list of samples. Typically, this value
                 can be used to assign unique primary keys for samples.
             generator: The generator to use when performing random sampling within the
@@ -331,10 +339,18 @@ class Collection(BaseCollection, ABC):
             primary keys.
         """
         if len(cls.members()) > 1 and len(cls.common_primary_keys()) > 0:
-            raise ValueError(
-                "`_preprocess_sample` must be overwritten for collections with more "
-                "than 1 member sharing a common primary key."
-            )
+            # If we have multiple members with a common primary key, we need to ensure
+            # that the samples have a value set for all common primary key columns.
+            # NOTE: This is experimental as we commit to a primary key that cannot be
+            #  changed at a later point (e.g. due to primary key violations).
+            first_member_columns = next(iter(cls.member_schemas().values())).columns()
+            for primary_key in cls.common_primary_keys():
+                if primary_key in sample:
+                    continue
+
+                value = first_member_columns[primary_key].sample(generator).item()
+                sample[primary_key] = value
+
         return sample
 
     # ---------------------------------- VALIDATION ---------------------------------- #
@@ -638,9 +654,6 @@ class Collection(BaseCollection, ABC):
         Each parquet file is named ``<member>.parquet``. No file is written for optional
         members which are not provided in the current collection.
 
-        In addition, one JSON file named ``schema.json`` is written, serializing the
-        collection's definition for fast reads.
-
         Args:
             directory: The directory where the Parquet files should be written to. If
                 the directory does not exist, it is created automatically, including all
@@ -661,9 +674,6 @@ class Collection(BaseCollection, ABC):
         Each parquet file is named ``<member>.parquet``. No file is written for optional
         members which are not provided in the current collection.
 
-        In addition, one JSON file named ``schema.json`` is written, serializing the
-        collection's definition for fast reads.
-
         Args:
             directory: The directory where the Parquet files should be written to. If
                 the directory does not exist, it is created automatically, including all
@@ -680,8 +690,11 @@ class Collection(BaseCollection, ABC):
     def _to_parquet(self, directory: str | Path, *, sink: bool, **kwargs: Any) -> None:
         path = Path(directory) if isinstance(directory, str) else directory
         path.mkdir(parents=True, exist_ok=True)
-        with open(path / "schema.json", "w") as f:
-            f.write(self.serialize())
+
+        # The collection schema is serialized as part of the member parquet metadata
+        kwargs["metadata"] = kwargs.get("metadata", {}) | {
+            COLLECTION_METADATA_KEY: self.serialize()
+        }
 
         member_schemas = self.member_schemas()
         for key, lf in self.to_dict().items():
@@ -719,16 +732,16 @@ class Collection(BaseCollection, ABC):
                 Parquet files may have been written with Hive partitioning.
             validation: The strategy for running validation when reading the data:
 
-                - ``"allow"`: The method tries to read the ``schema.json`` file in the
-                  directory. If the stored collection schema matches this collection
+                - ``"allow"`: The method tries to read the schema data from the parquet
+                  files. If the stored collection schema matches this collection
                   schema, the collection is read without validation. If the stored
-                  schema mismatches this schema or no ``schema.json`` can be found in
-                  the directory, this method automatically runs :meth:`validate` with
-                  ``cast=True``.
+                  schema mismatches this schema no metadata can be found in
+                  the parquets, or the files have conflicting metadata,
+                  this method automatically runs :meth:`validate` with ``cast=True``.
                 - ``"warn"`: The method behaves similarly to ``"allow"``. However,
                   it prints a warning if validation is necessary.
                 - ``"forbid"``: The method never runs validation automatically and only
-                  returns if the ``schema.json`` stores a collection schema that matches
+                  returns if the metadata stores a collection schema that matches
                   this collection.
                 - ``"skip"``: The method never runs validation and simply reads the
                   data, entrusting the user that the schema is valid. _Use this option
@@ -747,13 +760,20 @@ class Collection(BaseCollection, ABC):
                 all required members.
             ValidationError: If the collection cannot be validate.
 
+        Note:
+            This method is backward compatible with older versions of dataframely
+            in which the schema metadata was saved to `schema.json` files instead of
+            being encoded into the parquet files.
+
         Attention:
             Be aware that this method suffers from the same limitations as
             :meth:`serialize`.
         """
         path = Path(directory)
-        data = cls._from_parquet(path, scan=True, **kwargs)
-        if not cls._requires_validation_for_reading_parquets(path, validation):
+        data, collection_type = cls._from_parquet(path, scan=False, **kwargs)
+        if not cls._requires_validation_for_reading_parquets(
+            path, collection_type, validation
+        ):
             cls._validate_input_keys(data)
             return cls._init(data)
         return cls.validate(data, cast=True)
@@ -776,16 +796,16 @@ class Collection(BaseCollection, ABC):
                 Parquet files may have been written with Hive partitioning.
             validation: The strategy for running validation when reading the data:
 
-                - ``"allow"`: The method tries to read the ``schema.json`` file in the
-                  directory. If the stored collection schema matches this collection
+                - ``"allow"`: The method tries to read the schema data from the parquet
+                  files. If the stored collection schema matches this collection
                   schema, the collection is read without validation. If the stored
-                  schema mismatches this schema or no ``schema.json`` can be found in
-                  the directory, this method automatically runs :meth:`validate` with
-                  ``cast=True``.
+                  schema mismatches this schema no metadata can be found in
+                  the parquets, or the files have conflicting metadata,
+                  this method automatically runs :meth:`validate` with ``cast=True``.
                 - ``"warn"`: The method behaves similarly to ``"allow"``. However,
                   it prints a warning if validation is necessary.
                 - ``"forbid"``: The method never runs validation automatically and only
-                  returns if the ``schema.json`` stores a collection schema that matches
+                  returns if the metadata stores a collection schema that matches
                   this collection.
                 - ``"skip"``: The method never runs validation and simply reads the
                   data, entrusting the user that the schema is valid. _Use this option
@@ -808,13 +828,19 @@ class Collection(BaseCollection, ABC):
             parquet file into memory if ``"validation"`` is ``"warn"`` or ``"allow"``
             and validation is required.
 
+        Note: This method is backward compatible with older versions of dataframely
+            in which the schema metadata was saved to `schema.json` files instead of
+            being encoded into the parquet files.
+
         Attention:
             Be aware that this method suffers from the same limitations as
             :meth:`serialize`.
         """
         path = Path(directory)
-        data = cls._from_parquet(path, scan=True, **kwargs)
-        if not cls._requires_validation_for_reading_parquets(path, validation):
+        data, collection_type = cls._from_parquet(path, scan=True, **kwargs)
+        if not cls._requires_validation_for_reading_parquets(
+            path, collection_type, validation
+        ):
             cls._validate_input_keys(data)
             return cls._init(data)
         return cls.validate(data, cast=True)
@@ -822,8 +848,9 @@ class Collection(BaseCollection, ABC):
     @classmethod
     def _from_parquet(
         cls, path: Path, scan: bool, **kwargs: Any
-    ) -> dict[str, pl.LazyFrame]:
+    ) -> tuple[dict[str, pl.LazyFrame], type[Collection] | None]:
         data = {}
+        collection_types = set()
         for key in cls.members():
             if (source_path := cls._member_source_path(path, key)) is not None:
                 data[key] = (
@@ -831,7 +858,22 @@ class Collection(BaseCollection, ABC):
                     if scan
                     else pl.read_parquet(source_path, **kwargs).lazy()
                 )
-        return data
+                if source_path.is_file():
+                    collection_types.add(read_parquet_metadata_collection(source_path))
+                else:
+                    for file in source_path.glob("**/*.parquet"):
+                        collection_types.add(read_parquet_metadata_collection(file))
+        collection_type = _reconcile_collection_types(collection_types)
+
+        # Backward compatibility: If the parquets do not have schema information,
+        # fall back to looking for schema.json
+        if (collection_type is None) and (schema_file := path / "schema.json").exists():
+            try:
+                collection_type = deserialize_collection(schema_file.read_text())
+            except (JSONDecodeError, plexc.ComputeError):
+                pass
+
+        return data, collection_type
 
     @classmethod
     def _member_source_path(cls, base_path: Path, name: str) -> Path | None:
@@ -847,27 +889,20 @@ class Collection(BaseCollection, ABC):
     def _requires_validation_for_reading_parquets(
         cls,
         directory: Path,
+        collection_type: type[Collection] | None,
         validation: Validation,
     ) -> bool:
         if validation == "skip":
             return False
 
-        # First, we check whether the path provides the serialization of the collection.
-        # If it does, we check whether it matches this collection. If it does, we assume
-        # that the data adheres to the collection and we do not need to run validation.
-        if (json_serialization := directory / "schema.json").exists():
-            metadata = json_serialization.read_text()
-            serialized_collection = deserialize_collection(metadata)
-            if cls.matches(serialized_collection):
-                return False
-        else:
-            serialized_collection = None
+        if collection_type is not None and cls.matches(collection_type):
+            return False
 
-        # Otherwise, we definitely need to run validation. However, we emit different
+        # Now we definitely need to run validation. However, we emit different
         # information to the user depending on the value of `validate`.
         msg = (
             "current collection schema does not match stored collection schema"
-            if serialized_collection is not None
+            if collection_type is not None
             else "no collection schema to check validity can be read from the source"
         )
         if validation == "forbid":
@@ -903,6 +938,27 @@ class Collection(BaseCollection, ABC):
             )
 
 
+def read_parquet_metadata_collection(
+    source: str | Path | IO[bytes] | bytes,
+) -> type[Collection] | None:
+    """Read a dataframely Collection type from the metadata of a parquet file.
+
+    Args:
+        source: Path to a parquet file or a file-like object that contains the metadata.
+
+    Returns:
+        The collection that was serialized to the metadata. ``None`` if no collection
+        metadata is found or the deserialization fails.
+    """
+    metadata = pl.read_parquet_metadata(source)
+    if (schema_metadata := metadata.get(COLLECTION_METADATA_KEY)) is not None:
+        try:
+            return deserialize_collection(schema_metadata)
+        except (JSONDecodeError, plexc.ComputeError):
+            return None
+    return None
+
+
 def deserialize_collection(data: str) -> type[Collection]:
     """Deserialize a collection from a JSON string.
 
@@ -968,3 +1024,21 @@ def _extract_keys_if_exist(
     data: Mapping[str, Any], keys: Sequence[str]
 ) -> dict[str, Any]:
     return {key: data[key] for key in keys if key in data}
+
+
+def _reconcile_collection_types(
+    collection_types: Iterable[type[Collection] | None],
+) -> type[Collection] | None:
+    # When reading serialized collections, we may have collection type information from multiple sources
+    # (E.g. one set of information for each parquet file).
+    # This function determines which of them should finally be used
+    if not (collection_types := list(collection_types)):
+        return None
+    if (first_type := collection_types[0]) is None:
+        return None
+    for t in collection_types[1:]:
+        if t is None:
+            return None
+        if not first_type.matches(t):
+            return None
+    return first_type

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/columns/__init__.py

@@ -6,6 +6,7 @@ from ._registry import column_from_dict
 from .any import Any
 from .array import Array
 from .bool import Bool
+from .categorical import Categorical
 from .datetime import Date, Datetime, Duration, Time
 from .decimal import Decimal
 from .enum import Enum
@@ -22,6 +23,7 @@ __all__ = [
     "Any",
     "Array",
     "Bool",
+    "Categorical",
     "Date",
     "Datetime",
     "Decimal",

dataframely-1.8.0/dataframely/columns/categorical.py

@@ -0,0 +1,77 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+from typing import Any
+
+import polars as pl
+from sqlalchemy import Dialect
+
+from dataframely._compat import pa, sa, sa_TypeEngine
+from dataframely.random import Generator
+
+from ._base import Check, Column
+from ._registry import register
+
+
+@register
+class Categorical(Column):
+    """A column of categorical (string) values."""
+
+    def __init__(
+        self,
+        *,
+        nullable: bool | None = None,
+        primary_key: bool = False,
+        check: Check | None = None,
+        alias: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ):
+        """
+        Args:
+            nullable: Whether this column may contain null values.
+                Explicitly set `nullable=True` if you want your column to be nullable.
+                In a future release, `nullable=False` will be the default if `nullable`
+                is not specified.
+            primary_key: Whether this column is part of the primary key of the schema.
+                If ``True``, ``nullable`` is automatically set to ``False``.
+            check: A custom rule or multiple rules to run for this column. This can be:
+                - A single callable that returns a non-aggregated boolean expression.
+                The name of the rule is derived from the callable name, or defaults to
+                "check" for lambdas.
+                - A list of callables, where each callable returns a non-aggregated
+                boolean expression. The name of the rule is derived from the callable
+                name, or defaults to "check" for lambdas. Where multiple rules result
+                in the same name, the suffix __i is appended to the name.
+                - A dictionary mapping rule names to callables, where each callable
+                returns a non-aggregated boolean expression.
+                All rule names provided here are given the prefix "check_".
+            alias: An overwrite for this column's name which allows for using a column
+                name that is not a valid Python identifier. Especially note that setting
+                this option does _not_ allow to refer to the column with two different
+                names, the specified alias is the only valid name.
+            metadata: A dictionary of metadata to attach to the column.
+        """
+        super().__init__(
+            nullable=nullable,
+            primary_key=primary_key,
+            check=check,
+            alias=alias,
+            metadata=metadata,
+        )
+
+    @property
+    def dtype(self) -> pl.DataType:
+        return pl.Categorical()
+
+    def sqlalchemy_dtype(self, dialect: Dialect) -> sa_TypeEngine:
+        return sa.String()
+
+    @property
+    def pyarrow_dtype(self) -> pa.DataType:
+        return pa.dictionary(pa.uint32(), pa.large_string())
+
+    def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
+        # We simply sample low-cardinality strings here
+        return generator.sample_string(
+            n, regex=r"[a-z]{1,2}", null_probability=self._null_probability
+        ).cast(self.dtype)

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/columns/enum.py

@@ -82,7 +82,13 @@ class Enum(Column):
 
     @property
     def pyarrow_dtype(self) -> pa.DataType:
-        return pa.dictionary(pa.uint32(), pa.large_string())
+        if len(self.categories) <= 2**8 - 2:
+            dtype = pa.uint8()
+        elif len(self.categories) <= 2**16 - 2:
+            dtype = pa.uint16()
+        else:
+            dtype = pa.uint32()
+        return pa.dictionary(dtype, pa.large_string())
 
     def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
         return generator.sample_choice(

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/random.py

@@ -40,6 +40,7 @@ class Generator:
             seed: The seed to use for initializing the random number generator used
                 for all sampling methods.
         """
+        self.seed = seed
         self.numpy_generator = np.random.default_rng(seed)
 
     def sample_seed(self) -> int:

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/schema.py

@@ -8,6 +8,7 @@ import sys
 import warnings
 from abc import ABC
 from collections.abc import Iterable, Mapping, Sequence
+from json import JSONDecodeError
 from pathlib import Path
 from typing import IO, Any, Literal, overload
 
@@ -970,12 +971,15 @@ def read_parquet_metadata_schema(
         source: Path to a parquet file or a file-like object that contains the metadata.
 
     Returns:
-        The schema that was serialized to the metadata or ``None`` if no schema metadata
-        is found.
+        The schema that was serialized to the metadata. ``None`` if no schema metadata
+        is found or the deserialization fails.
     """
     metadata = pl.read_parquet_metadata(source)
     if (schema_metadata := metadata.get(SCHEMA_METADATA_KEY)) is not None:
-        return deserialize_schema(schema_metadata)
+        try:
+            return deserialize_schema(schema_metadata)
+        except (JSONDecodeError, plexc.ComputeError):
+            return None
     return None
 
 

{dataframely-1.7.6 → dataframely-1.8.0}/dataframely/testing/const.py

@@ -21,6 +21,7 @@ COLUMN_TYPES: list[type[dc.Column]] = [
     dc.UInt32,
     dc.UInt64,
     dc.String,
+    dc.Categorical,
 ]
 INTEGER_COLUMN_TYPES: list[type[dc.Column]] = [
     dc.Integer,