dataframely 1.12.1__tar.gz → 1.14.0__tar.gz

{dataframely-1.12.1 → dataframely-1.14.0}/.github/workflows/build.yml

@@ -13,11 +13,11 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
+        uses: prefix-dev/setup-pixi@194d461b21b6c5717c722ffc597fa91ed2ff29fa # v0.9.1
         with:
           environments: build
       - name: Set version
@@ -48,17 +48,17 @@ jobs:
           - target-platform: win-64
             os: windows-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
+        uses: prefix-dev/setup-pixi@194d461b21b6c5717c722ffc597fa91ed2ff29fa # v0.9.1
         with:
           environments: build
       - name: Set version
         run: pixi run -e build set-version
       - name: Build wheel
-        uses: PyO3/maturin-action@e10f6c464b90acceb5f640d31beda6d586ba7b4a # v1.49.3
+        uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380 # v1.49.4
         with:
           command: build
           args: --out dist -i python3.10
@@ -80,9 +80,9 @@ jobs:
       id-token: write
     environment: pypi
     steps:
-      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+      - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           path: dist
           merge-multiple: true
       - name: Publish package on PyPi
-        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

{dataframely-1.12.1 → dataframely-1.14.0}/.github/workflows/chore.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Check valid conventional commit message
         id: lint
-        uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3
+        uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
         with:
           subjectPattern: ^[A-Z].+[^. ]$ # subject must start with uppercase letter and may not end with a dot/space
         env:

{dataframely-1.12.1 → dataframely-1.14.0}/.github/workflows/ci.yml

@@ -19,12 +19,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           # needed for 'pre-commit-mirrors-insert-license'
           fetch-depth: 0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
+        uses: prefix-dev/setup-pixi@194d461b21b6c5717c722ffc597fa91ed2ff29fa # v0.9.1
         with:
           environments: default lint
       - name: Install repository
@@ -51,9 +51,9 @@ jobs:
             with_optionals: true
     steps:
       - name: Checkout branch
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
+        uses: prefix-dev/setup-pixi@194d461b21b6c5717c722ffc597fa91ed2ff29fa # v0.9.1
         with:
           environments: ${{ matrix.environment }}
       - name: Install repository
@@ -61,7 +61,7 @@ jobs:
       - name: Run pytest
         run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes ${{ matrix.with_optionals && '-m with_optionals' || '-m "not with_optionals"'}} --cov=dataframely --cov-report=xml
       - name: Upload codecov
-        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
+        uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
         with:
           files: ./coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}

{dataframely-1.12.1 → dataframely-1.14.0}/.github/workflows/nightly.yml

@@ -23,9 +23,9 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout branch
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@8ca4608ef7f4daeb54f5205b20d0b7cb42f11143 # v0.8.14
+        uses: prefix-dev/setup-pixi@194d461b21b6c5717c722ffc597fa91ed2ff29fa # v0.9.1
         with:
           environments: nightly
       - name: Install polars nightly

{dataframely-1.12.1 → dataframely-1.14.0}/.github/workflows/scorecard.yml

@@ -35,12 +35,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
+        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
         with:
           results_file: results.sarif
           results_format: sarif
@@ -74,6 +74,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
+        uses: github/codeql-action/upload-sarif@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.29.5
         with:
           sarif_file: results.sarif

{dataframely-1.12.1 → dataframely-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 1.12.1
+Version: 1.14.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -12,9 +12,11 @@ Requires-Dist: typing-extensions ; python_full_version < '3.11'
 Requires-Dist: deltalake ; extra == 'deltalake'
 Requires-Dist: sqlalchemy ; extra == 'sqlalchemy'
 Requires-Dist: pyarrow ; extra == 'pyarrow'
+Requires-Dist: pydantic>=2 ; extra == 'pydantic'
 Provides-Extra: deltalake
 Provides-Extra: sqlalchemy
 Provides-Extra: pyarrow
+Provides-Extra: pydantic
 License-File: LICENSE
 Summary: A declarative, polars-native data frame validation library
 Author-email: Andreas Albert <andreas.albert@quantco.com>, Daniel Elsner <daniel.elsner@quantco.com>, Oliver Borchert <oliver.borchert@quantco.com>

{dataframely-1.12.1 → dataframely-1.14.0}/dataframely/_base_schema.py

@@ -5,10 +5,10 @@ from __future__ import annotations
 
 import sys
 import textwrap
-from abc import ABCMeta
+from abc import ABCMeta, abstractmethod
 from copy import copy
 from dataclasses import dataclass, field
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import polars as pl
 
@@ -21,6 +21,10 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
+
+if TYPE_CHECKING:
+    from ._typing import DataFrame
+
 _COLUMN_ATTR = "__dataframely_columns__"
 _RULE_ATTR = "__dataframely_rules__"
 
@@ -133,6 +137,29 @@ class SchemaMeta(ABCMeta):
                         f"which are not in the schema: {missing_list}."
                     )
 
+        # 3) Check that all members are non-pathological (i.e., user errors).
+        for attr, value in namespace.items():
+            if attr.startswith("__"):
+                continue
+            # Check for tuple of column (commonly caused by trailing comma)
+            if (
+                isinstance(value, tuple)
+                and len(value) > 0
+                and isinstance(value[0], Column)
+            ):
+                raise TypeError(
+                    f"Column '{attr}' is defined as a tuple of dy.Column. "
+                    f"Did you accidentally add a trailing comma?"
+                )
+
+            # Check for column type instead of instance (e.g., dy.Float64 instead of dy.Float64())
+            if isinstance(value, type) and issubclass(value, Column):
+                raise TypeError(
+                    f"Column '{attr}' is a type, not an instance. "
+                    f"Schema members must be of type Column not type[Column]. "
+                    f"Did you forget to add parentheses?"
+                )
+
         return super().__new__(mcs, name, bases, namespace, *args, **kwargs)
 
     def __getattribute__(cls, name: str) -> Any:
@@ -198,11 +225,46 @@ class BaseSchema(metaclass=SchemaMeta):
             columns[name]._name = name
         return columns
 
+    @classmethod
+    @abstractmethod
+    def polars_schema(cls) -> pl.Schema:
+        """Obtain the polars schema for this schema.
+
+        Returns:
+            A :mod:`polars` schema that mirrors the schema defined by this class.
+        """
+
     @classmethod
     def primary_keys(cls) -> list[str]:
         """The primary key columns in this schema (possibly empty)."""
         return _primary_keys(cls.columns())
 
+    @classmethod
+    @abstractmethod
+    def validate(
+        cls, df: pl.DataFrame | pl.LazyFrame, /, *, cast: bool = False
+    ) -> DataFrame[Self]:
+        """Validate that a data frame satisfies the schema.
+
+        Args:
+            df: The data frame to validate.
+            cast: Whether columns with a wrong data type in the input data frame are
+                cast to the schema's defined data type if possible.
+
+        Returns:
+            The (collected) input data frame, wrapped in a generic version of the
+            input's data frame type to reflect schema adherence. The data frame is
+            guaranteed to maintain its order.
+
+        Raises:
+            ValidationError: If the input data frame does not satisfy the schema
+                definition.
+
+        Note:
+            This method _always_ collects the input data frame in order to raise
+            potential validation errors.
+        """
+
     @classmethod
     def _validation_rules(cls, *, with_cast: bool) -> dict[str, Rule]:
         return _build_rules(

{dataframely-1.12.1 → dataframely-1.14.0}/dataframely/_compat.py

@@ -54,6 +54,19 @@ try:
 except ImportError:  # pragma: no cover
     pa = _DummyModule("pyarrow")
 
+
+# -------------------------------------- PYDANTIC ------------------------------------ #
+
+try:
+    import pydantic
+except ImportError:  # pragma: no cover
+    pydantic = _DummyModule("pydantic")  # type: ignore
+
+try:
+    from pydantic_core import core_schema as pydantic_core_schema  # pragma: no cover
+except ImportError:
+    pydantic_core_schema = _DummyModule("pydantic_core_schema")  # type: ignore
+
 # ------------------------------------------------------------------------------------ #
 
 __all__ = [
@@ -64,4 +77,6 @@ __all__ = [
     "pa",
     "MSDialect_pyodbc",
     "PGDialect_psycopg2",
+    "pydantic",
+    "pydantic_core_schema",
 ]

dataframely-1.14.0/dataframely/_pydantic.py

@@ -0,0 +1,115 @@
+# Copyright (c) QuantCo 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+from __future__ import annotations
+
+from functools import partial
+from typing import TYPE_CHECKING, Literal, TypeVar, get_args, get_origin, overload
+
+import polars as pl
+
+from ._base_schema import BaseSchema
+from ._compat import pydantic, pydantic_core_schema
+from .exc import ValidationError
+
+if TYPE_CHECKING:
+    from ._typing import DataFrame, LazyFrame
+
+
+_S = TypeVar("_S", bound=BaseSchema)
+
+
+def _dict_to_df(schema_type: type[BaseSchema], data: dict) -> pl.DataFrame:
+    return pl.from_dict(
+        data,
+        schema=schema_type.polars_schema(),
+    )
+
+
+def _validate_df_schema(schema_type: type[_S], df: pl.DataFrame) -> DataFrame[_S]:
+    try:
+        return schema_type.validate(df, cast=False)
+    except ValidationError as e:
+        raise ValueError("DataFrame violates schema") from e
+
+
+def _serialize_df(df: pl.DataFrame) -> dict:
+    return df.to_dict(as_series=False)
+
+
+@overload
+def get_pydantic_core_schema(
+    source_type: type[DataFrame],
+    _handler: pydantic.GetCoreSchemaHandler,
+    lazy: Literal[False],
+) -> pydantic_core_schema.CoreSchema: ...
+
+
+@overload
+def get_pydantic_core_schema(
+    source_type: type[LazyFrame],
+    _handler: pydantic.GetCoreSchemaHandler,
+    lazy: Literal[True],
+) -> pydantic_core_schema.CoreSchema: ...
+
+
+def get_pydantic_core_schema(
+    source_type: type[DataFrame | LazyFrame],
+    _handler: pydantic.GetCoreSchemaHandler,
+    lazy: bool,
+) -> pydantic_core_schema.CoreSchema:
+    # https://docs.pydantic.dev/2.11/concepts/types/#handling-custom-generic-classes
+    origin = get_origin(source_type)
+    if origin is None:
+        # used as `x: dy.DataFrame` without schema
+        raise TypeError("DataFrame must be parametrized with a schema")
+
+    schema_type: type[BaseSchema] = get_args(source_type)[0]
+
+    # accept a DataFrame, a LazyFrame, or a dict that is converted to a DataFrame
+    # (-> output: DataFrame or LazyFrame)
+    polars_schema = pydantic_core_schema.union_schema(
+        [
+            pydantic_core_schema.is_instance_schema(pl.DataFrame),
+            pydantic_core_schema.is_instance_schema(pl.LazyFrame),
+            pydantic_core_schema.chain_schema(
+                [
+                    pydantic_core_schema.dict_schema(),
+                    pydantic_core_schema.no_info_plain_validator_function(
+                        partial(_dict_to_df, schema_type)
+                    ),
+                ]
+            ),
+        ]
+    )
+
+    to_lazy_schema = []
+    if lazy:
+        # If the Pydantic field type is LazyFrame, add a step to convert
+        # the model back to a LazyFrame.
+        to_lazy_schema.append(
+            pydantic_core_schema.no_info_plain_validator_function(
+                lambda df: df.lazy(),
+            )
+        )
+
+    return pydantic_core_schema.chain_schema(
+        [
+            polars_schema,
+            pydantic_core_schema.no_info_plain_validator_function(
+                partial(_validate_df_schema, schema_type)
+            ),
+            *to_lazy_schema,
+        ],
+        serialization=pydantic_core_schema.plain_serializer_function_ser_schema(
+            _serialize_df
+        ),
+    )
+
+
+def get_pydantic_json_schema(
+    handler: pydantic.GetJsonSchemaHandler,
+) -> pydantic.json_schema.JsonSchemaValue:
+    from pydantic_core import core_schema
+
+    # This could be made more sophisticated by actually reflecting the schema.
+    return handler(core_schema.dict_schema())

{dataframely-1.12.1 → dataframely-1.14.0}/dataframely/_typing.py

@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING, Any, Concatenate, Generic, Literal, ParamSpec,
 import polars as pl
 
 from ._base_schema import BaseSchema
+from ._compat import pydantic, pydantic_core_schema
+from ._pydantic import get_pydantic_core_schema, get_pydantic_json_schema
 
 S = TypeVar("S", bound=BaseSchema, covariant=True)
 
@@ -70,6 +72,20 @@ class DataFrame(pl.DataFrame, Generic[S]):
     def shrink_to_fit(self, *args: Any, **kwargs: Any) -> DataFrame[S]:
         raise NotImplementedError  # pragma: no cover
 
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: pydantic.GetCoreSchemaHandler
+    ) -> pydantic_core_schema.CoreSchema:
+        return get_pydantic_core_schema(source_type, handler, lazy=False)
+
+    @classmethod
+    def __get_pydantic_json_schema__(
+        cls,
+        _core_schema: pydantic_core_schema.CoreSchema,
+        handler: pydantic.GetJsonSchemaHandler,
+    ) -> pydantic.json_schema.JsonSchemaValue:
+        return get_pydantic_json_schema(handler)
+
 
 class LazyFrame(pl.LazyFrame, Generic[S]):
     """Generic wrapper around a :class:`polars.LazyFrame` to attach schema information.
@@ -113,3 +129,17 @@ class LazyFrame(pl.LazyFrame, Generic[S]):
     @inherit_signature(pl.LazyFrame.set_sorted)
     def set_sorted(self, *args: Any, **kwargs: Any) -> LazyFrame[S]:
         raise NotImplementedError  # pragma: no cover
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: pydantic.GetCoreSchemaHandler
+    ) -> pydantic_core_schema.CoreSchema:
+        return get_pydantic_core_schema(source_type, handler, lazy=True)
+
+    @classmethod
+    def __get_pydantic_json_schema__(
+        cls,
+        _core_schema: pydantic_core_schema.CoreSchema,
+        handler: pydantic.GetJsonSchemaHandler,
+    ) -> pydantic.json_schema.JsonSchemaValue:
+        return get_pydantic_json_schema(handler)

{dataframely-1.12.1 → dataframely-1.14.0}/dataframely/columns/_mixins.py

@@ -76,7 +76,7 @@ class OrdinalMixin(Generic[T], Base):
             result["min_exclusive"] = expr > self.min_exclusive  # type: ignore
         if self.max is not None:
             result["max"] = expr <= self.max  # type: ignore
-        if self.max_exclusive:
+        if self.max_exclusive is not None:
             result["max_exclusive"] = expr < self.max_exclusive  # type: ignore
         return result
 

{dataframely-1.12.1 → dataframely-1.14.0}/dataframely/schema.py

@@ -414,26 +414,6 @@ class Schema(BaseSchema, ABC):
     def validate(
         cls, df: pl.DataFrame | pl.LazyFrame, /, *, cast: bool = False
     ) -> DataFrame[Self]:
-        """Validate that a data frame satisfies the schema.
-
-        Args:
-            df: The data frame to validate.
-            cast: Whether columns with a wrong data type in the input data frame are
-                cast to the schema's defined data type if possible.
-
-        Returns:
-            The (collected) input data frame, wrapped in a generic version of the
-            input's data frame type to reflect schema adherence. The data frame is
-            guaranteed to maintain its order.
-
-        Raises:
-            ValidationError: If the input data frame does not satisfy the schema
-                definition.
-
-        Note:
-            This method _always_ collects the input data frame in order to raise
-            potential validation errors.
-        """
         # We can dispatch to the `filter` method and raise an error if any row cannot
         # be validated
         df_valid, failures = cls.filter(df, cast=cast)
@@ -1118,11 +1098,6 @@ class Schema(BaseSchema, ABC):
 
     @classmethod
     def polars_schema(cls) -> pl.Schema:
-        """Obtain the polars schema for this schema.
-
-        Returns:
-            A :mod:`polars` schema that mirrors the schema defined by this class.
-        """
         return pl.Schema({name: col.dtype for name, col in cls.columns().items()})
 
     @classmethod

{dataframely-1.12.1 → dataframely-1.14.0}/docs/_api/dataframely.columns.rst

@@ -25,6 +25,14 @@ dataframely.columns.array module
    :show-inheritance:
    :undoc-members:
 
+dataframely.columns.binary module
+---------------------------------
+
+.. automodule:: dataframely.columns.binary
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
 dataframely.columns.bool module
 -------------------------------
 
@@ -33,6 +41,14 @@ dataframely.columns.bool module
    :show-inheritance:
    :undoc-members:
 
+dataframely.columns.categorical module
+--------------------------------------
+
+.. automodule:: dataframely.columns.categorical
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
 dataframely.columns.datetime module
 -----------------------------------
 

{dataframely-1.12.1 → dataframely-1.14.0}/docs/_api/dataframely.testing.rst

@@ -41,6 +41,14 @@ dataframely.testing.rules module
    :show-inheritance:
    :undoc-members:
 
+dataframely.testing.storage module
+----------------------------------
+
+.. automodule:: dataframely.testing.storage
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
 dataframely.testing.typing module
 ---------------------------------
 

{dataframely-1.12.1 → dataframely-1.14.0}/docs/index.rst

@@ -22,11 +22,12 @@ Contents
 
 .. toctree::
     :caption: Contents
-    :maxdepth: 1
+    :maxdepth: 2
 
     Installation <sites/installation.rst>
     Quickstart <sites/quickstart.rst>
     Real-world Example <sites/examples/real-world.ipynb>
+    Features <sites/features/index.rst>
     FAQ <sites/faq.rst>
     Development Guide <sites/development.rst>
     Versioning <sites/versioning.rst>

dataframely-1.14.0/docs/sites/faq.rst

@@ -0,0 +1,30 @@
+FAQ
+===
+
+Whenever you find out something that you were surprised by or needed some non-trivial
+thinking, please add it here.
+
+How do I define additional unique keys in a ``dy.Schema``?
+----------------------------------------------------------
+
+By default, ``dataframely`` only supports defining a single non-nullable (composite) primary key in ``dy.Schema``.
+However, in some scenarios it may be useful to define additional unique keys (which support nullable fields and/or which are additionally unique).
+
+Consider the following example, which demonstrates two rules: one for validating that a field is entirely unique, and another for validating that a field, when provided, is unique.
+
+::
+
+    class UserSchema(dy.Schema):
+        user_id = dy.UInt64(primary_key=True, nullable=False)
+        username = dy.String(nullable=False)
+        email = dy.String(nullable=True)  # Must be unique, or null.
+
+        @dy.rule(group_by=["username"])
+        def unique_username() -> pl.Expr:
+            """Username, a non-nullable field, must be total unique."""
+            return pl.len() == 1
+
+        @dy.rule()
+        def unique_email_or_null() -> pl.Expr:
+            """Email must be unique, if provided."""
+            return pl.col("email").is_null() | pl.col("email").is_unique()

dataframely-1.14.0/docs/sites/features/index.rst

@@ -0,0 +1,7 @@
+Features
+========
+
+.. toctree::
+    :maxdepth: 1
+
+   primary-keys.rst

dataframely-1.14.0/docs/sites/features/primary-keys.rst

@@ -0,0 +1,47 @@
+Primary keys
+============
+
+Defining primary keys in ``dy.Schema``
+--------------------------------------
+
+When working with tabular data, it is often useful to define a `primary key <https://en.wikipedia.org/wiki/Primary_key>`_. A primary key is a set of one or multiple column, the combined values of which form a unique identifier for every record in a table.
+
+Dataframely supports marking columns as part of the primary key when defining a ``dy.Schema`` by setting ``primary_key=True`` on the respective column(s).
+
+.. note::
+
+  Primary key columns must not be nullable.
+
+Single primary keys
+^^^^^^^^^^^^^^^^^^^
+
+For example, when managing data about users, we might use an ``id`` column to uniquely identify users:
+
+::
+
+    class UserSchema(dy.Schema):
+      name = dy.String(primary_key=True)
+      name = dy.String()
+
+When we later validate data with this schema, ``dataframely`` checks that the values of the primary key are unique, i.e. there are no two users with the same value of ``id``. Having multiple users with the same ``name`` but different ``id`` but be allowed in this case.
+
+Composite primary keys
+^^^^^^^^^^^^^^^^^^^^^^
+
+In another scenario, we might be tracking line items on invoices. We have many invoices, and each invoice may contain any number of line items. To uniquely identify a line item, we need to specify the invoice, as well as the line items position within the invoice. To encode this, we set ``primary_key=True`` on both the ``invoice_id`` and ``item_id`` columns:
+
+::
+
+    class LineItemSchema(dy.Schema):
+      invoice_id = dy.Int64(primary_key=True)
+      item_id = dy.Int64(primary_key=True)
+      price = dy.Decimal()
+
+Validation will now ensure that all pairs of (``invoice_id``, ``item_id``) are unique.
+
+
+Primary keys in ``dy.Collection``
+---------------------------------
+
+The central idea behind ``dy.Collection`` is to unify multiple tables relating to the same set of underlying entities.
+This is useful because it allows us to write ``dy.filter``s that use information from multiple tables to identify whether the underlying entity is valid or not. If any ``dy.filter``s are defined, ``dataframely`` requires the tables in a ``dy.Collection`` to have an overlapping primary key (i.e., there must be at least one column that is a primary key in all tables).

{dataframely-1.12.1 → dataframely-1.14.0}/docs/sites/quickstart.rst

@@ -253,6 +253,7 @@ Lastly, ``dataframely`` schemas can be used to integrate with external tools:
 - ``HouseSchema.create_empty()`` creates an empty ``dy.DataFrame[HouseSchema]`` that can be used for testing
 - ``HouseSchema.sql_schema()`` provides a list of `sqlalchemy <https://www.sqlalchemy.org>`_ columns that can be used to create SQL tables using types and constraints in line with the schema
 - ``HouseSchema.pyarrow_schema()`` provides a `pyarrow <https://arrow.apache.org/docs/python/index.html>`_ schema with appropriate column dtypes and nullability information
+- You can use ``dy.DataFrame[HouseSchema]`` (or the ``LazyFrame`` equivalent) as fields in `pydantic <https://pydantic.dev>`_ models, including support for validation and serialization. Integration with pydantic is unstable.
 
 
 Outlook