dyff-schema 0.30.0__py3-none-any.whl → 0.30.1__py3-none-any.whl

dyff/schema/_version.py

@@ -1,2 +1,2 @@
-__version__ = version = "0.30.0"
-__version_tuple__ = version_tuple = (0, 30, 0)
+__version__ = version = "0.30.1"
+__version_tuple__ = version_tuple = (0, 30, 1)

dyff/schema/v0/r1/base.py

@@ -431,13 +431,11 @@ def list_(
     item_type: type[_ListElementT], *, list_size: Optional[int] = None
 ) -> type[list]:
     if list_size is None:
-        return Annotated[list[_ListElementT], Field()]  # type: ignore [return-value]
+        return Annotated[list[item_type], Field()]  # type: ignore [return-value, valid-type]
     else:
         if list_size <= 0:
             raise ValueError(f"list_size {list_size} must be > 0")
-        return Annotated[
-            list[_ListElementT], Field(min_length=list_size, max_length=list_size)
-        ]  # type: ignore [return-value]
+        return Annotated[list[item_type], Field(min_length=list_size, max_length=list_size)]  # type: ignore [return-value, valid-type]
 
 
 # mypy gets confused because 'dict' is the name of a method in DyffBaseModel

dyff/schema/v0/r1/dataset/arrow.py

@@ -7,11 +7,13 @@ from __future__ import annotations
 import functools
 import inspect
 import typing
+import uuid
 from typing import Any, Iterable, Literal, Optional
 
 import pyarrow
 import pyarrow.dataset
 import pydantic
+from pydantic.fields import FieldInfo
 
 from ..base import DType
 from . import binary
@@ -90,9 +92,12 @@ def subset_schema(schema: pyarrow.Schema, field_names: list[str]) -> pyarrow.Sch
 
 
 def arrow_type(annotation: type) -> pyarrow.DataType:
-    """Determine a suitable arrow type for a pydantic model field."""
+    """Determine a suitable arrow type for a pydantic model field.
 
-    # Handle generic types first (List, Union, etc.)
+    Supports primitive types as well as pydantic sub-models, lists, and optional types.
+    Numeric types must have appropriate bounds specified, as Arrow cannot represent the
+    unbounded integer types used by Python 3.
+    """
     if origin := typing.get_origin(annotation):
         if origin == list:
             annotation_args = typing.get_args(annotation)
@@ -114,40 +119,61 @@ def arrow_type(annotation: type) -> pyarrow.DataType:
                 raise ValueError(
                     f"annotation {annotation}: only Optional[T] supported, not general Union"
                 )
-            return arrow_type(inner_type)
+            return arrow_type(inner_type)  # All Arrow types are nullable
 
         raise NotImplementedError(f"Python type {annotation}")
 
-    # Guard against non-types (TypeVars, etc.)
-    if not isinstance(annotation, type):
-        return pyarrow.string()
-
-    # Handle custom types
-    if issubclass(annotation, DType):
-        # The dtype is in the metaclass
-        return pyarrow.from_numpy_dtype(type(annotation).dtype)  # type: ignore[attr-defined]
+    if issubclass(annotation, pydantic.BaseModel):
+        subfields = []
+        for _name, subfield in annotation.model_fields.items():
+            subfields.append(arrow_field(_name, subfield))
+        return pyarrow.struct(subfields)
 
     # Handle numpy-like types
     if hasattr(annotation, "dtype"):
         return pyarrow.from_numpy_dtype(annotation.dtype)
 
-    # Handle pydantic models
-    if issubclass(annotation, pydantic.BaseModel):
-        subfields = []
-        for field_name, subfield in annotation.model_fields.items():
-            subfields.append(arrow_field(field_name, subfield))
-        return pyarrow.struct(subfields)
+    # Handle Annotated list types (e.g., Annotated[list[str], Field(max_length=10)])
+    # This covers lists created by our list_() function in base.py which returns
+    # Annotated types with Field metadata for length constraints.
+    #
+    # We need custom logic here because:
+    # 1. Standard typing.List doesn't carry Pydantic Field constraints
+    # 2. Our list_() function wraps list[T] in Annotated[list[T], Field(...)]
+    #    to embed validation metadata (min/max length) at the type level
+    # 3. PyArrow needs to know these constraints upfront to create proper schemas
+    # 4. The nested generic structure requires careful extraction:
+    #    Annotated[list[str], Field(max_length=10)] needs to become
+    #    pyarrow.list_(pyarrow.string(), 10)
+    if (
+        typing.get_origin(annotation) is typing.Annotated
+        and typing.get_args(annotation)[0] is list
+    ):
+        metadata = typing.get_args(annotation)[1:]
+        item_type = typing.get_args(typing.get_args(annotation)[0])[0]
+        max_length = -1
+        for meta in metadata:
+            if isinstance(meta, FieldInfo):
+                max_length = getattr(meta, "max_length", -1)
+        return pyarrow.list_(arrow_type(item_type), max_length)
 
-    # Handle built-in types
-    type_map = {
-        str: pyarrow.string(),
-        int: pyarrow.int64(),
-        float: pyarrow.float64(),
-        bool: pyarrow.bool_(),
-    }
+    if issubclass(annotation, DType):
+        # The dtype is in the metaclass
+        return pyarrow.from_numpy_dtype(type(annotation).dtype)  # type: ignore[attr-defined]
 
-    if annotation in type_map:
-        return type_map[annotation]
+    if annotation == bool:
+        return pyarrow.bool_()
+    if annotation == bytes:
+        return pyarrow.binary()
+    if annotation == float:
+        return pyarrow.float64()
+    if annotation == int:
+        raise ValueError("unconstrained integers cannot be represented in Arrow")
+    if annotation == uuid.UUID:
+        return pyarrow.binary(16)
+
+    if annotation == str:
+        return pyarrow.string()
 
     raise NotImplementedError(f"Python type {annotation}")
 

{dyff_schema-0.30.0.dist-info → dyff_schema-0.30.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dyff-schema
-Version: 0.30.0
+Version: 0.30.1
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{dyff_schema-0.30.0.dist-info → dyff_schema-0.30.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dyff/schema/__init__.py,sha256=w7OWDFuyGKd6xt_yllNtKzHahPgywrfU4Ue02psYaMA,2244
-dyff/schema/_version.py,sha256=jaJNt0cIPw7pKpa1ZksSexFHwIRD7mvffKUdrE657pk,80
+dyff/schema/_version.py,sha256=b5YBuhpx4KBNLup_QY5e0xhsx1Q2LttYULeEoU-8uhY,80
 dyff/schema/adapters.py,sha256=YMTHv_2VlLGFp-Kqwa6H51hjffHmk8gXjZilHysIF5Q,123
 dyff/schema/annotations.py,sha256=nE6Jk1PLqlShj8uqjE_EzZC9zYnTDW5AVtQcjysiK8M,10018
 dyff/schema/base.py,sha256=jvaNtsSZyFfsdUZTcY_U-yfLY5_GyrMxSXhON2R9XR0,119
@@ -25,14 +25,14 @@ dyff/schema/io/vllm.py,sha256=2q05M_-lTzq9oywKXHPPpCFCSDVCSsRQqtmERzWTtio,123
 dyff/schema/v0/__init__.py,sha256=L5y8UhRnojerPYHumsxQJRcHCNz8Hj9NM8b47mewMNs,92
 dyff/schema/v0/r1/__init__.py,sha256=L5y8UhRnojerPYHumsxQJRcHCNz8Hj9NM8b47mewMNs,92
 dyff/schema/v0/r1/adapters.py,sha256=hpwCSW8lkMkUKCLe0zaMUDu-VS_caSxJvPsECEi_XRA,33069
-dyff/schema/v0/r1/base.py,sha256=WEy-xSL_FoC1gwAij1V_TEqPsITiIF_KlBDaDpPLWr8,20359
+dyff/schema/v0/r1/base.py,sha256=zaxU2fIu1Ca-nZsZwG0eb7COJmnPkDZ_yLrieHXqr0s,20353
 dyff/schema/v0/r1/commands.py,sha256=wDNMB8lry-H9G5hlT4m6y4fysoq4glY5qKsHytfehqU,9052
 dyff/schema/v0/r1/platform.py,sha256=Hp9hTaDqWCDsPGHouCmiCMLXByjW8UadXIlgRtb0vZc,82305
 dyff/schema/v0/r1/requests.py,sha256=VLdhyAKpdQytNdZZW60Y5Kg35GmB6BezDwDengl18Ws,17168
 dyff/schema/v0/r1/test.py,sha256=X6dUyVd5svcPCI-PBMOAqEfK9jv3bRDvkQTJzwS96c0,10720
 dyff/schema/v0/r1/version.py,sha256=NONebgcv5Thsw_ymud6PacZdGjV6ndBrmLnap-obcpo,428
 dyff/schema/v0/r1/dataset/__init__.py,sha256=LbVlkO2asyGYBKk2z49xjJYTM-pu9y9e4eQDXgTDLnM,2553
-dyff/schema/v0/r1/dataset/arrow.py,sha256=2XzJ1D2RqwfYRfLrJrsz-czOXZaPuGiWqXGfTGDBomY,12324
+dyff/schema/v0/r1/dataset/arrow.py,sha256=3AVKmxE-dFY7l-5A_riq0Uk4x_KshVF3V_uo2SNHuQs,13795
 dyff/schema/v0/r1/dataset/binary.py,sha256=KXvn79SUt3e_ZZXrju2atT_yMFwgAkCgDYXBtfv0E_I,636
 dyff/schema/v0/r1/dataset/classification.py,sha256=pbbEXhxyZ0pgYwzaTlM8hVHPNEJDCdHKOeGowPXgWYc,311
 dyff/schema/v0/r1/dataset/embedding.py,sha256=2cmcoV6AsvUBRrefmJ-_Vdfg9NVvdMr0-s7dqki15ls,792
@@ -40,9 +40,9 @@ dyff/schema/v0/r1/dataset/text.py,sha256=MYG5seGODDryRSCy-g0Unh5dD0HCytmZ3FeElC-
 dyff/schema/v0/r1/dataset/vision.py,sha256=aIe0fbfM_g3DsrDTdg2K803YKLjZBpurM_VJcJFuZLc,369
 dyff/schema/v0/r1/io/__init__.py,sha256=L5y8UhRnojerPYHumsxQJRcHCNz8Hj9NM8b47mewMNs,92
 dyff/schema/v0/r1/io/vllm.py,sha256=vWyLg-susbg0JDfv6VExBpgFdU2GHP2a14ChOdbckvs,5321
-dyff_schema-0.30.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dyff_schema-0.30.0.dist-info/licenses/NOTICE,sha256=YONACu0s_Ui6jNi-wtEsVQbTU1JIkh8wvLH6d1-Ni_w,43
-dyff_schema-0.30.0.dist-info/METADATA,sha256=g0DxR2SxqA7qTpENblTBKliqpHsnAqwcoJr7J78dAIg,3623
-dyff_schema-0.30.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dyff_schema-0.30.0.dist-info/top_level.txt,sha256=9e3VVdeX73t_sUJOPQPCcGtYO1JhoErhHIi3WoWGcFI,5
-dyff_schema-0.30.0.dist-info/RECORD,,
+dyff_schema-0.30.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dyff_schema-0.30.1.dist-info/licenses/NOTICE,sha256=YONACu0s_Ui6jNi-wtEsVQbTU1JIkh8wvLH6d1-Ni_w,43
+dyff_schema-0.30.1.dist-info/METADATA,sha256=q23-qxTt8hJXIjMnf3sM4VA4JgFhdyWkmPM_xeM5qgE,3623
+dyff_schema-0.30.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dyff_schema-0.30.1.dist-info/top_level.txt,sha256=9e3VVdeX73t_sUJOPQPCcGtYO1JhoErhHIi3WoWGcFI,5
+dyff_schema-0.30.1.dist-info/RECORD,,