dyff-schema 0.30.0__tar.gz → 0.31.0__tar.gz

{dyff_schema-0.30.0 → dyff_schema-0.31.0}/.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
 
 include:
   - project: saferatday0/library/gitlab
-    ref: 0.7.0
+    ref: 0.16.0
     file:
       - gitlab-release.yml
   - project: saferatday0/library/detect-secrets
@@ -20,7 +20,7 @@ include:
     file:
       - prettier.yml
   - project: saferatday0/library/python
-    ref: 0.20.0
+    ref: 0.21.0
     file:
       - python-autoflake.yml
       - python-black.yml
@@ -37,6 +37,10 @@ include:
     file:
       - license-eye-header-check.yml
 
+# Override image to use Go 1.24 to avoid golang.org/x/tools bug
+license-eye-header-check:
+  image: "${CONTAINER_PROXY}golang:1.24-alpine"
+
 variables:
   PYTHON_PACKAGE: dyff/schema
   PYTHON_REQUIREMENTS: .
@@ -58,4 +62,4 @@ python-twine-upload-pypi:
 python-pytest:
   parallel:
     matrix:
-      - PYTHON_VERSION: ['3.9', '3.10', '3.11', '3.12']
+      - PYTHON_VERSION: ['3.10', '3.11', '3.12']

{dyff_schema-0.30.0/dyff_schema.egg-info → dyff_schema-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dyff-schema
-Version: 0.30.0
+Version: 0.31.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0
@@ -12,12 +12,11 @@ Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: NOTICE

dyff_schema-0.31.0/dyff/schema/_version.py

@@ -0,0 +1,2 @@
+__version__ = version = "0.31.0"
+__version_tuple__ = version_tuple = (0, 31, 0)

{dyff_schema-0.30.0 → dyff_schema-0.31.0}/dyff/schema/v0/r1/base.py

@@ -431,13 +431,11 @@ def list_(
     item_type: type[_ListElementT], *, list_size: Optional[int] = None
 ) -> type[list]:
     if list_size is None:
-        return Annotated[list[_ListElementT], Field()]  # type: ignore [return-value]
+        return Annotated[list[item_type], Field()]  # type: ignore [return-value, valid-type]
     else:
         if list_size <= 0:
             raise ValueError(f"list_size {list_size} must be > 0")
-        return Annotated[
-            list[_ListElementT], Field(min_length=list_size, max_length=list_size)
-        ]  # type: ignore [return-value]
+        return Annotated[list[item_type], Field(min_length=list_size, max_length=list_size)]  # type: ignore [return-value, valid-type]
 
 
 # mypy gets confused because 'dict' is the name of a method in DyffBaseModel

{dyff_schema-0.30.0 → dyff_schema-0.31.0}/dyff/schema/v0/r1/dataset/arrow.py

@@ -7,11 +7,13 @@ from __future__ import annotations
 import functools
 import inspect
 import typing
+import uuid
 from typing import Any, Iterable, Literal, Optional
 
 import pyarrow
 import pyarrow.dataset
 import pydantic
+from pydantic.fields import FieldInfo
 
 from ..base import DType
 from . import binary
@@ -90,9 +92,12 @@ def subset_schema(schema: pyarrow.Schema, field_names: list[str]) -> pyarrow.Sch
 
 
 def arrow_type(annotation: type) -> pyarrow.DataType:
-    """Determine a suitable arrow type for a pydantic model field."""
+    """Determine a suitable arrow type for a pydantic model field.
 
-    # Handle generic types first (List, Union, etc.)
+    Supports primitive types as well as pydantic sub-models, lists, and optional types.
+    Numeric types must have appropriate bounds specified, as Arrow cannot represent the
+    unbounded integer types used by Python 3.
+    """
     if origin := typing.get_origin(annotation):
         if origin == list:
             annotation_args = typing.get_args(annotation)
@@ -114,40 +119,61 @@ def arrow_type(annotation: type) -> pyarrow.DataType:
                 raise ValueError(
                     f"annotation {annotation}: only Optional[T] supported, not general Union"
                 )
-            return arrow_type(inner_type)
+            return arrow_type(inner_type)  # All Arrow types are nullable
 
         raise NotImplementedError(f"Python type {annotation}")
 
-    # Guard against non-types (TypeVars, etc.)
-    if not isinstance(annotation, type):
-        return pyarrow.string()
-
-    # Handle custom types
-    if issubclass(annotation, DType):
-        # The dtype is in the metaclass
-        return pyarrow.from_numpy_dtype(type(annotation).dtype)  # type: ignore[attr-defined]
+    if issubclass(annotation, pydantic.BaseModel):
+        subfields = []
+        for _name, subfield in annotation.model_fields.items():
+            subfields.append(arrow_field(_name, subfield))
+        return pyarrow.struct(subfields)
 
     # Handle numpy-like types
     if hasattr(annotation, "dtype"):
         return pyarrow.from_numpy_dtype(annotation.dtype)
 
-    # Handle pydantic models
-    if issubclass(annotation, pydantic.BaseModel):
-        subfields = []
-        for field_name, subfield in annotation.model_fields.items():
-            subfields.append(arrow_field(field_name, subfield))
-        return pyarrow.struct(subfields)
+    # Handle Annotated list types (e.g., Annotated[list[str], Field(max_length=10)])
+    # This covers lists created by our list_() function in base.py which returns
+    # Annotated types with Field metadata for length constraints.
+    #
+    # We need custom logic here because:
+    # 1. Standard typing.List doesn't carry Pydantic Field constraints
+    # 2. Our list_() function wraps list[T] in Annotated[list[T], Field(...)]
+    #    to embed validation metadata (min/max length) at the type level
+    # 3. PyArrow needs to know these constraints upfront to create proper schemas
+    # 4. The nested generic structure requires careful extraction:
+    #    Annotated[list[str], Field(max_length=10)] needs to become
+    #    pyarrow.list_(pyarrow.string(), 10)
+    if (
+        typing.get_origin(annotation) is typing.Annotated
+        and typing.get_args(annotation)[0] is list
+    ):
+        metadata = typing.get_args(annotation)[1:]
+        item_type = typing.get_args(typing.get_args(annotation)[0])[0]
+        max_length = -1
+        for meta in metadata:
+            if isinstance(meta, FieldInfo):
+                max_length = getattr(meta, "max_length", -1)
+        return pyarrow.list_(arrow_type(item_type), max_length)
 
-    # Handle built-in types
-    type_map = {
-        str: pyarrow.string(),
-        int: pyarrow.int64(),
-        float: pyarrow.float64(),
-        bool: pyarrow.bool_(),
-    }
+    if issubclass(annotation, DType):
+        # The dtype is in the metaclass
+        return pyarrow.from_numpy_dtype(type(annotation).dtype)  # type: ignore[attr-defined]
 
-    if annotation in type_map:
-        return type_map[annotation]
+    if annotation == bool:
+        return pyarrow.bool_()
+    if annotation == bytes:
+        return pyarrow.binary()
+    if annotation == float:
+        return pyarrow.float64()
+    if annotation == int:
+        raise ValueError("unconstrained integers cannot be represented in Arrow")
+    if annotation == uuid.UUID:
+        return pyarrow.binary(16)
+
+    if annotation == str:
+        return pyarrow.string()
 
     raise NotImplementedError(f"Python type {annotation}")
 

{dyff_schema-0.30.0 → dyff_schema-0.31.0}/dyff/schema/v0/r1/platform.py

@@ -1955,11 +1955,20 @@ class Method(DyffEntity, MethodBase):
         return None
 
 
+class EntityIDMarker:
+    """Marker class to indicate that a field contains an entity ID that supports
+    family@tag resolution."""
+
+
+# Type alias for entity ID fields that support family@tag resolution
+EntityIDField: TypeAlias = Annotated[str, EntityIDMarker]
+
+
 class AnalysisInput(DyffSchemaBaseModel):
     keyword: str = pydantic.Field(
         description="The 'keyword' specified for this input in the MethodSpec."
     )
-    entity: str = pydantic.Field(
+    entity: EntityIDField = pydantic.Field(
         description="The ID of the entity whose data should be made available as 'keyword'."
     )
 
@@ -2495,6 +2504,8 @@ __all__ = [
     "Entities",
     "EntityID",
     "EntityIdentifier",
+    "EntityIDField",
+    "EntityIDMarker",
     "EntityKindLiteral",
     "Evaluation",
     "EvaluationBase",

{dyff_schema-0.30.0 → dyff_schema-0.31.0}/dyff/schema/v0/r1/requests.py

@@ -30,6 +30,7 @@ from .platform import (
     DatasetBase,
     DataView,
     DocumentationBase,
+    EntityIDField,
     Evaluation,
     EvaluationBase,
     FamilyBase,
@@ -95,7 +96,7 @@ class AnalysisCreateRequest(DyffEntityCreateRequest, AnalysisBase):
     """An Analysis transforms Datasets, Evaluations, and Measurements into new
     Measurements or SafetyCases."""
 
-    method: str = pydantic.Field(description="Method ID")
+    method: EntityIDField = pydantic.Field(description="Method ID")
 
     @pydantic.field_validator("scope", check_fields=False)
     def _validate_scope(cls, scope: AnalysisScope) -> AnalysisScope:

{dyff_schema-0.30.0 → dyff_schema-0.31.0/dyff_schema.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dyff-schema
-Version: 0.30.0
+Version: 0.31.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0
@@ -12,12 +12,11 @@ Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: NOTICE

{dyff_schema-0.30.0 → dyff_schema-0.31.0}/pyproject.toml

@@ -23,7 +23,6 @@ classifiers=[
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
@@ -52,7 +51,7 @@ license = {"text" = "Apache-2.0"}
 
 readme = "README.md"
 
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 
 [project.urls]
 "Home" = "https://gitlab.com/dyff/packages/dyff-schema"

dyff_schema-0.30.0/dyff/schema/_version.py

@@ -1,2 +0,0 @@
-__version__ = version = "0.30.0"
-__version_tuple__ = version_tuple = (0, 30, 0)