dyff-schema 0.17.0__tar.gz → 0.19.0__tar.gz

{dyff_schema-0.17.0 → dyff_schema-0.19.0}/.gitlab-ci.yml

@@ -20,7 +20,7 @@ include:
     file:
       - prettier.yml
   - project: buildgarden/pipelines/python
-    ref: 0.14.0
+    ref: 0.14.1
     file:
       - python-autoflake.yml
       - python-black.yml

{dyff_schema-0.17.0/dyff_schema.egg-info → dyff_schema-0.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dyff-schema
-Version: 0.17.0
+Version: 0.19.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

dyff_schema-0.19.0/dyff/schema/dataset/embedding.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2024 UL Research Institutes
+# SPDX-License-Identifier: Apache-2.0
+
+from ..v0.r1.dataset.embedding import *

{dyff_schema-0.17.0 → dyff_schema-0.19.0}/dyff/schema/v0/r1/base.py

@@ -547,8 +547,11 @@ def uint64(
     return type("UInt64Value", (UInt64,), namespace)
 
 
+_ListElementT = TypeVar("_ListElementT")
+
+
 def list_(
-    item_type: Type[pydantic.BaseModel], *, list_size: Optional[int] = None
+    item_type: Type[_ListElementT], *, list_size: Optional[int] = None
 ) -> Type[list]:
     if list_size is None:
         return pydantic.conlist(item_type)

{dyff_schema-0.17.0 → dyff_schema-0.19.0}/dyff/schema/v0/r1/dataset/arrow.py

@@ -8,7 +8,7 @@ import functools
 import inspect
 import typing
 import uuid
-from typing import Any, Iterable, Optional
+from typing import Any, Iterable, Literal, Optional
 
 import pyarrow
 import pyarrow.dataset
@@ -51,7 +51,10 @@ def make_response_schema(schema: pyarrow.Schema) -> pyarrow.Schema:
     """Given an Arrow schema, create a new one that has the extra ``ResponseItem``
     fields added."""
     response_item_schema = make_response_item_schema(schema)
-    fields = list(zip(response_item_schema.names, response_item_schema.types))
+    fields = [
+        pyarrow.field(n, t)
+        for n, t in zip(response_item_schema.names, response_item_schema.types)
+    ]
     item_type = pyarrow.struct(fields)
     responses_type = pyarrow.list_(item_type)
     return pyarrow.schema(
@@ -65,7 +68,8 @@ def make_response_schema(schema: pyarrow.Schema) -> pyarrow.Schema:
 
 def encode_schema(schema: pyarrow.Schema) -> str:
     """Encode an Arrow schema as a string."""
-    return binary.encode(schema.serialize())
+    # pyarrow.Buffer doesn't satisfy ReadableBuffer but it still works
+    return binary.encode(schema.serialize())  # type: ignore[arg-type]
 
 
 def decode_schema(schema: str) -> pyarrow.Schema:
@@ -84,7 +88,7 @@ def subset_schema(schema: pyarrow.Schema, field_names: list[str]) -> pyarrow.Sch
     return pyarrow.schema(fields)
 
 
-def arrow_type(annotation: type):
+def arrow_type(annotation: type) -> pyarrow.DataType:
     """Determine a suitable arrow type for a pydantic model field.
 
     Supports primitive types as well as pydantic sub-models, lists, and optional types.
@@ -130,8 +134,7 @@ def arrow_type(annotation: type):
 
     if issubclass(annotation, DType):
         # The dtype is in the metaclass
-        return type(annotation).dtype  # type: ignore
-        # return pyarrow.from_numpy_dtype(type(annotation).dtype)  # type: ignore
+        return pyarrow.from_numpy_dtype(type(annotation).dtype)  # type: ignore[attr-defined]
 
     if annotation == bool:
         return pyarrow.bool_()
@@ -246,6 +249,7 @@ def _construct_field_docs(
         if pyarrow.types.is_struct(field.type):
             children = [field.type.field(i) for i in range(field.type.num_fields)]
         elif pyarrow.types.is_list(field.type):
+            assert isinstance(field.type, pyarrow.ListType)
             children = [field.type.value_field]
         else:
             raise ValueError(f"Unsupported nested type {field.type}")
@@ -275,8 +279,10 @@ def write_dataset(
     *,
     output_path: str,
     feature_schema: pyarrow.Schema,
-    partition_schema: pyarrow.Schema = None,
-    existing_data_behavior: str = "overwrite_or_ignore",
+    partition_schema: Optional[pyarrow.Schema] = None,
+    existing_data_behavior: Literal[
+        "error", "overwrite_or_ignore", "delete_matching"
+    ] = "overwrite_or_ignore",
     **kwargs,
 ):
     """Creates a ``pyarrow.dataset.Dataset`` from a data generator.
@@ -291,15 +297,19 @@ def write_dataset(
       existing_data_behavior: Same as ``pyarrow.dataset.write_dataset``, but
         defaults to ``"overwrite_or_ignore"``, which is typically what we want.
     """
-    partitioning = partition_schema and pyarrow.dataset.partitioning(
-        partition_schema, flavor="hive"
+    partitioning = (
+        pyarrow.dataset.partitioning(partition_schema, flavor="hive")
+        if partition_schema is not None
+        else None
     )
     pyarrow.dataset.write_dataset(
         data_generator,
         output_path,
         format="parquet",
         schema=feature_schema,
-        partitioning=partitioning,
+        # Type annotation doesn't include PartitioningFactory even though
+        # you're clearly meant to pass the output of partitioning() here
+        partitioning=partitioning,  # type: ignore[arg-type]
         existing_data_behavior=existing_data_behavior,
         **kwargs,
     )
@@ -326,10 +336,10 @@ def batches(
     for instance in instances:
         batch.append(instance)
         if len(batch) == batch_size:
-            yield pyarrow.RecordBatch.from_pylist(batch, schema=schema)
+            yield pyarrow.RecordBatch.from_pylist(batch, schema=schema)  # type: ignore[attr-defined]
             batch = []
     if batch:  # Final (incomplete) batch
-        yield pyarrow.RecordBatch.from_pylist(batch, schema=schema)
+        yield pyarrow.RecordBatch.from_pylist(batch, schema=schema)  # type: ignore[attr-defined]
 
 
 __all__ = [

{dyff_schema-0.17.0 → dyff_schema-0.19.0}/dyff/schema/v0/r1/dataset/binary.py

@@ -3,9 +3,13 @@
 
 import base64
 import hashlib
+import typing
 
+if typing.TYPE_CHECKING:
+    from _typeshed import ReadableBuffer
 
-def encode(data: bytes) -> str:
+
+def encode(data: "ReadableBuffer") -> str:
     return base64.b64encode(data).decode("utf-8")
 
 

dyff_schema-0.19.0/dyff/schema/v0/r1/dataset/embedding.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2024 UL Research Institutes
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Type
+
+import pydantic
+
+from ..base import DyffSchemaBaseModel, FixedWidthFloat, list_
+
+
+def embedding(
+    element_type: Type[FixedWidthFloat], size: int
+) -> Type[DyffSchemaBaseModel]:
+    """Returns a schema type representing a list of fixed-length embedding vectors."""
+
+    class _Embedding(DyffSchemaBaseModel):
+        embedding: list_(element_type, list_size=size) = pydantic.Field(  # type: ignore[valid-type]
+            description="An embedding vector"
+        )
+
+    return _Embedding
+
+
+__all__ = [
+    "embedding",
+]

{dyff_schema-0.17.0 → dyff_schema-0.19.0}/dyff/schema/v0/r1/platform.py

@@ -23,7 +23,6 @@ import abc
 import enum
 import urllib.parse
 from datetime import datetime
-from decimal import Decimal
 from enum import Enum
 from typing import Any, Literal, NamedTuple, Optional, Type, Union
 
@@ -160,6 +159,34 @@ def _oci_image_tag_maxlen():
     return 127
 
 
+def identifier_regex():
+    """Python identifiers start with a letter or an underscore, and consist of letters,
+    numbers, and underscores."""
+    return r"^[a-zA-Z_][a-zA-Z0-9_]*$"
+
+
+def identifier_maxlen():
+    """There isn't really a max length for Python identifiers, but this seems like a
+    reasonable limit for our use."""
+    return 127
+
+
+def title_maxlen() -> int:
+    return 140
+
+
+def summary_maxlen() -> int:
+    return 280
+
+
+def entity_id_regex() -> str:
+    """An entity ID is a 32-character HEX string.
+
+    TODO: This doesn't check whether the hex string is a valid UUID.
+    """
+    return r"^[a-f0-9]{32}$"
+
+
 class Entities(str, enum.Enum):
     """The kinds of entities in the dyff system."""
 
@@ -226,6 +253,9 @@ class Resources(str, enum.Enum):
             raise ValueError(f"No Resources for Entity kind: {kind}")
 
 
+EntityID: TypeAlias = pydantic.constr(regex=entity_id_regex())  # type: ignore
+
+
 class DyffModelWithID(DyffSchemaBaseModel):
     id: str = pydantic.Field(description="Unique identifier of the entity")
     account: str = pydantic.Field(description="Account that owns the entity")
@@ -399,6 +429,10 @@ class DyffEntity(Status, Labeled, SchemaVersion, DyffModelWithID):
         default=None, description="Resource creation time (assigned by system)"
     )
 
+    lastTransitionTime: Optional[datetime] = pydantic.Field(
+        default=None, description="Time of last (status, reason) change."
+    )
+
     @abc.abstractmethod
     def dependencies(self) -> list[str]:
         """List of IDs of resources that this resource depends on.
@@ -824,7 +858,7 @@ class DataSchema(DyffSchemaBaseModel):
     def make_output_schema(
         schema: Union[pyarrow.Schema, Type[DyffSchemaBaseModel], DyffDataSchema],
     ) -> "DataSchema":
-        """Construct a complete ``DataSchema`` for inference inputs.
+        """Construct a complete ``DataSchema`` for inference outputs.
 
         This function will add required special fields for input data and then
         convert the augmented schema as necessary to populate at least the
@@ -1559,42 +1593,51 @@ class ScoreSpec(DyffSchemaBaseModel):
     name: str = pydantic.Field(
         description="The name of the score. Used as a key for retrieving score data."
         " Must be unique within the Method context.",
+        regex=identifier_regex(),
+        max_length=identifier_maxlen(),
     )
 
     title: str = pydantic.Field(
         description="The title text to use when displaying score information.",
-        max_length=140,
+        max_length=title_maxlen(),
     )
 
     summary: str = pydantic.Field(
         description="A short text description of what the score measures.",
-        max_length=140,
+        max_length=summary_maxlen(),
     )
 
     valence: Literal["positive", "negative", "neutral"] = pydantic.Field(
+        default="neutral",
         description="A score has 'positive' valence if 'more is better',"
         " 'negative' valence if 'less is better', and 'neutral' valence if"
-        " 'better' is not meaningful for this score."
+        " 'better' is not meaningful for this score.",
     )
 
     priority: Literal["primary", "secondary"] = pydantic.Field(
+        default="primary",
         description="The 'primary' score will be displayed in any UI widgets"
-        " that expect a single score."
+        " that expect a single score. There must be exactly 1 primary score.",
     )
 
-    minimum: Optional[Decimal] = pydantic.Field(
+    minimum: Optional[float] = pydantic.Field(
         default=None, description="The minimum possible value, if known."
     )
 
-    maximum: Optional[Decimal] = pydantic.Field(
+    maximum: Optional[float] = pydantic.Field(
         default=None, description="The maximum possible value, if known."
     )
 
     format: str = pydantic.Field(
-        default="{quantity}",
+        default="{quantity:.1f}",
+        # Must use the 'quantity' key in the format string:
+        # (Maybe string not ending in '}')(something like '{quantity:f}')(maybe another string)
+        regex=r"^(.*[^{])?[{]quantity(:[^}]*)?[}]([^}].*)?$",
         description="A Python 'format' string describing how to render the score"
-        " as a string. You can use the keys 'quantity' and 'unit' in the format"
-        " string (e.g., '{quantity} {unit}').",
+        " as a string. You *must* use the keyword 'quantity' in the format"
+        " string, and you may use 'unit' as well (e.g., '{quantity:.2f} {unit}')."
+        " It is *strongly recommended* that you limit the output precision"
+        " appropriately; use ':.0f' for integer-valued scores.",
     )
 
     unit: Optional[str] = pydantic.Field(
@@ -1603,6 +1646,11 @@ class ScoreSpec(DyffSchemaBaseModel):
         " Use standard SI abbreviations where possible for better indexing.",
     )
 
+    def quantity_string(self, quantity: float) -> str:
+        """Formats the given quantity as a string, according to the formatting
+        information stored in this ScoreSpec."""
+        return self.format_quantity(self.format, quantity, unit=self.unit)
+
     @pydantic.root_validator
     def _validate_minimum_maximum(cls, values):
         minimum = values.get("minimum")
@@ -1613,15 +1661,17 @@ class ScoreSpec(DyffSchemaBaseModel):
 
     @pydantic.validator("format")
     def _validate_format(cls, v):
-        try:
-            cls._format_quantity(v, Decimal("3.14"), unit="kg")
-        except Exception:
-            raise ValueError(f"invalid format: '{v}'")
+        x = cls.format_quantity(v, 3.14, unit="kg")
+        y = cls.format_quantity(v, -2.03, unit="kg")
+        if x == y:
+            # Formatted results for different quantities should be different
+            raise ValueError("format string does not mention 'quantity'")
+
         return v
 
     @classmethod
-    def _format_quantity(
-        cls, format: str, quantity: Decimal, *, unit: Optional[str] = None
+    def format_quantity(
+        cls, format: str, quantity: float, *, unit: Optional[str] = None
     ) -> str:
         return format.format(quantity=quantity, unit=unit)
 
@@ -1824,19 +1874,20 @@ class ScoreData(ScoreSpec):
         description="The Analysis that generated the current score instance."
     )
 
-    quantity: Decimal = pydantic.Field(
+    quantity: float = pydantic.Field(
         description="The numeric quantity associated with the score."
-        " Callers should set the desired precision explicitly."
+    )
+
+    quantityString: str = pydantic.Field(
+        description="The formatted string representation of .quantity,"
+        " after processing with the .format specification."
     )
 
     text: str = pydantic.Field(
         description="A short text description of what the quantity means.",
-        max_length=140,
+        max_length=summary_maxlen(),
     )
 
-    def format_quantity(self) -> str:
-        return self._format_quantity(self.format, self.quantity, unit=self.unit)
-
 
 class Score(ScoreData):
     """A Score is a numeric quantity describing an aspect of system performance.
@@ -2160,6 +2211,7 @@ DyffEntityType = Union[DyffEntityTypeExceptRevision, Revision]
 
 
 __all__ = [
+    "SYSTEM_ATTRIBUTES",
     "Accelerator",
     "AcceleratorGPU",
     "AccessGrant",
@@ -2193,9 +2245,11 @@ __all__ = [
     "DocumentationBase",
     "DyffDataSchema",
     "DyffEntity",
+    "DyffEntityType",
     "DyffModelWithID",
     "DyffSchemaBaseModel",
     "Entities",
+    "EntityID",
     "Evaluation",
     "EvaluationBase",
     "ExtractorStep",
@@ -2295,9 +2349,11 @@ __all__ = [
     "ModelStatusReason",
     "ReportStatus",
     "ReportStatusReason",
+    "identifier_regex",
+    "identifier_maxlen",
     "is_status_terminal",
     "is_status_failure",
     "is_status_success",
-    "DyffEntityType",
-    "SYSTEM_ATTRIBUTES",
+    "summary_maxlen",
+    "title_maxlen",
 ]

{dyff_schema-0.17.0 → dyff_schema-0.19.0}/dyff/schema/v0/r1/requests.py

@@ -14,6 +14,7 @@ in response.
 
 from __future__ import annotations
 
+import re
 from datetime import datetime
 from typing import Optional, Union
 
@@ -22,6 +23,7 @@ import pydantic
 from .base import DyffBaseModel
 from .platform import (
     AnalysisBase,
+    AnalysisScope,
     DatasetBase,
     DataView,
     DocumentationBase,
@@ -70,6 +72,25 @@ class AnalysisCreateRequest(DyffEntityCreateRequest, AnalysisBase):
 
     method: str = pydantic.Field(description="Method ID")
 
+    @pydantic.validator("scope", check_fields=False)
+    def _validate_scope(cls, scope: AnalysisScope) -> AnalysisScope:
+        # TODO: This has to be a validator function because we can't apply the
+        # regex contraint to AnalysisScope, since there are already entities
+        # with invalid IDs in the data store. Fix in Schema v1.
+        uuid4 = r"^[0-9a-f]{8}[0-9a-f]{4}[4][0-9a-f]{3}[89ab][0-9a-f]{3}[0-9a-f]{12}$"
+        id_pattern = re.compile(uuid4)
+        if scope.dataset is not None and not re.match(id_pattern, scope.dataset):
+            raise ValueError("scope.dataset must be an entity ID")
+        if scope.evaluation is not None and not re.match(id_pattern, scope.evaluation):
+            raise ValueError("scope.evaluation must be an entity ID")
+        if scope.inferenceService is not None and not re.match(
+            id_pattern, scope.inferenceService
+        ):
+            raise ValueError("scope.inferenceService must be an entity ID")
+        if scope.model is not None and not re.match(id_pattern, scope.model):
+            raise ValueError("scope.model must be an entity ID")
+        return scope
+
 
 class DatasetCreateRequest(DyffEntityCreateRequest, DatasetBase):
     pass

{dyff_schema-0.17.0 → dyff_schema-0.19.0/dyff_schema.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dyff-schema
-Version: 0.17.0
+Version: 0.19.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{dyff_schema-0.17.0 → dyff_schema-0.19.0}/dyff_schema.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ dyff/schema/dataset/__init__.py
 dyff/schema/dataset/arrow.py
 dyff/schema/dataset/binary.py
 dyff/schema/dataset/classification.py
+dyff/schema/dataset/embedding.py
 dyff/schema/dataset/text.py
 dyff/schema/dataset/vision.py
 dyff/schema/io/__init__.py
@@ -43,6 +44,7 @@ dyff/schema/v0/r1/dataset/__init__.py
 dyff/schema/v0/r1/dataset/arrow.py
 dyff/schema/v0/r1/dataset/binary.py
 dyff/schema/v0/r1/dataset/classification.py
+dyff/schema/v0/r1/dataset/embedding.py
 dyff/schema/v0/r1/dataset/text.py
 dyff/schema/v0/r1/dataset/vision.py
 dyff/schema/v0/r1/io/__init__.py