dyff-schema 0.16.1__tar.gz → 0.18.0__tar.gz

{dyff_schema-0.16.1/dyff_schema.egg-info → dyff_schema-0.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dyff-schema
-Version: 0.16.1
+Version: 0.18.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{dyff_schema-0.16.1 → dyff_schema-0.18.0}/dyff/schema/v0/r1/platform.py

@@ -23,7 +23,6 @@ import abc
 import enum
 import urllib.parse
 from datetime import datetime
-from decimal import Decimal
 from enum import Enum
 from typing import Any, Literal, NamedTuple, Optional, Type, Union
 
@@ -160,6 +159,26 @@ def _oci_image_tag_maxlen():
     return 127
 
 
+def identifier_regex():
+    """Python identifiers start with a letter or an underscore, and consist of letters,
+    numbers, and underscores."""
+    return r"^[a-zA-Z_][a-zA-Z0-9_]*$"
+
+
+def identifier_maxlen():
+    """There isn't really a max length for Python identifiers, but this seems like a
+    reasonable limit for our use."""
+    return 127
+
+
+def title_maxlen() -> int:
+    return 140
+
+
+def summary_maxlen() -> int:
+    return 280
+
+
 class Entities(str, enum.Enum):
     """The kinds of entities in the dyff system."""
 
@@ -1559,42 +1578,51 @@ class ScoreSpec(DyffSchemaBaseModel):
     name: str = pydantic.Field(
         description="The name of the score. Used as a key for retrieving score data."
         " Must be unique within the Method context.",
+        regex=identifier_regex(),
+        max_length=identifier_maxlen(),
     )
 
     title: str = pydantic.Field(
         description="The title text to use when displaying score information.",
-        max_length=140,
+        max_length=title_maxlen(),
     )
 
     summary: str = pydantic.Field(
         description="A short text description of what the score measures.",
-        max_length=140,
+        max_length=summary_maxlen(),
     )
 
     valence: Literal["positive", "negative", "neutral"] = pydantic.Field(
+        default="neutral",
         description="A score has 'positive' valence if 'more is better',"
         " 'negative' valence if 'less is better', and 'neutral' valence if"
-        " 'better' is not meaningful for this score."
+        " 'better' is not meaningful for this score.",
     )
 
     priority: Literal["primary", "secondary"] = pydantic.Field(
+        default="primary",
         description="The 'primary' score will be displayed in any UI widgets"
-        " that expect a single score."
+        " that expect a single score. There must be exactly 1 primary score.",
     )
 
-    minimum: Optional[Decimal] = pydantic.Field(
+    minimum: Optional[float] = pydantic.Field(
         default=None, description="The minimum possible value, if known."
     )
 
-    maximum: Optional[Decimal] = pydantic.Field(
+    maximum: Optional[float] = pydantic.Field(
         default=None, description="The maximum possible value, if known."
     )
 
     format: str = pydantic.Field(
-        default="{quantity}",
+        default="{quantity:.1f}",
+        # Must use the 'quantity' key in the format string:
+        # (Maybe string not ending in '}')(something like '{quantity:f}')(maybe another string)
+        regex=r"^(.*[^{])?[{]quantity(:[^}]*)?[}]([^}].*)?$",
         description="A Python 'format' string describing how to render the score"
-        " as a string. You can use the keys 'quantity' and 'unit' in the format"
-        " string (e.g., '{quantity} {unit}').",
+        " as a string. You *must* use the keyword 'quantity' in the format"
+        " string, and you may use 'unit' as well (e.g., '{quantity:.2f} {unit}')."
+        " It is *strongly recommended* that you limit the output precision"
+        " appropriately; use ':.0f' for integer-valued scores.",
     )
 
     unit: Optional[str] = pydantic.Field(
@@ -1603,6 +1631,11 @@ class ScoreSpec(DyffSchemaBaseModel):
         " Use standard SI abbreviations where possible for better indexing.",
     )
 
+    def quantity_string(self, quantity: float) -> str:
+        """Formats the given quantity as a string, according to the formatting
+        information stored in this ScoreSpec."""
+        return self.format_quantity(self.format, quantity, unit=self.unit)
+
     @pydantic.root_validator
     def _validate_minimum_maximum(cls, values):
         minimum = values.get("minimum")
@@ -1613,15 +1646,17 @@ class ScoreSpec(DyffSchemaBaseModel):
 
     @pydantic.validator("format")
     def _validate_format(cls, v):
-        try:
-            cls._format_quantity(v, Decimal("3.14"), unit="kg")
-        except Exception:
-            raise ValueError(f"invalid format: '{v}'")
+        x = cls.format_quantity(v, 3.14, unit="kg")
+        y = cls.format_quantity(v, -2.03, unit="kg")
+        if x == y:
+            # Formatted results for different quantities should be different
+            raise ValueError("format string does not mention 'quantity'")
+
         return v
 
     @classmethod
-    def _format_quantity(
-        cls, format: str, quantity: Decimal, *, unit: Optional[str] = None
+    def format_quantity(
+        cls, format: str, quantity: float, *, unit: Optional[str] = None
     ) -> str:
         return format.format(quantity=quantity, unit=unit)
 
@@ -1824,19 +1859,20 @@ class ScoreData(ScoreSpec):
         description="The Analysis that generated the current score instance."
     )
 
-    quantity: Decimal = pydantic.Field(
+    quantity: float = pydantic.Field(
         description="The numeric quantity associated with the score."
-        " Callers should set the desired precision explicitly."
+    )
+
+    quantityString: str = pydantic.Field(
+        description="The formatted string representation of .quantity,"
+        " after processing with the .format specification."
     )
 
     text: str = pydantic.Field(
         description="A short text description of what the quantity means.",
-        max_length=140,
+        max_length=summary_maxlen(),
     )
 
-    def format_quantity(self) -> str:
-        return self._format_quantity(self.format, self.quantity, unit=self.unit)
-
 
 class Score(ScoreData):
     """A Score is a numeric quantity describing an aspect of system performance.
@@ -2160,6 +2196,7 @@ DyffEntityType = Union[DyffEntityTypeExceptRevision, Revision]
 
 
 __all__ = [
+    "SYSTEM_ATTRIBUTES",
     "Accelerator",
     "AcceleratorGPU",
     "AccessGrant",
@@ -2193,6 +2230,7 @@ __all__ = [
     "DocumentationBase",
     "DyffDataSchema",
     "DyffEntity",
+    "DyffEntityType",
     "DyffModelWithID",
     "DyffSchemaBaseModel",
     "Entities",
@@ -2295,9 +2333,11 @@ __all__ = [
     "ModelStatusReason",
     "ReportStatus",
     "ReportStatusReason",
+    "identifier_regex",
+    "identifier_maxlen",
     "is_status_terminal",
     "is_status_failure",
     "is_status_success",
-    "DyffEntityType",
-    "SYSTEM_ATTRIBUTES",
+    "summary_maxlen",
+    "title_maxlen",
 ]

{dyff_schema-0.16.1 → dyff_schema-0.18.0}/dyff/schema/v0/r1/requests.py

@@ -290,6 +290,25 @@ class SafetyCaseQueryRequest(_AnalysisProductQueryRequest):
     pass
 
 
+class ScoreQueryRequest(DyffRequestDefaultValidators):
+    query: Optional[str] = pydantic.Field(
+        default=None,
+        description="A JSON structure describing a query, encoded as a string."
+        " Valid keys are the same as the valid query keys for the corresponding"
+        " endpoint. Values can be scalars or lists. Lists are treated as"
+        " disjunctive queries (i.e., 'value $in list').",
+    )
+
+    id: Optional[str] = pydantic.Field(default=None)
+    name: Optional[str] = pydantic.Field(default=None)
+    method: Optional[str] = pydantic.Field(default=None)
+    methodName: Optional[str] = pydantic.Field(default=None)
+    dataset: Optional[str] = pydantic.Field(default=None)
+    evaluation: Optional[str] = pydantic.Field(default=None)
+    inferenceService: Optional[str] = pydantic.Field(default=None)
+    model: Optional[str] = pydantic.Field(default=None)
+
+
 __all__ = [
     "AnalysisCreateRequest",
     "AuditQueryRequest",
@@ -323,4 +342,5 @@ __all__ = [
     "ReportCreateRequest",
     "ReportQueryRequest",
     "SafetyCaseQueryRequest",
+    "ScoreQueryRequest",
 ]

{dyff_schema-0.16.1 → dyff_schema-0.18.0/dyff_schema.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dyff-schema
-Version: 0.16.1
+Version: 0.18.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0