dyff-schema 0.13.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

dyff/schema/ids.py

@@ -24,6 +24,24 @@ def null_id() -> str:
     return uuid.UUID(int=0).hex
 
 
+def namespaced_id(namespace_id: str, entity_name: str) -> str:
+    """Return an ID for a named entity whose name is unique within a namespace
+    corresponding to another entity ID.
+
+    This is used to model "child" resources that exist within the context of
+    a "parent".
+
+    :param namespace_id: The ID of the namespace.
+    :type namespace_id: str
+    :param entity_name: A name that is unique in the context of the namespace.
+    :type entity_name: str
+    :returns: The unique identifier of this (namespace, name)
+        combination -- a hex string representation of a UUID.
+    :rtype: str
+    """
+    return uuid.uuid5(uuid.UUID(hex=namespace_id), entity_name).hex
+
+
 def replication_id(evaluation_id: str, replication_index: int) -> str:
     """Return a unique identifier for a replication within an evaluation. Replications
     in different evaluations will have different identifiers, so datasets from different
@@ -37,4 +55,4 @@ def replication_id(evaluation_id: str, replication_index: int) -> str:
         combination -- a hex string representation of a UUID.
     :rtype: str
     """
-    return uuid.uuid5(uuid.UUID(hex=evaluation_id), str(replication_index)).hex
+    return namespaced_id(evaluation_id, str(replication_index))

dyff/schema/v0/r1/platform.py

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2024 UL Research Institutes
 # SPDX-License-Identifier: Apache-2.0
+# fmt: off
 """Schema for the internal data representation of Dyff entities.
 
 We use the following naming convention:
@@ -15,11 +16,14 @@ We use the following naming convention:
       We include the full dependency data structure so that downstream
       components don't need to be able to look it up by ID.
 """
+
+# fmt: on
 # mypy: disable-error-code="import-untyped"
 import abc
 import enum
 import urllib.parse
 from datetime import datetime
+from decimal import Decimal
 from enum import Enum
 from typing import Any, Literal, NamedTuple, Optional, Type, Union
 
@@ -178,6 +182,7 @@ class Entities(str, enum.Enum):
     Report = "Report"
     Revision = "Revision"
     SafetyCase = "SafetyCase"
+    Score = "Score"
 
 
 class Resources(str, enum.Enum):
@@ -201,6 +206,7 @@ class Resources(str, enum.Enum):
     Report = "reports"
     Revision = "revisions"
     SafetyCase = "safetycases"
+    Score = "scores"
 
     Task = "tasks"
     """
@@ -1549,6 +1555,77 @@ class MethodScope(str, enum.Enum):
     Evaluation = Entities.Evaluation.value
 
 
+class ScoreSpec(DyffSchemaBaseModel):
+    name: str = pydantic.Field(
+        description="The name of the score. Used as a key for retrieving score data."
+        " Must be unique within the Method context.",
+    )
+
+    title: str = pydantic.Field(
+        description="The title text to use when displaying score information.",
+        max_length=140,
+    )
+
+    summary: str = pydantic.Field(
+        description="A short text description of what the score measures.",
+        max_length=140,
+    )
+
+    valence: Literal["positive", "negative", "neutral"] = pydantic.Field(
+        description="A score has 'positive' valence if 'more is better',"
+        " 'negative' valence if 'less is better', and 'neutral' valence if"
+        " 'better' is not meaningful for this score."
+    )
+
+    priority: Literal["primary", "secondary"] = pydantic.Field(
+        description="The 'primary' score will be displayed in any UI widgets"
+        " that expect a single score."
+    )
+
+    minimum: Optional[Decimal] = pydantic.Field(
+        default=None, description="The minimum possible value, if known."
+    )
+
+    maximum: Optional[Decimal] = pydantic.Field(
+        default=None, description="The maximum possible value, if known."
+    )
+
+    format: str = pydantic.Field(
+        default="{quantity}",
+        description="A Python 'format' string describing how to render the score"
+        " as a string. You can use the keys 'quantity' and 'unit' in the format"
+        " string (e.g., '{quantity} {unit}').",
+    )
+
+    unit: Optional[str] = pydantic.Field(
+        default=None,
+        description="The unit of measure, if applicable (e.g., 'meters', 'kJ/g')."
+        " Use standard SI abbreviations where possible for better indexing.",
+    )
+
+    @pydantic.root_validator
+    def _validate_minimum_maximum(cls, values):
+        minimum = values.get("minimum")
+        maximum = values.get("maximum")
+        if minimum is not None and maximum is not None and minimum > maximum:
+            raise ValueError(f"minimum {minimum} is greater than maximum {maximum}")
+        return values
+
+    @pydantic.validator("format")
+    def _validate_format(cls, v):
+        try:
+            cls._format_quantity(v, Decimal("3.14"), unit="kg")
+        except Exception:
+            raise ValueError(f"invalid format: '{v}'")
+        return v
+
+    @classmethod
+    def _format_quantity(
+        cls, format: str, quantity: Decimal, *, unit: Optional[str] = None
+    ) -> str:
+        return format.format(quantity=quantity, unit=unit)
+
+
 class MethodBase(DyffSchemaBaseModel):
     name: str = pydantic.Field(description="Descriptive name of the Method.")
 
@@ -1579,11 +1656,26 @@ class MethodBase(DyffSchemaBaseModel):
         description="Specification of the Method output."
     )
 
+    scores: list[ScoreSpec] = pydantic.Field(
+        default_factory=list,
+        description="Specifications of the Scores that this Method produces.",
+    )
+
     modules: list[str] = pydantic.Field(
         default_factory=list,
         description="Modules to load into the analysis environment",
     )
 
+    @pydantic.validator("scores")
+    def _scores_validator(cls, scores: list[ScoreSpec]):
+        if len(scores) > 0:
+            primary_count = sum(score.priority == "primary" for score in scores)
+            if primary_count != 1:
+                raise ValueError(
+                    "scores: Must have exactly one score with .priority == 'primary'"
+                )
+        return scores
+
 
 class Method(DyffEntity, MethodBase):
     kind: Literal["Method"] = Entities.Method.value
@@ -1659,11 +1751,32 @@ class AnalysisBase(DyffSchemaBaseModel):
     )
 
 
+class AnalysisData(DyffSchemaBaseModel):
+    """Arbitrary additional data for the Analysis, specified as a key-value pair where
+    the value is the data encoded in base64."""
+
+    key: str = pydantic.Field(
+        description="Key identifying the data. For data about a Dyff entity,"
+        " this should be the entity's ID."
+    )
+
+    value: str = pydantic.Field(
+        # Canonical base64 encoding
+        # https://stackoverflow.com/a/64467300/3709935
+        regex=r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?$",
+        description="Arbitrary data encoded in base64.",
+    )
+
+
 class Analysis(AnalysisBase):
     method: ForeignMethod = pydantic.Field(
         description="The analysis Method to run.",
     )
 
+    data: list[AnalysisData] = pydantic.Field(
+        default_factory=list, description="Additional data to supply to the analysis."
+    )
+
 
 class Measurement(DyffEntity, MeasurementSpec, Analysis):
     kind: Literal["Measurement"] = Entities.Measurement.value
@@ -1685,25 +1798,55 @@ class SafetyCase(DyffEntity, SafetyCaseSpec, Analysis):
         return None
 
 
-# class Analysis(DyffEntity, AnalysisBase):
-#     kind: Literal["Analysis"] = Entities.Analysis.value
+class ScoreMetadataRefs(AnalysisScope):
+    """References to other Dyff entities related to a Score."""
+
+    method: str = pydantic.Field(description="The Method that generates the score.")
+
+
+class ScoreMetadata(AnalysisScope):
+    """Metadata about a Score entity."""
+
+    refs: ScoreMetadataRefs = pydantic.Field(
+        description="References to other related Dyff entities."
+    )
+
+
+class ScoreData(ScoreSpec):
+    """ScoreData is an "instance" of a ScoreSpec containing the concrete measured value
+    for the score."""
+
+    analysis: str = pydantic.Field(
+        description="The Analysis that generated the current score instance."
+    )
 
-#     method: ForeignMethod = pydantic.Field(
-#         description="The analysis Method to run.",
-#     )
+    quantity: Decimal = pydantic.Field(
+        description="The numeric quantity associated with the score."
+        " Callers should set the desired precision explicitly."
+    )
 
-#     def dependencies(self) -> list[str]:
-#         return [self.method.id] + [x.entity for x in self.inputs]
+    text: str = pydantic.Field(
+        description="A short text description of what the quantity means.",
+        max_length=140,
+    )
 
-#     def resource_allocation(self) -> Optional[ResourceAllocation]:
-#         return None
+    def format_quantity(self) -> str:
+        return self._format_quantity(self.format, self.quantity, unit=self.unit)
 
 
-# class AnalysisAndOutputs(DyffSchemaBaseModel):
-#     analysis: Analysis = pydantic.Field(description="The Analysis entity")
-#     outputs: list[AnalysisOutputType] = pydantic.Field(
-#         description="Concrete outputs of the Analysis"
-#     )
+class Score(ScoreSpec):
+    """A Score is a numeric quantity describing an aspect of system performance.
+
+    Conceptually, a Score is an "instance" of a ScoreSpec.
+    """
+
+    kind: Literal["Score"] = Entities.Score.value
+
+    id: str = pydantic.Field(description="Unique identifier of the entity")
+
+    metadata: ScoreMetadata = pydantic.Field(
+        description="Metadata about the score; used for indexing.",
+    )
 
 
 # ---------------------------------------------------------------------------
@@ -2024,6 +2167,7 @@ __all__ = [
     "Analysis",
     "AnalysisArgument",
     "AnalysisBase",
+    "AnalysisData",
     "AnalysisInput",
     "AnalysisOutputQueryFields",
     "AnalysisScope",
@@ -2126,6 +2270,11 @@ __all__ = [
     "SafetyCase",
     "SafetyCaseSpec",
     "SchemaAdapter",
+    "Score",
+    "ScoreData",
+    "ScoreSpec",
+    "ScoreMetadata",
+    "ScoreMetadataRefs",
     "Status",
     "StorageSignedURL",
     "TagName",

{dyff_schema-0.13.0.dist-info → dyff_schema-0.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dyff-schema
-Version: 0.13.0
+Version: 0.15.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{dyff_schema-0.13.0.dist-info → dyff_schema-0.15.0.dist-info}/RECORD

@@ -4,7 +4,7 @@ dyff/schema/annotations.py,sha256=nE6Jk1PLqlShj8uqjE_EzZC9zYnTDW5AVtQcjysiK8M,10
 dyff/schema/base.py,sha256=jvaNtsSZyFfsdUZTcY_U-yfLY5_GyrMxSXhON2R9XR0,119
 dyff/schema/copydoc.py,sha256=B4ZRpQmbFxi-3l9LCHvaJiVKb9VxADgC5vey804Febc,1075
 dyff/schema/errors.py,sha256=ow3yiucU4wGkeLmapUURp3eyaebwGUwDaVTXpPcrA7M,1542
-dyff/schema/ids.py,sha256=Z3JQzlAJQC2Pam7ehxb4TXA4MIuFQN5SyzL5Ql0RukA,1422
+dyff/schema/ids.py,sha256=M3kdGRYGPCMnFkRRo0nzHlntdJtBrRwGJ_9R7LlN0bI,2085
 dyff/schema/platform.py,sha256=peHzGGSd5dQ-EFXrWDjBqMUtoOL3iCHxcV3XzW6Rjag,123
 dyff/schema/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dyff/schema/quantity.py,sha256=hhS9ybqW_6I--acPhkoZbFWASxFTEs7CUjO4pmpBJ98,3788
@@ -23,7 +23,7 @@ dyff/schema/v0/__init__.py,sha256=L5y8UhRnojerPYHumsxQJRcHCNz8Hj9NM8b47mewMNs,92
 dyff/schema/v0/r1/__init__.py,sha256=L5y8UhRnojerPYHumsxQJRcHCNz8Hj9NM8b47mewMNs,92
 dyff/schema/v0/r1/adapters.py,sha256=2t2oxsnGfSEDKKDIEYw4qqLXMH7qlFIwPVuLyUmbsHs,23552
 dyff/schema/v0/r1/base.py,sha256=i7eOKXDGS8_J9k2aVObUTpSOnA8CAgRW7Quj1fSbyRg,19403
-dyff/schema/v0/r1/platform.py,sha256=fFi75T8AKqfxzA_6c43fz9Edb9bBVIYocoSLoHo_Eog,67594
+dyff/schema/v0/r1/platform.py,sha256=Vg-JplBIQSBIgPWNwuFj_HFwJS8o7TSNyC-SvZc3BoU,72503
 dyff/schema/v0/r1/requests.py,sha256=MMMtSfT9JpE2X7PLQGIuI4v7mMlJ-8DiZ65jc2hE0zc,10976
 dyff/schema/v0/r1/test.py,sha256=X6dUyVd5svcPCI-PBMOAqEfK9jv3bRDvkQTJzwS96c0,10720
 dyff/schema/v0/r1/version.py,sha256=isKAGuGxsdru8vDaYmI4YiZdJOu_wNxXK7u6QzD6FE4,392
@@ -35,9 +35,9 @@ dyff/schema/v0/r1/dataset/text.py,sha256=nLIn91Zlt0tNdXUklSgjJ-kEDxoPX32ISLkiv2D
 dyff/schema/v0/r1/dataset/vision.py,sha256=aIe0fbfM_g3DsrDTdg2K803YKLjZBpurM_VJcJFuZLc,369
 dyff/schema/v0/r1/io/__init__.py,sha256=L5y8UhRnojerPYHumsxQJRcHCNz8Hj9NM8b47mewMNs,92
 dyff/schema/v0/r1/io/vllm.py,sha256=CUE9y8KthtUI7sD49S875rDmPvKotSXVIRaBS79aBZs,5320
-dyff_schema-0.13.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dyff_schema-0.13.0.dist-info/METADATA,sha256=83jGlADrQ7anWRYJNf00XwaVHkAiPdpJmRu6rVIU6fw,3482
-dyff_schema-0.13.0.dist-info/NOTICE,sha256=YONACu0s_Ui6jNi-wtEsVQbTU1JIkh8wvLH6d1-Ni_w,43
-dyff_schema-0.13.0.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-dyff_schema-0.13.0.dist-info/top_level.txt,sha256=9e3VVdeX73t_sUJOPQPCcGtYO1JhoErhHIi3WoWGcFI,5
-dyff_schema-0.13.0.dist-info/RECORD,,
+dyff_schema-0.15.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dyff_schema-0.15.0.dist-info/METADATA,sha256=QS7nIjfIS4L0D8v0Q6FBB9gyeP_0aIh-F1W1Dxy0_F0,3482
+dyff_schema-0.15.0.dist-info/NOTICE,sha256=YONACu0s_Ui6jNi-wtEsVQbTU1JIkh8wvLH6d1-Ni_w,43
+dyff_schema-0.15.0.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+dyff_schema-0.15.0.dist-info/top_level.txt,sha256=9e3VVdeX73t_sUJOPQPCcGtYO1JhoErhHIi3WoWGcFI,5
+dyff_schema-0.15.0.dist-info/RECORD,,

{dyff_schema-0.13.0.dist-info → dyff_schema-0.15.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any