dyff-schema 0.38.5__tar.gz → 0.39.0__tar.gz

{dyff_schema-0.38.5/dyff_schema.egg-info → dyff_schema-0.39.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dyff-schema
-Version: 0.38.5
+Version: 0.39.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

dyff_schema-0.39.0/dyff/schema/_version.py

@@ -0,0 +1,2 @@
+__version__ = version = "0.39.0"
+__version_tuple__ = version_tuple = (0, 39, 0)

{dyff_schema-0.38.5 → dyff_schema-0.39.0}/dyff/schema/v0/r1/platform.py

@@ -216,6 +216,7 @@ class Entities(str, enum.Enum):
     AuditProcedure = "AuditProcedure"
     Challenge = "Challenge"
     Concern = "Concern"
+    Curve = "Curve"
     DataSource = "DataSource"
     Dataset = "Dataset"
     Documentation = "Documentation"
@@ -256,6 +257,7 @@ class Resources(str, enum.Enum):
     AuditProcedure = "auditprocedures"
     Challenge = "challenges"
     Concern = "concerns"
+    Curve = "curves"
     Dataset = "datasets"
     DataSource = "datasources"
     Descriptor = "descriptors"
@@ -309,6 +311,7 @@ EntityKindLiteral = Literal[
     "Audit",
     "AuditProcedure",
     "Challenge",
+    "Curve",
     "DataSource",
     "Dataset",
     "Evaluation",
@@ -435,7 +438,7 @@ class Labeled(DyffSchemaBaseModel):
 class Annotation(DyffSchemaBaseModel):
     key: str = pydantic.Field(
         pattern=_k8s_label_key_regex(),
-        max_length=_k8s_domain_maxlen(),
+        max_length=_k8s_label_key_maxlen(),
         description="The annotation key. A DNS label with an optional DNS domain prefix."
         " For example: 'my-key', 'your.com/key_0'. Names prefixed with"
         " 'dyff.io/', 'subdomain.dyff.io/', etc. are reserved.\n\n"
@@ -541,6 +544,7 @@ class DyffEntity(Status, Labeled, SchemaVersion, DyffModelWithID):
         "Audit",
         "AuditProcedure",
         "Challenge",
+        "Curve",
         "DataSource",
         "Dataset",
         "Evaluation",
@@ -1442,6 +1446,21 @@ class VolumeMount(DyffSchemaBaseModel):
         default=None, description="Configuration for Scratch volume mounts."
     )
 
+    @pydantic.model_validator(mode="after")
+    def _validate_kind_matches_payload(self):
+        """Ensure payload fields match declared kind."""
+        if self.kind == VolumeMountKind.data:
+            if self.data is None or self.scratch is not None:
+                raise ValueError(
+                    "VolumeMount(kind='Data') requires .data and forbids .scratch"
+                )
+        elif self.kind == VolumeMountKind.scratch:
+            if self.scratch is None or self.data is not None:
+                raise ValueError(
+                    "VolumeMount(kind='Scratch') requires .scratch and forbids .data"
+                )
+        return self
+
 
 class Container(DyffSchemaBaseModel):
     """Configuration of a runnable container backed by either an image hosted in an
@@ -1489,6 +1508,15 @@ class Container(DyffSchemaBaseModel):
         " which might be configured in the container image.",
     )
 
+    @pydantic.model_validator(mode="after")
+    def _validate_image_or_ref(self):
+        """Exactly one of image or imageRef must be set."""
+        has_image = self.image is not None
+        has_ref = self.imageRef is not None
+        if has_image == has_ref:
+            raise ValueError("Exactly one of 'image' or 'imageRef' must be set.")
+        return self
+
 
 class InferenceServiceRunner(Container):
     """Configuration of the runtime environment to use to run an inference service.
@@ -2101,6 +2129,34 @@ class ScoreSpec(DyffSchemaBaseModel):
         return format.format(quantity=quantity, unit=unit)
 
 
+class CurveSpec(DyffSchemaBaseModel):
+    """Metadata describing the curve dimensions and how to read them.
+
+    - `dimensions` supplies per-dimension metadata (name -> ScoreSpec).
+    - Use these keys as columns in `CurveData.points`.
+    """
+
+    name: str = pydantic.Field(
+        description="Unique key for this curve within the Method context.",
+        pattern=identifier_regex(),
+        max_length=identifier_maxlen(),
+    )
+    title: str = pydantic.Field(
+        description="Human-friendly title for the curve.",
+        max_length=title_maxlen(),
+    )
+    summary: str = pydantic.Field(
+        description="Short description of what the curve shows.",
+        max_length=summary_maxlen(),
+    )
+    dimensions: dict[str, ScoreSpec] = pydantic.Field(
+        description=(
+            "Per-dimension metadata. Keys must match the columns stored in CurveData.points. "
+            "Typical keys might include 'x', 'y', 'threshold', 'tp', 'fp', etc."
+        )
+    )
+
+
 class MethodBase(DyffSchemaBaseModel):
     name: str = pydantic.Field(description="Descriptive name of the Method.")
 
@@ -2131,7 +2187,7 @@ class MethodBase(DyffSchemaBaseModel):
         description="Specification of the Method output."
     )
 
-    scores: list[ScoreSpec] = pydantic.Field(
+    scores: list[Union[ScoreSpec, CurveSpec]] = pydantic.Field(
         default_factory=list,
         description="Specifications of the Scores that this Method produces.",
     )
@@ -2149,12 +2205,13 @@ class MethodBase(DyffSchemaBaseModel):
     )
 
     @pydantic.field_validator("scores")
-    def _scores_validator(cls, scores: list[ScoreSpec]):
-        if len(scores) > 0:
-            primary_count = sum(score.priority == "primary" for score in scores)
+    def _scores_validator(cls, scores):
+        score_specs = [s for s in scores if isinstance(s, ScoreSpec)]
+        if score_specs:
+            primary_count = sum(s.priority == "primary" for s in score_specs)
             if primary_count != 1:
                 raise ValueError(
-                    "scores: Must have exactly one score with .priority == 'primary'"
+                    "scores: Must have exactly one primary ScoreSpec (curves excluded)"
                 )
         return scores
 
@@ -2329,17 +2386,19 @@ class ScoreData(ScoreSpec):
         description="The Analysis that generated the current score instance."
     )
 
-    quantity: float = pydantic.Field(
-        description="The numeric quantity associated with the score."
+    quantity: float | None = pydantic.Field(
+        default=None, description="Numeric quantity for scalar scores."
+    )
+    points: dict[str, list[float]] | None = pydantic.Field(
+        default=None,
+        description="Aligned vectors for curve scores. Keys should match curve.dimensions.",
     )
 
     quantityString: str = pydantic.Field(
-        description="The formatted string representation of .quantity,"
-        " after processing with the .format specification."
+        description="Formatted string representation of .quantity (scalar) or a summary string (curve)."
     )
-
     text: str = pydantic.Field(
-        description="A short text description of what the quantity means.",
+        description="Short text description of what the value/curve means.",
         max_length=summary_maxlen(),
     )
 
@@ -2355,6 +2414,53 @@ class Score(ScoreData):
     id: str = pydantic.Field(description="Unique identifier of the entity")
 
 
+class CurveData(CurveSpec):
+    """An instance of a curve produced by an analysis.
+
+    - Inherits the curve spec (name/title/summary/dimensions) directly.
+    - `points` is a dict-of-lists: each key is a dimension name defined in `dimensions`.
+    - All lists must be the same (non-zero) length.
+
+    spec: CurveSpec = pydantic.Field(description="The CurveSpec this data conforms to.")
+    """
+
+    metadata: ScoreMetadata = pydantic.Field(
+        description="Indexing metadata (method required).",
+    )
+    analysis: str = pydantic.Field(
+        description="ID of the Analysis that produced this curve."
+    )
+    points: dict[str, list[float]] = pydantic.Field(
+        description="Aligned vectors for each dimension; all lists must have equal length >= 1."
+    )
+
+    @pydantic.model_validator(mode="after")
+    def _validate_points(self):
+        if not self.points:
+            raise ValueError("CurveData.points must not be empty")
+
+        lengths = {k: len(v) for k, v in self.points.items()}
+        uniq = set(lengths.values())
+        if len(uniq) != 1:
+            raise ValueError(
+                f"All vectors must have equal length; got lengths {lengths}"
+            )
+        n = next(iter(uniq))
+        if n < 1:
+            raise ValueError("Vectors must contain at least 1 point")
+
+        missing = [k for k in self.dimensions.keys() if k not in self.points]
+        if missing:
+            raise ValueError(f"points is missing required dimensions: {missing}")
+        return self
+
+
+class Curve(CurveData):
+    kind: Literal["Curve"] = Entities.Curve.value
+
+    id: str = pydantic.Field(description="Unique identifier of the entity")
+
+
 # ---------------------------------------------------------------------------
 # OCI artifacts
 
@@ -2635,6 +2741,19 @@ class ChallengeTaskSchedule(DyffSchemaBaseModel):
         description="Teams are limited to this many submissions per cycle.",
     )
 
+    @pydantic.model_validator(mode="after")
+    def _validate_times(self):
+        """Basic temporal sanity checks."""
+        if (
+            self.openingTime
+            and self.closingTime
+            and self.closingTime <= self.openingTime
+        ):
+            raise ValueError("closingTime must be after openingTime")
+        if self.submissionCycleDuration <= timedelta(0):
+            raise ValueError("submissionCycleDuration must be positive")
+        return self
+
 
 class ChallengeTaskRules(DyffSchemaBaseModel):
     """The rules of the challenge."""
@@ -2656,6 +2775,18 @@ class ChallengeTaskContent(DyffSchemaBaseModel):
     )
 
 
+class SubmissionStructure(DyffSchemaBaseModel):
+    submissionKind: EntityKindLiteral = pydantic.Field(
+        default=Entities.InferenceService.value,
+        description="The kind of entity that you can submit to this task.",
+    )
+    pipelineKeyword: str = pydantic.Field(
+        default="submission",
+        description="The keyword parameter where the submitted entity ID"
+        " should be passed to the assessment Pipeline.",
+    )
+
+
 class ChallengeTaskBase(DyffSchemaBaseModel):
     """The part of the ChallengeTask spec that is not system-populated."""
 
@@ -2667,9 +2798,14 @@ class ChallengeTaskBase(DyffSchemaBaseModel):
         " This may appear in URLs and must follow naming restrictions.",
         max_length=title_maxlen(),
         pattern=r"[a-z0-9-]*",
+        min_length=1,
     )
     assessment: str = pydantic.Field(
-        description="ID of the Assessment that the task runs."
+        description="ID of the Pipeline used to assess the submission."
+    )
+    submissionStructure: SubmissionStructure = pydantic.Field(
+        default_factory=SubmissionStructure,
+        description="How to run the assessment pipeline on a new submission.",
     )
     rules: ChallengeTaskRules = pydantic.Field(description="The rules for submissions.")
     content: ChallengeTaskContent = pydantic.Field(
@@ -2729,7 +2865,14 @@ class Challenge(DyffEntity):
         return None
 
 
-class ChallengeSubmission(DyffEntity):
+class SubmissionBase(DyffSchemaBaseModel):
+    team: str = pydantic.Field(description="The ID of the team making the submission.")
+    submission: EntityIdentifier = pydantic.Field(
+        description="The resource being submitted for assessment."
+    )
+
+
+class Submission(DyffEntity, SubmissionBase):
     """A submission of an inference system to a challenge by a team.
 
     All of the constituent resources must already exist. Creating a Submission simply
@@ -2746,9 +2889,8 @@ class ChallengeSubmission(DyffEntity):
     task: str = pydantic.Field(
         description="The key of the task within the challenge being submitted to."
     )
-    team: str = pydantic.Field(description="The ID of the team making the submission.")
-    inferenceService: str = pydantic.Field(
-        description="The ID of the inference service being submitted."
+    pipelineRun: str = pydantic.Field(
+        description="The ID of the PipelineRun for this submission."
     )
 
     def dependencies(self) -> list[str]:
@@ -2765,14 +2907,33 @@ class ChallengeSubmission(DyffEntity):
 class PipelineEvaluationRequest(EvaluationRequestBase):
     kind: Literal["PipelineEvaluationRequest"] = "PipelineEvaluationRequest"
 
+    def dependencies(self) -> list[str]:
+        if self.inferenceSession is None:
+            raise AssertionError()
+        return [self.dataset, self.inferenceSession.inferenceService]
+
+    @pydantic.model_validator(mode="after")
+    def check_session_not_reference(self):
+        if self.inferenceSession is None or self.inferenceSessionReference is not None:
+            raise ValueError(
+                "evaluations in pipelines must set inferenceSession, not inferenceSessionReference"
+            )
+        return self
+
 
 class PipelineMeasurementRequest(AnalysisRequestBase):
     kind: Literal["PipelineMeasurementRequest"] = "PipelineMeasurementRequest"
 
+    def dependencies(self) -> list[str]:
+        return [self.method] + [x.entity for x in self.inputs]
+
 
 class PipelineSafetyCaseRequest(AnalysisRequestBase):
     kind: Literal["PipelineSafetyCaseRequest"] = "PipelineSafetyCaseRequest"
 
+    def dependencies(self) -> list[str]:
+        return [self.method] + [x.entity for x in self.inputs]
+
 
 PipelineNodeRequest = Union[
     PipelineEvaluationRequest,
@@ -2814,16 +2975,25 @@ class PipelineNode(DyffSchemaBaseModel):
 
 class PipelineParameter(DyffSchemaBaseModel):
     """Declares a parameter that can be passed to the pipeline to customize its
-    behavior."""
+    behavior.
+
+    By default, the argument will be substituted for the placeholder value
+    ``$(keyword)``, where ``keyword`` is the ``.keyword`` property of the
+    parameter. In this case, the value of the argument must be a string.
+
+    If ``.destination`` is specified, the argument will instead overwrite
+    the value at the specified JSON path.
+    """
 
-    keyword: str = pydantic.Field(description="The keyword of the argument.")
-    destination: str = pydantic.Field(
+    keyword: str = pydantic.Field(description="The keyword of the parameter.")
+    destination: Optional[str] = pydantic.Field(
+        default=None,
         description="The field in a pipeline node to substitute with the"
-        " parameter value. Should be a string like 'node_name.field1.field2'."
+        " parameter value. Should be a string like 'node_name.field1.field2'.",
     )
     description: Optional[str] = pydantic.Field(
         default=None,
-        description="A description of the argument.",
+        description="A description of the parameter.",
         max_length=summary_maxlen(),
     )
 
@@ -2834,6 +3004,10 @@ class PipelineBase(DyffSchemaBaseModel):
         description="The nodes in the pipeline graph.",
         min_length=1,
     )
+    parameters: dict[str, PipelineParameter] = pydantic.Field(
+        default_factory=dict,
+        description="Input parameters used to customize the pipeline.",
+    )
 
     @pydantic.field_validator("nodes", mode="after")
     def validate_node_names_match(cls, nodes: dict[str, PipelineNode]):
@@ -2842,6 +3016,15 @@ class PipelineBase(DyffSchemaBaseModel):
                 raise ValueError(f"nodes[{k}]: dict key must match value.name")
         return nodes
 
+    @pydantic.field_validator("parameters", mode="after")
+    def validate_parameter_keywords_match(
+        cls, parameters: dict[str, PipelineParameter]
+    ):
+        for k, v in parameters.items():
+            if k != v.keyword:
+                raise ValueError(f"parameters[{k}]: dict key must match value.keyword")
+        return parameters
+
 
 class Pipeline(DyffEntity, PipelineBase):
     """A set of Dyff workflows that can be executed as a group.
@@ -3160,6 +3343,7 @@ _DyffEntityTypeRevisable = Union[
     PipelineRun,
     Report,
     SafetyCase,
+    Submission,
     Team,
     UseCase,
 ]
@@ -3239,6 +3423,7 @@ _ENTITY_CLASS: dict[Entities, type[DyffEntityType]] = {
     Entities.Report: Report,
     Entities.Revision: Revision,
     Entities.SafetyCase: SafetyCase,
+    Entities.Submission: Submission,
     Entities.Team: Team,
     Entities.UseCase: UseCase,
 }
@@ -3272,6 +3457,7 @@ DyffEntityT = TypeVar(
     Report,
     Revision,
     SafetyCase,
+    Submission,
     Team,
     UseCase,
 )
@@ -3303,7 +3489,6 @@ __all__ = [
     "Challenge",
     "ChallengeContent",
     "ChallengeContentPage",
-    "ChallengeSubmission",
     "ChallengeTask",
     "ChallengeTaskBase",
     "ChallengeTaskContent",
@@ -3315,6 +3500,9 @@ __all__ = [
     "ConcernBase",
     "Container",
     "ContainerImageSource",
+    "Curve",
+    "CurveData",
+    "CurveSpec",
     "DataSchema",
     "Dataset",
     "DatasetBase",
@@ -3440,6 +3628,9 @@ __all__ = [
     "ScoreMetadataRefs",
     "Status",
     "StorageSignedURL",
+    "Submission",
+    "SubmissionBase",
+    "SubmissionStructure",
     "TagName",
     "TagNameType",
     "TaskSchema",

{dyff_schema-0.38.5 → dyff_schema-0.39.0}/dyff/schema/v0/r1/requests.py

@@ -45,6 +45,7 @@ from .platform import (
     PipelineBase,
     PipelineRunBase,
     ReportBase,
+    SubmissionBase,
     TagNameType,
     TeamBase,
     summary_maxlen,
@@ -148,6 +149,10 @@ class ChallengeCreateRequest(DyffEntityCreateRequest):
     )
 
 
+class SubmissionCreateRequest(DyffEntityCreateRequest, SubmissionBase):
+    pass
+
+
 class ChallengeTaskCreateRequest(DyffEntityCreateRequest, ChallengeTaskBase):
     pass
 
@@ -398,6 +403,10 @@ class AuditQueryRequest(DyffEntityQueryRequest):
     name: Optional[str] = pydantic.Field(default=None)
 
 
+class ChallengeQueryRequest(DyffEntityQueryRequest):
+    pass
+
+
 class DatasetQueryRequest(DyffEntityQueryRequest):
     name: Optional[str] = pydantic.Field(default=None)
 
@@ -492,6 +501,7 @@ __all__ = [
     "AuditQueryRequest",
     "ChallengeContentEditRequest",
     "ChallengeCreateRequest",
+    "ChallengeQueryRequest",
     "ChallengeTaskCreateRequest",
     "ChallengeTaskRulesEditRequest",
     "ChallengeTeamCreateRequest",
@@ -531,6 +541,7 @@ __all__ = [
     "ReportQueryRequest",
     "SafetyCaseQueryRequest",
     "ScoreQueryRequest",
+    "SubmissionCreateRequest",
     "TeamEditRequest",
     "TeamQueryRequest",
     "UseCaseQueryRequest",

{dyff_schema-0.38.5 → dyff_schema-0.39.0/dyff_schema.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dyff-schema
-Version: 0.38.5
+Version: 0.39.0
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{dyff_schema-0.38.5 → dyff_schema-0.39.0}/dyff_schema.egg-info/SOURCES.txt

@@ -62,4 +62,5 @@ dyff_schema.egg-info/dependency_links.txt
 dyff_schema.egg-info/requires.txt
 dyff_schema.egg-info/top_level.txt
 tests/test_adapters.py
-tests/test_import.py
+tests/test_import.py
+tests/test_score_curves.py

dyff_schema-0.39.0/tests/test_score_curves.py

@@ -0,0 +1,139 @@
+# SPDX-FileCopyrightText: 2024 UL Research Institutes
+# SPDX-License-Identifier: Apache-2.0
+
+import pydantic
+import pytest
+
+from dyff.schema.platform import (
+    CurveData,
+    CurveSpec,
+    ScoreData,
+    ScoreMetadata,
+    ScoreMetadataRefs,
+    ScoreSpec,
+)
+
+
+def _dim_spec(name: str, title: str) -> ScoreSpec:
+    return ScoreSpec(
+        name=name,
+        title=title,
+        summary=f"{title} dimension",
+    )
+
+
+def _minimal_curve_spec() -> CurveSpec:
+    return CurveSpec(
+        name="roc",
+        title="ROC Curve",
+        summary="Receiver operating characteristic.",
+        dimensions={
+            "x": _dim_spec("x", "False Positive Rate"),
+            "y": _dim_spec("y", "True Positive Rate"),
+            "threshold": _dim_spec("threshold", "Threshold"),
+        },
+    )
+
+
+def _metadata(method_id: str = "method-123") -> ScoreMetadata:
+    return ScoreMetadata(refs=ScoreMetadataRefs(method=method_id))
+
+
+def test_curve_data_ok_equal_length_vectors_and_required_dimensions():
+    spec = _minimal_curve_spec()
+
+    cd = CurveData(
+        name=spec.name,
+        title=spec.title,
+        summary=spec.summary,
+        dimensions=spec.dimensions,
+        metadata=_metadata(),
+        analysis="analysis-abc",
+        points={
+            "x": [0.0, 0.5, 1.0],
+            "y": [0.0, 0.7, 1.0],
+            "threshold": [1.0, 0.5, 0.0],
+        },
+    )
+
+    assert len(cd.points["x"]) == 3
+    for k in spec.dimensions.keys():
+        assert k in cd.points
+
+
+def test_curve_data_raises_when_points_missing_required_dimension():
+    spec = _minimal_curve_spec()
+
+    with pytest.raises(pydantic.ValidationError):
+        CurveData(
+            name=spec.name,
+            title=spec.title,
+            summary=spec.summary,
+            dimensions=spec.dimensions,
+            metadata=_metadata(),
+            analysis="analysis-abc",
+            points={"x": [0.0, 1.0], "y": [0.0, 1.0]},
+        )
+
+
+def test_curve_data_raises_when_vector_lengths_differ():
+    spec = _minimal_curve_spec()
+
+    with pytest.raises(pydantic.ValidationError):
+        CurveData(
+            name=spec.name,
+            title=spec.title,
+            summary=spec.summary,
+            dimensions=spec.dimensions,
+            metadata=_metadata(),
+            analysis="analysis-abc",
+            points={
+                "x": [0.0, 0.5, 1.0],
+                "y": [0.0, 1.0],
+                "threshold": [1.0, 0.0, -1.0],
+            },
+        )
+
+
+def test_curve_data_raises_when_points_empty():
+    spec = _minimal_curve_spec()
+
+    with pytest.raises(pydantic.ValidationError):
+        CurveData(
+            name=spec.name,
+            title=spec.title,
+            summary=spec.summary,
+            dimensions=spec.dimensions,
+            metadata=_metadata(),
+            analysis="analysis-abc",
+            points={},
+        )
+
+
+def test_score_data_scalar_accepts_quantity():
+    sd_scalar = ScoreData(
+        name="auc",
+        title="AUC",
+        summary="Area under ROC",
+        metadata=_metadata(),
+        analysis="analysis-abc",
+        quantity=0.9123,
+        points=None,
+        quantityString="0.912",
+        text="Higher is better.",
+    )
+    assert sd_scalar.quantity == 0.9123
+
+    with pytest.raises(pydantic.ValidationError):
+        ScoreData(
+            scoreKind="scalar",
+            name="auc_bad",
+            title="AUC",
+            summary="Scalar must not carry points",
+            metadata=_metadata(),
+            analysis="analysis-abc",
+            quantity=0.9,
+            points={"x": [0.0, 1.0], "y": [0.0, 1.0]},  # forbidden
+            quantityString="0.900",
+            text="This should fail.",
+        )

dyff_schema-0.38.5/dyff/schema/_version.py

@@ -1,2 +0,0 @@
-__version__ = version = "0.38.5"
-__version_tuple__ = version_tuple = (0, 38, 5)