dyff-schema 0.38.3__py3-none-any.whl → 0.38.13__py3-none-any.whl

dyff/schema/_version.py

@@ -1,2 +1,2 @@
-__version__ = version = "0.38.3"
-__version_tuple__ = version_tuple = (0, 38, 3)
+__version__ = version = "0.38.13"
+__version_tuple__ = version_tuple = (0, 38, 13)

dyff/schema/v0/r1/platform.py

@@ -2656,6 +2656,18 @@ class ChallengeTaskContent(DyffSchemaBaseModel):
     )
 
 
+class SubmissionStructure(DyffSchemaBaseModel):
+    submissionKind: EntityKindLiteral = pydantic.Field(
+        default=Entities.InferenceService.value,
+        description="The kind of entity that you can submit to this task.",
+    )
+    pipelineKeyword: str = pydantic.Field(
+        default="submission",
+        description="The keyword parameter where the submitted entity ID"
+        " should be passed to the assessment Pipeline.",
+    )
+
+
 class ChallengeTaskBase(DyffSchemaBaseModel):
     """The part of the ChallengeTask spec that is not system-populated."""
 
@@ -2669,7 +2681,11 @@ class ChallengeTaskBase(DyffSchemaBaseModel):
         pattern=r"[a-z0-9-]*",
     )
     assessment: str = pydantic.Field(
-        description="ID of the Assessment that the task runs."
+        description="ID of the Pipeline used to assess the submission."
+    )
+    submissionStructure: SubmissionStructure = pydantic.Field(
+        default_factory=SubmissionStructure,
+        description="How to run the assessment pipeline on a new submission.",
     )
     rules: ChallengeTaskRules = pydantic.Field(description="The rules for submissions.")
     content: ChallengeTaskContent = pydantic.Field(
@@ -2729,7 +2745,14 @@ class Challenge(DyffEntity):
         return None
 
 
-class ChallengeSubmission(DyffEntity):
+class SubmissionBase(DyffSchemaBaseModel):
+    team: str = pydantic.Field(description="The ID of the team making the submission.")
+    submission: EntityIdentifier = pydantic.Field(
+        description="The resource being submitted for assessment."
+    )
+
+
+class Submission(DyffEntity, SubmissionBase):
     """A submission of an inference system to a challenge by a team.
 
     All of the constituent resources must already exist. Creating a Submission simply
@@ -2746,9 +2769,8 @@ class ChallengeSubmission(DyffEntity):
     task: str = pydantic.Field(
         description="The key of the task within the challenge being submitted to."
     )
-    team: str = pydantic.Field(description="The ID of the team making the submission.")
-    inferenceService: str = pydantic.Field(
-        description="The ID of the inference service being submitted."
+    pipelineRun: str = pydantic.Field(
+        description="The ID of the PipelineRun for this submission."
     )
 
     def dependencies(self) -> list[str]:
@@ -2765,16 +2787,35 @@ class ChallengeSubmission(DyffEntity):
 class PipelineEvaluationRequest(EvaluationRequestBase):
     kind: Literal["PipelineEvaluationRequest"] = "PipelineEvaluationRequest"
 
+    def dependencies(self) -> list[str]:
+        if self.inferenceSession is None:
+            raise AssertionError()
+        return [self.dataset, self.inferenceSession.inferenceService]
+
+    @pydantic.model_validator(mode="after")
+    def check_session_not_reference(self):
+        if self.inferenceSession is None or self.inferenceSessionReference is not None:
+            raise ValueError(
+                "evaluations in pipelines must set inferenceSession, not inferenceSessionReference"
+            )
+        return self
+
 
 class PipelineMeasurementRequest(AnalysisRequestBase):
     kind: Literal["PipelineMeasurementRequest"] = "PipelineMeasurementRequest"
 
+    def dependencies(self) -> list[str]:
+        return [self.method] + [x.entity for x in self.inputs]
+
 
 class PipelineSafetyCaseRequest(AnalysisRequestBase):
     kind: Literal["PipelineSafetyCaseRequest"] = "PipelineSafetyCaseRequest"
 
+    def dependencies(self) -> list[str]:
+        return [self.method] + [x.entity for x in self.inputs]
+
 
-PipelineNodeRequest: TypeAlias = Union[
+PipelineNodeRequest = Union[
     PipelineEvaluationRequest,
     PipelineMeasurementRequest,
     PipelineSafetyCaseRequest,
@@ -2793,8 +2834,7 @@ class PipelineNode(DyffSchemaBaseModel):
         description="The name of the node. Must be unique in the context of the pipeline."
     )
 
-    request: PipelineNodeRequest = pydantic.Field(  # type: ignore[valid-type]
-        discriminator="kind",
+    request: PipelineNodeRequest = pydantic.Field(
         description="The request template that will be executed when this node"
         ' executes. You can use the syntax "$(node_name)" in request fields'
         " that reference another entity to indicate that the request depends"
@@ -2803,19 +2843,37 @@ class PipelineNode(DyffSchemaBaseModel):
         " dependency graph structure from these placeholders.",
     )
 
+    # The .kind field is dropped when we serialize nodes as part of a
+    # PipelineCreateRequest because of the default exclude_unset=True
+    # behavior for requests, so we add it back.
+    @pydantic.field_serializer("request")
+    def _serialize_request(self, request: PipelineNodeRequest, _info):
+        data = request.model_dump(mode=_info.mode)
+        data["kind"] = request.kind
+        return data
+
 
 class PipelineParameter(DyffSchemaBaseModel):
     """Declares a parameter that can be passed to the pipeline to customize its
-    behavior."""
+    behavior.
+
+    By default, the argument will be substituted for the placeholder value
+    ``$(keyword)``, where ``keyword`` is the ``.keyword`` property of the
+    parameter. In this case, the value of the argument must be a string.
 
-    keyword: str = pydantic.Field(description="The keyword of the argument.")
-    destination: str = pydantic.Field(
+    If ``.destination`` is specified, the argument will instead overwrite
+    the value at the specified JSON path.
+    """
+
+    keyword: str = pydantic.Field(description="The keyword of the parameter.")
+    destination: Optional[str] = pydantic.Field(
+        default=None,
         description="The field in a pipeline node to substitute with the"
-        " parameter value. Should be a string like 'node_name.field1.field2'."
+        " parameter value. Should be a string like 'node_name.field1.field2'.",
     )
     description: Optional[str] = pydantic.Field(
         default=None,
-        description="A description of the argument.",
+        description="A description of the parameter.",
         max_length=summary_maxlen(),
     )
 
@@ -2826,6 +2884,10 @@ class PipelineBase(DyffSchemaBaseModel):
         description="The nodes in the pipeline graph.",
         min_length=1,
     )
+    parameters: dict[str, PipelineParameter] = pydantic.Field(
+        default_factory=dict,
+        description="Input parameters used to customize the pipeline.",
+    )
 
     @pydantic.field_validator("nodes", mode="after")
     def validate_node_names_match(cls, nodes: dict[str, PipelineNode]):
@@ -2834,6 +2896,15 @@ class PipelineBase(DyffSchemaBaseModel):
                 raise ValueError(f"nodes[{k}]: dict key must match value.name")
         return nodes
 
+    @pydantic.field_validator("parameters", mode="after")
+    def validate_parameter_keywords_match(
+        cls, parameters: dict[str, PipelineParameter]
+    ):
+        for k, v in parameters.items():
+            if k != v.keyword:
+                raise ValueError(f"parameters[{k}]: dict key must match value.keyword")
+        return parameters
+
 
 class Pipeline(DyffEntity, PipelineBase):
     """A set of Dyff workflows that can be executed as a group.
@@ -3152,6 +3223,7 @@ _DyffEntityTypeRevisable = Union[
     PipelineRun,
     Report,
     SafetyCase,
+    Submission,
     Team,
     UseCase,
 ]
@@ -3231,6 +3303,7 @@ _ENTITY_CLASS: dict[Entities, type[DyffEntityType]] = {
     Entities.Report: Report,
     Entities.Revision: Revision,
     Entities.SafetyCase: SafetyCase,
+    Entities.Submission: Submission,
     Entities.Team: Team,
     Entities.UseCase: UseCase,
 }
@@ -3264,6 +3337,7 @@ DyffEntityT = TypeVar(
     Report,
     Revision,
     SafetyCase,
+    Submission,
     Team,
     UseCase,
 )
@@ -3295,7 +3369,6 @@ __all__ = [
     "Challenge",
     "ChallengeContent",
     "ChallengeContentPage",
-    "ChallengeSubmission",
     "ChallengeTask",
     "ChallengeTaskBase",
     "ChallengeTaskContent",
@@ -3432,6 +3505,9 @@ __all__ = [
     "ScoreMetadataRefs",
     "Status",
     "StorageSignedURL",
+    "Submission",
+    "SubmissionBase",
+    "SubmissionStructure",
     "TagName",
     "TagNameType",
     "TaskSchema",

dyff/schema/v0/r1/requests.py

@@ -45,6 +45,7 @@ from .platform import (
     PipelineBase,
     PipelineRunBase,
     ReportBase,
+    SubmissionBase,
     TagNameType,
     TeamBase,
     summary_maxlen,
@@ -148,6 +149,10 @@ class ChallengeCreateRequest(DyffEntityCreateRequest):
     )
 
 
+class SubmissionCreateRequest(DyffEntityCreateRequest, SubmissionBase):
+    pass
+
+
 class ChallengeTaskCreateRequest(DyffEntityCreateRequest, ChallengeTaskBase):
     pass
 
@@ -398,6 +403,10 @@ class AuditQueryRequest(DyffEntityQueryRequest):
     name: Optional[str] = pydantic.Field(default=None)
 
 
+class ChallengeQueryRequest(DyffEntityQueryRequest):
+    pass
+
+
 class DatasetQueryRequest(DyffEntityQueryRequest):
     name: Optional[str] = pydantic.Field(default=None)
 
@@ -492,6 +501,7 @@ __all__ = [
     "AuditQueryRequest",
     "ChallengeContentEditRequest",
     "ChallengeCreateRequest",
+    "ChallengeQueryRequest",
     "ChallengeTaskCreateRequest",
     "ChallengeTaskRulesEditRequest",
     "ChallengeTeamCreateRequest",
@@ -531,6 +541,7 @@ __all__ = [
     "ReportQueryRequest",
     "SafetyCaseQueryRequest",
     "ScoreQueryRequest",
+    "SubmissionCreateRequest",
     "TeamEditRequest",
     "TeamQueryRequest",
     "UseCaseQueryRequest",

{dyff_schema-0.38.3.dist-info → dyff_schema-0.38.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dyff-schema
-Version: 0.38.3
+Version: 0.38.13
 Summary: Data models for the Dyff AI auditing platform.
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{dyff_schema-0.38.3.dist-info → dyff_schema-0.38.13.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dyff/schema/__init__.py,sha256=w7OWDFuyGKd6xt_yllNtKzHahPgywrfU4Ue02psYaMA,2244
-dyff/schema/_version.py,sha256=-CCWFr62vN0w7SrCBH-IeTHMoX8ScRHffpLPQafa5pY,80
+dyff/schema/_version.py,sha256=50V94fT_56OCMvK0CeEH1SwgrRmZ0di-SpcyDei7ROA,82
 dyff/schema/adapters.py,sha256=YMTHv_2VlLGFp-Kqwa6H51hjffHmk8gXjZilHysIF5Q,123
 dyff/schema/annotations.py,sha256=nE6Jk1PLqlShj8uqjE_EzZC9zYnTDW5AVtQcjysiK8M,10018
 dyff/schema/base.py,sha256=jvaNtsSZyFfsdUZTcY_U-yfLY5_GyrMxSXhON2R9XR0,119
@@ -29,8 +29,8 @@ dyff/schema/v0/r1/adapters.py,sha256=hpwCSW8lkMkUKCLe0zaMUDu-VS_caSxJvPsECEi_XRA
 dyff/schema/v0/r1/base.py,sha256=1VJXVLKldOq3aH2HdVgxXXk4DJTZEIiaTa4GzMKzziU,20228
 dyff/schema/v0/r1/commands.py,sha256=jqbEvDRLpcVGWXqlbniuSKg3UIjlrpKt6_0hiwUpPfQ,20153
 dyff/schema/v0/r1/oci.py,sha256=YjHDVBJ2IIxqijll70OK6pM-qT6pq8tvU7D3YB9vGM0,6700
-dyff/schema/v0/r1/platform.py,sha256=bQO3gQ6G-7Cl_FHHNLW1ozAG-ShYPOTIf-NfB6uvnps,110880
-dyff/schema/v0/r1/requests.py,sha256=JMpdx9WBlUpePshacG85XMmlGPBkkaK5453VaK3kH9s,18321
+dyff/schema/v0/r1/platform.py,sha256=x6-JVK_gw1io-8A89zJfH_3pOu9qZxactsuhFrNNQmU,113785
+dyff/schema/v0/r1/requests.py,sha256=OhkM9QpSimsGaX8TtjSHcWlHg1UYyPCiq-WW9DJ7FdQ,18548
 dyff/schema/v0/r1/responses.py,sha256=nxy7FPtfw2B_bljz5UGGuSE79HTkDQxKH56AJVmd4Qo,1287
 dyff/schema/v0/r1/test.py,sha256=X6dUyVd5svcPCI-PBMOAqEfK9jv3bRDvkQTJzwS96c0,10720
 dyff/schema/v0/r1/version.py,sha256=NONebgcv5Thsw_ymud6PacZdGjV6ndBrmLnap-obcpo,428
@@ -43,9 +43,9 @@ dyff/schema/v0/r1/dataset/text.py,sha256=MYG5seGODDryRSCy-g0Unh5dD0HCytmZ3FeElC-
 dyff/schema/v0/r1/dataset/vision.py,sha256=tJFF4dkhHX0UXTj1sPW-G22xTSI40gbYO465FuvmvAU,443
 dyff/schema/v0/r1/io/__init__.py,sha256=L5y8UhRnojerPYHumsxQJRcHCNz8Hj9NM8b47mewMNs,92
 dyff/schema/v0/r1/io/vllm.py,sha256=vWyLg-susbg0JDfv6VExBpgFdU2GHP2a14ChOdbckvs,5321
-dyff_schema-0.38.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dyff_schema-0.38.3.dist-info/licenses/NOTICE,sha256=YONACu0s_Ui6jNi-wtEsVQbTU1JIkh8wvLH6d1-Ni_w,43
-dyff_schema-0.38.3.dist-info/METADATA,sha256=LRDYIEaUA4AMUGCTGCLb1ABwrCh6rT28AU8ywiYU60o,3734
-dyff_schema-0.38.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dyff_schema-0.38.3.dist-info/top_level.txt,sha256=9e3VVdeX73t_sUJOPQPCcGtYO1JhoErhHIi3WoWGcFI,5
-dyff_schema-0.38.3.dist-info/RECORD,,
+dyff_schema-0.38.13.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dyff_schema-0.38.13.dist-info/licenses/NOTICE,sha256=YONACu0s_Ui6jNi-wtEsVQbTU1JIkh8wvLH6d1-Ni_w,43
+dyff_schema-0.38.13.dist-info/METADATA,sha256=0_GHYLYCJl_ivN-40TZwmtwey-qsgf63pHQgIqBaOf0,3735
+dyff_schema-0.38.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dyff_schema-0.38.13.dist-info/top_level.txt,sha256=9e3VVdeX73t_sUJOPQPCcGtYO1JhoErhHIi3WoWGcFI,5
+dyff_schema-0.38.13.dist-info/RECORD,,