dyff-schema 0.29.0__py3-none-any.whl → 0.30.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dyff-schema might be problematic. Click here for more details.
- dyff/schema/__init__.py +2 -2
- dyff/schema/_version.py +2 -0
- dyff/schema/copydoc.py +1 -1
- dyff/schema/v0/r1/adapters.py +2 -2
- dyff/schema/v0/r1/base.py +167 -249
- dyff/schema/v0/r1/commands.py +11 -25
- dyff/schema/v0/r1/dataset/arrow.py +39 -14
- dyff/schema/v0/r1/dataset/embedding.py +10 -3
- dyff/schema/v0/r1/dataset/text.py +9 -5
- dyff/schema/v0/r1/io/vllm.py +1 -1
- dyff/schema/v0/r1/platform.py +53 -40
- dyff/schema/v0/r1/requests.py +22 -18
- dyff/schema/v0/r1/version.py +2 -1
- {dyff_schema-0.29.0.dist-info → dyff_schema-0.30.1.dist-info}/METADATA +2 -2
- {dyff_schema-0.29.0.dist-info → dyff_schema-0.30.1.dist-info}/RECORD +19 -18
- {dyff_schema-0.29.0.dist-info → dyff_schema-0.30.1.dist-info}/WHEEL +0 -0
- {dyff_schema-0.29.0.dist-info → dyff_schema-0.30.1.dist-info}/licenses/LICENSE +0 -0
- {dyff_schema-0.29.0.dist-info → dyff_schema-0.30.1.dist-info}/licenses/NOTICE +0 -0
- {dyff_schema-0.29.0.dist-info → dyff_schema-0.30.1.dist-info}/top_level.txt +0 -0
dyff/schema/v0/r1/commands.py
CHANGED
|
@@ -10,8 +10,10 @@ from __future__ import annotations
|
|
|
10
10
|
from typing import Literal, Optional, Union
|
|
11
11
|
|
|
12
12
|
import pydantic
|
|
13
|
+
from pydantic import StringConstraints
|
|
14
|
+
from typing_extensions import Annotated
|
|
13
15
|
|
|
14
|
-
from .base import DyffSchemaBaseModel, JsonMergePatchSemantics
|
|
16
|
+
from .base import DyffSchemaBaseModel, JsonMergePatchSemantics
|
|
15
17
|
from .platform import (
|
|
16
18
|
DyffEntityType,
|
|
17
19
|
EntityIdentifier,
|
|
@@ -26,20 +28,6 @@ from .platform import (
|
|
|
26
28
|
title_maxlen,
|
|
27
29
|
)
|
|
28
30
|
|
|
29
|
-
# ----------------------------------------------------------------------------
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# class _NoneMeansUndefined(DyffSchemaBaseModel):
|
|
33
|
-
# """Fields with the value None will not be emitted in the JSON output."""
|
|
34
|
-
|
|
35
|
-
# # TODO: (DYFF-223) This should perhaps be the default for all schema
|
|
36
|
-
# # objects.
|
|
37
|
-
# def dict(self, *, exclude_none=True, **kwargs) -> _ModelAsDict:
|
|
38
|
-
# return super().dict(exclude_none=True, **kwargs)
|
|
39
|
-
|
|
40
|
-
# def json(self, *, exclude_none=True, **kwargs) -> str:
|
|
41
|
-
# return super().json(exclude_none=True, **kwargs)
|
|
42
|
-
|
|
43
31
|
|
|
44
32
|
class FamilyIdentifier(EntityIdentifier):
|
|
45
33
|
"""Identifies a single Family entity."""
|
|
@@ -87,7 +75,7 @@ class EditEntityDocumentationPatch(JsonMergePatchSemantics):
|
|
|
87
75
|
Fields that are assigned explicitly remain unchanged.
|
|
88
76
|
"""
|
|
89
77
|
|
|
90
|
-
title: Optional[
|
|
78
|
+
title: Optional[Annotated[str, StringConstraints(max_length=title_maxlen())]] = ( # type: ignore
|
|
91
79
|
pydantic.Field(
|
|
92
80
|
default=None,
|
|
93
81
|
description='A short plain string suitable as a title or "headline".'
|
|
@@ -95,7 +83,7 @@ class EditEntityDocumentationPatch(JsonMergePatchSemantics):
|
|
|
95
83
|
)
|
|
96
84
|
)
|
|
97
85
|
|
|
98
|
-
summary: Optional[
|
|
86
|
+
summary: Optional[Annotated[str, StringConstraints(max_length=summary_maxlen())]] = ( # type: ignore
|
|
99
87
|
pydantic.Field(
|
|
100
88
|
default=None,
|
|
101
89
|
description="A brief summary, suitable for display in"
|
|
@@ -104,7 +92,7 @@ class EditEntityDocumentationPatch(JsonMergePatchSemantics):
|
|
|
104
92
|
)
|
|
105
93
|
)
|
|
106
94
|
|
|
107
|
-
fullPage: Optional[
|
|
95
|
+
fullPage: Optional[str] = pydantic.Field(
|
|
108
96
|
default=None,
|
|
109
97
|
description="Long-form documentation. Interpreted as"
|
|
110
98
|
" Markdown. There are no length constraints, but be reasonable."
|
|
@@ -146,7 +134,7 @@ class EditEntityDocumentation(Command):
|
|
|
146
134
|
class EditEntityLabelsAttributes(JsonMergePatchSemantics):
|
|
147
135
|
"""Attributes for the EditEntityLabels command."""
|
|
148
136
|
|
|
149
|
-
labels: dict[LabelKeyType, Optional[
|
|
137
|
+
labels: dict[LabelKeyType, Optional[LabelValueType]] = pydantic.Field(
|
|
150
138
|
default_factory=dict,
|
|
151
139
|
description="A set of key-value labels for the resource."
|
|
152
140
|
" Existing label keys that are not provided in the edit remain unchanged."
|
|
@@ -180,7 +168,7 @@ class EditEntityLabels(Command):
|
|
|
180
168
|
class EditFamilyMembersAttributes(JsonMergePatchSemantics):
|
|
181
169
|
"""Attributes for the EditFamilyMembers command."""
|
|
182
170
|
|
|
183
|
-
members: dict[TagNameType, Optional[
|
|
171
|
+
members: dict[TagNameType, Optional[FamilyMember]] = pydantic.Field(
|
|
184
172
|
description="Mapping of names to IDs of member resources.",
|
|
185
173
|
)
|
|
186
174
|
|
|
@@ -259,12 +247,10 @@ class RestoreEntity(Command):
|
|
|
259
247
|
class UpdateEntityStatusAttributes(JsonMergePatchSemantics):
|
|
260
248
|
"""Attributes for the UpdateEntityStatus command."""
|
|
261
249
|
|
|
262
|
-
status: str = pydantic.Field(
|
|
263
|
-
description=Status.__fields__["status"].field_info.description
|
|
264
|
-
)
|
|
250
|
+
status: str = pydantic.Field(description=Status.model_fields["status"].description)
|
|
265
251
|
|
|
266
|
-
reason: Optional[
|
|
267
|
-
description=Status.
|
|
252
|
+
reason: Optional[str] = pydantic.Field(
|
|
253
|
+
description=Status.model_fields["reason"].description
|
|
268
254
|
)
|
|
269
255
|
|
|
270
256
|
|
|
@@ -13,6 +13,7 @@ from typing import Any, Iterable, Literal, Optional
|
|
|
13
13
|
import pyarrow
|
|
14
14
|
import pyarrow.dataset
|
|
15
15
|
import pydantic
|
|
16
|
+
from pydantic.fields import FieldInfo
|
|
16
17
|
|
|
17
18
|
from ..base import DType
|
|
18
19
|
from . import binary
|
|
@@ -31,7 +32,9 @@ def arrow_schema(
|
|
|
31
32
|
We support a very basic subset of pydantic model features currently. The intention
|
|
32
33
|
is to expand this.
|
|
33
34
|
"""
|
|
34
|
-
arrow_fields = [
|
|
35
|
+
arrow_fields = [
|
|
36
|
+
arrow_field(name, field) for name, field in model_type.model_fields.items()
|
|
37
|
+
]
|
|
35
38
|
return pyarrow.schema(arrow_fields, metadata=metadata)
|
|
36
39
|
|
|
37
40
|
|
|
@@ -122,15 +125,37 @@ def arrow_type(annotation: type) -> pyarrow.DataType:
|
|
|
122
125
|
|
|
123
126
|
if issubclass(annotation, pydantic.BaseModel):
|
|
124
127
|
subfields = []
|
|
125
|
-
for _name, subfield in annotation.
|
|
126
|
-
subfields.append(arrow_field(subfield))
|
|
128
|
+
for _name, subfield in annotation.model_fields.items():
|
|
129
|
+
subfields.append(arrow_field(_name, subfield))
|
|
127
130
|
return pyarrow.struct(subfields)
|
|
128
131
|
|
|
129
|
-
#
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
132
|
+
# Handle numpy-like types
|
|
133
|
+
if hasattr(annotation, "dtype"):
|
|
134
|
+
return pyarrow.from_numpy_dtype(annotation.dtype)
|
|
135
|
+
|
|
136
|
+
# Handle Annotated list types (e.g., Annotated[list[str], Field(max_length=10)])
|
|
137
|
+
# This covers lists created by our list_() function in base.py which returns
|
|
138
|
+
# Annotated types with Field metadata for length constraints.
|
|
139
|
+
#
|
|
140
|
+
# We need custom logic here because:
|
|
141
|
+
# 1. Standard typing.List doesn't carry Pydantic Field constraints
|
|
142
|
+
# 2. Our list_() function wraps list[T] in Annotated[list[T], Field(...)]
|
|
143
|
+
# to embed validation metadata (min/max length) at the type level
|
|
144
|
+
# 3. PyArrow needs to know these constraints upfront to create proper schemas
|
|
145
|
+
# 4. The nested generic structure requires careful extraction:
|
|
146
|
+
# Annotated[list[str], Field(max_length=10)] needs to become
|
|
147
|
+
# pyarrow.list_(pyarrow.string(), 10)
|
|
148
|
+
if (
|
|
149
|
+
typing.get_origin(annotation) is typing.Annotated
|
|
150
|
+
and typing.get_args(annotation)[0] is list
|
|
151
|
+
):
|
|
152
|
+
metadata = typing.get_args(annotation)[1:]
|
|
153
|
+
item_type = typing.get_args(typing.get_args(annotation)[0])[0]
|
|
154
|
+
max_length = -1
|
|
155
|
+
for meta in metadata:
|
|
156
|
+
if isinstance(meta, FieldInfo):
|
|
157
|
+
max_length = getattr(meta, "max_length", -1)
|
|
158
|
+
return pyarrow.list_(arrow_type(item_type), max_length)
|
|
134
159
|
|
|
135
160
|
if issubclass(annotation, DType):
|
|
136
161
|
# The dtype is in the metaclass
|
|
@@ -138,7 +163,7 @@ def arrow_type(annotation: type) -> pyarrow.DataType:
|
|
|
138
163
|
|
|
139
164
|
if annotation == bool:
|
|
140
165
|
return pyarrow.bool_()
|
|
141
|
-
if annotation == bytes
|
|
166
|
+
if annotation == bytes:
|
|
142
167
|
return pyarrow.binary()
|
|
143
168
|
if annotation == float:
|
|
144
169
|
return pyarrow.float64()
|
|
@@ -147,13 +172,13 @@ def arrow_type(annotation: type) -> pyarrow.DataType:
|
|
|
147
172
|
if annotation == uuid.UUID:
|
|
148
173
|
return pyarrow.binary(16)
|
|
149
174
|
|
|
150
|
-
if annotation == str
|
|
175
|
+
if annotation == str:
|
|
151
176
|
return pyarrow.string()
|
|
152
177
|
|
|
153
178
|
raise NotImplementedError(f"Python type {annotation}")
|
|
154
179
|
|
|
155
180
|
|
|
156
|
-
def arrow_field(
|
|
181
|
+
def arrow_field(field_name: str, field_info: pydantic.fields.FieldInfo):
|
|
157
182
|
"""Create a named ``pyarrow.Field`` from a pydantic model ``ModelField``.
|
|
158
183
|
|
|
159
184
|
If present, the ``.alias`` property of the ``ModelField`` takes precedence
|
|
@@ -161,10 +186,10 @@ def arrow_field(pydantic_field: pydantic.fields.ModelField):
|
|
|
161
186
|
function. The ``.description`` property, if present, becomes the docstring
|
|
162
187
|
for the arrow field.
|
|
163
188
|
"""
|
|
164
|
-
name =
|
|
165
|
-
docstring =
|
|
189
|
+
name = field_info.alias if field_info.alias else field_name
|
|
190
|
+
docstring = field_info.description
|
|
166
191
|
return field_with_docstring(
|
|
167
|
-
name, arrow_type(
|
|
192
|
+
name, arrow_type(field_info.annotation or str), docstring=docstring
|
|
168
193
|
)
|
|
169
194
|
|
|
170
195
|
|
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2024 UL Research Institutes
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
from typing import Type
|
|
4
|
+
from typing import Type, Union
|
|
5
5
|
|
|
6
6
|
import pydantic
|
|
7
|
+
from typing_extensions import Annotated
|
|
7
8
|
|
|
8
|
-
from ..base import DyffSchemaBaseModel,
|
|
9
|
+
from ..base import DyffSchemaBaseModel, float32, float64, list_
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
def embedding(
|
|
12
|
-
element_type: Type[
|
|
13
|
+
element_type: Type[
|
|
14
|
+
Union[
|
|
15
|
+
Annotated[float, float32()],
|
|
16
|
+
Annotated[float, float64()],
|
|
17
|
+
]
|
|
18
|
+
],
|
|
19
|
+
size: int,
|
|
13
20
|
) -> Type[DyffSchemaBaseModel]:
|
|
14
21
|
"""Returns a schema type representing a list of fixed-length embedding vectors."""
|
|
15
22
|
|
|
@@ -3,17 +3,21 @@
|
|
|
3
3
|
|
|
4
4
|
import pydantic
|
|
5
5
|
|
|
6
|
-
from ..base import DyffSchemaBaseModel
|
|
6
|
+
from ..base import DyffSchemaBaseModel
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class TaggedSpan(DyffSchemaBaseModel):
|
|
10
10
|
"""A contiguous subsequence of text with a corresponding tag."""
|
|
11
11
|
|
|
12
|
-
start:
|
|
13
|
-
description="The index of the first character in the span"
|
|
12
|
+
start: int = pydantic.Field(
|
|
13
|
+
description="The index of the first character in the span",
|
|
14
|
+
ge=0, # Text indices should be non-negative
|
|
15
|
+
json_schema_extra={"dyff.io/dtype": "int64"},
|
|
14
16
|
)
|
|
15
|
-
end:
|
|
16
|
-
description="The index one past the final character in the span"
|
|
17
|
+
end: int = pydantic.Field(
|
|
18
|
+
description="The index one past the final character in the span",
|
|
19
|
+
ge=0, # Text indices should be non-negative
|
|
20
|
+
json_schema_extra={"dyff.io/dtype": "int64"},
|
|
17
21
|
)
|
|
18
22
|
tag: str = pydantic.Field(description="The tag of the span")
|
|
19
23
|
|
dyff/schema/v0/r1/io/vllm.py
CHANGED
|
@@ -119,6 +119,6 @@ class GenerateEndpointOutput(DyffSchemaBaseModel):
|
|
|
119
119
|
text: list[str] = pydantic.Field(
|
|
120
120
|
# TODO: hypothesis plugin doesn't respect constraints
|
|
121
121
|
# See: https://github.com/pydantic/pydantic/issues/2875
|
|
122
|
-
#
|
|
122
|
+
# min_length=1,
|
|
123
123
|
description="List of generated responses. The prompt is prepended to each response.",
|
|
124
124
|
)
|
dyff/schema/v0/r1/platform.py
CHANGED
|
@@ -28,7 +28,8 @@ from typing import Any, Literal, NamedTuple, Optional, Type, Union
|
|
|
28
28
|
|
|
29
29
|
import pyarrow
|
|
30
30
|
import pydantic
|
|
31
|
-
from
|
|
31
|
+
from pydantic import StringConstraints
|
|
32
|
+
from typing_extensions import Annotated, TypeAlias
|
|
32
33
|
|
|
33
34
|
from ... import named_data_schema, product_schema
|
|
34
35
|
from ...version import SomeSchemaVersion
|
|
@@ -283,7 +284,7 @@ EntityKindLiteral = Literal[
|
|
|
283
284
|
]
|
|
284
285
|
|
|
285
286
|
|
|
286
|
-
EntityID: TypeAlias =
|
|
287
|
+
EntityID: TypeAlias = Annotated[str, StringConstraints(pattern=entity_id_regex())] # type: ignore
|
|
287
288
|
|
|
288
289
|
|
|
289
290
|
class DyffModelWithID(DyffSchemaBaseModel):
|
|
@@ -306,21 +307,30 @@ class EntityIdentifier(DyffSchemaBaseModel):
|
|
|
306
307
|
|
|
307
308
|
|
|
308
309
|
def LabelKey() -> type[str]:
|
|
309
|
-
return
|
|
310
|
-
|
|
311
|
-
|
|
310
|
+
return Annotated[
|
|
311
|
+
str,
|
|
312
|
+
StringConstraints(
|
|
313
|
+
pattern=_k8s_label_key_regex(), max_length=_k8s_label_key_maxlen()
|
|
314
|
+
),
|
|
315
|
+
] # type: ignore [return-value]
|
|
312
316
|
|
|
313
317
|
|
|
314
318
|
def LabelValue() -> type[str]:
|
|
315
|
-
return
|
|
316
|
-
|
|
317
|
-
|
|
319
|
+
return Annotated[
|
|
320
|
+
str,
|
|
321
|
+
StringConstraints(
|
|
322
|
+
pattern=_k8s_label_value_regex(), max_length=_k8s_label_value_maxlen()
|
|
323
|
+
),
|
|
324
|
+
] # type: ignore [return-value]
|
|
318
325
|
|
|
319
326
|
|
|
320
327
|
def TagName() -> type[str]:
|
|
321
|
-
return
|
|
322
|
-
|
|
323
|
-
|
|
328
|
+
return Annotated[
|
|
329
|
+
str,
|
|
330
|
+
StringConstraints(
|
|
331
|
+
pattern=_oci_image_tag_regex(), max_length=_oci_image_tag_maxlen()
|
|
332
|
+
),
|
|
333
|
+
] # type: ignore [return-value]
|
|
324
334
|
|
|
325
335
|
|
|
326
336
|
LabelKeyType: TypeAlias = LabelKey() # type: ignore
|
|
@@ -364,12 +374,14 @@ class Labeled(DyffSchemaBaseModel):
|
|
|
364
374
|
" '.', '-', or '_'.\n\n"
|
|
365
375
|
"We follow the kubernetes label conventions closely."
|
|
366
376
|
" See: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels",
|
|
377
|
+
# Forbid entries that don't match the key patternProperties
|
|
378
|
+
json_schema_extra={"additionalProperties": False},
|
|
367
379
|
)
|
|
368
380
|
|
|
369
381
|
|
|
370
382
|
class Annotation(DyffSchemaBaseModel):
|
|
371
383
|
key: str = pydantic.Field(
|
|
372
|
-
|
|
384
|
+
pattern=_k8s_label_key_regex(),
|
|
373
385
|
max_length=_k8s_domain_maxlen(),
|
|
374
386
|
description="The annotation key. A DNS label with an optional DNS domain prefix."
|
|
375
387
|
" For example: 'my-key', 'your.com/key_0'. Names prefixed with"
|
|
@@ -383,7 +395,7 @@ class Annotation(DyffSchemaBaseModel):
|
|
|
383
395
|
)
|
|
384
396
|
|
|
385
397
|
|
|
386
|
-
Quantity: TypeAlias =
|
|
398
|
+
Quantity: TypeAlias = Annotated[str, StringConstraints(pattern=_k8s_quantity_regex())] # type: ignore
|
|
387
399
|
|
|
388
400
|
|
|
389
401
|
class ServiceClass(str, enum.Enum):
|
|
@@ -609,10 +621,10 @@ class AccessGrant(DyffSchemaBaseModel):
|
|
|
609
621
|
"""
|
|
610
622
|
|
|
611
623
|
resources: list[Resources] = pydantic.Field(
|
|
612
|
-
|
|
624
|
+
min_length=1, description="List of resource types to which the grant applies"
|
|
613
625
|
)
|
|
614
626
|
functions: list[APIFunctions] = pydantic.Field(
|
|
615
|
-
|
|
627
|
+
min_length=1,
|
|
616
628
|
description="List of functions on those resources to which the grant applies",
|
|
617
629
|
)
|
|
618
630
|
accounts: list[str] = pydantic.Field(
|
|
@@ -700,7 +712,7 @@ class FamilyMemberBase(DyffSchemaBaseModel):
|
|
|
700
712
|
description="ID of the resource this member references.",
|
|
701
713
|
)
|
|
702
714
|
|
|
703
|
-
description: Optional[
|
|
715
|
+
description: Optional[Annotated[str, StringConstraints(max_length=summary_maxlen())]] = pydantic.Field( # type: ignore
|
|
704
716
|
default=None,
|
|
705
717
|
description="A short description of the member."
|
|
706
718
|
" This should describe how this version of the resource"
|
|
@@ -719,7 +731,7 @@ class FamilyMember(FamilyMemberBase):
|
|
|
719
731
|
)
|
|
720
732
|
|
|
721
733
|
creationTime: datetime = pydantic.Field(
|
|
722
|
-
|
|
734
|
+
description="Tag creation time (assigned by system)"
|
|
723
735
|
)
|
|
724
736
|
|
|
725
737
|
|
|
@@ -916,12 +928,13 @@ class ExtractorStep(DyffSchemaBaseModel):
|
|
|
916
928
|
|
|
917
929
|
class DyffDataSchema(DyffSchemaBaseModel):
|
|
918
930
|
components: list[str] = pydantic.Field(
|
|
919
|
-
|
|
931
|
+
min_length=1,
|
|
920
932
|
description="A list of named dyff data schemas. The final schema is"
|
|
921
933
|
" the composition of these component schemas.",
|
|
922
934
|
)
|
|
923
935
|
schemaVersion: SomeSchemaVersion = pydantic.Field(
|
|
924
|
-
default=SCHEMA_VERSION,
|
|
936
|
+
default=SCHEMA_VERSION, # type: ignore [arg-type]
|
|
937
|
+
description="The dyff schema version",
|
|
925
938
|
)
|
|
926
939
|
|
|
927
940
|
def model_type(self) -> Type[DyffSchemaBaseModel]:
|
|
@@ -947,7 +960,7 @@ class DataSchema(DyffSchemaBaseModel):
|
|
|
947
960
|
@staticmethod
|
|
948
961
|
def from_model(model: Type[DyffSchemaBaseModel]) -> "DataSchema":
|
|
949
962
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(model))
|
|
950
|
-
jsonSchema = model.
|
|
963
|
+
jsonSchema = model.model_json_schema()
|
|
951
964
|
return DataSchema(arrowSchema=arrowSchema, jsonSchema=jsonSchema)
|
|
952
965
|
|
|
953
966
|
@staticmethod
|
|
@@ -966,14 +979,14 @@ class DataSchema(DyffSchemaBaseModel):
|
|
|
966
979
|
elif isinstance(schema, DyffDataSchema):
|
|
967
980
|
item_model = make_item_type(schema.model_type())
|
|
968
981
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(item_model))
|
|
969
|
-
jsonSchema = item_model.
|
|
982
|
+
jsonSchema = item_model.model_json_schema()
|
|
970
983
|
return DataSchema(
|
|
971
984
|
arrowSchema=arrowSchema, dyffSchema=schema, jsonSchema=jsonSchema
|
|
972
985
|
)
|
|
973
986
|
else:
|
|
974
987
|
item_model = make_item_type(schema)
|
|
975
988
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(item_model))
|
|
976
|
-
jsonSchema = item_model.
|
|
989
|
+
jsonSchema = item_model.model_json_schema()
|
|
977
990
|
return DataSchema(arrowSchema=arrowSchema, jsonSchema=jsonSchema)
|
|
978
991
|
|
|
979
992
|
@staticmethod
|
|
@@ -992,14 +1005,14 @@ class DataSchema(DyffSchemaBaseModel):
|
|
|
992
1005
|
elif isinstance(schema, DyffDataSchema):
|
|
993
1006
|
response_model = make_response_type(schema.model_type())
|
|
994
1007
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(response_model))
|
|
995
|
-
jsonSchema = response_model.
|
|
1008
|
+
jsonSchema = response_model.model_json_schema()
|
|
996
1009
|
return DataSchema(
|
|
997
1010
|
arrowSchema=arrowSchema, dyffSchema=schema, jsonSchema=jsonSchema
|
|
998
1011
|
)
|
|
999
1012
|
else:
|
|
1000
1013
|
response_model = make_response_type(schema)
|
|
1001
1014
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(response_model))
|
|
1002
|
-
jsonSchema = response_model.
|
|
1015
|
+
jsonSchema = response_model.model_json_schema()
|
|
1003
1016
|
return DataSchema(arrowSchema=arrowSchema, jsonSchema=jsonSchema)
|
|
1004
1017
|
|
|
1005
1018
|
|
|
@@ -1030,7 +1043,7 @@ class DataView(DyffSchemaBaseModel):
|
|
|
1030
1043
|
class DatasetBase(DyffSchemaBaseModel):
|
|
1031
1044
|
name: str = pydantic.Field(description="The name of the Dataset")
|
|
1032
1045
|
artifacts: list[Artifact] = pydantic.Field(
|
|
1033
|
-
|
|
1046
|
+
min_length=1, description="Artifacts that comprise the dataset"
|
|
1034
1047
|
)
|
|
1035
1048
|
schema_: DataSchema = pydantic.Field(
|
|
1036
1049
|
alias="schema", description="Schema of the dataset"
|
|
@@ -1108,7 +1121,7 @@ class ModelSource(DyffSchemaBaseModel):
|
|
|
1108
1121
|
|
|
1109
1122
|
class AcceleratorGPU(DyffSchemaBaseModel):
|
|
1110
1123
|
hardwareTypes: list[str] = pydantic.Field(
|
|
1111
|
-
|
|
1124
|
+
min_length=1,
|
|
1112
1125
|
description="Acceptable GPU hardware types.",
|
|
1113
1126
|
)
|
|
1114
1127
|
count: int = pydantic.Field(default=1, description="Number of GPUs required.")
|
|
@@ -1229,12 +1242,12 @@ class ContainerImageSource(DyffSchemaBaseModel):
|
|
|
1229
1242
|
name: str = pydantic.Field(
|
|
1230
1243
|
description="The name of the image",
|
|
1231
1244
|
# https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pull
|
|
1232
|
-
|
|
1245
|
+
pattern=r"^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$",
|
|
1233
1246
|
)
|
|
1234
1247
|
digest: str = pydantic.Field(
|
|
1235
1248
|
description="The digest of the image. The image is always pulled by"
|
|
1236
1249
|
" digest, even if 'tag' is specified.",
|
|
1237
|
-
|
|
1250
|
+
pattern=r"^sha256:[0-9a-f]{64}$",
|
|
1238
1251
|
)
|
|
1239
1252
|
tag: Optional[TagNameType] = pydantic.Field(
|
|
1240
1253
|
default=None,
|
|
@@ -1246,7 +1259,7 @@ class ContainerImageSource(DyffSchemaBaseModel):
|
|
|
1246
1259
|
def url(self) -> str:
|
|
1247
1260
|
return f"{self.host}/{self.name}@{self.digest}"
|
|
1248
1261
|
|
|
1249
|
-
@pydantic.
|
|
1262
|
+
@pydantic.field_validator("host")
|
|
1250
1263
|
def validate_host(cls, v: str):
|
|
1251
1264
|
if "/" in v:
|
|
1252
1265
|
raise ValueError(
|
|
@@ -1547,7 +1560,7 @@ class Evaluation(DyffEntity, EvaluationBase):
|
|
|
1547
1560
|
class ModuleBase(DyffSchemaBaseModel):
|
|
1548
1561
|
name: str = pydantic.Field(description="The name of the Module")
|
|
1549
1562
|
artifacts: list[Artifact] = pydantic.Field(
|
|
1550
|
-
|
|
1563
|
+
min_length=1, description="Artifacts that comprise the Module implementation"
|
|
1551
1564
|
)
|
|
1552
1565
|
|
|
1553
1566
|
|
|
@@ -1629,7 +1642,7 @@ class MeasurementLevel(str, enum.Enum):
|
|
|
1629
1642
|
|
|
1630
1643
|
|
|
1631
1644
|
class AnalysisOutputQueryFields(DyffSchemaBaseModel):
|
|
1632
|
-
analysis: str = pydantic.Field(
|
|
1645
|
+
analysis: Optional[str] = pydantic.Field(
|
|
1633
1646
|
default=None,
|
|
1634
1647
|
description="ID of the Analysis that produced the output.",
|
|
1635
1648
|
)
|
|
@@ -1638,7 +1651,7 @@ class AnalysisOutputQueryFields(DyffSchemaBaseModel):
|
|
|
1638
1651
|
description="Identifying information about the Method that was run to produce the output."
|
|
1639
1652
|
)
|
|
1640
1653
|
|
|
1641
|
-
inputs: list[str] = pydantic.Field(
|
|
1654
|
+
inputs: Optional[list[str]] = pydantic.Field(
|
|
1642
1655
|
default=None,
|
|
1643
1656
|
description="IDs of resources that were inputs to the Analysis.",
|
|
1644
1657
|
)
|
|
@@ -1781,7 +1794,7 @@ class ScoreSpec(DyffSchemaBaseModel):
|
|
|
1781
1794
|
name: str = pydantic.Field(
|
|
1782
1795
|
description="The name of the score. Used as a key for retrieving score data."
|
|
1783
1796
|
" Must be unique within the Method context.",
|
|
1784
|
-
|
|
1797
|
+
pattern=identifier_regex(),
|
|
1785
1798
|
max_length=identifier_maxlen(),
|
|
1786
1799
|
)
|
|
1787
1800
|
|
|
@@ -1820,7 +1833,7 @@ class ScoreSpec(DyffSchemaBaseModel):
|
|
|
1820
1833
|
default="{quantity:.1f}",
|
|
1821
1834
|
# Must use the 'quantity' key in the format string:
|
|
1822
1835
|
# (Maybe string not ending in '}')(something like '{quantity:f}')(maybe another string)
|
|
1823
|
-
|
|
1836
|
+
pattern=r"^(.*[^{])?[{]quantity(:[^}]*)?[}]([^}].*)?$",
|
|
1824
1837
|
description="A Python 'format' string describing how to render the score"
|
|
1825
1838
|
" as a string. You *must* use the keyword 'quantity' in the format"
|
|
1826
1839
|
" string, and you may use 'unit' as well (e.g., '{quantity:.2f} {unit}')."
|
|
@@ -1839,15 +1852,15 @@ class ScoreSpec(DyffSchemaBaseModel):
|
|
|
1839
1852
|
information stored in this ScoreSpec."""
|
|
1840
1853
|
return self.format_quantity(self.format, quantity, unit=self.unit)
|
|
1841
1854
|
|
|
1842
|
-
@pydantic.
|
|
1855
|
+
@pydantic.model_validator(mode="after")
|
|
1843
1856
|
def _validate_minimum_maximum(cls, values):
|
|
1844
|
-
minimum = values.
|
|
1845
|
-
maximum = values.
|
|
1857
|
+
minimum = values.minimum
|
|
1858
|
+
maximum = values.maximum
|
|
1846
1859
|
if minimum is not None and maximum is not None and minimum > maximum:
|
|
1847
1860
|
raise ValueError(f"minimum {minimum} is greater than maximum {maximum}")
|
|
1848
1861
|
return values
|
|
1849
1862
|
|
|
1850
|
-
@pydantic.
|
|
1863
|
+
@pydantic.field_validator("format")
|
|
1851
1864
|
def _validate_format(cls, v):
|
|
1852
1865
|
x = cls.format_quantity(v, 3.14, unit="kg")
|
|
1853
1866
|
y = cls.format_quantity(v, -2.03, unit="kg")
|
|
@@ -1911,7 +1924,7 @@ class MethodBase(DyffSchemaBaseModel):
|
|
|
1911
1924
|
" the default analysis environment.",
|
|
1912
1925
|
)
|
|
1913
1926
|
|
|
1914
|
-
@pydantic.
|
|
1927
|
+
@pydantic.field_validator("scores")
|
|
1915
1928
|
def _scores_validator(cls, scores: list[ScoreSpec]):
|
|
1916
1929
|
if len(scores) > 0:
|
|
1917
1930
|
primary_count = sum(score.priority == "primary" for score in scores)
|
|
@@ -2018,7 +2031,7 @@ class AnalysisData(DyffSchemaBaseModel):
|
|
|
2018
2031
|
value: str = pydantic.Field(
|
|
2019
2032
|
# Canonical base64 encoding
|
|
2020
2033
|
# https://stackoverflow.com/a/64467300/3709935
|
|
2021
|
-
|
|
2034
|
+
pattern=r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?$",
|
|
2022
2035
|
description="Arbitrary data encoded in base64.",
|
|
2023
2036
|
)
|
|
2024
2037
|
|
dyff/schema/v0/r1/requests.py
CHANGED
|
@@ -22,7 +22,7 @@ import pydantic
|
|
|
22
22
|
|
|
23
23
|
from ... import upcast
|
|
24
24
|
from . import commands
|
|
25
|
-
from .base import DyffBaseModel, JsonMergePatchSemantics
|
|
25
|
+
from .base import DyffBaseModel, JsonMergePatchSemantics
|
|
26
26
|
from .platform import (
|
|
27
27
|
AnalysisBase,
|
|
28
28
|
AnalysisScope,
|
|
@@ -58,9 +58,9 @@ class DyffRequestDefaultValidators(DyffBaseModel):
|
|
|
58
58
|
but we allow requests to have non-UTC timezones for user convenience.
|
|
59
59
|
"""
|
|
60
60
|
|
|
61
|
-
@pydantic.
|
|
61
|
+
@pydantic.model_validator(mode="after")
|
|
62
62
|
def _require_datetime_timezone_aware(cls, values):
|
|
63
|
-
for k, v in values.items():
|
|
63
|
+
for k, v in values.__dict__.items():
|
|
64
64
|
if isinstance(v, datetime):
|
|
65
65
|
if v.tzinfo is None:
|
|
66
66
|
raise ValueError(f"{cls.__qualname__}.{k}: timezone not set")
|
|
@@ -74,10 +74,14 @@ class DyffRequestBase(SchemaVersion, DyffRequestDefaultValidators):
|
|
|
74
74
|
def dict(
|
|
75
75
|
self, *, by_alias: bool = True, exclude_unset=True, **kwargs
|
|
76
76
|
) -> _ModelAsDict:
|
|
77
|
-
return super().
|
|
77
|
+
return super().model_dump(
|
|
78
|
+
by_alias=by_alias, exclude_unset=exclude_unset, **kwargs
|
|
79
|
+
)
|
|
78
80
|
|
|
79
81
|
def json(self, *, by_alias: bool = True, exclude_unset=True, **kwargs) -> str:
|
|
80
|
-
return super().
|
|
82
|
+
return super().model_dump_json(
|
|
83
|
+
by_alias=by_alias, exclude_unset=exclude_unset, **kwargs
|
|
84
|
+
)
|
|
81
85
|
|
|
82
86
|
|
|
83
87
|
# ----------------------------------------------------------------------------
|
|
@@ -93,7 +97,7 @@ class AnalysisCreateRequest(DyffEntityCreateRequest, AnalysisBase):
|
|
|
93
97
|
|
|
94
98
|
method: str = pydantic.Field(description="Method ID")
|
|
95
99
|
|
|
96
|
-
@pydantic.
|
|
100
|
+
@pydantic.field_validator("scope", check_fields=False)
|
|
97
101
|
def _validate_scope(cls, scope: AnalysisScope) -> AnalysisScope:
|
|
98
102
|
# TODO: This has to be a validator function because we can't apply the
|
|
99
103
|
# regex contraint to AnalysisScope, since there are already entities
|
|
@@ -114,7 +118,7 @@ class AnalysisCreateRequest(DyffEntityCreateRequest, AnalysisBase):
|
|
|
114
118
|
|
|
115
119
|
|
|
116
120
|
class ConcernCreateRequest(DyffEntityCreateRequest, ConcernBase):
|
|
117
|
-
@pydantic.
|
|
121
|
+
@pydantic.field_validator("documentation", check_fields=False)
|
|
118
122
|
def _validate_documentation(
|
|
119
123
|
cls, documentation: DocumentationBase
|
|
120
124
|
) -> DocumentationBase:
|
|
@@ -148,7 +152,7 @@ class InferenceServiceCreateRequest(DyffEntityCreateRequest, InferenceServiceBas
|
|
|
148
152
|
)
|
|
149
153
|
|
|
150
154
|
# TODO: (DYFF-421) Make .image required and remove this validator
|
|
151
|
-
@pydantic.
|
|
155
|
+
@pydantic.field_validator("image", check_fields=False)
|
|
152
156
|
def validate_image_not_none(cls, v):
|
|
153
157
|
if v is None:
|
|
154
158
|
raise ValueError(
|
|
@@ -189,10 +193,10 @@ class EvaluationCreateRequest(DyffEntityCreateRequest, EvaluationBase):
|
|
|
189
193
|
" for the evaluation, instead of starting a new one.",
|
|
190
194
|
)
|
|
191
195
|
|
|
192
|
-
@pydantic.
|
|
196
|
+
@pydantic.model_validator(mode="after")
|
|
193
197
|
def check_session_exactly_one(cls, values):
|
|
194
|
-
session = values.
|
|
195
|
-
session_ref = values.
|
|
198
|
+
session = values.inferenceSession is not None
|
|
199
|
+
session_ref = values.inferenceSessionReference is not None
|
|
196
200
|
if not (session ^ session_ref):
|
|
197
201
|
raise ValueError(
|
|
198
202
|
"must specify exactly one of {inferenceSession, inferenceSessionReference}"
|
|
@@ -208,16 +212,18 @@ class EvaluationCreateRequest(DyffEntityCreateRequest, EvaluationBase):
|
|
|
208
212
|
return EvaluationCreateRequest(
|
|
209
213
|
account=evaluation.account,
|
|
210
214
|
inferenceSessionReference=evaluation.inferenceSessionReference,
|
|
211
|
-
**base.
|
|
215
|
+
**base.model_dump(),
|
|
212
216
|
)
|
|
213
217
|
else:
|
|
214
218
|
return EvaluationCreateRequest(
|
|
215
219
|
account=evaluation.account,
|
|
216
220
|
inferenceSession=EvaluationInferenceSessionRequest(
|
|
217
221
|
inferenceService=evaluation.inferenceSession.inferenceService.id,
|
|
218
|
-
**upcast(
|
|
222
|
+
**upcast(
|
|
223
|
+
InferenceSessionBase, evaluation.inferenceSession
|
|
224
|
+
).model_dump(),
|
|
219
225
|
),
|
|
220
|
-
**base.
|
|
226
|
+
**base.model_dump(),
|
|
221
227
|
)
|
|
222
228
|
|
|
223
229
|
|
|
@@ -268,10 +274,8 @@ class DocumentationEditRequest(
|
|
|
268
274
|
|
|
269
275
|
|
|
270
276
|
class FamilyMembersEditRequest(DyffRequestBase, JsonMergePatchSemantics):
|
|
271
|
-
members: dict[TagNameType, Optional[
|
|
272
|
-
|
|
273
|
-
description="Mapping of names to IDs of member resources.",
|
|
274
|
-
)
|
|
277
|
+
members: dict[TagNameType, Optional[FamilyMemberBase]] = pydantic.Field(
|
|
278
|
+
description="Mapping of names to IDs of member resources.",
|
|
275
279
|
)
|
|
276
280
|
|
|
277
281
|
|
dyff/schema/v0/r1/version.py
CHANGED
|
@@ -10,7 +10,8 @@ SCHEMA_VERSION: str = "0.1"
|
|
|
10
10
|
|
|
11
11
|
class SchemaVersion(pydantic.BaseModel):
|
|
12
12
|
schemaVersion: Literal["0.1"] = pydantic.Field(
|
|
13
|
-
default=SCHEMA_VERSION,
|
|
13
|
+
default=SCHEMA_VERSION, # type: ignore [arg-type]
|
|
14
|
+
description="The schema version.",
|
|
14
15
|
)
|
|
15
16
|
|
|
16
17
|
|