dyff-schema 0.28.0__py3-none-any.whl → 0.30.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dyff-schema might be problematic. Click here for more details.
- dyff/schema/__init__.py +2 -2
- dyff/schema/_version.py +2 -0
- dyff/schema/copydoc.py +1 -1
- dyff/schema/v0/r1/adapters.py +2 -2
- dyff/schema/v0/r1/base.py +169 -249
- dyff/schema/v0/r1/commands.py +11 -25
- dyff/schema/v0/r1/dataset/arrow.py +35 -36
- dyff/schema/v0/r1/dataset/embedding.py +10 -3
- dyff/schema/v0/r1/dataset/text.py +9 -5
- dyff/schema/v0/r1/io/vllm.py +1 -1
- dyff/schema/v0/r1/platform.py +60 -40
- dyff/schema/v0/r1/requests.py +22 -18
- dyff/schema/v0/r1/version.py +2 -1
- {dyff_schema-0.28.0.dist-info → dyff_schema-0.30.0.dist-info}/METADATA +4 -3
- {dyff_schema-0.28.0.dist-info → dyff_schema-0.30.0.dist-info}/RECORD +19 -18
- {dyff_schema-0.28.0.dist-info → dyff_schema-0.30.0.dist-info}/WHEEL +1 -1
- {dyff_schema-0.28.0.dist-info → dyff_schema-0.30.0.dist-info}/licenses/LICENSE +0 -0
- {dyff_schema-0.28.0.dist-info → dyff_schema-0.30.0.dist-info}/licenses/NOTICE +0 -0
- {dyff_schema-0.28.0.dist-info → dyff_schema-0.30.0.dist-info}/top_level.txt +0 -0
dyff/schema/v0/r1/commands.py
CHANGED
|
@@ -10,8 +10,10 @@ from __future__ import annotations
|
|
|
10
10
|
from typing import Literal, Optional, Union
|
|
11
11
|
|
|
12
12
|
import pydantic
|
|
13
|
+
from pydantic import StringConstraints
|
|
14
|
+
from typing_extensions import Annotated
|
|
13
15
|
|
|
14
|
-
from .base import DyffSchemaBaseModel, JsonMergePatchSemantics
|
|
16
|
+
from .base import DyffSchemaBaseModel, JsonMergePatchSemantics
|
|
15
17
|
from .platform import (
|
|
16
18
|
DyffEntityType,
|
|
17
19
|
EntityIdentifier,
|
|
@@ -26,20 +28,6 @@ from .platform import (
|
|
|
26
28
|
title_maxlen,
|
|
27
29
|
)
|
|
28
30
|
|
|
29
|
-
# ----------------------------------------------------------------------------
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# class _NoneMeansUndefined(DyffSchemaBaseModel):
|
|
33
|
-
# """Fields with the value None will not be emitted in the JSON output."""
|
|
34
|
-
|
|
35
|
-
# # TODO: (DYFF-223) This should perhaps be the default for all schema
|
|
36
|
-
# # objects.
|
|
37
|
-
# def dict(self, *, exclude_none=True, **kwargs) -> _ModelAsDict:
|
|
38
|
-
# return super().dict(exclude_none=True, **kwargs)
|
|
39
|
-
|
|
40
|
-
# def json(self, *, exclude_none=True, **kwargs) -> str:
|
|
41
|
-
# return super().json(exclude_none=True, **kwargs)
|
|
42
|
-
|
|
43
31
|
|
|
44
32
|
class FamilyIdentifier(EntityIdentifier):
|
|
45
33
|
"""Identifies a single Family entity."""
|
|
@@ -87,7 +75,7 @@ class EditEntityDocumentationPatch(JsonMergePatchSemantics):
|
|
|
87
75
|
Fields that are assigned explicitly remain unchanged.
|
|
88
76
|
"""
|
|
89
77
|
|
|
90
|
-
title: Optional[
|
|
78
|
+
title: Optional[Annotated[str, StringConstraints(max_length=title_maxlen())]] = ( # type: ignore
|
|
91
79
|
pydantic.Field(
|
|
92
80
|
default=None,
|
|
93
81
|
description='A short plain string suitable as a title or "headline".'
|
|
@@ -95,7 +83,7 @@ class EditEntityDocumentationPatch(JsonMergePatchSemantics):
|
|
|
95
83
|
)
|
|
96
84
|
)
|
|
97
85
|
|
|
98
|
-
summary: Optional[
|
|
86
|
+
summary: Optional[Annotated[str, StringConstraints(max_length=summary_maxlen())]] = ( # type: ignore
|
|
99
87
|
pydantic.Field(
|
|
100
88
|
default=None,
|
|
101
89
|
description="A brief summary, suitable for display in"
|
|
@@ -104,7 +92,7 @@ class EditEntityDocumentationPatch(JsonMergePatchSemantics):
|
|
|
104
92
|
)
|
|
105
93
|
)
|
|
106
94
|
|
|
107
|
-
fullPage: Optional[
|
|
95
|
+
fullPage: Optional[str] = pydantic.Field(
|
|
108
96
|
default=None,
|
|
109
97
|
description="Long-form documentation. Interpreted as"
|
|
110
98
|
" Markdown. There are no length constraints, but be reasonable."
|
|
@@ -146,7 +134,7 @@ class EditEntityDocumentation(Command):
|
|
|
146
134
|
class EditEntityLabelsAttributes(JsonMergePatchSemantics):
|
|
147
135
|
"""Attributes for the EditEntityLabels command."""
|
|
148
136
|
|
|
149
|
-
labels: dict[LabelKeyType, Optional[
|
|
137
|
+
labels: dict[LabelKeyType, Optional[LabelValueType]] = pydantic.Field(
|
|
150
138
|
default_factory=dict,
|
|
151
139
|
description="A set of key-value labels for the resource."
|
|
152
140
|
" Existing label keys that are not provided in the edit remain unchanged."
|
|
@@ -180,7 +168,7 @@ class EditEntityLabels(Command):
|
|
|
180
168
|
class EditFamilyMembersAttributes(JsonMergePatchSemantics):
|
|
181
169
|
"""Attributes for the EditFamilyMembers command."""
|
|
182
170
|
|
|
183
|
-
members: dict[TagNameType, Optional[
|
|
171
|
+
members: dict[TagNameType, Optional[FamilyMember]] = pydantic.Field(
|
|
184
172
|
description="Mapping of names to IDs of member resources.",
|
|
185
173
|
)
|
|
186
174
|
|
|
@@ -259,12 +247,10 @@ class RestoreEntity(Command):
|
|
|
259
247
|
class UpdateEntityStatusAttributes(JsonMergePatchSemantics):
|
|
260
248
|
"""Attributes for the UpdateEntityStatus command."""
|
|
261
249
|
|
|
262
|
-
status: str = pydantic.Field(
|
|
263
|
-
description=Status.__fields__["status"].field_info.description
|
|
264
|
-
)
|
|
250
|
+
status: str = pydantic.Field(description=Status.model_fields["status"].description)
|
|
265
251
|
|
|
266
|
-
reason: Optional[
|
|
267
|
-
description=Status.
|
|
252
|
+
reason: Optional[str] = pydantic.Field(
|
|
253
|
+
description=Status.model_fields["reason"].description
|
|
268
254
|
)
|
|
269
255
|
|
|
270
256
|
|
|
@@ -7,7 +7,6 @@ from __future__ import annotations
|
|
|
7
7
|
import functools
|
|
8
8
|
import inspect
|
|
9
9
|
import typing
|
|
10
|
-
import uuid
|
|
11
10
|
from typing import Any, Iterable, Literal, Optional
|
|
12
11
|
|
|
13
12
|
import pyarrow
|
|
@@ -31,7 +30,9 @@ def arrow_schema(
|
|
|
31
30
|
We support a very basic subset of pydantic model features currently. The intention
|
|
32
31
|
is to expand this.
|
|
33
32
|
"""
|
|
34
|
-
arrow_fields = [
|
|
33
|
+
arrow_fields = [
|
|
34
|
+
arrow_field(name, field) for name, field in model_type.model_fields.items()
|
|
35
|
+
]
|
|
35
36
|
return pyarrow.schema(arrow_fields, metadata=metadata)
|
|
36
37
|
|
|
37
38
|
|
|
@@ -89,12 +90,9 @@ def subset_schema(schema: pyarrow.Schema, field_names: list[str]) -> pyarrow.Sch
|
|
|
89
90
|
|
|
90
91
|
|
|
91
92
|
def arrow_type(annotation: type) -> pyarrow.DataType:
|
|
92
|
-
"""Determine a suitable arrow type for a pydantic model field.
|
|
93
|
+
"""Determine a suitable arrow type for a pydantic model field."""
|
|
93
94
|
|
|
94
|
-
|
|
95
|
-
Numeric types must have appropriate bounds specified, as Arrow cannot represent the
|
|
96
|
-
unbounded integer types used by Python 3.
|
|
97
|
-
"""
|
|
95
|
+
# Handle generic types first (List, Union, etc.)
|
|
98
96
|
if origin := typing.get_origin(annotation):
|
|
99
97
|
if origin == list:
|
|
100
98
|
annotation_args = typing.get_args(annotation)
|
|
@@ -116,44 +114,45 @@ def arrow_type(annotation: type) -> pyarrow.DataType:
|
|
|
116
114
|
raise ValueError(
|
|
117
115
|
f"annotation {annotation}: only Optional[T] supported, not general Union"
|
|
118
116
|
)
|
|
119
|
-
return arrow_type(inner_type)
|
|
117
|
+
return arrow_type(inner_type)
|
|
120
118
|
|
|
121
119
|
raise NotImplementedError(f"Python type {annotation}")
|
|
122
120
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
subfields.append(arrow_field(subfield))
|
|
127
|
-
return pyarrow.struct(subfields)
|
|
128
|
-
|
|
129
|
-
# This doesn't get caught in the 'origin == list' case above because
|
|
130
|
-
# ConstrainedList isn't a generic type, but the desired result is the same
|
|
131
|
-
if issubclass(annotation, pydantic.ConstrainedList):
|
|
132
|
-
list_size = annotation.max_items if annotation.max_items is not None else -1
|
|
133
|
-
return pyarrow.list_(arrow_type(annotation.item_type), list_size)
|
|
121
|
+
# Guard against non-types (TypeVars, etc.)
|
|
122
|
+
if not isinstance(annotation, type):
|
|
123
|
+
return pyarrow.string()
|
|
134
124
|
|
|
125
|
+
# Handle custom types
|
|
135
126
|
if issubclass(annotation, DType):
|
|
136
127
|
# The dtype is in the metaclass
|
|
137
128
|
return pyarrow.from_numpy_dtype(type(annotation).dtype) # type: ignore[attr-defined]
|
|
138
129
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
return pyarrow.
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
130
|
+
# Handle numpy-like types
|
|
131
|
+
if hasattr(annotation, "dtype"):
|
|
132
|
+
return pyarrow.from_numpy_dtype(annotation.dtype)
|
|
133
|
+
|
|
134
|
+
# Handle pydantic models
|
|
135
|
+
if issubclass(annotation, pydantic.BaseModel):
|
|
136
|
+
subfields = []
|
|
137
|
+
for field_name, subfield in annotation.model_fields.items():
|
|
138
|
+
subfields.append(arrow_field(field_name, subfield))
|
|
139
|
+
return pyarrow.struct(subfields)
|
|
140
|
+
|
|
141
|
+
# Handle built-in types
|
|
142
|
+
type_map = {
|
|
143
|
+
str: pyarrow.string(),
|
|
144
|
+
int: pyarrow.int64(),
|
|
145
|
+
float: pyarrow.float64(),
|
|
146
|
+
bool: pyarrow.bool_(),
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if annotation in type_map:
|
|
150
|
+
return type_map[annotation]
|
|
152
151
|
|
|
153
152
|
raise NotImplementedError(f"Python type {annotation}")
|
|
154
153
|
|
|
155
154
|
|
|
156
|
-
def arrow_field(
|
|
155
|
+
def arrow_field(field_name: str, field_info: pydantic.fields.FieldInfo):
|
|
157
156
|
"""Create a named ``pyarrow.Field`` from a pydantic model ``ModelField``.
|
|
158
157
|
|
|
159
158
|
If present, the ``.alias`` property of the ``ModelField`` takes precedence
|
|
@@ -161,10 +160,10 @@ def arrow_field(pydantic_field: pydantic.fields.ModelField):
|
|
|
161
160
|
function. The ``.description`` property, if present, becomes the docstring
|
|
162
161
|
for the arrow field.
|
|
163
162
|
"""
|
|
164
|
-
name =
|
|
165
|
-
docstring =
|
|
163
|
+
name = field_info.alias if field_info.alias else field_name
|
|
164
|
+
docstring = field_info.description
|
|
166
165
|
return field_with_docstring(
|
|
167
|
-
name, arrow_type(
|
|
166
|
+
name, arrow_type(field_info.annotation or str), docstring=docstring
|
|
168
167
|
)
|
|
169
168
|
|
|
170
169
|
|
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2024 UL Research Institutes
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
from typing import Type
|
|
4
|
+
from typing import Type, Union
|
|
5
5
|
|
|
6
6
|
import pydantic
|
|
7
|
+
from typing_extensions import Annotated
|
|
7
8
|
|
|
8
|
-
from ..base import DyffSchemaBaseModel,
|
|
9
|
+
from ..base import DyffSchemaBaseModel, float32, float64, list_
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
def embedding(
|
|
12
|
-
element_type: Type[
|
|
13
|
+
element_type: Type[
|
|
14
|
+
Union[
|
|
15
|
+
Annotated[float, float32()],
|
|
16
|
+
Annotated[float, float64()],
|
|
17
|
+
]
|
|
18
|
+
],
|
|
19
|
+
size: int,
|
|
13
20
|
) -> Type[DyffSchemaBaseModel]:
|
|
14
21
|
"""Returns a schema type representing a list of fixed-length embedding vectors."""
|
|
15
22
|
|
|
@@ -3,17 +3,21 @@
|
|
|
3
3
|
|
|
4
4
|
import pydantic
|
|
5
5
|
|
|
6
|
-
from ..base import DyffSchemaBaseModel
|
|
6
|
+
from ..base import DyffSchemaBaseModel
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class TaggedSpan(DyffSchemaBaseModel):
|
|
10
10
|
"""A contiguous subsequence of text with a corresponding tag."""
|
|
11
11
|
|
|
12
|
-
start:
|
|
13
|
-
description="The index of the first character in the span"
|
|
12
|
+
start: int = pydantic.Field(
|
|
13
|
+
description="The index of the first character in the span",
|
|
14
|
+
ge=0, # Text indices should be non-negative
|
|
15
|
+
json_schema_extra={"dyff.io/dtype": "int64"},
|
|
14
16
|
)
|
|
15
|
-
end:
|
|
16
|
-
description="The index one past the final character in the span"
|
|
17
|
+
end: int = pydantic.Field(
|
|
18
|
+
description="The index one past the final character in the span",
|
|
19
|
+
ge=0, # Text indices should be non-negative
|
|
20
|
+
json_schema_extra={"dyff.io/dtype": "int64"},
|
|
17
21
|
)
|
|
18
22
|
tag: str = pydantic.Field(description="The tag of the span")
|
|
19
23
|
|
dyff/schema/v0/r1/io/vllm.py
CHANGED
|
@@ -119,6 +119,6 @@ class GenerateEndpointOutput(DyffSchemaBaseModel):
|
|
|
119
119
|
text: list[str] = pydantic.Field(
|
|
120
120
|
# TODO: hypothesis plugin doesn't respect constraints
|
|
121
121
|
# See: https://github.com/pydantic/pydantic/issues/2875
|
|
122
|
-
#
|
|
122
|
+
# min_length=1,
|
|
123
123
|
description="List of generated responses. The prompt is prepended to each response.",
|
|
124
124
|
)
|
dyff/schema/v0/r1/platform.py
CHANGED
|
@@ -28,7 +28,8 @@ from typing import Any, Literal, NamedTuple, Optional, Type, Union
|
|
|
28
28
|
|
|
29
29
|
import pyarrow
|
|
30
30
|
import pydantic
|
|
31
|
-
from
|
|
31
|
+
from pydantic import StringConstraints
|
|
32
|
+
from typing_extensions import Annotated, TypeAlias
|
|
32
33
|
|
|
33
34
|
from ... import named_data_schema, product_schema
|
|
34
35
|
from ...version import SomeSchemaVersion
|
|
@@ -283,7 +284,7 @@ EntityKindLiteral = Literal[
|
|
|
283
284
|
]
|
|
284
285
|
|
|
285
286
|
|
|
286
|
-
EntityID: TypeAlias =
|
|
287
|
+
EntityID: TypeAlias = Annotated[str, StringConstraints(pattern=entity_id_regex())] # type: ignore
|
|
287
288
|
|
|
288
289
|
|
|
289
290
|
class DyffModelWithID(DyffSchemaBaseModel):
|
|
@@ -306,21 +307,30 @@ class EntityIdentifier(DyffSchemaBaseModel):
|
|
|
306
307
|
|
|
307
308
|
|
|
308
309
|
def LabelKey() -> type[str]:
|
|
309
|
-
return
|
|
310
|
-
|
|
311
|
-
|
|
310
|
+
return Annotated[
|
|
311
|
+
str,
|
|
312
|
+
StringConstraints(
|
|
313
|
+
pattern=_k8s_label_key_regex(), max_length=_k8s_label_key_maxlen()
|
|
314
|
+
),
|
|
315
|
+
] # type: ignore [return-value]
|
|
312
316
|
|
|
313
317
|
|
|
314
318
|
def LabelValue() -> type[str]:
|
|
315
|
-
return
|
|
316
|
-
|
|
317
|
-
|
|
319
|
+
return Annotated[
|
|
320
|
+
str,
|
|
321
|
+
StringConstraints(
|
|
322
|
+
pattern=_k8s_label_value_regex(), max_length=_k8s_label_value_maxlen()
|
|
323
|
+
),
|
|
324
|
+
] # type: ignore [return-value]
|
|
318
325
|
|
|
319
326
|
|
|
320
327
|
def TagName() -> type[str]:
|
|
321
|
-
return
|
|
322
|
-
|
|
323
|
-
|
|
328
|
+
return Annotated[
|
|
329
|
+
str,
|
|
330
|
+
StringConstraints(
|
|
331
|
+
pattern=_oci_image_tag_regex(), max_length=_oci_image_tag_maxlen()
|
|
332
|
+
),
|
|
333
|
+
] # type: ignore [return-value]
|
|
324
334
|
|
|
325
335
|
|
|
326
336
|
LabelKeyType: TypeAlias = LabelKey() # type: ignore
|
|
@@ -364,12 +374,14 @@ class Labeled(DyffSchemaBaseModel):
|
|
|
364
374
|
" '.', '-', or '_'.\n\n"
|
|
365
375
|
"We follow the kubernetes label conventions closely."
|
|
366
376
|
" See: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels",
|
|
377
|
+
# Forbid entries that don't match the key patternProperties
|
|
378
|
+
json_schema_extra={"additionalProperties": False},
|
|
367
379
|
)
|
|
368
380
|
|
|
369
381
|
|
|
370
382
|
class Annotation(DyffSchemaBaseModel):
|
|
371
383
|
key: str = pydantic.Field(
|
|
372
|
-
|
|
384
|
+
pattern=_k8s_label_key_regex(),
|
|
373
385
|
max_length=_k8s_domain_maxlen(),
|
|
374
386
|
description="The annotation key. A DNS label with an optional DNS domain prefix."
|
|
375
387
|
" For example: 'my-key', 'your.com/key_0'. Names prefixed with"
|
|
@@ -383,7 +395,7 @@ class Annotation(DyffSchemaBaseModel):
|
|
|
383
395
|
)
|
|
384
396
|
|
|
385
397
|
|
|
386
|
-
Quantity: TypeAlias =
|
|
398
|
+
Quantity: TypeAlias = Annotated[str, StringConstraints(pattern=_k8s_quantity_regex())] # type: ignore
|
|
387
399
|
|
|
388
400
|
|
|
389
401
|
class ServiceClass(str, enum.Enum):
|
|
@@ -609,10 +621,10 @@ class AccessGrant(DyffSchemaBaseModel):
|
|
|
609
621
|
"""
|
|
610
622
|
|
|
611
623
|
resources: list[Resources] = pydantic.Field(
|
|
612
|
-
|
|
624
|
+
min_length=1, description="List of resource types to which the grant applies"
|
|
613
625
|
)
|
|
614
626
|
functions: list[APIFunctions] = pydantic.Field(
|
|
615
|
-
|
|
627
|
+
min_length=1,
|
|
616
628
|
description="List of functions on those resources to which the grant applies",
|
|
617
629
|
)
|
|
618
630
|
accounts: list[str] = pydantic.Field(
|
|
@@ -700,7 +712,7 @@ class FamilyMemberBase(DyffSchemaBaseModel):
|
|
|
700
712
|
description="ID of the resource this member references.",
|
|
701
713
|
)
|
|
702
714
|
|
|
703
|
-
description: Optional[
|
|
715
|
+
description: Optional[Annotated[str, StringConstraints(max_length=summary_maxlen())]] = pydantic.Field( # type: ignore
|
|
704
716
|
default=None,
|
|
705
717
|
description="A short description of the member."
|
|
706
718
|
" This should describe how this version of the resource"
|
|
@@ -719,7 +731,7 @@ class FamilyMember(FamilyMemberBase):
|
|
|
719
731
|
)
|
|
720
732
|
|
|
721
733
|
creationTime: datetime = pydantic.Field(
|
|
722
|
-
|
|
734
|
+
description="Tag creation time (assigned by system)"
|
|
723
735
|
)
|
|
724
736
|
|
|
725
737
|
|
|
@@ -916,12 +928,13 @@ class ExtractorStep(DyffSchemaBaseModel):
|
|
|
916
928
|
|
|
917
929
|
class DyffDataSchema(DyffSchemaBaseModel):
|
|
918
930
|
components: list[str] = pydantic.Field(
|
|
919
|
-
|
|
931
|
+
min_length=1,
|
|
920
932
|
description="A list of named dyff data schemas. The final schema is"
|
|
921
933
|
" the composition of these component schemas.",
|
|
922
934
|
)
|
|
923
935
|
schemaVersion: SomeSchemaVersion = pydantic.Field(
|
|
924
|
-
default=SCHEMA_VERSION,
|
|
936
|
+
default=SCHEMA_VERSION, # type: ignore [arg-type]
|
|
937
|
+
description="The dyff schema version",
|
|
925
938
|
)
|
|
926
939
|
|
|
927
940
|
def model_type(self) -> Type[DyffSchemaBaseModel]:
|
|
@@ -947,7 +960,7 @@ class DataSchema(DyffSchemaBaseModel):
|
|
|
947
960
|
@staticmethod
|
|
948
961
|
def from_model(model: Type[DyffSchemaBaseModel]) -> "DataSchema":
|
|
949
962
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(model))
|
|
950
|
-
jsonSchema = model.
|
|
963
|
+
jsonSchema = model.model_json_schema()
|
|
951
964
|
return DataSchema(arrowSchema=arrowSchema, jsonSchema=jsonSchema)
|
|
952
965
|
|
|
953
966
|
@staticmethod
|
|
@@ -966,14 +979,14 @@ class DataSchema(DyffSchemaBaseModel):
|
|
|
966
979
|
elif isinstance(schema, DyffDataSchema):
|
|
967
980
|
item_model = make_item_type(schema.model_type())
|
|
968
981
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(item_model))
|
|
969
|
-
jsonSchema = item_model.
|
|
982
|
+
jsonSchema = item_model.model_json_schema()
|
|
970
983
|
return DataSchema(
|
|
971
984
|
arrowSchema=arrowSchema, dyffSchema=schema, jsonSchema=jsonSchema
|
|
972
985
|
)
|
|
973
986
|
else:
|
|
974
987
|
item_model = make_item_type(schema)
|
|
975
988
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(item_model))
|
|
976
|
-
jsonSchema = item_model.
|
|
989
|
+
jsonSchema = item_model.model_json_schema()
|
|
977
990
|
return DataSchema(arrowSchema=arrowSchema, jsonSchema=jsonSchema)
|
|
978
991
|
|
|
979
992
|
@staticmethod
|
|
@@ -992,14 +1005,14 @@ class DataSchema(DyffSchemaBaseModel):
|
|
|
992
1005
|
elif isinstance(schema, DyffDataSchema):
|
|
993
1006
|
response_model = make_response_type(schema.model_type())
|
|
994
1007
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(response_model))
|
|
995
|
-
jsonSchema = response_model.
|
|
1008
|
+
jsonSchema = response_model.model_json_schema()
|
|
996
1009
|
return DataSchema(
|
|
997
1010
|
arrowSchema=arrowSchema, dyffSchema=schema, jsonSchema=jsonSchema
|
|
998
1011
|
)
|
|
999
1012
|
else:
|
|
1000
1013
|
response_model = make_response_type(schema)
|
|
1001
1014
|
arrowSchema = arrow.encode_schema(arrow.arrow_schema(response_model))
|
|
1002
|
-
jsonSchema = response_model.
|
|
1015
|
+
jsonSchema = response_model.model_json_schema()
|
|
1003
1016
|
return DataSchema(arrowSchema=arrowSchema, jsonSchema=jsonSchema)
|
|
1004
1017
|
|
|
1005
1018
|
|
|
@@ -1030,7 +1043,7 @@ class DataView(DyffSchemaBaseModel):
|
|
|
1030
1043
|
class DatasetBase(DyffSchemaBaseModel):
|
|
1031
1044
|
name: str = pydantic.Field(description="The name of the Dataset")
|
|
1032
1045
|
artifacts: list[Artifact] = pydantic.Field(
|
|
1033
|
-
|
|
1046
|
+
min_length=1, description="Artifacts that comprise the dataset"
|
|
1034
1047
|
)
|
|
1035
1048
|
schema_: DataSchema = pydantic.Field(
|
|
1036
1049
|
alias="schema", description="Schema of the dataset"
|
|
@@ -1108,7 +1121,7 @@ class ModelSource(DyffSchemaBaseModel):
|
|
|
1108
1121
|
|
|
1109
1122
|
class AcceleratorGPU(DyffSchemaBaseModel):
|
|
1110
1123
|
hardwareTypes: list[str] = pydantic.Field(
|
|
1111
|
-
|
|
1124
|
+
min_length=1,
|
|
1112
1125
|
description="Acceptable GPU hardware types.",
|
|
1113
1126
|
)
|
|
1114
1127
|
count: int = pydantic.Field(default=1, description="Number of GPUs required.")
|
|
@@ -1229,12 +1242,12 @@ class ContainerImageSource(DyffSchemaBaseModel):
|
|
|
1229
1242
|
name: str = pydantic.Field(
|
|
1230
1243
|
description="The name of the image",
|
|
1231
1244
|
# https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pull
|
|
1232
|
-
|
|
1245
|
+
pattern=r"^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$",
|
|
1233
1246
|
)
|
|
1234
1247
|
digest: str = pydantic.Field(
|
|
1235
1248
|
description="The digest of the image. The image is always pulled by"
|
|
1236
1249
|
" digest, even if 'tag' is specified.",
|
|
1237
|
-
|
|
1250
|
+
pattern=r"^sha256:[0-9a-f]{64}$",
|
|
1238
1251
|
)
|
|
1239
1252
|
tag: Optional[TagNameType] = pydantic.Field(
|
|
1240
1253
|
default=None,
|
|
@@ -1246,7 +1259,7 @@ class ContainerImageSource(DyffSchemaBaseModel):
|
|
|
1246
1259
|
def url(self) -> str:
|
|
1247
1260
|
return f"{self.host}/{self.name}@{self.digest}"
|
|
1248
1261
|
|
|
1249
|
-
@pydantic.
|
|
1262
|
+
@pydantic.field_validator("host")
|
|
1250
1263
|
def validate_host(cls, v: str):
|
|
1251
1264
|
if "/" in v:
|
|
1252
1265
|
raise ValueError(
|
|
@@ -1547,7 +1560,7 @@ class Evaluation(DyffEntity, EvaluationBase):
|
|
|
1547
1560
|
class ModuleBase(DyffSchemaBaseModel):
|
|
1548
1561
|
name: str = pydantic.Field(description="The name of the Module")
|
|
1549
1562
|
artifacts: list[Artifact] = pydantic.Field(
|
|
1550
|
-
|
|
1563
|
+
min_length=1, description="Artifacts that comprise the Module implementation"
|
|
1551
1564
|
)
|
|
1552
1565
|
|
|
1553
1566
|
|
|
@@ -1629,7 +1642,7 @@ class MeasurementLevel(str, enum.Enum):
|
|
|
1629
1642
|
|
|
1630
1643
|
|
|
1631
1644
|
class AnalysisOutputQueryFields(DyffSchemaBaseModel):
|
|
1632
|
-
analysis: str = pydantic.Field(
|
|
1645
|
+
analysis: Optional[str] = pydantic.Field(
|
|
1633
1646
|
default=None,
|
|
1634
1647
|
description="ID of the Analysis that produced the output.",
|
|
1635
1648
|
)
|
|
@@ -1638,7 +1651,7 @@ class AnalysisOutputQueryFields(DyffSchemaBaseModel):
|
|
|
1638
1651
|
description="Identifying information about the Method that was run to produce the output."
|
|
1639
1652
|
)
|
|
1640
1653
|
|
|
1641
|
-
inputs: list[str] = pydantic.Field(
|
|
1654
|
+
inputs: Optional[list[str]] = pydantic.Field(
|
|
1642
1655
|
default=None,
|
|
1643
1656
|
description="IDs of resources that were inputs to the Analysis.",
|
|
1644
1657
|
)
|
|
@@ -1781,7 +1794,7 @@ class ScoreSpec(DyffSchemaBaseModel):
|
|
|
1781
1794
|
name: str = pydantic.Field(
|
|
1782
1795
|
description="The name of the score. Used as a key for retrieving score data."
|
|
1783
1796
|
" Must be unique within the Method context.",
|
|
1784
|
-
|
|
1797
|
+
pattern=identifier_regex(),
|
|
1785
1798
|
max_length=identifier_maxlen(),
|
|
1786
1799
|
)
|
|
1787
1800
|
|
|
@@ -1820,7 +1833,7 @@ class ScoreSpec(DyffSchemaBaseModel):
|
|
|
1820
1833
|
default="{quantity:.1f}",
|
|
1821
1834
|
# Must use the 'quantity' key in the format string:
|
|
1822
1835
|
# (Maybe string not ending in '}')(something like '{quantity:f}')(maybe another string)
|
|
1823
|
-
|
|
1836
|
+
pattern=r"^(.*[^{])?[{]quantity(:[^}]*)?[}]([^}].*)?$",
|
|
1824
1837
|
description="A Python 'format' string describing how to render the score"
|
|
1825
1838
|
" as a string. You *must* use the keyword 'quantity' in the format"
|
|
1826
1839
|
" string, and you may use 'unit' as well (e.g., '{quantity:.2f} {unit}')."
|
|
@@ -1839,15 +1852,15 @@ class ScoreSpec(DyffSchemaBaseModel):
|
|
|
1839
1852
|
information stored in this ScoreSpec."""
|
|
1840
1853
|
return self.format_quantity(self.format, quantity, unit=self.unit)
|
|
1841
1854
|
|
|
1842
|
-
@pydantic.
|
|
1855
|
+
@pydantic.model_validator(mode="after")
|
|
1843
1856
|
def _validate_minimum_maximum(cls, values):
|
|
1844
|
-
minimum = values.
|
|
1845
|
-
maximum = values.
|
|
1857
|
+
minimum = values.minimum
|
|
1858
|
+
maximum = values.maximum
|
|
1846
1859
|
if minimum is not None and maximum is not None and minimum > maximum:
|
|
1847
1860
|
raise ValueError(f"minimum {minimum} is greater than maximum {maximum}")
|
|
1848
1861
|
return values
|
|
1849
1862
|
|
|
1850
|
-
@pydantic.
|
|
1863
|
+
@pydantic.field_validator("format")
|
|
1851
1864
|
def _validate_format(cls, v):
|
|
1852
1865
|
x = cls.format_quantity(v, 3.14, unit="kg")
|
|
1853
1866
|
y = cls.format_quantity(v, -2.03, unit="kg")
|
|
@@ -1904,7 +1917,14 @@ class MethodBase(DyffSchemaBaseModel):
|
|
|
1904
1917
|
description="Modules to load into the analysis environment",
|
|
1905
1918
|
)
|
|
1906
1919
|
|
|
1907
|
-
|
|
1920
|
+
analysisImage: Optional[ContainerImageSource] = pydantic.Field(
|
|
1921
|
+
default=None,
|
|
1922
|
+
description="Optional container image to use for running analysis methods."
|
|
1923
|
+
" If specified, analysis will run in this custom container instead of"
|
|
1924
|
+
" the default analysis environment.",
|
|
1925
|
+
)
|
|
1926
|
+
|
|
1927
|
+
@pydantic.field_validator("scores")
|
|
1908
1928
|
def _scores_validator(cls, scores: list[ScoreSpec]):
|
|
1909
1929
|
if len(scores) > 0:
|
|
1910
1930
|
primary_count = sum(score.priority == "primary" for score in scores)
|
|
@@ -2011,7 +2031,7 @@ class AnalysisData(DyffSchemaBaseModel):
|
|
|
2011
2031
|
value: str = pydantic.Field(
|
|
2012
2032
|
# Canonical base64 encoding
|
|
2013
2033
|
# https://stackoverflow.com/a/64467300/3709935
|
|
2014
|
-
|
|
2034
|
+
pattern=r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?$",
|
|
2015
2035
|
description="Arbitrary data encoded in base64.",
|
|
2016
2036
|
)
|
|
2017
2037
|
|