dkist-processing-common 10.8.1rc1__py3-none-any.whl → 10.8.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_common/codecs/fits.py +6 -12
- dkist_processing_common/manual.py +5 -3
- dkist_processing_common/models/graphql.py +3 -13
- dkist_processing_common/models/parameters.py +28 -65
- dkist_processing_common/tasks/mixin/input_dataset.py +166 -0
- dkist_processing_common/tasks/mixin/metadata_store.py +4 -7
- dkist_processing_common/tasks/transfer_input_data.py +70 -61
- dkist_processing_common/tasks/write_l1.py +20 -1
- dkist_processing_common/tests/conftest.py +7 -24
- dkist_processing_common/tests/test_codecs.py +0 -38
- dkist_processing_common/tests/test_input_dataset.py +308 -79
- dkist_processing_common/tests/test_parameters.py +22 -71
- dkist_processing_common/tests/test_transfer_input_data.py +45 -131
- dkist_processing_common/tests/test_write_l1.py +110 -2
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.2.dist-info}/METADATA +2 -2
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.2.dist-info}/RECORD +18 -23
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.2.dist-info}/WHEEL +1 -1
- changelog/235.feature.rst +0 -3
- changelog/235.misc.1.rst +0 -2
- changelog/235.misc.rst +0 -1
- dkist_processing_common/codecs/array.py +0 -19
- dkist_processing_common/codecs/basemodel.py +0 -21
- dkist_processing_common/models/input_dataset.py +0 -113
- {dkist_processing_common-10.8.1rc1.dist-info → dkist_processing_common-10.8.2.dist-info}/top_level.txt +0 -0
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
"""Input dataset models for the inputDatasetPartDocument from the metadata store api."""
|
|
2
|
-
import json
|
|
3
|
-
from datetime import datetime
|
|
4
|
-
from typing import Any
|
|
5
|
-
|
|
6
|
-
from pydantic import BaseModel
|
|
7
|
-
from pydantic import ConfigDict
|
|
8
|
-
from pydantic import Field
|
|
9
|
-
from pydantic import field_serializer
|
|
10
|
-
from pydantic import field_validator
|
|
11
|
-
from pydantic import Json
|
|
12
|
-
from pydantic import PlainSerializer
|
|
13
|
-
from pydantic.alias_generators import to_camel
|
|
14
|
-
from typing_extensions import Annotated
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class InputDatasetBaseModel(BaseModel):
|
|
18
|
-
"""Custom BaseModel for input datasets."""
|
|
19
|
-
|
|
20
|
-
model_config = ConfigDict(
|
|
21
|
-
alias_generator=to_camel, validate_by_name=True, validate_by_alias=True
|
|
22
|
-
)
|
|
23
|
-
|
|
24
|
-
def model_dump(self, **kwargs) -> dict:
|
|
25
|
-
"""Dump models as they were in the metadata store."""
|
|
26
|
-
kwargs.setdefault("exclude_defaults", True)
|
|
27
|
-
kwargs.setdefault("by_alias", True) # will not be needed in Pydantic v3
|
|
28
|
-
return super().model_dump(**kwargs)
|
|
29
|
-
|
|
30
|
-
def model_dump_json(self, **kwargs) -> str:
|
|
31
|
-
"""Dump models as they were in the metadata store."""
|
|
32
|
-
kwargs.setdefault("exclude_defaults", True)
|
|
33
|
-
kwargs.setdefault("by_alias", True) # will not be needed in Pydantic v3
|
|
34
|
-
return super().model_dump_json(**kwargs)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
class InputDatasetObject(InputDatasetBaseModel):
|
|
38
|
-
"""Input dataset object validator for a single file."""
|
|
39
|
-
|
|
40
|
-
bucket: str
|
|
41
|
-
object_key: str
|
|
42
|
-
tag: str | None = None
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
class InputDatasetFilePointer(InputDatasetBaseModel):
|
|
46
|
-
"""Wrapper for InputDatasetObject files."""
|
|
47
|
-
|
|
48
|
-
file_pointer: InputDatasetObject = Field(alias="__file__")
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
class InputDatasetParameterValue(InputDatasetBaseModel):
|
|
52
|
-
"""Input dataset parameter value validator."""
|
|
53
|
-
|
|
54
|
-
parameter_value_id: int
|
|
55
|
-
# parameter_value: Json[InputDatasetFilePointer] | Json[Any] # will work in gqlclient v2
|
|
56
|
-
parameter_value: Json[Any]
|
|
57
|
-
parameter_value_start_date: Annotated[
|
|
58
|
-
datetime, Field(default=datetime(1, 1, 1)), PlainSerializer(lambda x: x.isoformat())
|
|
59
|
-
]
|
|
60
|
-
|
|
61
|
-
@field_validator("parameter_value", mode="after")
|
|
62
|
-
@classmethod
|
|
63
|
-
def validate_parameter_value(cls, param_val):
|
|
64
|
-
"""Decode and provide additional validation for parameter_value types."""
|
|
65
|
-
match param_val:
|
|
66
|
-
case {"__file__": _}:
|
|
67
|
-
return InputDatasetFilePointer.model_validate(param_val)
|
|
68
|
-
case _:
|
|
69
|
-
return param_val
|
|
70
|
-
|
|
71
|
-
@field_serializer("parameter_value")
|
|
72
|
-
def serialize_parameter_value(self, param_val):
|
|
73
|
-
"""Serialize the parameter_value types."""
|
|
74
|
-
if isinstance(param_val, InputDatasetBaseModel):
|
|
75
|
-
return json.dumps(param_val.model_dump())
|
|
76
|
-
return json.dumps(param_val)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
class InputDatasetParameter(InputDatasetBaseModel):
|
|
80
|
-
"""Parsing of the inputDatasetPartDocument that is relevant for parameters."""
|
|
81
|
-
|
|
82
|
-
parameter_name: str
|
|
83
|
-
parameter_values: list[InputDatasetParameterValue]
|
|
84
|
-
|
|
85
|
-
@property
|
|
86
|
-
def input_dataset_objects(self) -> list[InputDatasetObject]:
|
|
87
|
-
"""Find and return list of InputDatasetObjects."""
|
|
88
|
-
object_list = []
|
|
89
|
-
for param in self.parameter_values:
|
|
90
|
-
if isinstance(param.parameter_value, InputDatasetFilePointer):
|
|
91
|
-
object_list.append(param.parameter_value.file_pointer)
|
|
92
|
-
return object_list
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
class InputDatasetFrames(InputDatasetBaseModel):
|
|
96
|
-
"""Parsing of the inputDatasetPartDocument that is relevant for frames."""
|
|
97
|
-
|
|
98
|
-
bucket: str
|
|
99
|
-
object_keys: list[str] = Field(alias="object_keys") # not camel case in metadata store
|
|
100
|
-
|
|
101
|
-
@property
|
|
102
|
-
def input_dataset_objects(self) -> list[InputDatasetObject]:
|
|
103
|
-
"""Convert a single bucket and a list of object_keys list into a list of InputDatasetObjects."""
|
|
104
|
-
object_list = []
|
|
105
|
-
for frame in self.object_keys:
|
|
106
|
-
object_list.append(InputDatasetObject(bucket=self.bucket, object_key=frame))
|
|
107
|
-
return object_list
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
class InputDatasetPartDocumentList(InputDatasetBaseModel):
|
|
111
|
-
"""List of either InputDatasetFrames or InputDatasetParameter objects."""
|
|
112
|
-
|
|
113
|
-
doc_list: list[InputDatasetFrames] | list[InputDatasetParameter] = Field(alias="doc_list")
|
|
File without changes
|