dkist-processing-test 1.21.4__py3-none-any.whl → 1.22.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dkist-processing-test might be problematic. Click here for more details.
- dkist_processing_test/models/parameters.py +16 -0
- dkist_processing_test/tasks/fake_science.py +10 -1
- dkist_processing_test/tests/conftest.py +41 -1
- dkist_processing_test/tests/parameter_models.py +130 -0
- dkist_processing_test/tests/test_parameters.py +57 -9
- dkist_processing_test/tests/test_tasks.py +107 -47
- {dkist_processing_test-1.21.4.dist-info → dkist_processing_test-1.22.0rc1.dist-info}/METADATA +1 -1
- {dkist_processing_test-1.21.4.dist-info → dkist_processing_test-1.22.0rc1.dist-info}/RECORD +10 -9
- {dkist_processing_test-1.21.4.dist-info → dkist_processing_test-1.22.0rc1.dist-info}/WHEEL +1 -1
- {dkist_processing_test-1.21.4.dist-info → dkist_processing_test-1.22.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
from datetime import datetime
|
|
2
3
|
|
|
3
4
|
import numpy as np
|
|
@@ -33,3 +34,18 @@ class TestParameters(ParameterBase, ParameterWavelengthMixin):
|
|
|
33
34
|
def wavelength_category(self) -> str:
|
|
34
35
|
"""A dummy parameter that depends on wavelength."""
|
|
35
36
|
return self._find_parameter_closest_wavelength("test_wavelength_category")
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def value_message(self) -> str:
|
|
40
|
+
"""A dummy parameter that returns a message."""
|
|
41
|
+
return self._find_most_recent_past_value("test_message")
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def file_message(self) -> str:
|
|
45
|
+
"""A dummy parameter that returns a message from a file."""
|
|
46
|
+
param_dict = self._find_most_recent_past_value("test_message_file")
|
|
47
|
+
file_path = param_dict["param_path"]
|
|
48
|
+
with open(file_path, "r") as f:
|
|
49
|
+
message = json.load(f)
|
|
50
|
+
|
|
51
|
+
return message
|
|
@@ -57,10 +57,19 @@ class GenerateCalibratedData(WorkflowTaskBase, InputDatasetMixin):
|
|
|
57
57
|
|
|
58
58
|
self.write(data={"test": "dictionary"}, tags=["BAZ"], encoder=json_encoder)
|
|
59
59
|
|
|
60
|
-
with self.apm_task_step(
|
|
60
|
+
with self.apm_task_step(
|
|
61
|
+
"Creating frames that won't be used or transferred as trial outputs"
|
|
62
|
+
):
|
|
61
63
|
self.write(data=b"123", tags=[Tag.intermediate(), Tag.task("NOT_USED"), Tag.frame()])
|
|
62
64
|
self.write(data=b"123", tags=["FOO"])
|
|
63
65
|
|
|
66
|
+
# Here is where we record the easy-to-check parameterValues for downstream checking
|
|
67
|
+
param_check_dict = {
|
|
68
|
+
"file_message": self.parameters.file_message,
|
|
69
|
+
"value_message": self.parameters.value_message,
|
|
70
|
+
}
|
|
71
|
+
self.write(data=param_check_dict, tags=["PARAM_CHECK"], encoder=json_encoder)
|
|
72
|
+
|
|
64
73
|
with self.apm_task_step("Loop over inputs"):
|
|
65
74
|
count = 1 # keep a running count to increment the dsps repeat number
|
|
66
75
|
for hdu in self.read(tags=Tag.input(), decoder=fits_hdu_decoder):
|
|
@@ -48,10 +48,20 @@ def generate_214_l0_fits_frame(
|
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
@pytest.fixture(scope="session")
|
|
51
|
-
def
|
|
51
|
+
def array_parameter_file_object_key() -> str:
|
|
52
52
|
return "random.fits"
|
|
53
53
|
|
|
54
54
|
|
|
55
|
+
@pytest.fixture(scope="session")
|
|
56
|
+
def early_json_parameter_file_object_key() -> str:
|
|
57
|
+
return "early_message.json"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@pytest.fixture(scope="session")
|
|
61
|
+
def late_json_parameter_file_object_key() -> str:
|
|
62
|
+
return "late_message.json"
|
|
63
|
+
|
|
64
|
+
|
|
55
65
|
@pytest.fixture(scope="session")
|
|
56
66
|
def random_parameter_hdulist() -> (fits.HDUList, float, float, float):
|
|
57
67
|
rng = np.random.default_rng()
|
|
@@ -62,3 +72,33 @@ def random_parameter_hdulist() -> (fits.HDUList, float, float, float):
|
|
|
62
72
|
hdul = fits.HDUList([fits.PrimaryHDU(rand_data), fits.ImageHDU(const_data)])
|
|
63
73
|
|
|
64
74
|
return hdul, mu, std, const
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@pytest.fixture(scope="session")
|
|
78
|
+
def early_file_message_str() -> str:
|
|
79
|
+
return "Early in a file"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@pytest.fixture(scope="session")
|
|
83
|
+
def late_file_message_str() -> str:
|
|
84
|
+
return "Late in a file"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@pytest.fixture(scope="session")
|
|
88
|
+
def early_value_message_str() -> str:
|
|
89
|
+
return "Early"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@pytest.fixture(scope="session")
|
|
93
|
+
def late_value_message_str() -> str:
|
|
94
|
+
return "Late"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@pytest.fixture(scope="session")
|
|
98
|
+
def early_date() -> str:
|
|
99
|
+
return "1980-01-01"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@pytest.fixture(scope="session")
|
|
103
|
+
def late_date() -> str:
|
|
104
|
+
return "2000-01-01"
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from random import randint
|
|
4
|
+
from typing import Any
|
|
5
|
+
from typing import Union
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel
|
|
8
|
+
from pydantic import ConfigDict
|
|
9
|
+
from pydantic import Field
|
|
10
|
+
from pydantic import field_serializer
|
|
11
|
+
from pydantic import field_validator
|
|
12
|
+
from pydantic import model_serializer
|
|
13
|
+
|
|
14
|
+
param_value_type_hint = Union[
|
|
15
|
+
int, str, list, tuple, "WavelengthParameterValue", "RawFileParameterValue"
|
|
16
|
+
]
|
|
17
|
+
multi_str_type = Union[str, "MultipleParameterValues"]
|
|
18
|
+
multi_file_type = Union["RawFileParameterValue", "MultipleParameterValues"]
|
|
19
|
+
multi_wave_type = Union["WavelengthParameterValue", "MultipleParameterValues"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ParameterValue(BaseModel):
|
|
23
|
+
"""A single parameterValue entry"""
|
|
24
|
+
|
|
25
|
+
parameterValue: param_value_type_hint
|
|
26
|
+
parameterValueId: int = Field(default_factory=lambda: randint(1000, 2000))
|
|
27
|
+
parameterValueStartDate: datetime = datetime(1946, 11, 20)
|
|
28
|
+
|
|
29
|
+
@field_serializer("parameterValue")
|
|
30
|
+
def json_parameter_value(self, parameter_value: param_value_type_hint) -> str:
|
|
31
|
+
"""Encode the actual value in a JSON string."""
|
|
32
|
+
try:
|
|
33
|
+
parameter_value = parameter_value.model_dump()
|
|
34
|
+
except:
|
|
35
|
+
# If the value is just a basic type (i.e., not a `BaseModel`)
|
|
36
|
+
pass
|
|
37
|
+
return json.dumps(parameter_value)
|
|
38
|
+
|
|
39
|
+
@field_serializer("parameterValueStartDate")
|
|
40
|
+
def datetime_iso_format(self, start_datetime: datetime) -> str:
|
|
41
|
+
"""Encode the start date as an ISO-formatted string."""
|
|
42
|
+
return start_datetime.isoformat()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class MultipleParameterValues(BaseModel):
|
|
46
|
+
"""
|
|
47
|
+
Container for a list of parameterValues.
|
|
48
|
+
|
|
49
|
+
This exists to be different than a raw `list`, which is a valid `parameterValue.parameterValue` type
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
parameter_value_list: list[ParameterValue]
|
|
53
|
+
|
|
54
|
+
@field_validator("parameter_value_list", mode="before")
|
|
55
|
+
@classmethod
|
|
56
|
+
def ensure_list_of_parameter_values(cls, input_list: list) -> list[ParameterValue]:
|
|
57
|
+
"""Convert any raw types to `ParameterValue` objects."""
|
|
58
|
+
output_list = []
|
|
59
|
+
for parameter_value in input_list:
|
|
60
|
+
if not isinstance(parameter_value, ParameterValue):
|
|
61
|
+
parameter_value = ParameterValue(parameterValue=parameter_value)
|
|
62
|
+
|
|
63
|
+
output_list.append(parameter_value)
|
|
64
|
+
|
|
65
|
+
return output_list
|
|
66
|
+
|
|
67
|
+
@field_validator("parameter_value_list")
|
|
68
|
+
@classmethod
|
|
69
|
+
def no_repeat_start_dates(
|
|
70
|
+
cls, parameter_value_list: list[ParameterValue]
|
|
71
|
+
) -> list[ParameterValue]:
|
|
72
|
+
"""Fail validation if any of the `ParameterValues` have the same `parameterValueStartDate."""
|
|
73
|
+
start_dates = [pv.parameterValueStartDate for pv in parameter_value_list]
|
|
74
|
+
if len(set(start_dates)) != len(start_dates):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"parameterValueStartDates must be unique. Got {set(start_dates)} over {len(start_dates)} parameters."
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
return parameter_value_list
|
|
80
|
+
|
|
81
|
+
@model_serializer
|
|
82
|
+
def parameter_value_list(self):
|
|
83
|
+
"""Return just the list of `ParameterValues`."""
|
|
84
|
+
return self.parameter_value_list
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class WavelengthParameterValue(BaseModel):
|
|
88
|
+
values: tuple
|
|
89
|
+
wavelength: tuple = (1.0, 2.0, 3.0)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class RawFileParameterValue(BaseModel):
|
|
93
|
+
"""
|
|
94
|
+
For parameters that are files on disk.
|
|
95
|
+
|
|
96
|
+
"Raw" in the sense that it still has the `__file__`-level dictionary.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
objectKey: str
|
|
100
|
+
bucket: str = "doesn't_matter"
|
|
101
|
+
|
|
102
|
+
@model_serializer
|
|
103
|
+
def file_dict(self) -> dict:
|
|
104
|
+
"""Wrap the input values in a `__file__` dict."""
|
|
105
|
+
return {"__file__": dict(self)}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class TestParameterValues(BaseModel):
|
|
109
|
+
test_random_data: multi_file_type = RawFileParameterValue(objectKey="")
|
|
110
|
+
test_wavelength_category: multi_wave_type = WavelengthParameterValue(
|
|
111
|
+
values=("one", "two", "three")
|
|
112
|
+
)
|
|
113
|
+
test_message: multi_str_type = "Weird?"
|
|
114
|
+
test_message_file: multi_file_type = RawFileParameterValue(objectKey="")
|
|
115
|
+
|
|
116
|
+
model_config = ConfigDict(validate_default=True)
|
|
117
|
+
|
|
118
|
+
@field_validator("*")
|
|
119
|
+
@classmethod
|
|
120
|
+
def ensure_parameter_value_lists(cls, parameter: Any) -> MultipleParameterValues:
|
|
121
|
+
"""Convert all values to `MultipleParameterValues`, if they aren't already."""
|
|
122
|
+
if not isinstance(parameter, MultipleParameterValues):
|
|
123
|
+
return MultipleParameterValues(parameter_value_list=[parameter])
|
|
124
|
+
|
|
125
|
+
return parameter
|
|
126
|
+
|
|
127
|
+
@model_serializer
|
|
128
|
+
def input_dataset_document_parameters_part(self) -> list:
|
|
129
|
+
"""Place all parameters into higher-level dictionaries required by the input dataset document."""
|
|
130
|
+
return [{"parameterName": f, "parameterValues": v} for f, v in self]
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
from datetime import datetime
|
|
2
3
|
|
|
3
4
|
import numpy as np
|
|
@@ -9,16 +10,16 @@ from dkist_processing_test.models.parameters import TestParameters
|
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
@pytest.fixture(scope="session")
|
|
12
|
-
def
|
|
13
|
-
tmp_path_factory, random_parameter_hdulist,
|
|
13
|
+
def array_file_parameter(
|
|
14
|
+
tmp_path_factory, random_parameter_hdulist, array_parameter_file_object_key
|
|
14
15
|
) -> tuple[InputDatasetParameterValue, float, float, float]:
|
|
15
16
|
hdul, mu, std, const = random_parameter_hdulist
|
|
16
|
-
file_path = tmp_path_factory.mktemp("parameters") /
|
|
17
|
+
file_path = tmp_path_factory.mktemp("parameters") / array_parameter_file_object_key
|
|
17
18
|
hdul.writeto(file_path)
|
|
18
19
|
|
|
19
20
|
value = {
|
|
20
21
|
"bucket": "raw",
|
|
21
|
-
"objectKey":
|
|
22
|
+
"objectKey": array_parameter_file_object_key,
|
|
22
23
|
"param_path": file_path,
|
|
23
24
|
"is_file": True,
|
|
24
25
|
}
|
|
@@ -31,12 +32,48 @@ def file_parameter(
|
|
|
31
32
|
return parameter_value, mu, std, const
|
|
32
33
|
|
|
33
34
|
|
|
35
|
+
@pytest.fixture(scope="session")
|
|
36
|
+
def message_file_parameter(
|
|
37
|
+
tmp_path_factory, early_file_message_str, early_json_parameter_file_object_key
|
|
38
|
+
):
|
|
39
|
+
|
|
40
|
+
file_path = tmp_path_factory.mktemp("parameters") / early_json_parameter_file_object_key
|
|
41
|
+
with open(file_path, "w") as f:
|
|
42
|
+
json.dump(early_file_message_str, f)
|
|
43
|
+
|
|
44
|
+
value = {
|
|
45
|
+
"bucket": "raw",
|
|
46
|
+
"objectKey": early_json_parameter_file_object_key,
|
|
47
|
+
"param_path": file_path,
|
|
48
|
+
"is_file": True,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
parameter_value = InputDatasetParameterValue(
|
|
52
|
+
parameter_value_id=2,
|
|
53
|
+
parameter_value=value,
|
|
54
|
+
parameter_value_start_date=datetime(1946, 11, 20),
|
|
55
|
+
)
|
|
56
|
+
return parameter_value
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@pytest.fixture(scope="session")
|
|
60
|
+
def message_parameter(early_value_message_str):
|
|
61
|
+
|
|
62
|
+
parameter_value = InputDatasetParameterValue(
|
|
63
|
+
parameter_value_id=3,
|
|
64
|
+
parameter_value=early_value_message_str,
|
|
65
|
+
parameter_value_start_date=datetime(1946, 11, 20),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
return parameter_value
|
|
69
|
+
|
|
70
|
+
|
|
34
71
|
@pytest.fixture(scope="session")
|
|
35
72
|
def wavelength_parameter() -> InputDatasetParameterValue:
|
|
36
73
|
value = {"wavelength": [1.0, 2.0, 3.0], "values": ["one", "two", "three"]}
|
|
37
74
|
|
|
38
75
|
parameter_value = InputDatasetParameterValue(
|
|
39
|
-
parameter_value_id=
|
|
76
|
+
parameter_value_id=4,
|
|
40
77
|
parameter_value=value,
|
|
41
78
|
parameter_value_start_date=datetime(1946, 11, 20),
|
|
42
79
|
)
|
|
@@ -44,12 +81,16 @@ def wavelength_parameter() -> InputDatasetParameterValue:
|
|
|
44
81
|
|
|
45
82
|
|
|
46
83
|
@pytest.fixture(scope="session")
|
|
47
|
-
def parameter_dict_with_path(
|
|
84
|
+
def parameter_dict_with_path(
|
|
85
|
+
array_file_parameter, message_file_parameter, message_parameter, wavelength_parameter
|
|
86
|
+
):
|
|
48
87
|
"""Enough of an input dataset parameters part to exercise file loading parameters."""
|
|
49
|
-
file_parameter_value, _, _, _ =
|
|
88
|
+
file_parameter_value, _, _, _ = array_file_parameter
|
|
50
89
|
|
|
51
90
|
param_dict = {
|
|
52
91
|
"test_random_data": [file_parameter_value],
|
|
92
|
+
"test_message": [message_parameter],
|
|
93
|
+
"test_message_file": [message_file_parameter],
|
|
53
94
|
"test_wavelength_category": [wavelength_parameter],
|
|
54
95
|
}
|
|
55
96
|
|
|
@@ -87,17 +128,24 @@ def task_with_parameters(parameter_dict_with_path):
|
|
|
87
128
|
return task
|
|
88
129
|
|
|
89
130
|
|
|
90
|
-
def test_parameter(
|
|
131
|
+
def test_parameter(
|
|
132
|
+
task_with_parameters,
|
|
133
|
+
array_file_parameter,
|
|
134
|
+
early_file_message_str,
|
|
135
|
+
early_value_message_str,
|
|
136
|
+
):
|
|
91
137
|
"""
|
|
92
138
|
Given: A task with parameters that depend on files
|
|
93
139
|
When: Accessing those parameters
|
|
94
140
|
Then: The correct values are returned
|
|
95
141
|
"""
|
|
96
142
|
task = task_with_parameters
|
|
97
|
-
_, mu, std, const =
|
|
143
|
+
_, mu, std, const = array_file_parameter
|
|
98
144
|
|
|
99
145
|
assert type(task.parameters.randomness) is tuple
|
|
100
146
|
np.testing.assert_allclose(np.array(task.parameters.randomness), np.array([mu, std]), rtol=1)
|
|
101
147
|
|
|
102
148
|
assert task.parameters.constant == const
|
|
103
149
|
assert task.parameters.wavelength_category == "two"
|
|
150
|
+
assert task.parameters.value_message == early_value_message_str
|
|
151
|
+
assert task.parameters.file_message == early_file_message_str
|
|
@@ -6,8 +6,8 @@ from dataclasses import asdict
|
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from dataclasses import is_dataclass
|
|
8
8
|
from datetime import datetime
|
|
9
|
+
from datetime import timedelta
|
|
9
10
|
from random import randint
|
|
10
|
-
from typing import Type
|
|
11
11
|
from uuid import uuid4
|
|
12
12
|
|
|
13
13
|
import numpy as np
|
|
@@ -18,11 +18,11 @@ from dkist_header_validator import spec122_validator
|
|
|
18
18
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
19
19
|
from dkist_processing_common.codecs.fits import fits_hdu_decoder
|
|
20
20
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
21
|
+
from dkist_processing_common.codecs.json import json_encoder
|
|
21
22
|
from dkist_processing_common.models.constants import BudName
|
|
22
23
|
from dkist_processing_common.models.constants import ConstantsBase
|
|
23
24
|
from dkist_processing_common.models.tags import Tag
|
|
24
25
|
from dkist_processing_common.models.task_name import TaskName
|
|
25
|
-
from dkist_processing_common.tasks import QualityL0Metrics
|
|
26
26
|
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
27
27
|
from dkist_service_configuration.logging import logger
|
|
28
28
|
|
|
@@ -38,12 +38,16 @@ from dkist_processing_test.tasks.trial_output_data import TransferTestTrialData
|
|
|
38
38
|
from dkist_processing_test.tasks.write_l1 import WriteL1Data
|
|
39
39
|
from dkist_processing_test.tests.conftest import generate_214_l0_fits_frame
|
|
40
40
|
from dkist_processing_test.tests.conftest import S122Headers
|
|
41
|
+
from dkist_processing_test.tests.parameter_models import MultipleParameterValues
|
|
42
|
+
from dkist_processing_test.tests.parameter_models import ParameterValue
|
|
43
|
+
from dkist_processing_test.tests.parameter_models import RawFileParameterValue
|
|
44
|
+
from dkist_processing_test.tests.parameter_models import TestParameterValues
|
|
41
45
|
|
|
42
46
|
|
|
43
47
|
@dataclass
|
|
44
48
|
class FakeConstantDb:
|
|
45
49
|
NUM_DSPS_REPEATS: int = 2
|
|
46
|
-
OBS_IP_START_TIME: str = "
|
|
50
|
+
OBS_IP_START_TIME: str = "1990-06-12T12:00:00"
|
|
47
51
|
INSTRUMENT: str = "TEST"
|
|
48
52
|
AVERAGE_CADENCE: float = 10.0
|
|
49
53
|
MINIMUM_CADENCE: float = 10.0
|
|
@@ -94,12 +98,25 @@ def generate_calibrated_data_task(
|
|
|
94
98
|
recipe_run_id,
|
|
95
99
|
assign_input_dataset_doc_to_task,
|
|
96
100
|
link_constants_db,
|
|
97
|
-
|
|
101
|
+
array_parameter_file_object_key,
|
|
98
102
|
random_parameter_hdulist,
|
|
103
|
+
early_json_parameter_file_object_key,
|
|
104
|
+
early_file_message_str,
|
|
105
|
+
late_json_parameter_file_object_key,
|
|
106
|
+
late_file_message_str,
|
|
107
|
+
early_or_late,
|
|
108
|
+
late_date,
|
|
99
109
|
):
|
|
100
110
|
number_of_frames = 10
|
|
111
|
+
if early_or_late == "early":
|
|
112
|
+
obs_ip_start_time_str = (datetime.fromisoformat(late_date) - timedelta(days=30)).isoformat()
|
|
113
|
+
elif early_or_late == "late":
|
|
114
|
+
obs_ip_start_time_str = (datetime.fromisoformat(late_date) + timedelta(days=30)).isoformat()
|
|
101
115
|
link_constants_db(
|
|
102
|
-
recipe_run_id=recipe_run_id,
|
|
116
|
+
recipe_run_id=recipe_run_id,
|
|
117
|
+
constants_obj=FakeConstantDb(
|
|
118
|
+
NUM_DSPS_REPEATS=number_of_frames, OBS_IP_START_TIME=obs_ip_start_time_str
|
|
119
|
+
),
|
|
103
120
|
)
|
|
104
121
|
with GenerateCalibratedData(
|
|
105
122
|
recipe_run_id=recipe_run_id, workflow_name="GenerateCalibratedData", workflow_version="VX.Y"
|
|
@@ -125,13 +142,25 @@ def generate_calibrated_data_task(
|
|
|
125
142
|
data=hdul, tags=Tag.input(), relative_path=file_name, encoder=fits_hdulist_encoder
|
|
126
143
|
)
|
|
127
144
|
|
|
128
|
-
# Write parameter
|
|
145
|
+
# Write parameter files
|
|
129
146
|
hdul = random_parameter_hdulist[0]
|
|
130
147
|
task.write(
|
|
131
|
-
data=hdul,
|
|
148
|
+
data=hdul,
|
|
149
|
+
tags=Tag.parameter(array_parameter_file_object_key),
|
|
150
|
+
encoder=fits_hdulist_encoder,
|
|
151
|
+
)
|
|
152
|
+
task.write(
|
|
153
|
+
data=early_file_message_str,
|
|
154
|
+
tags=Tag.parameter(early_json_parameter_file_object_key),
|
|
155
|
+
encoder=json_encoder,
|
|
156
|
+
)
|
|
157
|
+
task.write(
|
|
158
|
+
data=late_file_message_str,
|
|
159
|
+
tags=Tag.parameter(late_json_parameter_file_object_key),
|
|
160
|
+
encoder=json_encoder,
|
|
132
161
|
)
|
|
133
162
|
|
|
134
|
-
# This needs to be after we've tagged the parameter
|
|
163
|
+
# This needs to be after we've written and tagged the parameter files
|
|
135
164
|
assign_input_dataset_doc_to_task(task, obs_ip_start_time=task.constants.obs_ip_start_time)
|
|
136
165
|
|
|
137
166
|
# result
|
|
@@ -142,55 +171,61 @@ def generate_calibrated_data_task(
|
|
|
142
171
|
|
|
143
172
|
|
|
144
173
|
@pytest.fixture(scope="session")
|
|
145
|
-
def
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
"parameterValue": json.dumps(value),
|
|
155
|
-
"parameterValueStartDate": "1946-11-20",
|
|
156
|
-
}
|
|
157
|
-
]
|
|
158
|
-
parameter = {"parameterName": param_name, "parameterValues": values}
|
|
159
|
-
parameters_list = [parameter]
|
|
174
|
+
def input_dataset_document_parameters_part_json(
|
|
175
|
+
array_parameter_file_object_key,
|
|
176
|
+
early_json_parameter_file_object_key,
|
|
177
|
+
late_json_parameter_file_object_key,
|
|
178
|
+
early_value_message_str,
|
|
179
|
+
late_value_message_str,
|
|
180
|
+
early_date,
|
|
181
|
+
late_date,
|
|
182
|
+
):
|
|
160
183
|
|
|
161
|
-
|
|
184
|
+
message_file_values = MultipleParameterValues(
|
|
185
|
+
parameter_value_list=[
|
|
186
|
+
ParameterValue(
|
|
187
|
+
parameterValue=RawFileParameterValue(
|
|
188
|
+
objectKey=early_json_parameter_file_object_key
|
|
189
|
+
),
|
|
190
|
+
parameterValueStartDate=early_date,
|
|
191
|
+
),
|
|
192
|
+
ParameterValue(
|
|
193
|
+
parameterValue=RawFileParameterValue(objectKey=late_json_parameter_file_object_key),
|
|
194
|
+
parameterValueStartDate=late_date,
|
|
195
|
+
),
|
|
196
|
+
]
|
|
197
|
+
)
|
|
198
|
+
message_value_values = MultipleParameterValues(
|
|
199
|
+
parameter_value_list=[
|
|
200
|
+
ParameterValue(
|
|
201
|
+
parameterValue=early_value_message_str,
|
|
202
|
+
parameterValueStartDate=early_date,
|
|
203
|
+
),
|
|
204
|
+
ParameterValue(
|
|
205
|
+
parameterValue=late_value_message_str,
|
|
206
|
+
parameterValueStartDate=late_date,
|
|
207
|
+
),
|
|
208
|
+
]
|
|
209
|
+
)
|
|
210
|
+
parameters_obj = TestParameterValues(
|
|
211
|
+
test_random_data=RawFileParameterValue(objectKey=array_parameter_file_object_key),
|
|
212
|
+
test_message_file=message_file_values,
|
|
213
|
+
test_message=message_value_values,
|
|
214
|
+
)
|
|
162
215
|
|
|
216
|
+
part_json_str = parameters_obj.model_dump_json()
|
|
163
217
|
|
|
164
|
-
|
|
165
|
-
def input_dataset_document_parameters_part_with_wavelength():
|
|
166
|
-
param_name = "test_wavelength_category"
|
|
167
|
-
value = {"wavelength": [1.0, 2.0, 3.0], "values": ["one", "two", "three"]}
|
|
168
|
-
value_id = randint(2000, 3000)
|
|
169
|
-
values = [
|
|
170
|
-
{
|
|
171
|
-
"parameterValueId": value_id,
|
|
172
|
-
"parameterValue": json.dumps(value),
|
|
173
|
-
"parameterValueStartDate": "1946-11-20",
|
|
174
|
-
}
|
|
175
|
-
]
|
|
176
|
-
parameter = {"parameterName": param_name, "parameterValues": values}
|
|
177
|
-
parameter_list = [parameter]
|
|
178
|
-
return parameter_list
|
|
218
|
+
return part_json_str
|
|
179
219
|
|
|
180
220
|
|
|
181
221
|
@pytest.fixture(scope="session")
|
|
182
222
|
def assign_input_dataset_doc_to_task(
|
|
183
|
-
|
|
184
|
-
input_dataset_document_parameters_part_with_wavelength,
|
|
223
|
+
input_dataset_document_parameters_part_json,
|
|
185
224
|
):
|
|
186
225
|
def update_task(task, obs_ip_start_time=None):
|
|
187
226
|
doc_path = task.scratch.workflow_base_path / "dataset_parameters.json"
|
|
188
|
-
full_parameters = (
|
|
189
|
-
input_dataset_document_parameters_part_with_file
|
|
190
|
-
+ input_dataset_document_parameters_part_with_wavelength
|
|
191
|
-
)
|
|
192
227
|
with open(doc_path, "w") as f:
|
|
193
|
-
f.write(
|
|
228
|
+
f.write(input_dataset_document_parameters_part_json)
|
|
194
229
|
task.tag(doc_path, Tag.input_dataset_parameters())
|
|
195
230
|
task.parameters = TestParameters(
|
|
196
231
|
task.input_dataset_parameters, wavelength=2.0, obs_ip_start_time=obs_ip_start_time
|
|
@@ -214,7 +249,16 @@ def constants_linker(recipe_run_id: int, constants_obj):
|
|
|
214
249
|
return
|
|
215
250
|
|
|
216
251
|
|
|
217
|
-
|
|
252
|
+
@pytest.mark.parametrize("early_or_late", [pytest.param("early"), pytest.param("late")])
|
|
253
|
+
def test_generate_calibrated_data(
|
|
254
|
+
generate_calibrated_data_task,
|
|
255
|
+
early_file_message_str,
|
|
256
|
+
late_file_message_str,
|
|
257
|
+
early_value_message_str,
|
|
258
|
+
late_value_message_str,
|
|
259
|
+
early_or_late,
|
|
260
|
+
mocker,
|
|
261
|
+
):
|
|
218
262
|
"""
|
|
219
263
|
Given: A GenerateCalibratedData task
|
|
220
264
|
When: Calling the task instance
|
|
@@ -241,6 +285,22 @@ def test_generate_calibrated_data(generate_calibrated_data_task, mocker):
|
|
|
241
285
|
assert len(debug_frame_paths) == 1
|
|
242
286
|
assert debug_frame_paths[0].exists()
|
|
243
287
|
|
|
288
|
+
# Verify correct date params were used
|
|
289
|
+
param_check_paths = list(task.read(tags=["PARAM_CHECK"]))
|
|
290
|
+
assert len(param_check_paths) == 1
|
|
291
|
+
with open(param_check_paths[0], "r") as f:
|
|
292
|
+
date_dict = json.load(f)
|
|
293
|
+
|
|
294
|
+
if early_or_late == "early":
|
|
295
|
+
expected_file_message = early_file_message_str
|
|
296
|
+
expected_value_message = early_value_message_str
|
|
297
|
+
elif early_or_late == "late":
|
|
298
|
+
expected_file_message = late_file_message_str
|
|
299
|
+
expected_value_message = late_value_message_str
|
|
300
|
+
|
|
301
|
+
assert date_dict["file_message"] == expected_file_message
|
|
302
|
+
assert date_dict["value_message"] == expected_value_message
|
|
303
|
+
|
|
244
304
|
|
|
245
305
|
class CommonDataset(Spec122Dataset):
|
|
246
306
|
# NOTE: We use ViSP data for unit tests because ViSP can be polarimetric
|
{dkist_processing_test-1.21.4.dist-info → dkist_processing_test-1.22.0rc1.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dkist-processing-test
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.22.0rc1
|
|
4
4
|
Summary: Example instrument code used by the DKIST science data processing pipelines to test processing infrastructure
|
|
5
5
|
Author-email: NSO / AURA <dkistdc@nso.edu>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -2,11 +2,11 @@ dkist_processing_test/__init__.py,sha256=t-Daj1wOIg5dgI1gklg6suSnI-xi6DKSb6DxHDW
|
|
|
2
2
|
dkist_processing_test/config.py,sha256=LF80ReTk0ggJ3eZI4NZRZCgmR6E84adWU5iP6-JN-XY,383
|
|
3
3
|
dkist_processing_test/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
dkist_processing_test/models/constants.py,sha256=TxHYZHl3YQXN7-Voj7QgjgAajbxxB_qR8tRI9iTdERI,726
|
|
5
|
-
dkist_processing_test/models/parameters.py,sha256=
|
|
5
|
+
dkist_processing_test/models/parameters.py,sha256=OLrS1O_xZptjdNW882Otx5cHWyd4FH7nzi8fkZnj9GM,1778
|
|
6
6
|
dkist_processing_test/tasks/__init__.py,sha256=ZWoqlNgFrn2auTMGNX9YWmHW_3JwQzn04XE_FmCAU24,517
|
|
7
7
|
dkist_processing_test/tasks/exercise_numba.py,sha256=XfMVffPUgeiPoxSgo39tIjzFBJKKt3fnIYGaDFyBibc,1437
|
|
8
8
|
dkist_processing_test/tasks/fail.py,sha256=jiOiuqoX_JR5hx81REgvcSvb0GBlVKc9MIXKUO0Nhr4,245
|
|
9
|
-
dkist_processing_test/tasks/fake_science.py,sha256=
|
|
9
|
+
dkist_processing_test/tasks/fake_science.py,sha256=L5994d1amaeOT24noWoEsBKfuoabeJ7Pdyp3TnbLHe8,4496
|
|
10
10
|
dkist_processing_test/tasks/high_memory.py,sha256=J3vBsivFkqs8eZZwBPpEORlHnFX1bGsbO3CjllJMRsc,540
|
|
11
11
|
dkist_processing_test/tasks/manual.py,sha256=gjol_EieMbclv0ZDkVqR2Xd7abutali6KNwoqaCJiAI,583
|
|
12
12
|
dkist_processing_test/tasks/movie.py,sha256=uFPWQlRFP5zzzYGT58IXebbG49su2I4AEpN2bIy7vGc,1831
|
|
@@ -16,9 +16,10 @@ dkist_processing_test/tasks/quality.py,sha256=9QynI9R5mkYueb46wNBUhK62QSd2n791L0
|
|
|
16
16
|
dkist_processing_test/tasks/trial_output_data.py,sha256=T34kGFxyR0IVaYnJhP3s9kv1_ExSNlnxx0n76cN32NU,1553
|
|
17
17
|
dkist_processing_test/tasks/write_l1.py,sha256=lpNyCI3kH_pnL0Wi1LHhNRjRHNxu6q76xccCNHm7uwA,2074
|
|
18
18
|
dkist_processing_test/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
dkist_processing_test/tests/conftest.py,sha256=
|
|
20
|
-
dkist_processing_test/tests/
|
|
21
|
-
dkist_processing_test/tests/
|
|
19
|
+
dkist_processing_test/tests/conftest.py,sha256=9H8_tqNKofHnl1IRqYM2-M2_K_TJ3b6oaIfb1dVkbsg,2728
|
|
20
|
+
dkist_processing_test/tests/parameter_models.py,sha256=c-SqCBqHP4KkFs0_Y7HLED4mH4oMCpAYu-rYVlTHy1M,4635
|
|
21
|
+
dkist_processing_test/tests/test_parameters.py,sha256=KGeEUOhtC_l-LN6hjB2jgwUZIWAMBx8wlmEeIja-eKo,4572
|
|
22
|
+
dkist_processing_test/tests/test_tasks.py,sha256=t3jNEBucxLndzbE2qClKd_huYo-7T4w7qqp6ImqyLLc,27628
|
|
22
23
|
dkist_processing_test/tests/test_workflows.py,sha256=NqRkstqcqwH80cKJ1uDw16G4MsDOuOiZRWdfH0STarQ,286
|
|
23
24
|
dkist_processing_test/workflows/__init__.py,sha256=Ryn_HXsVkZM_lLKyVteXC909Td2geylmz_1Z-8SE8c8,78
|
|
24
25
|
dkist_processing_test/workflows/common_tasks.py,sha256=4H1TaUNa5-HyE1NJQiAc3NAN62ZOTYb7MbR1yjKMg-k,6843
|
|
@@ -28,7 +29,7 @@ dkist_processing_test/workflows/fail.py,sha256=KGhyAF7gKYYutP4aGa-1tDqPsC7Nr4xKY
|
|
|
28
29
|
dkist_processing_test/workflows/noop.py,sha256=k2-6BpRkl1JDGaHPavxDzIlVx11KQtxKmsHmHTNQ9o0,666
|
|
29
30
|
dkist_processing_test/workflows/resource_queue.py,sha256=_jOT99O0MFN_ACne6i8T5ZIwe_lBQqGz2Ccb8JlqQMI,500
|
|
30
31
|
dkist_processing_test/workflows/trial_end_to_end.py,sha256=wkl_gk0zQ5TJ41CY0gEREPSIldZdTtLGwPDL_iwObe4,2425
|
|
31
|
-
dkist_processing_test-1.
|
|
32
|
-
dkist_processing_test-1.
|
|
33
|
-
dkist_processing_test-1.
|
|
34
|
-
dkist_processing_test-1.
|
|
32
|
+
dkist_processing_test-1.22.0rc1.dist-info/METADATA,sha256=x2uSqgJndQx33siO1UZE6pq20yrNufSrLPrdk8kE4AI,3295
|
|
33
|
+
dkist_processing_test-1.22.0rc1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
34
|
+
dkist_processing_test-1.22.0rc1.dist-info/top_level.txt,sha256=Hs4oTIrG_r-svhk_RGFTEO4e3vqQoYlBzdv5mvJVF24,22
|
|
35
|
+
dkist_processing_test-1.22.0rc1.dist-info/RECORD,,
|
{dkist_processing_test-1.21.4.dist-info → dkist_processing_test-1.22.0rc1.dist-info}/top_level.txt
RENAMED
|
File without changes
|