dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- changelog/280.misc.rst +1 -0
- changelog/282.feature.2.rst +2 -0
- changelog/282.feature.rst +2 -0
- changelog/284.feature.rst +1 -0
- changelog/285.feature.rst +2 -0
- changelog/285.misc.rst +2 -0
- changelog/286.feature.rst +2 -0
- changelog/287.misc.rst +1 -0
- dkist_processing_common/__init__.py +1 -0
- dkist_processing_common/_util/constants.py +1 -0
- dkist_processing_common/_util/graphql.py +1 -0
- dkist_processing_common/_util/scratch.py +9 -9
- dkist_processing_common/_util/tags.py +1 -0
- dkist_processing_common/codecs/array.py +20 -0
- dkist_processing_common/codecs/asdf.py +9 -3
- dkist_processing_common/codecs/basemodel.py +22 -0
- dkist_processing_common/codecs/bytes.py +1 -0
- dkist_processing_common/codecs/fits.py +37 -9
- dkist_processing_common/codecs/iobase.py +1 -0
- dkist_processing_common/codecs/json.py +1 -0
- dkist_processing_common/codecs/path.py +1 -0
- dkist_processing_common/codecs/quality.py +1 -1
- dkist_processing_common/codecs/str.py +1 -0
- dkist_processing_common/config.py +64 -25
- dkist_processing_common/manual.py +6 -8
- dkist_processing_common/models/constants.py +373 -37
- dkist_processing_common/models/dkist_location.py +27 -0
- dkist_processing_common/models/fits_access.py +48 -0
- dkist_processing_common/models/flower_pot.py +231 -9
- dkist_processing_common/models/fried_parameter.py +41 -0
- dkist_processing_common/models/graphql.py +66 -75
- dkist_processing_common/models/input_dataset.py +117 -0
- dkist_processing_common/models/message.py +1 -1
- dkist_processing_common/models/message_queue_binding.py +1 -1
- dkist_processing_common/models/metric_code.py +2 -0
- dkist_processing_common/models/parameters.py +65 -28
- dkist_processing_common/models/quality.py +50 -5
- dkist_processing_common/models/tags.py +23 -21
- dkist_processing_common/models/task_name.py +3 -2
- dkist_processing_common/models/telemetry.py +28 -0
- dkist_processing_common/models/wavelength.py +3 -1
- dkist_processing_common/parsers/average_bud.py +46 -0
- dkist_processing_common/parsers/cs_step.py +13 -12
- dkist_processing_common/parsers/dsps_repeat.py +6 -4
- dkist_processing_common/parsers/experiment_id_bud.py +12 -4
- dkist_processing_common/parsers/id_bud.py +42 -27
- dkist_processing_common/parsers/l0_fits_access.py +5 -3
- dkist_processing_common/parsers/l1_fits_access.py +51 -23
- dkist_processing_common/parsers/lookup_bud.py +125 -0
- dkist_processing_common/parsers/near_bud.py +21 -20
- dkist_processing_common/parsers/observing_program_id_bud.py +24 -0
- dkist_processing_common/parsers/proposal_id_bud.py +13 -5
- dkist_processing_common/parsers/quality.py +2 -0
- dkist_processing_common/parsers/retarder.py +32 -0
- dkist_processing_common/parsers/single_value_single_key_flower.py +6 -1
- dkist_processing_common/parsers/task.py +8 -6
- dkist_processing_common/parsers/time.py +178 -72
- dkist_processing_common/parsers/unique_bud.py +21 -22
- dkist_processing_common/parsers/wavelength.py +5 -3
- dkist_processing_common/tasks/__init__.py +3 -2
- dkist_processing_common/tasks/assemble_movie.py +4 -3
- dkist_processing_common/tasks/base.py +59 -60
- dkist_processing_common/tasks/l1_output_data.py +54 -53
- dkist_processing_common/tasks/mixin/globus.py +24 -27
- dkist_processing_common/tasks/mixin/interservice_bus.py +1 -0
- dkist_processing_common/tasks/mixin/metadata_store.py +108 -243
- dkist_processing_common/tasks/mixin/object_store.py +22 -0
- dkist_processing_common/tasks/mixin/quality/__init__.py +1 -0
- dkist_processing_common/tasks/mixin/quality/_base.py +8 -1
- dkist_processing_common/tasks/mixin/quality/_metrics.py +166 -14
- dkist_processing_common/tasks/output_data_base.py +4 -3
- dkist_processing_common/tasks/parse_l0_input_data.py +277 -15
- dkist_processing_common/tasks/quality_metrics.py +9 -9
- dkist_processing_common/tasks/teardown.py +7 -7
- dkist_processing_common/tasks/transfer_input_data.py +67 -69
- dkist_processing_common/tasks/trial_catalog.py +77 -17
- dkist_processing_common/tasks/trial_output_data.py +16 -17
- dkist_processing_common/tasks/write_l1.py +102 -72
- dkist_processing_common/tests/conftest.py +32 -173
- dkist_processing_common/tests/mock_metadata_store.py +271 -0
- dkist_processing_common/tests/test_assemble_movie.py +4 -4
- dkist_processing_common/tests/test_assemble_quality.py +32 -4
- dkist_processing_common/tests/test_base.py +5 -19
- dkist_processing_common/tests/test_codecs.py +103 -12
- dkist_processing_common/tests/test_constants.py +15 -0
- dkist_processing_common/tests/test_dkist_location.py +15 -0
- dkist_processing_common/tests/test_fits_access.py +56 -19
- dkist_processing_common/tests/test_flower_pot.py +147 -5
- dkist_processing_common/tests/test_fried_parameter.py +27 -0
- dkist_processing_common/tests/test_input_dataset.py +78 -361
- dkist_processing_common/tests/test_interservice_bus.py +1 -0
- dkist_processing_common/tests/test_interservice_bus_mixin.py +1 -1
- dkist_processing_common/tests/test_manual_processing.py +33 -0
- dkist_processing_common/tests/test_output_data_base.py +5 -7
- dkist_processing_common/tests/test_parameters.py +71 -22
- dkist_processing_common/tests/test_parse_l0_input_data.py +115 -32
- dkist_processing_common/tests/test_publish_catalog_messages.py +2 -24
- dkist_processing_common/tests/test_quality.py +1 -0
- dkist_processing_common/tests/test_quality_mixin.py +255 -23
- dkist_processing_common/tests/test_scratch.py +2 -1
- dkist_processing_common/tests/test_stems.py +511 -168
- dkist_processing_common/tests/test_submit_dataset_metadata.py +3 -7
- dkist_processing_common/tests/test_tags.py +1 -0
- dkist_processing_common/tests/test_task_name.py +1 -1
- dkist_processing_common/tests/test_task_parsing.py +17 -7
- dkist_processing_common/tests/test_teardown.py +28 -24
- dkist_processing_common/tests/test_transfer_input_data.py +270 -125
- dkist_processing_common/tests/test_transfer_l1_output_data.py +2 -3
- dkist_processing_common/tests/test_trial_catalog.py +83 -8
- dkist_processing_common/tests/test_trial_output_data.py +46 -73
- dkist_processing_common/tests/test_workflow_task_base.py +8 -10
- dkist_processing_common/tests/test_write_l1.py +298 -76
- dkist_processing_common-12.1.0rc1.dist-info/METADATA +265 -0
- dkist_processing_common-12.1.0rc1.dist-info/RECORD +134 -0
- {dkist_processing_common-10.5.4.dist-info → dkist_processing_common-12.1.0rc1.dist-info}/WHEEL +1 -1
- docs/conf.py +1 -0
- docs/index.rst +1 -1
- docs/landing_page.rst +13 -0
- dkist_processing_common/tasks/mixin/input_dataset.py +0 -166
- dkist_processing_common-10.5.4.dist-info/METADATA +0 -175
- dkist_processing_common-10.5.4.dist-info/RECORD +0 -112
- {dkist_processing_common-10.5.4.dist-info → dkist_processing_common-12.1.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,14 +1,21 @@
|
|
|
1
1
|
"""Experiment Id parser."""
|
|
2
|
+
|
|
2
3
|
from dkist_processing_common.models.constants import BudName
|
|
4
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
5
|
+
from dkist_processing_common.models.task_name import TaskName
|
|
3
6
|
from dkist_processing_common.parsers.id_bud import ContributingIdsBud
|
|
4
|
-
from dkist_processing_common.parsers.
|
|
7
|
+
from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
|
|
5
8
|
|
|
6
9
|
|
|
7
|
-
class ExperimentIdBud(
|
|
10
|
+
class ExperimentIdBud(TaskUniqueBud):
|
|
8
11
|
"""Class to create a Bud for the experiment_id."""
|
|
9
12
|
|
|
10
13
|
def __init__(self):
|
|
11
|
-
super().__init__(
|
|
14
|
+
super().__init__(
|
|
15
|
+
constant_name=BudName.experiment_id,
|
|
16
|
+
metadata_key=MetadataKey.experiment_id,
|
|
17
|
+
ip_task_types=TaskName.observe,
|
|
18
|
+
)
|
|
12
19
|
|
|
13
20
|
|
|
14
21
|
class ContributingExperimentIdsBud(ContributingIdsBud):
|
|
@@ -16,5 +23,6 @@ class ContributingExperimentIdsBud(ContributingIdsBud):
|
|
|
16
23
|
|
|
17
24
|
def __init__(self):
|
|
18
25
|
super().__init__(
|
|
19
|
-
|
|
26
|
+
constant_name=BudName.contributing_experiment_ids,
|
|
27
|
+
metadata_key=MetadataKey.experiment_id,
|
|
20
28
|
)
|
|
@@ -1,32 +1,25 @@
|
|
|
1
1
|
"""Base classes for ID bud parsing."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
from typing import Callable
|
|
2
5
|
from typing import Type
|
|
3
6
|
|
|
7
|
+
from dkist_processing_common.models.flower_pot import SetStem
|
|
4
8
|
from dkist_processing_common.models.flower_pot import SpilledDirt
|
|
5
|
-
from dkist_processing_common.models.flower_pot import Stem
|
|
6
|
-
from dkist_processing_common.models.task_name import TaskName
|
|
7
9
|
from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
|
|
8
|
-
from dkist_processing_common.parsers.
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class IdBud(TaskUniqueBud):
|
|
12
|
-
"""Base class for ID buds."""
|
|
13
|
-
|
|
14
|
-
def __init__(self, constant_name, metadata_key):
|
|
15
|
-
super().__init__(
|
|
16
|
-
constant_name=constant_name,
|
|
17
|
-
metadata_key=metadata_key,
|
|
18
|
-
ip_task_type=TaskName.observe.value,
|
|
19
|
-
)
|
|
10
|
+
from dkist_processing_common.parsers.task import passthrough_header_ip_task
|
|
20
11
|
|
|
21
12
|
|
|
22
|
-
class ContributingIdsBud(
|
|
13
|
+
class ContributingIdsBud(SetStem):
|
|
23
14
|
"""Base class for contributing ID buds."""
|
|
24
15
|
|
|
25
|
-
def __init__(self,
|
|
26
|
-
super().__init__(stem_name=
|
|
16
|
+
def __init__(self, constant_name: str, metadata_key: str | StrEnum):
|
|
17
|
+
super().__init__(stem_name=constant_name)
|
|
18
|
+
if isinstance(metadata_key, StrEnum):
|
|
19
|
+
metadata_key = metadata_key.name
|
|
27
20
|
self.metadata_key = metadata_key
|
|
28
21
|
|
|
29
|
-
def setter(self, fits_obj: L0FitsAccess) -> str
|
|
22
|
+
def setter(self, fits_obj: L0FitsAccess) -> str:
|
|
30
23
|
"""
|
|
31
24
|
Set the id for any type of frame.
|
|
32
25
|
|
|
@@ -40,17 +33,39 @@ class ContributingIdsBud(Stem):
|
|
|
40
33
|
"""
|
|
41
34
|
return getattr(fits_obj, self.metadata_key)
|
|
42
35
|
|
|
43
|
-
def getter(self
|
|
36
|
+
def getter(self) -> tuple[str, ...]:
|
|
44
37
|
"""
|
|
45
|
-
Get all ids seen
|
|
46
|
-
|
|
47
|
-
Parameters
|
|
48
|
-
----------
|
|
49
|
-
key
|
|
50
|
-
The input key
|
|
38
|
+
Get all ids seen for any type of frame.
|
|
51
39
|
|
|
52
40
|
Returns
|
|
53
41
|
-------
|
|
54
|
-
IDs from
|
|
42
|
+
IDs from all types of frames
|
|
55
43
|
"""
|
|
56
|
-
return tuple(
|
|
44
|
+
return tuple(self.value_set)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TaskContributingIdsBud(ContributingIdsBud):
|
|
48
|
+
"""Base class for contributing ID buds for a particular task type."""
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
constant_name: str,
|
|
53
|
+
metadata_key: str | StrEnum,
|
|
54
|
+
ip_task_types: str | list[str],
|
|
55
|
+
task_type_parsing_function: Callable = passthrough_header_ip_task,
|
|
56
|
+
):
|
|
57
|
+
super().__init__(constant_name=constant_name, metadata_key=metadata_key)
|
|
58
|
+
|
|
59
|
+
if isinstance(ip_task_types, str):
|
|
60
|
+
ip_task_types = [ip_task_types]
|
|
61
|
+
self.ip_task_types = [task.casefold() for task in ip_task_types]
|
|
62
|
+
self.parsing_function = task_type_parsing_function
|
|
63
|
+
|
|
64
|
+
def setter(self, fits_obj: L0FitsAccess) -> str | Type[SpilledDirt]:
|
|
65
|
+
"""Ingest an object only if its parsed IP task type matches what's desired."""
|
|
66
|
+
task = self.parsing_function(fits_obj)
|
|
67
|
+
|
|
68
|
+
if task.casefold() in self.ip_task_types:
|
|
69
|
+
return super().setter(fits_obj)
|
|
70
|
+
|
|
71
|
+
return SpilledDirt
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"""By-frame 214 L0 header keywords that are not instrument specific."""
|
|
2
|
+
|
|
2
3
|
from astropy.io import fits
|
|
3
4
|
|
|
5
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
4
6
|
from dkist_processing_common.parsers.l1_fits_access import L1FitsAccess
|
|
5
7
|
|
|
6
8
|
|
|
@@ -25,6 +27,6 @@ class L0FitsAccess(L1FitsAccess):
|
|
|
25
27
|
auto_squeeze: bool = True,
|
|
26
28
|
):
|
|
27
29
|
super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
|
|
28
|
-
self.ip_task_type: str = self.header[
|
|
29
|
-
self.ip_start_time: str = self.header[
|
|
30
|
-
self.ip_end_time: str = self.header[
|
|
30
|
+
self.ip_task_type: str = self.header[MetadataKey.ip_task_type]
|
|
31
|
+
self.ip_start_time: str = self.header[MetadataKey.ip_start_time]
|
|
32
|
+
self.ip_end_time: str = self.header[MetadataKey.ip_end_time]
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
"""By-frame 214 L1 only header keywords that are not instrument specific."""
|
|
2
|
+
|
|
2
3
|
from astropy.io import fits
|
|
3
4
|
|
|
5
|
+
from dkist_processing_common.models.fits_access import HEADER_KEY_NOT_FOUND
|
|
4
6
|
from dkist_processing_common.models.fits_access import FitsAccessBase
|
|
7
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
8
|
+
|
|
9
|
+
NOT_A_FLOAT = -999
|
|
5
10
|
|
|
6
11
|
|
|
7
12
|
class L1FitsAccess(FitsAccessBase):
|
|
@@ -26,38 +31,61 @@ class L1FitsAccess(FitsAccessBase):
|
|
|
26
31
|
):
|
|
27
32
|
super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
|
|
28
33
|
|
|
29
|
-
self.elevation: float = self.header[
|
|
30
|
-
self.azimuth: float = self.header[
|
|
31
|
-
self.table_angle: float = self.header[
|
|
32
|
-
self.gos_level3_status: str = self.header[
|
|
33
|
-
self.gos_level3_lamp_status: str = self.header[
|
|
34
|
-
self.gos_polarizer_status: str = self.header[
|
|
35
|
-
self.gos_retarder_status: str = self.header[
|
|
36
|
-
self.gos_level0_status: str = self.header[
|
|
37
|
-
self.time_obs: str = self.header[
|
|
38
|
-
self.ip_id: str = self.header[
|
|
39
|
-
self.instrument: str = self.header[
|
|
40
|
-
self.wavelength: float = self.header[
|
|
41
|
-
self.proposal_id: str = self.header[
|
|
42
|
-
self.experiment_id: str = self.header[
|
|
43
|
-
self.num_dsps_repeats: int = self.header[
|
|
44
|
-
self.current_dsps_repeat: int = self.header[
|
|
45
|
-
self.fpa_exposure_time_ms: float = self.header[
|
|
46
|
-
self.sensor_readout_exposure_time_ms: float = self.header[
|
|
47
|
-
|
|
34
|
+
self.elevation: float = self.header[MetadataKey.elevation]
|
|
35
|
+
self.azimuth: float = self.header[MetadataKey.azimuth]
|
|
36
|
+
self.table_angle: float = self.header[MetadataKey.table_angle]
|
|
37
|
+
self.gos_level3_status: str = self.header[MetadataKey.gos_level3_status]
|
|
38
|
+
self.gos_level3_lamp_status: str = self.header[MetadataKey.gos_level3_lamp_status]
|
|
39
|
+
self.gos_polarizer_status: str = self.header[MetadataKey.gos_polarizer_status]
|
|
40
|
+
self.gos_retarder_status: str = self.header[MetadataKey.gos_retarder_status]
|
|
41
|
+
self.gos_level0_status: str = self.header[MetadataKey.gos_level0_status]
|
|
42
|
+
self.time_obs: str = self.header[MetadataKey.time_obs]
|
|
43
|
+
self.ip_id: str = self.header[MetadataKey.ip_id]
|
|
44
|
+
self.instrument: str = self.header[MetadataKey.instrument]
|
|
45
|
+
self.wavelength: float = self.header[MetadataKey.wavelength]
|
|
46
|
+
self.proposal_id: str = self.header[MetadataKey.proposal_id]
|
|
47
|
+
self.experiment_id: str = self.header[MetadataKey.experiment_id]
|
|
48
|
+
self.num_dsps_repeats: int = self.header[MetadataKey.num_dsps_repeats]
|
|
49
|
+
self.current_dsps_repeat: int = self.header[MetadataKey.current_dsps_repeat]
|
|
50
|
+
self.fpa_exposure_time_ms: float = self.header[MetadataKey.fpa_exposure_time_ms]
|
|
51
|
+
self.sensor_readout_exposure_time_ms: float = self.header[
|
|
52
|
+
MetadataKey.sensor_readout_exposure_time_ms
|
|
53
|
+
]
|
|
54
|
+
self.num_raw_frames_per_fpa: int = self.header[MetadataKey.num_raw_frames_per_fpa]
|
|
55
|
+
self.camera_id: str = self.header[MetadataKey.camera_id]
|
|
56
|
+
self.camera_name: str = self.header[MetadataKey.camera_name]
|
|
57
|
+
self.camera_bit_depth: int = self.header[MetadataKey.camera_bit_depth]
|
|
58
|
+
self.hardware_binning_x: int = self.header[MetadataKey.hardware_binning_x]
|
|
59
|
+
self.hardware_binning_y: int = self.header[MetadataKey.hardware_binning_y]
|
|
60
|
+
self.software_binning_x: int = self.header[MetadataKey.software_binning_x]
|
|
61
|
+
self.software_binning_y: int = self.header[MetadataKey.software_binning_y]
|
|
62
|
+
self.observing_program_execution_id: str = self.header[
|
|
63
|
+
MetadataKey.observing_program_execution_id
|
|
64
|
+
]
|
|
65
|
+
self.telescope_tracking_mode: str = self.header.get(
|
|
66
|
+
MetadataKey.telescope_tracking_mode, HEADER_KEY_NOT_FOUND
|
|
67
|
+
)
|
|
68
|
+
self.coude_table_tracking_mode: str = self.header.get(
|
|
69
|
+
MetadataKey.coude_table_tracking_mode, HEADER_KEY_NOT_FOUND
|
|
70
|
+
)
|
|
71
|
+
self.telescope_scanning_mode: str = self.header.get(
|
|
72
|
+
MetadataKey.telescope_scanning_mode, HEADER_KEY_NOT_FOUND
|
|
73
|
+
)
|
|
74
|
+
self.light_level: float = self.header[MetadataKey.light_level]
|
|
75
|
+
self.hls_version: str = self.header[MetadataKey.hls_version]
|
|
48
76
|
|
|
49
77
|
@property
|
|
50
78
|
def gos_polarizer_angle(self) -> float:
|
|
51
79
|
"""Convert the polarizer angle to a float if possible before returning."""
|
|
52
80
|
try:
|
|
53
|
-
return float(self.header[
|
|
81
|
+
return float(self.header[MetadataKey.gos_polarizer_angle])
|
|
54
82
|
except ValueError:
|
|
55
|
-
return
|
|
83
|
+
return NOT_A_FLOAT # The angle is only used if the polarizer is in the beam
|
|
56
84
|
|
|
57
85
|
@property
|
|
58
86
|
def gos_retarder_angle(self) -> float:
|
|
59
87
|
"""Convert the retarder angle to a float if possible before returning."""
|
|
60
88
|
try:
|
|
61
|
-
return float(self.header[
|
|
89
|
+
return float(self.header[MetadataKey.gos_retarder_angle])
|
|
62
90
|
except ValueError:
|
|
63
|
-
return
|
|
91
|
+
return NOT_A_FLOAT # The angle is only used if the retarder is in the beam
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Simple bud that is used to set a constant to a mapping dictionary."""
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from enum import StrEnum
|
|
5
|
+
from typing import Any
|
|
6
|
+
from typing import Callable
|
|
7
|
+
from typing import DefaultDict
|
|
8
|
+
|
|
9
|
+
from dkist_processing_common.models.flower_pot import SetStem
|
|
10
|
+
from dkist_processing_common.models.flower_pot import SpilledDirt
|
|
11
|
+
from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
|
|
12
|
+
from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
|
|
13
|
+
from dkist_processing_common.parsers.task import passthrough_header_ip_task
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TimeLookupBud(SetStem):
|
|
17
|
+
"""
|
|
18
|
+
Bud that reads two header keys from all files and creates a dictionary mapping a time KEY value to sets of a VALUE value.
|
|
19
|
+
|
|
20
|
+
Parameters
|
|
21
|
+
----------
|
|
22
|
+
constant_name
|
|
23
|
+
The name for the constant to be defined
|
|
24
|
+
|
|
25
|
+
key_metadata_key
|
|
26
|
+
The time metadata key for the resulting dictionary key
|
|
27
|
+
|
|
28
|
+
value_metadata_key
|
|
29
|
+
The metadata key for the resulting dictionary value
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
constant_name: str,
|
|
35
|
+
key_metadata_key: str | StrEnum,
|
|
36
|
+
value_metadata_key: str | StrEnum,
|
|
37
|
+
):
|
|
38
|
+
super().__init__(stem_name=constant_name)
|
|
39
|
+
|
|
40
|
+
if isinstance(key_metadata_key, StrEnum):
|
|
41
|
+
key_metadata_key = key_metadata_key.name
|
|
42
|
+
self.key_metadata_key = key_metadata_key
|
|
43
|
+
if isinstance(value_metadata_key, StrEnum):
|
|
44
|
+
value_metadata_key = value_metadata_key.name
|
|
45
|
+
self.value_metadata_key = value_metadata_key
|
|
46
|
+
|
|
47
|
+
self.mapping: DefaultDict[float, set[Any]] = defaultdict(set)
|
|
48
|
+
|
|
49
|
+
def setter(self, fits_obj: L0FitsAccess):
|
|
50
|
+
"""
|
|
51
|
+
Update the mapping dictionary.
|
|
52
|
+
|
|
53
|
+
Parameters
|
|
54
|
+
----------
|
|
55
|
+
fits_obj
|
|
56
|
+
The input fits object
|
|
57
|
+
Returns
|
|
58
|
+
-------
|
|
59
|
+
Updates the dictionary and returns None
|
|
60
|
+
"""
|
|
61
|
+
key = getattr(fits_obj, self.key_metadata_key)
|
|
62
|
+
rounded_key = round(key, EXP_TIME_ROUND_DIGITS)
|
|
63
|
+
value = getattr(fits_obj, self.value_metadata_key)
|
|
64
|
+
self.mapping[rounded_key].add(value)
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
def getter(self):
|
|
68
|
+
"""
|
|
69
|
+
Get the dictionary mapping created by the setter with values converted to JSON-able lists.
|
|
70
|
+
|
|
71
|
+
Returns
|
|
72
|
+
-------
|
|
73
|
+
The mapping dictionary with values converted to JSON-able lists
|
|
74
|
+
"""
|
|
75
|
+
mapping_lists = {k: list(v) for k, v in self.mapping.items()}
|
|
76
|
+
return mapping_lists
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class TaskTimeLookupBud(TimeLookupBud):
|
|
80
|
+
"""
|
|
81
|
+
Subclass of `TimeLookupBud` that only considers objects that have specific task types.
|
|
82
|
+
|
|
83
|
+
Parameters
|
|
84
|
+
----------
|
|
85
|
+
constant_name
|
|
86
|
+
The name for the constant to be defined
|
|
87
|
+
|
|
88
|
+
key_metadata_key
|
|
89
|
+
The time metadata key for the resulting dictionary key
|
|
90
|
+
|
|
91
|
+
value_metadata_key
|
|
92
|
+
The metadata key for the resulting dictionary value
|
|
93
|
+
|
|
94
|
+
ip_task_types
|
|
95
|
+
Only consider objects whose parsed header IP task type matches a string in this list
|
|
96
|
+
|
|
97
|
+
task_type_parsing_function
|
|
98
|
+
The function used to convert a header into an IP task type
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
def __init__(
|
|
102
|
+
self,
|
|
103
|
+
constant_name: str,
|
|
104
|
+
key_metadata_key: str | StrEnum,
|
|
105
|
+
value_metadata_key: str | StrEnum,
|
|
106
|
+
ip_task_types: str | list[str],
|
|
107
|
+
task_type_parsing_function: Callable = passthrough_header_ip_task,
|
|
108
|
+
):
|
|
109
|
+
super().__init__(
|
|
110
|
+
constant_name=constant_name,
|
|
111
|
+
key_metadata_key=key_metadata_key,
|
|
112
|
+
value_metadata_key=value_metadata_key,
|
|
113
|
+
)
|
|
114
|
+
if isinstance(ip_task_types, str):
|
|
115
|
+
ip_task_types = [ip_task_types]
|
|
116
|
+
self.ip_task_types = [task.casefold() for task in ip_task_types]
|
|
117
|
+
self.parsing_function = task_type_parsing_function
|
|
118
|
+
|
|
119
|
+
def setter(self, fits_obj: L0FitsAccess):
|
|
120
|
+
"""Ingest an object only if its parsed IP task type matches what's desired."""
|
|
121
|
+
task = self.parsing_function(fits_obj)
|
|
122
|
+
if task.casefold() in self.ip_task_types:
|
|
123
|
+
return super().setter(fits_obj)
|
|
124
|
+
|
|
125
|
+
return SpilledDirt
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
"""Pre-made flower that reads a single header key from all files and raises a ValueError if the values are not in a supplied range."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
2
4
|
from statistics import mean
|
|
3
5
|
from typing import Callable
|
|
4
6
|
|
|
7
|
+
from dkist_processing_common.models.flower_pot import ListStem
|
|
5
8
|
from dkist_processing_common.models.flower_pot import SpilledDirt
|
|
6
|
-
from dkist_processing_common.models.flower_pot import Stem
|
|
7
9
|
from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
|
|
8
10
|
from dkist_processing_common.parsers.task import passthrough_header_ip_task
|
|
9
11
|
|
|
10
12
|
|
|
11
|
-
class NearFloatBud(
|
|
13
|
+
class NearFloatBud(ListStem):
|
|
12
14
|
"""
|
|
13
|
-
Pre-made
|
|
15
|
+
Pre-made `ListStem` that reads a single header key from all files and raises a ValueError if the values are not within a given tolerance.
|
|
14
16
|
|
|
15
17
|
This is intended for use with floats where the values may be slightly different, but should be the same.
|
|
16
18
|
|
|
@@ -29,10 +31,12 @@ class NearFloatBud(Stem):
|
|
|
29
31
|
def __init__(
|
|
30
32
|
self,
|
|
31
33
|
constant_name: str,
|
|
32
|
-
metadata_key: str,
|
|
34
|
+
metadata_key: str | StrEnum,
|
|
33
35
|
tolerance: float,
|
|
34
36
|
):
|
|
35
37
|
super().__init__(stem_name=constant_name)
|
|
38
|
+
if isinstance(metadata_key, StrEnum):
|
|
39
|
+
metadata_key = metadata_key.name
|
|
36
40
|
self.metadata_key = metadata_key
|
|
37
41
|
self.tolerance = tolerance
|
|
38
42
|
|
|
@@ -50,31 +54,26 @@ class NearFloatBud(Stem):
|
|
|
50
54
|
"""
|
|
51
55
|
return getattr(fits_obj, self.metadata_key)
|
|
52
56
|
|
|
53
|
-
def getter(self
|
|
57
|
+
def getter(self):
|
|
54
58
|
"""
|
|
55
59
|
Get the value for this key and raise an error if the data spans more than the given tolerance.
|
|
56
60
|
|
|
57
|
-
Parameters
|
|
58
|
-
----------
|
|
59
|
-
key
|
|
60
|
-
The input key
|
|
61
61
|
Returns
|
|
62
62
|
-------
|
|
63
63
|
The mean value associated with this input key
|
|
64
64
|
"""
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
smallest_value = min(value_set)
|
|
65
|
+
biggest_value = max(self.value_list)
|
|
66
|
+
smallest_value = min(self.value_list)
|
|
68
67
|
if biggest_value - smallest_value > self.tolerance:
|
|
69
68
|
raise ValueError(
|
|
70
69
|
f"{self.stem_name} values are not close enough. Max: {biggest_value}, Min: {smallest_value}, Tolerance: {self.tolerance}"
|
|
71
70
|
)
|
|
72
|
-
return mean(
|
|
71
|
+
return mean(self.value_list)
|
|
73
72
|
|
|
74
73
|
|
|
75
74
|
class TaskNearFloatBud(NearFloatBud):
|
|
76
75
|
"""
|
|
77
|
-
Subclass of `NearFloatBud` that only considers objects that have
|
|
76
|
+
Subclass of `NearFloatBud` that only considers objects that have specific task types.
|
|
78
77
|
|
|
79
78
|
Parameters
|
|
80
79
|
----------
|
|
@@ -84,8 +83,8 @@ class TaskNearFloatBud(NearFloatBud):
|
|
|
84
83
|
metadata_key
|
|
85
84
|
The metadata key associated with the constant
|
|
86
85
|
|
|
87
|
-
|
|
88
|
-
Only consider objects whose parsed header IP task type matches this
|
|
86
|
+
ip_task_types
|
|
87
|
+
Only consider objects whose parsed header IP task type matches a string in this list
|
|
89
88
|
|
|
90
89
|
task_type_parsing_function
|
|
91
90
|
The function used to convert a header into an IP task type
|
|
@@ -97,8 +96,8 @@ class TaskNearFloatBud(NearFloatBud):
|
|
|
97
96
|
def __init__(
|
|
98
97
|
self,
|
|
99
98
|
constant_name: str,
|
|
100
|
-
metadata_key: str,
|
|
101
|
-
|
|
99
|
+
metadata_key: str | StrEnum,
|
|
100
|
+
ip_task_types: str | list[str],
|
|
102
101
|
tolerance: float,
|
|
103
102
|
task_type_parsing_function: Callable = passthrough_header_ip_task,
|
|
104
103
|
):
|
|
@@ -106,14 +105,16 @@ class TaskNearFloatBud(NearFloatBud):
|
|
|
106
105
|
constant_name=constant_name, metadata_key=metadata_key, tolerance=tolerance
|
|
107
106
|
)
|
|
108
107
|
|
|
109
|
-
|
|
108
|
+
if isinstance(ip_task_types, str):
|
|
109
|
+
ip_task_types = [ip_task_types]
|
|
110
|
+
self.ip_task_types = [task.casefold() for task in ip_task_types]
|
|
110
111
|
self.parsing_function = task_type_parsing_function
|
|
111
112
|
|
|
112
113
|
def setter(self, fits_obj: L0FitsAccess):
|
|
113
114
|
"""Ingest an object only if its parsed IP task type matches what's desired."""
|
|
114
115
|
task = self.parsing_function(fits_obj)
|
|
115
116
|
|
|
116
|
-
if task.casefold()
|
|
117
|
+
if task.casefold() in self.ip_task_types:
|
|
117
118
|
return super().setter(fits_obj)
|
|
118
119
|
|
|
119
120
|
return SpilledDirt
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Observing Program Id parser."""
|
|
2
|
+
|
|
3
|
+
from typing import Callable
|
|
4
|
+
|
|
5
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
6
|
+
from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
|
|
7
|
+
from dkist_processing_common.parsers.task import passthrough_header_ip_task
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TaskContributingObservingProgramExecutionIdsBud(TaskContributingIdsBud):
|
|
11
|
+
"""Class to create a Bud for the supporting observing_program_execution_ids."""
|
|
12
|
+
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
constant_name: str,
|
|
16
|
+
ip_task_types: str | list[str],
|
|
17
|
+
task_type_parsing_function: Callable = passthrough_header_ip_task,
|
|
18
|
+
):
|
|
19
|
+
super().__init__(
|
|
20
|
+
constant_name=constant_name,
|
|
21
|
+
metadata_key=MetadataKey.observing_program_execution_id,
|
|
22
|
+
ip_task_types=ip_task_types,
|
|
23
|
+
task_type_parsing_function=task_type_parsing_function,
|
|
24
|
+
)
|
|
@@ -1,20 +1,28 @@
|
|
|
1
1
|
"""Proposal Id parser."""
|
|
2
|
+
|
|
2
3
|
from dkist_processing_common.models.constants import BudName
|
|
4
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
5
|
+
from dkist_processing_common.models.task_name import TaskName
|
|
3
6
|
from dkist_processing_common.parsers.id_bud import ContributingIdsBud
|
|
4
|
-
from dkist_processing_common.parsers.
|
|
7
|
+
from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
|
|
5
8
|
|
|
6
9
|
|
|
7
|
-
class ProposalIdBud(
|
|
10
|
+
class ProposalIdBud(TaskUniqueBud):
|
|
8
11
|
"""Class to create a Bud for the proposal_id."""
|
|
9
12
|
|
|
10
13
|
def __init__(self):
|
|
11
|
-
super().__init__(
|
|
14
|
+
super().__init__(
|
|
15
|
+
constant_name=BudName.proposal_id,
|
|
16
|
+
metadata_key=MetadataKey.proposal_id,
|
|
17
|
+
ip_task_types=TaskName.observe,
|
|
18
|
+
)
|
|
12
19
|
|
|
13
20
|
|
|
14
21
|
class ContributingProposalIdsBud(ContributingIdsBud):
|
|
15
|
-
"""Class to create a Bud for the proposal_ids."""
|
|
22
|
+
"""Class to create a Bud for the supporting proposal_ids."""
|
|
16
23
|
|
|
17
24
|
def __init__(self):
|
|
18
25
|
super().__init__(
|
|
19
|
-
|
|
26
|
+
constant_name=BudName.contributing_proposal_ids,
|
|
27
|
+
metadata_key=MetadataKey.proposal_id,
|
|
20
28
|
)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Support classes to define object attributes from header information."""
|
|
2
|
+
|
|
2
3
|
from astropy.io import fits
|
|
3
4
|
|
|
4
5
|
from dkist_processing_common.parsers.l0_fits_access import L1FitsAccess
|
|
@@ -31,3 +32,4 @@ class L1QualityFitsAccess(L1FitsAccess):
|
|
|
31
32
|
self.light_level: float = self.header["LIGHTLVL"]
|
|
32
33
|
self.health_status: str = self.header["DSHEALTH"]
|
|
33
34
|
self.ao_status: int = self.header.get("AO_LOCK", None)
|
|
35
|
+
self.num_out_of_bounds_ao_values: int = self.header.get("OOBSHIFT", None)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Bud that parses the name of the retarder used during POLCAL task observations."""
|
|
2
|
+
|
|
3
|
+
from dkist_processing_common.models.constants import BudName
|
|
4
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
5
|
+
from dkist_processing_common.models.flower_pot import SpilledDirt
|
|
6
|
+
from dkist_processing_common.models.task_name import TaskName
|
|
7
|
+
from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
|
|
8
|
+
from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RetarderNameBud(TaskUniqueBud):
|
|
12
|
+
"""
|
|
13
|
+
Bud for determining the name of the retarder used during a polcal Calibration Sequence (CS).
|
|
14
|
+
|
|
15
|
+
This is *slightly* different than a simple `TaskUniqueBud` because we need to allow for CS steps when the retarder
|
|
16
|
+
is out of the beam (i.g., "clear"). We do this by returning `SpilledDirt` from the `setter` if the value is "clear".
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self):
|
|
20
|
+
super().__init__(
|
|
21
|
+
constant_name=BudName.retarder_name,
|
|
22
|
+
metadata_key=MetadataKey.gos_retarder_status,
|
|
23
|
+
ip_task_types=TaskName.polcal,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
def setter(self, fits_obj: L0FitsAccess) -> type[SpilledDirt] | str:
|
|
27
|
+
"""Drop the result if the retarder is out of the beam ("clear")."""
|
|
28
|
+
result = super().setter(fits_obj)
|
|
29
|
+
if result is not SpilledDirt and result.casefold() == "clear":
|
|
30
|
+
return SpilledDirt
|
|
31
|
+
|
|
32
|
+
return result
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
"""Pre-made flower that produces tag based on a single header key."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
2
5
|
from dkist_processing_common.models.flower_pot import Stem
|
|
3
6
|
from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
|
|
4
7
|
|
|
@@ -15,8 +18,10 @@ class SingleValueSingleKeyFlower(Stem):
|
|
|
15
18
|
The metadata key
|
|
16
19
|
"""
|
|
17
20
|
|
|
18
|
-
def __init__(self, tag_stem_name: str, metadata_key: str):
|
|
21
|
+
def __init__(self, tag_stem_name: str, metadata_key: str | StrEnum):
|
|
19
22
|
super().__init__(stem_name=tag_stem_name)
|
|
23
|
+
if isinstance(metadata_key, StrEnum):
|
|
24
|
+
metadata_key = metadata_key.name
|
|
20
25
|
self.metadata_key = metadata_key
|
|
21
26
|
|
|
22
27
|
def setter(self, fits_obj: L0FitsAccess):
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
"""Module for parsing IP task related things."""
|
|
2
|
+
|
|
2
3
|
from typing import Callable
|
|
3
4
|
from typing import Type
|
|
4
5
|
|
|
5
6
|
from dkist_processing_common.models.fits_access import FitsAccessBase
|
|
7
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
6
8
|
from dkist_processing_common.models.flower_pot import SpilledDirt
|
|
7
9
|
from dkist_processing_common.models.tags import StemName
|
|
8
10
|
from dkist_processing_common.models.task_name import TaskName
|
|
@@ -35,9 +37,9 @@ def parse_header_ip_task_with_gains(fits_obj: FitsAccessBase) -> str:
|
|
|
35
37
|
and fits_obj.gos_level3_status == "lamp"
|
|
36
38
|
and fits_obj.gos_level3_lamp_status == "on"
|
|
37
39
|
):
|
|
38
|
-
return TaskName.lamp_gain
|
|
40
|
+
return TaskName.lamp_gain
|
|
39
41
|
if fits_obj.ip_task_type == "gain" and fits_obj.gos_level3_status == "clear":
|
|
40
|
-
return TaskName.solar_gain
|
|
42
|
+
return TaskName.solar_gain
|
|
41
43
|
|
|
42
44
|
# Everything else is unchanged
|
|
43
45
|
return passthrough_header_ip_task(fits_obj)
|
|
@@ -58,14 +60,14 @@ def parse_polcal_task_type(fits_obj: Type[FitsAccessBase]) -> str | Type[Spilled
|
|
|
58
60
|
and fits_obj.gos_retarder_status == "clear"
|
|
59
61
|
and fits_obj.gos_polarizer_status == "clear"
|
|
60
62
|
):
|
|
61
|
-
return TaskName.polcal_dark
|
|
63
|
+
return TaskName.polcal_dark
|
|
62
64
|
|
|
63
65
|
elif (
|
|
64
66
|
fits_obj.gos_level0_status.startswith("FieldStop")
|
|
65
67
|
and fits_obj.gos_retarder_status == "clear"
|
|
66
68
|
and fits_obj.gos_polarizer_status == "clear"
|
|
67
69
|
):
|
|
68
|
-
return TaskName.polcal_gain
|
|
70
|
+
return TaskName.polcal_gain
|
|
69
71
|
|
|
70
72
|
# We don't care about a POLCAL frame that is neither dark nor clear
|
|
71
73
|
return SpilledDirt
|
|
@@ -77,7 +79,7 @@ class TaskTypeFlower(SingleValueSingleKeyFlower):
|
|
|
77
79
|
def __init__(
|
|
78
80
|
self, header_task_parsing_func: Callable[[FitsAccessBase], str] = passthrough_header_ip_task
|
|
79
81
|
):
|
|
80
|
-
super().__init__(tag_stem_name=StemName.task
|
|
82
|
+
super().__init__(tag_stem_name=StemName.task, metadata_key=MetadataKey.ip_task_type)
|
|
81
83
|
self.header_parsing_function = header_task_parsing_func
|
|
82
84
|
|
|
83
85
|
def setter(self, fits_obj: FitsAccessBase):
|
|
@@ -102,7 +104,7 @@ class PolcalTaskFlower(SingleValueSingleKeyFlower):
|
|
|
102
104
|
"""
|
|
103
105
|
|
|
104
106
|
def __init__(self):
|
|
105
|
-
super().__init__(tag_stem_name=StemName.task
|
|
107
|
+
super().__init__(tag_stem_name=StemName.task, metadata_key=MetadataKey.ip_task_type)
|
|
106
108
|
|
|
107
109
|
def setter(self, fits_obj: FitsAccessBase):
|
|
108
110
|
"""
|