dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. changelog/280.misc.rst +1 -0
  2. changelog/282.feature.2.rst +2 -0
  3. changelog/282.feature.rst +2 -0
  4. changelog/284.feature.rst +1 -0
  5. changelog/285.feature.rst +2 -0
  6. changelog/285.misc.rst +2 -0
  7. changelog/286.feature.rst +2 -0
  8. changelog/287.misc.rst +1 -0
  9. dkist_processing_common/__init__.py +1 -0
  10. dkist_processing_common/_util/constants.py +1 -0
  11. dkist_processing_common/_util/graphql.py +1 -0
  12. dkist_processing_common/_util/scratch.py +9 -9
  13. dkist_processing_common/_util/tags.py +1 -0
  14. dkist_processing_common/codecs/array.py +20 -0
  15. dkist_processing_common/codecs/asdf.py +9 -3
  16. dkist_processing_common/codecs/basemodel.py +22 -0
  17. dkist_processing_common/codecs/bytes.py +1 -0
  18. dkist_processing_common/codecs/fits.py +37 -9
  19. dkist_processing_common/codecs/iobase.py +1 -0
  20. dkist_processing_common/codecs/json.py +1 -0
  21. dkist_processing_common/codecs/path.py +1 -0
  22. dkist_processing_common/codecs/quality.py +1 -1
  23. dkist_processing_common/codecs/str.py +1 -0
  24. dkist_processing_common/config.py +64 -25
  25. dkist_processing_common/manual.py +6 -8
  26. dkist_processing_common/models/constants.py +373 -37
  27. dkist_processing_common/models/dkist_location.py +27 -0
  28. dkist_processing_common/models/fits_access.py +48 -0
  29. dkist_processing_common/models/flower_pot.py +231 -9
  30. dkist_processing_common/models/fried_parameter.py +41 -0
  31. dkist_processing_common/models/graphql.py +66 -75
  32. dkist_processing_common/models/input_dataset.py +117 -0
  33. dkist_processing_common/models/message.py +1 -1
  34. dkist_processing_common/models/message_queue_binding.py +1 -1
  35. dkist_processing_common/models/metric_code.py +2 -0
  36. dkist_processing_common/models/parameters.py +65 -28
  37. dkist_processing_common/models/quality.py +50 -5
  38. dkist_processing_common/models/tags.py +23 -21
  39. dkist_processing_common/models/task_name.py +3 -2
  40. dkist_processing_common/models/telemetry.py +28 -0
  41. dkist_processing_common/models/wavelength.py +3 -1
  42. dkist_processing_common/parsers/average_bud.py +46 -0
  43. dkist_processing_common/parsers/cs_step.py +13 -12
  44. dkist_processing_common/parsers/dsps_repeat.py +6 -4
  45. dkist_processing_common/parsers/experiment_id_bud.py +12 -4
  46. dkist_processing_common/parsers/id_bud.py +42 -27
  47. dkist_processing_common/parsers/l0_fits_access.py +5 -3
  48. dkist_processing_common/parsers/l1_fits_access.py +51 -23
  49. dkist_processing_common/parsers/lookup_bud.py +125 -0
  50. dkist_processing_common/parsers/near_bud.py +21 -20
  51. dkist_processing_common/parsers/observing_program_id_bud.py +24 -0
  52. dkist_processing_common/parsers/proposal_id_bud.py +13 -5
  53. dkist_processing_common/parsers/quality.py +2 -0
  54. dkist_processing_common/parsers/retarder.py +32 -0
  55. dkist_processing_common/parsers/single_value_single_key_flower.py +6 -1
  56. dkist_processing_common/parsers/task.py +8 -6
  57. dkist_processing_common/parsers/time.py +178 -72
  58. dkist_processing_common/parsers/unique_bud.py +21 -22
  59. dkist_processing_common/parsers/wavelength.py +5 -3
  60. dkist_processing_common/tasks/__init__.py +3 -2
  61. dkist_processing_common/tasks/assemble_movie.py +4 -3
  62. dkist_processing_common/tasks/base.py +59 -60
  63. dkist_processing_common/tasks/l1_output_data.py +54 -53
  64. dkist_processing_common/tasks/mixin/globus.py +24 -27
  65. dkist_processing_common/tasks/mixin/interservice_bus.py +1 -0
  66. dkist_processing_common/tasks/mixin/metadata_store.py +108 -243
  67. dkist_processing_common/tasks/mixin/object_store.py +22 -0
  68. dkist_processing_common/tasks/mixin/quality/__init__.py +1 -0
  69. dkist_processing_common/tasks/mixin/quality/_base.py +8 -1
  70. dkist_processing_common/tasks/mixin/quality/_metrics.py +166 -14
  71. dkist_processing_common/tasks/output_data_base.py +4 -3
  72. dkist_processing_common/tasks/parse_l0_input_data.py +277 -15
  73. dkist_processing_common/tasks/quality_metrics.py +9 -9
  74. dkist_processing_common/tasks/teardown.py +7 -7
  75. dkist_processing_common/tasks/transfer_input_data.py +67 -69
  76. dkist_processing_common/tasks/trial_catalog.py +77 -17
  77. dkist_processing_common/tasks/trial_output_data.py +16 -17
  78. dkist_processing_common/tasks/write_l1.py +102 -72
  79. dkist_processing_common/tests/conftest.py +32 -173
  80. dkist_processing_common/tests/mock_metadata_store.py +271 -0
  81. dkist_processing_common/tests/test_assemble_movie.py +4 -4
  82. dkist_processing_common/tests/test_assemble_quality.py +32 -4
  83. dkist_processing_common/tests/test_base.py +5 -19
  84. dkist_processing_common/tests/test_codecs.py +103 -12
  85. dkist_processing_common/tests/test_constants.py +15 -0
  86. dkist_processing_common/tests/test_dkist_location.py +15 -0
  87. dkist_processing_common/tests/test_fits_access.py +56 -19
  88. dkist_processing_common/tests/test_flower_pot.py +147 -5
  89. dkist_processing_common/tests/test_fried_parameter.py +27 -0
  90. dkist_processing_common/tests/test_input_dataset.py +78 -361
  91. dkist_processing_common/tests/test_interservice_bus.py +1 -0
  92. dkist_processing_common/tests/test_interservice_bus_mixin.py +1 -1
  93. dkist_processing_common/tests/test_manual_processing.py +33 -0
  94. dkist_processing_common/tests/test_output_data_base.py +5 -7
  95. dkist_processing_common/tests/test_parameters.py +71 -22
  96. dkist_processing_common/tests/test_parse_l0_input_data.py +115 -32
  97. dkist_processing_common/tests/test_publish_catalog_messages.py +2 -24
  98. dkist_processing_common/tests/test_quality.py +1 -0
  99. dkist_processing_common/tests/test_quality_mixin.py +255 -23
  100. dkist_processing_common/tests/test_scratch.py +2 -1
  101. dkist_processing_common/tests/test_stems.py +511 -168
  102. dkist_processing_common/tests/test_submit_dataset_metadata.py +3 -7
  103. dkist_processing_common/tests/test_tags.py +1 -0
  104. dkist_processing_common/tests/test_task_name.py +1 -1
  105. dkist_processing_common/tests/test_task_parsing.py +17 -7
  106. dkist_processing_common/tests/test_teardown.py +28 -24
  107. dkist_processing_common/tests/test_transfer_input_data.py +270 -125
  108. dkist_processing_common/tests/test_transfer_l1_output_data.py +2 -3
  109. dkist_processing_common/tests/test_trial_catalog.py +83 -8
  110. dkist_processing_common/tests/test_trial_output_data.py +46 -73
  111. dkist_processing_common/tests/test_workflow_task_base.py +8 -10
  112. dkist_processing_common/tests/test_write_l1.py +298 -76
  113. dkist_processing_common-12.1.0rc1.dist-info/METADATA +265 -0
  114. dkist_processing_common-12.1.0rc1.dist-info/RECORD +134 -0
  115. {dkist_processing_common-10.5.4.dist-info → dkist_processing_common-12.1.0rc1.dist-info}/WHEEL +1 -1
  116. docs/conf.py +1 -0
  117. docs/index.rst +1 -1
  118. docs/landing_page.rst +13 -0
  119. dkist_processing_common/tasks/mixin/input_dataset.py +0 -166
  120. dkist_processing_common-10.5.4.dist-info/METADATA +0 -175
  121. dkist_processing_common-10.5.4.dist-info/RECORD +0 -112
  122. {dkist_processing_common-10.5.4.dist-info → dkist_processing_common-12.1.0rc1.dist-info}/top_level.txt +0 -0
@@ -1,14 +1,21 @@
1
1
  """Experiment Id parser."""
2
+
2
3
  from dkist_processing_common.models.constants import BudName
4
+ from dkist_processing_common.models.fits_access import MetadataKey
5
+ from dkist_processing_common.models.task_name import TaskName
3
6
  from dkist_processing_common.parsers.id_bud import ContributingIdsBud
4
- from dkist_processing_common.parsers.id_bud import IdBud
7
+ from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
5
8
 
6
9
 
7
- class ExperimentIdBud(IdBud):
10
+ class ExperimentIdBud(TaskUniqueBud):
8
11
  """Class to create a Bud for the experiment_id."""
9
12
 
10
13
  def __init__(self):
11
- super().__init__(constant_name=BudName.experiment_id.value, metadata_key="experiment_id")
14
+ super().__init__(
15
+ constant_name=BudName.experiment_id,
16
+ metadata_key=MetadataKey.experiment_id,
17
+ ip_task_types=TaskName.observe,
18
+ )
12
19
 
13
20
 
14
21
  class ContributingExperimentIdsBud(ContributingIdsBud):
@@ -16,5 +23,6 @@ class ContributingExperimentIdsBud(ContributingIdsBud):
16
23
 
17
24
  def __init__(self):
18
25
  super().__init__(
19
- stem_name=BudName.contributing_experiment_ids.value, metadata_key="experiment_id"
26
+ constant_name=BudName.contributing_experiment_ids,
27
+ metadata_key=MetadataKey.experiment_id,
20
28
  )
@@ -1,32 +1,25 @@
1
1
  """Base classes for ID bud parsing."""
2
+
3
+ from enum import StrEnum
4
+ from typing import Callable
2
5
  from typing import Type
3
6
 
7
+ from dkist_processing_common.models.flower_pot import SetStem
4
8
  from dkist_processing_common.models.flower_pot import SpilledDirt
5
- from dkist_processing_common.models.flower_pot import Stem
6
- from dkist_processing_common.models.task_name import TaskName
7
9
  from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
8
- from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
9
-
10
-
11
- class IdBud(TaskUniqueBud):
12
- """Base class for ID buds."""
13
-
14
- def __init__(self, constant_name, metadata_key):
15
- super().__init__(
16
- constant_name=constant_name,
17
- metadata_key=metadata_key,
18
- ip_task_type=TaskName.observe.value,
19
- )
10
+ from dkist_processing_common.parsers.task import passthrough_header_ip_task
20
11
 
21
12
 
22
- class ContributingIdsBud(Stem):
13
+ class ContributingIdsBud(SetStem):
23
14
  """Base class for contributing ID buds."""
24
15
 
25
- def __init__(self, stem_name, metadata_key):
26
- super().__init__(stem_name=stem_name)
16
+ def __init__(self, constant_name: str, metadata_key: str | StrEnum):
17
+ super().__init__(stem_name=constant_name)
18
+ if isinstance(metadata_key, StrEnum):
19
+ metadata_key = metadata_key.name
27
20
  self.metadata_key = metadata_key
28
21
 
29
- def setter(self, fits_obj: L0FitsAccess) -> str | Type[SpilledDirt]:
22
+ def setter(self, fits_obj: L0FitsAccess) -> str:
30
23
  """
31
24
  Set the id for any type of frame.
32
25
 
@@ -40,17 +33,39 @@ class ContributingIdsBud(Stem):
40
33
  """
41
34
  return getattr(fits_obj, self.metadata_key)
42
35
 
43
- def getter(self, key) -> tuple:
36
+ def getter(self) -> tuple[str, ...]:
44
37
  """
45
- Get all ids seen in non observe frames.
46
-
47
- Parameters
48
- ----------
49
- key
50
- The input key
38
+ Get all ids seen for any type of frame.
51
39
 
52
40
  Returns
53
41
  -------
54
- IDs from non observe frames
42
+ IDs from all types of frames
55
43
  """
56
- return tuple(set(self.key_to_petal_dict.values()))
44
+ return tuple(self.value_set)
45
+
46
+
47
+ class TaskContributingIdsBud(ContributingIdsBud):
48
+ """Base class for contributing ID buds for a particular task type."""
49
+
50
+ def __init__(
51
+ self,
52
+ constant_name: str,
53
+ metadata_key: str | StrEnum,
54
+ ip_task_types: str | list[str],
55
+ task_type_parsing_function: Callable = passthrough_header_ip_task,
56
+ ):
57
+ super().__init__(constant_name=constant_name, metadata_key=metadata_key)
58
+
59
+ if isinstance(ip_task_types, str):
60
+ ip_task_types = [ip_task_types]
61
+ self.ip_task_types = [task.casefold() for task in ip_task_types]
62
+ self.parsing_function = task_type_parsing_function
63
+
64
+ def setter(self, fits_obj: L0FitsAccess) -> str | Type[SpilledDirt]:
65
+ """Ingest an object only if its parsed IP task type matches what's desired."""
66
+ task = self.parsing_function(fits_obj)
67
+
68
+ if task.casefold() in self.ip_task_types:
69
+ return super().setter(fits_obj)
70
+
71
+ return SpilledDirt
@@ -1,6 +1,8 @@
1
1
  """By-frame 214 L0 header keywords that are not instrument specific."""
2
+
2
3
  from astropy.io import fits
3
4
 
5
+ from dkist_processing_common.models.fits_access import MetadataKey
4
6
  from dkist_processing_common.parsers.l1_fits_access import L1FitsAccess
5
7
 
6
8
 
@@ -25,6 +27,6 @@ class L0FitsAccess(L1FitsAccess):
25
27
  auto_squeeze: bool = True,
26
28
  ):
27
29
  super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
28
- self.ip_task_type: str = self.header["IPTASK"]
29
- self.ip_start_time: str = self.header["DKIST011"]
30
- self.ip_end_time: str = self.header["DKIST012"]
30
+ self.ip_task_type: str = self.header[MetadataKey.ip_task_type]
31
+ self.ip_start_time: str = self.header[MetadataKey.ip_start_time]
32
+ self.ip_end_time: str = self.header[MetadataKey.ip_end_time]
@@ -1,7 +1,12 @@
1
1
  """By-frame 214 L1 only header keywords that are not instrument specific."""
2
+
2
3
  from astropy.io import fits
3
4
 
5
+ from dkist_processing_common.models.fits_access import HEADER_KEY_NOT_FOUND
4
6
  from dkist_processing_common.models.fits_access import FitsAccessBase
7
+ from dkist_processing_common.models.fits_access import MetadataKey
8
+
9
+ NOT_A_FLOAT = -999
5
10
 
6
11
 
7
12
  class L1FitsAccess(FitsAccessBase):
@@ -26,38 +31,61 @@ class L1FitsAccess(FitsAccessBase):
26
31
  ):
27
32
  super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
28
33
 
29
- self.elevation: float = self.header["ELEV_ANG"]
30
- self.azimuth: float = self.header["TAZIMUTH"]
31
- self.table_angle: float = self.header["TTBLANGL"]
32
- self.gos_level3_status: str = self.header["LVL3STAT"]
33
- self.gos_level3_lamp_status: str = self.header["LAMPSTAT"]
34
- self.gos_polarizer_status: str = self.header["LVL2STAT"]
35
- self.gos_retarder_status: str = self.header["LVL1STAT"]
36
- self.gos_level0_status: str = self.header["LVL0STAT"]
37
- self.time_obs: str = self.header["DATE-BEG"]
38
- self.ip_id: str = self.header["IP_ID"]
39
- self.instrument: str = self.header["INSTRUME"]
40
- self.wavelength: float = self.header["LINEWAV"]
41
- self.proposal_id: str = self.header["PROP_ID"]
42
- self.experiment_id: str = self.header["EXPER_ID"]
43
- self.num_dsps_repeats: int = self.header["DSPSREPS"]
44
- self.current_dsps_repeat: int = self.header["DSPSNUM"]
45
- self.fpa_exposure_time_ms: float = self.header["XPOSURE"]
46
- self.sensor_readout_exposure_time_ms: float = self.header["TEXPOSUR"]
47
- self.num_raw_frames_per_fpa: int = self.header["NSUMEXP"]
34
+ self.elevation: float = self.header[MetadataKey.elevation]
35
+ self.azimuth: float = self.header[MetadataKey.azimuth]
36
+ self.table_angle: float = self.header[MetadataKey.table_angle]
37
+ self.gos_level3_status: str = self.header[MetadataKey.gos_level3_status]
38
+ self.gos_level3_lamp_status: str = self.header[MetadataKey.gos_level3_lamp_status]
39
+ self.gos_polarizer_status: str = self.header[MetadataKey.gos_polarizer_status]
40
+ self.gos_retarder_status: str = self.header[MetadataKey.gos_retarder_status]
41
+ self.gos_level0_status: str = self.header[MetadataKey.gos_level0_status]
42
+ self.time_obs: str = self.header[MetadataKey.time_obs]
43
+ self.ip_id: str = self.header[MetadataKey.ip_id]
44
+ self.instrument: str = self.header[MetadataKey.instrument]
45
+ self.wavelength: float = self.header[MetadataKey.wavelength]
46
+ self.proposal_id: str = self.header[MetadataKey.proposal_id]
47
+ self.experiment_id: str = self.header[MetadataKey.experiment_id]
48
+ self.num_dsps_repeats: int = self.header[MetadataKey.num_dsps_repeats]
49
+ self.current_dsps_repeat: int = self.header[MetadataKey.current_dsps_repeat]
50
+ self.fpa_exposure_time_ms: float = self.header[MetadataKey.fpa_exposure_time_ms]
51
+ self.sensor_readout_exposure_time_ms: float = self.header[
52
+ MetadataKey.sensor_readout_exposure_time_ms
53
+ ]
54
+ self.num_raw_frames_per_fpa: int = self.header[MetadataKey.num_raw_frames_per_fpa]
55
+ self.camera_id: str = self.header[MetadataKey.camera_id]
56
+ self.camera_name: str = self.header[MetadataKey.camera_name]
57
+ self.camera_bit_depth: int = self.header[MetadataKey.camera_bit_depth]
58
+ self.hardware_binning_x: int = self.header[MetadataKey.hardware_binning_x]
59
+ self.hardware_binning_y: int = self.header[MetadataKey.hardware_binning_y]
60
+ self.software_binning_x: int = self.header[MetadataKey.software_binning_x]
61
+ self.software_binning_y: int = self.header[MetadataKey.software_binning_y]
62
+ self.observing_program_execution_id: str = self.header[
63
+ MetadataKey.observing_program_execution_id
64
+ ]
65
+ self.telescope_tracking_mode: str = self.header.get(
66
+ MetadataKey.telescope_tracking_mode, HEADER_KEY_NOT_FOUND
67
+ )
68
+ self.coude_table_tracking_mode: str = self.header.get(
69
+ MetadataKey.coude_table_tracking_mode, HEADER_KEY_NOT_FOUND
70
+ )
71
+ self.telescope_scanning_mode: str = self.header.get(
72
+ MetadataKey.telescope_scanning_mode, HEADER_KEY_NOT_FOUND
73
+ )
74
+ self.light_level: float = self.header[MetadataKey.light_level]
75
+ self.hls_version: str = self.header[MetadataKey.hls_version]
48
76
 
49
77
  @property
50
78
  def gos_polarizer_angle(self) -> float:
51
79
  """Convert the polarizer angle to a float if possible before returning."""
52
80
  try:
53
- return float(self.header["POLANGLE"])
81
+ return float(self.header[MetadataKey.gos_polarizer_angle])
54
82
  except ValueError:
55
- return -999 # The angle is only used if the polarizer is in the beam
83
+ return NOT_A_FLOAT # The angle is only used if the polarizer is in the beam
56
84
 
57
85
  @property
58
86
  def gos_retarder_angle(self) -> float:
59
87
  """Convert the retarder angle to a float if possible before returning."""
60
88
  try:
61
- return float(self.header["RETANGLE"])
89
+ return float(self.header[MetadataKey.gos_retarder_angle])
62
90
  except ValueError:
63
- return -999 # The angle is only used if the retarder is in the beam
91
+ return NOT_A_FLOAT # The angle is only used if the retarder is in the beam
@@ -0,0 +1,125 @@
1
+ """Simple bud that is used to set a constant to a mapping dictionary."""
2
+
3
+ from collections import defaultdict
4
+ from enum import StrEnum
5
+ from typing import Any
6
+ from typing import Callable
7
+ from typing import DefaultDict
8
+
9
+ from dkist_processing_common.models.flower_pot import SetStem
10
+ from dkist_processing_common.models.flower_pot import SpilledDirt
11
+ from dkist_processing_common.models.tags import EXP_TIME_ROUND_DIGITS
12
+ from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
13
+ from dkist_processing_common.parsers.task import passthrough_header_ip_task
14
+
15
+
16
+ class TimeLookupBud(SetStem):
17
+ """
18
+ Bud that reads two header keys from all files and creates a dictionary mapping a time KEY value to sets of a VALUE value.
19
+
20
+ Parameters
21
+ ----------
22
+ constant_name
23
+ The name for the constant to be defined
24
+
25
+ key_metadata_key
26
+ The time metadata key for the resulting dictionary key
27
+
28
+ value_metadata_key
29
+ The metadata key for the resulting dictionary value
30
+ """
31
+
32
+ def __init__(
33
+ self,
34
+ constant_name: str,
35
+ key_metadata_key: str | StrEnum,
36
+ value_metadata_key: str | StrEnum,
37
+ ):
38
+ super().__init__(stem_name=constant_name)
39
+
40
+ if isinstance(key_metadata_key, StrEnum):
41
+ key_metadata_key = key_metadata_key.name
42
+ self.key_metadata_key = key_metadata_key
43
+ if isinstance(value_metadata_key, StrEnum):
44
+ value_metadata_key = value_metadata_key.name
45
+ self.value_metadata_key = value_metadata_key
46
+
47
+ self.mapping: DefaultDict[float, set[Any]] = defaultdict(set)
48
+
49
+ def setter(self, fits_obj: L0FitsAccess):
50
+ """
51
+ Update the mapping dictionary.
52
+
53
+ Parameters
54
+ ----------
55
+ fits_obj
56
+ The input fits object
57
+ Returns
58
+ -------
59
+ Updates the dictionary and returns None
60
+ """
61
+ key = getattr(fits_obj, self.key_metadata_key)
62
+ rounded_key = round(key, EXP_TIME_ROUND_DIGITS)
63
+ value = getattr(fits_obj, self.value_metadata_key)
64
+ self.mapping[rounded_key].add(value)
65
+ return None
66
+
67
+ def getter(self):
68
+ """
69
+ Get the dictionary mapping created by the setter with values converted to JSON-able lists.
70
+
71
+ Returns
72
+ -------
73
+ The mapping dictionary with values converted to JSON-able lists
74
+ """
75
+ mapping_lists = {k: list(v) for k, v in self.mapping.items()}
76
+ return mapping_lists
77
+
78
+
79
+ class TaskTimeLookupBud(TimeLookupBud):
80
+ """
81
+ Subclass of `TimeLookupBud` that only considers objects that have specific task types.
82
+
83
+ Parameters
84
+ ----------
85
+ constant_name
86
+ The name for the constant to be defined
87
+
88
+ key_metadata_key
89
+ The time metadata key for the resulting dictionary key
90
+
91
+ value_metadata_key
92
+ The metadata key for the resulting dictionary value
93
+
94
+ ip_task_types
95
+ Only consider objects whose parsed header IP task type matches a string in this list
96
+
97
+ task_type_parsing_function
98
+ The function used to convert a header into an IP task type
99
+ """
100
+
101
+ def __init__(
102
+ self,
103
+ constant_name: str,
104
+ key_metadata_key: str | StrEnum,
105
+ value_metadata_key: str | StrEnum,
106
+ ip_task_types: str | list[str],
107
+ task_type_parsing_function: Callable = passthrough_header_ip_task,
108
+ ):
109
+ super().__init__(
110
+ constant_name=constant_name,
111
+ key_metadata_key=key_metadata_key,
112
+ value_metadata_key=value_metadata_key,
113
+ )
114
+ if isinstance(ip_task_types, str):
115
+ ip_task_types = [ip_task_types]
116
+ self.ip_task_types = [task.casefold() for task in ip_task_types]
117
+ self.parsing_function = task_type_parsing_function
118
+
119
+ def setter(self, fits_obj: L0FitsAccess):
120
+ """Ingest an object only if its parsed IP task type matches what's desired."""
121
+ task = self.parsing_function(fits_obj)
122
+ if task.casefold() in self.ip_task_types:
123
+ return super().setter(fits_obj)
124
+
125
+ return SpilledDirt
@@ -1,16 +1,18 @@
1
1
  """Pre-made flower that reads a single header key from all files and raises a ValueError if the values are not in a supplied range."""
2
+
3
+ from enum import StrEnum
2
4
  from statistics import mean
3
5
  from typing import Callable
4
6
 
7
+ from dkist_processing_common.models.flower_pot import ListStem
5
8
  from dkist_processing_common.models.flower_pot import SpilledDirt
6
- from dkist_processing_common.models.flower_pot import Stem
7
9
  from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
8
10
  from dkist_processing_common.parsers.task import passthrough_header_ip_task
9
11
 
10
12
 
11
- class NearFloatBud(Stem):
13
+ class NearFloatBud(ListStem):
12
14
  """
13
- Pre-made flower that reads a single header key from all files and raises a ValueError if the values are not within a given tolerance.
15
+ Pre-made `ListStem` that reads a single header key from all files and raises a ValueError if the values are not within a given tolerance.
14
16
 
15
17
  This is intended for use with floats where the values may be slightly different, but should be the same.
16
18
 
@@ -29,10 +31,12 @@ class NearFloatBud(Stem):
29
31
  def __init__(
30
32
  self,
31
33
  constant_name: str,
32
- metadata_key: str,
34
+ metadata_key: str | StrEnum,
33
35
  tolerance: float,
34
36
  ):
35
37
  super().__init__(stem_name=constant_name)
38
+ if isinstance(metadata_key, StrEnum):
39
+ metadata_key = metadata_key.name
36
40
  self.metadata_key = metadata_key
37
41
  self.tolerance = tolerance
38
42
 
@@ -50,31 +54,26 @@ class NearFloatBud(Stem):
50
54
  """
51
55
  return getattr(fits_obj, self.metadata_key)
52
56
 
53
- def getter(self, key):
57
+ def getter(self):
54
58
  """
55
59
  Get the value for this key and raise an error if the data spans more than the given tolerance.
56
60
 
57
- Parameters
58
- ----------
59
- key
60
- The input key
61
61
  Returns
62
62
  -------
63
63
  The mean value associated with this input key
64
64
  """
65
- value_set = list(self.key_to_petal_dict.values())
66
- biggest_value = max(value_set)
67
- smallest_value = min(value_set)
65
+ biggest_value = max(self.value_list)
66
+ smallest_value = min(self.value_list)
68
67
  if biggest_value - smallest_value > self.tolerance:
69
68
  raise ValueError(
70
69
  f"{self.stem_name} values are not close enough. Max: {biggest_value}, Min: {smallest_value}, Tolerance: {self.tolerance}"
71
70
  )
72
- return mean(value_set)
71
+ return mean(self.value_list)
73
72
 
74
73
 
75
74
  class TaskNearFloatBud(NearFloatBud):
76
75
  """
77
- Subclass of `NearFloatBud` that only considers objects that have a specific task type.
76
+ Subclass of `NearFloatBud` that only considers objects that have specific task types.
78
77
 
79
78
  Parameters
80
79
  ----------
@@ -84,8 +83,8 @@ class TaskNearFloatBud(NearFloatBud):
84
83
  metadata_key
85
84
  The metadata key associated with the constant
86
85
 
87
- ip_task_type
88
- Only consider objects whose parsed header IP task type matches this string
86
+ ip_task_types
87
+ Only consider objects whose parsed header IP task type matches a string in this list
89
88
 
90
89
  task_type_parsing_function
91
90
  The function used to convert a header into an IP task type
@@ -97,8 +96,8 @@ class TaskNearFloatBud(NearFloatBud):
97
96
  def __init__(
98
97
  self,
99
98
  constant_name: str,
100
- metadata_key: str,
101
- ip_task_type: str,
99
+ metadata_key: str | StrEnum,
100
+ ip_task_types: str | list[str],
102
101
  tolerance: float,
103
102
  task_type_parsing_function: Callable = passthrough_header_ip_task,
104
103
  ):
@@ -106,14 +105,16 @@ class TaskNearFloatBud(NearFloatBud):
106
105
  constant_name=constant_name, metadata_key=metadata_key, tolerance=tolerance
107
106
  )
108
107
 
109
- self.ip_task_type = ip_task_type.casefold()
108
+ if isinstance(ip_task_types, str):
109
+ ip_task_types = [ip_task_types]
110
+ self.ip_task_types = [task.casefold() for task in ip_task_types]
110
111
  self.parsing_function = task_type_parsing_function
111
112
 
112
113
  def setter(self, fits_obj: L0FitsAccess):
113
114
  """Ingest an object only if its parsed IP task type matches what's desired."""
114
115
  task = self.parsing_function(fits_obj)
115
116
 
116
- if task.casefold() == self.ip_task_type:
117
+ if task.casefold() in self.ip_task_types:
117
118
  return super().setter(fits_obj)
118
119
 
119
120
  return SpilledDirt
@@ -0,0 +1,24 @@
1
+ """Observing Program Id parser."""
2
+
3
+ from typing import Callable
4
+
5
+ from dkist_processing_common.models.fits_access import MetadataKey
6
+ from dkist_processing_common.parsers.id_bud import TaskContributingIdsBud
7
+ from dkist_processing_common.parsers.task import passthrough_header_ip_task
8
+
9
+
10
+ class TaskContributingObservingProgramExecutionIdsBud(TaskContributingIdsBud):
11
+ """Class to create a Bud for the supporting observing_program_execution_ids."""
12
+
13
+ def __init__(
14
+ self,
15
+ constant_name: str,
16
+ ip_task_types: str | list[str],
17
+ task_type_parsing_function: Callable = passthrough_header_ip_task,
18
+ ):
19
+ super().__init__(
20
+ constant_name=constant_name,
21
+ metadata_key=MetadataKey.observing_program_execution_id,
22
+ ip_task_types=ip_task_types,
23
+ task_type_parsing_function=task_type_parsing_function,
24
+ )
@@ -1,20 +1,28 @@
1
1
  """Proposal Id parser."""
2
+
2
3
  from dkist_processing_common.models.constants import BudName
4
+ from dkist_processing_common.models.fits_access import MetadataKey
5
+ from dkist_processing_common.models.task_name import TaskName
3
6
  from dkist_processing_common.parsers.id_bud import ContributingIdsBud
4
- from dkist_processing_common.parsers.id_bud import IdBud
7
+ from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
5
8
 
6
9
 
7
- class ProposalIdBud(IdBud):
10
+ class ProposalIdBud(TaskUniqueBud):
8
11
  """Class to create a Bud for the proposal_id."""
9
12
 
10
13
  def __init__(self):
11
- super().__init__(constant_name=BudName.proposal_id.value, metadata_key="proposal_id")
14
+ super().__init__(
15
+ constant_name=BudName.proposal_id,
16
+ metadata_key=MetadataKey.proposal_id,
17
+ ip_task_types=TaskName.observe,
18
+ )
12
19
 
13
20
 
14
21
  class ContributingProposalIdsBud(ContributingIdsBud):
15
- """Class to create a Bud for the proposal_ids."""
22
+ """Class to create a Bud for the supporting proposal_ids."""
16
23
 
17
24
  def __init__(self):
18
25
  super().__init__(
19
- stem_name=BudName.contributing_proposal_ids.value, metadata_key="proposal_id"
26
+ constant_name=BudName.contributing_proposal_ids,
27
+ metadata_key=MetadataKey.proposal_id,
20
28
  )
@@ -1,4 +1,5 @@
1
1
  """Support classes to define object attributes from header information."""
2
+
2
3
  from astropy.io import fits
3
4
 
4
5
  from dkist_processing_common.parsers.l0_fits_access import L1FitsAccess
@@ -31,3 +32,4 @@ class L1QualityFitsAccess(L1FitsAccess):
31
32
  self.light_level: float = self.header["LIGHTLVL"]
32
33
  self.health_status: str = self.header["DSHEALTH"]
33
34
  self.ao_status: int = self.header.get("AO_LOCK", None)
35
+ self.num_out_of_bounds_ao_values: int = self.header.get("OOBSHIFT", None)
@@ -0,0 +1,32 @@
1
+ """Bud that parses the name of the retarder used during POLCAL task observations."""
2
+
3
+ from dkist_processing_common.models.constants import BudName
4
+ from dkist_processing_common.models.fits_access import MetadataKey
5
+ from dkist_processing_common.models.flower_pot import SpilledDirt
6
+ from dkist_processing_common.models.task_name import TaskName
7
+ from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
8
+ from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
9
+
10
+
11
+ class RetarderNameBud(TaskUniqueBud):
12
+ """
13
+ Bud for determining the name of the retarder used during a polcal Calibration Sequence (CS).
14
+
15
+ This is *slightly* different than a simple `TaskUniqueBud` because we need to allow for CS steps when the retarder
16
+ is out of the beam (i.g., "clear"). We do this by returning `SpilledDirt` from the `setter` if the value is "clear".
17
+ """
18
+
19
+ def __init__(self):
20
+ super().__init__(
21
+ constant_name=BudName.retarder_name,
22
+ metadata_key=MetadataKey.gos_retarder_status,
23
+ ip_task_types=TaskName.polcal,
24
+ )
25
+
26
+ def setter(self, fits_obj: L0FitsAccess) -> type[SpilledDirt] | str:
27
+ """Drop the result if the retarder is out of the beam ("clear")."""
28
+ result = super().setter(fits_obj)
29
+ if result is not SpilledDirt and result.casefold() == "clear":
30
+ return SpilledDirt
31
+
32
+ return result
@@ -1,4 +1,7 @@
1
1
  """Pre-made flower that produces tag based on a single header key."""
2
+
3
+ from enum import StrEnum
4
+
2
5
  from dkist_processing_common.models.flower_pot import Stem
3
6
  from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
4
7
 
@@ -15,8 +18,10 @@ class SingleValueSingleKeyFlower(Stem):
15
18
  The metadata key
16
19
  """
17
20
 
18
- def __init__(self, tag_stem_name: str, metadata_key: str):
21
+ def __init__(self, tag_stem_name: str, metadata_key: str | StrEnum):
19
22
  super().__init__(stem_name=tag_stem_name)
23
+ if isinstance(metadata_key, StrEnum):
24
+ metadata_key = metadata_key.name
20
25
  self.metadata_key = metadata_key
21
26
 
22
27
  def setter(self, fits_obj: L0FitsAccess):
@@ -1,8 +1,10 @@
1
1
  """Module for parsing IP task related things."""
2
+
2
3
  from typing import Callable
3
4
  from typing import Type
4
5
 
5
6
  from dkist_processing_common.models.fits_access import FitsAccessBase
7
+ from dkist_processing_common.models.fits_access import MetadataKey
6
8
  from dkist_processing_common.models.flower_pot import SpilledDirt
7
9
  from dkist_processing_common.models.tags import StemName
8
10
  from dkist_processing_common.models.task_name import TaskName
@@ -35,9 +37,9 @@ def parse_header_ip_task_with_gains(fits_obj: FitsAccessBase) -> str:
35
37
  and fits_obj.gos_level3_status == "lamp"
36
38
  and fits_obj.gos_level3_lamp_status == "on"
37
39
  ):
38
- return TaskName.lamp_gain.value
40
+ return TaskName.lamp_gain
39
41
  if fits_obj.ip_task_type == "gain" and fits_obj.gos_level3_status == "clear":
40
- return TaskName.solar_gain.value
42
+ return TaskName.solar_gain
41
43
 
42
44
  # Everything else is unchanged
43
45
  return passthrough_header_ip_task(fits_obj)
@@ -58,14 +60,14 @@ def parse_polcal_task_type(fits_obj: Type[FitsAccessBase]) -> str | Type[Spilled
58
60
  and fits_obj.gos_retarder_status == "clear"
59
61
  and fits_obj.gos_polarizer_status == "clear"
60
62
  ):
61
- return TaskName.polcal_dark.value
63
+ return TaskName.polcal_dark
62
64
 
63
65
  elif (
64
66
  fits_obj.gos_level0_status.startswith("FieldStop")
65
67
  and fits_obj.gos_retarder_status == "clear"
66
68
  and fits_obj.gos_polarizer_status == "clear"
67
69
  ):
68
- return TaskName.polcal_gain.value
70
+ return TaskName.polcal_gain
69
71
 
70
72
  # We don't care about a POLCAL frame that is neither dark nor clear
71
73
  return SpilledDirt
@@ -77,7 +79,7 @@ class TaskTypeFlower(SingleValueSingleKeyFlower):
77
79
  def __init__(
78
80
  self, header_task_parsing_func: Callable[[FitsAccessBase], str] = passthrough_header_ip_task
79
81
  ):
80
- super().__init__(tag_stem_name=StemName.task.value, metadata_key="ip_task_type")
82
+ super().__init__(tag_stem_name=StemName.task, metadata_key=MetadataKey.ip_task_type)
81
83
  self.header_parsing_function = header_task_parsing_func
82
84
 
83
85
  def setter(self, fits_obj: FitsAccessBase):
@@ -102,7 +104,7 @@ class PolcalTaskFlower(SingleValueSingleKeyFlower):
102
104
  """
103
105
 
104
106
  def __init__(self):
105
- super().__init__(tag_stem_name=StemName.task.value, metadata_key="ip_task_type")
107
+ super().__init__(tag_stem_name=StemName.task, metadata_key=MetadataKey.ip_task_type)
106
108
 
107
109
  def setter(self, fits_obj: FitsAccessBase):
108
110
  """