dkist-processing-trend 0.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- changelog/.gitempty +0 -0
- changelog/1.feature.rst +1 -0
- dkist_processing_trend/__init__.py +10 -0
- dkist_processing_trend/config.py +11 -0
- dkist_processing_trend/models/__init__.py +1 -0
- dkist_processing_trend/models/constants.py +143 -0
- dkist_processing_trend/models/fit_options.py +15 -0
- dkist_processing_trend/models/fits_access.py +65 -0
- dkist_processing_trend/models/instrument.py +11 -0
- dkist_processing_trend/models/instrument_options.py +35 -0
- dkist_processing_trend/models/parameters.py +134 -0
- dkist_processing_trend/models/tags.py +117 -0
- dkist_processing_trend/models/task_name.py +20 -0
- dkist_processing_trend/parsers/__init__.py +1 -0
- dkist_processing_trend/parsers/arm_id.py +103 -0
- dkist_processing_trend/parsers/instrument_unique_bud.py +40 -0
- dkist_processing_trend/parsers/time.py +27 -0
- dkist_processing_trend/parsers/trend_l0_fits_access.py +123 -0
- dkist_processing_trend/tasks/__init__.py +11 -0
- dkist_processing_trend/tasks/dark.py +56 -0
- dkist_processing_trend/tasks/gain.py +74 -0
- dkist_processing_trend/tasks/parse.py +169 -0
- dkist_processing_trend/tasks/prepare_fit_data_base.py +257 -0
- dkist_processing_trend/tasks/run_pac_fitter.py +392 -0
- dkist_processing_trend/tasks/trend_base.py +97 -0
- dkist_processing_trend/tasks/trend_output_data.py +167 -0
- dkist_processing_trend/tasks/visp/__init__.py +6 -0
- dkist_processing_trend/tasks/visp/visp_dmpd.py +405 -0
- dkist_processing_trend/tasks/visp/visp_extract_beam.py +14 -0
- dkist_processing_trend/tasks/visp/visp_geometric.py +260 -0
- dkist_processing_trend/tasks/visp/visp_prep_fit_data.py +162 -0
- dkist_processing_trend/tasks/visp/visp_process_demod.py +236 -0
- dkist_processing_trend/tasks/write_trend.py +663 -0
- dkist_processing_trend/tests/__init__.py +1 -0
- dkist_processing_trend/tests/conftest.py +718 -0
- dkist_processing_trend/tests/local_trial_workflows/__init__.py +0 -0
- dkist_processing_trend/tests/local_trial_workflows/l0_to_trend_visp_polcal.py +294 -0
- dkist_processing_trend/tests/local_trial_workflows/local_trial_helpers.py +488 -0
- dkist_processing_trend/tests/test_arm_class_factory.py +39 -0
- dkist_processing_trend/tests/test_base_tasks.py +86 -0
- dkist_processing_trend/tests/test_constants.py +120 -0
- dkist_processing_trend/tests/test_dark.py +97 -0
- dkist_processing_trend/tests/test_gain.py +135 -0
- dkist_processing_trend/tests/test_parameters.py +149 -0
- dkist_processing_trend/tests/test_parse.py +276 -0
- dkist_processing_trend/tests/test_prep_fit_data_base.py +233 -0
- dkist_processing_trend/tests/test_publish_catalog_messages.py +45 -0
- dkist_processing_trend/tests/test_run_pac_fitter.py +371 -0
- dkist_processing_trend/tests/test_stems.py +75 -0
- dkist_processing_trend/tests/test_transfer_output_data.py +76 -0
- dkist_processing_trend/tests/test_trend_fits_access.py +173 -0
- dkist_processing_trend/tests/test_visp.py +870 -0
- dkist_processing_trend/tests/test_workflows.py +10 -0
- dkist_processing_trend/tests/test_write_trend.py +460 -0
- dkist_processing_trend/workflows/__init__.py +3 -0
- dkist_processing_trend/workflows/arm_class_factory.py +30 -0
- dkist_processing_trend/workflows/visp.py +59 -0
- dkist_processing_trend-0.1.0rc1.dist-info/METADATA +549 -0
- dkist_processing_trend-0.1.0rc1.dist-info/RECORD +66 -0
- dkist_processing_trend-0.1.0rc1.dist-info/WHEEL +5 -0
- dkist_processing_trend-0.1.0rc1.dist-info/top_level.txt +3 -0
- docs/conf.py +57 -0
- docs/index.rst +10 -0
- docs/l0_to_trend_visp_polcal.rst +4 -0
- docs/landing_page.rst +11 -0
- docs/requirements_table.rst +8 -0
changelog/.gitempty
ADDED
|
File without changes
|
changelog/1.feature.rst
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Add Parse tasks.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Package providing support classes and methods used by all workflow tasks."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError
|
|
4
|
+
from importlib.metadata import version
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
__version__ = version(distribution_name=__name__)
|
|
8
|
+
except PackageNotFoundError:
|
|
9
|
+
# package is not installed
|
|
10
|
+
__version__ = "unknown"
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Configuration for the dkist-processing-trend package and logging thereof."""
|
|
2
|
+
|
|
3
|
+
from dkist_processing_common.config import DKISTProcessingCommonConfiguration
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DKISTProcessingTrendConfigurations(DKISTProcessingCommonConfiguration):
|
|
7
|
+
"""Configurations custom to the dkist-processing-trend package."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
dkist_processing_trend_configurations = DKISTProcessingTrendConfigurations()
|
|
11
|
+
dkist_processing_trend_configurations.log_configurations(log_parent_configurations=False)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Objects containing controlled values and logic used by the rest of the pipeline."""
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Trend specific parsed constants and controlled name enums."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from enum import StrEnum
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from dkist_processing_common.models.constants import ConstantsBase
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TrendBudName(StrEnum):
|
|
11
|
+
"""Names to be used for Trend buds."""
|
|
12
|
+
|
|
13
|
+
earliest_ip_start_time = "EARLIEST_IP_START_TIME"
|
|
14
|
+
arm_id_list = "ARM_ID_LIST"
|
|
15
|
+
wavelength_for_arm = "WAVELENGTH_FOR_ARM"
|
|
16
|
+
observing_program_id_list = "OBSERVING_PROGRAM_ID_LIST"
|
|
17
|
+
experiment_id_list = "EXPERIMENT_ID_LIST"
|
|
18
|
+
proposal_id_list = "PROPOSAL_ID_LIST"
|
|
19
|
+
hls_version_list = "HLS_VERSION_LIST"
|
|
20
|
+
visp_grating_angle_for_arm = "VISP_GRATING_ANGLE_FOR_ARM"
|
|
21
|
+
visp_arm_position_for_arm = "VISP_ARM_POSITION_FOR_ARM"
|
|
22
|
+
visp_grating_constant_for_arm = "VISP_GRATING_CONSTANT_FOR_ARM"
|
|
23
|
+
cryonirsp_od_filter_name_for_arm = "CRYONIRSP_OD_FILTER_NAME_FOR_ARM"
|
|
24
|
+
cryonirsp_grating_angle_for_arm = "CRYONIRSP_GRATING_ANGLE_FOR_ARM"
|
|
25
|
+
cryonirsp_grating_littrow_angle_for_arm = "CRYONIRSP_GRATING_LITTROW_ANGLE_FOR_ARM"
|
|
26
|
+
cryonirsp_grating_constant_for_arm = "CRYONIRSP_GRATING_CONSTANT_FOR_ARM"
|
|
27
|
+
cryonirsp_slit_width_for_arm = "CRYONIRSP_SLIT_WIDTH_FOR_ARM"
|
|
28
|
+
dlnirsp_grating_angle_for_arm = "DLNIRSP_GRATING_ANGLE_FOR_ARM"
|
|
29
|
+
dlnirsp_arm_position_for_arm = "DLNIRSP_ARM_POSITION_FOR_ARM"
|
|
30
|
+
dlnirsp_grating_constant_for_arm = "DLNIRSP_GRATING_CONSTANT_FOR_ARM"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _unjsonify_dict_keys(input_dict: dict[str, Any]) -> dict[str | int, Any]:
|
|
34
|
+
"""Apply `json.loads` to the keys of the input dictionary."""
|
|
35
|
+
return {json.loads(k): v for k, v in input_dict.items()}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class TrendConstants(ConstantsBase):
|
|
39
|
+
"""Trend specific constants to add to the common constants."""
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def earliest_ip_start_time(self) -> str:
|
|
43
|
+
"""Return the start time of the earliest IP in the input data."""
|
|
44
|
+
return self._db_dict[TrendBudName.earliest_ip_start_time]
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def arm_id_list(self) -> list[str | int]:
|
|
48
|
+
"""List of all instrument arms represented in the input data."""
|
|
49
|
+
return self._db_dict[TrendBudName.arm_id_list]
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def wavelength_for_arm(self) -> dict[str | int, float]:
|
|
53
|
+
"""Return a dictionary whose keys are arm_id and value is the associated wavelength."""
|
|
54
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.wavelength_for_arm])
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def observing_program_id_list(self) -> list[str]:
|
|
58
|
+
"""Sorted list of OP IDs, ordered by frequency in the input data."""
|
|
59
|
+
return self._db_dict[TrendBudName.observing_program_id_list]
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def experiment_id_list(self) -> list[str]:
|
|
63
|
+
"""Sorted list of experiment IDs, ordered by frequency in the input data."""
|
|
64
|
+
return self._db_dict[TrendBudName.experiment_id_list]
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def proposal_id_list(self) -> list[str]:
|
|
68
|
+
"""Sorted list of proposal IDs, ordered by frequency in the input data."""
|
|
69
|
+
return self._db_dict[TrendBudName.proposal_id_list]
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def hls_version_list(self) -> list[str]:
|
|
73
|
+
"""Sorted list of HLS versions, ordered by frequency in the input data."""
|
|
74
|
+
return self._db_dict[TrendBudName.hls_version_list]
|
|
75
|
+
|
|
76
|
+
############
|
|
77
|
+
### ViSP ###
|
|
78
|
+
############
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def visp_grating_angle_for_arm(self) -> dict[int, float]:
|
|
82
|
+
"""Return the ViSP grating angle for all arms."""
|
|
83
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.visp_grating_angle_for_arm])
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def visp_arm_position_for_arm(self) -> dict[int, float]:
|
|
87
|
+
"""Return the ViSP grating angle for all arms."""
|
|
88
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.visp_arm_position_for_arm])
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def visp_grating_constant_for_arm(self) -> dict[int, float]:
|
|
92
|
+
"""Return the ViSP grating angle for all arms."""
|
|
93
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.visp_grating_constant_for_arm])
|
|
94
|
+
|
|
95
|
+
#################
|
|
96
|
+
### CryoNIRSP ###
|
|
97
|
+
#################
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def cryonirsp_od_filter_name_for_arm(self) -> dict[str, str]:
|
|
101
|
+
"""Return the Cryo OD filter name for all arms."""
|
|
102
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.cryonirsp_od_filter_name_for_arm])
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def cryonirsp_grating_angle_for_arm(self) -> dict[str, float]:
|
|
106
|
+
"""Return the Cryo grating angle for all arms."""
|
|
107
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.cryonirsp_grating_angle_for_arm])
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def cryonirsp_grating_littrow_angle_for_arm(self) -> dict[str, float]:
|
|
111
|
+
"""Return the Cryo grating littrow angle for all arms."""
|
|
112
|
+
return _unjsonify_dict_keys(
|
|
113
|
+
self._db_dict[TrendBudName.cryonirsp_grating_littrow_angle_for_arm]
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def cryonirsp_grating_constant_for_arm(self) -> dict[str, float]:
|
|
118
|
+
"""Return the Cryo grating constant for all arms."""
|
|
119
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.cryonirsp_grating_constant_for_arm])
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def cryonirsp_slit_width_for_arm(self) -> dict[str, float]:
|
|
123
|
+
"""Return the Cryo slit width for all arms."""
|
|
124
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.cryonirsp_slit_width_for_arm])
|
|
125
|
+
|
|
126
|
+
###############
|
|
127
|
+
### DLNIRSP ###
|
|
128
|
+
###############
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def dlnirsp_grating_angle_for_arm(self) -> dict[str, float]:
|
|
132
|
+
"""Return the DLNIRSP grating angle for all arms."""
|
|
133
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.dlnirsp_grating_angle_for_arm])
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def dlnirsp_arm_position_for_arm(self) -> dict[str, float]:
|
|
137
|
+
"""Return the DLNIRSP grating angle for all arms."""
|
|
138
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.dlnirsp_arm_position_for_arm])
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def dlnirsp_grating_constant_for_arm(self) -> dict[str, float]:
|
|
142
|
+
"""Return the DLNIRSP grating angle for all arms."""
|
|
143
|
+
return _unjsonify_dict_keys(self._db_dict[TrendBudName.dlnirsp_grating_constant_for_arm])
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""BaseModel to hold `dkist_processing_pac` fit options."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
from pydantic import ConfigDict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FitOptions(BaseModel):
|
|
8
|
+
"""Container for a single set of `dkist_processing_pac` fit options."""
|
|
9
|
+
|
|
10
|
+
model_config = ConfigDict(extra="forbid")
|
|
11
|
+
|
|
12
|
+
name: str
|
|
13
|
+
remove_I_trend: bool
|
|
14
|
+
fit_mode_name: str
|
|
15
|
+
init_set_name: str
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Controlled mapping from TrendFitsAccess property names to header key values."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
5
|
+
# Property names with this value must be set in a `*FitsAccess` class with an instrument-specific `*MetadataKey`.
|
|
6
|
+
# See `TrendL0FitsAccess` for examples.
|
|
7
|
+
INSTRUMENT_SPECIFIC_KEY_SLUG = "You must use the instrument-specific MetadataKeys when setting"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TrendMetadataKey(StrEnum):
|
|
11
|
+
"""
|
|
12
|
+
Controlled list of property names on `~dkist_processing_trend.parsers.trend_l0_fits_access.TrendL0FitsAccess`.
|
|
13
|
+
|
|
14
|
+
Unlike the instrument specific `*MetadataKey` enums defined below many of the property names defined here (the names)
|
|
15
|
+
do not have a corresponding FITS header key (the values). This is because, in these cases, `~dkist_processing_trend.parsers.trend_l0_fits_access.TrendL0FitsAccess`
|
|
16
|
+
matches a header's instrument value and then uses the appropriate instrument's `*MetadataKey` to actually match
|
|
17
|
+
property name to header key.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
arm_id = f"{INSTRUMENT_SPECIFIC_KEY_SLUG} arm_id"
|
|
21
|
+
number_of_modulator_states = f"{INSTRUMENT_SPECIFIC_KEY_SLUG} number_of_modulator_states"
|
|
22
|
+
modulator_state = f"{INSTRUMENT_SPECIFIC_KEY_SLUG} modulator_state"
|
|
23
|
+
visp_grating_angle = "VSPGRTAN"
|
|
24
|
+
visp_arm_position = "VSPARMPS"
|
|
25
|
+
visp_grating_constant = "VSPGRTCN"
|
|
26
|
+
cryonirsp_od_filter_name = "CNFILTNP"
|
|
27
|
+
cryonirsp_grating_angle = "CNGRTPOS"
|
|
28
|
+
cryonirsp_grating_littrow_angle = "CNGRTLAT"
|
|
29
|
+
cryonirsp_grating_constant = "CNGRTCON"
|
|
30
|
+
cryonirsp_slit_width = "CNSLITW"
|
|
31
|
+
dlnirsp_grating_angle = "DLGRTAN"
|
|
32
|
+
dlnirsp_arm_position = "DLARMPS"
|
|
33
|
+
dlnirsp_grating_constant = "DLGRTCN"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class VispMetadataKey(StrEnum):
|
|
37
|
+
"""Controlled list of names for ViSP FITS metadata header keys."""
|
|
38
|
+
|
|
39
|
+
arm_id = "VSPARMID"
|
|
40
|
+
number_of_modulator_states = "VSPNUMST"
|
|
41
|
+
modulator_state = "VSPSTNUM"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class CryonirspMetadataKey(StrEnum):
|
|
45
|
+
"""Controlled list of names for Cryo-NIRSP FITS metadata header keys."""
|
|
46
|
+
|
|
47
|
+
camera_readout_mode = "CNCAMMD"
|
|
48
|
+
curr_frame_in_ramp = "CNCNDR"
|
|
49
|
+
num_frames_in_ramp = "CNNNDR"
|
|
50
|
+
arm_id = "CNARMID"
|
|
51
|
+
number_of_modulator_states = "CNMODNST"
|
|
52
|
+
modulator_state = "CNMODCST"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DlnirspMetadataKey(StrEnum):
|
|
56
|
+
"""Controlled list of names for DL-NIRSP FITS metadata header keys."""
|
|
57
|
+
|
|
58
|
+
modulator_spin_mode = "DLMOD"
|
|
59
|
+
camera_readout_mode = "DLCAMSMD"
|
|
60
|
+
num_frames_in_ramp = "DLCAMNS"
|
|
61
|
+
current_frame_in_ramp = "DLCAMCUR"
|
|
62
|
+
arm_id = "DLARMID"
|
|
63
|
+
camera_sample_sequence = "DLCAMSSQ"
|
|
64
|
+
number_of_modulator_states = "DLNUMST"
|
|
65
|
+
modulator_state = "DLSTNUM"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""BaseModel to hold instrument-specific pre and post fit processing options."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
from pydantic import ConfigDict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class InstrumentOptionsBase(BaseModel):
|
|
8
|
+
"""Base container for a single set of instrument pre and post fit processing options."""
|
|
9
|
+
|
|
10
|
+
model_config = ConfigDict(extra="forbid")
|
|
11
|
+
|
|
12
|
+
name: str
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class VispInstrumentOptions(InstrumentOptionsBase):
|
|
16
|
+
"""Container for ViSP-specific instrument processing options."""
|
|
17
|
+
|
|
18
|
+
num_spatial_px: int
|
|
19
|
+
num_spectral_px: int
|
|
20
|
+
spatial_smoothing_fit_order: int
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class CryonirspInstrumentOptions(InstrumentOptionsBase):
|
|
24
|
+
"""Container for CryoNIRSP-specific instrument processing options."""
|
|
25
|
+
|
|
26
|
+
num_spatial_px: int
|
|
27
|
+
num_spectral_px: int
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class DlnirspInstrumentOptions(InstrumentOptionsBase):
|
|
31
|
+
"""Container for DLNIRSP-specific instrument processing options."""
|
|
32
|
+
|
|
33
|
+
num_spatial_px: int
|
|
34
|
+
num_spectral_px: int
|
|
35
|
+
spatial_smoothing_fit_order: int
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Classes for holding pipeline parameters read from the input dataset document."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from dkist_processing_common.models.parameters import ParameterBase
|
|
7
|
+
from dkist_processing_common.models.parameters import _ParamMixinBase
|
|
8
|
+
|
|
9
|
+
from dkist_processing_trend.models.fit_options import FitOptions
|
|
10
|
+
from dkist_processing_trend.models.instrument import InstrumentName
|
|
11
|
+
from dkist_processing_trend.models.instrument_options import CryonirspInstrumentOptions
|
|
12
|
+
from dkist_processing_trend.models.instrument_options import DlnirspInstrumentOptions
|
|
13
|
+
from dkist_processing_trend.models.instrument_options import VispInstrumentOptions
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ParameterInstrumentMixin(_ParamMixinBase):
|
|
17
|
+
"""
|
|
18
|
+
Mixin that provides support for instrument-dependent parameters.
|
|
19
|
+
|
|
20
|
+
It allows for the "instrument" kwarg to `__init__` and provides the `self._instrument` attribute and methods for
|
|
21
|
+
finding parameterValues based on that instrument.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(self, instrument: str, **kwargs):
|
|
25
|
+
# This lets us use the more common instrument names for parameter database values (e.g., "cryonirsp" instead of "CRYO-NIRSP").
|
|
26
|
+
# StrEnum called with a *value* returns that enum member as an `InstrumentName`. Magic.
|
|
27
|
+
self._instrument = InstrumentName(instrument).name
|
|
28
|
+
|
|
29
|
+
def _find_parameter_for_instrument(self, parameter_name: str) -> Any:
|
|
30
|
+
full_param_name = f"{parameter_name}_{self._instrument}"
|
|
31
|
+
param = self._find_most_recent_past_value(full_param_name)
|
|
32
|
+
return param
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TrendParsingParameters(ParameterBase):
|
|
36
|
+
"""
|
|
37
|
+
Parameters specifically (and only) for the Parse task.
|
|
38
|
+
|
|
39
|
+
Needed because the Parse task doesn't know what things like obs OP start time or wavelength are.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def max_cs_step_time_sec(self):
|
|
44
|
+
"""Time window within which CS steps with identical GOS configurations are considered to be the same."""
|
|
45
|
+
return self._find_most_recent_past_value(
|
|
46
|
+
"trend_max_cs_step_time_sec", start_date=datetime.now()
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TrendParameters(ParameterBase, ParameterInstrumentMixin):
|
|
51
|
+
"""Trend-specific parameter class that exposes values in the input dataset document as properties on this class."""
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def fit_options_list(self) -> list[FitOptions]:
|
|
55
|
+
"""Return a list of `FitOptions` objects where each element represents a set of fit options to use in a separate PAC fit."""
|
|
56
|
+
raw_value: list[dict] = self._find_most_recent_past_value("trend_fit_options_list")
|
|
57
|
+
return [FitOptions.model_validate(i) for i in raw_value]
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def instrument_processing_options(
|
|
61
|
+
self,
|
|
62
|
+
) -> list[VispInstrumentOptions, CryonirspInstrumentOptions, DlnirspInstrumentOptions]:
|
|
63
|
+
"""Return a list of dicts where each element represents a set of PAC input processing options."""
|
|
64
|
+
raw_value: list[dict] = self._find_parameter_for_instrument(
|
|
65
|
+
"trend_instrument_processing_options"
|
|
66
|
+
)
|
|
67
|
+
match self._instrument:
|
|
68
|
+
case InstrumentName.visp.name:
|
|
69
|
+
return [VispInstrumentOptions.model_validate(i) for i in raw_value]
|
|
70
|
+
case InstrumentName.cryonirsp.name:
|
|
71
|
+
return [CryonirspInstrumentOptions.model_validate(i) for i in raw_value]
|
|
72
|
+
case InstrumentName.dlnirsp.name:
|
|
73
|
+
return [DlnirspInstrumentOptions.model_validate(i) for i in raw_value]
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def visp_beam_border(self) -> int:
|
|
77
|
+
"""Return the pixel location of the border between the two ViSP beams."""
|
|
78
|
+
return self._find_most_recent_past_value("trend_visp_beam_border")
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def visp_geo_upsample_factor(self) -> int:
|
|
82
|
+
"""Return the upsample factor used by `phase_cross_correlation` in the ViSP geometric task."""
|
|
83
|
+
return self._find_most_recent_past_value("trend_visp_geo_upsample_factor")
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def visp_hairline_median_spatial_smoothing_width_px(self) -> int:
|
|
87
|
+
"""Size of median filter in the spatial dimension with which to smooth data for hairline identification."""
|
|
88
|
+
return self._find_most_recent_past_value(
|
|
89
|
+
"trend_visp_hairline_median_spatial_smoothing_width_px"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def visp_hairline_fraction(self):
|
|
94
|
+
"""Relative difference from median used to identify slit positions covered by the hairlines."""
|
|
95
|
+
return self._find_most_recent_past_value("trend_visp_hairline_fraction")
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def visp_hairline_mask_spatial_smoothing_width_px(self) -> float:
|
|
99
|
+
"""Amount to smooth the hairline mask in the spatial direction.
|
|
100
|
+
|
|
101
|
+
This helps capture the higher-flux wings of the hairlines that would otherwise require a `hairline_fraction`
|
|
102
|
+
that was so low it captures other optical features.
|
|
103
|
+
"""
|
|
104
|
+
return self._find_most_recent_past_value(
|
|
105
|
+
"trend_visp_hairline_mask_spatial_smoothing_width_px"
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def visp_hairline_mask_gaussian_peak_cutoff_fraction(self) -> float:
|
|
110
|
+
"""Fraction of the maximum smoothed mask value used to truncate the smoothed mask.
|
|
111
|
+
|
|
112
|
+
This ensures that very very small values way out in the wings are not included in the mask. For example, if
|
|
113
|
+
this value is 0.01 then any mask points less than 1% of the maximum will be ignored.
|
|
114
|
+
"""
|
|
115
|
+
return self._find_most_recent_past_value(
|
|
116
|
+
"trend_visp_hairline_mask_gaussian_peak_cutoff_fraction"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
@property
|
|
120
|
+
def visp_polcal_spatial_median_filter_width_px(self) -> int:
|
|
121
|
+
"""Return the size of the median filter to apply in the spatial dimension to polcal data."""
|
|
122
|
+
return self._find_most_recent_past_value("trend_visp_polcal_spatial_median_filter_width_px")
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def visp_polcal_demod_spatial_smooth_min_samples(self) -> float:
|
|
126
|
+
"""Return fractional number of samples required for the RANSAC regressor used to smooth ViSP demod matrices."""
|
|
127
|
+
return self._find_most_recent_past_value(
|
|
128
|
+
"trend_visp_polcal_demod_spatial_smooth_min_samples"
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def visp_polcal_demod_upsample_order(self) -> int:
|
|
133
|
+
"""Interpolation order to use when upsampling the demodulation matrices to the full frame."""
|
|
134
|
+
return self._find_most_recent_past_value("trend_visp_polcal_demod_upsample_order")
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Trend specific file tags and enums."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
5
|
+
from dkist_processing_common.models.tags import StemName
|
|
6
|
+
from dkist_processing_common.models.tags import Tag
|
|
7
|
+
|
|
8
|
+
from dkist_processing_trend.models.task_name import TrendTaskName
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TrendStemName(StrEnum):
|
|
12
|
+
"""Controlled list of Tag Stems."""
|
|
13
|
+
|
|
14
|
+
arm_id = "ARM_ID"
|
|
15
|
+
beam = "BEAM"
|
|
16
|
+
instrument_processing_options = "INST_PROC_OPT"
|
|
17
|
+
pac_fit_options = "PAC_FIT_OPT"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TrendTag(Tag):
|
|
21
|
+
"""Trend specific tag formatting."""
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def arm_id(cls, arm_id: str | int) -> str:
|
|
25
|
+
"""
|
|
26
|
+
Tags based on the arm_id from which the data is recorded.
|
|
27
|
+
|
|
28
|
+
Parameters
|
|
29
|
+
----------
|
|
30
|
+
arm_id
|
|
31
|
+
The arm ID in use; {1, 2, 3} for ViSP, {"CI", "SP"} for Cryo, and {"VIS", "JBand", "HBand"} for DL
|
|
32
|
+
|
|
33
|
+
Returns
|
|
34
|
+
-------
|
|
35
|
+
The formatted tag string
|
|
36
|
+
"""
|
|
37
|
+
return cls.format_tag(TrendStemName.arm_id, arm_id)
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def beam(cls, beam_num: int) -> str:
|
|
41
|
+
"""
|
|
42
|
+
Tags by beam number.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
beam_num: int
|
|
47
|
+
The beam number
|
|
48
|
+
|
|
49
|
+
"""
|
|
50
|
+
return cls.format_tag(TrendStemName.beam, beam_num)
|
|
51
|
+
|
|
52
|
+
@classmethod
|
|
53
|
+
def instrument_processing_options(cls, instrument_processing_options_name: str) -> str:
|
|
54
|
+
"""Tag on a given set of instrument processing options."""
|
|
55
|
+
return cls.format_tag(
|
|
56
|
+
TrendStemName.instrument_processing_options, instrument_processing_options_name
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
@classmethod
|
|
60
|
+
def pac_fit_options(cls, pac_fit_options_name: str) -> str:
|
|
61
|
+
"""Tag on a given set of pac fit options."""
|
|
62
|
+
return cls.format_tag(TrendStemName.pac_fit_options, pac_fit_options_name)
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def task_global_pac_input(cls) -> str:
|
|
66
|
+
"""Tags 'global' PAC inputs."""
|
|
67
|
+
return cls.format_tag(StemName.task, TrendTaskName.global_pac_input)
|
|
68
|
+
|
|
69
|
+
@classmethod
|
|
70
|
+
def task_local_pac_input(cls) -> str:
|
|
71
|
+
"""Tags 'local' PAC inputs."""
|
|
72
|
+
return cls.format_tag(StemName.task, TrendTaskName.local_pac_input)
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def task_calibration_sequence(cls) -> str:
|
|
76
|
+
"""Tags the full GOS configuration of input calibration sequence(s)."""
|
|
77
|
+
return cls.format_tag(StemName.task, TrendTaskName.calibration_sequence)
|
|
78
|
+
|
|
79
|
+
@classmethod
|
|
80
|
+
def task_best_fit_parameters(cls) -> str:
|
|
81
|
+
"""Tags the best-fit parameter values."""
|
|
82
|
+
return cls.format_tag(StemName.task, TrendTaskName.best_fit_parameters)
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def task_best_fit_demodulation_matrices(cls) -> str:
|
|
86
|
+
"""Tags the best-fit demodulation matrices."""
|
|
87
|
+
return cls.format_tag(StemName.task, TrendTaskName.best_fit_demodulation_matrices)
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def task_best_fit_flux(cls) -> str:
|
|
91
|
+
"""Tags the best-fit model flux."""
|
|
92
|
+
return cls.format_tag(StemName.task, TrendTaskName.best_fit_flux)
|
|
93
|
+
|
|
94
|
+
@classmethod
|
|
95
|
+
def task_fit_residuals(cls) -> str:
|
|
96
|
+
"""Tags the flux fit residuals for the best-fit parameters."""
|
|
97
|
+
return cls.format_tag(StemName.task, TrendTaskName.fit_residuals)
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
def task_processed_demodulation_matrices(cls) -> str:
|
|
101
|
+
"""Tags the processed demodulation matrices."""
|
|
102
|
+
return cls.format_tag(StemName.task, TrendTaskName.processed_demodulation_matrices)
|
|
103
|
+
|
|
104
|
+
@classmethod
|
|
105
|
+
def task_single_beam_demodulated_polcal_data(cls) -> str:
|
|
106
|
+
"""Tags the demodulated polcal data (DMPD)."""
|
|
107
|
+
return cls.format_tag(StemName.task, TrendTaskName.single_beam_demodulated_polcal_data)
|
|
108
|
+
|
|
109
|
+
@classmethod
|
|
110
|
+
def task_demodulated_polcal_data(cls) -> str:
|
|
111
|
+
"""Tags the demodulated polcal data (DMPD)."""
|
|
112
|
+
return cls.format_tag(StemName.task, TrendTaskName.demodulated_polcal_data)
|
|
113
|
+
|
|
114
|
+
@classmethod
|
|
115
|
+
def task_visp_geometric_calibration(cls) -> str:
|
|
116
|
+
"""Tags the ViSP geometric calibration (angles and shift) intermediate product."""
|
|
117
|
+
return cls.format_tag(StemName.task, TrendTaskName.visp_geometric_calibration)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Controlled list of trend-specific TASK tag names."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TrendTaskName(StrEnum):
|
|
7
|
+
"""Controlled set of trend task names."""
|
|
8
|
+
|
|
9
|
+
global_pac_input = "GLOBAL_PAC_INPUT"
|
|
10
|
+
local_pac_input = "LOCAL_PAC_INPUT"
|
|
11
|
+
calibration_sequence = "CALIBRATION_SEQUENCE"
|
|
12
|
+
best_fit_parameters = "BEST_FIT_PARAMETERS"
|
|
13
|
+
best_fit_demodulation_matrices = "BEST_FIT_DEMOD_MATRICES"
|
|
14
|
+
best_fit_flux = "BEST_FIT_FLUX"
|
|
15
|
+
fit_residuals = "FIT_RESIDUALS"
|
|
16
|
+
processed_demodulation_matrices = "PROCESSED_DEMODULATION_MATRICES"
|
|
17
|
+
single_beam_demodulated_polcal_data = "SINGLE_BEAM_DEMODULATED_POLCAL_DATA"
|
|
18
|
+
demodulated_polcal_data = "DEMODULATED_POLCAL_DATA"
|
|
19
|
+
|
|
20
|
+
visp_geometric_calibration = "VISP_GEO_CAL"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Custom Stems for parsing Trend data."""
|