dkist-processing-dlnirsp 0.14.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- changelog/.gitempty +0 -0
- dkist_processing_dlnirsp/__init__.py +9 -0
- dkist_processing_dlnirsp/config.py +12 -0
- dkist_processing_dlnirsp/dev_scripts/__init__.py +1 -0
- dkist_processing_dlnirsp/dev_scripts/test_slitbeam_group_assignment.py +118 -0
- dkist_processing_dlnirsp/models/__init__.py +1 -0
- dkist_processing_dlnirsp/models/constants.py +154 -0
- dkist_processing_dlnirsp/models/parameters.py +177 -0
- dkist_processing_dlnirsp/models/tags.py +157 -0
- dkist_processing_dlnirsp/models/task_name.py +11 -0
- dkist_processing_dlnirsp/parsers/__init__.py +1 -0
- dkist_processing_dlnirsp/parsers/dlnirsp_l0_fits_access.py +80 -0
- dkist_processing_dlnirsp/parsers/dlnirsp_l1_fits_acess.py +30 -0
- dkist_processing_dlnirsp/parsers/mosaic.py +264 -0
- dkist_processing_dlnirsp/parsers/task.py +99 -0
- dkist_processing_dlnirsp/parsers/time.py +49 -0
- dkist_processing_dlnirsp/parsers/wavelength.py +26 -0
- dkist_processing_dlnirsp/tasks/__init__.py +14 -0
- dkist_processing_dlnirsp/tasks/assemble_movie.py +146 -0
- dkist_processing_dlnirsp/tasks/dark.py +89 -0
- dkist_processing_dlnirsp/tasks/dlnirsp_base.py +99 -0
- dkist_processing_dlnirsp/tasks/geometric.py +625 -0
- dkist_processing_dlnirsp/tasks/ifu_drift.py +247 -0
- dkist_processing_dlnirsp/tasks/instrument_polarization.py +480 -0
- dkist_processing_dlnirsp/tasks/l1_output_data.py +13 -0
- dkist_processing_dlnirsp/tasks/lamp.py +126 -0
- dkist_processing_dlnirsp/tasks/linearity_correction.py +285 -0
- dkist_processing_dlnirsp/tasks/make_movie_frames.py +155 -0
- dkist_processing_dlnirsp/tasks/mixin/__init__.py +1 -0
- dkist_processing_dlnirsp/tasks/mixin/corrections.py +157 -0
- dkist_processing_dlnirsp/tasks/mixin/group_id.py +272 -0
- dkist_processing_dlnirsp/tasks/mixin/input_frame_loaders.py +27 -0
- dkist_processing_dlnirsp/tasks/mixin/intermediate_frame_helpers.py +140 -0
- dkist_processing_dlnirsp/tasks/mixin/linearized_frame_loaders.py +81 -0
- dkist_processing_dlnirsp/tasks/parse.py +201 -0
- dkist_processing_dlnirsp/tasks/quality_metrics.py +192 -0
- dkist_processing_dlnirsp/tasks/science.py +716 -0
- dkist_processing_dlnirsp/tasks/solar.py +278 -0
- dkist_processing_dlnirsp/tasks/trial_output_data.py +54 -0
- dkist_processing_dlnirsp/tasks/write_l1.py +231 -0
- dkist_processing_dlnirsp/tests/__init__.py +1 -0
- dkist_processing_dlnirsp/tests/conftest.py +1698 -0
- dkist_processing_dlnirsp/tests/local_trial_workflows/__init__.py +0 -0
- dkist_processing_dlnirsp/tests/local_trial_workflows/e2e_dev_mockers.py +259 -0
- dkist_processing_dlnirsp/tests/local_trial_workflows/e2e_helpers.py +558 -0
- dkist_processing_dlnirsp/tests/local_trial_workflows/l0_polcals_as_science.py +427 -0
- dkist_processing_dlnirsp/tests/local_trial_workflows/l0_solar_gain_as_science.py +442 -0
- dkist_processing_dlnirsp/tests/local_trial_workflows/l0_to_l1.py +405 -0
- dkist_processing_dlnirsp/tests/local_trial_workflows/translate_files.py +55 -0
- dkist_processing_dlnirsp/tests/test_assemble_movie.py +163 -0
- dkist_processing_dlnirsp/tests/test_assemble_quality.py +47 -0
- dkist_processing_dlnirsp/tests/test_corrections.py +223 -0
- dkist_processing_dlnirsp/tests/test_dark.py +127 -0
- dkist_processing_dlnirsp/tests/test_dlnirsp_base.py +75 -0
- dkist_processing_dlnirsp/tests/test_dlnirsp_constants.py +167 -0
- dkist_processing_dlnirsp/tests/test_dlnirsp_fits_access.py +76 -0
- dkist_processing_dlnirsp/tests/test_geometric.py +225 -0
- dkist_processing_dlnirsp/tests/test_group_id.py +195 -0
- dkist_processing_dlnirsp/tests/test_ifu_drift.py +210 -0
- dkist_processing_dlnirsp/tests/test_instrument_polarization_calibration.py +463 -0
- dkist_processing_dlnirsp/tests/test_lamp.py +134 -0
- dkist_processing_dlnirsp/tests/test_linearity_correction.py +431 -0
- dkist_processing_dlnirsp/tests/test_make_movie_frames.py +97 -0
- dkist_processing_dlnirsp/tests/test_parameters.py +152 -0
- dkist_processing_dlnirsp/tests/test_parse.py +562 -0
- dkist_processing_dlnirsp/tests/test_quality.py +185 -0
- dkist_processing_dlnirsp/tests/test_science.py +368 -0
- dkist_processing_dlnirsp/tests/test_solar.py +419 -0
- dkist_processing_dlnirsp/tests/test_trial_create_quality_report.py +78 -0
- dkist_processing_dlnirsp/tests/test_trial_output_data.py +235 -0
- dkist_processing_dlnirsp/tests/test_workflows.py +9 -0
- dkist_processing_dlnirsp/tests/test_write_l1.py +372 -0
- dkist_processing_dlnirsp/workflows/__init__.py +2 -0
- dkist_processing_dlnirsp/workflows/l0_processing.py +78 -0
- dkist_processing_dlnirsp/workflows/trial_workflow.py +97 -0
- dkist_processing_dlnirsp-0.14.3.dist-info/METADATA +194 -0
- dkist_processing_dlnirsp-0.14.3.dist-info/RECORD +90 -0
- dkist_processing_dlnirsp-0.14.3.dist-info/WHEEL +5 -0
- dkist_processing_dlnirsp-0.14.3.dist-info/entry_points.txt +2 -0
- dkist_processing_dlnirsp-0.14.3.dist-info/top_level.txt +4 -0
- docs/Makefile +134 -0
- docs/changelog.rst +11 -0
- docs/conf.py +53 -0
- docs/index.rst +13 -0
- docs/l0_to_l1_dlnirsp.rst +11 -0
- docs/l0_to_l1_dlnirsp_full-trial.rst +10 -0
- docs/make.bat +170 -0
- docs/requirements_table.rst +8 -0
- docs/scientific_changelog.rst +10 -0
- licenses/LICENSE.rst +11 -0
changelog/.gitempty
ADDED
|
File without changes
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Package providing support classes and methods used by all workflow tasks."""
|
|
2
|
+
from importlib.metadata import PackageNotFoundError
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version(distribution_name=__name__)
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
# package is not installed
|
|
9
|
+
__version__ = "unknown"
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Configuration for the dkist-processing-dlnirsp package and logging thereof."""
|
|
2
|
+
from dkist_processing_common.config import DKISTProcessingCommonConfiguration
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class DKISTProcessingDLNIRSPConfigurations(DKISTProcessingCommonConfiguration):
|
|
6
|
+
"""Configurations custom to the dkist-processing-dlnirsp package."""
|
|
7
|
+
|
|
8
|
+
pass # nothing custom.... yet
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
dkist_processing_dlnirsp_configurations = DKISTProcessingDLNIRSPConfigurations()
|
|
12
|
+
dkist_processing_dlnirsp_configurations.log_configurations()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Modules to allow easy testing of pipeline components by pipeline developers."""
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Script for showing the slitbeam assignment based on a group_id array."""
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from astropy.io import fits
|
|
8
|
+
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
9
|
+
from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetParameterValue
|
|
10
|
+
from dkist_service_configuration.logging import logger
|
|
11
|
+
|
|
12
|
+
from dkist_processing_dlnirsp.models.parameters import DlnirspParameters
|
|
13
|
+
from dkist_processing_dlnirsp.tasks.mixin.group_id import GroupIdMixin
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BlankTaskWithGroupId(WorkflowTaskBase, GroupIdMixin):
|
|
17
|
+
"""We just need this so we can use the group id mixin"""
|
|
18
|
+
|
|
19
|
+
def run(self) -> None:
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def mark_slitbeams(task: BlankTaskWithGroupId) -> np.ndarray:
|
|
24
|
+
"""Produce an array where each pixel's value is its slitbeam ID."""
|
|
25
|
+
group_id_array = task.group_id_drifted_id_array
|
|
26
|
+
slitbeam_group_dict = task.group_id_slitbeam_group_dict
|
|
27
|
+
|
|
28
|
+
slitbeam_id_array = np.empty(group_id_array.shape) * np.nan
|
|
29
|
+
for slitbeam, groups_in_slitbeam in slitbeam_group_dict.items():
|
|
30
|
+
for group_id in groups_in_slitbeam:
|
|
31
|
+
group_idx = task.group_id_get_idx(group_id=group_id)
|
|
32
|
+
slitbeam_id_array[group_idx] = slitbeam
|
|
33
|
+
|
|
34
|
+
return slitbeam_id_array
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def create_parameter_dict(
|
|
38
|
+
group_id_file_path: str | Path, slit_separation_px: int, arm_id: str
|
|
39
|
+
) -> dict[str, list[InputDatasetParameterValue]]:
|
|
40
|
+
"""Create a dictionary that can be used to instantiate a DlnirspParameters object."""
|
|
41
|
+
param_dict = defaultdict(list)
|
|
42
|
+
date = datetime(1946, 11, 20)
|
|
43
|
+
param_dict["dlnirsp_group_id_rough_slit_separation_px"].append(
|
|
44
|
+
InputDatasetParameterValue(
|
|
45
|
+
parameter_value=slit_separation_px,
|
|
46
|
+
parameter_value_id=0,
|
|
47
|
+
parameter_value_start_date=date,
|
|
48
|
+
)
|
|
49
|
+
)
|
|
50
|
+
file_param_dict = {"param_path": str(group_id_file_path.absolute()), "is_file": True}
|
|
51
|
+
param_dict[f"dlnirsp_group_id_file_{arm_id.casefold()}"].append(
|
|
52
|
+
InputDatasetParameterValue(
|
|
53
|
+
parameter_value=file_param_dict, parameter_value_id=1, parameter_value_start_date=date
|
|
54
|
+
)
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
return param_dict
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def main(
|
|
61
|
+
group_id_file_path: str | Path,
|
|
62
|
+
output_file_name: str,
|
|
63
|
+
arm_id: str,
|
|
64
|
+
rough_slit_separation_px: int = 300,
|
|
65
|
+
) -> int:
|
|
66
|
+
"""Use provided group id file to make a slitbeam id file."""
|
|
67
|
+
if isinstance(group_id_file_path, str):
|
|
68
|
+
group_id_file_path = Path(group_id_file_path)
|
|
69
|
+
|
|
70
|
+
logger.info("Setting up task and parameters")
|
|
71
|
+
task = BlankTaskWithGroupId(
|
|
72
|
+
recipe_run_id=0, workflow_name="test_group_id_assignment", workflow_version="0.0.0b1rc0"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
loaded_parameter_dict = create_parameter_dict(
|
|
76
|
+
group_id_file_path, rough_slit_separation_px, arm_id=arm_id
|
|
77
|
+
)
|
|
78
|
+
task.parameters = DlnirspParameters(
|
|
79
|
+
loaded_parameter_dict, wavelength=1.0, arm_id=arm_id
|
|
80
|
+
) # Wavelength doesn't matter
|
|
81
|
+
|
|
82
|
+
logger.info("Creating slitbeam id array")
|
|
83
|
+
slitbeam_array = mark_slitbeams(task)
|
|
84
|
+
|
|
85
|
+
fits.PrimaryHDU(slitbeam_array).writeto(output_file_name, overwrite=True)
|
|
86
|
+
logger.info(f"Slitbeam id array saved to {output_file_name}")
|
|
87
|
+
|
|
88
|
+
return 0
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def command_line():
|
|
92
|
+
"""Entry point for main() from command line."""
|
|
93
|
+
import argparse
|
|
94
|
+
import sys
|
|
95
|
+
|
|
96
|
+
parser = argparse.ArgumentParser(
|
|
97
|
+
description="Produce a FITS file showing the slitbeam each pixel is assigned to."
|
|
98
|
+
)
|
|
99
|
+
parser.add_argument(
|
|
100
|
+
"-s",
|
|
101
|
+
"--slit-separation-px",
|
|
102
|
+
help="Rough distance between 2 DL slits (NOT slitbeams)",
|
|
103
|
+
type=int,
|
|
104
|
+
default=300,
|
|
105
|
+
)
|
|
106
|
+
parser.add_argument("-a", "--arm-id", help="Dlnirsp arm", type=str, default="JBand")
|
|
107
|
+
parser.add_argument("group_id_file", help="Path to a group_id FITS file")
|
|
108
|
+
parser.add_argument("output_file", help="Where to save slitbeam assignment file")
|
|
109
|
+
|
|
110
|
+
args = parser.parse_args()
|
|
111
|
+
sys.exit(
|
|
112
|
+
main(
|
|
113
|
+
group_id_file_path=args.group_id_file,
|
|
114
|
+
output_file_name=args.output_file,
|
|
115
|
+
rough_slit_separation_px=args.slit_separation_px,
|
|
116
|
+
arm_id=args.arm_id,
|
|
117
|
+
)
|
|
118
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Data classifications and enumerations."""
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Dataset-level constants for a pipeline run."""
|
|
2
|
+
from enum import Enum
|
|
3
|
+
|
|
4
|
+
from dkist_processing_common.models.constants import BudName
|
|
5
|
+
from dkist_processing_common.models.constants import ConstantsBase
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DlnirspBudName(Enum):
|
|
9
|
+
"""Names to be used for DLNIRSP buds."""
|
|
10
|
+
|
|
11
|
+
arm_id = "ARM_ID"
|
|
12
|
+
num_beams = "NUM_BEAMS"
|
|
13
|
+
num_dsps_repeats = "NUM_DSPS_REPEATS" # TODO: Maybe we don't need this?
|
|
14
|
+
num_dither_steps = "NUM_DITHER_STEPS"
|
|
15
|
+
num_mosaic_repeats = "NUM_MOSAIC_REPEATS"
|
|
16
|
+
num_spatial_steps_X = "NUM_SPATIAL_STEPS_X"
|
|
17
|
+
num_spatial_steps_Y = "NUM_SPATIAL_STEPS_Y"
|
|
18
|
+
num_modstates = "NUM_MODSTATES"
|
|
19
|
+
wavelength = "WAVELENGTH"
|
|
20
|
+
camera_readout_mode = "CAMERA_READOUT_MODE"
|
|
21
|
+
polarimeter_mode = "POLARIMETER_MODE"
|
|
22
|
+
time_obs_list = "TIME_OBS_LIST"
|
|
23
|
+
lamp_gain_exposure_times = "LAMP_GAIN_EXPOSURE_TIMES"
|
|
24
|
+
solar_gain_exposure_times = "SOLAR_GAIN_EXPOSURE_TIMES"
|
|
25
|
+
polcal_exposure_times = "POLCAL_EXPOSURE_TIMES"
|
|
26
|
+
observe_exposure_times = "OBSERVE_EXPOSURE_TIMES"
|
|
27
|
+
non_dark_task_exposure_times = "NON_DARK_TASK_EXPOSURE_TIMES"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
VIS_ARM_NAMES = ["VIS".casefold()]
|
|
31
|
+
IR_ARM_NAMES = ["JBand".casefold(), "HBand".casefold()]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class DlnirspConstants(ConstantsBase):
|
|
35
|
+
"""DLNIRSP specific constants to add to the common constants."""
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def arm_id(self) -> str:
|
|
39
|
+
"""Arm used to record the data, either VIS or one of 2 IR bands."""
|
|
40
|
+
return self._db_dict[DlnirspBudName.arm_id]
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def is_ir_data(self) -> bool:
|
|
44
|
+
"""Return True if the data are from an IR camera and need to be linearized."""
|
|
45
|
+
if self.arm_id.casefold() in VIS_ARM_NAMES:
|
|
46
|
+
return False
|
|
47
|
+
if self.arm_id.casefold() in IR_ARM_NAMES:
|
|
48
|
+
return True
|
|
49
|
+
raise ValueError(f"Unable to determine the camera type of Arm ID {self.arm_id}")
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def num_beams(self) -> int:
|
|
53
|
+
"""Determine the number of beams present in the data."""
|
|
54
|
+
return 2
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def num_slits(self) -> int:
|
|
58
|
+
"""Return the number of slits on a single detector readout."""
|
|
59
|
+
return 4
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def num_dither_steps(self) -> int:
|
|
63
|
+
"""Return the number of dither steps."""
|
|
64
|
+
return self._db_dict[DlnirspBudName.num_dither_steps]
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def num_mosaic_repeats(self) -> int:
|
|
68
|
+
"""
|
|
69
|
+
Return the number of mosaic repeats.
|
|
70
|
+
|
|
71
|
+
I.e., the number of times the mosaic pattern was observed.
|
|
72
|
+
"""
|
|
73
|
+
return self._db_dict[DlnirspBudName.num_mosaic_repeats]
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def num_spatial_steps_X(self) -> int:
|
|
77
|
+
"""Return the number of spatial steps in the X direction in the mosaic pattern."""
|
|
78
|
+
return self._db_dict[DlnirspBudName.num_spatial_steps_X]
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def num_spatial_steps_Y(self) -> int:
|
|
82
|
+
"""Return the number of spatial steps in the Y direction in the mosaic pattern."""
|
|
83
|
+
return self._db_dict[DlnirspBudName.num_spatial_steps_Y]
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def num_mosaic_tiles(self) -> int:
|
|
87
|
+
"""Return the total number of tiles that make up the full mosaic."""
|
|
88
|
+
return self.num_spatial_steps_X * self.num_spatial_steps_Y
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def num_cs_steps(self):
|
|
92
|
+
"""Find the number of calibration sequence steps."""
|
|
93
|
+
return self._db_dict[BudName.num_cs_steps]
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def time_obs_list(self) -> list[str]:
|
|
97
|
+
"""Construct a list of all the dateobs for this dataset."""
|
|
98
|
+
return self._db_dict[DlnirspBudName.time_obs_list]
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def wavelength(self) -> float:
|
|
102
|
+
"""Wavelength."""
|
|
103
|
+
return self._db_dict[DlnirspBudName.wavelength]
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def camera_readout_mode(self) -> str:
|
|
107
|
+
"""Determine the readout mode of the camera."""
|
|
108
|
+
return self._db_dict[DlnirspBudName.camera_readout_mode]
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def correct_for_polarization(self) -> bool:
|
|
112
|
+
"""Return True if dataset is polarimetric."""
|
|
113
|
+
# TODO: Check what the option "Other" for DLPOLMD means
|
|
114
|
+
return self._db_dict[DlnirspBudName.polarimeter_mode] == "Full Stokes"
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def lamp_gain_exposure_times(self) -> list[float]:
|
|
118
|
+
"""Construct a list of lamp gain FPA exposure times for the dataset."""
|
|
119
|
+
return self._db_dict[DlnirspBudName.lamp_gain_exposure_times]
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def solar_gain_exposure_times(self) -> list[float]:
|
|
123
|
+
"""Construct a list of solar gain FPA exposure times for the dataset."""
|
|
124
|
+
return self._db_dict[DlnirspBudName.solar_gain_exposure_times]
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def polcal_exposure_times(self) -> list[float]:
|
|
128
|
+
"""Construct a list of polcal FPA exposure times for the dataset."""
|
|
129
|
+
if self.correct_for_polarization:
|
|
130
|
+
return self._db_dict[DlnirspBudName.polcal_exposure_times]
|
|
131
|
+
else:
|
|
132
|
+
return []
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def observe_exposure_times(self) -> list[float]:
|
|
136
|
+
"""Construct a list of observe FPA exposure times."""
|
|
137
|
+
return self._db_dict[DlnirspBudName.observe_exposure_times]
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def non_dark_task_exposure_times(self) -> list[float]:
|
|
141
|
+
"""Return a list of all exposure times required for all tasks other than dark."""
|
|
142
|
+
exposure_times = list()
|
|
143
|
+
exposure_times.extend(self.lamp_gain_exposure_times)
|
|
144
|
+
exposure_times.extend(self.solar_gain_exposure_times)
|
|
145
|
+
exposure_times.extend(self.observe_exposure_times)
|
|
146
|
+
if self.correct_for_polarization:
|
|
147
|
+
exposure_times.extend(self.polcal_exposure_times)
|
|
148
|
+
exposure_times = list(set(exposure_times))
|
|
149
|
+
return exposure_times
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def num_modstates(self) -> int:
|
|
153
|
+
"""Return the number of modulator states."""
|
|
154
|
+
return self._db_dict[DlnirspBudName.num_modstates]
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Machinery to access pipeline parameters served in input dataset document."""
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from functools import cached_property
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from dkist_processing_common.models.parameters import ParameterArmIdMixin
|
|
7
|
+
from dkist_processing_common.models.parameters import ParameterBase
|
|
8
|
+
from dkist_processing_common.models.parameters import ParameterWavelengthMixin
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DlnirspParsingParameters(ParameterBase):
|
|
12
|
+
"""
|
|
13
|
+
Parameters specifically (and only) for the Parse task.
|
|
14
|
+
|
|
15
|
+
Needed because the Parse task doesn't know what the wavelength is yet and therefore can't use the
|
|
16
|
+
`ParameterWaveLengthMixin`.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def max_cs_step_time_sec(self) -> float:
|
|
21
|
+
"""Time window within which CS steps with identical GOS configurations are considered to be the same."""
|
|
22
|
+
return self._find_most_recent_past_value(
|
|
23
|
+
"dlnirsp_max_cs_step_time_sec", start_date=datetime.now()
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DlnirspParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmIdMixin):
|
|
28
|
+
"""Put all DLNIRSP parameters parsed from the input dataset document in a single property."""
|
|
29
|
+
|
|
30
|
+
@cached_property
|
|
31
|
+
def raw_group_id_array(self) -> np.ndarray:
|
|
32
|
+
"""
|
|
33
|
+
Return an array containing 'group ids' of each array pixel.
|
|
34
|
+
|
|
35
|
+
The group id labels each IFU ribbon/mirror slice.
|
|
36
|
+
|
|
37
|
+
NOTE: This array has NOT been corrected for IFU drift. See `GroupIdMixin.group_id_drifted_id_array` for that.
|
|
38
|
+
"""
|
|
39
|
+
param_dict = self._find_parameter_for_arm("dlnirsp_group_id_file")
|
|
40
|
+
return self._load_param_value_from_fits(param_dict)
|
|
41
|
+
|
|
42
|
+
@cached_property
|
|
43
|
+
def raw_dispersion_array(self) -> np.ndarray:
|
|
44
|
+
"""
|
|
45
|
+
Return an array that provides the dispersion (in Angstrom / px) for each group.
|
|
46
|
+
|
|
47
|
+
NOTE: This array has NOT been corrected for IFU drift. Use the intermediate frame
|
|
48
|
+
`DlnirspTag.task_drifted_dispersion()` for that.
|
|
49
|
+
"""
|
|
50
|
+
param_dict = self._find_parameter_for_arm("dlnirsp_geo_dispersion_file")
|
|
51
|
+
return self._load_param_value_from_fits(param_dict)
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def raw_ifu_x_pos_array(self) -> np.ndarray:
|
|
55
|
+
"""
|
|
56
|
+
Return the array mapping raw pixel position to an X coordinate in the IFU.
|
|
57
|
+
|
|
58
|
+
NOTE: This array has NOT been corrected for IFU drift. Use the intermediate frame
|
|
59
|
+
`DlnirspTag.task_drifted_ifu_x_pos()` for that.
|
|
60
|
+
"""
|
|
61
|
+
param_dict = self._find_parameter_for_arm("dlnirsp_ifu_x_pos_file")
|
|
62
|
+
return self._load_param_value_from_fits(param_dict)
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def raw_ifu_y_pos_array(self) -> np.ndarray:
|
|
66
|
+
"""
|
|
67
|
+
Return the array mapping raw pixel position to an X coordinate in the IFU.
|
|
68
|
+
|
|
69
|
+
NOTE: This array has NOT been corrected for IFU drift. Use the intermediate frame
|
|
70
|
+
`DlnirspTag.task_drifted_ifu_y_pos()` for that.
|
|
71
|
+
"""
|
|
72
|
+
param_dict = self._find_parameter_for_arm("dlnirsp_ifu_y_pos_file")
|
|
73
|
+
return self._load_param_value_from_fits(param_dict)
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def group_id_max_drift_px(self) -> int:
|
|
77
|
+
"""
|
|
78
|
+
Define the maximum pixel shift allowed when computing group ID drift.
|
|
79
|
+
|
|
80
|
+
If the computed drift is larger than this then the raw group ID array will be used.
|
|
81
|
+
"""
|
|
82
|
+
return self._find_most_recent_past_value("dlnirsp_group_id_max_drift_px")
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def group_id_rough_slit_separation_px(self) -> float:
|
|
86
|
+
"""
|
|
87
|
+
Rough pixel distance between slits.
|
|
88
|
+
|
|
89
|
+
This is NOT the pixel distance between both beams of the same slit.
|
|
90
|
+
"""
|
|
91
|
+
return self._find_most_recent_past_value("dlnirsp_group_id_rough_slit_separation_px")
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def corrections_max_nan_frac(self) -> float:
|
|
95
|
+
"""
|
|
96
|
+
Maximum allowable fraction of NaN in a shifted pixel before that pixel gets converted to NaN.
|
|
97
|
+
|
|
98
|
+
Input NaN values are tracked and any shifted pixel that has a value made up of more than this fraction of NaN
|
|
99
|
+
pixels will be set to NaN.
|
|
100
|
+
"""
|
|
101
|
+
return self._find_most_recent_past_value("dlnirsp_corrections_max_nan_frac")
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def pac_remove_linear_I_trend(self) -> bool:
|
|
105
|
+
"""Flag that determines if a linear intensity trend is removed from the whole PolCal CS.
|
|
106
|
+
|
|
107
|
+
The trend is fit using the average flux in the starting and ending clear steps.
|
|
108
|
+
"""
|
|
109
|
+
return self._find_most_recent_past_value("dlnirsp_pac_remove_linear_I_trend")
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def pac_fit_mode(self) -> str:
|
|
113
|
+
"""Name of set of fitting flags to use during PAC Calibration Unit parameter fits."""
|
|
114
|
+
return self._find_most_recent_past_value("dlnirsp_pac_fit_mode")
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def pac_init_set(self):
|
|
118
|
+
"""Name of set of initial values for Calibration Unit parameter fit."""
|
|
119
|
+
return self._find_most_recent_past_value("dlnirsp_pac_init_set")
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def lamp_despike_kernel(self) -> (float, float):
|
|
123
|
+
"""Return the (x, y) stddev of the Gaussian kernel used for lamp despiking."""
|
|
124
|
+
return self._find_most_recent_past_value("dlnirsp_lamp_despike_kernel")
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def lamp_despike_threshold(self) -> float:
|
|
128
|
+
"""Return the threhold value used to identify spikes in lamp gains."""
|
|
129
|
+
return self._find_most_recent_past_value("dlnirsp_lamp_despike_threshold")
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def geo_spectral_edge_trim(self) -> int:
|
|
133
|
+
"""Return the +/- number of pixels to remove from the ends of all spectra prior to fitting."""
|
|
134
|
+
return self._find_most_recent_past_value("dlnirsp_geo_spectral_edge_trim")
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def geo_continuum_smoothing_sigma_px(self) -> float:
|
|
138
|
+
"""
|
|
139
|
+
Return the Gaussian sigma used to smooth out spectral lines when estimating the continuum background.
|
|
140
|
+
|
|
141
|
+
This should be roughly the width, in px, of typical spectral lines. Err on the side of too large.
|
|
142
|
+
"""
|
|
143
|
+
return self._find_most_recent_past_value("dlnirsp_geo_continuum_smoothing_sigma_px")
|
|
144
|
+
|
|
145
|
+
@property
|
|
146
|
+
def geo_max_shift_px(self) -> float:
|
|
147
|
+
"""
|
|
148
|
+
Return the maximum shift to consider when computing spectral curvature.
|
|
149
|
+
|
|
150
|
+
This is an absolute value: negative and positive shifts are constrained to the same magnitude.
|
|
151
|
+
"""
|
|
152
|
+
return self._find_most_recent_past_value("dlnirsp_geo_max_shift_px")
|
|
153
|
+
|
|
154
|
+
@property
|
|
155
|
+
def geo_shift_poly_fit_order(self) -> int:
|
|
156
|
+
"""Return the order of the polynomial used to fit spectral shifts as a function of slit position."""
|
|
157
|
+
return self._find_most_recent_past_value("dlnirsp_geo_shift_poly_fit_order")
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def geo_bad_px_sigma_threshold(self) -> float:
|
|
161
|
+
"""Any pixels larger than this many stddevs from a difference between a filtered and raw spectrum will be removed."""
|
|
162
|
+
return self._find_most_recent_past_value("dlnirsp_geo_bad_px_sigma_threshold")
|
|
163
|
+
|
|
164
|
+
@property
|
|
165
|
+
def geo_slitbeam_fit_sig_clip(self) -> int:
|
|
166
|
+
"""Plus/minus number of standard deviations away from the median used to reject outlier values when fitting along the slitbeams."""
|
|
167
|
+
return self._find_most_recent_past_value("dlnirsp_geo_slitbeam_fit_sig_clip")
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def geo_reference_wave_min_nonnan_frac(self) -> float:
|
|
171
|
+
"""
|
|
172
|
+
Minimum fraction of non-NaN values allowed in reference wavelength regions.
|
|
173
|
+
|
|
174
|
+
Wavelength regions with less than this fraction of non-NaN pixels (across all slitbeams) will be excluded from
|
|
175
|
+
the reference wavelength vector.
|
|
176
|
+
"""
|
|
177
|
+
return self._find_most_recent_past_value("dlnirsp_geo_reference_wave_min_nonnan_frac")
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""DLNIRSP-specific file tags."""
|
|
2
|
+
from enum import Enum
|
|
3
|
+
|
|
4
|
+
from dkist_processing_common.models.tags import Tag
|
|
5
|
+
|
|
6
|
+
from dkist_processing_dlnirsp.models.task_name import DlnirspTaskName
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class DlnirspStemName(str, Enum):
|
|
10
|
+
"""Controlled list of Tag Stems."""
|
|
11
|
+
|
|
12
|
+
linearized = "LINEARIZED"
|
|
13
|
+
beam = "BEAM"
|
|
14
|
+
arm_id = "ARM_ID"
|
|
15
|
+
scan_step = "SCAN_STEP"
|
|
16
|
+
current_frame_in_ramp = "CURRENT_FRAME_IN_RAMP"
|
|
17
|
+
time_obs = "TIME_OBS"
|
|
18
|
+
modstate = "MODSTATE"
|
|
19
|
+
dither_step = "DITHER_STEP"
|
|
20
|
+
mosaic_num = "MOSAIC_NUM"
|
|
21
|
+
tile_X_num = "TILE_X_NUM"
|
|
22
|
+
tile_Y_num = "TILE_Y_NUM"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DlnirspTag(Tag):
|
|
26
|
+
"""DLNIRSP specific tag formatting."""
|
|
27
|
+
|
|
28
|
+
@classmethod
|
|
29
|
+
def beam(cls, beam_num: int) -> str:
|
|
30
|
+
"""
|
|
31
|
+
Tags by beam number.
|
|
32
|
+
|
|
33
|
+
Parameters
|
|
34
|
+
----------
|
|
35
|
+
beam_num
|
|
36
|
+
The beam number
|
|
37
|
+
|
|
38
|
+
Returns
|
|
39
|
+
-------
|
|
40
|
+
The formatted tag string
|
|
41
|
+
"""
|
|
42
|
+
return cls.format_tag(DlnirspStemName.beam, beam_num)
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def modstate(cls, modstate: int) -> str:
|
|
46
|
+
"""
|
|
47
|
+
Tags by the current modstate number.
|
|
48
|
+
|
|
49
|
+
Parameters
|
|
50
|
+
----------
|
|
51
|
+
modstate
|
|
52
|
+
The current scan step number
|
|
53
|
+
|
|
54
|
+
Returns
|
|
55
|
+
-------
|
|
56
|
+
The formatted tag string
|
|
57
|
+
"""
|
|
58
|
+
return cls.format_tag(DlnirspStemName.modstate, modstate)
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def linearized(cls) -> str:
|
|
62
|
+
"""
|
|
63
|
+
Tags for linearized frames.
|
|
64
|
+
|
|
65
|
+
Returns
|
|
66
|
+
-------
|
|
67
|
+
The formatted tag string
|
|
68
|
+
"""
|
|
69
|
+
return cls.format_tag(DlnirspStemName.linearized)
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def arm_id(cls, arm_id: str) -> str:
|
|
73
|
+
"""
|
|
74
|
+
Tags based on the CryoNIRSP arm_id from which the data is recorded (SP or CI).
|
|
75
|
+
|
|
76
|
+
Parameters
|
|
77
|
+
----------
|
|
78
|
+
arm_id
|
|
79
|
+
The arm ID in use, SP or CI
|
|
80
|
+
|
|
81
|
+
Returns
|
|
82
|
+
-------
|
|
83
|
+
The formatted tag string
|
|
84
|
+
"""
|
|
85
|
+
return cls.format_tag(DlnirspStemName.arm_id, arm_id)
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def current_frame_in_ramp(cls, curr_frame_in_ramp: int) -> str:
|
|
89
|
+
"""
|
|
90
|
+
Tags based on the current frame number in the ramp.
|
|
91
|
+
|
|
92
|
+
Parameters
|
|
93
|
+
----------
|
|
94
|
+
curr_frame_in_ramp
|
|
95
|
+
The current frame number for this ramp
|
|
96
|
+
|
|
97
|
+
Returns
|
|
98
|
+
-------
|
|
99
|
+
The formatted tag string
|
|
100
|
+
"""
|
|
101
|
+
return cls.format_tag(DlnirspStemName.current_frame_in_ramp, curr_frame_in_ramp)
|
|
102
|
+
|
|
103
|
+
@classmethod
|
|
104
|
+
def time_obs(cls, time_obs: str) -> str:
|
|
105
|
+
"""
|
|
106
|
+
Tags by the observe date.
|
|
107
|
+
|
|
108
|
+
Parameters
|
|
109
|
+
----------
|
|
110
|
+
time_obs
|
|
111
|
+
The observe time
|
|
112
|
+
|
|
113
|
+
Returns
|
|
114
|
+
-------
|
|
115
|
+
The formatted tag string
|
|
116
|
+
"""
|
|
117
|
+
return cls.format_tag(DlnirspStemName.time_obs, time_obs)
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def mosaic_num(cls, mosaic_num: int) -> str:
|
|
121
|
+
"""Tags by the mosaic number."""
|
|
122
|
+
return cls.format_tag(DlnirspStemName.mosaic_num, mosaic_num)
|
|
123
|
+
|
|
124
|
+
@classmethod
|
|
125
|
+
def tile_X_num(cls, tile_X_num: int) -> str:
|
|
126
|
+
"""Tags by the current mosaic location in the X direction."""
|
|
127
|
+
return cls.format_tag(DlnirspStemName.tile_X_num, tile_X_num)
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def tile_Y_num(cls, tile_Y_num: int) -> str:
|
|
131
|
+
"""Tags by the current mosaic location in the Y direction."""
|
|
132
|
+
return cls.format_tag(DlnirspStemName.tile_Y_num, tile_Y_num)
|
|
133
|
+
|
|
134
|
+
@classmethod
|
|
135
|
+
def dither_step(cls, dither_step: int) -> str:
|
|
136
|
+
"""Tags by dither step."""
|
|
137
|
+
return cls.format_tag(DlnirspStemName.dither_step, dither_step)
|
|
138
|
+
|
|
139
|
+
@classmethod
|
|
140
|
+
def task_drifted_ifu_groups(cls) -> str:
|
|
141
|
+
"""Identify the IFU group ID array with drift applied."""
|
|
142
|
+
return cls.task(DlnirspTaskName.drifted_ifu_group_id)
|
|
143
|
+
|
|
144
|
+
@classmethod
|
|
145
|
+
def task_drifted_dispersion(cls) -> str:
|
|
146
|
+
"""Identify the IFU dispersion array with drift applied."""
|
|
147
|
+
return cls.task(DlnirspTaskName.drifted_dispersion)
|
|
148
|
+
|
|
149
|
+
@classmethod
|
|
150
|
+
def task_drifted_ifu_x_pos(cls) -> str:
|
|
151
|
+
"""Identify the IFU X Pos array with drift applied."""
|
|
152
|
+
return cls.task(DlnirspTaskName.drifted_ifu_x_pos)
|
|
153
|
+
|
|
154
|
+
@classmethod
|
|
155
|
+
def task_drifted_ifu_y_pos(cls) -> str:
|
|
156
|
+
"""Identify the IFU Y Pos array with drift applied."""
|
|
157
|
+
return cls.task(DlnirspTaskName.drifted_ifu_y_pos)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""List of intermediate task names."""
|
|
2
|
+
from enum import StrEnum
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class DlnirspTaskName(StrEnum):
|
|
6
|
+
"""Controlled list of DLNIRSP task tag names."""
|
|
7
|
+
|
|
8
|
+
drifted_ifu_group_id = "DRIFTED_IFU_GROUP_ID"
|
|
9
|
+
drifted_dispersion = "DRIFTED_DISPERSION"
|
|
10
|
+
drifted_ifu_x_pos = "DRIFTED_IFU_XPOS"
|
|
11
|
+
drifted_ifu_y_pos = "DRIFTED_IFU_YPOS"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Tools to read dataset and ingest metadata required for science processing."""
|