dkist-processing-cryonirsp 0.0.17rc1__tar.gz → 0.0.18__tar.gz

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/CHANGELOG.rst

@@ -1,3 +1,23 @@
+v0.0.18 (2023-11-08)
+====================
+
+Bugfixes
+--------
+
+- Removes cross-talk correction from SP science task. (`#72 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/72>`__)
+- Turn on bad-pixel correction of science observe frames for science team evaluation. (`#73 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/73>`__)
+- Fix computation of characteristic spectrum by removing incorrect median normalization. (`#74 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/74>`__)
+
+
+v0.0.17 (2023-11-06)
+====================
+
+Features
+--------
+
+- Implement relative photometric calibration. Linearized ramp sets are normalized to counts per second. Attenuation due to Optical Density filters is compensated. Solar gain image is no longer normalized to mean of 1. Observe images are now in units of flux relative to solar center. (`#70 <https://bitbucket.org/dkistdc/dkist-processing-cryonirsp/pull-requests/70>`__)
+
+
 v0.0.16 (2023-11-02)
 ====================
 

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist_processing_cryonirsp
-Version: 0.0.17rc1
+Version: 0.0.18
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/models/optical_density_filter_names.py

@@ -5,7 +5,8 @@ from enum import StrEnum
 class AllowableOpticalDensityFilterNames(StrEnum):
     """Enum to implement list of allowable Optical Density Filter names."""
 
+    G278 = "G278"
     G358 = "G358"
     G408 = "G408"
-    OPEN = "Open"
-    NONE = "none"
+    OPEN = "OPEN"
+    NONE = "NONE"

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/models/parameters.py

@@ -156,6 +156,7 @@ class CryonirspParameters(ParameterBase):
     def linearization_filter_attenuation_dict(self) -> dict:
         """Return a dict that maps the filter name to its attenuation parameter."""
         filter_attenuation_dict = {
+            AllowableOpticalDensityFilterNames.G278.value: self._linearization_optical_density_filter_attenuation_g278,
             AllowableOpticalDensityFilterNames.G358.value: self._linearization_optical_density_filter_attenuation_g358,
             AllowableOpticalDensityFilterNames.G408.value: self._linearization_optical_density_filter_attenuation_g408,
             AllowableOpticalDensityFilterNames.OPEN.value: self._linearization_optical_density_filter_attenuation_open,
@@ -163,6 +164,13 @@ class CryonirspParameters(ParameterBase):
         }
         return filter_attenuation_dict
 
+    @cached_property
+    def _linearization_optical_density_filter_attenuation_g278(self) -> float:
+        """Return the attenuation value for the G278 filter."""
+        return self._find_parameter_closest_wavelength(
+            "cryonirsp_linearization_optical_density_filter_attenuation_g278"
+        )
+
     @cached_property
     def _linearization_optical_density_filter_attenuation_g358(self) -> float:
         """Return the attenuation value for the G358 filter."""

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/parsers/cryonirsp_l0_fits_access.py

@@ -32,7 +32,7 @@ class CryonirspRampFitsAccess(L0FitsAccess):
         self.camera_readout_mode = self.header["CNCAMMD"]
         self.curr_frame_in_ramp: int = self.header["CNCNDR"]
         self.arm_id: str = self.header["CNARMID"]
-        self.filter_name = self.header["CNFILTNP"]
+        self.filter_name = self.header["CNFILTNP"].upper()
         self.roi_1_origin_x = self.header["HWROI1OX"]
         self.roi_1_origin_y = self.header["HWROI1OY"]
         self.roi_1_size_x = self.header["HWROI1SX"]

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/parsers/optical_density_filters.py

@@ -31,7 +31,7 @@ class OpticalDensityFiltersPickyBud(Stem):
         -------
         The optical density filter name associated with this fits object
         """
-        return fits_obj.filter_name
+        return fits_obj.filter_name.upper()
 
     def getter(self, key: Hashable) -> Thorn:
         """Return a Thorn for valid names or raise an exception for bad names."""

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tasks/science_base.py

@@ -168,17 +168,14 @@ class ScienceCalibrationBase(CryonirspTaskBase, ABC):
         # Dark correction
         dark_corrected_array = next(subtract_array_from_arrays(avg_observe_array, dark_array))
 
-        # TODO: Unclear how much this helps, so leaving it out for now...
-        # TODO: Do not remove this code until we have confirmed with the cryo team
         # Bad pixel correction
-        # bad_pixel_map = self.intermediate_frame_load_bad_pixel_map(beam=beam)
-        # bad_pixel_corrected_array = self.corrections_correct_bad_pixels(
-        #     dark_corrected_array, bad_pixel_map
-        # )
+        bad_pixel_map = self.intermediate_frame_load_bad_pixel_map(beam=beam)
+        bad_pixel_corrected_array = self.corrections_correct_bad_pixels(
+            dark_corrected_array, bad_pixel_map
+        )
 
         # Gain correction
-        # gain_corrected_array = next(divide_arrays_by_array(bad_pixel_corrected_array, gain_array))
-        gain_corrected_array = next(divide_arrays_by_array(dark_corrected_array, gain_array))
+        gain_corrected_array = next(divide_arrays_by_array(bad_pixel_corrected_array, gain_array))
 
         return gain_corrected_array, observe_header
 

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tasks/sp_science.py

@@ -99,7 +99,6 @@ class SPScienceCalibration(ScienceCalibrationBase):
 
         logger.info(f"Correcting telescope polarization for {logging_str}")
         calibrated = self.telescope_polarization_correction(combined)
-        calibrated = self.remove_crosstalk(calibrated)
 
         return calibrated
 
@@ -208,21 +207,6 @@ class SPScienceCalibration(ScienceCalibrationBase):
 
         return obj
 
-    @staticmethod
-    def remove_crosstalk(fits_obj: CryonirspL0FitsAccess) -> CryonirspL0FitsAccess:
-        """Remove the continuum from QUV (remove cross-talk)."""
-        array_stack = fits_obj.data
-        array_header = fits_obj.header
-        for i in range(1, 4):
-            array_stack[:, :, i] = (
-                array_stack[:, :, i]
-                - np.nanmedian(array_stack[:, :, i] / array_stack[:, :, 0], axis=1)[:, None]
-                * array_stack[:, :, 0]
-            )
-        hdu = fits.ImageHDU(data=array_stack, header=array_header)
-        obj = CryonirspL0FitsAccess(hdu=hdu, auto_squeeze=False)
-        return obj
-
     def collect_calibration_objects(self) -> CalibrationCollection:
         """
         Collect *all* calibration for all modstates, and exposure times.

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tasks/sp_solar_gain.py

@@ -154,15 +154,14 @@ class SPSolarGainCalibration(CryonirspTaskBase):
         return spectral_corrected_solar_array
 
     def _compute_char_spectrum(self, array: np.ndarray) -> np.ndarray:
-        """Estimate the characteristic colar spectrum from the corrected solar gain data."""
+        """Estimate the characteristic solar spectrum from the corrected solar gain data."""
         # Normalize data row by row
         pct = self.parameters.solar_characteristic_spatial_normalization_percentile
         array_row_norm = array / np.nanpercentile(array, pct, axis=1)[:, None]
         # Compute characteristic spectrum
         char_spec_1d = np.nanmedian(array_row_norm, axis=0)
-        median_char_spec_1d = char_spec_1d / np.nanmean(char_spec_1d)
         # Expand the 1D median along the columns (along the slit)
-        median_char_spec_2d = np.tile(median_char_spec_1d, (array_row_norm.shape[0], 1))
+        median_char_spec_2d = np.tile(char_spec_1d, (array_row_norm.shape[0], 1))
         return median_char_spec_2d
 
     def _distort_char_spectrum(self, char_spec: np.ndarray) -> np.ndarray:

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tests/conftest.py

@@ -329,6 +329,9 @@ def cryonirsp_testing_parameters_factory(
             )
         )
         cryonirsp_linearization_max_memory_gb: float = 4.0
+        cryonirsp_linearization_optical_density_filter_attenuation_g278: WavelengthParameter = (
+            WavelengthParameter(values=(-1.64, -1.64, -1.64, -1.64))
+        )
         cryonirsp_linearization_optical_density_filter_attenuation_g358: WavelengthParameter = (
             WavelengthParameter(values=(-3.75, -3.75, -3.75, -3.75))
         )
@@ -387,7 +390,7 @@ def assign_input_dataset_doc_to_task(
 def _write_frames_to_task(
     task: Type[WorkflowTaskBase],
     frame_generator: Spec122Dataset,
-    change_translated_headers: Callable[[fits.Header | None], fits.Header] = None,
+    change_translated_headers: Callable[[fits.Header], fits.Header] = lambda x: x,
     tag_ramp_frames: Callable[[fits.Header], list[str]] = lambda x: [],
     extra_tags: list[str] | None = None,
     tag_func: Callable[[CryonirspHeaders], list[str]] = lambda x: [],
@@ -404,11 +407,9 @@ def _write_frames_to_task(
     data = frame.data
     frame_tags = tags + tag_func(frame)
     translated_header = fits.Header(translate_spec122_to_spec214_l0(header))
-    if change_translated_headers is not None:
-        translated_header = change_translated_headers(translated_header)
-    if tag_ramp_frames is not None:
-        ramp_tags = tag_ramp_frames(translated_header)
-        frame_tags = frame_tags + ramp_tags
+    translated_header = change_translated_headers(translated_header)
+    ramp_tags = tag_ramp_frames(translated_header)
+    frame_tags = frame_tags + ramp_tags
 
     task.write(data=data, header=translated_header, tags=frame_tags, encoder=fits_array_encoder)
     num_frames += 1

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tests/header_models.py

@@ -108,7 +108,7 @@ class CryonirspHeadersValidNonLinearizedFrames(CryonirspHeaders):
     @key_function("XPOSURE")
     # set the exposure time for each frame in the ramp
     def exposure_time(self, key: float):
-        return 100 * self._index * 0.01
+        return 100 * self.index * 0.01
 
 
 class CryonirspHeadersValidLinearizedFrames(CryonirspHeaders):

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tests/test_linearity_correction.py

@@ -117,7 +117,7 @@ def linearity_correction(
 
 @pytest.mark.parametrize(
     "filter_name",
-    [pytest.param("G358", id="G358"), pytest.param("Open", id="Open")],
+    [pytest.param("G358", id="G358"), pytest.param("OPEN", id="OPEN")],
 )
 @pytest.mark.parametrize(
     "frames_in_ramp",

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tests/test_parameters.py

@@ -52,6 +52,27 @@ def _is_wavelength_param(param_value: Any) -> bool:
     return isinstance(param_value, dict) and "wavelength" in param_value
 
 
+@given(wave=st.floats(min_value=800.0, max_value=2000.0))
+@settings(suppress_health_check=[HealthCheck.function_scoped_fixture])
+@example(wave=1082.7)
+def test_filter_parameters(basic_science_task_with_parameter_mixin, wave):
+    """
+    Given: A Science task with the parameter mixin
+    When: Accessing properties for the optical density filters
+    Then: The correct value is returned
+    """
+    task, expected = basic_science_task_with_parameter_mixin
+    task_params = task.parameters
+    task_params._wavelength = wave
+    expected = {
+        "_linearization_optical_density_filter_attenuation_g278": -1.64,
+        "_linearization_optical_density_filter_attenuation_g358": -3.75,
+        "_linearization_optical_density_filter_attenuation_g408": -4.26,
+    }
+    for param in expected:
+        assert getattr(task_params, param) == expected[param]
+
+
 def _is_file_param(param_value: Any) -> bool:
     return isinstance(param_value, dict) and "is_file" in param_value and param_value["is_file"]
 

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tests/test_parse.py

@@ -107,7 +107,7 @@ def write_observe_frames_to_task(
     num_map_scans: int,
     num_sub_repeats: int,
     arm_id: str,
-    change_translated_headers: Callable[[fits.Header | None], fits.Header] = None,
+    change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
     exp_time_ms: float = 6.0,
     array_shape=(2, 2, 1),
     tags: list[str] | None = None,
@@ -154,7 +154,7 @@ def write_non_linearized_frames(
     arm_id: str,
     start_time: str,
     camera_readout_mode: str,
-    change_translated_headers: Callable[[fits.Header | None], fits.Header] = None,
+    change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
     tags: list[str] | None = None,
 ):
     frame_generator = CryonirspHeadersValidNonLinearizedFrames(
@@ -194,7 +194,7 @@ def make_linearized_test_frames(
     dark_exp_times: list[float],
     num_modstates: int,
     num_steps: int,
-    change_translated_headers: Callable[[fits.Header | None], fits.Header] = None,
+    change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
     num_map_scans: int = 1,
     num_sub_repeats: int = 1,
     extra_headers: dict | None = None,
@@ -232,7 +232,7 @@ def make_linearized_test_frames(
 
 def make_non_linearized_test_frames(
     task,
-    change_translated_headers: Callable[[fits.Header | None], fits.Header] = None,
+    change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
 ):
     arm_id = "SP"
     camera_readout_mode = "FastUpTheRamp"
@@ -335,11 +335,13 @@ def test_parse_cryonirsp_non_linearized_data_bad_filter_name(parse_non_linearize
 
     task = parse_non_linearized_task
 
-    def fix_ramp_frame_headers(translated_header: fits.Header):
+    def insert_bad_filter_name_into_header(translated_header: fits.Header):
         translated_header["CNFILTNP"] = "BAD_FILTER_NAME"
         return translated_header
 
-    make_non_linearized_test_frames(task, change_translated_headers=fix_ramp_frame_headers)
+    make_non_linearized_test_frames(
+        task, change_translated_headers=insert_bad_filter_name_into_header
+    )
 
     with pytest.raises(
         ValueError,
@@ -560,7 +562,6 @@ def test_parse_cryonirsp_linearized_incomplete_final_map(parse_linearized_task,
                         task=task,
                         frame_generator=frame_generator,
                         extra_tags=[CryonirspTag.linearized()],
-                        change_translated_headers=None,
                     )
 
     task()
@@ -720,6 +721,15 @@ def test_parse_cryonirsp_linearized_data_constants(parse_linearized_task, arm_id
 
 def test_optical_density_filter_names(parse_non_linearized_task):
     task = parse_non_linearized_task
-    filter_map = {k for k in task.parameters.linearization_filter_attenuation_dict.keys()}
-    difference = filter_map.symmetric_difference(ALLOWABLE_OPTICAL_DENSITY_FILTERS)
-    assert not difference
+    # List of filter attenuation parameters defined in CryonirspParameters:
+    defined_filter_params = {
+        item[-4:].upper()
+        for item in dir(task.parameters)
+        if item.startswith("_linearization_optical_density_filter_attenuation_")
+    }
+    # List of filters in the filter map:
+    filter_map_params = {k for k in task.parameters.linearization_filter_attenuation_dict.keys()}
+    # Make sure all filter parameters match the allowable list
+    assert not defined_filter_params.symmetric_difference(ALLOWABLE_OPTICAL_DENSITY_FILTERS)
+    # Make sure all filter map keys match the allowable list
+    assert not filter_map_params.symmetric_difference(ALLOWABLE_OPTICAL_DENSITY_FILTERS)

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp/tests/test_sp_science.py

@@ -67,7 +67,7 @@ def sp_science_calibration_task(
             assign_input_dataset_doc_to_task(task, param_class())
             # Create fake bad pixel map
             task.intermediate_frame_write_arrays(
-                arrays=np.zeros((1, 1)),
+                arrays=np.zeros((30, 60)),
                 task_tag=CryonirspTag.task_bad_pixel_map(),
             )
             # Create fake demodulation matrices

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-cryonirsp
-Version: 0.0.17rc1
+Version: 0.0.18
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Home-page: https://bitbucket.org/dkistdc/dkist_processing_cryonirsp/src/main/
 Author: NSO / AURA

{dkist_processing_cryonirsp-0.0.17rc1 → dkist_processing_cryonirsp-0.0.18}/dkist_processing_cryonirsp.egg-info/SOURCES.txt

@@ -12,7 +12,6 @@ setup.cfg
 setup.py
 towncrier_science.toml
 changelog/.gitempty
-changelog/70.feature.rst
 dkist_processing_cryonirsp/__init__.py
 dkist_processing_cryonirsp.egg-info/PKG-INFO
 dkist_processing_cryonirsp.egg-info/SOURCES.txt

dkist_processing_cryonirsp-0.0.17rc1/changelog/70.feature.rst

@@ -1 +0,0 @@
-Implement relative photometric calibration. Linearized ramp sets are normalized to counts per second. Attenuation due to Optical Density filters is compensated. Solar gain image is no longer normalized to mean of 1. Observe images are now in units of flux relative to solar center.