dkist-processing-common 10.2.2__tar.gz → 10.3.0__tar.gz

{dkist_processing_common-10.2.2 → dkist_processing_common-10.3.0}/CHANGELOG.rst

@@ -1,3 +1,21 @@
+v10.3.0 (2024-10-15)
+====================
+
+Features
+--------
+
+- Update the machinery in `quality_store_polcal_results` to handle NaN values.
+  This is required for the new error-handling paradigm in `dkist-processing-pac` v3.1.0. (`#214 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/214>`__)
+- Add new argument, `num_points_to_sample`, to `quality_store_polcal_results`, which allows a user to reduce the number of points saved for inclusion in the quality report.
+  This allows us to mitigate large quality metrics. (`#215 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/215>`__)
+
+
+Bugfixes
+--------
+
+- Correctly clean up tags used for file name uniqueness. (`#217 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/217>`__)
+
+
 v10.2.2 (2024-10-14)
 ====================
 

{dkist_processing_common-10.2.2 → dkist_processing_common-10.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-common
-Version: 10.2.2
+Version: 10.3.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,7 +17,7 @@ Requires-Dist: astropy>=5.1.1
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
 Requires-Dist: dkist-header-validator<6.0,>=5.0.0
 Requires-Dist: dkist-processing-core==4.2.1
-Requires-Dist: dkist-processing-pac<4.0,>=3.0
+Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
 Requires-Dist: globus-sdk>=3.12.0

{dkist_processing_common-10.2.2 → dkist_processing_common-10.3.0}/dkist_processing_common/tasks/mixin/quality/_metrics.py

@@ -651,8 +651,11 @@ class _PolcalQualityMixin:
         bin_nums: list[int],
         bin_labels: list[str],
         skip_recording_constant_pars: bool = False,
+        num_points_to_sample: int | None = None,
     ):
         """Compute and store all PolCal related metrics."""
+        thinning_stride = self._compute_thinning_stride(polcal_fitter, num_points_to_sample)
+
         if not skip_recording_constant_pars:
             logger.info("Storing constant parameter values")
             self._store_polcal_constant_parameter_values(polcal_fitter=polcal_fitter, label=label)
@@ -662,7 +665,11 @@ class _PolcalQualityMixin:
 
         logger.info("Storing local parameter values")
         self._store_polcal_local_parameter_values(
-            polcal_fitter=polcal_fitter, label=label, bin_nums=bin_nums, bin_labels=bin_labels
+            polcal_fitter=polcal_fitter,
+            label=label,
+            bin_nums=bin_nums,
+            bin_labels=bin_labels,
+            thinning_stride=thinning_stride,
         )
 
         logger.info("Storing fit residuals")
@@ -671,6 +678,7 @@ class _PolcalQualityMixin:
             label=label,
             bin_nums=bin_nums,
             bin_labels=bin_labels,
+            thinning_stride=thinning_stride,
         )
 
         logger.info("Storing modulation matrix efficiencies")
@@ -679,6 +687,7 @@ class _PolcalQualityMixin:
             label=label,
             bin_nums=bin_nums,
             bin_labels=bin_labels,
+            thinning_stride=thinning_stride,
         )
 
     @staticmethod
@@ -688,6 +697,32 @@ class _PolcalQualityMixin:
             return None
         return str(label).replace(" ", "_").upper()
 
+    @staticmethod
+    def _compute_thinning_stride(
+        polcal_fitter: PolcalFitter, num_points_to_sample: int
+    ) -> int | None:
+        """
+        Compute the stride needed to collect the requested number of data samples.
+
+        E.g., if there are 20,000 samples in the full `polcal_fitter` and the user wants to only save 200 then the
+        stride will be 100.
+
+        `None` is returned if no subsampling has been requested or if the number of requested samples is larger than
+        then number of samples in the full `polcal_fitter`.
+        """
+        if num_points_to_sample is None:
+            return None
+
+        num_total_points = np.prod(polcal_fitter.local_objects.dresser.shape)
+        if num_points_to_sample > num_total_points:
+            return None
+
+        remainder = num_total_points % num_points_to_sample
+        if remainder:
+            return num_total_points // (num_points_to_sample - 1)
+
+        return num_total_points // num_points_to_sample
+
     def _store_polcal_constant_parameter_values(
         self, *, polcal_fitter: PolcalFitter, label: str
     ) -> None:
@@ -896,6 +931,7 @@ class _PolcalQualityMixin:
         label: str,
         bin_nums: list[int],
         bin_labels: list[str],
+        thinning_stride: int | None,
     ) -> None:
         """Store local polcal parameter fits.
 
@@ -909,10 +945,16 @@ class _PolcalQualityMixin:
         flattened_demod = np.reshape(
             polcal_fitter.demodulation_matrices, (np.prod(fov_shape), 4, num_mod)
         )
-        flattened_mod = np.zeros((np.prod(fov_shape), num_mod, 4))
-        for i in range(flattened_demod.shape[0]):
-            flattened_mod[i] = np.linalg.pinv(flattened_demod[i])
+        flattened_mod_list = []
+        # Need to apply stride this was (instead of argument to `range` because `range` doesn't accept `None` for
+        #  step argument.
+        for i in range(flattened_demod.shape[0])[::thinning_stride]:
+            try:
+                flattened_mod_list.append(np.linalg.pinv(flattened_demod[i]))
+            except:
+                pass
 
+        flattened_mod = np.stack(flattened_mod_list)
         # Move axis so numpoints is the last dimension, which will be easier to understand when
         # plotting
         flattened_mod = np.moveaxis(flattened_mod, 0, -1)
@@ -937,8 +979,10 @@ class _PolcalQualityMixin:
                 num_varied_I_sys += 1
 
             fit_value_list = []
-            for point_param in fit_params._all_parameters:
-                fit_value_list.append(point_param[param].value)
+            for point_param in fit_params._all_parameters[::thinning_stride]:
+                value = point_param[param].value
+                if not np.isnan(value):
+                    fit_value_list.append(value)
 
             init_value = init_param.first_parameters[param].value
 
@@ -950,6 +994,7 @@ class _PolcalQualityMixin:
                 f"{num_bins} {bin_label}" for num_bins, bin_label in zip(bin_nums, bin_labels)
             ],
             "total_bins": int(np.prod(bin_nums)),
+            "sampled_bins": flattened_mod.shape[-1],
             "num_varied_I_sys": num_varied_I_sys,
             "modmat_list": mod_list,
             "free_param_dict": free_param_data,
@@ -1005,7 +1050,9 @@ class _PolcalQualityMixin:
         "Subsequent plots show the distribution of all other free parameters in the fit, along with their initial "
         "values. For I_sys there is a separate fit value for each CS step."
         description += self._compute_bin_description(
-            total_bins=data["total_bins"], bin_strs=data["bin_strs"]
+            total_bins=data["total_bins"],
+            bin_strs=data["bin_strs"],
+            sampled_bins=data["sampled_bins"],
         )
 
         metric = ReportMetric(
@@ -1019,7 +1066,13 @@ class _PolcalQualityMixin:
         return metric.model_dump()
 
     def _store_polcal_fit_resdiuals(
-        self, *, polcal_fitter: PolcalFitter, label: str, bin_nums: list[int], bin_labels: list[str]
+        self,
+        *,
+        polcal_fitter: PolcalFitter,
+        label: str,
+        bin_nums: list[int],
+        bin_labels: list[str],
+        thinning_stride: int | None,
     ):
         """Store flux residuals and chisq values for a local fit."""
         fit_container = polcal_fitter.local_objects
@@ -1029,7 +1082,7 @@ class _PolcalQualityMixin:
         num_points = np.prod(fov_shape)
         residual_array = np.zeros((num_mod, num_steps, num_points))
         red_chi_list = []
-        for i in range(num_points):
+        for i in range(num_points)[::thinning_stride]:
             ## Fit residuals
             point_TM = copy.deepcopy(fit_container.telescope)
             point_CM = copy.deepcopy(fit_container.calibration_unit)
@@ -1054,7 +1107,8 @@ class _PolcalQualityMixin:
             chisq = np.sum(flat_residual**2)
             num_free = sum([fit_params[p].vary for p in fit_params])
             red_chisq = chisq / num_free
-            red_chi_list.append(red_chisq)
+            if not np.isnan(red_chisq):
+                red_chi_list.append(red_chisq)
 
         # Convert residuals to panda DataFrame, which will greatly simplify plotting
         col_list = sum(
@@ -1076,6 +1130,7 @@ class _PolcalQualityMixin:
                 f"{num_bins} {bin_label}" for num_bins, bin_label in zip(bin_nums, bin_labels)
             ],
             "total_bins": int(np.prod(bin_nums)),
+            "sampled_bins": len(red_chi_list),
             "residual_json": dataframe_str,
             "red_chi_list": red_chi_list,
         }
@@ -1113,10 +1168,11 @@ class _PolcalQualityMixin:
             dataframe_json=data["residual_json"],
         )
 
-        description = "The top plot shows relative flux residual distributions for all polcal Calibration Sequence "
-        "steps. The bottom plot shows the reduced chi-squared distribution of all fits."
+        description = "The top plot shows relative flux residual distributions for all polcal Calibration Sequence steps. The bottom plot shows the reduced chi-squared distribution of all fits."
         description += self._compute_bin_description(
-            total_bins=data["total_bins"], bin_strs=data["bin_strs"]
+            total_bins=data["total_bins"],
+            bin_strs=data["bin_strs"],
+            sampled_bins=data["sampled_bins"],
         )
 
         metric = ReportMetric(
@@ -1136,15 +1192,19 @@ class _PolcalQualityMixin:
         label: str,
         bin_nums: list[int],
         bin_labels: list[str],
+        thinning_stride: int | None,
     ):
         """Compute modulation efficiency for all fit bins and store in a file."""
         fov_shape = polcal_fitter.local_objects.dresser.shape
         num_mod = polcal_fitter.local_objects.dresser.nummod
         num_points = np.prod(fov_shape)
         flat_demod = np.reshape(polcal_fitter.demodulation_matrices, (num_points, 4, num_mod))
-        flat_efficiency = 1.0 / np.sqrt(
-            num_mod * np.sum(flat_demod**2, axis=2)
-        )  # (num_points, 4)
+        thinned_demod = flat_demod[::thinning_stride, :, :]
+        # This will have shape (num_points, 4)
+        flat_efficiency = 1.0 / np.sqrt(num_mod * np.sum(thinned_demod**2, axis=2))
+
+        nan_idx = np.sum(np.isnan(flat_efficiency), axis=1).astype(bool)
+        flat_efficiency = flat_efficiency[~nan_idx, :]
 
         # Because ndarrays are not JSON-able
         # Also, transpose it so the Stokes parameters are the first dimension
@@ -1165,6 +1225,7 @@ class _PolcalQualityMixin:
                 f"{num_bins} {bin_label}" for num_bins, bin_label in zip(bin_nums, bin_labels)
             ],
             "total_bins": int(np.prod(bin_nums)),
+            "sampled_bins": flat_efficiency.shape[0],
             "efficiency_list": efficiency_list,
             "warnings": warnings,
         }
@@ -1183,7 +1244,9 @@ class _PolcalQualityMixin:
 
         description = "The modulation efficiencies for all fit modulation matrices."
         description += self._compute_bin_description(
-            total_bins=data["total_bins"], bin_strs=data["bin_strs"]
+            total_bins=data["total_bins"],
+            bin_strs=data["bin_strs"],
+            sampled_bins=data["sampled_bins"],
         )
 
         metric = ReportMetric(
@@ -1197,7 +1260,9 @@ class _PolcalQualityMixin:
         return metric.model_dump()
 
     @staticmethod
-    def _compute_bin_description(total_bins: int, bin_strs: list[str]) -> str:
+    def _compute_bin_description(
+        total_bins: int, bin_strs: list[str], sampled_bins: int | None = None
+    ) -> str:
         """
         Construct a grammatically correct string that describes the layout of bins present in polcal data.
 
@@ -1210,7 +1275,11 @@ class _PolcalQualityMixin:
         3. For greater than two bin-types we get a list with commas and a final "and":
            "...spanning N TYPE1, M TYPE2, ..., and K TYPEK bins."
         """
-        base_str = f" Data show {total_bins} total samples spanning "
+        base_str = f" Data show"
+        if sampled_bins is not None:
+            base_str += f" {sampled_bins} uniformly sampled points from"
+
+        base_str += f" {total_bins} total points spanning "
 
         if len(bin_strs) == 1:
             base_str += bin_strs[0]

{dkist_processing_common-10.2.2 → dkist_processing_common-10.3.0}/dkist_processing_common/tasks/teardown.py

@@ -46,6 +46,9 @@ class TeardownBase(WorkflowTaskBase, ABC):
         with self.apm_task_step("Remove Data and Tags"):
             self.scratch.purge()
 
+        with self.apm_task_step("Remove File Counters"):
+            self.filename_counter.purge()
+
         with self.apm_task_step("Remove Constants"):
             self.constants._purge()
 

{dkist_processing_common-10.2.2 → dkist_processing_common-10.3.0}/dkist_processing_common/tests/conftest.py

@@ -3,9 +3,11 @@ Global test fixtures
 """
 import json
 from collections import defaultdict
+from copy import deepcopy
 from datetime import datetime
 from datetime import timedelta
 from pathlib import Path
+from random import choice
 from random import randint
 from random import random
 from typing import Any
@@ -510,7 +512,12 @@ class InstAccess(L0FitsAccess):
 
 
 @pytest.fixture(scope="session")
-def cs_with_correct_geometry():
+def cs_data_shape():
+    return (10, 4, 3)
+
+
+@pytest.fixture(scope="session")
+def cs_with_correct_geometry(cs_data_shape):
     dark_status = [
         "DarkShutter",
         "FieldStop (5arcmin)",
@@ -561,7 +568,6 @@ def cs_with_correct_geometry():
         "clear",
         "clear",
     ]
-    data_shape = (1, 1, 1)
     num_steps = len(pol_theta)
     out_dict = dict()
     start_time = datetime.fromisoformat("2022-05-25T12:00:00")
@@ -587,14 +593,14 @@ def cs_with_correct_geometry():
         for m in range(ds.num_mod):
             hdu_list.append(
                 fits.PrimaryHDU(
-                    data=np.ones(data_shape) * 1e3, header=fits.Header(header_list.pop(0))
+                    data=np.ones(cs_data_shape) * 1e3, header=fits.Header(header_list.pop(0))
                 )
             )
 
         out_dict[n] = [InstAccess(h) for h in hdu_list]
         start_time = ds.start_time + timedelta(seconds=60)
 
-    return out_dict, data_shape
+    return out_dict
 
 
 @pytest.fixture(
@@ -630,11 +636,11 @@ def visp_modulation_matrix() -> np.ndarray:
 
 
 @pytest.fixture(scope="session")
-def fully_realistic_cs(
+def fully_realistic_local_cs(
     cs_with_correct_geometry, visp_modulation_matrix, pac_fit_mode, pac_init_set
 ):
 
-    cs_dict, data_shape = cs_with_correct_geometry
+    cs_dict = cs_with_correct_geometry
     dresser = Dresser()
     dresser.add_drawer(Drawer(cs_dict, skip_darks=False))
     CM = CalibrationUnit(dresser)
@@ -663,21 +669,75 @@ def fully_realistic_cs(
 
 
 @pytest.fixture(scope="session")
-def fully_realistic_dresser(fully_realistic_cs):
+def fully_realistic_local_dresser(fully_realistic_local_cs):
     dresser = Dresser()
-    dresser.add_drawer(Drawer(fully_realistic_cs))
+    dresser.add_drawer(Drawer(fully_realistic_local_cs))
+    return dresser
+
+
+@pytest.fixture(scope="session")
+def fully_realistic_global_dresser(fully_realistic_local_cs):
+
+    global_cs_dict = defaultdict(list)
+    for step, step_list in fully_realistic_local_cs.items():
+        for hdu in step_list:
+            hdu_copy = deepcopy(hdu)
+            hdu_copy.data = hdu_copy.data[0, 0, 0][None, None, None]
+            global_cs_dict[step].append(hdu_copy)
+
+    dresser = Dresser()
+    dresser.add_drawer(Drawer(global_cs_dict, remove_I_trend=False))
+
     return dresser
 
 
 @pytest.fixture(scope="session")
-def initialized_polcal_fitter(fully_realistic_dresser, pac_init_set, pac_fit_mode) -> PolcalFitter:
-    return PolcalFitter(
-        local_dresser=fully_realistic_dresser,
-        global_dresser=fully_realistic_dresser,
+def num_polcal_metrics_sample_points() -> int:
+    return 10
+
+
+@pytest.fixture(scope="session")
+def polcal_fit_nan_locations(cs_data_shape, num_polcal_metrics_sample_points):
+    # We want one nan that will be un-thinned so it can be ignored by downstream checks
+    # and one nan that will be thinned so we can confirm the interaction between thinning and
+    # downstream filtering
+    num_total_points = np.prod(cs_data_shape)
+    remainder = num_total_points % num_polcal_metrics_sample_points
+    if remainder:
+        stride = num_total_points // (num_polcal_metrics_sample_points - 1)
+    else:
+        stride = num_total_points // num_polcal_metrics_sample_points
+
+    thinned_index = list(range(num_total_points))[::stride]
+    discarded_indices = list(set(range(num_total_points)) - set(thinned_index))
+
+    included_nan_idx = np.unravel_index(choice(thinned_index), cs_data_shape)
+    discarded_nan_idx = np.unravel_index(choice(discarded_indices), cs_data_shape)
+
+    return [included_nan_idx, discarded_nan_idx]
+
+
+@pytest.fixture(scope="session")
+def post_fit_polcal_fitter(
+    fully_realistic_local_dresser,
+    fully_realistic_global_dresser,
+    pac_init_set,
+    pac_fit_mode,
+    polcal_fit_nan_locations,
+) -> PolcalFitter:
+    fitter = PolcalFitter(
+        local_dresser=fully_realistic_local_dresser,
+        global_dresser=fully_realistic_global_dresser,
         fit_mode=pac_fit_mode,
         init_set=pac_init_set,
         fit_TM=False,
+        suppress_local_starting_values=True,
     )
+    for idx in polcal_fit_nan_locations:
+        for parameter in fitter.fit_parameters[idx].values():
+            parameter.set(value=np.nan)
+
+    return fitter
 
 
 class InputDatasetTask(WorkflowTaskBase, InputDatasetMixin):

{dkist_processing_common-10.2.2 → dkist_processing_common-10.3.0}/dkist_processing_common/tests/test_assemble_quality.py

@@ -186,6 +186,7 @@ def quality_metrics(dataframe_json) -> list[Metric]:
                 },
                 "bin_strs": ["bin1", "bin2"],
                 "total_bins": 100,
+                "sampled_bins": 20,
                 "num_varied_I_sys": 2,
             },
             ["QUALITY_POLCAL_LOCAL_PAR_VALS", "QUALITY_TASK_BEAM 1"],
@@ -194,6 +195,7 @@ def quality_metrics(dataframe_json) -> list[Metric]:
             {
                 "bin_strs": ["bin1", "bin2"],
                 "total_bins": 100,
+                "sampled_bins": 20,
                 "red_chi_list": [1, 2, 3],
                 "residual_json": dataframe_json,
             },
@@ -203,6 +205,7 @@ def quality_metrics(dataframe_json) -> list[Metric]:
             {
                 "bin_strs": ["bin1", "bin2"],
                 "total_bins": 100,
+                "sampled_bins": 20,
                 "efficiency_list": ((np.random.randn(4, 100) - 0.5) * 0.3).tolist(),
                 "warnings": ["A warning"],
             },