dkist-processing-common 13.0.2__tar.gz → 14.0.0rc1__tar.gz

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/CHANGELOG.rst

@@ -1,12 +1,3 @@
-v13.0.2 (2026-04-08)
-====================
-
-Misc
-----
-
-- Upgrade `dkist-processing-core` to version 7.2.0 which includes an `apache-airflow` version upgrade to 3.2.0. (`#325 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/325>`__)
-
-
 v13.0.1 (2026-04-07)
 ====================
 

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 13.0.2
+Version: 14.0.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,7 +17,7 @@ Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.24.0
 Requires-Dist: dkist-header-validator<6.0,>=5.3.0
-Requires-Dist: dkist-processing-core==7.2.0
+Requires-Dist: dkist-processing-core==7.1.0
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
 Requires-Dist: dkist-service-configuration<5.0,>=4.3.0
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0

dkist_processing_common-14.0.0rc1/changelog/323.misc.2.rst

@@ -0,0 +1,2 @@
+Simplify the reading of L0 files in the `QualityL0Metrics` task; we now read `FitsAccess` objects directly via the
+`fits_access_decoder`.

dkist_processing_common-14.0.0rc1/changelog/323.misc.rst

@@ -0,0 +1,2 @@
+`FitsAccessBase` no longer uses an HDU object for internal storage. It is now instantiated with a `header` and
+(optionally) `data` directly.

dkist_processing_common-14.0.0rc1/changelog/323.removal.2.rst

@@ -0,0 +1 @@
+Remove `.from_path` constructor from `FitsAccessBase`. This should be replaced with a call to the `fits_access_decoder`.

dkist_processing_common-14.0.0rc1/changelog/323.removal.3.rst

@@ -0,0 +1 @@
+Remove the `.size` property from `FitsAccessBase`. This not always correct and no longer used anywhere.

dkist_processing_common-14.0.0rc1/changelog/323.removal.rst

@@ -0,0 +1,2 @@
+Remove `.from_header`  constructor from `FitsAccessBase`. This should be replaced by passing the header object directly
+on instantiation.

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/codecs/fits.py

@@ -144,14 +144,15 @@ def fits_access_decoder(
         disable_image_compression=disable_image_compression,
         memmap=memmap,
     )
-    return fits_access_class(hdu=data_hdu, name=str(path), **fits_access_kwargs)
+    return fits_access_class(
+        header=data_hdu.header, data=data_hdu.data, name=str(path), **fits_access_kwargs
+    )
 
 
 def fits_access_header_decoder(
     path: Path,
     fits_access_class: Type[FitsAccessBase],
     hdu: int | None = None,
-    **from_header_kwargs,
 ) -> FitsAccessBase:
     """
     Read a header from a Path with `~astropy.io.fits` and ingest into a `~dkist_processing_common.models.fits_access.FitsAccessBase`-type object.
@@ -163,7 +164,7 @@ def fits_access_header_decoder(
         path,
         hdu=hdu,
     )
-    return fits_access_class.from_header(header=header, name=str(path), **from_header_kwargs)
+    return fits_access_class(header=header, name=str(path))
 
 
 def _extract_hdu(hdul: fits.HDUList, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/models/fits_access.py

@@ -59,33 +59,46 @@ class FitsAccessBase:
     """
     Abstraction layer for accessing fits data via class attributes.
 
+    .. note::
+      The preferred method to load a FITS file into this class or its children is to use one of the `fits access codecs <dkist_processing_common.codecs.fits>`
+      in combination with `~dkist_processing_common.tasks.base.WorkflowTaskBase.read`. Direct instantiation should only
+      be used in unit tests or when `header` and `data` have been recently been generated an only live in memory.
+
     Parameters
     ----------
-    hdu
-        The fits object
+    header
+        The header from which to parse higher-level properties
+
+    data
+        A data array to associate with the current instance
+
     name
         An optional name that can be associated with the object
+
     auto_squeeze
         A boolean indicating whether to 'squeeze' out dimensions of size 1
     """
 
     def __init__(
         self,
-        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU | fits.BinTableHDU,
+        *,
+        header: fits.Header,
+        data: np.ndarray | None = None,
         name: str | Path | None = None,
         auto_squeeze: bool = True,
     ):
-        self._hdu = hdu
+        self.header = header
+        self._data = data
         self.name = name
         self.auto_squeeze = auto_squeeze
 
     def __repr__(self):
-        return f"{self.__class__.__name__}(hdu={self._hdu!r}, name={self.name!r}, auto_squeeze={self.auto_squeeze})"
+        return f"{self.__class__.__name__}(header={self.header!r}, data={self._data!r}, name={self.name!r}, auto_squeeze={self.auto_squeeze})"
 
     @property
-    def data(self) -> np.ndarray:
+    def data(self) -> np.ndarray | None:
         """
-        Return the data array from the associated FITS object, with axes of length 1 removed if the array has three dimensions and the unit axis is the zeroth one.
+        Return the data array, with axes of length 1 removed if the array has three dimensions and the unit axis is the zeroth one.
 
         This is intended solely to remove the dummy dimension that is in raw data from the summit.
 
@@ -95,17 +108,25 @@ class FitsAccessBase:
         -------
         data array
         """
+        if not self.auto_squeeze or self._data is None:
+            return self._data
+
         # This conditional is explicitly to catch summit data with a dummy first axis for WCS
         # purposes
-        if self.auto_squeeze and len(self._hdu.data.shape) == 3 and self._hdu.data.shape[0] == 1:
-            return np.squeeze(self._hdu.data)
-        return self._hdu.data
+        if len(self._data.shape) == 3 and self._data.shape[0] == 1:
+            return np.squeeze(self._data)
+
+        return self._data
 
     @data.setter
     def data(self, array: np.ndarray) -> None:
         """
         Set the data array using an input data array.
 
+        Note that this method does not update any keys in this object's `.header` (NAXIS, BITPIX, etc.). If the data and
+        header of this object are to be written to disk, make sure to use the `~astropy.io.fits` interface, which will
+        correctly update the header on write.
+
         Parameters
         ----------
         array
@@ -115,14 +136,7 @@ class FitsAccessBase:
         -------
         None
         """
-        # There is no shape magic stuff going on here right now because the tasks/services that care about
-        # it will deal with it themselves (I think (tm)).
-        self._hdu.data = array
-
-    @property
-    def header(self) -> fits.Header:
-        """Return the header for this fits object."""
-        return self._hdu.header
+        self._data = array
 
     @property
     def header_dict(self) -> dict:
@@ -134,46 +148,3 @@ class FitsAccessBase:
             else:
                 result[card] = value
         return result
-
-    @property
-    def size(self) -> float:
-        """Return the size in bytes of the data portion of this fits object."""
-        return self._hdu.size
-
-    @classmethod
-    def from_header(cls, header: fits.Header | dict, name: str | None = None) -> FitsAccessBase:
-        """
-        Convert a header to a FitsAccessBase (or child) object.
-
-        Parameters
-        ----------
-        header
-            A single `astropy.io.fits.header.Header` HDU object.
-        name
-            A unique name for the fits access instance
-        """
-        if isinstance(header, dict):
-            header = fits.Header(header)
-        hdu = fits.PrimaryHDU()
-
-        # We need to update the header after `PrimaryHDU` instantiation because some of the FITS controlled keys
-        # (e.g., NAXIS, NAXISn) would otherwise be changed by checks that occur during instantiation.
-        hdu.header.update(header)
-        return cls(hdu=hdu, name=name)
-
-    @classmethod
-    def from_path(cls, path: str | Path) -> FitsAccessBase:
-        """
-        Load the file at given path into a FitsAccess object.
-
-        Parameters
-        ----------
-        path
-            Location of fits file on disk
-        """
-        hdul = fits.open(path)
-        if hdul[0].data is not None:
-            hdu = hdul[0]
-        else:
-            hdu = hdul[1]
-        return cls(hdu=hdu, name=path)

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/parsers/l0_fits_access.py

@@ -1,5 +1,6 @@
 """By-frame 214 L0 header keywords that are not instrument specific."""
 
+import numpy as np
 from astropy.io import fits
 
 from dkist_processing_common.models.fits_access import MetadataKey
@@ -33,10 +34,15 @@ class L0FitsAccess(L1FitsAccess):
 
     Parameters
     ----------
-    hdu
-        The input fits hdu
+    header
+        The header from which to parse higher-level properties
+
+    data
+        A data array to associate with the current instance
+
     name
         An optional name to be associated with the hdu
+
     auto_squeeze
         A boolean indicating whether to 'squeeze' out dimensions of size 1
     """
@@ -45,11 +51,13 @@ class L0FitsAccess(L1FitsAccess):
 
     def __init__(
         self,
-        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU,
+        *,
+        header: fits.Header,
+        data: np.ndarray | None = None,
         name: str | None = None,
         auto_squeeze: bool = True,
     ):
-        super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
+        super().__init__(header=header, data=data, name=name, auto_squeeze=auto_squeeze)
         self.ip_task_type: str = self.header[MetadataKey.ip_task_type]
         self.ip_start_time: str = self.header[MetadataKey.ip_start_time]
         self.ip_end_time: str = self.header[MetadataKey.ip_end_time]

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/parsers/l1_fits_access.py

@@ -1,5 +1,6 @@
 """By-frame 214 L1 only header keywords that are not instrument specific."""
 
+import numpy as np
 from astropy.io import fits
 
 from dkist_processing_common.models.fits_access import HEADER_KEY_NOT_FOUND
@@ -15,21 +16,28 @@ class L1FitsAccess(FitsAccessBase):
 
     Parameters
     ----------
-    hdu
-        The input fits hdu
+    header
+        The header from which to parse higher-level properties
+
+    data
+        A data array to associate with the current instance
+
     name
         An optional name to be associated with the hdu
+
     auto_squeeze
         A boolean indicating whether to 'squeeze' out dimensions of size 1
     """
 
     def __init__(
         self,
-        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU,
+        *,
+        header: fits.Header,
+        data: np.ndarray | None = None,
         name: str | None = None,
         auto_squeeze: bool = False,  # Because L1 data should always have the right form, right?
     ):
-        super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
+        super().__init__(header=header, data=data, name=name, auto_squeeze=auto_squeeze)
 
         self.elevation: float = self.header[MetadataKey.elevation]
         self.azimuth: float = self.header[MetadataKey.azimuth]

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/parsers/quality.py

@@ -1,5 +1,6 @@
 """Support classes to define object attributes from header information."""
 
+import numpy as np
 from astropy.io import fits
 
 from dkist_processing_common.parsers.l0_fits_access import L1FitsAccess
@@ -21,11 +22,13 @@ class L1QualityFitsAccess(L1FitsAccess):
 
     def __init__(
         self,
-        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU,
+        *,
+        header: fits.Header,
+        data: np.ndarray | None = None,
         name: str | None = None,
         auto_squeeze: bool = True,
     ):
-        super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
+        super().__init__(header=header, data=data, name=name, auto_squeeze=auto_squeeze)
 
         self.fried_parameter: float = self.header["ATMOS_R0"]
         self.date_begin: str = self.header["DATE-BEG"]

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/tasks/quality_metrics.py

@@ -5,9 +5,7 @@ from collections import defaultdict
 from datetime import datetime
 from inspect import signature
 from itertools import chain
-from pathlib import Path
 from typing import Callable
-from typing import Generator
 from typing import Iterable
 from typing import Type
 
@@ -15,6 +13,7 @@ import numpy as np
 from pydantic import FiniteFloat
 
 from dkist_processing_common.codecs.basemodel import basemodel_encoder
+from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.models.fits_access import FitsAccessBase
 from dkist_processing_common.models.metric_code import MetricCode
 from dkist_processing_common.models.quality import QualityMetric
@@ -128,21 +127,6 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
         with self.telemetry_span("Writing L0 Quality Metrics to disk"):
             self.compute_and_write_l0_quality_metrics(segmented_quality_data)
 
-    # TODO FUTURE - this method should ultimately go away
-    def get_paths_for_modstate_and_task(
-        self, modstate: int | None, task_type: str
-    ) -> Generator[Path, None, None]:
-        """
-        Return Paths for all raw files that tagged with the given modstate and task type.
-
-        The tag that defines "raw" can be changed in the `raw_frame_tag` property.
-        """
-        tags = [self.raw_frame_tag, Tag.task(task_type)]
-        if modstate is not None:
-            tags.append(Tag.modstate(modstate))
-
-        return self.read(tags)
-
     def collect_l0_quality_data(self) -> dict[str, dict[str, L0QualityData]]:
         """Calculate L0 quality metrics for each task type and modstate combo."""
         intermediate_l0_quality_data_segmented_first_by_ip_task_and_then_by_modstate: dict[
@@ -152,25 +136,29 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
         for task_type in self.quality_task_types:
             with self.telemetry_span(f"Collecting {task_type = }"):
                 for modstate in modstate_list:
-                    # TODO FUTURE - this should do something like this: self.read(decoder=l0_fits_access_decoder)
-                    paths = self.get_paths_for_modstate_and_task(modstate, task_type)
+                    tags = [self.raw_frame_tag, Tag.task(task_type)]
+                    if modstate is not None:
+                        tags.append(Tag.modstate(modstate))
+
+                    fits_objs = self.read(
+                        tags=tags, decoder=fits_access_decoder, fits_access_class=L0FitsAccess
+                    )
 
                     # establish empty lists for this task type AND modstate
                     datetimes = []
                     average_values = []
                     rms_values = []
 
-                    for path in paths:
-                        frame = L0FitsAccess.from_path(path)
-                        data = frame.data.astype(float)
-                        exposure_time_sec = frame.fpa_exposure_time_ms / 1000
+                    for fo in fits_objs:
+                        data = fo.data.astype(float)
+                        exposure_time_sec = fo.fpa_exposure_time_ms / 1000
 
                         # Mean and RMS for Frame
                         normalized_mean = self.compute_normalized_mean(data, exposure_time_sec)
                         normalized_rms = self.compute_normalized_rms(data, exposure_time_sec)
 
                         # accumulate each value into its list
-                        datetimes.append(datetime.fromisoformat(frame.time_obs))
+                        datetimes.append(datetime.fromisoformat(fo.time_obs))
                         average_values.append(normalized_mean)
                         rms_values.append(normalized_rms)
 
@@ -488,13 +476,16 @@ class QualityL1Metrics(WorkflowTaskBase, QualityMixin):
         ]
 
         with self.telemetry_span("Reading L1 frames"):
+            # Read in paths and convert to `FitsAccess` objects below because we iterate throught `paths` multiple times
             paths = list(self.read(tags=[Tag.calibrated(), Tag.frame()]))
 
         with self.telemetry_span("Calculating L1 quality metrics"):
             for metric in metrics:
                 with self.telemetry_span(f"Calculating L1 metric {metric.value_source}"):
                     for path in paths:
-                        frame = L1QualityFitsAccess.from_path(path)
+                        frame = fits_access_decoder(
+                            path=path, fits_access_class=L1QualityFitsAccess
+                        )
                         metric.append_value(frame=frame)
 
         with self.telemetry_span("Sending lists for storage"):

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/tasks/write_l1.py

@@ -67,7 +67,6 @@ class WriteL1Frame(WriteL1Base, ABC):
                 l1_header = self.convert_l0_to_l1(
                     header=calibrated_fits_object.header,
                     data=calibrated_fits_object.data,
-                    hdu_size=calibrated_fits_object.size,
                     stokes_param=stokes_param,
                 )
 
@@ -160,7 +159,6 @@ class WriteL1Frame(WriteL1Base, ABC):
     def add_datacenter_headers(
         self,
         header: fits.Header,
-        hdu_size: float,
         stokes: Literal["I", "Q", "U", "V"],
     ) -> fits.Header:
         """Fill out the spec 214 datacenter header table."""
@@ -301,7 +299,6 @@ class WriteL1Frame(WriteL1Base, ABC):
         self,
         header: fits.Header,
         data: np.ndarray,
-        hdu_size: float,
         stokes_param: Literal["I", "Q", "U", "V"],
     ) -> fits.Header:
         """
@@ -311,10 +308,10 @@ class WriteL1Frame(WriteL1Base, ABC):
         ----------
         header
             The L0 header
+
         data
             The data array
-        hdu_size
-            The hdu size
+
         stokes_param
             The stokes parameter
 
@@ -335,7 +332,7 @@ class WriteL1Frame(WriteL1Base, ABC):
         # Add the stats table
         header = self.add_stats_headers(header=header, data=data)
         # Add the datacenter table
-        header = self.add_datacenter_headers(header=header, hdu_size=hdu_size, stokes=stokes_param)
+        header = self.add_datacenter_headers(header=header, stokes=stokes_param)
         # Add extra headers recommended by solarnet (not all in a single table)
         header = self.add_solarnet_headers(header=header)
         # Add the documentation headers

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/tests/conftest.py

@@ -297,11 +297,12 @@ class NonPolCalDataset(Spec122Dataset):
 class ModstateFitsAccess(L0FitsAccess):
     def __init__(
         self,
-        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU,
+        *,
+        header: fits.Header,
         name: str | None = None,
         auto_squeeze: bool = True,
     ):
-        super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
+        super().__init__(header=header, name=name, auto_squeeze=auto_squeeze)
         self.modulator_state: int = self.header["VSPSTNUM"]
 
 
@@ -348,7 +349,7 @@ def grouped_cal_sequence_fits_objs(
     for i in range(ds.num_steps):
         for m in range(ds.num_modstates):
             for j in range(ds.num_frames_per_CS_step):
-                expected_cs_dict[i].append(ModstateFitsAccess.from_header(header_list.pop(0)))
+                expected_cs_dict[i].append(ModstateFitsAccess(header=header_list.pop(0)))
 
     # Make sure we used all the dataset frames
     assert len(header_list) == 0
@@ -376,7 +377,7 @@ def grouped_cal_sequence_modstate_fits_objs(
     for i in range(ds.num_steps):
         for m in range(ds.num_modstates):
             for j in range(ds.num_frames_per_CS_step):
-                expected_cs_dict[i][m].append(ModstateFitsAccess.from_header(header_list.pop(0)))
+                expected_cs_dict[i][m].append(ModstateFitsAccess(header=header_list.pop(0)))
 
     # Make sure we used all the dataset frames
     assert len(header_list) == 0
@@ -392,7 +393,7 @@ def non_polcal_fits_objs() -> list[L0FitsAccess]:
         ].header
         for d in ds
     ]
-    obj_list = [L0FitsAccess.from_header(h) for h in header_list]
+    obj_list = [L0FitsAccess(header=h) for h in header_list]
     return obj_list
 
 
@@ -485,8 +486,8 @@ class CalibrationSequenceStepDataset(Spec122Dataset):
 
 
 class InstAccess(L0FitsAccess):
-    def __init__(self, hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU):
-        super().__init__(hdu, auto_squeeze=False)
+    def __init__(self, *, header: fits.Header, data: np.ndarray | None = None):
+        super().__init__(header=header, data=data, auto_squeeze=False)
         self.modulator_state = self.header["VSPSTNUM"]
         self.number_of_modulator_states = self.header["VSPNUMST"]
 
@@ -577,7 +578,7 @@ def cs_with_correct_geometry(cs_data_shape):
                 )
             )
 
-        out_dict[n] = [InstAccess(h) for h in hdu_list]
+        out_dict[n] = [InstAccess(header=h.header, data=h.data) for h in hdu_list]
         start_time = ds.start_time + timedelta(seconds=60)
 
     return out_dict

{dkist_processing_common-13.0.2 → dkist_processing_common-14.0.0rc1}/dkist_processing_common/tests/test_codecs.py

@@ -382,11 +382,13 @@ def check_open_file_descriptors() -> int | None:
 class DummyFitsAccess(FitsAccessBase):
     def __init__(
         self,
-        hdu: fits.ImageHDU | fits.PrimaryHDU | fits.CompImageHDU,
+        *,
+        header: fits.Header,
+        data: np.ndarray | None = None,
         name: str | None = None,
         auto_squeeze: bool = False,
     ):
-        super().__init__(hdu=hdu, name=name, auto_squeeze=auto_squeeze)
+        super().__init__(header=header, data=data, name=name, auto_squeeze=auto_squeeze)
         self.foo = self.header["foo"]
 
 
@@ -636,14 +638,6 @@ def test_fits_access_decoder(
         assert post_num_fd <= init_num_fd + 1 if memmap and not decompress else init_num_fd
     assert fits_obj.name == str(file_path)
     assert fits_obj.foo == fits_header["foo"]
-    if "compressed" in path_fixture_name:
-        if decompress:
-            assert isinstance(fits_obj._hdu, fits.CompImageHDU)
-        else:
-            assert isinstance(fits_obj._hdu, fits.BinTableHDU)
-    else:
-        assert isinstance(fits_obj._hdu, fits.PrimaryHDU)
-
     if "compressed" in path_fixture_name and not decompress:
         assert not np.array_equal(fits_obj.data, ndarray_object)
     else: