dkist-processing-ops 1.6.6rc1__tar.gz → 1.6.7__tar.gz

{dkist_processing_ops-1.6.6rc1 → dkist_processing_ops-1.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.6rc1
+Version: 1.6.7
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause

{dkist_processing_ops-1.6.6rc1 → dkist_processing_ops-1.6.7}/dkist_processing_ops/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.6rc1'
-__version_tuple__ = version_tuple = (1, 6, 6)
+__version__ = version = '1.6.7'
+__version_tuple__ = version_tuple = (1, 6, 7)

dkist_processing_ops-1.6.7/dkist_processing_ops/tasks/read_memory_leak.py

@@ -0,0 +1,51 @@
+import os
+from abc import ABC
+from pathlib import Path
+from typing import Any
+from typing import Generator
+
+import numpy as np
+from astropy.io import fits
+from dkist_processing_common.codecs.fits import fits_hdu_decoder
+from dkist_processing_common.codecs.path import path_decoder
+from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tasks import tag_type_hint
+from dkist_processing_common.tasks import WorkflowTaskBase
+
+
+def yield_from_read(
+    self, tags: tag_type_hint, decoder: callable = path_decoder, **decoder_kwargs
+) -> Generator[Any, None, None]:
+    yield from (decoder(p, **decoder_kwargs) for p in self.scratch.find_all(tags=tags))
+
+
+def binary_decoder(path: Path) -> bytes:
+    """Generates a random blob of binary data."""
+    return os.urandom(67109008)  # This is the size of the VBI data arrays, in bytes
+
+
+class FitsDataRead(WorkflowTaskBase, ABC):
+    @property
+    def data_type(self) -> str:
+        """Recipe run configuration indicating how fFITS data should be read."""
+        return self.metadata_store_recipe_run_configuration().get("data_type", "fits")
+
+    @property
+    def read_method(self):
+        return self.metadata_store_recipe_run_configuration().get("read_method", "return")
+
+    def run(self) -> None:
+        if self.data_type == "fits":
+            if self.read_method == "return":
+                hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
+            if self.read_method == "yield":
+                hdus = yield_from_read(
+                    self, tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder
+                )
+        if self.data_type == "binary":
+            if self.read_method == "return":
+                hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=binary_decoder)
+            if self.read_method == "yield":
+                hdus = yield_from_read(
+                    self, tags=[Tag.input(), Tag.frame()], decoder=binary_decoder
+                )

{dkist_processing_ops-1.6.6rc1 → dkist_processing_ops-1.6.7}/dkist_processing_ops.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.6rc1
+Version: 1.6.7
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause

dkist_processing_ops-1.6.6rc1/dkist_processing_ops/tasks/read_memory_leak.py

@@ -1,55 +0,0 @@
-from abc import ABC
-from pathlib import Path
-
-import numpy as np
-from astropy.io import fits
-from dkist_processing_common.codecs.fits import fits_hdu_decoder
-from dkist_processing_common.models.tags import Tag
-from dkist_processing_common.tasks import WorkflowTaskBase
-
-
-def _extract_hdu(hdul: fits.HDUList) -> fits.PrimaryHDU | fits.CompImageHDU:
-    """Return the fits hdu associated with the data in the hdu list."""
-    if hdul[0].data is not None:
-        return hdul[0]
-    return hdul[1]
-
-
-def fits_hdu_decoder_with_close(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
-    """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path, memmap=False)
-    hdu_to_return = _extract_hdu(hdu_list).copy()
-    hdu_list.close()
-    return hdu_to_return
-
-
-def fits_hdu_decoder_with_close_and_del(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
-    """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path, memmap=False)
-    hdu_to_return = _extract_hdu(hdu_list).copy()
-    hdu_list.close()
-    del hdu_list
-    return hdu_to_return
-
-
-class FitsDataRead(WorkflowTaskBase, ABC):
-    @property
-    def fits_data_read_strategy(self) -> str:
-        """Recipe run configuration indicating how fFITS data should be read."""
-        return self.metadata_store_recipe_run_configuration().get(
-            "fits_data_read_strategy", "leave_open"
-        )
-
-    def run(self) -> None:
-        if self.fits_data_read_strategy == "leave_open":
-            hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
-        if self.fits_data_read_strategy == "close":
-            hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder_with_close)
-        if self.fits_data_read_strategy == "close_and_del":
-            hdus = self.read(
-                tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder_with_close_and_del
-            )
-        for hdu in hdus:
-            data = hdu.data
-            header = hdu.header
-            total = np.sum(data)