dkist-processing-ops 1.6.12__py3-none-any.whl → 1.6.14__py3-none-any.whl

dkist_processing_ops/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.12'
-__version_tuple__ = version_tuple = (1, 6, 12)
+__version__ = version = '1.6.14'
+__version_tuple__ = version_tuple = (1, 6, 14)

dkist_processing_ops/tasks/read_memory_leak.py

@@ -1,79 +1,74 @@
-import logging
-import os
-import sys
+import gc
 from abc import ABC
 from pathlib import Path
-from typing import Any
-from typing import Generator
 
 import numpy as np
 from astropy.io import fits
+from dkist_processing_common.codecs.fits import fits_hdu_decoder
 from dkist_processing_common.codecs.path import path_decoder
 from dkist_processing_common.models.tags import Tag
-from dkist_processing_common.tasks import tag_type_hint
 from dkist_processing_common.tasks import WorkflowTaskBase
 
 
-def yield_from_read(
-    self, tags: tag_type_hint, decoder: callable = path_decoder, **decoder_kwargs
-) -> Generator[Any, None, None]:
-    yield from (decoder(p, **decoder_kwargs) for p in self.scratch.find_all(tags=tags))
+def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
+    data = np.random.rand(4096, 4096)
+    hdu = fits.CompImageHDU(data)
+    return hdu
 
 
-def binary_decoder(path: Path) -> bytes:
-    """Generates a random blob of binary data."""
-    data = os.urandom(67109008)  # This is the size of the VBI data arrays, in bytes
-    logging.info(f"Size of data: {sys.getsizeof(data)}")
-    logging.info(f"Bytes of data: {len(data)}")
-    return data
-
-
-def fits_smart_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
-    """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path, lazy_load_hdus=False)
-    return hdu_list[1]
+def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
+    hdul = fits.open(path)
+    hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
+    hdul.close()
+    del hdul
+    gc.collect()
+    return hdu
 
 
 class FitsDataRead(WorkflowTaskBase, ABC):
     @property
-    def data_type(self) -> str:
-        """Recipe run configuration indicating how fFITS data should be read."""
-        return self.metadata_store_recipe_run_configuration().get("data_type", "fits")
+    def run_type(self):
+        return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
 
-    @property
-    def read_method(self):
-        return self.metadata_store_recipe_run_configuration().get("read_method", "return")
+    def run(self) -> None:
+        if self.run_type == "garbage_collect_read":
+            hdus = self.read(
+                tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
+            )
+            for hdu in hdus:
+                h = hdu.header
+                d = hdu.data
 
-    @property
-    def manipulate_data(self):
-        return self.metadata_store_recipe_run_configuration().get("manipulate_data", True)
+        if self.run_type == "garbage_collect_task":
+            filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
+            for filepath in filepaths:
+                hdu = fits_garbage_collect_hdu_decoder(filepath)
+                h = hdu.header
+                d = hdu.data
 
-    def run(self) -> None:
-        if self.data_type == "fits":
-            if self.read_method == "return":
-                hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_smart_hdu_decoder)
-                for hdu in hdus:
-                    h = hdu.header
-                    d = hdu.data
-                    if self.manipulate_data:
-                        total = np.sum(d)
-            if self.read_method == "yield":
-                hdus = yield_from_read(
-                    self, tags=[Tag.input(), Tag.frame()], decoder=fits_smart_hdu_decoder
-                )
-                for hdu in hdus:
-                    h = hdu.header
-                    d = hdu.data
-                    if self.manipulate_data:
-                        total = np.sum(d)
-        if self.data_type == "binary":
-            if self.read_method == "return":
-                hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=binary_decoder)
-                for hdu in hdus:
-                    data = hdu
-            if self.read_method == "yield":
-                hdus = yield_from_read(
-                    self, tags=[Tag.input(), Tag.frame()], decoder=binary_decoder
-                )
-                for hdu in hdus:
-                    data = hdu
+        if self.run_type == "file_read":
+            hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
+            for hdu in hdus:
+                h = hdu.header
+                d = hdu.data
+        #
+        # if self.run_type == "generated_read":
+        #     hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)
+        #     for hdu in hdus:
+        #         h = hdu.header
+        #         d = hdu.data
+        #
+        # if self.run_type == "file_task":
+        #     filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
+        #     for filepath in filepaths:
+        #         hdu = fits.open(filepath)[1]
+        #         h = hdu.header
+        #         d = hdu.data
+        #
+        # if self.run_type == "generated_task":
+        #     filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
+        #     for filepath in filepaths:
+        #         data = np.random.rand(4096, 4096)
+        #         hdu = fits.CompImageHDU(data)
+        #         h = hdu.header
+        #         d = hdu.data

{dkist_processing_ops-1.6.12.dist-info → dkist_processing_ops-1.6.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.12
+Version: 1.6.14
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause

{dkist_processing_ops-1.6.12.dist-info → dkist_processing_ops-1.6.14.dist-info}/RECORD

@@ -1,16 +1,16 @@
 dkist_processing_ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dkist_processing_ops/_version.py,sha256=dUu2OxelDtAMXjHtstu0l-dD_T_j4hfz4LAPXSgLBBc,413
+dkist_processing_ops/_version.py,sha256=ENoeph2R-6hTwDBnYv-6HGbXQQBxCuxRZdUsEOyJGbk,413
 dkist_processing_ops/dags/scale.py,sha256=We5TYjNhkJ-5ykfbrOMgjTpXdzOCkIeyKyA-40sU9r0,2312
 dkist_processing_ops/tasks/__init__.py,sha256=P81O9cg4dlBMqBTaWitdsAte68RsMtDlhV30JSZfXUY,107
-dkist_processing_ops/tasks/read_memory_leak.py,sha256=gAhoOKp3m9PPXCHUFqTkeiUhWZzTUD-QCMkjvsFMzIE,2941
+dkist_processing_ops/tasks/read_memory_leak.py,sha256=WuwS7Jb-wUBqGsI6Qn8DvGvDHZDsj6jVyRp_vKf2QV4,2657
 dkist_processing_ops/tasks/wait.py,sha256=uObka-nH1dKPcGBDsp3t2RCtTV2F1kksM0V-lRewFuY,273
 dkist_processing_ops/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dkist_processing_ops/tests/test_workflows.py,sha256=Ch_8BlGeQyPJU_9hB_GOncwW-SoZwpRUVKMOEz0RQZk,285
 dkist_processing_ops/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dkist_processing_ops/workflows/memory_leak.py,sha256=lYXAYyJVjXif3Os9xPDp-bPTG_je6HOw1uvRJ4WMUi4,758
 dkist_processing_ops/workflows/smoke.py,sha256=ofXu0_iYF6L3zQy-BOVvS5VdzKhmXs1gyugqMNkd-GM,878
-dkist_processing_ops-1.6.12.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
-dkist_processing_ops-1.6.12.dist-info/METADATA,sha256=Rd4do0X7qmPP8hRsVV-M8B-rcLn2IRTXeiZvCBuYnVg,1500
-dkist_processing_ops-1.6.12.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-dkist_processing_ops-1.6.12.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
-dkist_processing_ops-1.6.12.dist-info/RECORD,,
+dkist_processing_ops-1.6.14.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
+dkist_processing_ops-1.6.14.dist-info/METADATA,sha256=KUW80lrBGyblfl5r9uSPaWF3ohu26KT2rSlQRV7qleA,1500
+dkist_processing_ops-1.6.14.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+dkist_processing_ops-1.6.14.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
+dkist_processing_ops-1.6.14.dist-info/RECORD,,