dkist-processing-ops 1.6.16__tar.gz → 1.6.18__tar.gz

{dkist_processing_ops-1.6.16 → dkist_processing_ops-1.6.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.16
+Version: 1.6.18
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause

{dkist_processing_ops-1.6.16 → dkist_processing_ops-1.6.18}/dkist_processing_ops/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.16'
-__version_tuple__ = version_tuple = (1, 6, 16)
+__version__ = version = '1.6.18'
+__version_tuple__ = version_tuple = (1, 6, 18)

dkist_processing_ops-1.6.18/dkist_processing_ops/tasks/read_memory_leak.py

@@ -0,0 +1,47 @@
+import gc
+import logging
+from abc import ABC
+from pathlib import Path
+
+import numpy as np
+from astropy.io import fits
+from dkist_processing_common.codecs.fits import fits_hdu_decoder
+from dkist_processing_common.codecs.path import path_decoder
+from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.tasks import WorkflowTaskBase
+
+
+def fits_bytes_decoder(path: Path) -> bytes:
+    with open(path, "rb") as f:
+        return f.read()
+
+
+class FitsDataRead(WorkflowTaskBase, ABC):
+    @property
+    def run_type(self):
+        return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
+
+    def run(self) -> None:
+        if self.run_type == "bytes_read":
+            bytes_objects = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_bytes_decoder)
+            for i, byte_object in enumerate(bytes_objects):
+                pass
+
+        if self.run_type == "bytes_task":
+            filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
+            for filepath in filepaths:
+                with open(filepath, "rb") as f:
+                    byte_object = f.read()
+
+        if self.run_type == "file_read":
+            hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
+            for hdu in hdus:
+                h = hdu.header
+                d = hdu.data
+
+        if self.run_type == "file_task":
+            filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
+            for filepath in filepaths:
+                hdu = fits.open(filepath)[1]
+                h = hdu.header
+                d = hdu.data

{dkist_processing_ops-1.6.16 → dkist_processing_ops-1.6.18}/dkist_processing_ops.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.16
+Version: 1.6.18
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause

dkist_processing_ops-1.6.16/dkist_processing_ops/tasks/read_memory_leak.py

@@ -1,95 +0,0 @@
-import gc
-import logging
-from abc import ABC
-from pathlib import Path
-
-import numpy as np
-from astropy.io import fits
-from dkist_processing_common.codecs.fits import fits_hdu_decoder
-from dkist_processing_common.codecs.path import path_decoder
-from dkist_processing_common.models.tags import Tag
-from dkist_processing_common.tasks import WorkflowTaskBase
-
-
-# def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
-#     data = np.random.rand(4096, 4096)
-#     hdu = fits.CompImageHDU(data)
-#     return hdu
-
-
-def fits_bytes_decoder(path: Path) -> bytes:
-    with open(path, "rb") as f:
-        return f.read()
-
-
-# def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
-#     hdul = fits.open(path)
-#     hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
-#     hdul.close()
-#     del hdul
-#     gc.collect()
-#     return hdu
-
-
-class FitsDataRead(WorkflowTaskBase, ABC):
-    @property
-    def run_type(self):
-        return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
-
-    def run(self) -> None:
-        if self.run_type == "bytes_read":
-            bytes_objects = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_bytes_decoder)
-            for i, byte_object in enumerate(bytes_objects):
-                pass
-
-        if self.run_type == "file_read":
-            hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
-            for i, hdu in enumerate(hdus):
-                h = hdu.header
-                d = hdu.data
-
-        # if self.run_type == "garbage_collect_read":
-        #     hdus = self.read(
-        #         tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
-        #     )
-        #     for i, hdu in enumerate(hdus):
-        #         h = hdu.header
-        #         d = hdu.data
-        #         if i % 50 == 0:
-        #             cur, peak = tracemalloc.get_traced_memory()
-        #             logging.info(
-        #                 f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
-        #             )
-        #
-        # if self.run_type == "garbage_collect_task":
-        #     filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
-        #     for i, filepath in enumerate(filepaths):
-        #         hdu = fits_garbage_collect_hdu_decoder(filepath)
-        #         h = hdu.header
-        #         d = hdu.data
-        #         if i % 50 == 0:
-        #             cur, peak = tracemalloc.get_traced_memory()
-        #             logging.info(
-        #                 f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
-        #             )
-        #
-        # if self.run_type == "generated_read":
-        #     hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)
-        #     for hdu in hdus:
-        #         h = hdu.header
-        #         d = hdu.data
-        #
-        # if self.run_type == "file_task":
-        #     filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
-        #     for filepath in filepaths:
-        #         hdu = fits.open(filepath)[1]
-        #         h = hdu.header
-        #         d = hdu.data
-        #
-        # if self.run_type == "generated_task":
-        #     filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
-        #     for filepath in filepaths:
-        #         data = np.random.rand(4096, 4096)
-        #         hdu = fits.CompImageHDU(data)
-        #         h = hdu.header
-        #         d = hdu.data