dkist-processing-ops 1.6.15__py3-none-any.whl → 1.6.16__py3-none-any.whl

dkist_processing_ops/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.15'
-__version_tuple__ = version_tuple = (1, 6, 15)
+__version__ = version = '1.6.16'
+__version_tuple__ = version_tuple = (1, 6, 16)

dkist_processing_ops/tasks/read_memory_leak.py

@@ -1,6 +1,5 @@
 import gc
 import logging
-import tracemalloc
 from abc import ABC
 from pathlib import Path
 
@@ -12,19 +11,24 @@ from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import WorkflowTaskBase
 
 
-def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
-    data = np.random.rand(4096, 4096)
-    hdu = fits.CompImageHDU(data)
-    return hdu
+# def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
+#     data = np.random.rand(4096, 4096)
+#     hdu = fits.CompImageHDU(data)
+#     return hdu
 
 
-def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
-    hdul = fits.open(path)
-    hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
-    hdul.close()
-    del hdul
-    gc.collect()
-    return hdu
+def fits_bytes_decoder(path: Path) -> bytes:
+    with open(path, "rb") as f:
+        return f.read()
+
+
+# def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
+#     hdul = fits.open(path)
+#     hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
+#     hdul.close()
+#     del hdul
+#     gc.collect()
+#     return hdu
 
 
 class FitsDataRead(WorkflowTaskBase, ABC):
@@ -33,42 +37,41 @@ class FitsDataRead(WorkflowTaskBase, ABC):
         return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
 
     def run(self) -> None:
-        tracemalloc.start()
-        if self.run_type == "garbage_collect_read":
-            hdus = self.read(
-                tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
-            )
-            for i, hdu in enumerate(hdus):
-                h = hdu.header
-                d = hdu.data
-                if i % 50 == 0:
-                    cur, peak = tracemalloc.get_traced_memory()
-                    logging.info(
-                        f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
-                    )
-
-        if self.run_type == "garbage_collect_task":
-            filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
-            for i, filepath in enumerate(filepaths):
-                hdu = fits_garbage_collect_hdu_decoder(filepath)
-                h = hdu.header
-                d = hdu.data
-                if i % 50 == 0:
-                    cur, peak = tracemalloc.get_traced_memory()
-                    logging.info(
-                        f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
-                    )
+        if self.run_type == "bytes_read":
+            bytes_objects = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_bytes_decoder)
+            for i, byte_object in enumerate(bytes_objects):
+                pass
 
         if self.run_type == "file_read":
             hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
             for i, hdu in enumerate(hdus):
                 h = hdu.header
                 d = hdu.data
-                if i % 50 == 0:
-                    cur, peak = tracemalloc.get_traced_memory()
-                    logging.info(
-                        f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
-                    )
+
+        # if self.run_type == "garbage_collect_read":
+        #     hdus = self.read(
+        #         tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
+        #     )
+        #     for i, hdu in enumerate(hdus):
+        #         h = hdu.header
+        #         d = hdu.data
+        #         if i % 50 == 0:
+        #             cur, peak = tracemalloc.get_traced_memory()
+        #             logging.info(
+        #                 f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
+        #             )
+        #
+        # if self.run_type == "garbage_collect_task":
+        #     filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
+        #     for i, filepath in enumerate(filepaths):
+        #         hdu = fits_garbage_collect_hdu_decoder(filepath)
+        #         h = hdu.header
+        #         d = hdu.data
+        #         if i % 50 == 0:
+        #             cur, peak = tracemalloc.get_traced_memory()
+        #             logging.info(
+        #                 f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
+        #             )
         #
         # if self.run_type == "generated_read":
         #     hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)

{dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.15
+Version: 1.6.16
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause

{dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.16.dist-info}/RECORD

@@ -1,16 +1,16 @@
 dkist_processing_ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dkist_processing_ops/_version.py,sha256=G6p4cF-LZDRxAX98giKjw_I7RSw9Ya1xCzihpDrYDsA,413
+dkist_processing_ops/_version.py,sha256=Krs1A6GFKeSDM3HlmJuBHtdakqIKMggtBrGf0aTD85E,413
 dkist_processing_ops/dags/scale.py,sha256=We5TYjNhkJ-5ykfbrOMgjTpXdzOCkIeyKyA-40sU9r0,2312
 dkist_processing_ops/tasks/__init__.py,sha256=P81O9cg4dlBMqBTaWitdsAte68RsMtDlhV30JSZfXUY,107
-dkist_processing_ops/tasks/read_memory_leak.py,sha256=HQzzQLn_45EauauLCuBYeXZYav5KiXekgKhEzkbSwGM,3559
+dkist_processing_ops/tasks/read_memory_leak.py,sha256=yQeWMjUm0YrgyRtC4XGuExAVDUgCjNetDB45M0jz4Ug,3649
 dkist_processing_ops/tasks/wait.py,sha256=uObka-nH1dKPcGBDsp3t2RCtTV2F1kksM0V-lRewFuY,273
 dkist_processing_ops/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dkist_processing_ops/tests/test_workflows.py,sha256=Ch_8BlGeQyPJU_9hB_GOncwW-SoZwpRUVKMOEz0RQZk,285
 dkist_processing_ops/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dkist_processing_ops/workflows/memory_leak.py,sha256=lYXAYyJVjXif3Os9xPDp-bPTG_je6HOw1uvRJ4WMUi4,758
 dkist_processing_ops/workflows/smoke.py,sha256=ofXu0_iYF6L3zQy-BOVvS5VdzKhmXs1gyugqMNkd-GM,878
-dkist_processing_ops-1.6.15.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
-dkist_processing_ops-1.6.15.dist-info/METADATA,sha256=Q1K-2rhN2vtaVdNEfFLb_UrzFw7p60mzgzgMZaOkCtA,1500
-dkist_processing_ops-1.6.15.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-dkist_processing_ops-1.6.15.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
-dkist_processing_ops-1.6.15.dist-info/RECORD,,
+dkist_processing_ops-1.6.16.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
+dkist_processing_ops-1.6.16.dist-info/METADATA,sha256=r2cyD9NBVw7w2D4O7RrOg4maweWbprBmSqMZKxEoYFw,1500
+dkist_processing_ops-1.6.16.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+dkist_processing_ops-1.6.16.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
+dkist_processing_ops-1.6.16.dist-info/RECORD,,