dkist-processing-ops 1.6.14__tar.gz → 1.6.15__tar.gz

{dkist_processing_ops-1.6.14 → dkist_processing_ops-1.6.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.14
+Version: 1.6.15
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause

{dkist_processing_ops-1.6.14 → dkist_processing_ops-1.6.15}/dkist_processing_ops/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.14'
-__version_tuple__ = version_tuple = (1, 6, 14)
+__version__ = version = '1.6.15'
+__version_tuple__ = version_tuple = (1, 6, 15)

{dkist_processing_ops-1.6.14 → dkist_processing_ops-1.6.15}/dkist_processing_ops/tasks/read_memory_leak.py

@@ -1,4 +1,6 @@
 import gc
+import logging
+import tracemalloc
 from abc import ABC
 from pathlib import Path
 
@@ -31,26 +33,42 @@ class FitsDataRead(WorkflowTaskBase, ABC):
         return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
 
     def run(self) -> None:
+        tracemalloc.start()
         if self.run_type == "garbage_collect_read":
             hdus = self.read(
                 tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
             )
-            for hdu in hdus:
+            for i, hdu in enumerate(hdus):
                 h = hdu.header
                 d = hdu.data
+                if i % 50 == 0:
+                    cur, peak = tracemalloc.get_traced_memory()
+                    logging.info(
+                        f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
+                    )
 
         if self.run_type == "garbage_collect_task":
             filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
-            for filepath in filepaths:
+            for i, filepath in enumerate(filepaths):
                 hdu = fits_garbage_collect_hdu_decoder(filepath)
                 h = hdu.header
                 d = hdu.data
+                if i % 50 == 0:
+                    cur, peak = tracemalloc.get_traced_memory()
+                    logging.info(
+                        f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
+                    )
 
         if self.run_type == "file_read":
             hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
-            for hdu in hdus:
+            for i, hdu in enumerate(hdus):
                 h = hdu.header
                 d = hdu.data
+                if i % 50 == 0:
+                    cur, peak = tracemalloc.get_traced_memory()
+                    logging.info(
+                        f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
+                    )
         #
         # if self.run_type == "generated_read":
         #     hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)

{dkist_processing_ops-1.6.14 → dkist_processing_ops-1.6.15}/dkist_processing_ops.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.6.14
+Version: 1.6.15
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause