dkist-processing-ops 1.6.13__py3-none-any.whl → 1.6.15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of dkist-processing-ops might be problematic. Click here for more details.

Files changed (7) hide show
  1. dkist_processing_ops/_version.py +2 -2
  2. dkist_processing_ops/tasks/read_memory_leak.py +60 -17
  3. {dkist_processing_ops-1.6.13.dist-info → dkist_processing_ops-1.6.15.dist-info}/METADATA +1 -1
  4. {dkist_processing_ops-1.6.13.dist-info → dkist_processing_ops-1.6.15.dist-info}/RECORD +7 -7
  5. {dkist_processing_ops-1.6.13.dist-info → dkist_processing_ops-1.6.15.dist-info}/LICENSE.rst +0 -0
  6. {dkist_processing_ops-1.6.13.dist-info → dkist_processing_ops-1.6.15.dist-info}/WHEEL +0 -0
  7. {dkist_processing_ops-1.6.13.dist-info → dkist_processing_ops-1.6.15.dist-info}/top_level.txt +0 -0
dkist_processing_ops/_version.py CHANGED
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '1.6.13'
16
- __version_tuple__ = version_tuple = (1, 6, 13)
15
+ __version__ = version = '1.6.15'
16
+ __version_tuple__ = version_tuple = (1, 6, 15)
dkist_processing_ops/tasks/read_memory_leak.py CHANGED
@@ -1,3 +1,6 @@
1
+ import gc
2
+ import logging
3
+ import tracemalloc
1
4
  from abc import ABC
2
5
  from pathlib import Path
3
6
 
@@ -15,35 +18,75 @@ def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
15
18
  return hdu
16
19
 
17
20
 
21
+ def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
22
+ hdul = fits.open(path)
23
+ hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
24
+ hdul.close()
25
+ del hdul
26
+ gc.collect()
27
+ return hdu
28
+
29
+
18
30
  class FitsDataRead(WorkflowTaskBase, ABC):
19
31
  @property
20
32
  def run_type(self):
21
33
  return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
22
34
 
23
35
  def run(self) -> None:
24
- if self.run_type == "file_read":
25
- hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
26
- for hdu in hdus:
36
+ tracemalloc.start()
37
+ if self.run_type == "garbage_collect_read":
38
+ hdus = self.read(
39
+ tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
40
+ )
41
+ for i, hdu in enumerate(hdus):
27
42
  h = hdu.header
28
43
  d = hdu.data
44
+ if i % 50 == 0:
45
+ cur, peak = tracemalloc.get_traced_memory()
46
+ logging.info(
47
+ f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
48
+ )
29
49
 
30
- if self.run_type == "generated_read":
31
- hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)
32
- for hdu in hdus:
33
- h = hdu.header
34
- d = hdu.data
35
-
36
- if self.run_type == "file_task":
50
+ if self.run_type == "garbage_collect_task":
37
51
  filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
38
- for filepath in filepaths:
39
- hdu = fits.open(filepath)[1]
52
+ for i, filepath in enumerate(filepaths):
53
+ hdu = fits_garbage_collect_hdu_decoder(filepath)
40
54
  h = hdu.header
41
55
  d = hdu.data
56
+ if i % 50 == 0:
57
+ cur, peak = tracemalloc.get_traced_memory()
58
+ logging.info(
59
+ f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
60
+ )
42
61
 
43
- if self.run_type == "generated_task":
44
- filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
45
- for filepath in filepaths:
46
- data = np.random.rand(4096, 4096)
47
- hdu = fits.CompImageHDU(data)
62
+ if self.run_type == "file_read":
63
+ hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
64
+ for i, hdu in enumerate(hdus):
48
65
  h = hdu.header
49
66
  d = hdu.data
67
+ if i % 50 == 0:
68
+ cur, peak = tracemalloc.get_traced_memory()
69
+ logging.info(
70
+ f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
71
+ )
72
+ #
73
+ # if self.run_type == "generated_read":
74
+ # hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)
75
+ # for hdu in hdus:
76
+ # h = hdu.header
77
+ # d = hdu.data
78
+ #
79
+ # if self.run_type == "file_task":
80
+ # filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
81
+ # for filepath in filepaths:
82
+ # hdu = fits.open(filepath)[1]
83
+ # h = hdu.header
84
+ # d = hdu.data
85
+ #
86
+ # if self.run_type == "generated_task":
87
+ # filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
88
+ # for filepath in filepaths:
89
+ # data = np.random.rand(4096, 4096)
90
+ # hdu = fits.CompImageHDU(data)
91
+ # h = hdu.header
92
+ # d = hdu.data
{dkist_processing_ops-1.6.13.dist-info → dkist_processing_ops-1.6.15.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dkist-processing-ops
3
- Version: 1.6.13
3
+ Version: 1.6.15
4
4
  Summary: Automated Processing smoke test and operations workflows
5
5
  Author-email: NSO / AURA <dkistdc@nso.edu>
6
6
  License: BSD 3-Clause
{dkist_processing_ops-1.6.13.dist-info → dkist_processing_ops-1.6.15.dist-info}/RECORD RENAMED
@@ -1,16 +1,16 @@
1
1
  dkist_processing_ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- dkist_processing_ops/_version.py,sha256=F6yOQF2i7n8OMZVy5diTWAk2RoBPwyUW2P5AsaXwnHo,413
2
+ dkist_processing_ops/_version.py,sha256=G6p4cF-LZDRxAX98giKjw_I7RSw9Ya1xCzihpDrYDsA,413
3
3
  dkist_processing_ops/dags/scale.py,sha256=We5TYjNhkJ-5ykfbrOMgjTpXdzOCkIeyKyA-40sU9r0,2312
4
4
  dkist_processing_ops/tasks/__init__.py,sha256=P81O9cg4dlBMqBTaWitdsAte68RsMtDlhV30JSZfXUY,107
5
- dkist_processing_ops/tasks/read_memory_leak.py,sha256=vlw-ao2GYObw3tCJGZDXmVbF5TPwEAQiOVRuukhimxk,1752
5
+ dkist_processing_ops/tasks/read_memory_leak.py,sha256=HQzzQLn_45EauauLCuBYeXZYav5KiXekgKhEzkbSwGM,3559
6
6
  dkist_processing_ops/tasks/wait.py,sha256=uObka-nH1dKPcGBDsp3t2RCtTV2F1kksM0V-lRewFuY,273
7
7
  dkist_processing_ops/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
8
  dkist_processing_ops/tests/test_workflows.py,sha256=Ch_8BlGeQyPJU_9hB_GOncwW-SoZwpRUVKMOEz0RQZk,285
9
9
  dkist_processing_ops/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
10
  dkist_processing_ops/workflows/memory_leak.py,sha256=lYXAYyJVjXif3Os9xPDp-bPTG_je6HOw1uvRJ4WMUi4,758
11
11
  dkist_processing_ops/workflows/smoke.py,sha256=ofXu0_iYF6L3zQy-BOVvS5VdzKhmXs1gyugqMNkd-GM,878
12
- dkist_processing_ops-1.6.13.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
13
- dkist_processing_ops-1.6.13.dist-info/METADATA,sha256=3QbaFLhReEeqB_QEdo9j9dfQXA69FEAqOng4BnQAXl4,1500
14
- dkist_processing_ops-1.6.13.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
15
- dkist_processing_ops-1.6.13.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
16
- dkist_processing_ops-1.6.13.dist-info/RECORD,,
12
+ dkist_processing_ops-1.6.15.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
13
+ dkist_processing_ops-1.6.15.dist-info/METADATA,sha256=Q1K-2rhN2vtaVdNEfFLb_UrzFw7p60mzgzgMZaOkCtA,1500
14
+ dkist_processing_ops-1.6.15.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
15
+ dkist_processing_ops-1.6.15.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
16
+ dkist_processing_ops-1.6.15.dist-info/RECORD,,