dkist-processing-ops 1.6.15__py3-none-any.whl → 1.6.17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of dkist-processing-ops might be problematic. Click here for more details.

Files changed (7) hide show
  1. dkist_processing_ops/_version.py +2 -2
  2. dkist_processing_ops/tasks/read_memory_leak.py +58 -47
  3. {dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.17.dist-info}/METADATA +1 -1
  4. {dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.17.dist-info}/RECORD +7 -7
  5. {dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.17.dist-info}/LICENSE.rst +0 -0
  6. {dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.17.dist-info}/WHEEL +0 -0
  7. {dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.17.dist-info}/top_level.txt +0 -0
dkist_processing_ops/_version.py CHANGED
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '1.6.15'
16
- __version_tuple__ = version_tuple = (1, 6, 15)
15
+ __version__ = version = '1.6.17'
16
+ __version_tuple__ = version_tuple = (1, 6, 17)
dkist_processing_ops/tasks/read_memory_leak.py CHANGED
@@ -1,6 +1,5 @@
1
1
  import gc
2
2
  import logging
3
- import tracemalloc
4
3
  from abc import ABC
5
4
  from pathlib import Path
6
5
 
@@ -12,19 +11,24 @@ from dkist_processing_common.models.tags import Tag
12
11
  from dkist_processing_common.tasks import WorkflowTaskBase
13
12
 
14
13
 
15
- def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
16
- data = np.random.rand(4096, 4096)
17
- hdu = fits.CompImageHDU(data)
18
- return hdu
14
+ # def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
15
+ # data = np.random.rand(4096, 4096)
16
+ # hdu = fits.CompImageHDU(data)
17
+ # return hdu
19
18
 
20
19
 
21
- def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
22
- hdul = fits.open(path)
23
- hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
24
- hdul.close()
25
- del hdul
26
- gc.collect()
27
- return hdu
20
+ def fits_bytes_decoder(path: Path) -> bytes:
21
+ with open(path, "rb") as f:
22
+ return f.read()
23
+
24
+
25
+ # def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
26
+ # hdul = fits.open(path)
27
+ # hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
28
+ # hdul.close()
29
+ # del hdul
30
+ # gc.collect()
31
+ # return hdu
28
32
 
29
33
 
30
34
  class FitsDataRead(WorkflowTaskBase, ABC):
@@ -33,42 +37,54 @@ class FitsDataRead(WorkflowTaskBase, ABC):
33
37
  return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
34
38
 
35
39
  def run(self) -> None:
36
- tracemalloc.start()
37
- if self.run_type == "garbage_collect_read":
38
- hdus = self.read(
39
- tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
40
- )
41
- for i, hdu in enumerate(hdus):
42
- h = hdu.header
43
- d = hdu.data
44
- if i % 50 == 0:
45
- cur, peak = tracemalloc.get_traced_memory()
46
- logging.info(
47
- f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
48
- )
40
+ if self.run_type == "bytes_read":
41
+ bytes_objects = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_bytes_decoder)
42
+ for i, byte_object in enumerate(bytes_objects):
43
+ pass
49
44
 
50
- if self.run_type == "garbage_collect_task":
45
+ if self.run_type == "bytes_task":
51
46
  filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
52
- for i, filepath in enumerate(filepaths):
53
- hdu = fits_garbage_collect_hdu_decoder(filepath)
54
- h = hdu.header
55
- d = hdu.data
56
- if i % 50 == 0:
57
- cur, peak = tracemalloc.get_traced_memory()
58
- logging.info(
59
- f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
60
- )
47
+ for filepath in filepaths:
48
+ with open(filepath, "rb") as f:
49
+ byte_object = f.read()
61
50
 
62
51
  if self.run_type == "file_read":
63
52
  hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
64
- for i, hdu in enumerate(hdus):
53
+ for hdu in hdus:
65
54
  h = hdu.header
66
55
  d = hdu.data
67
- if i % 50 == 0:
68
- cur, peak = tracemalloc.get_traced_memory()
69
- logging.info(
70
- f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
71
- )
56
+
57
+ if self.run_type == "file_task":
58
+ filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
59
+ for filepath in filepaths:
60
+ hdu = fits.open(filepath)[1]
61
+ h = hdu.header
62
+ d = hdu.data
63
+
64
+ # if self.run_type == "garbage_collect_read":
65
+ # hdus = self.read(
66
+ # tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
67
+ # )
68
+ # for i, hdu in enumerate(hdus):
69
+ # h = hdu.header
70
+ # d = hdu.data
71
+ # if i % 50 == 0:
72
+ # cur, peak = tracemalloc.get_traced_memory()
73
+ # logging.info(
74
+ # f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
75
+ # )
76
+ #
77
+ # if self.run_type == "garbage_collect_task":
78
+ # filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
79
+ # for i, filepath in enumerate(filepaths):
80
+ # hdu = fits_garbage_collect_hdu_decoder(filepath)
81
+ # h = hdu.header
82
+ # d = hdu.data
83
+ # if i % 50 == 0:
84
+ # cur, peak = tracemalloc.get_traced_memory()
85
+ # logging.info(
86
+ # f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
87
+ # )
72
88
  #
73
89
  # if self.run_type == "generated_read":
74
90
  # hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)
@@ -76,12 +92,7 @@ class FitsDataRead(WorkflowTaskBase, ABC):
76
92
  # h = hdu.header
77
93
  # d = hdu.data
78
94
  #
79
- # if self.run_type == "file_task":
80
- # filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
81
- # for filepath in filepaths:
82
- # hdu = fits.open(filepath)[1]
83
- # h = hdu.header
84
- # d = hdu.data
95
+
85
96
  #
86
97
  # if self.run_type == "generated_task":
87
98
  # filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
{dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.17.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dkist-processing-ops
3
- Version: 1.6.15
3
+ Version: 1.6.17
4
4
  Summary: Automated Processing smoke test and operations workflows
5
5
  Author-email: NSO / AURA <dkistdc@nso.edu>
6
6
  License: BSD 3-Clause
{dkist_processing_ops-1.6.15.dist-info → dkist_processing_ops-1.6.17.dist-info}/RECORD RENAMED
@@ -1,16 +1,16 @@
1
1
  dkist_processing_ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- dkist_processing_ops/_version.py,sha256=G6p4cF-LZDRxAX98giKjw_I7RSw9Ya1xCzihpDrYDsA,413
2
+ dkist_processing_ops/_version.py,sha256=bUNPwAsnC5Y4wwmtKw5lZfH0PuKwJg_38vSI9Pv11Qo,413
3
3
  dkist_processing_ops/dags/scale.py,sha256=We5TYjNhkJ-5ykfbrOMgjTpXdzOCkIeyKyA-40sU9r0,2312
4
4
  dkist_processing_ops/tasks/__init__.py,sha256=P81O9cg4dlBMqBTaWitdsAte68RsMtDlhV30JSZfXUY,107
5
- dkist_processing_ops/tasks/read_memory_leak.py,sha256=HQzzQLn_45EauauLCuBYeXZYav5KiXekgKhEzkbSwGM,3559
5
+ dkist_processing_ops/tasks/read_memory_leak.py,sha256=p_jIqhx3-en8X9QrBkQO9tH0IIeSfCtb369VGnU_f3I,3887
6
6
  dkist_processing_ops/tasks/wait.py,sha256=uObka-nH1dKPcGBDsp3t2RCtTV2F1kksM0V-lRewFuY,273
7
7
  dkist_processing_ops/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
8
  dkist_processing_ops/tests/test_workflows.py,sha256=Ch_8BlGeQyPJU_9hB_GOncwW-SoZwpRUVKMOEz0RQZk,285
9
9
  dkist_processing_ops/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
10
  dkist_processing_ops/workflows/memory_leak.py,sha256=lYXAYyJVjXif3Os9xPDp-bPTG_je6HOw1uvRJ4WMUi4,758
11
11
  dkist_processing_ops/workflows/smoke.py,sha256=ofXu0_iYF6L3zQy-BOVvS5VdzKhmXs1gyugqMNkd-GM,878
12
- dkist_processing_ops-1.6.15.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
13
- dkist_processing_ops-1.6.15.dist-info/METADATA,sha256=Q1K-2rhN2vtaVdNEfFLb_UrzFw7p60mzgzgMZaOkCtA,1500
14
- dkist_processing_ops-1.6.15.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
15
- dkist_processing_ops-1.6.15.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
16
- dkist_processing_ops-1.6.15.dist-info/RECORD,,
12
+ dkist_processing_ops-1.6.17.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
13
+ dkist_processing_ops-1.6.17.dist-info/METADATA,sha256=d1tDJKVwe12ytA2PRpP9PRX_csWjF4jefNtZngzwqoY,1500
14
+ dkist_processing_ops-1.6.17.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
15
+ dkist_processing_ops-1.6.17.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
16
+ dkist_processing_ops-1.6.17.dist-info/RECORD,,