dkist-processing-ops 1.6.14__py3-none-any.whl → 1.6.16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of dkist-processing-ops might be problematic. Click here for more details.

Files changed (7) hide show
  1. dkist_processing_ops/_version.py +2 -2
  2. dkist_processing_ops/tasks/read_memory_leak.py +47 -26
  3. {dkist_processing_ops-1.6.14.dist-info → dkist_processing_ops-1.6.16.dist-info}/METADATA +1 -1
  4. {dkist_processing_ops-1.6.14.dist-info → dkist_processing_ops-1.6.16.dist-info}/RECORD +7 -7
  5. {dkist_processing_ops-1.6.14.dist-info → dkist_processing_ops-1.6.16.dist-info}/LICENSE.rst +0 -0
  6. {dkist_processing_ops-1.6.14.dist-info → dkist_processing_ops-1.6.16.dist-info}/WHEEL +0 -0
  7. {dkist_processing_ops-1.6.14.dist-info → dkist_processing_ops-1.6.16.dist-info}/top_level.txt +0 -0
dkist_processing_ops/_version.py CHANGED
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '1.6.14'
16
- __version_tuple__ = version_tuple = (1, 6, 14)
15
+ __version__ = version = '1.6.16'
16
+ __version_tuple__ = version_tuple = (1, 6, 16)
dkist_processing_ops/tasks/read_memory_leak.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import gc
2
+ import logging
2
3
  from abc import ABC
3
4
  from pathlib import Path
4
5
 
@@ -10,19 +11,24 @@ from dkist_processing_common.models.tags import Tag
10
11
  from dkist_processing_common.tasks import WorkflowTaskBase
11
12
 
12
13
 
13
- def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
14
- data = np.random.rand(4096, 4096)
15
- hdu = fits.CompImageHDU(data)
16
- return hdu
14
+ # def generated_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
15
+ # data = np.random.rand(4096, 4096)
16
+ # hdu = fits.CompImageHDU(data)
17
+ # return hdu
17
18
 
18
19
 
19
- def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
20
- hdul = fits.open(path)
21
- hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
22
- hdul.close()
23
- del hdul
24
- gc.collect()
25
- return hdu
20
+ def fits_bytes_decoder(path: Path) -> bytes:
21
+ with open(path, "rb") as f:
22
+ return f.read()
23
+
24
+
25
+ # def fits_garbage_collect_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
26
+ # hdul = fits.open(path)
27
+ # hdu = fits.CompImageHDU(header=hdul[1].header, data=hdul[1].data)
28
+ # hdul.close()
29
+ # del hdul
30
+ # gc.collect()
31
+ # return hdu
26
32
 
27
33
 
28
34
  class FitsDataRead(WorkflowTaskBase, ABC):
@@ -31,26 +37,41 @@ class FitsDataRead(WorkflowTaskBase, ABC):
31
37
  return self.metadata_store_recipe_run_configuration().get("run_type", "file_read")
32
38
 
33
39
  def run(self) -> None:
34
- if self.run_type == "garbage_collect_read":
35
- hdus = self.read(
36
- tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
37
- )
38
- for hdu in hdus:
39
- h = hdu.header
40
- d = hdu.data
41
-
42
- if self.run_type == "garbage_collect_task":
43
- filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
44
- for filepath in filepaths:
45
- hdu = fits_garbage_collect_hdu_decoder(filepath)
46
- h = hdu.header
47
- d = hdu.data
40
+ if self.run_type == "bytes_read":
41
+ bytes_objects = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_bytes_decoder)
42
+ for i, byte_object in enumerate(bytes_objects):
43
+ pass
48
44
 
49
45
  if self.run_type == "file_read":
50
46
  hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=fits_hdu_decoder)
51
- for hdu in hdus:
47
+ for i, hdu in enumerate(hdus):
52
48
  h = hdu.header
53
49
  d = hdu.data
50
+
51
+ # if self.run_type == "garbage_collect_read":
52
+ # hdus = self.read(
53
+ # tags=[Tag.input(), Tag.frame()], decoder=fits_garbage_collect_hdu_decoder
54
+ # )
55
+ # for i, hdu in enumerate(hdus):
56
+ # h = hdu.header
57
+ # d = hdu.data
58
+ # if i % 50 == 0:
59
+ # cur, peak = tracemalloc.get_traced_memory()
60
+ # logging.info(
61
+ # f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
62
+ # )
63
+ #
64
+ # if self.run_type == "garbage_collect_task":
65
+ # filepaths = self.read(tags=[Tag.input(), Tag.frame()], decoder=path_decoder)
66
+ # for i, filepath in enumerate(filepaths):
67
+ # hdu = fits_garbage_collect_hdu_decoder(filepath)
68
+ # h = hdu.header
69
+ # d = hdu.data
70
+ # if i % 50 == 0:
71
+ # cur, peak = tracemalloc.get_traced_memory()
72
+ # logging.info(
73
+ # f"After {i} files current memory usage is {cur / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB"
74
+ # )
54
75
  #
55
76
  # if self.run_type == "generated_read":
56
77
  # hdus = self.read(tags=[Tag.input(), Tag.frame()], decoder=generated_hdu_decoder)
{dkist_processing_ops-1.6.14.dist-info → dkist_processing_ops-1.6.16.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dkist-processing-ops
3
- Version: 1.6.14
3
+ Version: 1.6.16
4
4
  Summary: Automated Processing smoke test and operations workflows
5
5
  Author-email: NSO / AURA <dkistdc@nso.edu>
6
6
  License: BSD 3-Clause
{dkist_processing_ops-1.6.14.dist-info → dkist_processing_ops-1.6.16.dist-info}/RECORD RENAMED
@@ -1,16 +1,16 @@
1
1
  dkist_processing_ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- dkist_processing_ops/_version.py,sha256=ENoeph2R-6hTwDBnYv-6HGbXQQBxCuxRZdUsEOyJGbk,413
2
+ dkist_processing_ops/_version.py,sha256=Krs1A6GFKeSDM3HlmJuBHtdakqIKMggtBrGf0aTD85E,413
3
3
  dkist_processing_ops/dags/scale.py,sha256=We5TYjNhkJ-5ykfbrOMgjTpXdzOCkIeyKyA-40sU9r0,2312
4
4
  dkist_processing_ops/tasks/__init__.py,sha256=P81O9cg4dlBMqBTaWitdsAte68RsMtDlhV30JSZfXUY,107
5
- dkist_processing_ops/tasks/read_memory_leak.py,sha256=WuwS7Jb-wUBqGsI6Qn8DvGvDHZDsj6jVyRp_vKf2QV4,2657
5
+ dkist_processing_ops/tasks/read_memory_leak.py,sha256=yQeWMjUm0YrgyRtC4XGuExAVDUgCjNetDB45M0jz4Ug,3649
6
6
  dkist_processing_ops/tasks/wait.py,sha256=uObka-nH1dKPcGBDsp3t2RCtTV2F1kksM0V-lRewFuY,273
7
7
  dkist_processing_ops/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
8
  dkist_processing_ops/tests/test_workflows.py,sha256=Ch_8BlGeQyPJU_9hB_GOncwW-SoZwpRUVKMOEz0RQZk,285
9
9
  dkist_processing_ops/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
10
  dkist_processing_ops/workflows/memory_leak.py,sha256=lYXAYyJVjXif3Os9xPDp-bPTG_je6HOw1uvRJ4WMUi4,758
11
11
  dkist_processing_ops/workflows/smoke.py,sha256=ofXu0_iYF6L3zQy-BOVvS5VdzKhmXs1gyugqMNkd-GM,878
12
- dkist_processing_ops-1.6.14.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
13
- dkist_processing_ops-1.6.14.dist-info/METADATA,sha256=KUW80lrBGyblfl5r9uSPaWF3ohu26KT2rSlQRV7qleA,1500
14
- dkist_processing_ops-1.6.14.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
15
- dkist_processing_ops-1.6.14.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
16
- dkist_processing_ops-1.6.14.dist-info/RECORD,,
12
+ dkist_processing_ops-1.6.16.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
13
+ dkist_processing_ops-1.6.16.dist-info/METADATA,sha256=r2cyD9NBVw7w2D4O7RrOg4maweWbprBmSqMZKxEoYFw,1500
14
+ dkist_processing_ops-1.6.16.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
15
+ dkist_processing_ops-1.6.16.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
16
+ dkist_processing_ops-1.6.16.dist-info/RECORD,,