tomwer 1.3.5__py3-none-any.whl → 1.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. orangecontrib/tomwer/test/TestAcquisition.py +246 -0
  2. orangecontrib/tomwer/widgets/cluster/test/test_future_supervisorow.py +87 -0
  3. orangecontrib/tomwer/widgets/cluster/test/test_slurm_clusterow.py +67 -0
  4. orangecontrib/tomwer/widgets/control/test/test_advancement.py +51 -0
  5. orangecontrib/tomwer/widgets/control/test/test_data_validator.py +55 -0
  6. orangecontrib/tomwer/widgets/control/test/test_datadiscovery.py +131 -0
  7. orangecontrib/tomwer/widgets/control/test/test_datalist.py +70 -0
  8. orangecontrib/tomwer/widgets/control/test/test_datalistener.py +137 -0
  9. orangecontrib/tomwer/widgets/control/test/test_dataselector.py +95 -0
  10. orangecontrib/tomwer/widgets/control/test/test_datawatcher.py +436 -0
  11. orangecontrib/tomwer/widgets/control/test/test_emailow.py +29 -0
  12. orangecontrib/tomwer/widgets/control/test/test_notifier.py +51 -0
  13. orangecontrib/tomwer/widgets/control/test/test_nxtomo_concatenate_ow.py +64 -0
  14. orangecontrib/tomwer/widgets/control/test/test_nxtomomill.py +160 -0
  15. orangecontrib/tomwer/widgets/control/test/test_reduce_dark_flat_selector.py +40 -0
  16. orangecontrib/tomwer/widgets/control/test/test_singletomoobj.py +40 -0
  17. orangecontrib/tomwer/widgets/control/test/test_timerow.py +51 -0
  18. orangecontrib/tomwer/widgets/control/test/test_tomoobj_serie.py +96 -0
  19. orangecontrib/tomwer/widgets/control/test/test_volume_selector.py +69 -0
  20. orangecontrib/tomwer/widgets/control/test/test_volumesymlink.py +51 -0
  21. orangecontrib/tomwer/widgets/debugtools/test/test_dataset_generator.py +57 -0
  22. orangecontrib/tomwer/widgets/debugtools/test/test_object_inspector.py +62 -0
  23. orangecontrib/tomwer/widgets/other/test/test_pythonscript.py +31 -0
  24. orangecontrib/tomwer/widgets/reconstruction/test/test_axis.py +224 -0
  25. orangecontrib/tomwer/widgets/reconstruction/test/test_cast_volumeow.py +85 -0
  26. orangecontrib/tomwer/widgets/reconstruction/test/test_dark_refs_widget.py +136 -0
  27. orangecontrib/tomwer/widgets/reconstruction/test/test_delta_beta_selector.py +15 -0
  28. orangecontrib/tomwer/widgets/reconstruction/test/test_i_norm.py +226 -0
  29. orangecontrib/tomwer/widgets/reconstruction/test/test_nabu_helical_prepare_weights_double.py +20 -0
  30. orangecontrib/tomwer/widgets/reconstruction/test/test_nabu_volume.py +100 -0
  31. orangecontrib/tomwer/widgets/reconstruction/test/test_nabu_widget.py +107 -0
  32. orangecontrib/tomwer/widgets/reconstruction/test/test_sa_delta_beta.py +194 -0
  33. orangecontrib/tomwer/widgets/reconstruction/test/test_saaxis.py +220 -0
  34. orangecontrib/tomwer/widgets/stitching/test/test_zstitching.py +308 -0
  35. orangecontrib/tomwer/widgets/test/test_conditions.py +111 -0
  36. orangecontrib/tomwer/widgets/test/test_darkref.py +251 -0
  37. orangecontrib/tomwer/widgets/test/test_foldertransfert.py +131 -0
  38. orangecontrib/tomwer/widgets/visualization/test/test_dataviewerow.py +83 -0
  39. orangecontrib/tomwer/widgets/visualization/test/test_diffviewerow.py +65 -0
  40. orangecontrib/tomwer/widgets/visualization/test/test_live_sliceow.py +63 -0
  41. orangecontrib/tomwer/widgets/visualization/test/test_nxtomo_metadata_viewer.py +29 -0
  42. orangecontrib/tomwer/widgets/visualization/test/test_radio_stackow.py +56 -0
  43. orangecontrib/tomwer/widgets/visualization/test/test_sample_movedow.py +72 -0
  44. orangecontrib/tomwer/widgets/visualization/test/test_sinogram_viewerow.py +56 -0
  45. orangecontrib/tomwer/widgets/visualization/test/test_slice_stackow.py +57 -0
  46. orangecontrib/tomwer/widgets/visualization/test/test_volume_viewerow.py +57 -0
  47. tomwer/core/log/test/test_processlog.py +41 -0
  48. tomwer/core/process/control/datalistener/datalistener.py +11 -11
  49. tomwer/core/process/edit/test/test_darkflatpatch.py +269 -0
  50. tomwer/core/process/edit/test/test_imagekey_editor.py +125 -0
  51. tomwer/core/process/icat/test/test_create_screenshots.py +98 -0
  52. tomwer/core/process/icat/test/test_gallery.py +170 -0
  53. tomwer/core/process/reconstruction/axis/axis.py +3 -3
  54. tomwer/core/process/reconstruction/darkref/darkrefscopy.py +3 -2
  55. tomwer/core/process/reconstruction/nabu/nabucommon.py +3 -4
  56. tomwer/core/process/reconstruction/nabu/nabuslices.py +4 -4
  57. tomwer/core/process/reconstruction/nabu/nabuvolume.py +2 -5
  58. tomwer/core/process/reconstruction/nabu/test/test_castvolume.py +143 -0
  59. tomwer/core/process/reconstruction/nabu/test/test_nabu_utils.py +203 -0
  60. tomwer/core/process/reconstruction/nabu/test/test_nabunormalization.py +222 -0
  61. tomwer/core/process/script/test/test_script.py +68 -0
  62. tomwer/core/process/stitching/test/test_metadataholder.py +17 -0
  63. tomwer/core/process/task.py +3 -2
  64. tomwer/core/process/test/test_data_transfer.py +4 -3
  65. tomwer/core/process/visualization/test/test_data_viewer.py +39 -0
  66. tomwer/core/process/visualization/test/test_diff_viewer.py +39 -0
  67. tomwer/core/process/visualization/test/test_image_stack_viewer.py +41 -0
  68. tomwer/core/process/visualization/test/test_radio_stack.py +39 -0
  69. tomwer/core/process/visualization/test/test_sample_moved.py +41 -0
  70. tomwer/core/process/visualization/test/test_sinogram_viewer.py +39 -0
  71. tomwer/core/process/visualization/test/test_slice_stack.py +39 -0
  72. tomwer/core/process/visualization/test/test_volume_viewer.py +39 -0
  73. tomwer/core/scan/blissscan.py +3 -3
  74. tomwer/core/scan/nxtomoscan.py +2 -2
  75. tomwer/core/scan/scanbase.py +5 -6
  76. tomwer/core/utils/test/test_image.py +30 -0
  77. tomwer/core/utils/test/test_nxtomo.py +66 -0
  78. tomwer/core/utils/test/test_scan_utils.py +46 -0
  79. tomwer/core/utils/test/test_time.py +6 -0
  80. tomwer/core/volume/test/test_volumes.py +21 -0
  81. tomwer/gui/control/reducedarkflatselector.py +2 -2
  82. tomwer/gui/control/serie/test/test_creator.py +451 -0
  83. tomwer/gui/control/serie/test/test_nxtomo_concatenate.py +21 -0
  84. tomwer/gui/edit/dkrfpatch.py +4 -4
  85. tomwer/gui/edit/nxtomowarmer.py +3 -2
  86. tomwer/gui/icat/test/test_create_screenshots_gui.py +23 -0
  87. tomwer/gui/icat/test/test_gallery_gui.py +37 -0
  88. tomwer/gui/imagefromfile.py +2 -2
  89. tomwer/gui/reconstruction/nabu/test/test_check.py +92 -0
  90. tomwer/gui/reconstruction/nabu/test/test_ctf.py +46 -0
  91. tomwer/gui/reconstruction/nabu/test/test_helical.py +21 -0
  92. tomwer/gui/reconstruction/nabu/test/test_nabu_preprocessing.py +81 -0
  93. tomwer/gui/reconstruction/normalization/test/test_intensity.py +119 -0
  94. tomwer/gui/stitching/config/tests/test_axisparams.py +25 -0
  95. tomwer/gui/stitching/tests/test_axis_ordered_list.py +21 -0
  96. tomwer/gui/stitching/tests/test_normalization.py +27 -0
  97. tomwer/gui/stitching/tests/test_preview.py +85 -0
  98. tomwer/gui/stitching/tests/test_stitching_raw.py +110 -0
  99. tomwer/gui/stitching/tests/test_z_stitching.py +67 -0
  100. tomwer/gui/stitching/tests/utils.py +79 -0
  101. tomwer/gui/stitching/z_stitching/tests/test_fine_estimation.py +35 -0
  102. tomwer/gui/stitching/z_stitching/tests/test_raw_estimation.py +215 -0
  103. tomwer/gui/stitching/z_stitching/tests/test_stitching_window.py +51 -0
  104. tomwer/gui/utils/test/test_completer.py +67 -0
  105. tomwer/gui/utils/test/test_line_selector.py +21 -0
  106. tomwer/gui/utils/test/test_splashscreen.py +8 -0
  107. tomwer/gui/utils/test/test_vignettes.py +68 -0
  108. tomwer/io/utils/h5pyutils.py +3 -7
  109. tomwer/io/utils/test/test_raw_and_processed_data.py +10 -0
  110. tomwer/io/utils/test/test_utils.py +92 -0
  111. tomwer/io/utils/utils.py +3 -3
  112. tomwer/synctools/stacks/reconstruction/castvolume.py +20 -5
  113. tomwer/tests/test_ewoks/test_conversion.py +104 -0
  114. tomwer/tests/test_ewoks/test_single_node_execution.py +112 -0
  115. tomwer/tests/test_ewoks/test_workflows.py +160 -0
  116. tomwer/version.py +1 -1
  117. {tomwer-1.3.5.dist-info → tomwer-1.3.7.dist-info}/METADATA +1 -1
  118. {tomwer-1.3.5.dist-info → tomwer-1.3.7.dist-info}/RECORD +124 -27
  119. /tomwer-1.3.5-py3.11-nspkg.pth → /tomwer-1.3.7-py3.11-nspkg.pth +0 -0
  120. {tomwer-1.3.5.dist-info → tomwer-1.3.7.dist-info}/LICENSE +0 -0
  121. {tomwer-1.3.5.dist-info → tomwer-1.3.7.dist-info}/WHEEL +0 -0
  122. {tomwer-1.3.5.dist-info → tomwer-1.3.7.dist-info}/entry_points.txt +0 -0
  123. {tomwer-1.3.5.dist-info → tomwer-1.3.7.dist-info}/namespace_packages.txt +0 -0
  124. {tomwer-1.3.5.dist-info → tomwer-1.3.7.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,170 @@
1
+ import os
2
+
3
+ import h5py
4
+ import numpy
5
+ import pytest
6
+ from silx.io.url import DataUrl
7
+
8
+ from tomwer.core.process.icat.gallery import (
9
+ SaveScreenshotsToGalleryTask,
10
+ IcatScreenshots,
11
+ deduce_dataset_gallery_location,
12
+ deduce_proposal_GALLERY_location,
13
+ select_screenshot_from_volume,
14
+ PROPOSAL_GALLERY_DIR_NAME,
15
+ )
16
+ from tomwer.core.scan.edfscan import EDFTomoScan
17
+ from tomwer.core.scan.nxtomoscan import NXtomoScan
18
+ from tomwer.core.volume.edfvolume import EDFVolume
19
+ from tomwer.core.volume.hdf5volume import HDF5Volume
20
+
21
+
22
+ def test_deduce_dataset_gallery_dir():
23
+ """test the deduce_gallery_dir function"""
24
+ assert (
25
+ deduce_dataset_gallery_location(
26
+ NXtomoScan(scan="/path/to/PROCESSED_DATA/my_scan.nx", entry="entry")
27
+ )
28
+ == "/path/to/PROCESSED_DATA/gallery"
29
+ )
30
+ assert (
31
+ deduce_dataset_gallery_location(
32
+ NXtomoScan(scan="/path/to/RAW_DATA/collection/my_scan.nx", entry="entry")
33
+ )
34
+ == "/path/to/PROCESSED_DATA/collection/gallery"
35
+ )
36
+ assert (
37
+ deduce_dataset_gallery_location(
38
+ NXtomoScan(scan="/any/random/path/my_scan.nx", entry="entry")
39
+ )
40
+ == "/any/random/path/gallery"
41
+ )
42
+ assert (
43
+ deduce_dataset_gallery_location(
44
+ EDFTomoScan(scan="/path/to/PROCESSED_DATA/dataset/toto")
45
+ )
46
+ == "/path/to/PROCESSED_DATA/dataset/toto/gallery"
47
+ )
48
+ assert (
49
+ deduce_dataset_gallery_location(EDFTomoScan(scan="/path/to/dataset/toto"))
50
+ == "/path/to/dataset/toto/gallery"
51
+ )
52
+
53
+
54
+ def test_deduce_proposal_gallery_dir():
55
+ """test the deduce_gallery_dir function"""
56
+ assert (
57
+ deduce_proposal_GALLERY_location(
58
+ NXtomoScan(scan="/path/to/PROCESSED_DATA/my_scan.nx", entry="entry")
59
+ )
60
+ == f"/path/to/{PROPOSAL_GALLERY_DIR_NAME}"
61
+ )
62
+ assert (
63
+ deduce_proposal_GALLERY_location(
64
+ NXtomoScan(scan="/path/to/PROCESSED_DATA/dataset/my_scan.nx", entry="entry")
65
+ )
66
+ == f"/path/to/{PROPOSAL_GALLERY_DIR_NAME}/dataset"
67
+ )
68
+ assert (
69
+ deduce_proposal_GALLERY_location(
70
+ NXtomoScan(scan="/any/random/path/my_scan.nx", entry="entry")
71
+ )
72
+ == f"/any/random/path/{PROPOSAL_GALLERY_DIR_NAME}"
73
+ )
74
+ assert (
75
+ deduce_proposal_GALLERY_location(
76
+ EDFTomoScan(scan="/path/to/PROCESSED_DATA/dataset/toto")
77
+ )
78
+ == "/path/to/GALLERY/dataset/toto"
79
+ )
80
+ assert (
81
+ deduce_proposal_GALLERY_location(EDFTomoScan(scan="/path/to/dataset/toto"))
82
+ == "/path/to/dataset/toto/GALLERY"
83
+ )
84
+
85
+
86
+ @pytest.mark.parametrize("MetaVolumeClass", (HDF5Volume, EDFVolume))
87
+ def test_select_screenshot_from_volume(MetaVolumeClass, tmp_path):
88
+ """test the 'select_screenshot_from_volume' function"""
89
+ output_dir = tmp_path / "volumes"
90
+
91
+ if MetaVolumeClass is HDF5Volume:
92
+ volume = MetaVolumeClass(
93
+ file_path=os.path.join(output_dir, "test.hdf5"),
94
+ data_path="data",
95
+ )
96
+ elif MetaVolumeClass is EDFVolume:
97
+ volume = MetaVolumeClass(folder=os.path.join(output_dir, "folder"))
98
+ else:
99
+ raise NotImplementedError
100
+
101
+ # test with 'standard' volume
102
+ ## setup the volume
103
+ volume.data = numpy.arange(1000000).reshape(100, 100, 100)
104
+ volume.save() # needed because brosing urls check if url exists or not...
105
+ ## make sure we get the expected number of screenshot
106
+ screenshots_as_url = select_screenshot_from_volume(volume=volume)
107
+ assert len(screenshots_as_url) == 3
108
+ ## test slices
109
+ for url in screenshots_as_url.values():
110
+ if isinstance(volume, HDF5Volume):
111
+ assert url.data_slice() in (33, 50, 66)
112
+ elif isinstance(volume, EDFVolume):
113
+ assert url.file_path().endswith(("33.edf", "50.edf", "66.edf"))
114
+ else:
115
+ raise NotImplementedError
116
+
117
+ # test with a single frame volume
118
+ volume.data = numpy.arange(100).reshape(1, 10, 10)
119
+ volume.overwrite = True
120
+ volume.save()
121
+ screenshots_as_url = select_screenshot_from_volume(volume=volume)
122
+ ## make sure we have a single screenshot in this case
123
+ screenshot_url = tuple(screenshots_as_url.values())[0]
124
+ if isinstance(volume, HDF5Volume):
125
+ assert screenshot_url.data_slice() == 0
126
+ assert len(screenshots_as_url) == 1
127
+ # warning: for now this doesn't work for EDF because the volume class will not remove any existing slices...
128
+ # which is a clear limitation. But not sure we should remove those either...
129
+ elif isinstance(volume, EDFVolume):
130
+ # warning: for now this doesn't work for EDF because the volume class will not remove any existing slices...
131
+ # which is a clear limitation. But not sure we should remove those either...
132
+ # so here we will still find 3 urls for the screenshot
133
+ pass
134
+ else:
135
+ raise NotImplementedError
136
+
137
+
138
+ def test_MoveScreenshotsToGalleryTask(tmp_path):
139
+ """test the 'MoveScreenshotsToGalleryTask' task"""
140
+ raw_data_dir = tmp_path / "raw_data"
141
+ raw_data_dir.mkdir()
142
+ screenshot_output_data_dir = tmp_path / "screenshot"
143
+ screenshot_output_data_dir.mkdir()
144
+
145
+ raw_data_file = raw_data_dir / "data.hdf5"
146
+ with h5py.File(raw_data_file, mode="w") as h5f:
147
+ h5f["data"] = numpy.random.random(100).reshape(10, 10)
148
+
149
+ screenshots = IcatScreenshots(
150
+ data_dir=str(screenshot_output_data_dir),
151
+ screenshots={
152
+ "my_screenshot": DataUrl(
153
+ file_path=raw_data_file,
154
+ data_path="data",
155
+ scheme="silx",
156
+ )
157
+ },
158
+ scan=None,
159
+ )
160
+
161
+ task = SaveScreenshotsToGalleryTask(
162
+ inputs={
163
+ "screenshots": screenshots,
164
+ "format": "png",
165
+ }
166
+ )
167
+ task.run()
168
+
169
+ expected_output_file = os.path.join(screenshot_output_data_dir, "my_screenshot.png")
170
+ assert os.path.exists(expected_output_file)
@@ -76,7 +76,7 @@ else:
76
76
  has_composite_cor_finder = True
77
77
  from silx.io.url import DataUrl
78
78
  from silx.io.utils import h5py_read_dataset
79
- from tomoscan.io import HDF5File, get_swmr_mode
79
+ from silx.io.utils import open as open_hdf5
80
80
 
81
81
  _logger = logging.getLogger(__name__)
82
82
  if not has_composite_cor_finder:
@@ -1164,7 +1164,7 @@ class AxisTask(
1164
1164
  :return:
1165
1165
  """
1166
1166
  if entry is None:
1167
- with HDF5File(process_file, "r", swmr=get_swmr_mode()) as h5f:
1167
+ with open_hdf5(process_file) as h5f:
1168
1168
  entries = AxisTask._get_process_nodes(root_node=h5f, process=AxisTask)
1169
1169
  if len(entries) == 0:
1170
1170
  _logger.info("unable to find a Axis process in %s" % process_file)
@@ -1175,7 +1175,7 @@ class AxisTask(
1175
1175
  entry = list(entries.keys())[0]
1176
1176
  _logger.info("take %s as default entry" % entry)
1177
1177
 
1178
- with HDF5File(process_file, "r", swmr=get_swmr_mode()) as h5f:
1178
+ with open_hdf5(process_file) as h5f:
1179
1179
  axis_nodes = AxisTask._get_process_nodes(
1180
1180
  root_node=h5f[entry], process=AxisTask
1181
1181
  )
@@ -39,6 +39,7 @@ from typing import Union
39
39
 
40
40
  from silx.io.dictdump import dicttoh5, h5todict
41
41
  from silx.io.url import DataUrl
42
+ from silx.io.utils import open as open_hdf5
42
43
  from tomoscan.esrf.scan.utils import (
43
44
  copy_h5_dict_darks_to,
44
45
  copy_h5_dict_flats_to,
@@ -297,7 +298,7 @@ class DarkRefsCopy(DarkRefsTask):
297
298
  if not os.path.exists(self._save_file):
298
299
  return False
299
300
  else:
300
- with HDF5File(self._save_file, mode="r") as h5f:
301
+ with open_hdf5(self._save_file) as h5f:
301
302
  return self._flats_url.data_path() in h5f
302
303
 
303
304
  def has_dark_stored(self) -> bool:
@@ -309,7 +310,7 @@ class DarkRefsCopy(DarkRefsTask):
309
310
  if not os.path.exists(self._save_file):
310
311
  return False
311
312
  else:
312
- with HDF5File(self._save_file, mode="r") as h5f:
313
+ with open_hdf5(self._save_file) as h5f:
313
314
  return self._darks_url.data_path() in h5f
314
315
 
315
316
  def contains_dark(self, scan: TomwerScanBase) -> bool:
@@ -43,9 +43,10 @@ import sys
43
43
 
44
44
  import numpy
45
45
  from silx.io.url import DataUrl
46
+ from silx.io.utils import open as open_hdf5
46
47
  from sluurp.executor import submit as submit_to_slurm_cluster
47
48
  from sluurp.job import SBatchScriptJob
48
- from tomoscan.io import HDF5File, get_swmr_mode
49
+ from tomoscan.io import HDF5File
49
50
  from tomoscan.normalization import Method as INormMethod
50
51
  from tomoscan.identifier import VolumeIdentifier
51
52
 
@@ -581,9 +582,7 @@ class _NabuBaseReconstructor:
581
582
  # if an url exists insure we can access it
582
583
  dataset_url = DataUrl(path=dataset_url)
583
584
  if os.path.exists(dataset_url.file_path()):
584
- with HDF5File(
585
- dataset_url.file_path(), mode="r", swmr=get_swmr_mode()
586
- ) as h5f:
585
+ with open_hdf5(dataset_url.file_path()) as h5f:
587
586
  if dataset_url.data_path() not in h5f:
588
587
  dataset_url = None
589
588
  else:
@@ -35,6 +35,7 @@ import os
35
35
  import gc
36
36
  from tomwer.io.utils import format_stderr_stdout
37
37
  from silx.utils.deprecation import deprecated, deprecated_warning
38
+ from silx.io.utils import open as open_hdf5
38
39
 
39
40
  from processview.core.manager.manager import ProcessManager, DatasetState
40
41
 
@@ -84,7 +85,6 @@ from processview.core.superviseprocess import SuperviseProcess
84
85
  from silx.io.dictdump import h5todict
85
86
  from silx.io.utils import h5py_read_dataset
86
87
  from silx.utils.enum import Enum as _Enum
87
- from tomoscan.io import HDF5File, get_swmr_mode
88
88
 
89
89
  from tomwer.core.process.task import Task
90
90
  from tomwer.core.scan.edfscan import EDFTomoScan
@@ -494,7 +494,7 @@ class NabuSlicesTask(
494
494
  :rtype:dict
495
495
  """
496
496
  if entry is None:
497
- with HDF5File(process_file, "r", swmr=get_swmr_mode()) as h5f:
497
+ with open_hdf5(process_file) as h5f:
498
498
  entries = NabuSlicesTask._get_process_nodes(
499
499
  root_node=h5f, process=NabuSlicesTask
500
500
  )
@@ -510,7 +510,7 @@ class NabuSlicesTask(
510
510
  configuration_path = None
511
511
  res = {}
512
512
 
513
- with HDF5File(process_file, "r", swmr=get_swmr_mode()) as h5f:
513
+ with open_hdf5(process_file) as h5f:
514
514
  nabu_nodes = NabuSlicesTask._get_process_nodes(
515
515
  root_node=h5f[entry], process=NabuSlicesTask
516
516
  )
@@ -845,7 +845,7 @@ class SingleSliceRunner(_NabuBaseReconstructor):
845
845
  basename, _ = os.path.splitext(scan.master_file)
846
846
  basename = os.path.basename(basename)
847
847
  try:
848
- with HDF5File(scan.master_file, mode="r", swmr=get_swmr_mode()) as h5f:
848
+ with open_hdf5(scan.master_file) as h5f:
849
849
  if len(h5f.keys()) > 1:
850
850
  # if there is more than one entry in the file append the entry name to the file basename
851
851
  basename = "_".join((basename, scan.entry.lstrip("/")))
@@ -43,10 +43,9 @@ from processview.core.manager.manager import ProcessManager, DatasetState
43
43
  from processview.core.superviseprocess import SuperviseProcess
44
44
 
45
45
  from silx.io.utils import h5py_read_dataset
46
+ from silx.io.utils import open as open_hdf5
46
47
  from tomwer.core.utils.deprecation import deprecated_warning
47
48
 
48
- from tomoscan.io import HDF5File, get_swmr_mode
49
-
50
49
  from tomwer.core.cluster.cluster import SlurmClusterConfiguration
51
50
  from tomwer.core.futureobject import FutureTomwerObject
52
51
  from tomwer.core.process.reconstruction.nabu.plane import NabuPlane
@@ -354,9 +353,7 @@ class VolumeRunner(_NabuBaseReconstructor):
354
353
  basename = os.path.basename(basename)
355
354
  try:
356
355
  # if there is more than one entry in the file append the entry name to the file basename
357
- with HDF5File(
358
- self.scan.master_file, mode="r", swmr=get_swmr_mode()
359
- ) as h5f:
356
+ with open_hdf5(self.scan.master_file) as h5f:
360
357
  if len(h5f.keys()) > 1:
361
358
  basename = "_".join((basename, self.scan.entry.strip("/")))
362
359
  except Exception:
@@ -0,0 +1,143 @@
1
+ # coding: utf-8
2
+ # /*##########################################################################
3
+ #
4
+ # Copyright (c) 2016-2017 European Synchrotron Radiation Facility
5
+ #
6
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ # of this software and associated documentation files (the "Software"), to deal
8
+ # in the Software without restriction, including without limitation the rights
9
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ # copies of the Software, and to permit persons to whom the Software is
11
+ # furnished to do so, subject to the following conditions:
12
+ #
13
+ # The above copyright notice and this permission notice shall be included in
14
+ # all copies or substantial portions of the Software.
15
+ #
16
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
+ # THE SOFTWARE.
23
+ #
24
+ # ###########################################################################*/
25
+
26
+ __authors__ = ["H. Payno"]
27
+ __license__ = "MIT"
28
+ __date__ = "05/07/2022"
29
+
30
+
31
+ import os
32
+
33
+ import numpy
34
+ import pytest
35
+
36
+ from tomwer.core.process.reconstruction.nabu.castvolume import CastVolumeTask
37
+ from tomwer.core.process.reconstruction.output import NabuOutputFileFormat
38
+ from tomwer.core.scan.nxtomoscan import NXtomoScan
39
+ from tomwer.core.volume.edfvolume import EDFVolume
40
+ from tomwer.core.volume.hdf5volume import HDF5Volume
41
+ from tomwer.core.volume.tiffvolume import TIFFVolume
42
+
43
+
44
+ def test_cast_volume_32bitstiff_to_16bits_tiff(tmp_path):
45
+ """test 32 bits tiffs to 16 bits works"""
46
+ dir_test = tmp_path / "test_dir"
47
+ dir_test.mkdir()
48
+ save_dir = tmp_path / "output_dir"
49
+ save_dir.mkdir()
50
+
51
+ vol_data = numpy.random.random(20 * 100 * 100) * 3600.4
52
+ vol_data = vol_data.reshape(20, 100, 100)
53
+ vol_data = vol_data.astype(numpy.float32)
54
+ volume = TIFFVolume(
55
+ folder=dir_test,
56
+ volume_basename="test_dir_vol_",
57
+ data=vol_data,
58
+ )
59
+ volume.save()
60
+
61
+ # test with output datadir as None
62
+ with pytest.raises(ValueError):
63
+ CastVolumeTask(
64
+ inputs={
65
+ "volume": volume,
66
+ "configuration": {
67
+ "output_file_path": None,
68
+ "output_file_format": NabuOutputFileFormat.EDF,
69
+ "output_data_type": numpy.uint16,
70
+ },
71
+ },
72
+ ).run()
73
+
74
+ # test providing save dir and tiff
75
+ assert len(os.listdir(save_dir)) == 0
76
+ task = CastVolumeTask(
77
+ inputs={
78
+ "volume": volume,
79
+ "configuration": {
80
+ "output_dir": str(save_dir),
81
+ "output_type": NabuOutputFileFormat.EDF,
82
+ "output_data_type": numpy.uint16,
83
+ },
84
+ }
85
+ )
86
+ task.run()
87
+
88
+ assert len(os.listdir(save_dir)) == 20, "no files have been generated"
89
+ assert task.outputs.volume.load_data().shape == (20, 100, 100)
90
+ assert task.outputs.volume.load_data().dtype == numpy.uint16
91
+
92
+ # test providing save dir, a scan and hdf5
93
+ scan = NXtomoScan(scan=os.sep.join([str(tmp_path), "scan"]), entry="entry0000")
94
+ task = CastVolumeTask(
95
+ inputs={
96
+ "volume": volume,
97
+ "configuration": {
98
+ "output_dir": str(save_dir),
99
+ "output_type": NabuOutputFileFormat.HDF5,
100
+ "output_data_type": numpy.float32,
101
+ },
102
+ "scan": scan,
103
+ },
104
+ )
105
+ task.run()
106
+ assert task.outputs.volume.load_data().shape == (20, 100, 100)
107
+ assert task.outputs.volume.load_data().dtype == numpy.float32
108
+
109
+ # test providing a volume as output volume
110
+ output_volume = HDF5Volume(
111
+ file_path=os.path.join(str(tmp_path), "my_volume.hdf5"), data_path="entry0002"
112
+ )
113
+ task = CastVolumeTask(
114
+ inputs={
115
+ "volume": volume,
116
+ "configuration": {
117
+ "output_dir": str(save_dir),
118
+ "output_type": NabuOutputFileFormat.HDF5,
119
+ "output_data_type": numpy.uint8,
120
+ },
121
+ "output_volume": output_volume,
122
+ },
123
+ )
124
+ task.run()
125
+ assert task.outputs.volume.load_data().shape == (20, 100, 100)
126
+ assert task.outputs.volume.load_data().dtype == numpy.uint8
127
+
128
+ # test providing a volume us as output volume
129
+ output_volume = EDFVolume(folder=os.path.join(str(tmp_path), "my_edf_vol"))
130
+ task = CastVolumeTask(
131
+ inputs={
132
+ "volume": volume,
133
+ "configuration": {
134
+ "output_dir": str(save_dir),
135
+ "output_type": NabuOutputFileFormat.HDF5,
136
+ "output_data_type": numpy.int16,
137
+ },
138
+ "output_volume": output_volume.get_identifier().to_str(),
139
+ },
140
+ )
141
+ task.run()
142
+ assert task.outputs.volume.load_data().shape == (20, 100, 100)
143
+ assert task.outputs.volume.load_data().dtype == numpy.int16
@@ -0,0 +1,203 @@
1
+ import pytest
2
+
3
+ from tomwer.core.process.reconstruction.nabu.utils import (
4
+ get_recons_volume_identifier,
5
+ get_multi_cor_recons_volume_identifiers,
6
+ nabu_std_err_has_error,
7
+ )
8
+ from tomwer.core.scan.edfscan import EDFTomoScan
9
+ from tomwer.core.scan.nxtomoscan import NXtomoScan
10
+
11
+ _scans = (
12
+ NXtomoScan(
13
+ scan="/my_scan_file.nx",
14
+ entry="entry_test",
15
+ ),
16
+ EDFTomoScan("/my_scan_folder"),
17
+ )
18
+
19
+
20
+ @pytest.mark.parametrize("scan", _scans)
21
+ @pytest.mark.parametrize("axis", ("YZ", "XZ", "XY"))
22
+ def test_get_recons_volume_identifier(scan, axis):
23
+ """
24
+ test get_recons_volume_identifier behavior
25
+ """
26
+
27
+ entry = scan.entry if isinstance(scan, NXtomoScan) else "entry"
28
+ # check some exceptions
29
+ with pytest.raises(ValueError):
30
+ get_recons_volume_identifier(
31
+ file_prefix="volume_rec",
32
+ location="/this/is/a/test",
33
+ file_format="toto",
34
+ slice_index=1080,
35
+ scan=scan,
36
+ axis=axis,
37
+ )
38
+
39
+ # check hdf5 reconstructions
40
+ id_rec_vols = get_recons_volume_identifier(
41
+ file_prefix="volume_rec",
42
+ location="/this/is/a/test",
43
+ file_format="hdf5",
44
+ slice_index="1080",
45
+ scan=scan,
46
+ axis=axis,
47
+ )
48
+ assert len(id_rec_vols) == 1
49
+ assert (
50
+ id_rec_vols[0].to_str()
51
+ == f"hdf5:volume:/this/is/a/test/volume_rec_plane_{axis}_001080.hdf5?path={entry}/reconstruction"
52
+ )
53
+
54
+ # check edf and jp2k slice reconstructions (are expected to have the same behavior)
55
+ id_rec_vols = get_recons_volume_identifier(
56
+ file_prefix="volume_rec",
57
+ location="/this/is/a/test",
58
+ file_format="edf",
59
+ slice_index="1080",
60
+ scan=scan,
61
+ axis=axis,
62
+ )
63
+ assert len(id_rec_vols) == 1
64
+ assert (
65
+ id_rec_vols[0].to_str() == "edf:volume:/this/is/a/test?file_prefix=volume_rec"
66
+ )
67
+
68
+ id_rec_vols = get_recons_volume_identifier(
69
+ file_prefix="volume_recslice",
70
+ location="/this/is/a/test",
71
+ file_format="jp2k",
72
+ slice_index="1080",
73
+ scan=scan,
74
+ axis=axis,
75
+ )
76
+ assert len(id_rec_vols) == 1
77
+ assert (
78
+ id_rec_vols[0].to_str()
79
+ == "jp2k:volume:/this/is/a/test?file_prefix=volume_recslice"
80
+ )
81
+
82
+ # check tiff slice reconstructions
83
+ id_rec_vols = get_recons_volume_identifier(
84
+ file_prefix="volume_rec",
85
+ location="/this/is/a/test",
86
+ file_format="tiff",
87
+ slice_index="1080",
88
+ scan=scan,
89
+ axis=axis,
90
+ )
91
+ assert len(id_rec_vols) == 1
92
+ assert (
93
+ id_rec_vols[0].to_str() == "tiff:volume:/this/is/a/test?file_prefix=volume_rec"
94
+ )
95
+
96
+
97
+ @pytest.mark.parametrize("scan", _scans)
98
+ def test_get_multi_cor_recons_volume_identifier(scan):
99
+ """
100
+ test the get_multi_cor_recons_volume_identifier function
101
+ """
102
+ entry = scan.entry if isinstance(scan, NXtomoScan) else "entry"
103
+
104
+ # dummy test with hdf5
105
+ id_rec_vols = get_multi_cor_recons_volume_identifiers(
106
+ scan=scan,
107
+ slice_index="middle",
108
+ location="/this/is/a/test",
109
+ file_prefix="rec",
110
+ cors=(10, 12),
111
+ file_format="hdf5",
112
+ )
113
+ assert isinstance(id_rec_vols, dict)
114
+ assert len(id_rec_vols) == 2
115
+ assert 10 in id_rec_vols.keys()
116
+ assert 12 in id_rec_vols.keys()
117
+ assert (
118
+ id_rec_vols[10].to_str()
119
+ == f"hdf5:volume:/this/is/a/test/rec_10.000_01024.hdf5?path={entry}/reconstruction"
120
+ )
121
+
122
+ # dummy test with tiff
123
+ id_rec_vols = get_multi_cor_recons_volume_identifiers(
124
+ scan=scan,
125
+ slice_index="middle",
126
+ location="/this/is/a/test",
127
+ file_prefix="rec",
128
+ cors=(10, 12),
129
+ file_format="tiff",
130
+ )
131
+ assert isinstance(id_rec_vols, dict)
132
+ assert len(id_rec_vols) == 2
133
+ assert 10 in id_rec_vols.keys()
134
+ assert 12 in id_rec_vols.keys()
135
+ assert id_rec_vols[10].to_str() == "tiff:volume:/this/is/a?file_prefix=test"
136
+
137
+
138
+ def test_nabu_std_err_has_error():
139
+ assert nabu_std_err_has_error(None) is False
140
+ assert nabu_std_err_has_error(b"") is False
141
+ assert nabu_std_err_has_error(b"this is an error") is True
142
+ assert (
143
+ nabu_std_err_has_error(
144
+ b"warnings.warn('creating CUBLAS context to get version num"
145
+ )
146
+ is False
147
+ )
148
+ assert (
149
+ nabu_std_err_has_error(
150
+ b"warnings.warn('creating CUBLAS context to get version num\n this is an error"
151
+ )
152
+ is True
153
+ )
154
+
155
+ assert (
156
+ nabu_std_err_has_error(
157
+ b"""/nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/skcuda/cublas.py:284: UserWarning: creating CUBLAS context to get version number
158
+ warnings.warn('creating CUBLAS context to get version number')
159
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
160
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
161
+ self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
162
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/pycuda/elementwise.py:47: UserWarning: The CUDA compiler succeeded, but said the following:
163
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
164
+ return SourceModule(
165
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
166
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
167
+ self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
168
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/pycuda/elementwise.py:47: UserWarning: The CUDA compiler succeeded, but said the following:
169
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
170
+ return SourceModule(
171
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
172
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
173
+ self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
174
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
175
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
176
+ self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
177
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
178
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
179
+ kernel.cu(111): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
180
+ /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
181
+ kernel.cu(112): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
182
+ /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
183
+ kernel.cu(113): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
184
+ /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
185
+ kernel.cu(114): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
186
+ /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
187
+ self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
188
+ /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/pycuda/elementwise.py:47: UserWarning: The CUDA compiler succeeded, but said the following:
189
+ nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
190
+ return SourceModule("""
191
+ )
192
+ is False
193
+ )
194
+
195
+ assert (
196
+ nabu_std_err_has_error(
197
+ b"""/cvmfs/tomo.esrf.fr/software/packages/linux/x86_64/tomotools/2024_02_26/lib/python3.11/site-packages/cupyx/jit/_interface.py:173: FutureWarning: cupyx.jit.rawkernel is experimental. The interface can change in the future.
198
+ cupy._util.experimental('cupyx.jit.rawkernel')
199
+ /cvmfs/tomo.esrf.fr/software/packages/linux/x86_64/tomotools/2024_02_26/lib/python3.11/site-packages/skcuda/cublas.py:284: UserWarning: creating CUBLAS context to get version number
200
+ warnings.warn('creating CUBLAS context to get version number')"""
201
+ )
202
+ is False
203
+ )