tomwer 1.2.0a1__py3-none-any.whl → 1.2.0a3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. orangecontrib/tomwer/tutorials/append_raw_darks_and_flats_frames_to_NXtomos.ows +44 -0
  2. orangecontrib/tomwer/tutorials/copy_reduced_darks_and_flats_meth1.ows +55 -0
  3. orangecontrib/tomwer/tutorials/copy_reduced_darks_and_flats_meth2.ows +48 -0
  4. orangecontrib/tomwer/tutorials/default_cor_search.ows +40 -0
  5. orangecontrib/tomwer/tutorials/hello_world_python_script.ows +50 -0
  6. orangecontrib/tomwer/tutorials/simple_slice_reconstruction_on_slurm.ows +50 -0
  7. orangecontrib/tomwer/tutorials/simple_volume_to_slurm_reconstruction.ows +8 -8
  8. orangecontrib/tomwer/widgets/__init__.py +1 -1
  9. orangecontrib/tomwer/widgets/cluster/FutureSupervisorOW.py +0 -1
  10. orangecontrib/tomwer/widgets/cluster/SlurmClusterOW.py +8 -6
  11. orangecontrib/tomwer/widgets/control/AdvancementOW.py +0 -1
  12. orangecontrib/tomwer/widgets/control/DataDiscoveryOW.py +1 -6
  13. orangecontrib/tomwer/widgets/control/DataListOW.py +0 -1
  14. orangecontrib/tomwer/widgets/control/DataListenerOW.py +4 -4
  15. orangecontrib/tomwer/widgets/control/DataSelectorOW.py +0 -1
  16. orangecontrib/tomwer/widgets/control/DataTransfertOW.py +7 -7
  17. orangecontrib/tomwer/widgets/control/DataValidatorOW.py +0 -1
  18. orangecontrib/tomwer/widgets/control/DataWatcherOW.py +0 -3
  19. orangecontrib/tomwer/widgets/control/EDF2NXTomomillOW.py +3 -2
  20. orangecontrib/tomwer/widgets/control/EmailOW.py +82 -0
  21. orangecontrib/tomwer/widgets/control/FilterOW.py +3 -3
  22. orangecontrib/tomwer/widgets/control/NXTomomillOW.py +1 -1
  23. orangecontrib/tomwer/widgets/control/NotifierOW.py +0 -1
  24. orangecontrib/tomwer/widgets/control/ReduceDarkFlatSelectorOW.py +93 -0
  25. orangecontrib/tomwer/widgets/control/SingleTomoObjOW.py +29 -5
  26. orangecontrib/tomwer/widgets/control/TimerOW.py +1 -2
  27. orangecontrib/tomwer/widgets/control/TomoObjSerieOW.py +0 -1
  28. orangecontrib/tomwer/widgets/control/VolumeSelector.py +0 -1
  29. orangecontrib/tomwer/widgets/control/VolumeSymLinkOW.py +4 -10
  30. orangecontrib/tomwer/widgets/control/icons/email.png +0 -0
  31. orangecontrib/tomwer/widgets/control/icons/email.svg +58 -0
  32. orangecontrib/tomwer/widgets/control/icons/reduced_darkflat_selector.png +0 -0
  33. orangecontrib/tomwer/widgets/control/icons/reduced_darkflat_selector.svg +199 -0
  34. orangecontrib/tomwer/widgets/debugtools/DatasetGeneratorOW.py +0 -1
  35. orangecontrib/tomwer/widgets/debugtools/ObjectInspectorOW.py +0 -1
  36. orangecontrib/tomwer/widgets/edit/DarkFlatPatchOW.py +1 -2
  37. orangecontrib/tomwer/widgets/edit/ImageKeyEditorOW.py +1 -2
  38. orangecontrib/tomwer/widgets/edit/ImageKeyUpgraderOW.py +0 -1
  39. orangecontrib/tomwer/widgets/edit/NXtomoEditorOW.py +0 -1
  40. orangecontrib/tomwer/widgets/other/PythonScriptOW.py +29 -1
  41. orangecontrib/tomwer/widgets/other/TomoObjsHub.py +28 -0
  42. orangecontrib/tomwer/widgets/other/icons/hub.png +0 -0
  43. orangecontrib/tomwer/widgets/other/icons/hub.svg +113 -0
  44. orangecontrib/tomwer/widgets/reconstruction/AxisOW.py +18 -12
  45. orangecontrib/tomwer/widgets/reconstruction/CastNabuVolumeOW.py +0 -2
  46. orangecontrib/tomwer/widgets/reconstruction/DarkRefAndCopyOW.py +21 -6
  47. orangecontrib/tomwer/widgets/reconstruction/NabuOW.py +29 -7
  48. orangecontrib/tomwer/widgets/reconstruction/NabuVolumeOW.py +18 -5
  49. orangecontrib/tomwer/widgets/reconstruction/SAAxisOW.py +40 -13
  50. orangecontrib/tomwer/widgets/reconstruction/SADeltaBetaOW.py +37 -10
  51. orangecontrib/tomwer/widgets/reconstruction/SinoNormOW.py +2 -3
  52. orangecontrib/tomwer/widgets/reconstruction/TofuOW.py +5 -4
  53. orangecontrib/tomwer/widgets/stitching/StitcherOW.py +0 -1
  54. orangecontrib/tomwer/widgets/stitching/ZStitchingConfigOW.py +0 -1
  55. orangecontrib/tomwer/widgets/visualization/DataViewerOW.py +10 -4
  56. orangecontrib/tomwer/widgets/visualization/DiffViewerOW.py +1 -1
  57. orangecontrib/tomwer/widgets/visualization/LivesliceOW.py +0 -1
  58. orangecontrib/tomwer/widgets/visualization/NXtomoMetadataViewerOW.py +0 -1
  59. orangecontrib/tomwer/widgets/visualization/RadioStackOW.py +7 -5
  60. orangecontrib/tomwer/widgets/visualization/SampleMovedOW.py +1 -1
  61. orangecontrib/tomwer/widgets/visualization/SinogramViewerOW.py +0 -3
  62. orangecontrib/tomwer/widgets/visualization/SliceStackOW.py +7 -5
  63. orangecontrib/tomwer/widgets/visualization/VolumeViewerOW.py +4 -4
  64. tomwer/__main__.py +139 -5
  65. tomwer/app/axis.py +16 -5
  66. tomwer/app/canvas_launcher/config.py +1 -1
  67. tomwer/app/canvas_launcher/mainwindow.py +164 -6
  68. tomwer/app/darkref.py +10 -181
  69. tomwer/app/darkrefpatch.py +10 -131
  70. tomwer/app/diffframe.py +11 -0
  71. tomwer/app/imagekeyeditor.py +12 -19
  72. tomwer/app/intensitynormalization.py +1 -0
  73. tomwer/app/lamino.py +7 -2
  74. tomwer/app/patchrawdarkflat.py +131 -0
  75. tomwer/app/radiostack.py +10 -0
  76. tomwer/app/reducedarkflat.py +205 -0
  77. tomwer/app/saaxis.py +27 -8
  78. tomwer/app/sadeltabeta.py +29 -8
  79. tomwer/app/samplemoved.py +11 -0
  80. tomwer/app/scanviewer.py +12 -0
  81. tomwer/app/sinogramviewer.py +11 -0
  82. tomwer/app/slicestack.py +11 -0
  83. tomwer/app/zstitching.py +12 -0
  84. tomwer/core/futureobject.py +4 -2
  85. tomwer/core/process/conditions/filters.py +26 -4
  86. tomwer/core/process/control/datadiscovery.py +4 -0
  87. tomwer/core/process/control/datawatcher/datawatcher.py +5 -1
  88. tomwer/core/process/control/email.py +148 -0
  89. tomwer/core/process/control/nxtomoconcatenate.py +9 -2
  90. tomwer/core/process/control/nxtomomill.py +58 -16
  91. tomwer/core/process/control/scanselector.py +4 -0
  92. tomwer/core/process/control/scantransfer.py +52 -23
  93. tomwer/core/process/control/test/test_concatenate_nxtomos.py +1 -0
  94. tomwer/core/process/control/test/test_email.py +52 -0
  95. tomwer/core/process/control/test/test_h52nx_process.py +106 -0
  96. tomwer/core/process/control/test/test_volume_link.py +5 -4
  97. tomwer/core/process/control/timer.py +27 -6
  98. tomwer/core/process/control/tomoobjserie.py +4 -0
  99. tomwer/core/process/control/volumeselector.py +4 -0
  100. tomwer/core/process/control/volumesymlink.py +47 -8
  101. tomwer/core/process/edit/darkflatpatch.py +49 -8
  102. tomwer/core/process/edit/imagekeyeditor.py +63 -13
  103. tomwer/core/process/reconstruction/axis/__init__.py +1 -1
  104. tomwer/core/process/reconstruction/axis/axis.py +61 -41
  105. tomwer/core/process/reconstruction/axis/params.py +4 -6
  106. tomwer/core/process/reconstruction/darkref/darkrefs.py +53 -16
  107. tomwer/core/process/reconstruction/darkref/darkrefscopy.py +12 -2
  108. tomwer/core/process/reconstruction/lamino/__init__.py +1 -1
  109. tomwer/core/process/reconstruction/lamino/tofu.py +22 -2
  110. tomwer/core/process/reconstruction/nabu/nabucommon.py +93 -14
  111. tomwer/core/process/reconstruction/nabu/nabuscores.py +70 -33
  112. tomwer/core/process/reconstruction/nabu/nabuslices.py +219 -41
  113. tomwer/core/process/reconstruction/nabu/nabuvolume.py +240 -108
  114. tomwer/core/process/reconstruction/nabu/utils.py +10 -36
  115. tomwer/core/process/reconstruction/normalization/normalization.py +10 -3
  116. tomwer/core/process/reconstruction/saaxis/__init__.py +1 -0
  117. tomwer/core/process/reconstruction/saaxis/saaxis.py +564 -376
  118. tomwer/core/process/reconstruction/sadeltabeta/__init__.py +1 -0
  119. tomwer/core/process/reconstruction/sadeltabeta/sadeltabeta.py +481 -268
  120. tomwer/core/process/reconstruction/scores/params.py +21 -8
  121. tomwer/core/process/reconstruction/test/test_darkref_copy.py +2 -0
  122. tomwer/core/process/reconstruction/test/test_saaxis.py +21 -8
  123. tomwer/core/process/reconstruction/test/test_sadeltabeta.py +8 -5
  124. tomwer/core/process/script/python.py +7 -2
  125. tomwer/core/process/stitching/nabustitcher.py +10 -3
  126. tomwer/core/process/task.py +2 -9
  127. tomwer/core/process/test/test_axis.py +25 -15
  128. tomwer/core/process/test/test_conditions.py +6 -6
  129. tomwer/core/process/test/test_dark_and_flat.py +20 -15
  130. tomwer/core/process/test/test_data_transfer.py +8 -8
  131. tomwer/core/process/test/test_data_watcher.py +1 -1
  132. tomwer/core/process/test/test_lamino.py +6 -6
  133. tomwer/core/process/test/test_nabu.py +13 -8
  134. tomwer/core/process/test/test_normalization.py +1 -0
  135. tomwer/core/process/test/test_timer.py +6 -6
  136. tomwer/core/process/visualization/dataviewer.py +4 -0
  137. tomwer/core/process/visualization/diffviewer.py +4 -0
  138. tomwer/core/process/visualization/imagestackviewer.py +4 -0
  139. tomwer/core/process/visualization/radiostack.py +4 -0
  140. tomwer/core/process/visualization/samplemoved.py +4 -0
  141. tomwer/core/process/visualization/sinogramviewer.py +4 -0
  142. tomwer/core/process/visualization/slicestack.py +4 -0
  143. tomwer/core/process/visualization/volumeviewer.py +4 -0
  144. tomwer/core/scan/hdf5scan.py +4 -4
  145. tomwer/core/scan/scanbase.py +5 -1
  146. tomwer/core/scan/test/test_process_registration.py +9 -9
  147. tomwer/core/settings.py +59 -1
  148. tomwer/core/test/test_lamino.py +2 -1
  149. tomwer/core/utils/__init__.py +16 -0
  150. tomwer/core/utils/locker.py +0 -1
  151. tomwer/core/utils/resource.py +6 -11
  152. tomwer/core/utils/scanutils.py +2 -0
  153. tomwer/gui/cluster/slurm.py +91 -7
  154. tomwer/gui/cluster/supervisor.py +16 -11
  155. tomwer/gui/cluster/test/test_cluster.py +16 -1
  156. tomwer/gui/conditions/filter.py +3 -3
  157. tomwer/gui/control/datalist.py +24 -11
  158. tomwer/gui/control/email.py +183 -0
  159. tomwer/gui/control/reducedarkflatselector.py +545 -0
  160. tomwer/gui/control/singletomoobj.py +23 -1
  161. tomwer/gui/control/test/test_email.py +35 -0
  162. tomwer/gui/control/test/test_reducedarkflat_selector.py +280 -0
  163. tomwer/gui/reconstruction/axis/CompareImages.py +1 -1
  164. tomwer/gui/reconstruction/axis/axis.py +10 -6
  165. tomwer/gui/reconstruction/axis/radioaxis.py +14 -6
  166. tomwer/gui/reconstruction/darkref/darkrefcopywidget.py +2 -0
  167. tomwer/gui/reconstruction/darkref/darkrefwidget.py +4 -4
  168. tomwer/gui/reconstruction/nabu/nabuconfig/phase.py +3 -1
  169. tomwer/gui/reconstruction/nabu/nabuconfig/preprocessing.py +34 -33
  170. tomwer/gui/reconstruction/nabu/nabuconfig/reconstruction.py +1 -1
  171. tomwer/gui/reconstruction/normalization/intensity.py +5 -5
  172. tomwer/gui/reconstruction/saaxis/corrangeselector.py +1 -0
  173. tomwer/gui/reconstruction/saaxis/saaxis.py +6 -6
  174. tomwer/gui/reconstruction/sadeltabeta/saadeltabeta.py +6 -6
  175. tomwer/gui/reconstruction/scores/scoreplot.py +6 -4
  176. tomwer/gui/samplemoved/__init__.py +2 -2
  177. tomwer/gui/stackplot.py +18 -7
  178. tomwer/gui/stacks.py +2 -2
  179. tomwer/gui/stitching/stitchandbackground.py +2 -2
  180. tomwer/gui/stitching/stitching.py +1 -1
  181. tomwer/gui/stitching/stitching_raw.py +1 -1
  182. tomwer/gui/utils/__init__.py +1 -85
  183. tomwer/gui/utils/illustrations.py +1 -1
  184. tomwer/gui/utils/inputwidget.py +41 -36
  185. tomwer/gui/utils/slider.py +2 -2
  186. tomwer/gui/utils/utils.py +93 -0
  187. tomwer/gui/visualization/dataviewer.py +8 -5
  188. tomwer/gui/visualization/diffviewer/diffviewer.py +4 -4
  189. tomwer/gui/visualization/reconstructionparameters.py +26 -6
  190. tomwer/gui/visualization/sinogramviewer.py +7 -1
  191. tomwer/gui/visualization/test/test_reconstruction_parameters.py +2 -4
  192. tomwer/gui/visualization/volumeviewer.py +2 -0
  193. tomwer/resources/__init__.py +55 -43
  194. tomwer/resources/gui/icons/compose.png +0 -0
  195. tomwer/resources/gui/icons/compose.svg +75 -0
  196. tomwer/synctools/datatransfert.py +3 -1
  197. tomwer/synctools/stacks/edit/darkflatpatch.py +39 -34
  198. tomwer/synctools/stacks/edit/imagekeyeditor.py +8 -27
  199. tomwer/synctools/stacks/processingstack.py +45 -9
  200. tomwer/synctools/stacks/reconstruction/axis.py +6 -5
  201. tomwer/synctools/stacks/reconstruction/dkrefcopy.py +1 -0
  202. tomwer/synctools/stacks/reconstruction/lamino.py +3 -3
  203. tomwer/synctools/stacks/reconstruction/nabu.py +49 -140
  204. tomwer/synctools/stacks/reconstruction/normalization.py +1 -0
  205. tomwer/synctools/stacks/reconstruction/saaxis.py +19 -33
  206. tomwer/synctools/stacks/reconstruction/sadeltabeta.py +16 -32
  207. tomwer/synctools/test/test_darkRefs.py +19 -10
  208. tomwer/synctools/test/test_foldertransfer.py +7 -7
  209. tomwer/third_party/nabu/preproc/phase.py +6 -8
  210. tomwer/third_party/nabu/utils.py +2 -3
  211. tomwer/version.py +1 -1
  212. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/METADATA +15 -54
  213. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/RECORD +219 -192
  214. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/WHEEL +1 -1
  215. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/entry_points.txt +3 -3
  216. /tomwer-1.2.0a1-py3.9-nspkg.pth → /tomwer-1.2.0a3-py3.11-nspkg.pth +0 -0
  217. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/LICENSE +0 -0
  218. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/namespace_packages.txt +0 -0
  219. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/top_level.txt +0 -0
@@ -49,6 +49,7 @@ else:
49
49
  has_nabu = True
50
50
  import logging
51
51
  import os
52
+ from copy import deepcopy
52
53
  from typing import Iterable, Optional, Union
53
54
 
54
55
  from nabu.pipeline.fullfield.nabu_config import (
@@ -64,7 +65,6 @@ from tomwer.core.process.reconstruction.nabu.nabuslices import (
64
65
  )
65
66
  from tomwer.core.process.reconstruction.nabu.target import Target
66
67
  from tomwer.core.progress import Progress
67
- from tomwer.core.scan.edfscan import EDFTomoScan
68
68
  from tomwer.core.scan.scanbase import TomwerScanBase
69
69
  from tomwer.core.utils.slurm import is_slurm_available
70
70
  from tomwer.utils import docstring
@@ -85,6 +85,8 @@ def run_nabu_one_slice_several_config(
85
85
  file_format: str,
86
86
  advancement: Optional[Progress] = None,
87
87
  process_id: Optional[int] = None,
88
+ instanciate_classes_only: bool = False,
89
+ output_file_prefix_pattern=None,
88
90
  ) -> tuple:
89
91
  """
90
92
  Run several reconstruction of a specific slice.
@@ -97,6 +99,7 @@ def run_nabu_one_slice_several_config(
97
99
  :param Optional[int] process_id: id of the process requesting this computation
98
100
  :param Optional[dict] cluster_config: cluster configuration if
99
101
  :return: success, recons_urls (list of output urls), tuple of outs, tuples of errs, dict future_scans (key is cor, value is future_scan)
102
+ if `instanciate_classes_only` set to True then return a list of :class:`_Reconstructor`
100
103
  :rtype: tuple
101
104
  """
102
105
  if cluster_config in (None, {}):
@@ -128,7 +131,11 @@ def run_nabu_one_slice_several_config(
128
131
  file_format=file_format,
129
132
  cluster_config=cluster_config,
130
133
  process_name=process_name,
134
+ output_file_prefix_pattern=output_file_prefix_pattern,
131
135
  )
136
+ if instanciate_classes_only:
137
+ return (reconstructor,)
138
+
132
139
  try:
133
140
  results = reconstructor.run()
134
141
  except TimeoutError as e:
@@ -137,26 +144,26 @@ def run_nabu_one_slice_several_config(
137
144
  else:
138
145
  assert isinstance(
139
146
  results, dict
140
- ), "results should be a dictionary with cor as key and urls as value"
147
+ ), "results should be a dictionary with var_value as key and urls as value"
141
148
  success = True
142
149
  recons_urls = {}
143
150
  std_outs = []
144
151
  std_errs = []
145
152
  future_tomo_objs = {}
146
- for cor, res in results.items():
153
+ for var_value, res in results.items():
147
154
  success = success and res.success
148
155
  if isinstance(res, ResultsWithStd):
149
156
  std_outs.append(res.std_out)
150
157
  std_errs.append(res.std_err)
151
158
  if isinstance(res, ResultsLocalRun):
152
- recons_urls[cor] = res.results_urls
159
+ recons_urls[var_value] = res.results_urls
153
160
  if isinstance(res, ResultSlurmRun):
154
161
  future_tomo_obj = FutureTomwerObject(
155
162
  tomo_obj=scan,
156
163
  process_requester_id=process_id,
157
164
  futures=res.future_slurm_jobs,
158
165
  )
159
- future_tomo_objs[cor] = future_tomo_obj
166
+ future_tomo_objs[var_value] = future_tomo_obj
160
167
  return success, recons_urls, std_outs, std_errs, future_tomo_objs
161
168
 
162
169
 
@@ -172,7 +179,14 @@ class _Reconstructor(_NabuBaseReconstructor):
172
179
  file_format: str,
173
180
  cluster_config: Optional[dict],
174
181
  process_name: str,
182
+ output_file_prefix_pattern=None,
175
183
  ) -> None:
184
+ """
185
+ :param str extra_output_file_pattern: possible extra file name pattern like for cor we want to add 'cor_' as prefix and cor value as suffix.
186
+ To make the file name unique. For delta/beta it is already forseen to be unique. For now keywords are:
187
+ * file_name: default file name according to db values and dataset name
188
+ * value: value of the nabu_configs keys
189
+ """
176
190
  super().__init__(
177
191
  scan=scan,
178
192
  dry_run=dry_run,
@@ -189,6 +203,7 @@ class _Reconstructor(_NabuBaseReconstructor):
189
203
  self.slice_index = slice_index
190
204
  self.nabu_configs = nabu_configs
191
205
  self.file_format = file_format
206
+ self._output_file_prefix_pattern = output_file_prefix_pattern
192
207
 
193
208
  @docstring(_NabuBaseReconstructor)
194
209
  def run(self) -> Iterable:
@@ -204,8 +219,12 @@ class _Reconstructor(_NabuBaseReconstructor):
204
219
  results = {}
205
220
  if self.advancement:
206
221
  self.advancement.setMaxAdvancement(len(self.nabu_configs))
207
- for cor, config in self.nabu_configs.items():
208
- config, conf_file = self.preprocess_config(config, cor)
222
+ for var_value, config in self.nabu_configs.items():
223
+ if self._cancelled:
224
+ break
225
+ config, conf_file = self.preprocess_config(deepcopy(config), var_value)
226
+ print("conf file is", conf_file)
227
+ print("confif output is", config.get("output"))
209
228
 
210
229
  # add some tomwer metadata and save the configuration
211
230
  # note: for now the section is ignored by nabu but shouldn't stay that way
@@ -217,7 +236,7 @@ class _Reconstructor(_NabuBaseReconstructor):
217
236
  options_level="advanced",
218
237
  )
219
238
 
220
- results[cor] = self._process_config(
239
+ results[var_value] = self._process_config(
221
240
  config_to_dump=config_to_dump,
222
241
  config_file=conf_file,
223
242
  file_format=self.file_format,
@@ -231,7 +250,36 @@ class _Reconstructor(_NabuBaseReconstructor):
231
250
  self.advancement.increaseAdvancement(1)
232
251
  return results
233
252
 
234
- def treateOutputConfig(self, _config, cor):
253
+ def _format_file_prefix(self, file_prefix, value):
254
+ if self._output_file_prefix_pattern is None:
255
+ return file_prefix
256
+
257
+ keywords = {
258
+ "file_name": file_prefix,
259
+ "value": value,
260
+ }
261
+
262
+ # filter necessary keywords
263
+ def get_necessary_keywords():
264
+ import string
265
+
266
+ formatter = string.Formatter()
267
+ return [
268
+ field
269
+ for _, field, _, _ in formatter.parse(self._output_file_prefix_pattern)
270
+ if field
271
+ ]
272
+
273
+ requested_keywords = get_necessary_keywords()
274
+
275
+ def keyword_needed(pair):
276
+ keyword, _ = pair
277
+ return keyword in requested_keywords
278
+
279
+ keywords = dict(filter(keyword_needed, keywords.items()))
280
+ return self._output_file_prefix_pattern.format(**keywords)
281
+
282
+ def treateOutputConfig(self, _config, value):
235
283
  """
236
284
  - add or overwrite some parameters of the dictionary
237
285
  - create the output directory if does not exist
@@ -249,33 +297,26 @@ class _Reconstructor(_NabuBaseReconstructor):
249
297
  if "delta_beta" in _config["phase"]:
250
298
  db = round(float(_config["phase"]["delta_beta"]))
251
299
  if "output" in _config:
252
- _file_name = SingleSliceRunner.get_file_basename_reconstruction(
300
+ file_prefix = SingleSliceRunner.get_file_basename_reconstruction(
253
301
  scan=self.scan,
254
302
  slice_index=self.slice_index,
255
303
  pag=pag,
256
304
  db=db,
257
305
  ctf=ctf,
258
306
  )
259
- _config["output"]["file_prefix"] = f"cor_{_file_name}_{cor}"
260
- if _config["output"]["location"] not in ("", None):
261
- # if user specify the location
262
- if not os.path.isdir(_config["output"]["location"]):
263
- os.makedirs(_config["output"]["location"])
264
- else:
265
- # otherwise default location will be the data root level
266
- _config["output"]["location"] = os.sep.join(
267
- [
268
- self.scan.path,
269
- "saaxis_results",
270
- ]
271
- )
307
+ file_prefix = self._format_file_prefix(file_prefix=file_prefix, value=value)
308
+ _config["output"]["file_prefix"] = file_prefix
309
+ assert _config["output"]["location"] not in ("", None)
310
+ if not os.path.isdir(_config["output"]["location"]):
311
+ os.makedirs(_config["output"]["location"])
312
+
272
313
  if "reconstruction" not in _config:
273
314
  _config["reconstruction"] = {}
274
315
  _config["reconstruction"]["start_z"] = self.slice_index
275
316
  _config["reconstruction"]["end_z"] = self.slice_index
276
- return _config
317
+ return _config, file_prefix
277
318
 
278
- def preprocess_config(self, config, cor: float):
319
+ def preprocess_config(self, config, value: float):
279
320
  dataset_params = self.scan.get_nabu_dataset_info()
280
321
  if "dataset" in config:
281
322
  dataset_params.update(config["dataset"])
@@ -289,7 +330,7 @@ class _Reconstructor(_NabuBaseReconstructor):
289
330
  config["output"] = {}
290
331
  config["output"].update({"overwrite_results": 1})
291
332
 
292
- config = self.treateOutputConfig(config, cor=cor)
333
+ config, file_prefix = self.treateOutputConfig(config, value=value)
293
334
  # the policy is to save nabu .cfg file at the same location as the
294
335
  # force overwrite results
295
336
 
@@ -297,13 +338,9 @@ class _Reconstructor(_NabuBaseReconstructor):
297
338
  config["output"]["location"],
298
339
  nabu_settings.NABU_CFG_FILE_FOLDER,
299
340
  )
300
- if not os.path.exists(cfg_folder):
301
- os.makedirs(cfg_folder)
341
+ os.makedirs(cfg_folder, exist_ok=True)
302
342
 
303
- name = (
304
- config["output"]["file_prefix"] + nabu_settings.NABU_CONFIG_FILE_EXTENSION
343
+ conf_file = os.path.join(
344
+ cfg_folder, file_prefix + nabu_settings.NABU_CONFIG_FILE_EXTENSION
305
345
  )
306
- if not isinstance(self.scan, EDFTomoScan):
307
- name = "_".join((self.scan.entry.lstrip("/"), name))
308
- conf_file = os.path.join(cfg_folder, f"cor_{cor}_{name}")
309
346
  return config, conf_file
@@ -32,8 +32,11 @@ import copy
32
32
  import functools
33
33
  import logging
34
34
  import os
35
+ import gc
36
+ from tomwer.io.utils import format_stderr_stdout
37
+ from silx.utils.deprecation import deprecated, deprecated_warning
35
38
 
36
- from processview.core.manager.manager import ProcessManager
39
+ from processview.core.manager.manager import ProcessManager, DatasetState
37
40
 
38
41
  from tomwer.core.futureobject import FutureTomwerObject
39
42
  from tomwer.core.utils.scanutils import data_identifier_to_scan
@@ -105,6 +108,7 @@ def run_slices_reconstruction(
105
108
  dry_run: bool = False,
106
109
  advancement=None,
107
110
  process_id=None,
111
+ instanciate_classes_only: bool = False,
108
112
  ) -> tuple:
109
113
  """
110
114
  call nabu for a reconstruction on scan with the given configuration
@@ -119,6 +123,7 @@ def run_slices_reconstruction(
119
123
  :param stdout: file to redirect stdout
120
124
  :param advancement: optional Progress class to display advancement
121
125
  :param int process_id: optional process id
126
+ :param bool instanciate_class_only: if we don't want to run the SingleSliceRunner but only return them. Use case: we want to keep a hand on processing and it can be cancelled
122
127
 
123
128
  :return: (all_succeed, stdouts, stderrs, final_configs, future_scan)
124
129
  * all_succeed: bool, True if all the reconstruction succeed or if all job request succeed.
@@ -174,6 +179,7 @@ def run_slices_reconstruction(
174
179
  stdouts = []
175
180
  final_configs = []
176
181
  futures = []
182
+ instanciated_classes = []
177
183
  all_succeed = True
178
184
  if advancement is not None:
179
185
  advancement.setMaxAdvancement(len(nabu_configurations))
@@ -188,14 +194,18 @@ def run_slices_reconstruction(
188
194
  dry_run=dry_run,
189
195
  ask_sinogram_registration=ask_sinogram_registration,
190
196
  ask_sinogram_load=ask_sinogram_load,
197
+ instanciate_class_only=instanciate_classes_only,
191
198
  )
192
- if slice_index is None:
193
- continue
194
199
 
195
200
  # specific treatments of results
196
201
  if result is None:
197
202
  # in case of timeout or another issue. Log should already have been provided
198
203
  pass
204
+ elif instanciate_classes_only:
205
+ instanciated_classes.append(result)
206
+ continue
207
+ if slice_index is None:
208
+ continue
199
209
  elif isinstance(result, ResultsLocalRun):
200
210
  assert not is_cluster_job, "cluster job should not return ResultsLocalRun"
201
211
  stderrs.append(result.std_err)
@@ -226,6 +236,8 @@ def run_slices_reconstruction(
226
236
  if advancement is not None:
227
237
  advancement.increaseAdvancement(1)
228
238
 
239
+ if instanciate_classes_only:
240
+ return instanciated_classes
229
241
  if is_cluster_job:
230
242
  future_tomo_obj = FutureTomwerObject(
231
243
  tomo_obj=scan,
@@ -240,16 +252,26 @@ def run_slices_reconstruction(
240
252
  return all_succeed, stdouts, stderrs, final_configs, None
241
253
 
242
254
 
243
- class NabuSlices(
255
+ class NabuSlicesTask(
244
256
  Task,
245
257
  SuperviseProcess,
246
- input_names=("data",),
258
+ input_names=(
259
+ "data",
260
+ "nabu_params",
261
+ ),
262
+ optional_input_names=(
263
+ "dry_run",
264
+ "serialize_output_data",
265
+ ),
247
266
  output_names=(
248
267
  "data",
249
268
  "nabu_params",
269
+ "future_tomo_obj",
250
270
  ),
251
271
  ):
252
- """Definition of the nabu reconstruction Single process"""
272
+ """
273
+ Definition of the nabu reconstruction volume reconstruction process
274
+ """
253
275
 
254
276
  def __init__(
255
277
  self,
@@ -269,14 +291,15 @@ class NabuSlices(
269
291
  node_attrs=node_attrs,
270
292
  execinfo=execinfo,
271
293
  )
272
- if "recons_params" in inputs:
273
- raise KeyError("Do not use recons_params but `nabu_recons_params` instead")
274
- recons_params = inputs.get("nabu_params", {})
275
- self.set_configuration(recons_params)
276
294
  self._dry_run = inputs.get("dry_run", False)
295
+ self._current_processing = None
296
+ # we can sometime call several time a nabu subprocess. The idea is to keep track of it
297
+ # if we want to stop the processing
277
298
 
278
299
  def run(self):
279
300
  scan = data_identifier_to_scan(self.inputs.data)
301
+ configuration = self.inputs.nabu_params
302
+
280
303
  self.outputs.nabu_params = None
281
304
  if scan is None:
282
305
  self.outputs.data = None
@@ -287,39 +310,143 @@ class NabuSlices(
287
310
  scan = ScanFactory.create_scan_object_frm_dict(scan)
288
311
  else:
289
312
  raise ValueError(f"input type of {scan}: {type(scan)} is not managed" "")
290
- assert self.get_configuration() is not None, "configuration should be set"
291
- run_slices_reconstruction(
292
- scan=scan,
293
- config=self.get_configuration(),
294
- dry_run=self.dry_run,
295
- process_id=self.process_id,
296
- )
297
- # register result
313
+ assert isinstance(configuration, dict), "configuration is expected to be a dict"
314
+
298
315
  entry = "entry"
299
316
  if isinstance(scan, HDF5TomoScan):
300
317
  entry = scan.entry
301
318
 
319
+ output_urls = []
320
+ stderrs = []
321
+ stdouts = []
322
+ final_configs = []
323
+ futures = []
324
+ all_succeed = True
325
+ is_cluster_job = configuration.get("cluster_config", None) is not None
326
+ # loop is required for distributed since version 2021
327
+ try:
328
+ single_slice_runner_instances = run_slices_reconstruction(
329
+ config=configuration,
330
+ scan=scan,
331
+ dry_run=self._dry_run,
332
+ process_id=self.process_id,
333
+ instanciate_classes_only=True,
334
+ )
335
+ except Exception as e:
336
+ mess = f"Fail to instanciate slice reconstructor for {str(scan)}. Reason is {e}."
337
+ _logger.processFailed(mess)
338
+ ProcessManager().notify_dataset_state(
339
+ dataset=scan,
340
+ process=self,
341
+ state=DatasetState.FAILED,
342
+ details=mess,
343
+ )
344
+ self.outputs.future_tomo_obj = None
345
+ return
346
+
347
+ for slice_runner in single_slice_runner_instances:
348
+ self._current_processing = slice_runner
349
+ result = self._current_processing.run()[
350
+ 0
351
+ ] # we are expecting a single slice per run in this configuration
352
+ if result is None:
353
+ # in case of timeout or another issue. Log should already have been provided
354
+ pass
355
+ elif isinstance(result, ResultsLocalRun):
356
+ assert (
357
+ not is_cluster_job
358
+ ), "cluster job should not return ResultsLocalRun"
359
+ stderrs.append(result.std_err)
360
+ stdouts.append(result.std_out)
361
+ output_urls.extend(result.results_urls)
362
+ # if slice_index is None this mean that we are simply creating the
363
+ # .cfg file for nabu full volume.
364
+ elif isinstance(result, ResultSlurmRun):
365
+ assert (
366
+ is_cluster_job
367
+ ), "local reconstruction should not return ResultSlurmRun"
368
+ stderrs.append(result.std_err)
369
+ stdouts.append(result.std_out)
370
+ futures.extend(result.future_slurm_jobs)
371
+ elif not isinstance(result, ResultsRun):
372
+ raise ValueError(
373
+ f"result is expected to be an instance of {ResultsRun} not {type(result)}"
374
+ )
375
+
376
+ # common treatments of results
377
+ if result is not None:
378
+ final_configs.append(result.config)
379
+ all_succeed = all_succeed and result.success
380
+
381
+ # update future object and scan latest reconstructions
382
+ self._current_processing = None
383
+ if not self._cancelled and is_cluster_job:
384
+ future_tomo_obj = FutureTomwerObject(
385
+ tomo_obj=scan,
386
+ futures=tuple(futures),
387
+ process_requester_id=self.process_id,
388
+ )
389
+ scan.set_latest_reconstructions(output_urls)
390
+ else:
391
+ # tag latest reconstructions
392
+ scan.set_latest_reconstructions(output_urls)
393
+ future_tomo_obj = None
394
+
395
+ process_index = scan.pop_process_index()
396
+ # update processes information / registration
397
+ gc.collect()
398
+
399
+ # TODO: check output files with the tomoscan validator ?
400
+ if self._cancelled:
401
+ state = DatasetState.CANCELLED
402
+ details = "cancelled by user"
403
+ future_tomo_obj = None
404
+ _logger.info(f"Slices computation for {scan} cancelled")
405
+ else:
406
+ if not all_succeed:
407
+ mess = f"Slices computation for {scan} failed."
408
+ state = DatasetState.FAILED
409
+ _logger.processFailed(mess)
410
+ else:
411
+ state = DatasetState.SUCCEED
412
+ mess = f"Slices computed for {scan}."
413
+ _logger.processSucceed(mess)
414
+ elmts = [
415
+ format_stderr_stdout(stderr=stderr, stdout=stdout)
416
+ for stderr, stdout in zip(stderrs, stdouts)
417
+ ]
418
+ elmts.insert(0, mess)
419
+ details = "\n".join(elmts)
420
+
421
+ ProcessManager().notify_dataset_state(
422
+ dataset=scan,
423
+ process=self,
424
+ state=state,
425
+ details=details,
426
+ )
427
+
428
+ # register result
302
429
  with scan.acquire_process_file_lock():
303
430
  self.register_process(
304
431
  process_file=scan.process_file,
305
432
  entry=entry,
306
- configuration=self.get_configuration(),
433
+ configuration=configuration,
307
434
  results={},
308
- process_index=scan.pop_process_index(),
435
+ process_index=process_index,
309
436
  overwrite=True,
310
437
  )
311
- self.outputs.data = scan
438
+ if self.get_input_value("serialize_output_data", True):
439
+ self.outputs.data = scan.to_dict()
440
+ else:
441
+ self.outputs.data = scan
312
442
  self.outputs.nabu_params = scan.nabu_recons_params
443
+ self.outputs.future_tomo_obj = future_tomo_obj
313
444
 
314
445
  def set_configuration(self, configuration: dict) -> None:
315
446
  Task.set_configuration(self, configuration=configuration)
316
447
  if "dry_run" in configuration:
317
448
  self.set_dry_run(bool(configuration["dry_run"]))
318
449
 
319
- # TODO: those get / set configuration should be removed now that Task is used
320
- def get_configuration(self):
321
- return self._settings
322
-
323
450
  @staticmethod
324
451
  def program_name():
325
452
  return "nabu-slices"
@@ -328,10 +455,12 @@ class NabuSlices(
328
455
  def program_version():
329
456
  return nabu_version
330
457
 
458
+ @deprecated(replacement="provide dry_run to the task inputs", since_version="1.2")
331
459
  def set_dry_run(self, dry_run):
332
460
  self._dry_run = dry_run
333
461
 
334
462
  @property
463
+ @deprecated(replacement="Task.inputs.dry_run", since_version="1.2")
335
464
  def dry_run(self):
336
465
  return self._dry_run
337
466
 
@@ -348,8 +477,8 @@ class NabuSlices(
348
477
  """
349
478
  if entry is None:
350
479
  with HDF5File(process_file, "r", swmr=True) as h5f:
351
- entries = NabuSlices._get_process_nodes(
352
- root_node=h5f, process=NabuSlices
480
+ entries = NabuSlicesTask._get_process_nodes(
481
+ root_node=h5f, process=NabuSlicesTask
353
482
  )
354
483
  if len(entries) == 0:
355
484
  _logger.info("unable to find a Axis process in %s" % process_file)
@@ -364,8 +493,8 @@ class NabuSlices(
364
493
  res = {}
365
494
 
366
495
  with HDF5File(process_file, "r", swmr=True) as h5f:
367
- nabu_nodes = NabuSlices._get_process_nodes(
368
- root_node=h5f[entry], process=NabuSlices
496
+ nabu_nodes = NabuSlicesTask._get_process_nodes(
497
+ root_node=h5f[entry], process=NabuSlicesTask
369
498
  )
370
499
  index_to_path = {}
371
500
  for key, index in nabu_nodes.items():
@@ -402,10 +531,18 @@ class NabuSlices(
402
531
  )
403
532
  return res
404
533
 
534
+ def cancel(self):
535
+ """
536
+ stop current processing
537
+ """
538
+ self._cancelled = True
539
+ if self._current_processing is not None:
540
+ self._current_processing.cancel()
541
+
405
542
  @staticmethod
406
543
  def retrieve_last_relative_cor(scan):
407
544
  with EntryReader(scan.process_file_url) as h5f:
408
- latest_nabu_node = Task.get_most_recent_process(h5f, NabuSlices)
545
+ latest_nabu_node = Task.get_most_recent_process(h5f, NabuSlicesTask)
409
546
  path = "configuration/reconstruction/rotation_axis_position"
410
547
  if latest_nabu_node is not None and path in latest_nabu_node:
411
548
  return h5py_read_dataset(latest_nabu_node[path])
@@ -446,6 +583,17 @@ def interpret_tomwer_configuration(
446
583
  else:
447
584
  pag_dbs = (None,)
448
585
 
586
+ # remove slices that 'cannot' be reconstructed (out of bounds)
587
+ def filter_slice(slice_index: int):
588
+ if scan.dim_2 is not None and slice_index > scan.dim_2:
589
+ _logger.error(
590
+ f"slice index {slice_index} requested. But slice index must be in 0-{scan.dim_2} - ignore this request"
591
+ )
592
+ return False
593
+ return True
594
+
595
+ slices = list(filter(lambda slice_index: filter_slice(int(slice_index)), slices))
596
+
449
597
  # by default add the slice 'None' which is the slice for the volume
450
598
  slices.append(None)
451
599
  nabu_config = get_nabu_config(config=config)
@@ -568,17 +716,15 @@ class SingleSliceRunner(_NabuBaseReconstructor):
568
716
  "steps_file": steps_file,
569
717
  }
570
718
 
571
- config = self._treateOutputSliceConfig(config)
719
+ config, cfg_folder = self._treateOutputSliceConfig(config)
572
720
  # the policy is to save nabu .cfg file at the same location as the
573
721
  # force overwrite results
574
722
  if self.slice_index is not None:
575
723
  config["reconstruction"]["start_z"] = self.slice_index
576
724
  config["reconstruction"]["end_z"] = self.slice_index
577
- cfg_folder = os.path.join(
578
- config["output"]["location"], nabu_settings.NABU_CFG_FILE_FOLDER
579
- )
580
- if not os.path.exists(cfg_folder):
581
- os.makedirs(cfg_folder)
725
+
726
+ if self.slice_index is not None:
727
+ os.makedirs(config["output"]["location"], exist_ok=True)
582
728
 
583
729
  name = (
584
730
  config["output"]["file_prefix"] + nabu_settings.NABU_CONFIG_FILE_EXTENSION
@@ -638,8 +784,8 @@ class SingleSliceRunner(_NabuBaseReconstructor):
638
784
  file_format=file_format,
639
785
  scan=self.scan,
640
786
  slice_index=None,
641
- start_z=None, # self.slice_index,
642
- end_z=None, # self.slice_index,
787
+ start_z=None,
788
+ end_z=None,
643
789
  expects_single_slice=True,
644
790
  )
645
791
 
@@ -659,6 +805,13 @@ class SingleSliceRunner(_NabuBaseReconstructor):
659
805
  if isinstance(scan, HDF5TomoScan):
660
806
  basename, _ = os.path.splitext(scan.master_file)
661
807
  basename = os.path.basename(basename)
808
+ try:
809
+ with HDF5File(scan.master_file, mode="r") as h5f:
810
+ if len(h5f.keys()) > 1:
811
+ # if there is more than one entry in the file append the entry name to the file basename
812
+ basename = "_".join((basename, scan.entry.lstrip("/")))
813
+ except Exception:
814
+ pass
662
815
  else:
663
816
  basename = os.path.basename(scan.path)
664
817
  if slice_index is None:
@@ -673,7 +826,7 @@ class SingleSliceRunner(_NabuBaseReconstructor):
673
826
  return "_".join(
674
827
  (
675
828
  basename + "slice_pag",
676
- str(slice_index).zfill(4),
829
+ str(slice_index).zfill(6),
677
830
  "db" + str(db).zfill(4),
678
831
  )
679
832
  )
@@ -681,12 +834,12 @@ class SingleSliceRunner(_NabuBaseReconstructor):
681
834
  return "_".join(
682
835
  (
683
836
  basename + "slice_ctf",
684
- str(slice_index).zfill(4),
837
+ str(slice_index).zfill(6),
685
838
  "db" + str(db).zfill(4),
686
839
  )
687
840
  )
688
841
  else:
689
- return "_".join((basename + "slice", str(slice_index).zfill(4)))
842
+ return "_".join((basename + "slice", str(slice_index).zfill(6)))
690
843
 
691
844
  @docstring(_NabuBaseReconstructor)
692
845
  def _get_file_basename_reconstruction(self, pag, db, ctf):
@@ -715,6 +868,7 @@ def run_single_slice_reconstruction(
715
868
  process_id: Optional[int] = None,
716
869
  cluster_config: Optional[dict] = None,
717
870
  add_to_latest_reconstructions=True,
871
+ instanciate_class_only=False,
718
872
  ) -> Optional[ResultsRun]:
719
873
  """
720
874
  # TODO: might need something like a context or an option "keep" slice in memory
@@ -732,6 +886,7 @@ def run_single_slice_reconstruction(
732
886
  :param bool ask_sinogram_load: should we ask nabu to load sinogram
733
887
  :param bool add_to_latest_reconstructions: if true add reconstructed slice to the latest reconstruction.
734
888
  We wan't to avoid this treatment for saaxis and sadeltebeta for example
889
+ :param bool instanciate_class_only: if we don't want to run the SingleSliceRunner but only return them. Use case: we want to keep a hand on processing and it can be cancelled
735
890
  :return: result of the slice reconstruction if succeed to launch it.
736
891
  :rtype: Optional[ResultsRun]
737
892
  """
@@ -758,6 +913,9 @@ def run_single_slice_reconstruction(
758
913
  add_to_latest_reconstructions=add_to_latest_reconstructions,
759
914
  process_name=process_name,
760
915
  )
916
+ if instanciate_class_only:
917
+ return slice_reconstructor
918
+
761
919
  try:
762
920
  results = slice_reconstructor.run()
763
921
  except TimeoutError as e:
@@ -793,3 +951,23 @@ class NabuSliceMode(_Enum):
793
951
  "an unique value or a list or a tuple"
794
952
  )
795
953
  return tuple(res)
954
+
955
+
956
+ class NabuSlices(NabuSlicesTask):
957
+ def __init__(
958
+ self,
959
+ process_id=None,
960
+ varinfo=None,
961
+ inputs=None,
962
+ node_id=None,
963
+ node_attrs=None,
964
+ execinfo=None,
965
+ ):
966
+ deprecated_warning(
967
+ name="tomwer.core.process.reconstruction.nabu.nabuslices.Nabuslices",
968
+ type_="class",
969
+ reason="improve readibility",
970
+ since_version="1.2",
971
+ replacement="NabuSlicesTask",
972
+ )
973
+ super().__init__(process_id, varinfo, inputs, node_id, node_attrs, execinfo)