tomwer 1.2.0a1__py3-none-any.whl → 1.2.0a3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. orangecontrib/tomwer/tutorials/append_raw_darks_and_flats_frames_to_NXtomos.ows +44 -0
  2. orangecontrib/tomwer/tutorials/copy_reduced_darks_and_flats_meth1.ows +55 -0
  3. orangecontrib/tomwer/tutorials/copy_reduced_darks_and_flats_meth2.ows +48 -0
  4. orangecontrib/tomwer/tutorials/default_cor_search.ows +40 -0
  5. orangecontrib/tomwer/tutorials/hello_world_python_script.ows +50 -0
  6. orangecontrib/tomwer/tutorials/simple_slice_reconstruction_on_slurm.ows +50 -0
  7. orangecontrib/tomwer/tutorials/simple_volume_to_slurm_reconstruction.ows +8 -8
  8. orangecontrib/tomwer/widgets/__init__.py +1 -1
  9. orangecontrib/tomwer/widgets/cluster/FutureSupervisorOW.py +0 -1
  10. orangecontrib/tomwer/widgets/cluster/SlurmClusterOW.py +8 -6
  11. orangecontrib/tomwer/widgets/control/AdvancementOW.py +0 -1
  12. orangecontrib/tomwer/widgets/control/DataDiscoveryOW.py +1 -6
  13. orangecontrib/tomwer/widgets/control/DataListOW.py +0 -1
  14. orangecontrib/tomwer/widgets/control/DataListenerOW.py +4 -4
  15. orangecontrib/tomwer/widgets/control/DataSelectorOW.py +0 -1
  16. orangecontrib/tomwer/widgets/control/DataTransfertOW.py +7 -7
  17. orangecontrib/tomwer/widgets/control/DataValidatorOW.py +0 -1
  18. orangecontrib/tomwer/widgets/control/DataWatcherOW.py +0 -3
  19. orangecontrib/tomwer/widgets/control/EDF2NXTomomillOW.py +3 -2
  20. orangecontrib/tomwer/widgets/control/EmailOW.py +82 -0
  21. orangecontrib/tomwer/widgets/control/FilterOW.py +3 -3
  22. orangecontrib/tomwer/widgets/control/NXTomomillOW.py +1 -1
  23. orangecontrib/tomwer/widgets/control/NotifierOW.py +0 -1
  24. orangecontrib/tomwer/widgets/control/ReduceDarkFlatSelectorOW.py +93 -0
  25. orangecontrib/tomwer/widgets/control/SingleTomoObjOW.py +29 -5
  26. orangecontrib/tomwer/widgets/control/TimerOW.py +1 -2
  27. orangecontrib/tomwer/widgets/control/TomoObjSerieOW.py +0 -1
  28. orangecontrib/tomwer/widgets/control/VolumeSelector.py +0 -1
  29. orangecontrib/tomwer/widgets/control/VolumeSymLinkOW.py +4 -10
  30. orangecontrib/tomwer/widgets/control/icons/email.png +0 -0
  31. orangecontrib/tomwer/widgets/control/icons/email.svg +58 -0
  32. orangecontrib/tomwer/widgets/control/icons/reduced_darkflat_selector.png +0 -0
  33. orangecontrib/tomwer/widgets/control/icons/reduced_darkflat_selector.svg +199 -0
  34. orangecontrib/tomwer/widgets/debugtools/DatasetGeneratorOW.py +0 -1
  35. orangecontrib/tomwer/widgets/debugtools/ObjectInspectorOW.py +0 -1
  36. orangecontrib/tomwer/widgets/edit/DarkFlatPatchOW.py +1 -2
  37. orangecontrib/tomwer/widgets/edit/ImageKeyEditorOW.py +1 -2
  38. orangecontrib/tomwer/widgets/edit/ImageKeyUpgraderOW.py +0 -1
  39. orangecontrib/tomwer/widgets/edit/NXtomoEditorOW.py +0 -1
  40. orangecontrib/tomwer/widgets/other/PythonScriptOW.py +29 -1
  41. orangecontrib/tomwer/widgets/other/TomoObjsHub.py +28 -0
  42. orangecontrib/tomwer/widgets/other/icons/hub.png +0 -0
  43. orangecontrib/tomwer/widgets/other/icons/hub.svg +113 -0
  44. orangecontrib/tomwer/widgets/reconstruction/AxisOW.py +18 -12
  45. orangecontrib/tomwer/widgets/reconstruction/CastNabuVolumeOW.py +0 -2
  46. orangecontrib/tomwer/widgets/reconstruction/DarkRefAndCopyOW.py +21 -6
  47. orangecontrib/tomwer/widgets/reconstruction/NabuOW.py +29 -7
  48. orangecontrib/tomwer/widgets/reconstruction/NabuVolumeOW.py +18 -5
  49. orangecontrib/tomwer/widgets/reconstruction/SAAxisOW.py +40 -13
  50. orangecontrib/tomwer/widgets/reconstruction/SADeltaBetaOW.py +37 -10
  51. orangecontrib/tomwer/widgets/reconstruction/SinoNormOW.py +2 -3
  52. orangecontrib/tomwer/widgets/reconstruction/TofuOW.py +5 -4
  53. orangecontrib/tomwer/widgets/stitching/StitcherOW.py +0 -1
  54. orangecontrib/tomwer/widgets/stitching/ZStitchingConfigOW.py +0 -1
  55. orangecontrib/tomwer/widgets/visualization/DataViewerOW.py +10 -4
  56. orangecontrib/tomwer/widgets/visualization/DiffViewerOW.py +1 -1
  57. orangecontrib/tomwer/widgets/visualization/LivesliceOW.py +0 -1
  58. orangecontrib/tomwer/widgets/visualization/NXtomoMetadataViewerOW.py +0 -1
  59. orangecontrib/tomwer/widgets/visualization/RadioStackOW.py +7 -5
  60. orangecontrib/tomwer/widgets/visualization/SampleMovedOW.py +1 -1
  61. orangecontrib/tomwer/widgets/visualization/SinogramViewerOW.py +0 -3
  62. orangecontrib/tomwer/widgets/visualization/SliceStackOW.py +7 -5
  63. orangecontrib/tomwer/widgets/visualization/VolumeViewerOW.py +4 -4
  64. tomwer/__main__.py +139 -5
  65. tomwer/app/axis.py +16 -5
  66. tomwer/app/canvas_launcher/config.py +1 -1
  67. tomwer/app/canvas_launcher/mainwindow.py +164 -6
  68. tomwer/app/darkref.py +10 -181
  69. tomwer/app/darkrefpatch.py +10 -131
  70. tomwer/app/diffframe.py +11 -0
  71. tomwer/app/imagekeyeditor.py +12 -19
  72. tomwer/app/intensitynormalization.py +1 -0
  73. tomwer/app/lamino.py +7 -2
  74. tomwer/app/patchrawdarkflat.py +131 -0
  75. tomwer/app/radiostack.py +10 -0
  76. tomwer/app/reducedarkflat.py +205 -0
  77. tomwer/app/saaxis.py +27 -8
  78. tomwer/app/sadeltabeta.py +29 -8
  79. tomwer/app/samplemoved.py +11 -0
  80. tomwer/app/scanviewer.py +12 -0
  81. tomwer/app/sinogramviewer.py +11 -0
  82. tomwer/app/slicestack.py +11 -0
  83. tomwer/app/zstitching.py +12 -0
  84. tomwer/core/futureobject.py +4 -2
  85. tomwer/core/process/conditions/filters.py +26 -4
  86. tomwer/core/process/control/datadiscovery.py +4 -0
  87. tomwer/core/process/control/datawatcher/datawatcher.py +5 -1
  88. tomwer/core/process/control/email.py +148 -0
  89. tomwer/core/process/control/nxtomoconcatenate.py +9 -2
  90. tomwer/core/process/control/nxtomomill.py +58 -16
  91. tomwer/core/process/control/scanselector.py +4 -0
  92. tomwer/core/process/control/scantransfer.py +52 -23
  93. tomwer/core/process/control/test/test_concatenate_nxtomos.py +1 -0
  94. tomwer/core/process/control/test/test_email.py +52 -0
  95. tomwer/core/process/control/test/test_h52nx_process.py +106 -0
  96. tomwer/core/process/control/test/test_volume_link.py +5 -4
  97. tomwer/core/process/control/timer.py +27 -6
  98. tomwer/core/process/control/tomoobjserie.py +4 -0
  99. tomwer/core/process/control/volumeselector.py +4 -0
  100. tomwer/core/process/control/volumesymlink.py +47 -8
  101. tomwer/core/process/edit/darkflatpatch.py +49 -8
  102. tomwer/core/process/edit/imagekeyeditor.py +63 -13
  103. tomwer/core/process/reconstruction/axis/__init__.py +1 -1
  104. tomwer/core/process/reconstruction/axis/axis.py +61 -41
  105. tomwer/core/process/reconstruction/axis/params.py +4 -6
  106. tomwer/core/process/reconstruction/darkref/darkrefs.py +53 -16
  107. tomwer/core/process/reconstruction/darkref/darkrefscopy.py +12 -2
  108. tomwer/core/process/reconstruction/lamino/__init__.py +1 -1
  109. tomwer/core/process/reconstruction/lamino/tofu.py +22 -2
  110. tomwer/core/process/reconstruction/nabu/nabucommon.py +93 -14
  111. tomwer/core/process/reconstruction/nabu/nabuscores.py +70 -33
  112. tomwer/core/process/reconstruction/nabu/nabuslices.py +219 -41
  113. tomwer/core/process/reconstruction/nabu/nabuvolume.py +240 -108
  114. tomwer/core/process/reconstruction/nabu/utils.py +10 -36
  115. tomwer/core/process/reconstruction/normalization/normalization.py +10 -3
  116. tomwer/core/process/reconstruction/saaxis/__init__.py +1 -0
  117. tomwer/core/process/reconstruction/saaxis/saaxis.py +564 -376
  118. tomwer/core/process/reconstruction/sadeltabeta/__init__.py +1 -0
  119. tomwer/core/process/reconstruction/sadeltabeta/sadeltabeta.py +481 -268
  120. tomwer/core/process/reconstruction/scores/params.py +21 -8
  121. tomwer/core/process/reconstruction/test/test_darkref_copy.py +2 -0
  122. tomwer/core/process/reconstruction/test/test_saaxis.py +21 -8
  123. tomwer/core/process/reconstruction/test/test_sadeltabeta.py +8 -5
  124. tomwer/core/process/script/python.py +7 -2
  125. tomwer/core/process/stitching/nabustitcher.py +10 -3
  126. tomwer/core/process/task.py +2 -9
  127. tomwer/core/process/test/test_axis.py +25 -15
  128. tomwer/core/process/test/test_conditions.py +6 -6
  129. tomwer/core/process/test/test_dark_and_flat.py +20 -15
  130. tomwer/core/process/test/test_data_transfer.py +8 -8
  131. tomwer/core/process/test/test_data_watcher.py +1 -1
  132. tomwer/core/process/test/test_lamino.py +6 -6
  133. tomwer/core/process/test/test_nabu.py +13 -8
  134. tomwer/core/process/test/test_normalization.py +1 -0
  135. tomwer/core/process/test/test_timer.py +6 -6
  136. tomwer/core/process/visualization/dataviewer.py +4 -0
  137. tomwer/core/process/visualization/diffviewer.py +4 -0
  138. tomwer/core/process/visualization/imagestackviewer.py +4 -0
  139. tomwer/core/process/visualization/radiostack.py +4 -0
  140. tomwer/core/process/visualization/samplemoved.py +4 -0
  141. tomwer/core/process/visualization/sinogramviewer.py +4 -0
  142. tomwer/core/process/visualization/slicestack.py +4 -0
  143. tomwer/core/process/visualization/volumeviewer.py +4 -0
  144. tomwer/core/scan/hdf5scan.py +4 -4
  145. tomwer/core/scan/scanbase.py +5 -1
  146. tomwer/core/scan/test/test_process_registration.py +9 -9
  147. tomwer/core/settings.py +59 -1
  148. tomwer/core/test/test_lamino.py +2 -1
  149. tomwer/core/utils/__init__.py +16 -0
  150. tomwer/core/utils/locker.py +0 -1
  151. tomwer/core/utils/resource.py +6 -11
  152. tomwer/core/utils/scanutils.py +2 -0
  153. tomwer/gui/cluster/slurm.py +91 -7
  154. tomwer/gui/cluster/supervisor.py +16 -11
  155. tomwer/gui/cluster/test/test_cluster.py +16 -1
  156. tomwer/gui/conditions/filter.py +3 -3
  157. tomwer/gui/control/datalist.py +24 -11
  158. tomwer/gui/control/email.py +183 -0
  159. tomwer/gui/control/reducedarkflatselector.py +545 -0
  160. tomwer/gui/control/singletomoobj.py +23 -1
  161. tomwer/gui/control/test/test_email.py +35 -0
  162. tomwer/gui/control/test/test_reducedarkflat_selector.py +280 -0
  163. tomwer/gui/reconstruction/axis/CompareImages.py +1 -1
  164. tomwer/gui/reconstruction/axis/axis.py +10 -6
  165. tomwer/gui/reconstruction/axis/radioaxis.py +14 -6
  166. tomwer/gui/reconstruction/darkref/darkrefcopywidget.py +2 -0
  167. tomwer/gui/reconstruction/darkref/darkrefwidget.py +4 -4
  168. tomwer/gui/reconstruction/nabu/nabuconfig/phase.py +3 -1
  169. tomwer/gui/reconstruction/nabu/nabuconfig/preprocessing.py +34 -33
  170. tomwer/gui/reconstruction/nabu/nabuconfig/reconstruction.py +1 -1
  171. tomwer/gui/reconstruction/normalization/intensity.py +5 -5
  172. tomwer/gui/reconstruction/saaxis/corrangeselector.py +1 -0
  173. tomwer/gui/reconstruction/saaxis/saaxis.py +6 -6
  174. tomwer/gui/reconstruction/sadeltabeta/saadeltabeta.py +6 -6
  175. tomwer/gui/reconstruction/scores/scoreplot.py +6 -4
  176. tomwer/gui/samplemoved/__init__.py +2 -2
  177. tomwer/gui/stackplot.py +18 -7
  178. tomwer/gui/stacks.py +2 -2
  179. tomwer/gui/stitching/stitchandbackground.py +2 -2
  180. tomwer/gui/stitching/stitching.py +1 -1
  181. tomwer/gui/stitching/stitching_raw.py +1 -1
  182. tomwer/gui/utils/__init__.py +1 -85
  183. tomwer/gui/utils/illustrations.py +1 -1
  184. tomwer/gui/utils/inputwidget.py +41 -36
  185. tomwer/gui/utils/slider.py +2 -2
  186. tomwer/gui/utils/utils.py +93 -0
  187. tomwer/gui/visualization/dataviewer.py +8 -5
  188. tomwer/gui/visualization/diffviewer/diffviewer.py +4 -4
  189. tomwer/gui/visualization/reconstructionparameters.py +26 -6
  190. tomwer/gui/visualization/sinogramviewer.py +7 -1
  191. tomwer/gui/visualization/test/test_reconstruction_parameters.py +2 -4
  192. tomwer/gui/visualization/volumeviewer.py +2 -0
  193. tomwer/resources/__init__.py +55 -43
  194. tomwer/resources/gui/icons/compose.png +0 -0
  195. tomwer/resources/gui/icons/compose.svg +75 -0
  196. tomwer/synctools/datatransfert.py +3 -1
  197. tomwer/synctools/stacks/edit/darkflatpatch.py +39 -34
  198. tomwer/synctools/stacks/edit/imagekeyeditor.py +8 -27
  199. tomwer/synctools/stacks/processingstack.py +45 -9
  200. tomwer/synctools/stacks/reconstruction/axis.py +6 -5
  201. tomwer/synctools/stacks/reconstruction/dkrefcopy.py +1 -0
  202. tomwer/synctools/stacks/reconstruction/lamino.py +3 -3
  203. tomwer/synctools/stacks/reconstruction/nabu.py +49 -140
  204. tomwer/synctools/stacks/reconstruction/normalization.py +1 -0
  205. tomwer/synctools/stacks/reconstruction/saaxis.py +19 -33
  206. tomwer/synctools/stacks/reconstruction/sadeltabeta.py +16 -32
  207. tomwer/synctools/test/test_darkRefs.py +19 -10
  208. tomwer/synctools/test/test_foldertransfer.py +7 -7
  209. tomwer/third_party/nabu/preproc/phase.py +6 -8
  210. tomwer/third_party/nabu/utils.py +2 -3
  211. tomwer/version.py +1 -1
  212. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/METADATA +15 -54
  213. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/RECORD +219 -192
  214. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/WHEEL +1 -1
  215. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/entry_points.txt +3 -3
  216. /tomwer-1.2.0a1-py3.9-nspkg.pth → /tomwer-1.2.0a3-py3.11-nspkg.pth +0 -0
  217. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/LICENSE +0 -0
  218. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/namespace_packages.txt +0 -0
  219. {tomwer-1.2.0a1.dist-info → tomwer-1.2.0a3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,148 @@
1
+ import smtplib
2
+ import platform
3
+ import os
4
+ import string
5
+
6
+ from processview.core.manager import ProcessManager
7
+
8
+ from tomwer.core.process.task import Task
9
+ from tomwer.core.tomwer_object import TomwerObject
10
+ from tomwer.core.scan.scanbase import TomwerScanBase
11
+ from tomwer.core.volume.volumebase import TomwerVolumeBase
12
+ from tomwer.core.volume.hdf5volume import HDF5Volume
13
+
14
+ from datetime import datetime
15
+ from tomwer.version import version as __version
16
+
17
+
18
+ class EmailTask(
19
+ Task,
20
+ input_names=("subject", "from_addr", "to_addrs", "text"),
21
+ optional_input_names=("mail_options", "rcpt_options", "host", "port"),
22
+ ):
23
+ """
24
+ Generic task to send a simple email
25
+ """
26
+
27
+ def run(self):
28
+ port = self.get_input_value("port", 0)
29
+ assert isinstance(port, int), "port is expected to be an int"
30
+ host = self.get_input_value("host", "smtps.esrf.fr")
31
+ server = smtplib.SMTP(host, port, timeout=5)
32
+ server.sendmail(
33
+ self.inputs.from_addr,
34
+ self.inputs.to_addrs,
35
+ f"Subject: {self.inputs.subject}\n\n{self.inputs.text}",
36
+ )
37
+ server.quit()
38
+
39
+
40
+ class TomoEmailTask(
41
+ Task,
42
+ input_names=("tomo_obj", "configuration"),
43
+ output_names=("tomo_obj",),
44
+ ):
45
+ """Dedicated task for tomography and gui approach"""
46
+
47
+ def __init__(
48
+ self, varinfo=None, inputs=None, node_id=None, node_attrs=None, execinfo=None
49
+ ):
50
+ super().__init__(varinfo, inputs, node_id, node_attrs, execinfo)
51
+
52
+ def run(self):
53
+ tomo_obj = self.inputs.tomo_obj
54
+ configuration = self.inputs.configuration
55
+ configuration["text"] = format_email_info(
56
+ configuration.get("text", ""), tomo_obj=tomo_obj
57
+ )
58
+ configuration["subject"] = format_email_info(
59
+ configuration.get("subject", ""), tomo_obj=tomo_obj
60
+ )
61
+ task = EmailTask(inputs=configuration)
62
+ task.run()
63
+ self.outputs.tomo_obj = self.inputs.tomo_obj
64
+
65
+
66
+ def _ls_tomo_obj(tomo_obj) -> tuple:
67
+ """
68
+ list information regarding a tomo obj
69
+ """
70
+ if isinstance(tomo_obj, TomwerScanBase):
71
+ # for tomoscan base use the `path` attribut
72
+ file_path_to_list = tomo_obj.path
73
+ elif isinstance(tomo_obj, TomwerVolumeBase):
74
+ if isinstance(tomo_obj, HDF5Volume):
75
+ file_path_to_list = os.path.dirname(tomo_obj.url.file_path())
76
+ else:
77
+ file_path_to_list = tomo_obj.url.file_path()
78
+ else:
79
+ raise TypeError
80
+
81
+ def get_size(file_path, decimal_places=2) -> str:
82
+ size = os.path.getsize(file_path)
83
+ for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"):
84
+ if size < 1024.0 or unit == "PiB":
85
+ break
86
+ size /= 1024.0
87
+ return f"{size:.{decimal_places}f} {unit}"
88
+
89
+ return tuple(
90
+ [
91
+ f"{get_size(os.path.join(file_path_to_list, file_path))} - {file_path}"
92
+ for file_path in os.listdir(file_path_to_list)
93
+ ]
94
+ )
95
+
96
+
97
+ def _ls_dataset_state(tomo_obj) -> str:
98
+ """
99
+ list the status of all met processes by the tomo_obj
100
+ """
101
+ states = {}
102
+ for process in ProcessManager().get_processes():
103
+ state = ProcessManager().get_dataset_state(
104
+ dataset_id=tomo_obj.get_identifier(),
105
+ process=process,
106
+ )
107
+ if state is not None:
108
+ states[process] = state
109
+
110
+ return "\n".join(
111
+ [f"* {process.name}: {state.value}" for process, state in states.items()]
112
+ )
113
+
114
+
115
+ def format_email_info(my_str: str, tomo_obj: TomwerObject) -> str:
116
+ """
117
+ format `my_str` string. It can contain one of the following keyword:
118
+
119
+ - {tomo_obj_short_id}: tomo_obj 'short id' (calling identifier.short_description)
120
+ - {tomo_obj_id}: tomo_obj id
121
+ - {ls_tomo_obj}: ls of the scan folder
122
+ - {timestamp}: current time
123
+ - {footnote}: some footnote defined by tomwer
124
+ - {dataset_processing_states}: list the status of all met processing
125
+ """
126
+
127
+ keywords = {
128
+ "tomo_obj_short_id": tomo_obj.get_identifier().short_description(),
129
+ "tomo_obj_id": tomo_obj.get_identifier().to_str(),
130
+ "ls_tomo_obj": "\n".join(_ls_tomo_obj(tomo_obj)),
131
+ "timestamp": datetime.utcnow().isoformat(timespec="seconds"),
132
+ "footnote": f"email send by tomwer - {__version} from {platform.node()}",
133
+ "dataset_processing_states": _ls_dataset_state(tomo_obj),
134
+ }
135
+
136
+ # filter necessary keywords
137
+ def get_necessary_keywords():
138
+ formatter = string.Formatter()
139
+ return [field for _, field, _, _ in formatter.parse(my_str) if field]
140
+
141
+ requested_keywords = get_necessary_keywords()
142
+
143
+ def keyword_needed(pair):
144
+ keyword, _ = pair
145
+ return keyword in requested_keywords
146
+
147
+ keywords = dict(filter(keyword_needed, keywords.items()))
148
+ return my_str.format(**keywords)
@@ -21,7 +21,10 @@ class ConcatenateNXtomoTask(
21
21
  "output_entry",
22
22
  "overwrite",
23
23
  ),
24
- optional_input_names=("progress",),
24
+ optional_input_names=(
25
+ "progress",
26
+ "serialize_output_data",
27
+ ),
25
28
  output_names=("data",),
26
29
  ):
27
30
  """
@@ -66,10 +69,14 @@ class ConcatenateNXtomoTask(
66
69
  )
67
70
 
68
71
  # cast back nxtomomill NXtomo to HDF5TomoScan (reference object for tomwer)
69
- self.outputs.data = HDF5TomoScan(
72
+ scan = HDF5TomoScan(
70
73
  scan=output_file,
71
74
  entry=self.inputs.output_entry,
72
75
  )
76
+ if self.get_input_value("serialize_output_data", True):
77
+ self.outputs.data = scan.to_dict()
78
+ else:
79
+ self.outputs.data = scan
73
80
 
74
81
 
75
82
  def format_output_location(file_path, serie: Serie):
@@ -30,34 +30,62 @@ __date__ = "30/07/2020"
30
30
 
31
31
  import logging
32
32
  import os
33
+ import pathlib
33
34
 
34
35
  from nxtomomill import converter as nxtomomill_converter
35
36
  from nxtomomill.io.config import TomoEDFConfig as EDFConfig
36
37
  from nxtomomill.io.config import TomoHDF5Config as HDF5Config
38
+ from nxtomomill.converter.hdf5.utils import get_default_output_file
37
39
 
38
40
  from tomwer.core.process.task import TaskWithProgress
39
41
  from tomwer.core.scan.hdf5scan import HDF5TomoScan
40
42
  from tomwer.core.utils.scanutils import format_output_location
41
43
 
44
+ from silx.utils.enum import Enum as _Enum
45
+
42
46
  _logger = logging.getLogger(__name__)
43
47
 
44
48
 
49
+ class NXtomomillNXDefaultOutput(_Enum):
50
+ NEAR_INPUT_FILE = "near input"
51
+ PROCESSED_DATA = "processed data dir"
52
+
53
+
45
54
  class H5ToNxProcess(
46
55
  TaskWithProgress,
47
56
  input_names=("h5_to_nx_configuration",),
48
- optional_input_names=("progress", "hdf5_scan"),
57
+ optional_input_names=(
58
+ "progress",
59
+ "hdf5_scan",
60
+ "serialize_output_data",
61
+ ),
49
62
  output_names=("data",),
50
63
  ):
51
64
  """
52
- Process to convert from a bliss dataset to a nexus compliant dataset
65
+ Task to convert from a bliss dataset to a nexus compliant dataset
53
66
  """
54
67
 
55
68
  @staticmethod
56
- def deduce_output_file_path(master_file_name, scan, entry, outputdir=None):
57
- if outputdir is not None:
58
- file_dir = outputdir
69
+ def deduce_output_file_path(master_file_name, scan, entry, outputdir):
70
+ assert isinstance(outputdir, str), "outputdir is expected to be a str"
71
+
72
+ master_file_name = os.path.realpath(master_file_name)
73
+ # step 1: get output dir
74
+ try:
75
+ outputdir = NXtomomillNXDefaultOutput.from_value(outputdir)
76
+ except ValueError:
77
+ output_folder = format_output_location(outputdir, scan=scan)
59
78
  else:
60
- file_dir = os.path.dirname(master_file_name)
79
+ if outputdir is NXtomomillNXDefaultOutput.PROCESSED_DATA:
80
+ path = pathlib.Path(
81
+ get_default_output_file(input_file=master_file_name)
82
+ )
83
+ output_folder = str(path.parent)
84
+ elif outputdir is NXtomomillNXDefaultOutput.NEAR_INPUT_FILE:
85
+ output_folder = os.path.dirname(master_file_name)
86
+ else:
87
+ raise RuntimeError(f"output dir {outputdir} not handled")
88
+
61
89
  file_name = os.path.basename(master_file_name)
62
90
  if "." in file_name:
63
91
  file_name = "".join(file_name.split(".")[:-1])
@@ -69,8 +97,7 @@ class H5ToNxProcess(
69
97
  output_file_name = "_".join(
70
98
  (os.path.splitext(file_name)[0], entry_for_file_name + ".nx")
71
99
  )
72
- file_dir = format_output_location(file_dir, scan=scan)
73
- return os.path.join(file_dir, output_file_name)
100
+ return os.path.join(output_folder, output_file_name)
74
101
 
75
102
  def run(self):
76
103
  config = self.inputs.h5_to_nx_configuration
@@ -102,13 +129,20 @@ class H5ToNxProcess(
102
129
  _logger.processSucceed(
103
130
  f"{config.input_file} {config.entries} has been translated to {scan_converted}"
104
131
  )
105
- self.outputs.data = scan_converted
132
+ if self.get_input_value("serialize_output_data", True):
133
+ self.outputs.data = scan_converted.to_dict()
134
+ else:
135
+ self.outputs.data = scan_converted
106
136
 
107
137
 
108
138
  class EDFToNxProcess(
109
139
  TaskWithProgress,
110
140
  input_names=("edf_to_nx_configuration",),
111
- optional_input_names=("progress", "edf_scan"),
141
+ optional_input_names=(
142
+ "progress",
143
+ "edf_scan",
144
+ "serialize_output_data",
145
+ ),
112
146
  output_names=("data",),
113
147
  ):
114
148
  """
@@ -126,12 +160,20 @@ class EDFToNxProcess(
126
160
  file_path, entry = nxtomomill_converter.from_edf_to_nx(
127
161
  configuration=config, progress=self.progress
128
162
  )
129
- self.outputs.data = HDF5TomoScan(entry=entry, scan=file_path)
163
+ scan = HDF5TomoScan(entry=entry, scan=file_path)
164
+ if self.get_input_value("serialize_output_data", True):
165
+ self.outputs.data = scan.to_dict()
166
+ else:
167
+ self.outputs.data = scan
130
168
 
131
169
  @staticmethod
132
170
  def deduce_output_file_path(folder_path, output_dir, scan):
133
- if output_dir is None:
134
- output_dir = os.path.dirname(folder_path)
135
-
136
- folder_path = format_output_location(folder_path, scan=scan)
137
- return os.path.join(output_dir, os.path.basename(folder_path) + ".nx")
171
+ if output_dir in (None, NXtomomillNXDefaultOutput.NEAR_INPUT_FILE.value):
172
+ output_folder = os.path.dirname(folder_path)
173
+ elif output_dir == NXtomomillNXDefaultOutput.PROCESSED_DATA.value:
174
+ path = pathlib.Path(get_default_output_file(folder_path))
175
+ output_folder = str(path.parent)
176
+ else:
177
+ output_folder = format_output_location(output_dir, scan=scan)
178
+ print("output output_folder is", output_folder)
179
+ return os.path.join(output_folder, os.path.basename(folder_path) + ".nx")
@@ -4,5 +4,9 @@ from tomwer.core.utils.scanutils import data_identifier_to_scan
4
4
 
5
5
 
6
6
  class _ScanSelectorPlaceHolder(EwoksTask, input_names=["data"], output_names=["data"]):
7
+ """
8
+ task to select one or several scan / data to be processed
9
+ """
10
+
7
11
  def run(self):
8
12
  self.outputs.data = data_identifier_to_scan(self.inputs.data)
@@ -32,6 +32,7 @@ import fnmatch
32
32
  import logging
33
33
  import os
34
34
  import shutil
35
+ from silx.utils.deprecation import deprecated_warning
35
36
 
36
37
  import tomwer.version
37
38
  from tomwer.core.process.reconstruction.nabu.settings import NABU_CFG_FILE_FOLDER
@@ -58,7 +59,21 @@ else:
58
59
  has_rsync = True
59
60
 
60
61
 
61
- class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
62
+ class ScanTransferTask(
63
+ Task,
64
+ input_names=("data",),
65
+ optional_input_names=(
66
+ "serialize_output_data",
67
+ "copying",
68
+ "block",
69
+ "turn_off_print",
70
+ "dest_dir",
71
+ "move",
72
+ "noRsync",
73
+ "overwrite",
74
+ ),
75
+ output_names=("data",),
76
+ ):
62
77
  """Manage the copy of scan.
63
78
 
64
79
  .. warning : the destination directory is find out from the file system
@@ -99,17 +114,19 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
99
114
  else:
100
115
  force_sync = False
101
116
 
102
- self._block = inputs.get("block", force_sync)
117
+ self._block = self.get_input_value("block", force_sync)
103
118
 
104
- self._move = inputs.get("move", False)
119
+ self._move = self.get_input_value("move", False)
105
120
  if not isinstance(self._move, bool):
106
121
  raise TypeError("move is expected to be a boolean")
107
122
 
108
- self._force = inputs.get("force", False)
109
- if not isinstance(self._force, bool):
123
+ self._overwrite = self.get_input_value("overwrite", False)
124
+ if not isinstance(self._overwrite, bool):
110
125
  raise TypeError("move is expected to be a boolean")
111
126
 
112
- self._noRsync = inputs.get("noRsync", False) # TODO: rename noRsync to no_rsync
127
+ self._noRsync = self.get_input_value(
128
+ "noRsync", False
129
+ ) # TODO: rename noRsync to no_rsync
113
130
  if not isinstance(self._noRsync, bool):
114
131
  raise TypeError("move is expected to be a boolean")
115
132
 
@@ -144,7 +161,7 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
144
161
 
145
162
  return ""
146
163
 
147
- def _process_edf_scan(self, scan, move=False, force=True, noRsync=False):
164
+ def _process_edf_scan(self, scan, move=False, overwrite=True, noRsync=False):
148
165
  if not isinstance(scan, EDFTomoScan):
149
166
  raise TypeError(f"{scan} is expected to be an instance of {EDFTomoScan}")
150
167
  outputdir = self.getDestinationDir(scan.path)
@@ -154,7 +171,7 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
154
171
  self._pretransfertOperations(scan.path, outputdir)
155
172
  # as we are in the workflow we want this function to be bloking.
156
173
  # so we will not used a thread for folder synchronization
157
- # for now rsync is not delaing with force option
174
+ # for now rsync is not delaing with overwrite option
158
175
  if not has_rsync or noRsync is True or RSyncManager().has_rsync() is False:
159
176
  logger.info("Can't use rsync, copying files")
160
177
  try:
@@ -162,11 +179,11 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
162
179
  self._moveFiles(
163
180
  scanPath=scan.path,
164
181
  outputdir=os.path.dirname(outputdir),
165
- force=force,
182
+ overwrite=overwrite,
166
183
  )
167
184
  else:
168
185
  self._copyFiles(
169
- scanPath=scan.path, outputdir=outputdir, force=force
186
+ scanPath=scan.path, outputdir=outputdir, overwrite=overwrite
170
187
  )
171
188
  except shutil.Error as e:
172
189
  raise e
@@ -440,7 +457,7 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
440
457
  :param scan: the path to the file we want to move/process
441
458
  :type scan: :class:`.TomoBase`
442
459
  :param move: if True, directly move the files. Otherwise copy the files
443
- :param force: if True then force the copy even if the file/folder already
460
+ :param overwrite: if True then force the copy even if the file/folder already
444
461
  exists
445
462
  :param bool noRSync: True if we wan't do sue shutil instead of rsync.
446
463
  """
@@ -450,7 +467,6 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
450
467
 
451
468
  if scan is None:
452
469
  self.outputs.data = None
453
- force = self._force
454
470
 
455
471
  _scan = scan
456
472
  if type(_scan) is dict:
@@ -461,7 +477,10 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
461
477
  logger.info("synchronisation with scanPath")
462
478
  if isinstance(scan, EDFTomoScan):
463
479
  output_scan = self._process_edf_scan(
464
- scan=scan, move=self._move, force=force, noRsync=self._noRsync
480
+ scan=scan,
481
+ move=self._move,
482
+ overwrite=self.get_input_value("overwrite", True),
483
+ noRsync=self._noRsync,
465
484
  )
466
485
  elif isinstance(scan, HDF5TomoScan):
467
486
  if self._move is True:
@@ -471,7 +490,7 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
471
490
  output_scan = self._process_hdf5_scan(scan=scan)
472
491
  else:
473
492
  raise TypeError("Other scan than EDF or HDF5 are not managed")
474
- if self._return_dict:
493
+ if self.get_input_value("serialize_output_data", True):
475
494
  self.outputs.data = output_scan.to_dict()
476
495
  else:
477
496
  self.outputs.data = output_scan
@@ -537,13 +556,11 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
537
556
  assert isinstance(output_scan, TomwerScanBase)
538
557
  self.scanready.emit(output_scan)
539
558
 
540
- def _copyFiles(self, scanPath, outputdir, force):
559
+ def _copyFiles(self, scanPath, outputdir, overwrite):
541
560
  """Copying files and removing them"""
542
561
  assert type(scanPath) is str
543
562
  assert type(outputdir) is str
544
563
  assert os.path.isdir(scanPath)
545
- # if force is False:
546
- # assert(os.path.isdir(outputdir))
547
564
  # create the destination dir
548
565
  if not os.path.isdir(outputdir):
549
566
 
@@ -557,7 +574,7 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
557
574
  for f in os.listdir(scanPath):
558
575
  file = os.path.join(scanPath, f)
559
576
  fileDest = os.path.join(outputdir, f)
560
- if force is True:
577
+ if overwrite is True:
561
578
  if os.path.isdir(fileDest):
562
579
  shutil.rmtree(fileDest)
563
580
  if os.path.isfile(fileDest):
@@ -575,11 +592,9 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
575
592
  info = "sucessfuly removed file at %s !!!" % scanPath
576
593
  logger.info(info)
577
594
 
578
- def _moveFiles(self, scanPath, outputdir, force):
595
+ def _moveFiles(self, scanPath, outputdir, overwrite):
579
596
  """Function simply moving files"""
580
597
  assert os.path.isdir(scanPath)
581
- if force is False:
582
- assert os.path.isdir(outputdir)
583
598
 
584
599
  logger.debug(
585
600
  "synchronisation with scanPath",
@@ -587,7 +602,7 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
587
602
  )
588
603
 
589
604
  target = os.path.join(outputdir, os.path.basename(scanPath))
590
- if force is True and os.path.isdir(target):
605
+ if overwrite is True and os.path.isdir(target):
591
606
  shutil.rmtree(target)
592
607
  shutil.move(scanPath, outputdir)
593
608
 
@@ -629,7 +644,7 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
629
644
  def setDestDir(self, dist):
630
645
  """Force the outpudir to dist.
631
646
 
632
- :param str dist: path to the folder. If None remove force behavior
647
+ :param str dist: path to the folder. If None remove overwrite behavior
633
648
  """
634
649
  self._destDir = dist
635
650
  if self._destDir is not None and os.path.isdir(self._destDir):
@@ -688,3 +703,17 @@ class ScanTransfer(Task, input_names=("data",), output_names=("data",)):
688
703
  :return:
689
704
  """
690
705
  return os.path.join(self.getDestinationDir(scan), os.path.basename(scan))
706
+
707
+
708
+ class ScanTransfer(ScanTransferTask):
709
+ def __init__(
710
+ self, varinfo=None, inputs=None, node_id=None, node_attrs=None, execinfo=None
711
+ ):
712
+ deprecated_warning(
713
+ name="tomwer.core.process.control.scantransfer.ScanTransfer",
714
+ type_="class",
715
+ reason="improve readibility",
716
+ since_version="1.2",
717
+ replacement="ScanTransferTask",
718
+ )
719
+ super().__init__(varinfo, inputs, node_id, node_attrs, execinfo)
@@ -38,6 +38,7 @@ def test_concatenate_nx_tomo_task(tmp_path):
38
38
  "output_file": output_scan_file,
39
39
  "output_entry": "my_entry",
40
40
  "overwrite": False,
41
+ "serialize_output_data": False,
41
42
  }
42
43
  )
43
44
  task.run()
@@ -0,0 +1,52 @@
1
+ import os
2
+ import numpy
3
+ from tomwer.core.process.control.email import format_email_info, _ls_tomo_obj
4
+ from tomwer.core.scan.hdf5scan import HDF5TomoScan
5
+ from tomwer.core.volume.hdf5volume import HDF5Volume
6
+ from tomwer.core.utils.scanutils import HDF5MockContext
7
+
8
+
9
+ def test__ls_tomo_obj(tmp_path):
10
+ """simple test of the `_ls_tomo_obj` function"""
11
+ for i in range(2):
12
+ open(os.path.join(tmp_path, f"{i}.nx"), "a").close()
13
+
14
+ scan = HDF5TomoScan(
15
+ scan=os.path.join(tmp_path, "1.nx"),
16
+ entry="entry0000",
17
+ )
18
+ assert isinstance(scan, HDF5TomoScan)
19
+
20
+ assert len(_ls_tomo_obj(scan)) == 2
21
+
22
+ volume = HDF5Volume(
23
+ file_path=os.path.join(tmp_path, "myvolume.hdf5"),
24
+ data_path="myvolume",
25
+ data=numpy.linspace(0, 10, 100 * 100 * 3).reshape( # pylint: disable=E1121
26
+ 3, 100, 100
27
+ ),
28
+ )
29
+ volume.save()
30
+ assert len(_ls_tomo_obj(volume)) == 3
31
+
32
+
33
+ def test_format_email_info(tmp_path):
34
+ """
35
+ simple test of formatting some information related to an email
36
+ """
37
+ with HDF5MockContext(
38
+ scan_path=os.path.join(tmp_path, "test", "scan"), n_proj=100
39
+ ) as scan:
40
+ res = format_email_info(
41
+ my_str="{tomo_obj_short_id} \n {tomo_obj_id} \n {ls_tomo_obj} \n {timestamp} \n {footnote}",
42
+ tomo_obj=scan,
43
+ )
44
+
45
+ for keyword in (
46
+ "{tomo_obj_short_id}",
47
+ "{tomo_obj_id}",
48
+ "{ls_tomo_obj}",
49
+ "{timestamp}",
50
+ "{footnote}",
51
+ ):
52
+ assert keyword not in res
@@ -0,0 +1,106 @@
1
+ import os
2
+ from tomwer.core.scan.edfscan import EDFTomoScan
3
+ from tomwer.core.process.control.nxtomomill import (
4
+ EDFToNxProcess,
5
+ H5ToNxProcess,
6
+ NXtomomillNXDefaultOutput,
7
+ )
8
+ from tomwer.core.utils.scanutils import MockHDF5, MockEDF
9
+ from nxtomomill.converter.hdf5.utils import PROCESSED_DATA_DIR_NAME, RAW_DATA_DIR_NAME
10
+
11
+
12
+ def test_h52nx_process_deduce_output_file_path(tmp_path):
13
+ """test H5ToNxProcess.deduce_output_file_path function"""
14
+ scan_path = str(tmp_path / "path" / RAW_DATA_DIR_NAME / "my_scan")
15
+ os.makedirs(scan_path)
16
+
17
+ scan = MockHDF5(scan_path=scan_path, n_proj=0).scan
18
+
19
+ # test H52NXDefaultOutput.PROCESSED_DATA
20
+ assert H5ToNxProcess.deduce_output_file_path(
21
+ master_file_name=scan.master_file,
22
+ scan=scan,
23
+ entry=scan.entry,
24
+ outputdir=NXtomomillNXDefaultOutput.PROCESSED_DATA.value,
25
+ ) == str(
26
+ tmp_path
27
+ / "path"
28
+ / PROCESSED_DATA_DIR_NAME
29
+ / "my_scan"
30
+ / f"my_scan_{scan.entry}.nx"
31
+ )
32
+
33
+ # test H52NXDefaultOutput.NEAR_BLISS_FILE
34
+ assert H5ToNxProcess.deduce_output_file_path(
35
+ master_file_name=scan.master_file,
36
+ scan=scan,
37
+ entry=scan.entry,
38
+ outputdir=NXtomomillNXDefaultOutput.NEAR_INPUT_FILE.value,
39
+ ) == str(
40
+ tmp_path / "path" / RAW_DATA_DIR_NAME / "my_scan" / f"my_scan_{scan.entry}.nx"
41
+ )
42
+
43
+ # test providing output dir with some formatting to be done
44
+ assert H5ToNxProcess.deduce_output_file_path(
45
+ master_file_name=scan.master_file,
46
+ scan=scan,
47
+ entry=scan.entry,
48
+ outputdir="{scan_parent_dir_basename}/../../toto/{scan_dir_name}",
49
+ ) == str(tmp_path / "toto" / "my_scan" / f"my_scan_{scan.entry}.nx")
50
+
51
+ # test providing output folder directly
52
+ assert (
53
+ H5ToNxProcess.deduce_output_file_path(
54
+ master_file_name=scan.master_file,
55
+ scan=scan,
56
+ entry=scan.entry,
57
+ outputdir="/tmp/",
58
+ )
59
+ == "/tmp/my_scan_entry.nx"
60
+ )
61
+
62
+
63
+ def test_edf2nx_process_deduce_output_file_path(tmp_path):
64
+ """test EDFToNxProcess.deduce_output_file_path function"""
65
+ scan_path = str(tmp_path / "path" / RAW_DATA_DIR_NAME / "my_edf_scan")
66
+ MockEDF(
67
+ scan_path=scan_path,
68
+ n_radio=10,
69
+ n_ini_radio=10,
70
+ n_extra_radio=0,
71
+ dim=128,
72
+ dark_n=1,
73
+ flat_n=1,
74
+ )
75
+ scan = EDFTomoScan(scan_path)
76
+
77
+ # test NEAR_INPUT_FILE
78
+ assert EDFToNxProcess.deduce_output_file_path(
79
+ folder_path=scan_path,
80
+ output_dir=NXtomomillNXDefaultOutput.NEAR_INPUT_FILE.value,
81
+ scan=scan,
82
+ ) == os.path.join(tmp_path, "path", RAW_DATA_DIR_NAME, "my_edf_scan.nx")
83
+
84
+ # test PROCESSED_DATA
85
+ assert EDFToNxProcess.deduce_output_file_path(
86
+ folder_path=scan_path,
87
+ output_dir=NXtomomillNXDefaultOutput.PROCESSED_DATA.value,
88
+ scan=scan,
89
+ ) == os.path.join(tmp_path, "path", PROCESSED_DATA_DIR_NAME, "my_edf_scan.nx")
90
+
91
+ # test providing output dir with some formatting to be done
92
+ assert EDFToNxProcess.deduce_output_file_path(
93
+ folder_path=scan_path,
94
+ output_dir="{scan_parent_dir_basename}/../../toto/",
95
+ scan=scan,
96
+ ) == str(tmp_path / "toto" / "my_edf_scan.nx")
97
+
98
+ # test providing output folder directly
99
+ assert (
100
+ EDFToNxProcess.deduce_output_file_path(
101
+ folder_path=scan_path,
102
+ output_dir="/tmp/output",
103
+ scan=scan,
104
+ )
105
+ == "/tmp/output/my_edf_scan.nx"
106
+ )