tomwer 1.3.12__py3-none-any.whl → 1.3.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orangecontrib/tomwer/widgets/reconstruction/DarkRefAndCopyOW.py +1 -0
- orangecontrib/tomwer/widgets/reconstruction/NabuHelicalPrepareWeightsDoubleOW.py +184 -184
- tomwer/app/canvas_launcher/mainwindow.py +0 -1
- tomwer/app/zstitching.py +0 -1
- tomwer/core/cluster/cluster.py +0 -9
- tomwer/core/process/control/datalistener/datalistener.py +4 -1
- tomwer/core/process/reconstruction/axis/params.py +3 -3
- tomwer/core/process/reconstruction/darkref/darkrefscopy.py +34 -6
- tomwer/core/process/reconstruction/nabu/utils.py +4 -2
- tomwer/core/utils/scanutils.py +5 -1
- tomwer/gui/cluster/slurm.py +1 -21
- tomwer/gui/cluster/test/test_cluster.py +0 -1
- tomwer/gui/control/selectorwidgetbase.py +3 -1
- tomwer/gui/edit/nxtomoeditor.py +28 -20
- tomwer/gui/edit/test/test_nx_editor.py +58 -1
- tomwer/gui/reconstruction/axis/axis.py +16 -13
- tomwer/gui/reconstruction/axis/radioaxis.py +0 -1
- tomwer/gui/reconstruction/darkref/darkrefcopywidget.py +11 -0
- tomwer/gui/reconstruction/nabu/nabuconfig/preprocessing.py +5 -4
- tomwer/gui/stitching/stitching.py +8 -3
- tomwer/gui/visualization/dataviewer.py +27 -15
- tomwer/gui/visualization/diffviewer/diffviewer.py +9 -8
- tomwer/synctools/stacks/reconstruction/dkrefcopy.py +10 -0
- tomwer/tests/datasets.py +5 -1
- tomwer/utils.py +1 -4
- tomwer/version.py +1 -1
- {tomwer-1.3.12.dist-info → tomwer-1.3.26.dist-info}/METADATA +34 -48
- {tomwer-1.3.12.dist-info → tomwer-1.3.26.dist-info}/RECORD +34 -34
- {tomwer-1.3.12.dist-info → tomwer-1.3.26.dist-info}/WHEEL +1 -1
- /tomwer-1.3.12-py3.11-nspkg.pth → /tomwer-1.3.26-py3.11-nspkg.pth +0 -0
- {tomwer-1.3.12.dist-info → tomwer-1.3.26.dist-info}/LICENSE +0 -0
- {tomwer-1.3.12.dist-info → tomwer-1.3.26.dist-info}/entry_points.txt +0 -0
- {tomwer-1.3.12.dist-info → tomwer-1.3.26.dist-info}/namespace_packages.txt +0 -0
- {tomwer-1.3.12.dist-info → tomwer-1.3.26.dist-info}/top_level.txt +0 -0
@@ -152,6 +152,7 @@ class DarkRefAndCopyOW(SuperviseOW, WidgetLongProcessing):
|
|
152
152
|
self.widget.recons_params.sigChanged.connect(self._updateSettingsVals)
|
153
153
|
self.widget.sigModeAutoChanged.connect(self._updateSettingsVals)
|
154
154
|
self.widget.sigCopyActivationChanged.connect(self._updateSettingsVals)
|
155
|
+
self.widget.sigClearCache.connect(self._processing_stack.clear_cache)
|
155
156
|
self._processing_stack.sigComputationStarted.connect(self._startProcessing)
|
156
157
|
self._processing_stack.sigComputationEnded.connect(self._endProcessing)
|
157
158
|
self._processing_stack.sigRefSetted.connect(self.widget.setRefSetBy)
|
@@ -1,184 +1,184 @@
|
|
1
|
-
import logging
|
2
|
-
|
3
|
-
from orangewidget import gui
|
4
|
-
from orangewidget.settings import Setting
|
5
|
-
from orangewidget.widget import Input, Output
|
6
|
-
from orangecontrib.tomwer.orange.managedprocess import TomwerWithStackStack
|
7
|
-
from silx.gui import qt
|
8
|
-
from typing import Optional
|
9
|
-
|
10
|
-
from tomwer.core.futureobject import FutureTomwerObject
|
11
|
-
from tomwer.core.scan.scanbase import TomwerScanBase
|
12
|
-
from tomwer.gui.reconstruction.nabu.helical import HelicalPrepareWeightsDouble
|
13
|
-
from tomwer.core.process.reconstruction.nabu.helical import (
|
14
|
-
|
15
|
-
)
|
16
|
-
from tomwer.core.scan.nxtomoscan import NXtomoScan
|
17
|
-
|
18
|
-
_logger = logging.getLogger(__name__)
|
19
|
-
|
20
|
-
|
21
|
-
class NabuHelicalPrepareWeightsDoubleOW(
|
22
|
-
|
23
|
-
|
24
|
-
):
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
1
|
+
# import logging
|
2
|
+
|
3
|
+
# from orangewidget import gui
|
4
|
+
# from orangewidget.settings import Setting
|
5
|
+
# from orangewidget.widget import Input, Output
|
6
|
+
# from orangecontrib.tomwer.orange.managedprocess import TomwerWithStackStack
|
7
|
+
# from silx.gui import qt
|
8
|
+
# from typing import Optional
|
9
|
+
|
10
|
+
# from tomwer.core.futureobject import FutureTomwerObject
|
11
|
+
# from tomwer.core.scan.scanbase import TomwerScanBase
|
12
|
+
# from tomwer.gui.reconstruction.nabu.helical import HelicalPrepareWeightsDouble
|
13
|
+
# from tomwer.core.process.reconstruction.nabu.helical import (
|
14
|
+
# NabuHelicalPrepareWeightsDouble,
|
15
|
+
# )
|
16
|
+
# from tomwer.core.scan.nxtomoscan import NXtomoScan
|
17
|
+
|
18
|
+
# _logger = logging.getLogger(__name__)
|
19
|
+
|
20
|
+
|
21
|
+
# class NabuHelicalPrepareWeightsDoubleOW(
|
22
|
+
# TomwerWithStackStack,
|
23
|
+
# ewokstaskclass=NabuHelicalPrepareWeightsDouble,
|
24
|
+
# ):
|
25
|
+
# """
|
26
|
+
# widget used to call the `nabu-helical-prepare-weights-double` application on a dedicated thread. It define weights map and double flat field.
|
27
|
+
|
28
|
+
# :param parent: the parent widget
|
29
|
+
# """
|
30
|
+
|
31
|
+
# # note of this widget should be the one registered on the documentation
|
32
|
+
# name = "helical prerate weights double"
|
33
|
+
# id = "orangecontrib.tomwer.widgets.reconstruction.NabuHelicalPrepareWeightsDoubleOW.NabuHelicalPrepareWeightsDoubleOW"
|
34
|
+
# description = "compute map of weights requested for nabu helical reconstruction"
|
35
|
+
# icon = "icons/nabu_prepare_weights_double.svg"
|
36
|
+
# priority = 199
|
37
|
+
# keywords = [
|
38
|
+
# "tomography",
|
39
|
+
# "nabu",
|
40
|
+
# "reconstruction",
|
41
|
+
# "nabu-helical",
|
42
|
+
# "helical",
|
43
|
+
# "weights",
|
44
|
+
# "prepare",
|
45
|
+
# ]
|
46
|
+
|
47
|
+
# want_main_area = True
|
48
|
+
# want_control_area = False
|
49
|
+
# resizing_enabled = True
|
50
|
+
|
51
|
+
# _ewoks_default_inputs = Setting(
|
52
|
+
# {
|
53
|
+
# "data": None,
|
54
|
+
# "transition_width_vertical": 50,
|
55
|
+
# "transition_width_horizontal": 50,
|
56
|
+
# "processes_file": "",
|
57
|
+
# "rotation_axis_position": 0,
|
58
|
+
# }
|
59
|
+
# )
|
60
|
+
|
61
|
+
# _ewoks_inputs_to_hide_from_orange = (
|
62
|
+
# "progress",
|
63
|
+
# "processes_file",
|
64
|
+
# "transition_width_vertical",
|
65
|
+
# "transition_width_horizontal",
|
66
|
+
# "rotation_axis_position",
|
67
|
+
# )
|
68
|
+
|
69
|
+
# sigScanReady = qt.Signal(TomwerScanBase)
|
70
|
+
# "Signal emitted when a scan is ended"
|
71
|
+
|
72
|
+
# TIMEOUT = 30
|
73
|
+
|
74
|
+
# class Inputs:
|
75
|
+
# data = Input(
|
76
|
+
# name="data",
|
77
|
+
# type=TomwerScanBase,
|
78
|
+
# doc="one scan to be process",
|
79
|
+
# default=True,
|
80
|
+
# multiple=False,
|
81
|
+
# )
|
82
|
+
|
83
|
+
# class Outputs:
|
84
|
+
# data = Output(name="data", type=TomwerScanBase, doc="one scan to be process")
|
85
|
+
# future_tomo_obj = Output(
|
86
|
+
# name="future_tomo_obj",
|
87
|
+
# type=FutureTomwerObject,
|
88
|
+
# doc="future object (process remotely)",
|
89
|
+
# )
|
90
|
+
|
91
|
+
# LOGGER = _logger
|
92
|
+
|
93
|
+
# def __init__(self, parent=None):
|
94
|
+
# super().__init__(parent)
|
95
|
+
# self.__scan = None
|
96
|
+
|
97
|
+
# # gui definition
|
98
|
+
# _layout = gui.vBox(self.mainArea, self.name).layout()
|
99
|
+
# self.widget = HelicalPrepareWeightsDouble(parent=self)
|
100
|
+
# _layout.addWidget(self.widget)
|
101
|
+
|
102
|
+
# ## connect signal / slot
|
103
|
+
# self.widget.sigConfigChanged.connect(self._updateSettings)
|
104
|
+
|
105
|
+
# if isinstance(self.task_output_changed_callbacks, set):
|
106
|
+
# self.task_output_changed_callbacks.add(self._notify_state)
|
107
|
+
# elif isinstance(self.task_output_changed_callbacks, list):
|
108
|
+
# self.task_output_changed_callbacks.append(self._notify_state)
|
109
|
+
# else:
|
110
|
+
# raise NotImplementedError
|
111
|
+
|
112
|
+
# ## handle settings
|
113
|
+
# self._loadSettings()
|
114
|
+
# self.task_executor_queue.sigComputationStarted.connect(self._newTaskStarted)
|
115
|
+
|
116
|
+
# def _updateSettings(self):
|
117
|
+
# config = self.widget.getConfiguration()
|
118
|
+
# for key in ("transition_width", "processes_file"):
|
119
|
+
# self._ewoks_default_inputs[key] = config[key] # pylint: disable=E1137
|
120
|
+
|
121
|
+
# @property
|
122
|
+
# def request_input(self):
|
123
|
+
# return self.__request_input
|
124
|
+
|
125
|
+
# @request_input.setter
|
126
|
+
# def request_input(self, request):
|
127
|
+
# self.__request_input = request
|
128
|
+
|
129
|
+
# def get_task_inputs(self):
|
130
|
+
# assert self.__scan is not None
|
131
|
+
# return {
|
132
|
+
# "data": self.__scan,
|
133
|
+
# "transition_width": self.widget.getConfiguration()["transition_width"],
|
134
|
+
# "processes_file": self.widget.getConfiguration()["processes_file"],
|
135
|
+
# }
|
136
|
+
|
137
|
+
# def handleNewSignals(self) -> None:
|
138
|
+
# """Invoked by the workflow signal propagation manager after all
|
139
|
+
# signals handlers have been called.
|
140
|
+
# """
|
141
|
+
# # for now we want to avoid propagation any processing.
|
142
|
+
# # task will be executed only when the user validates the dialog
|
143
|
+
# data = super().get_task_inputs().get("data", None)
|
144
|
+
# if data is not None:
|
145
|
+
# if not isinstance(data, NXtomoScan):
|
146
|
+
# raise TypeError(
|
147
|
+
# f"data is expected to be an instance of NXtomoScan. {type(data)} are not handled"
|
148
|
+
# )
|
149
|
+
# self.add(data.path)
|
150
|
+
|
151
|
+
# def _loadSettings(self):
|
152
|
+
# self.widget.setConfiguration(self._ewoks_default_inputs)
|
153
|
+
|
154
|
+
# def _newTaskStarted(self):
|
155
|
+
# try:
|
156
|
+
# task_executor = self.sender()
|
157
|
+
# scan = task_executor.current_task.inputs.data
|
158
|
+
# self.notify_on_going(scan)
|
159
|
+
# except Exception:
|
160
|
+
# pass
|
161
|
+
|
162
|
+
# def _notify_state(self):
|
163
|
+
# try:
|
164
|
+
# task_executor = self.sender()
|
165
|
+
# task_suceeded = task_executor.succeeded
|
166
|
+
# scan = task_executor.current_task.outputs.data
|
167
|
+
# if task_suceeded:
|
168
|
+
# self.notify_succeed(scan=scan)
|
169
|
+
# else:
|
170
|
+
# self.notify_failed(scan=scan)
|
171
|
+
# except Exception as e:
|
172
|
+
# _logger.error(f"failed to handle task finished callback. Raiseon is {e}")
|
173
|
+
|
174
|
+
# @Inputs.data
|
175
|
+
# def process_data(self, scan: Optional[TomwerScanBase]):
|
176
|
+
# if scan is None:
|
177
|
+
# return
|
178
|
+
# else:
|
179
|
+
# self.__scan = scan
|
180
|
+
# self.notify_pending(scan=scan)
|
181
|
+
# self.execute_ewoks_task()
|
182
|
+
|
183
|
+
# def sizeHint(self) -> qt.QSize:
|
184
|
+
# return qt.QSize(650, 60)
|
@@ -281,7 +281,6 @@ class _OMain(ocMain):
|
|
281
281
|
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
282
282
|
# Disable pyqtgraph's atexit and QApplication.aboutToQuit cleanup handlers.
|
283
283
|
pyqtgraph.setConfigOption("exitCleanup", False)
|
284
|
-
print("argv are", argv)
|
285
284
|
super().run(argv)
|
286
285
|
|
287
286
|
def argument_parser(self) -> argparse.ArgumentParser:
|
tomwer/app/zstitching.py
CHANGED
@@ -56,7 +56,6 @@ class MainWidget(qt.QTabWidget):
|
|
56
56
|
"partition": settingsClass.PARTITION,
|
57
57
|
"n_gpus": settingsClass.N_GPUS_PER_WORKER,
|
58
58
|
"job_name": settingsClass.PROJECT_NAME,
|
59
|
-
"walltime": settingsClass.DEFAULT_WALLTIME,
|
60
59
|
"python_venv": settingsClass.PYTHON_VENV,
|
61
60
|
}
|
62
61
|
)
|
tomwer/core/cluster/cluster.py
CHANGED
@@ -48,7 +48,6 @@ class SlurmClusterConfiguration:
|
|
48
48
|
queue=_SlurmSettings.PARTITION,
|
49
49
|
n_gpus=_SlurmSettings.N_GPUS_PER_WORKER,
|
50
50
|
project_name=_SlurmSettings.PROJECT_NAME,
|
51
|
-
walltime=_SlurmSettings.DEFAULT_WALLTIME,
|
52
51
|
python_venv=_SlurmSettings.PYTHON_VENV,
|
53
52
|
n_jobs=_SlurmSettings.N_JOBS,
|
54
53
|
modules_to_load: tuple = _SlurmSettings.MODULES_TO_LOAD,
|
@@ -60,7 +59,6 @@ class SlurmClusterConfiguration:
|
|
60
59
|
self._queue = queue
|
61
60
|
self._n_gpus = n_gpus
|
62
61
|
self._project_name = project_name
|
63
|
-
self._walltime = walltime
|
64
62
|
self._python_venv = python_venv
|
65
63
|
self._modules_to_load = modules_to_load
|
66
64
|
self._n_jobs = n_jobs
|
@@ -98,10 +96,6 @@ class SlurmClusterConfiguration:
|
|
98
96
|
def project_name(self):
|
99
97
|
return self._project_name
|
100
98
|
|
101
|
-
@property
|
102
|
-
def walltime(self):
|
103
|
-
return self._walltime
|
104
|
-
|
105
99
|
@property
|
106
100
|
def python_venv(self):
|
107
101
|
return self._python_venv
|
@@ -132,7 +126,6 @@ class SlurmClusterConfiguration:
|
|
132
126
|
"partition": self.queue,
|
133
127
|
"n_gpus": self.n_gpus,
|
134
128
|
"job_name": self.project_name,
|
135
|
-
"walltime": self.walltime,
|
136
129
|
"python_venv": self.python_venv,
|
137
130
|
"modules": self.modules_to_load,
|
138
131
|
"sbatch_extra_params": self.sbatch_extra_params,
|
@@ -157,8 +150,6 @@ class SlurmClusterConfiguration:
|
|
157
150
|
self._n_gpus = dict_["n_gpus"]
|
158
151
|
if "job_name" in dict_:
|
159
152
|
self._project_name = dict_["job_name"]
|
160
|
-
if "walltime" in dict_:
|
161
|
-
self._walltime = dict_["walltime"]
|
162
153
|
if "python_venv" in dict_:
|
163
154
|
self._python_venv = dict_["python_venv"]
|
164
155
|
modules = dict_.get("modules", None)
|
@@ -60,7 +60,10 @@ try:
|
|
60
60
|
except ImportError:
|
61
61
|
from nxtomomill.converter.hdf5.acquisition.baseacquisition import BaseAcquisition
|
62
62
|
|
63
|
-
|
63
|
+
try:
|
64
|
+
_SCAN_NUMBER_PATH = BaseAcquisition._SCAN_NUMBER_PATH
|
65
|
+
except ImportError:
|
66
|
+
_SCAN_NUMBER_PATH = None
|
64
67
|
|
65
68
|
_logger = logging.getLogger(__name__)
|
66
69
|
|
@@ -62,10 +62,10 @@ _WITH_PAG = "withpag"
|
|
62
62
|
_NO_PAG = "nopag"
|
63
63
|
|
64
64
|
|
65
|
-
DEFAULT_CMP_THETA =
|
66
|
-
DEFAULT_CMP_N_SUBSAMPLING_Y =
|
65
|
+
DEFAULT_CMP_THETA = 5
|
66
|
+
DEFAULT_CMP_N_SUBSAMPLING_Y = 40
|
67
67
|
DEFAULT_CMP_NEAR_POS = 0
|
68
|
-
DEFAULT_CMP_NEAR_WIDTH =
|
68
|
+
DEFAULT_CMP_NEAR_WIDTH = 40
|
69
69
|
DEFAULT_CMP_OVERSAMPLING = 4
|
70
70
|
DEFAULT_CMP_TAKE_LOG = True
|
71
71
|
|
@@ -34,6 +34,7 @@ __date__ = "17/08/2021"
|
|
34
34
|
|
35
35
|
import logging
|
36
36
|
import os
|
37
|
+
import h5py
|
37
38
|
import tempfile
|
38
39
|
from typing import Union
|
39
40
|
|
@@ -198,10 +199,37 @@ class DarkRefsCopy(DarkRefsTask):
|
|
198
199
|
update_mode="replace",
|
199
200
|
)
|
200
201
|
|
202
|
+
def clear_cache(self):
|
203
|
+
"""
|
204
|
+
remove the file used to cache the reduced darks / flats.
|
205
|
+
This can be used in the case it contain unrelevant data. Like frame with another shape...
|
206
|
+
"""
|
207
|
+
cache_file = DarkRefsCopy.get_save_file(self._savedir)
|
208
|
+
if os.path.exists(cache_file):
|
209
|
+
os.remove(cache_file)
|
210
|
+
|
211
|
+
def _clear_cache_data_path(self, file_path: str, data_path: str):
|
212
|
+
if not os.path.exists(file_path):
|
213
|
+
return
|
214
|
+
with h5py.File(file_path, mode="a") as h5f:
|
215
|
+
if data_path in h5f:
|
216
|
+
del h5f[data_path]
|
217
|
+
|
201
218
|
def set_darks_and_flats_from_scan(self, scan: TomwerScanBase) -> bool:
|
202
|
-
|
219
|
+
has_flats = scan.reduced_flats not in (None, {})
|
220
|
+
has_darks = scan.reduced_darks not in (None, {})
|
221
|
+
if has_flats and has_darks:
|
222
|
+
# if the scan has darks and flats remove directly the cache file
|
223
|
+
# else in append mode HDF5 is not removing the dataset and
|
224
|
+
# the cache size will continue to increase
|
225
|
+
self.clear_cache()
|
226
|
+
if not has_flats:
|
203
227
|
logger.warning(f"No flat found for {scan}. Unable to copy them")
|
204
228
|
else:
|
229
|
+
self._clear_cache_data_path(
|
230
|
+
file_path=self._flats_url.file_path(),
|
231
|
+
data_path=self._flats_url.data_path(),
|
232
|
+
)
|
205
233
|
dicttoh5(
|
206
234
|
scan.reduced_flats,
|
207
235
|
h5file=self._flats_url.file_path(),
|
@@ -209,9 +237,13 @@ class DarkRefsCopy(DarkRefsTask):
|
|
209
237
|
mode="a",
|
210
238
|
update_mode="replace",
|
211
239
|
)
|
212
|
-
if
|
240
|
+
if not has_darks:
|
213
241
|
logger.warning(f"No dark found for {scan}. Unable to copy them")
|
214
242
|
else:
|
243
|
+
self._clear_cache_data_path(
|
244
|
+
file_path=self._darks_url.file_path(),
|
245
|
+
data_path=self._darks_url.data_path(),
|
246
|
+
)
|
215
247
|
dicttoh5(
|
216
248
|
scan.reduced_darks,
|
217
249
|
h5file=self._darks_url.file_path(),
|
@@ -247,10 +279,6 @@ class DarkRefsCopy(DarkRefsTask):
|
|
247
279
|
overwrite=True,
|
248
280
|
)
|
249
281
|
|
250
|
-
def clean_save_files(self):
|
251
|
-
if os.path.exists(self._save_file):
|
252
|
-
os.remove(self._save_file)
|
253
|
-
|
254
282
|
def run(self):
|
255
283
|
"""
|
256
284
|
This is function triggered when a new scan / data is received.
|
@@ -128,7 +128,6 @@ def get_nabu_resources_desc(scan: TomwerScanBase, method, workers=1) -> dict:
|
|
128
128
|
"partition": "gpu",
|
129
129
|
"memory_per_node": "90%",
|
130
130
|
"threads_per_node": "100%",
|
131
|
-
"walltime": "01:00:00",
|
132
131
|
}
|
133
132
|
return res
|
134
133
|
|
@@ -305,7 +304,7 @@ def get_multi_cor_recons_volume_identifiers(
|
|
305
304
|
class _NabuMode(_Enum):
|
306
305
|
FULL_FIELD = "standard acquisition"
|
307
306
|
HALF_ACQ = "half acquisition"
|
308
|
-
HELICAL = "helical acquisition"
|
307
|
+
# HELICAL = "helical acquisition"
|
309
308
|
|
310
309
|
|
311
310
|
class _NabuStages(_Enum):
|
@@ -467,6 +466,9 @@ def nabu_std_err_has_error(errs: typing.Optional[bytes]):
|
|
467
466
|
in line # debatable but very disturbing from the gui side... anyway EDF days are coming to an end
|
468
467
|
or "PerformanceWarning" in line
|
469
468
|
or "jitify._init_module()" in line
|
469
|
+
or " unable to import 'siphash24.siphash13" in line
|
470
|
+
or "_create_built_program_from_source_cached" in line
|
471
|
+
or "prg.build(options_bytes," in line
|
470
472
|
)
|
471
473
|
|
472
474
|
if errs is None:
|
tomwer/core/utils/scanutils.py
CHANGED
@@ -40,7 +40,11 @@ from silx.io.utils import h5py_read_dataset
|
|
40
40
|
from tomoscan.esrf.scan.mock import MockEDF as _MockEDF
|
41
41
|
from tomoscan.esrf.scan.mock import MockNXtomo as _MockNXtomo
|
42
42
|
from tomoscan.io import HDF5File
|
43
|
-
|
43
|
+
|
44
|
+
try:
|
45
|
+
from tomoscan.test.utils import MockContext
|
46
|
+
except ImportError:
|
47
|
+
from tomoscan.tests.utils import MockContext
|
44
48
|
|
45
49
|
from tomwer.core.scan.blissscan import BlissScan
|
46
50
|
from tomwer.core.scan.nxtomoscan import NXtomoScan
|
tomwer/gui/cluster/slurm.py
CHANGED
@@ -166,7 +166,6 @@ class SlurmSettingsWindow(qt.QMainWindow):
|
|
166
166
|
"partition": settingsClass.PARTITION,
|
167
167
|
"n_gpus": settingsClass.N_GPUS_PER_WORKER,
|
168
168
|
"job_name": settingsClass.PROJECT_NAME,
|
169
|
-
"walltime": settingsClass.DEFAULT_WALLTIME,
|
170
169
|
"python_venv": settingsClass.PYTHON_VENV,
|
171
170
|
"modules": settingsClass.MODULES_TO_LOAD,
|
172
171
|
}
|
@@ -261,11 +260,6 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
261
260
|
self._nJobsLabel.setVisible(jobLimitation is None)
|
262
261
|
self._nJobs.setVisible(jobLimitation is None)
|
263
262
|
|
264
|
-
# wall time
|
265
|
-
self._wallTimeQLE = qt.QLineEdit("", self)
|
266
|
-
self._wallTimeLabel = qt.QLabel("wall time", self)
|
267
|
-
self.layout().addRow(self._wallTimeLabel, self._wallTimeQLE)
|
268
|
-
|
269
263
|
# python exe / modules
|
270
264
|
self._preProcessingGroup = qt.QGroupBox("pre-processing", self)
|
271
265
|
self._preProcessingGroup.setLayout(qt.QFormLayout())
|
@@ -356,8 +350,6 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
356
350
|
self._sbatchAdvancedParameters.layout().addRow("-C (gpu card)", self._gpuCardCB)
|
357
351
|
|
358
352
|
# simplify gui
|
359
|
-
self._wallTimeLabel.hide()
|
360
|
-
self._wallTimeQLE.hide()
|
361
353
|
self._dashboardPort.hide()
|
362
354
|
self._dashboardPortLabel.hide()
|
363
355
|
self._jobName.hide()
|
@@ -376,7 +368,6 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
376
368
|
self._queue.setCurrentText(SlurmSettings.PARTITION)
|
377
369
|
self._nGpu.setValue(n_gpu)
|
378
370
|
self._jobName.setText(SlurmSettings.PROJECT_NAME)
|
379
|
-
self._wallTimeQLE.setText(SlurmSettings.DEFAULT_WALLTIME)
|
380
371
|
self._modulesCB.setChecked(True) # by default we go for modules
|
381
372
|
self._preProcessingModeChanged()
|
382
373
|
self._partitionChanged()
|
@@ -389,7 +380,6 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
389
380
|
self._queue.currentTextChanged.connect(self._configurationChanged)
|
390
381
|
self._nGpu.valueChanged.connect(self._configurationChanged)
|
391
382
|
self._jobName.editingFinished.connect(self._configurationChanged)
|
392
|
-
self._wallTimeQLE.editingFinished.connect(self._configurationChanged)
|
393
383
|
self._pythonVenv.editingFinished.connect(self._configurationChanged)
|
394
384
|
self._modulesQLE.editingFinished.connect(self._configurationChanged)
|
395
385
|
self._preProcessingButtonGroup.buttonClicked.connect(self._configurationChanged)
|
@@ -401,6 +391,7 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
401
391
|
self._queue.currentTextChanged.connect(self._partitionChanged)
|
402
392
|
self._nGpu.valueChanged.connect(self._nGpuChanged)
|
403
393
|
self._gpuCardCB.currentTextChanged.connect(self._configurationChanged)
|
394
|
+
self._exportValueCM.currentIndexChanged.connect(self._configurationChanged)
|
404
395
|
|
405
396
|
def _nGpuChanged(self, *args, **kwargs):
|
406
397
|
nGpu = self.getNGPU()
|
@@ -451,12 +442,6 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
451
442
|
def setProjectName(self, name):
|
452
443
|
self._jobName.setText(name)
|
453
444
|
|
454
|
-
def getWallTime(self):
|
455
|
-
return self._wallTimeQLE.text()
|
456
|
-
|
457
|
-
def setWallTime(self, walltime):
|
458
|
-
self._wallTimeQLE.setText(walltime)
|
459
|
-
|
460
445
|
def getPythonExe(self):
|
461
446
|
if self._sourceScriptCB.isChecked():
|
462
447
|
return self._pythonVenv.text()
|
@@ -547,10 +532,6 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
547
532
|
if project_name is not None:
|
548
533
|
self.setProjectName(project_name)
|
549
534
|
|
550
|
-
wall_time = config.get("walltime", None)
|
551
|
-
if wall_time is not None:
|
552
|
-
self.setWallTime(wall_time)
|
553
|
-
|
554
535
|
python_venv = config.get("python_venv", None)
|
555
536
|
if python_venv is not None:
|
556
537
|
python_venv = python_venv.rstrip("'").rstrip('"')
|
@@ -581,7 +562,6 @@ class SlurmSettingsWidget(qt.QWidget):
|
|
581
562
|
"partition": self.getQueue(),
|
582
563
|
"n_gpus": self.getNGPU(),
|
583
564
|
"job_name": self.getProjectName(),
|
584
|
-
"walltime": self.getWallTime(),
|
585
565
|
"sbatch_extra_params": self.getSBatchExtraParams(),
|
586
566
|
}
|
587
567
|
if self._modulesCB.isChecked():
|