tomwer 1.2.1__py3-none-any.whl → 1.3.12__py3-none-any.whl

tomwer/app/{saaxis.py → multicor.py}

@@ -8,6 +8,7 @@ import sys
 import time
 from typing import Union
 
+import silx
 from silx.gui import qt
 from silx.io.utils import h5py_read_dataset
 
@@ -120,7 +121,6 @@ class SAAxisWindow(_SAAxisWindow):
 
     def stop(self):
         self._stopProcessingThread()
-        super().stop()
 
     def _computeEstimatedCor(self) -> Union[float, None]:
         scan = self.getScan()
@@ -234,9 +234,9 @@ def main(argv):
     requires_reduced_dark_and_flat(scan=scan, logger_=_logger)
 
     if options.use_opengl_plot:
-        import silx
-
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     window = SAAxisWindow(dump_roi=options.dump_roi)
     window.setWindowTitle("saaxis")

tomwer/app/{sadeltabeta.py → multipag.py}

@@ -8,6 +8,7 @@ import sys
 import time
 from typing import Union
 
+import silx
 from silx.gui import qt
 from silx.io.utils import h5py_read_dataset
 
@@ -198,7 +199,6 @@ class SADeltaBetaWindow(_SADeltaBetaWindow):
 
     def stop(self):
         self._stopProcessingThread()
-        super().stop()
 
     def _computeEstimatedCor(self) -> Union[float, None]:
         scan = self.getScan()
@@ -283,9 +283,9 @@ def main(argv):
         logging.root.setLevel(logging.DEBUG)
 
     if options.use_opengl_plot:
-        import silx
-
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     increase_max_number_file()
     scan = ScanFactory.create_scan_object(

tomwer/app/nabuapp.py

@@ -44,12 +44,6 @@ class DataViewer(_DataViewer):
         if event.key() is qt.Qt.Key_F5:
             self._updateDisplay()
 
-    def event(self, event) -> bool:
-        if event.type() == qt.QEvent.DeferredDelete:
-            self.cleanBeforeQuit()
-            self.closed.emit()
-        return super().event(event)
-
 
 class NabuProcessingThread(qt.QThread):
     def init(self, data, config, dry_run=False):
@@ -185,10 +179,6 @@ class NabuWindow(NabuDialog):
             self._dataViewer.close()
             self._dataViewer = None
 
-    def _aboutToQuit(self):
-        if self._dataViewer is not None:
-            self._dataViewer.cleanBeforeQuit()
-
 
 def main(argv):
     import os
@@ -248,7 +238,6 @@ def main(argv):
     widget.setWindowIcon(icons.getQIcon("tomwer"))
     splash.finish(widget)
     widget.show()
-    qt.QApplication.instance().aboutToQuit.connect(widget._aboutToQuit)
 
     app.exec_()
 

tomwer/app/radiostack.py

@@ -7,9 +7,10 @@ import os
 import signal
 import sys
 
+import silx
 from silx.gui import qt
 
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.utils.resource import increase_max_number_file
 from tomwer.gui import icons
 from tomwer.gui.stacks import RadioStack
@@ -29,7 +30,7 @@ def addFolderAndSubFolder(stack, path):
         _path = os.path.join(path, f)
         if os.path.isdir(_path) is True:
             addFolderAndSubFolder(stack, _path)
-        elif os.path.isfile(_path) and HDF5TomoScan.is_nexus_nxtomo_file(_path):
+        elif os.path.isfile(_path) and NXtomoScan.is_nexus_nxtomo_file(_path):
             stack.addLeafScan(_path)
 
 
@@ -60,10 +61,11 @@ def main(argv):
     increase_max_number_file()
 
     options = parser.parse_args(argv[1:])
-    if options.use_opengl_plot:
-        import silx
 
+    if options.use_opengl_plot:
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     global app  # QApplication must be global to avoid seg fault on quit
     app = qt.QApplication.instance() or qt.QApplication(["tomwer"])

tomwer/app/samplemoved.py

@@ -7,6 +7,7 @@ import os
 import signal
 import sys
 
+import silx
 from silx.gui import qt
 
 from tomwer.core.scan.scanfactory import ScanFactory
@@ -54,9 +55,9 @@ def main(argv):
     options = parser.parse_args(argv[1:])
 
     if options.use_opengl_plot:
-        import silx
-
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     increase_max_number_file()
 

tomwer/app/scanviewer.py

@@ -45,10 +45,12 @@ def main(argv):
         scan_path=options.scan_path, entry=options.entry
     )
 
-    if options.use_opengl_plot:
-        import silx
+    import silx
 
+    if options.use_opengl_plot:
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     increase_max_number_file()
 

tomwer/app/sinogramviewer.py

@@ -7,6 +7,7 @@ import signal
 import sys
 import weakref
 
+import silx
 from silx.gui import qt
 
 from tomwer.core.scan.scanfactory import ScanFactory
@@ -72,9 +73,9 @@ def main(argv):
     options = parser.parse_args(argv[1:])
 
     if options.use_opengl_plot:
-        import silx
-
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     scan = ScanFactory.create_scan_object(options.scan_path, entry=options.entry)
     increase_max_number_file()

tomwer/app/slicestack.py

@@ -7,6 +7,7 @@ import os
 import signal
 import sys
 
+import silx
 from silx.gui import qt
 
 from tomwer.core.utils.resource import increase_max_number_file
@@ -57,9 +58,9 @@ def main(argv):
     options = parser.parse_args(argv[1:])
 
     if options.use_opengl_plot:
-        import silx
-
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     increase_max_number_file()
 

tomwer/app/zstitching.py

@@ -5,18 +5,24 @@ import argparse
 import logging
 import signal
 import sys
+import functools
 
+import silx
 from silx.gui import qt
 
 from nabu.utils import Progress
 from nabu.stitching.config import StitchingType
 
+from tomwer.core.settings import SlurmSettingsMode
 from tomwer.core.utils.resource import increase_max_number_file
+from tomwer.gui import icons
 from tomwer.core.process.stitching.nabustitcher import StitcherTask
 from tomwer.io.utils.tomoobj import get_tomo_objs_instances
 from tomwer.gui.utils.splashscreen import getMainSplashScreen
 from tomwer.gui.stitching.stitching import ZStitchingWindow as _ZStitchingWindow
+from tomwer.gui.stitching import action as stitching_action
 from tomwer.gui.cluster.slurm import SlurmSettingsWidget
+from tomwer.core.scan.scanbase import TomwerScanBase
 
 logging.basicConfig(level=logging.WARNING)
 
@@ -31,12 +37,30 @@ class MainWidget(qt.QTabWidget):
         parent=None,
     ) -> None:
         super().__init__(parent)
-        self._stitchingConfigWindow = _ZStitchingWindow(parent=self)
+        self._stitchingConfigWindow = _ZStitchingWindow(
+            parent=self, with_configuration_action=False
+        )
         self.addTab(self._stitchingConfigWindow, "stitching config")
+        settingsClass = SlurmSettingsMode.get_settings_class(
+            SlurmSettingsMode.STITCHING
+        )
         self._slurmConfig = SlurmSettingsWidget(
-            parent=self, n_gpu=0, jobLimitation=None
+            parent=self, n_gpu=settingsClass.N_GPUS_PER_WORKER, jobLimitation=None
+        )
+        self._slurmConfig.setConfiguration(
+            {
+                "cpu-per-task": settingsClass.N_CORES_PER_TASK,
+                "n_tasks": settingsClass.N_TASKS,
+                "n_jobs": settingsClass.N_JOBS,
+                "memory": settingsClass.MEMORY_PER_WORKER,
+                "partition": settingsClass.PARTITION,
+                "n_gpus": settingsClass.N_GPUS_PER_WORKER,
+                "job_name": settingsClass.PROJECT_NAME,
+                "walltime": settingsClass.DEFAULT_WALLTIME,
+                "python_venv": settingsClass.PYTHON_VENV,
+            }
         )
-        self._slurmConfig.setNJobs(20)
+
         self._stitchingConfigWindow.setCallbackToGetSlurmConfig(
             self._slurmConfig.getConfiguration
         )
@@ -133,16 +157,63 @@ class MainWindow(qt.QDialog):
     def __init__(self, parent=None) -> None:
         super().__init__(parent)
         self.setLayout(qt.QVBoxLayout())
+        # menu
+        self._menuBar = qt.QMenuBar(self)
+        self.layout().addWidget(self._menuBar)
+        self._menu = qt.QMenu("&Stitching")
+        self._menuBar.addMenu(self._menu)
+        # save / load configuration
+        self._loadConfigurationAction = stitching_action.LoadConfigurationAction(
+            self, "&Load configuration"
+        )
+        self._menu.addAction(self._loadConfigurationAction)
+        self._saveConfigurationAction = stitching_action.SaveConfigurationAction(
+            self, "&Save configuration"
+        )
+        self._menu.addAction(self._saveConfigurationAction)
+        # separator
+        self._menu.addSeparator()
+        # add tomo object
+        self._addTomoObjectAction = stitching_action.AddTomoObjectAction(
+            self, "&Add tomo object (volume or NXtomo)"
+        )
+        self._menu.addAction(self._addTomoObjectAction)
+
+        # main window
         self.setWindowTitle("tomwer z-stitching")
         self._mainWindow = MainWidget(parent=self)
         self.layout().addWidget(self._mainWindow)
 
+        # button
         types = qt.QDialogButtonBox.Apply
         self._buttons = qt.QDialogButtonBox(self)
         self._buttons.setStandardButtons(types)
         self.layout().addWidget(self._buttons)
         self._buttons.button(qt.QDialogButtonBox.Apply).setText("Launch stitching")
+
+        # connect signal / slot
         self._buttons.button(qt.QDialogButtonBox.Apply).clicked.connect(self.accept)
+        self._loadConfigurationAction.triggered.connect(
+            functools.partial(
+                self._mainWindow._stitchingConfigWindow._loadSettings, file_path=None
+            )
+        )
+        self._saveConfigurationAction.triggered.connect(
+            functools.partial(
+                self._mainWindow._stitchingConfigWindow._saveSettings, file_path=None
+            )
+        )
+        self._addTomoObjectAction.triggered.connect(self._callbackAddTomoObj)
+
+    def _callbackAddTomoObj(self, *args, **kwargs):
+        """move interface to the z-ordered list and call the 'add tomo obj' callback"""
+        orderedListWidget = (
+            self._mainWindow._stitchingConfigWindow._widget._mainWidget._zOrderedList
+        )
+        self._mainWindow._stitchingConfigWindow._widget._mainWidget.setCurrentWidget(
+            orderedListWidget
+        )
+        orderedListWidget._callbackAddTomoObj()
 
     def close(self):
         self._mainWindow.close()
@@ -227,13 +298,18 @@ def main(argv):
         action="store_true",
         default=False,
     )
-
+    parser.add_argument(
+        "--check-scans",
+        help="If True will check scans before adding them. This check scan validity (virtual dataset contains data...)",
+        action="store_true",
+        default=False,
+    )
     options = parser.parse_args(argv[1:])
 
     if options.use_opengl_plot:
-        import silx
-
         silx.config.DEFAULT_PLOT_BACKEND = "gl"
+    else:
+        silx.config.DEFAULT_PLOT_BACKEND = "matplotlib"
 
     if options.debug:
         logging.root.setLevel(logging.DEBUG)
@@ -258,6 +334,8 @@ def main(argv):
 
     window = MainWindow()
     window.setWindowFlags(qt.Qt.Window)
+    window.setWindowTitle("tomwer: z-stitching")
+    window.setWindowIcon(icons.getQIcon("tomwer"))
 
     window.show()
     from sluurp.utils import has_sbatch_available
@@ -271,6 +349,10 @@ def main(argv):
 
     tomo_objs, (has_scans, has_vols) = get_tomo_objs_instances(options.tomo_objs)
     for tomo_obj in tomo_objs:
+        if isinstance(tomo_obj, TomwerScanBase):
+            tomo_obj.set_check_behavior(
+                run_check=options.check_scans
+            )  # speed up processing
         window.addTomoObj(tomo_obj=tomo_obj)
     if has_scans and has_vols:
         _logger.warning(

tomwer/core/cluster/cluster.py

@@ -31,8 +31,11 @@ __license__ = "MIT"
 __date__ = "12/10/2021"
 
 
+import logging
 from tomwer.core.settings import SlurmSettings as _SlurmSettings
 
+_logger = logging.getLogger(__name__)
+
 
 class SlurmClusterConfiguration:
     """Object shipping the configuration of a slurm cluster"""
@@ -48,6 +51,8 @@ class SlurmClusterConfiguration:
         walltime=_SlurmSettings.DEFAULT_WALLTIME,
         python_venv=_SlurmSettings.PYTHON_VENV,
         n_jobs=_SlurmSettings.N_JOBS,
+        modules_to_load: tuple = _SlurmSettings.MODULES_TO_LOAD,
+        sbatch_extra_params: dict = _SlurmSettings.SBATCH_EXTRA_PARAMS,
     ) -> None:
         self._n_cpu_per_task = n_cpu_per_task
         self._n_task = n_tasks
@@ -57,7 +62,13 @@ class SlurmClusterConfiguration:
         self._project_name = project_name
         self._walltime = walltime
         self._python_venv = python_venv
+        self._modules_to_load = modules_to_load
         self._n_jobs = n_jobs
+        self._sbatch_extra_params = sbatch_extra_params
+        if python_venv not in (None, "") and len(modules_to_load) > 0:
+            _logger.warning(
+                "Either 'modules to load' of 'python venv' can be provided. Not both"
+            )
 
     @property
     def n_cpu_per_task(self):
@@ -95,11 +106,19 @@ class SlurmClusterConfiguration:
     def python_venv(self):
         return self._python_venv
 
+    @property
+    def modules_to_load(self) -> tuple:
+        return self._modules_to_load
+
     @property
     def port_range(self) -> tuple:
         """port range as (start:int, strop:int, step: int)"""
         return self._port_range
 
+    @property
+    def sbatch_extra_params(self) -> dict:
+        return self._sbatch_extra_params
+
     @property
     def dashboard_port(self):
         return self._dashboard_port
@@ -115,6 +134,8 @@ class SlurmClusterConfiguration:
             "job_name": self.project_name,
             "walltime": self.walltime,
             "python_venv": self.python_venv,
+            "modules": self.modules_to_load,
+            "sbatch_extra_params": self.sbatch_extra_params,
         }
 
     @staticmethod
@@ -140,8 +161,13 @@ class SlurmClusterConfiguration:
             self._walltime = dict_["walltime"]
         if "python_venv" in dict_:
             self._python_venv = dict_["python_venv"]
+        modules = dict_.get("modules", None)
+        if modules is not None:
+            self._modules_to_load = modules
         if "port_range" in dict_:
             self._port_range = dict_["port_range"]
         if "dashboard_port" in dict_:
             self._dashboard_port = dict_["dashboard_port"]
+        if "sbatch_extra_params" in dict_:
+            self._sbatch_extra_params = dict_["sbatch_extra_params"]
         return self

tomwer/core/log/logger.py

@@ -51,10 +51,14 @@ from tomwer.core.log.processlog import (
 
 # These are the sequences need to get colored ouput
 _RESET_SEQ = "\033[0m"
-_COLOR_SEQ = "\033[1;%dm"
 _BOLD_SEQ = "\033[1m"
 
-_BLACK, _RED, _GREEN, _YELLOW, _BLUE, _MAGENTA, _CYAN, _WHITE = range(8)
+_BLACK = "\033[30m"
+_RED = "\033[31m"
+_GREEN = "\033[32m"
+_YELLOW = "\033[33m"
+_BLUE = "\033[34m"
+_MAGENTA = "\033[35m"
 
 LOG_COLORS = {
     "WARNING": _MAGENTA,
@@ -90,9 +94,7 @@ class _ColoredFormatter(logging.Formatter):
     def format(self, record):
         levelname = record.levelname
         if self.use_color is True and levelname in LOG_COLORS:
-            levelname_color = (
-                _COLOR_SEQ % (30 + LOG_COLORS[levelname]) + levelname + _RESET_SEQ
-            )
+            levelname_color = LOG_COLORS[levelname] + levelname + _RESET_SEQ
             record.levelname = levelname_color
         record.asctime = self.formatTime(record, self.datefmt)
         return logging.Formatter.format(self, record)

tomwer/core/process/conditions/filters.py

@@ -41,7 +41,7 @@ from tomwer.core.process.task import Task
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 
-from silx.utils.deprecation import deprecated_warning
+from tomwer.core.utils.deprecation import deprecated_warning
 
 _logger = logging.getLogger(__name__)
 

tomwer/core/process/control/datalistener/datalistener.py

@@ -37,13 +37,13 @@ import typing
 from ewokscore.task import Task as EwoksTask
 from nxtomomill import converter as nxtomomill_converter
 from silx.io.utils import h5py_read_dataset
-from tomoscan.io import HDF5File
+from silx.io.utils import open as open_hdf5
 
 import tomwer.version
 from tomwer.core import settings
 from tomwer.core.process.task import BaseProcessInfo
 from tomwer.core.scan.blissscan import BlissScan
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 
 from .rpcserver import DataListenerThread
 
@@ -102,6 +102,11 @@ class DataListener(BaseProcessInfo):
     TIMOUT_READ_FILE = 30
     "When the event 'scan_ended' is received all data might not have been write" " yet"
 
+    SWMR_MODE = None
+    """The bliss writer is not using the swmr mode. This class has independant behavior regarding the tomoscan / nxotmo get_swmr_mode which is
+    dedicated to the internal tomotools behavior
+    """
+
     def __init__(self):
         super().__init__()
         self._host = settings.JSON_RPC_HOST
@@ -191,7 +196,7 @@ class DataListener(BaseProcessInfo):
         for mf_emf in self.convert(bliss_file=sample_file, entry=entry):
             master_file, entry_master_file = mf_emf
             if master_file is not None and entry_master_file is not None:
-                scan = HDF5TomoScan(scan=master_file, entry=entry_master_file)
+                scan = NXtomoScan(scan=master_file, entry=entry_master_file)
 
                 try:
                     # register process.
@@ -224,7 +229,7 @@ class DataListener(BaseProcessInfo):
         """Return the list of scans dir for this bliss_file / entry"""
 
         def get_scan_indexes():
-            with HDF5File(bliss_file, "r", swmr=True) as h5f:
+            with open_hdf5(bliss_file) as h5f:
                 entry_node = h5f[entry]
                 if _SCAN_NUMBER_PATH in entry_node:
                     return h5py_read_dataset(entry_node[_SCAN_NUMBER_PATH])
@@ -249,7 +254,7 @@ class DataListener(BaseProcessInfo):
         """Return the proposal file of the experimentation if registred by the
         data listener"""
         if entry is None:
-            with HDF5File(process_file, "r", swmr=True) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -264,7 +269,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=True) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -306,7 +311,7 @@ class DataListener(BaseProcessInfo):
         """Return the proposal file of the experimentation if registred by the
         data listener"""
         if entry is None:
-            with HDF5File(process_file, "r", swmr=True) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -321,7 +326,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=True) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -355,7 +360,7 @@ class DataListener(BaseProcessInfo):
         """Return the proposal file of the experimentation if registred by the
         data listener"""
         if entry is None:
-            with HDF5File(process_file, "r", swmr=True) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -370,7 +375,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=True) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -405,7 +410,7 @@ class DataListener(BaseProcessInfo):
         specific sequence data
         """
         if entry is None:
-            with HDF5File(process_file, "r", swmr=True) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -420,7 +425,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=True) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -498,7 +503,7 @@ class DataListener(BaseProcessInfo):
         # write
         def sequence_is_finished():
             try:
-                with HDF5File(bliss_file, "r", swmr=True) as h5f:
+                with open_hdf5(bliss_file) as h5f:
                     end_scan_path = "/".join((entry, "end_time"))
                     return end_scan_path in h5f
             except Exception:
@@ -548,7 +553,7 @@ class DataListener(BaseProcessInfo):
         return res == "Y"
 
     def _signal_scan_ready(self, scan):
-        assert isinstance(scan, HDF5TomoScan)
+        assert isinstance(scan, NXtomoScan)
         pass
 
     def activate(self, activate=True):

tomwer/core/process/control/datawatcher/edfdwprocess.py

@@ -50,15 +50,6 @@ except ImportError:
     has_rsync = True
 
 
-def get_info_val(lines, key):
-    r = range(len(lines))
-    key = key + "="
-    for i in r:
-        if key in lines[i]:
-            val = float(lines[i].split("=")[1])
-    return val
-
-
 class _DataWatcherEDFProcess(_DataWatcherProcess):
     """
     Base class for edf acquisition observation