tomwer 1.2.0a1__py3-none-any.whl → 1.2.0a3__py3-none-any.whl

tomwer/core/process/test/test_nabu.py

@@ -127,7 +127,7 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + ".cfg",
-            os.path.basename(self.scan.path) + "slice_0001" + ".cfg",
+            os.path.basename(self.scan.path) + "slice_000001" + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -151,7 +151,7 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + "pag_db0100.cfg",
-            os.path.basename(self.scan.path) + "slice_pag_0001_db0100" + ".cfg",
+            os.path.basename(self.scan.path) + "slice_pag_000001_db0100" + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -178,7 +178,7 @@ class TestNabuIni(unittest.TestCase):
             ini_files.append(
                 os.path.basename(self.scan.path)
                 + "slice_"
-                + str(slice_index).zfill(4)
+                + str(slice_index).zfill(6)
                 + ".cfg"
             )
 
@@ -222,7 +222,7 @@ class TestNabuIni(unittest.TestCase):
             # add slice ini file
             ini_files.append(
                 os.path.basename(self.scan.path)
-                + "slice_pag_0001_db"
+                + "slice_pag_000001_db"
                 + str(int(pag_db)).zfill(4)
                 + ".cfg"
             )
@@ -265,7 +265,7 @@ class TestNabuIni(unittest.TestCase):
                 ini_files.append(
                     os.path.basename(self.scan.path)
                     + "slice_pag_"
-                    + str(slice_index).zfill(4)
+                    + str(slice_index).zfill(6)
                     + "_db"
                     + str(pag_db).zfill(4)
                     + ".cfg"
@@ -365,9 +365,14 @@ class TestNabuAndAxis(unittest.TestCase):
         # initial configuration
         self.conf = get_nabu_default_config(output_dir=self.scan.path)
 
-        self.nabu_process = nabu.NabuSlices(inputs={"data": self.scan})
-        self.nabu_process.set_dry_run(True)
-        self.nabu_process.set_configuration(self.conf)
+        self.nabu_process = nabu.NabuSlicesTask(
+            inputs={
+                "data": self.scan,
+                "nabu_params": self.conf,
+                "dry_run": True,
+                "serialize_output_data": False,
+            },
+        )
         self.scan._axis_params = AxisRP()
         self.nabu_cfg_folder = os.path.join(
             self.scan.path, nabu_settings.NABU_CFG_FILE_FOLDER

tomwer/core/process/test/test_normalization.py

@@ -93,6 +93,7 @@ class TestNormalization(unittest.TestCase):
                     inputs={
                         "data": self.scan,
                         "configuration": process_params,
+                        "serialize_output_data": False,
                     }
                 )
                 process.run()

tomwer/core/process/test/test_timer.py

@@ -32,7 +32,7 @@ import shutil
 import tempfile
 import unittest
 
-from tomwer.core.process.control.timer import Timer
+from tomwer.core.process.control.timer import TimerTask
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.scanutils import MockEDF
 
@@ -53,25 +53,25 @@ class TestTimerIO(unittest.TestCase):
     def testInputOutput(self):
         """Test that io using TomoBase instance work"""
         for input_type in (dict, TomwerScanBase):
-            for return_dict in (True, False):
+            for serialize_output_data in (True, False):
                 with self.subTest(
-                    return_dict=return_dict,
+                    return_dict=serialize_output_data,
                     input_type=input_type,
                 ):
                     input_obj = self.scan
                     if input_obj is dict:
                         input_obj = input_obj.to_dict()
 
-                    timer_process = Timer(
+                    timer_process = TimerTask(
                         inputs={
                             "wait": 0.1,
-                            "return_dict": return_dict,
+                            "serialize_output_data": serialize_output_data,
                             "data": input_obj,
                         }
                     )
                     timer_process.run()
                     out = timer_process.outputs.data
-                    if return_dict:
+                    if serialize_output_data:
                         self.assertTrue(isinstance(out, dict))
                     else:
                         self.assertTrue(isinstance(out, TomwerScanBase))

tomwer/core/process/visualization/dataviewer.py

@@ -31,5 +31,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _DataViewerPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    Task to browse a scan and associated processing (display raw frames, normalized projections, recosntructed slices...)
+    """
+
     def run(self):
         pass

tomwer/core/process/visualization/diffviewer.py

@@ -31,5 +31,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _DiffViewerPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    task to compare frames (between one or several dataset)
+    """
+
     def run(self):
         pass

tomwer/core/process/visualization/imagestackviewer.py

@@ -31,5 +31,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _ImageStackViewerPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    Task to aggregate several images
+    """
+
     def run(self):
         pass

tomwer/core/process/visualization/radiostack.py

@@ -31,5 +31,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _RadioStackPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    Task to aggregate radios of several scans and allow to browse along them all
+    """
+
     def run(self):
         pass

tomwer/core/process/visualization/samplemoved.py

@@ -32,5 +32,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _SampleMovedPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    Task to check if a sample has moved along time / projections
+    """
+
     def run(self):
         pass

tomwer/core/process/visualization/sinogramviewer.py

@@ -31,5 +31,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _SinogramViewerPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    Task to display a sinogram
+    """
+
     def run(self):
         pass

tomwer/core/process/visualization/slicestack.py

@@ -32,5 +32,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _SliceStackPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    Task to aggregate reconstructed slices of several scans and allow users to along them
+    """
+
     def run(self):
         pass

tomwer/core/process/visualization/volumeviewer.py

@@ -32,5 +32,9 @@ from ewokscore.task import Task as EwoksTask
 
 
 class _VolumeViewerPlaceHolder(EwoksTask, input_names=("data",)):
+    """
+    Task to plot a (reconstructed) volume
+    """
+
     def run(self):
         pass

tomwer/core/scan/hdf5scan.py

@@ -311,10 +311,10 @@ class HDF5TomoScan(_tsHDF5TomoScan, TomwerScanBase):
         if self._reduced_darks is None:
             if self.process_file is not None and os.path.exists(self.process_file):
                 from tomwer.core.process.reconstruction.darkref.darkrefs import (
-                    DarkRefs,
+                    DarkRefsTask,
                 )  # avoid cyclic import
 
-                reduced_darks = DarkRefs.get_darks_frm_process_file(
+                reduced_darks = DarkRefsTask.get_darks_frm_process_file(
                     process_file=self.process_file, entry=self.entry
                 )
                 if reduced_darks is not None:
@@ -326,10 +326,10 @@ class HDF5TomoScan(_tsHDF5TomoScan, TomwerScanBase):
         if self._reduced_flats is None:
             if self.process_file is not None and os.path.exists(self.process_file):
                 from tomwer.core.process.reconstruction.darkref.darkrefs import (
-                    DarkRefs,
+                    DarkRefsTask,
                 )  # avoid cyclic import
 
-                reduced_flats = DarkRefs.get_flats_frm_process_file(
+                reduced_flats = DarkRefsTask.get_flats_frm_process_file(
                     process_file=self.process_file, entry=self.entry
                 )
                 if reduced_flats is not None:

tomwer/core/scan/scanbase.py

@@ -411,7 +411,11 @@ class TomwerScanBase(TomwerObject):
         res = {}
         # nabu reconstruction parameters
         if self._nabu_params:
-            res[self._DICT_NABU_RP_KEY] = self.nabu_recons_params.to_dict()
+            res[self._DICT_NABU_RP_KEY] = (
+                self.nabu_recons_params
+                if isinstance(self.nabu_recons_params, dict)
+                else self.nabu_recons_params.to_dict()
+            )
         else:
             res[self._DICT_NABU_RP_KEY] = None
         # axis reconstruction parameters

tomwer/core/scan/test/test_process_registration.py

@@ -68,55 +68,55 @@ class TestProcessRegistration(unittest.TestCase):
         shutil.rmtree(self.tmp_dir)
 
     def testGetCorValues(self):
-        from tomwer.core.process.reconstruction.axis import AxisProcess
+        from tomwer.core.process.reconstruction.axis import AxisTask
 
         for i in range(20):
             results = {"center_of_rotation": i}
             Task._register_process(
                 self.scan.process_file,
-                process=AxisProcess,
+                process=AxisTask,
                 entry=self.scan.entry,
                 configuration=None,
                 results=results,
                 process_index=i,
             )
-        cor_value = AxisProcess.get_cor_frm_process_file(
+        cor_value = AxisTask.get_cor_frm_process_file(
             self.scan.process_file, entry=self.scan.entry
         )
         self.assertEqual(cor_value, 19)
 
     def testGetDarks(self):
-        from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefs
+        from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefsTask
 
         for i in range(20):
             results = {"darks": {"0": i * 2}, "flats": {"1": i}}
             Task._register_process(
                 self.scan.process_file,
-                process=DarkRefs,
+                process=DarkRefsTask,
                 entry=self.scan.entry,
                 configuration=None,
                 results=results,
                 process_index=i,
             )
-        flats = DarkRefs.get_darks_frm_process_file(
+        flats = DarkRefsTask.get_darks_frm_process_file(
             self.scan.process_file, entry=self.scan.entry
         )
         self.assertEqual(flats[list(flats.keys())[0]], 19 * 2)
 
     def testGetFlats(self):
-        from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefs
+        from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefsTask
 
         for i in range(20):
             results = {"darks": {"0": i}, "flats": {"1": i}}
             Task._register_process(
                 self.scan.process_file,
-                process=DarkRefs,
+                process=DarkRefsTask,
                 entry=self.scan.entry,
                 configuration=None,
                 results=results,
                 process_index=i,
             )
-        flats = DarkRefs.get_flats_frm_process_file(
+        flats = DarkRefsTask.get_flats_frm_process_file(
             self.scan.process_file, entry=self.scan.entry
         )
         self.assertEqual(flats[list(flats.keys())[0]], 19)

tomwer/core/settings.py

@@ -29,6 +29,8 @@ __date__ = "02/06/2017"
 
 
 import os
+from silx.utils.enum import Enum as _Enum
+
 
 __LBSRAM_PATH = '/lbsram'
 
@@ -84,8 +86,33 @@ def _set_dest_path(path):
     __DEST_PATH = path
 
 
+class SlurmSettingsMode(_Enum):
+    MANUAL = "manual"
+    GENERIC = "generic"
+    CAST_VOLUME = "cast_volume"
+    SLICE_RECONSTRUCTION = "slice_reconstruction"
+    VOLUME_RECONSTRUCTION = "volume_reconstruction"
+
+    @staticmethod
+    def get_settings_class(mode):
+        assert isinstance(mode, SlurmSettingsMode)
+        if mode is SlurmSettingsMode.MANUAL:
+            return None
+        elif mode is SlurmSettingsMode.GENERIC:
+            return SlurmSettings
+        elif mode is SlurmSettingsMode.CAST_VOLUME:
+            return DefaultSlurmSettingsCastVolume
+        elif mode is SlurmSettingsMode.SLICE_RECONSTRUCTION:
+            return DefaultSlurmSettingsSliceReconstruction
+        elif mode is SlurmSettingsMode.VOLUME_RECONSTRUCTION:
+            return DefaultSlurmSettingsVolumeReconstruction
+        else:
+            raise ValueError(f"{mode} not handled")
+
+
 class SlurmSettings:
     # Default slurm cluster configuration
+
     N_CORES_PER_TASK = 4
     """Number of CPU per worker"""
 
@@ -95,7 +122,7 @@ class SlurmSettings:
     N_JOBS = 1
     """on how many job we want to split the EwoksTask"""
 
-    MEMORY_PER_WORKER = 50  # memory in GB
+    MEMORY_PER_WORKER = 64  # memory in GB
     """Amount of memory per worker"""
 
     PARTITION = "p9gpu"
@@ -113,3 +140,34 @@ class SlurmSettings:
     PROJECT_NAME = "tomwer_{scan}_-_{process}_-_{info}"
     """Slurm cluster project name. `scan`, `process` and `info` will be format.
     """
+
+
+class DefaultSlurmSettingsCastVolume(SlurmSettings):
+    """
+    default proposed configuration for casting a volume remotly
+    """
+    N_GPUS_PER_WORKER = 0
+
+    N_CORES_PER_TASK = 8
+
+    MEMORY_PER_WORKER = 128  # memory in GB
+
+    PARTITION = "nice*"
+
+
+class DefaultSlurmSettingsSliceReconstruction(SlurmSettings):
+    """
+    default proposed configuration for reconstructing a single slice remotly
+    """
+    MEMORY_PER_WORKER = 256  # memory in GB
+
+    PARTITION = None
+
+
+class DefaultSlurmSettingsVolumeReconstruction(SlurmSettings):
+    """
+    default proposed configuration for reconstructing a full volume remotly
+    """
+    MEMORY_PER_WORKER = 256  # memory in GB
+
+    PARTITION = None

tomwer/core/test/test_lamino.py

@@ -44,7 +44,7 @@ class TestTofuReconstruction(unittest.TestCase):
     def setUp(self):
         self._scan_path = tempfile.mkdtemp()
         self.scan = EDFTomoScan(self._scan_path)
-        self.reconstruction = tofu.LaminoReconstruction(
+        self.reconstruction = tofu.LaminoReconstructionTask(
             inputs={
                 "dry_run": True,
                 "data": self.scan,
@@ -53,6 +53,7 @@ class TestTofuReconstruction(unittest.TestCase):
                     "projections": "tata-yoyo",
                     "output": "output_dir",
                 },
+                "serialize_output_data": False,
             }
         )
 

tomwer/core/utils/__init__.py

@@ -448,3 +448,19 @@ def DownloadDataset(dataset, output_folder, timeout, unpack=False):
     if unpack is True:
         shutil.unpack_archive(archive_folder, extract_dir=output_folder, format="bztar")
         os.remove(archive_folder)
+
+
+def concatenate_dict(dict_1, dict_2) -> dict:
+    """update dict which has dict as values. And we want concatenate those values to"""
+    res = dict_1.copy()
+    for key in dict_2:
+        if key in dict_1:
+            if key in [f"axis_{axis}_params" for axis in (0, 1, 2)]:
+                res[key] = ";".join((dict_1[key], dict_2[key]))
+            elif isinstance(dict_1[key], dict):
+                res[key] = concatenate_dict(dict_1=dict_1[key], dict_2=dict_2[key])
+            else:
+                res.update({key: dict_2[key]})
+        else:
+            res[key] = dict_2[key]
+    return res

tomwer/core/utils/locker.py

@@ -81,6 +81,5 @@ class FileLockerManager:
         #     from pathlib import Path
         #     Path(lock_file_path).touch()
         if lock_file_path not in _FILE_LOCKERS:
-            print("create lock to", lock_file_path)
             _FILE_LOCKERS[lock_file_path] = Lock(lock_file_path, default_timeout=3)
         return _FILE_LOCKERS[lock_file_path]

tomwer/core/utils/resource.py

@@ -27,6 +27,7 @@ __license__ = "MIT"
 __date__ = "25/11/2021"
 
 import logging
+import resource
 
 _logger = logging.getLogger(__name__)
 
@@ -34,15 +35,9 @@ _logger = logging.getLogger(__name__)
 def increase_max_number_file():
     """increase the maximal number of file the software can open within respect of the hard limit"""
     try:
-        import resource
-    except ImportError:
-        _logger.debug("No resource module available")
+        hard_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+        resource.setrlimit(resource.RLIMIT_NOFILE, (hard_nofile, hard_nofile))
+    except (ValueError, OSError):
+        _logger.warning("Failed to retrieve and set the max opened files limit")
     else:
-        if hasattr(resource, "RLIMIT_NOFILE"):
-            try:
-                hard_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-                resource.setrlimit(resource.RLIMIT_NOFILE, (hard_nofile, hard_nofile))
-            except (ValueError, OSError):
-                _logger.warning("Failed to retrieve and set the max opened files limit")
-            else:
-                _logger.debug("Set max opened files to %d", hard_nofile)
+        _logger.debug("Set max opened files to %d", hard_nofile)

tomwer/core/utils/scanutils.py

@@ -407,6 +407,8 @@ def data_identifier_to_scan(data_identifier: Any):
     """
     if isinstance(data_identifier, str):
         return ScanFactory.create_tomo_object_from_identifier(data_identifier)
+    elif isinstance(data_identifier, dict):
+        return ScanFactory.create_scan_object_frm_dict(data_identifier)
     else:
         return data_identifier
 

tomwer/gui/cluster/slurm.py

@@ -33,7 +33,8 @@ from typing import Optional
 from silx.gui import qt
 from sluurp.utils import get_partitions
 
-from tomwer.core.settings import SlurmSettings
+from tomwer.core.settings import SlurmSettings, SlurmSettingsMode
+from tomwer.gui.utils.qt_utils import block_signals
 
 
 class SlurmSettingsDialog(qt.QDialog):
@@ -44,7 +45,8 @@ class SlurmSettingsDialog(qt.QDialog):
         super().__init__(parent=parent)
 
         self.setLayout(qt.QVBoxLayout())
-        self._mainWidget = SlurmSettingsWidget(parent=self)
+        self._mainWidget = SlurmSettingsWindow(parent=self)
+        self._mainWidget.setWindowFlags(qt.Qt.Widget)
         self.layout().addWidget(self._mainWidget)
 
         # buttons for validation
@@ -71,6 +73,88 @@ class SlurmSettingsDialog(qt.QDialog):
         self._mainWidget.setConfiguration(config=config)
 
 
+class SlurmSettingsWindow(qt.QMainWindow):
+    """
+    Main window to define slurm settings.
+    Composed of the SlurmSettingsWidget and a combobox with some predefined settings
+    """
+
+    sigConfigChanged = qt.Signal()
+    """Signal emit when the SlurmSetting changed"""
+
+    def __init__(self, parent: Optional[qt.QWidget] = None) -> None:
+        super().__init__(parent)
+        self._mainWidget = qt.QWidget(self)
+        self._mainWidget.setLayout(qt.QFormLayout())
+
+        self._modeCombox = qt.QComboBox(self)
+        self._mainWidget.layout().addRow("configuration: ", self._modeCombox)
+        self._modeCombox.addItems(SlurmSettingsMode.values())
+        self._modeCombox.setCurrentText(SlurmSettingsMode.GENERIC.value)
+
+        self._settingsWidget = SlurmSettingsWidget(self)
+        self._mainWidget.layout().addRow(self._settingsWidget)
+
+        self.setCentralWidget(self._mainWidget)
+
+        # set up
+        self._reloadPredefinedSettings()
+
+        # connect signal / slot
+        self._modeCombox.currentIndexChanged.connect(self._reloadPredefinedSettings)
+        self._settingsWidget.sigConfigChanged.connect(self._configChanged)
+        # when the settings widget is edited them we automatically move to 'manual' settings. To notify visually the user
+        self._settingsWidget.sigConfigChanged.connect(self._switchToManual)
+
+    def _reloadPredefinedSettings(self, *args, **kkwargs):
+        """
+        reload settings from some predefined configuration
+        """
+        mode = self.getCurrentSettingsMode()
+        settingsClass = SlurmSettingsMode.get_settings_class(mode)
+        if settingsClass:
+            with block_signals(self._settingsWidget):
+                self.setConfiguration(
+                    {
+                        "cpu-per-task": settingsClass.N_CORES_PER_TASK,
+                        "n_tasks": settingsClass.N_TASKS,
+                        "n_jobs": settingsClass.N_JOBS,
+                        "memory": settingsClass.MEMORY_PER_WORKER,
+                        "partition": settingsClass.PARTITION,
+                        "n_gpus": settingsClass.N_GPUS_PER_WORKER,
+                        "job_name": settingsClass.PROJECT_NAME,
+                        "walltime": settingsClass.DEFAULT_WALLTIME,
+                        "python_venv": settingsClass.PYTHON_VENV,
+                    }
+                )
+
+    def _configChanged(self, *args, **kwargs):
+        self.sigConfigChanged.emit()
+
+    def _switchToManual(self):
+        self._modeCombox.setCurrentText(SlurmSettingsMode.MANUAL.value)
+
+    def getCurrentSettingsMode(self) -> SlurmSettingsMode:
+        return SlurmSettingsMode.from_value(self._modeCombox.currentText())
+
+    def setCurrentSettingsMode(self, mode: SlurmSettingsMode) -> SlurmSettingsMode:
+        mode = SlurmSettingsMode.from_value(mode)
+        self._modeCombox.setCurrentText(mode.value)
+
+    # expose API
+    def getConfiguration(self) -> dict:
+        return self._settingsWidget.getConfiguration()
+
+    def getSlurmClusterConfiguration(self) -> dict:
+        return self._settingsWidget.getSlurmClusterConfiguration()
+
+    def setConfiguration(self, config: dict) -> None:
+        self._settingsWidget.setConfiguration(config=config)
+
+    def isSlurmActive(self):
+        return self._settingsWidget.isSlurmActive()
+
+
 class SlurmSettingsWidget(qt.QWidget):
     """Widget used to define Slurm configuration to be used"""
 
@@ -212,19 +296,19 @@ class SlurmSettingsWidget(qt.QWidget):
         return self._nCores.value()
 
     def setNCores(self, n: int) -> None:
-        self._nCores.setValue(n)
+        self._nCores.setValue(int(n))
 
     def getNWorkers(self) -> int:
         return self._nWorkers.value()
 
     def setNWorkers(self, n) -> None:
-        self._nWorkers.setValue(n)
+        self._nWorkers.setValue(int(n))
 
     def getMemory(self) -> int:
         return self._memory.value()
 
     def setMemory(self, memory: int) -> None:
-        self._memory.setValue(memory)
+        self._memory.setValue(int(memory))
 
     def getQueue(self) -> str:
         return self._queue.currentText()
@@ -236,13 +320,13 @@ class SlurmSettingsWidget(qt.QWidget):
         return self._nGpu.value()
 
     def setNGPU(self, n: int) -> None:
-        self._nGpu.setValue(n)
+        self._nGpu.setValue(int(n))
 
     def getNJobs(self) -> int:
         return self._nJobs.value()
 
     def setNJobs(self, value: int):
-        self._nJobs.setValue(value)
+        self._nJobs.setValue(int(value))
 
     def getProjectName(self):
         return self._jobName.text()