tomwer 1.3.0.dev2__py3-none-any.whl → 1.3.0rc10__py3-none-any.whl

tomwer/core/process/task.py

@@ -46,7 +46,8 @@ from ewokscore.task import Task as _EwoksTask
 from ewokscore.taskwithprogress import TaskWithProgress as _EwoksTaskWithProgress
 from silx.io.dictdump import dicttoh5, h5todict
 from silx.io.utils import h5py_read_dataset
-from tomoscan.io import HDF5File
+from tomoscan.io import HDF5File, get_swmr_mode
+from tomwer.core.utils.locker import FileLockerManager
 
 
 _logger = logging.getLogger(__name__)
@@ -173,19 +174,20 @@ class BaseProcessInfo:
         :type interpretations: Union[None, dict
         """
         assert process_file is not None
-        try:
-            self._register_process(
-                process_file=process_file,
-                entry=entry,
-                process=self,
-                configuration=configuration,
-                results=results,
-                process_index=process_index,
-                interpretations=interpretations,
-                overwrite=overwrite,
-            )
-        except IOError as e:
-            _logger.error(e)
+        with FileLockerManager().get_lock(file_name=process_file):
+            try:
+                self._register_process(
+                    process_file=process_file,
+                    entry=entry,
+                    process=self,
+                    configuration=configuration,
+                    results=results,
+                    process_index=process_index,
+                    interpretations=interpretations,
+                    overwrite=overwrite,
+                )
+            except IOError as e:
+                _logger.error(e)
 
     @staticmethod
     def _register_process(
@@ -373,8 +375,8 @@ class BaseProcessInfo:
         :return: list of _process_desc(sequence_index, configuration, results)
         :rtype: list
         """
-        processes = []
-        with HDF5File(process_file, mode="r", swmr=True) as h5f:
+        # retrieve process to load
+        with HDF5File(process_file, mode="r", swmr=get_swmr_mode()) as h5f:
             if entry is None:
                 if len(h5f.keys()) > 0:
                     root = h5f[list(h5f.keys())[0]]
@@ -382,24 +384,28 @@ class BaseProcessInfo:
                     _logger.warning("no process find")
             else:
                 root = h5f[entry]
+
+            processes_to_load = {}
             for process in root.keys():
                 nx_process = root[process]
                 if (
                     h5py_read_dataset(nx_process["program"])
                     == process_type.program_name()
                 ):
-                    p_order = h5py_read_dataset(nx_process["sequence_index"])
-                    config = h5todict(
-                        process_file, "/".join((nx_process.name, "configuration"))
-                    )
-                    results = h5todict(
-                        process_file, "/".join((nx_process.name, "results"))
-                    )
-                    processes.append(
-                        _process_desc(
-                            process_order=p_order, configuration=config, results=results
-                        )
+                    processes_to_load[nx_process.name] = h5py_read_dataset(
+                        nx_process["sequence_index"]
                     )
+
+        # load process
+        processes = []
+        for process_name, p_order in processes_to_load.items():
+            config = h5todict(process_file, "/".join((process_name, "configuration")))
+            results = h5todict(process_file, "/".join((process_name, "results")))
+            processes.append(
+                _process_desc(
+                    process_order=p_order, configuration=config, results=results
+                )
+            )
         return processes
 
 

tomwer/core/process/test/test_axis.py

@@ -34,6 +34,7 @@ import tempfile
 import unittest
 
 import h5py
+from tomoscan.io import get_swmr_mode
 import numpy
 from silx.io.utils import h5py_read_dataset
 
@@ -93,9 +94,9 @@ class TestAxisIO(unittest.TestCase):
                     )
 
                     # patch the axis process
-                    axis_process._CALCULATIONS_METHODS[
-                        AxisMode.centered
-                    ] = TestAxisIO._random_calc
+                    axis_process._CALCULATIONS_METHODS[AxisMode.centered] = (
+                        TestAxisIO._random_calc
+                    )
 
                     axis_process.run()
                     out = axis_process.outputs.data
@@ -131,7 +132,7 @@ class TestAxis(unittest.TestCase):
         axis_process.run()
         self.assertTrue(os.path.exists(scan.process_file))
 
-        with h5py.File(scan.process_file, "r", swmr=True) as h5f:
+        with h5py.File(scan.process_file, "r", swmr=get_swmr_mode()) as h5f:
             self.assertTrue("entry" in h5f)
             self.assertTrue("tomwer_process_0" in h5f["entry"])
             group_axis = h5f["entry"]["tomwer_process_0"]
@@ -183,7 +184,7 @@ class TestAxis(unittest.TestCase):
         # make sure center of position has been computed
         self.assertTrue(os.path.exists(scan.process_file))
 
-        with h5py.File(scan.process_file, "r", swmr=True) as h5f:
+        with h5py.File(scan.process_file, "r", swmr=get_swmr_mode()) as h5f:
             self.assertTrue("entry" in h5f)
             self.assertTrue("tomwer_process_0" in h5f["entry"])
             group_axis = h5f["entry"]["tomwer_process_0"]
@@ -260,7 +261,7 @@ class TestSinogramAlgorithm(unittest.TestCase):
             n_proj=10,
             n_ini_proj=10,
             create_ini_dark=True,
-            create_final_ref=True,
+            create_final_flat=True,
             scan_range=360,
             dim=dim,
         ).scan

tomwer/core/process/test/test_data_transfer.py

@@ -42,7 +42,7 @@ from nxtomomill.converter import from_h5_to_nx
 from nxtomomill.io.config.hdf5config import TomoHDF5Config
 from nxtomo.nxobject.nxdetector import ImageKey
 
-from tomoscan.io import HDF5File
+from tomoscan.io import HDF5File, get_swmr_mode
 from tomoscan.validator import is_valid_for_reconstruction
 
 from tomwer.core.process.control.datalistener import DataListener
@@ -335,9 +335,9 @@ class NXTomoDataTransferBase(unittest.TestCase):
         dst_scan = process.outputs.data
         assert dst_scan.master_file != self.nexus_file_path
         assert is_valid_for_reconstruction(dst_scan, check_values=True)
-        with HDF5File(src_scan.master_file, mode="r") as h5s_src:
+        with HDF5File(src_scan.master_file, mode="r", swmr=get_swmr_mode()) as h5s_src:
             src_dataset = h5s_src["entry0000/detector/data"][...]
-        with HDF5File(dst_scan.master_file, mode="r") as h5s_dst:
+        with HDF5File(dst_scan.master_file, mode="r", swmr=get_swmr_mode()) as h5s_dst:
             dst_dataset = h5s_dst["entry0000/detector/data"][...]
         assert numpy.array_equal(src_dataset, dst_dataset)
 

tomwer/core/process/test/test_nabu.py

@@ -127,7 +127,7 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + ".cfg",
-            os.path.basename(self.scan.path) + "slice_000001" + ".cfg",
+            os.path.basename(self.scan.path) + "slice_000001_plane_XY" + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -151,7 +151,9 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + "pag_db0100.cfg",
-            os.path.basename(self.scan.path) + "slice_pag_000001_db0100" + ".cfg",
+            os.path.basename(self.scan.path)
+            + "slice_pag_000001_db0100_plane_XY"
+            + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -179,10 +181,13 @@ class TestNabuIni(unittest.TestCase):
                 os.path.basename(self.scan.path)
                 + "slice_"
                 + str(slice_index).zfill(6)
+                + "_plane_XY"
                 + ".cfg"
             )
 
+        print("nabu_files are", nabu_files)
         for ini_file in ini_files:
+            print("check ini_file", ini_file)
             with self.subTest(file_name=ini_file):
                 self.assertTrue(
                     os.path.join(self._nabu_cfg_folder, ini_file) in nabu_files
@@ -220,10 +225,12 @@ class TestNabuIni(unittest.TestCase):
                 + ".cfg"
             )
             # add slice ini file
+            # note: plane is only specify for slice.
             ini_files.append(
                 os.path.basename(self.scan.path)
                 + "slice_pag_000001_db"
                 + str(int(pag_db)).zfill(4)
+                + "_plane_XY"
                 + ".cfg"
             )
 
@@ -268,6 +275,7 @@ class TestNabuIni(unittest.TestCase):
                     + str(slice_index).zfill(6)
                     + "_db"
                     + str(pag_db).zfill(4)
+                    + "_plane_XY"
                     + ".cfg"
                 )
 

tomwer/core/process/test/test_normalization.py

@@ -54,8 +54,8 @@ class TestNormalization(unittest.TestCase):
             scan_range=180,
             dim=dim,
             create_ini_dark=False,
-            create_ini_ref=False,
-            create_final_ref=False,
+            create_ini_flat=False,
+            create_final_flat=False,
         ).scan
 
     def tearDown(self) -> None:

tomwer/core/scan/blissscan.py

@@ -54,7 +54,7 @@ import logging
 from typing import Optional
 
 from silx.io.utils import h5py_read_dataset
-from tomoscan.io import HDF5File
+from tomoscan.io import HDF5File, get_swmr_mode
 
 _logger = logging.getLogger(__name__)
 
@@ -203,7 +203,7 @@ class BlissScan:
                     return True
             return False
 
-        with HDF5File(file_path, mode="r", swmr=True) as h5s:
+        with HDF5File(file_path, mode="r", swmr=get_swmr_mode()) as h5s:
             if not isinstance(h5s, h5py.Group) or not isinstance(
                 h5s[entry], h5py.Group
             ):
@@ -227,7 +227,7 @@ class BlissScan:
             return tuple()
         else:
             res = []
-            with HDF5File(file_path, mode="r", swmr=True) as h5s:
+            with HDF5File(file_path, mode="r", swmr=get_swmr_mode()) as h5s:
                 for entry in h5s:
                     if BlissScan.is_bliss_valid_entry(file_path=file_path, entry=entry):
                         res.append(entry)

tomwer/core/scan/edfscan.py

@@ -281,9 +281,9 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
                                     files.append(gfile)
                     if f.endswith(".vol"):
                         if withIndex is True:
-                            files[
-                                EDFTomoScan.get_index_reconstructed(f, scanID)
-                            ] = os.path.join(scanID, f)
+                            files[EDFTomoScan.get_index_reconstructed(f, scanID)] = (
+                                os.path.join(scanID, f)
+                            )
                         else:
                             files.append(os.path.join(scanID, f))
             return files
@@ -552,12 +552,12 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
             file_prefix=self.dataset_basename,
             metadata={
                 "name": self.path,
-                "creation_time": datetime.fromtimestamp(stat.st_ctime)
-                if stat
-                else None,
-                "modification_time": datetime.fromtimestamp(stat.st_ctime)
-                if stat
-                else None,
+                "creation_time": (
+                    datetime.fromtimestamp(stat.st_ctime) if stat else None
+                ),
+                "modification_time": (
+                    datetime.fromtimestamp(stat.st_ctime) if stat else None
+                ),
             },
         )
 

tomwer/core/scan/nxtomoscan.py

@@ -45,7 +45,7 @@ from tomoscan.esrf.identifier.hdf5Identifier import (
 )
 from tomoscan.esrf.identifier.url_utils import UrlSettings, split_path, split_query
 from tomoscan.esrf.scan.nxtomoscan import NXtomoScan as _tsNXtomoScan
-from tomoscan.io import HDF5File
+from tomoscan.io import HDF5File, get_swmr_mode
 from nxtomo.nxobject.nxdetector import ImageKey
 
 from tomwer.utils import docstring
@@ -169,12 +169,12 @@ class NXtomoScan(_tsNXtomoScan, TomwerScanBase):
             entry=self.entry,
             metadata={
                 "name": self.master_file,
-                "creation_time": datetime.fromtimestamp(stat.st_ctime)
-                if stat
-                else None,
-                "modification_time": datetime.fromtimestamp(stat.st_ctime)
-                if stat
-                else None,
+                "creation_time": (
+                    datetime.fromtimestamp(stat.st_ctime) if stat else None
+                ),
+                "modification_time": (
+                    datetime.fromtimestamp(stat.st_ctime) if stat else None
+                ),
             },
         )
 
@@ -406,9 +406,9 @@ class NXtomoScan(_tsNXtomoScan, TomwerScanBase):
                 )
                 self._projections_with_angles = {}
                 for proj_frame in proj_frames:
-                    self._projections_with_angles[
-                        proj_frame.rotation_angle
-                    ] = proj_frame.url
+                    self._projections_with_angles[proj_frame.rotation_angle] = (
+                        proj_frame.url
+                    )
         return self._projections_with_angles
 
     @staticmethod
@@ -422,7 +422,7 @@ class NXtomoScan(_tsNXtomoScan, TomwerScanBase):
             return tuple()
         else:
             res = []
-            with HDF5File(file_path, mode="r", swmr=True, libver="latest") as h5s:
+            with HDF5File(file_path, mode="r", swmr=get_swmr_mode()) as h5s:
                 for entry_name, node in h5s.items():
                     if isinstance(node, h5py.Group):
                         if NXtomoScan.entry_is_nx_tomo(node):

tomwer/core/scan/scanbase.py

@@ -41,7 +41,7 @@ import numpy
 from silx.io.url import DataUrl
 from silx.utils.enum import Enum as _Enum
 from tomoscan.identifier import VolumeIdentifier
-from tomoscan.io import HDF5File
+from tomoscan.io import HDF5File, get_swmr_mode
 from tomoscan.normalization import IntensityNormalization
 from tomoscan.volumebase import VolumeBase
 from tomoscan.identifier import BaseIdentifier
@@ -65,8 +65,6 @@ class TomwerScanBase(TomwerObject):
 
     _DICT_DARK_REF_KEYS = "dark_ref_params"
 
-    _DICT_LAMINO_RP_KEY = "lamino_params"
-
     _DICT_NABU_RP_KEY = "nabu_params"
 
     _DICT_AXIS_KEYS = "axis_params"
@@ -88,8 +86,6 @@ class TomwerScanBase(TomwerObject):
 
         self._nabu_params = None
         """nabu reconstruction parameters"""
-        self._lamino_params = None
-        """Set of reconstructions parameters for laminography"""
         self._axis_params = None
         """Axis parameters"""
         self._saaxis_params = None
@@ -135,7 +131,7 @@ class TomwerScanBase(TomwerObject):
             and self.process_file is not None
             and os.path.exists(self.process_file)
         ):
-            with HDF5File(self.process_file, mode="r", swmr=True) as h5s:
+            with HDF5File(self.process_file, mode="r", swmr=get_swmr_mode()) as h5s:
                 self._process_index = len(h5s.items())
         else:
             self._process_index = 0
@@ -253,14 +249,6 @@ class TomwerScanBase(TomwerObject):
     def nabu_recons_params(self, recons_params):
         self._nabu_params = recons_params
 
-    @property
-    def lamino_recons_params(self):
-        return self._lamino_params
-
-    @lamino_recons_params.setter
-    def lamino_recons_params(self, recons_params):
-        self._lamino_params = recons_params
-
     @property
     def axis_params(self):
         return self._axis_params
@@ -448,8 +436,6 @@ class TomwerScanBase(TomwerObject):
         # process index
         res[self._DICT_PROCESS_INDEX_KEY] = self._process_index
 
-        # lamino reconstruction parameters
-        res[self._DICT_LAMINO_RP_KEY] = self.lamino_recons_params
         return res
 
     def load_from_dict(self, desc):
@@ -468,8 +454,6 @@ class TomwerScanBase(TomwerObject):
             raise ValueError("Description is not an EDFScan json description")
 
         assert self.DICT_PATH_KEY in data  # pylint: disable=E1101
-        assert self._DICT_LAMINO_RP_KEY in data
-        self.lamino_recons_params = data[self._DICT_LAMINO_RP_KEY]
         # load axis reconstruction parameters
         axis_params = data.get(self._DICT_AXIS_KEYS, None)
         if axis_params is not None:
@@ -520,7 +504,6 @@ class TomwerScanBase(TomwerObject):
             isinstance(other, self.__class__)
             or isinstance(self, other.__class__)
             and self.type == other.type  # pylint: disable=E1101
-            and self.lamino_recons_params == other.lamino_recons_params
             and self.nabu_recons_params == other.nabu_recons_params
             and self.path == other.path  # pylint: disable=E1101
         )
@@ -602,8 +585,19 @@ class TomwerScanBase(TomwerObject):
         :return: list of urls that contains reconstruction from nabu
         :rtype: list
         """
+        from tomwer.core.process.reconstruction.output import (
+            PROCESS_FOLDER_NAME,
+        )  # avoid cyclic import
+
         all_recons_urls = set()
         recons_paths = self.reconstruction_paths
+        if self.working_directory:
+            recons_paths.add(
+                os.path.join(
+                    self.working_directory,
+                    PROCESS_FOLDER_NAME,
+                )
+            )
         for path in recons_paths:
             all_recons_urls.update(self.get_reconstructions_urls_from_path(path))
         return all_recons_urls
@@ -612,6 +606,17 @@ class TomwerScanBase(TomwerObject):
         if path is None or not os.path.isdir(path):
             return []
         else:
+
+            from tomwer.core.process.reconstruction.saaxis.saaxis import (
+                DEFAULT_RECONS_FOLDER as MULTI_COR_DEFAULT_FOLDER,
+            )
+            from tomwer.core.process.reconstruction.sadeltabeta.sadeltabeta import (
+                DEFAULT_RECONS_FOLDER as MULTI_DB_DEFAULT_FOLDER,
+            )
+            from tomwer.core.process.reconstruction.nabu.settings import (
+                NABU_CFG_FILE_FOLDER,
+            )
+
             res = set()
             for f in os.listdir(path):
                 current_path = os.path.join(path, f)
@@ -626,10 +631,10 @@ class TomwerScanBase(TomwerObject):
                     f
                     in (
                         "matlab",
-                        "saaxis_results",
-                        "sadeltabeta_results",
                         "steps_file_basename_nabu_sinogram_save_step",
-                        "nabu_cfg_files",
+                        MULTI_COR_DEFAULT_FOLDER,
+                        MULTI_DB_DEFAULT_FOLDER,
+                        NABU_CFG_FILE_FOLDER,
                     )
                     or f.endswith("_vol")
                     or "slice" not in f
@@ -751,7 +756,7 @@ class TomwerScanBase(TomwerObject):
         from tomwer.core.process.task import Task
 
         if os.path.exists(self.process_file):
-            with HDF5File(self.process_file, mode="r") as h5s:
+            with HDF5File(self.process_file, mode="r", swmr=get_swmr_mode()) as h5s:
                 if not hasattr(self, "entry"):
                     entry = "entry"
                 else:
@@ -859,7 +864,9 @@ def _get_reconstructed_single_file_volume(
 
                 if check_url is True:
                     try:
-                        with HDF5File(os.path.abspath(file_), "r", swmr=True) as h5f:
+                        with HDF5File(
+                            os.path.abspath(file_), "r", swmr=get_swmr_mode()
+                        ) as h5f:
                             if entry not in h5f:
                                 logger.info("{volume} does not exists")
                                 return None

tomwer/core/scan/test/test_future_scan.py

@@ -43,7 +43,7 @@ def test_simple_future_tomo_obj(tmpdir):
         n_proj=10,
         n_ini_proj=10,
         create_ini_dark=False,
-        create_ini_ref=False,
+        create_ini_flat=False,
         dim=10,
     ).scan
     future = asyncio.Future()

tomwer/core/scan/test/test_h5.py

@@ -65,9 +65,9 @@ class TestHDF5Scan(unittest.TestCase):
             n_ini_proj=20,
             n_proj=20,
             n_alignement_proj=2,
-            create_final_ref=True,
+            create_final_flat=True,
             create_ini_dark=True,
-            create_ini_ref=True,
+            create_ini_flat=True,
             n_refs=5,
         ).scan
         self.assertEqual(scan_1.ff_interval, 20)
@@ -78,9 +78,9 @@ class TestHDF5Scan(unittest.TestCase):
             n_ini_proj=10,
             n_proj=10,
             n_alignement_proj=2,
-            create_final_ref=False,
+            create_final_flat=False,
             create_ini_dark=True,
-            create_ini_ref=True,
+            create_ini_flat=True,
             n_refs=1,
         ).scan
         self.assertEqual(scan_2.ff_interval, 0)

tomwer/core/scan/test/test_process_registration.py

@@ -32,7 +32,7 @@ import shutil
 import tempfile
 import unittest
 
-from tomoscan.io import HDF5File
+from tomoscan.io import HDF5File, get_swmr_mode
 
 from tomwer.core.process.task import Task
 from tomwer.core.utils.scanutils import MockNXtomo
@@ -98,7 +98,7 @@ class TestProcessRegistration(unittest.TestCase):
                 process_index=i,
             )
 
-        with HDF5File(self.scan.process_file, "r", swmr=True) as h5f:
+        with HDF5File(self.scan.process_file, "r", swmr=get_swmr_mode()) as h5f:
             nodes = Task._get_process_nodes(
                 root_node=h5f[self.scan.entry], process=self.DummyProcess
             )

tomwer/core/scan/test/test_scan.py

@@ -1,45 +1,14 @@
-# coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "24/01/2017"
-
 import contextlib
 import logging
 import os
 import shutil
 import tempfile
 import unittest
-from multiprocessing import Process
 
 import fabio.edfimage
 import numpy
 
 from tomwer.core.process.reconstruction.nabu import nabuslices
-from tomwer.core.process.task import Task
 from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import MockEDF
@@ -103,6 +72,7 @@ class TestScanValidatorFindNabuFiles(unittest.TestCase):
                             pag=pag if delta_beta is not None else False,
                             ctf=ctf if delta_beta is not None else False,
                             db=delta_beta,
+                            axis="XY",
                         )
                         with self.subTest(
                             slice_index=slice_index,
@@ -401,47 +371,3 @@ def test_get_sinogram(tmp_path):
         sinogram_1,
         numpy.array([[2, 3], [6, 7], [10, 11], [14, 15]]),
     )
-
-
-class TestSeveralWriteWithThreading(unittest.TestCase):
-    class DummyProcess(Task):
-        @staticmethod
-        def program_name():
-            """Name of the program used for this processing"""
-            return "dummy program"
-
-        @staticmethod
-        def program_version():
-            """version of the program used for this processing"""
-            return "0.0.0"
-
-        @staticmethod
-        def definition():
-            """definition of the process"""
-            return "no definition"
-
-    def setUp(self):
-        self.tmp_dir = tempfile.mkdtemp()
-        self._file = os.path.join(self.tmp_dir, "XXXtomwer_processes.h5")
-
-    def tearDown(self):
-        shutil.rmtree(self.tmp_dir)
-
-    def testSeveralWrite(self):
-        processes = []
-        for i in range(20):
-            args = (
-                self._file,
-                TestSeveralWriteWithThreading.DummyProcess,
-                "entry0000",
-                {"info": "no configuration"},
-                {"data": numpy.random.random(2000)},
-                i,
-                None,
-                True,
-            )
-            processes.append(Process(target=Task._register_process, args=args))
-        for process in processes:
-            process.start()
-        for process in processes:
-            process.join()

tomwer/core/settings.py

@@ -125,7 +125,7 @@ class SlurmSettings:
     N_JOBS = 1
     """on how many job we want to split the EwoksTask"""
 
-    MEMORY_PER_WORKER = 64  # memory in GB
+    MEMORY_PER_WORKER = 128  # memory in GB
     """Amount of memory per worker"""
 
     PARTITION = "p9gpu"
@@ -136,7 +136,7 @@ class SlurmSettings:
     N_GPUS_PER_WORKER = 1
     """number of gpu per worker"""
 
-    PYTHON_VENV = "/scisoft/tomotools/activate stable"
+    PYTHON_VENV = None
     """Python executable to take. Useful if compute nodes have a different environment from the front end.
     """
 
@@ -144,7 +144,7 @@ class SlurmSettings:
     """Slurm cluster project name. `scan`, `process` and `info` will be format.
     """
 
-    MODULES_TO_LOAD = tuple()  # in the future might be 'tomotools'. In this case we must remove the PYTHON_VENV
+    MODULES_TO_LOAD = ("tomotools/stable", )
 
     SBATCH_EXTRA_PARAMS = {
         "export": "NONE",  # value to provide to sbatch --export={}