tomwer 1.4.0rc6__py3-none-any.whl → 1.4.2__py3-none-any.whl

tomwer/core/process/task.py

@@ -8,17 +8,9 @@ from __future__ import annotations
 
 import logging
 from collections import namedtuple
-from datetime import datetime
 
-import h5py
-import numpy
 from ewokscore.task import Task as _EwoksTask
 from ewokscore.taskwithprogress import TaskWithProgress as _EwoksTaskWithProgress
-from silx.io.dictdump import dicttoh5, h5todict
-from silx.io.utils import h5py_read_dataset
-from silx.io.utils import open as open_hdf5
-from tomoscan.io import HDF5File
-from tomwer.core.utils.locker import FileLockerManager
 
 
 _logger = logging.getLogger(__name__)
@@ -31,15 +23,10 @@ _process_desc = namedtuple(
 class BaseProcessInfo:
     """Tomwer base process class"""
 
-    _output_values = {}
-    # TODO: look at this: not sure this is needed anymore
-
     def __init__(self, inputs=None):
         """
         :param return_dict: if True serialize (to_dict / from_dict functions) between each task
         """
-        if inputs is None:
-            inputs = {}
 
         self._scheme_title = (
             "scheme_title"  # TODO: have a look, this must be get somewhere and reused ?
@@ -60,29 +47,6 @@ class BaseProcessInfo:
         # TODO: use argsparse instead of this dict ?
         raise NotImplementedError("BaseProcess is an abstract class")
 
-    def get_output_value(self, key):
-        """
-
-        :param key:
-        :return:
-        """
-        assert type(key) is str
-        if key in self._output_values:
-            return self._output_values[key]
-        else:
-            return None
-
-    def clear_output_values(self):
-        self._output_values.clear()
-
-    def register_output(self, key, value):
-        """
-
-        :param key: name of the output
-        :param value: value of the output
-        """
-        self._output_values[key] = value
-
     @staticmethod
     def program_name():
         """Name of the program used for this processing"""
@@ -113,260 +77,6 @@ class BaseProcessInfo:
     def set_configuration(self, configuration: dict) -> None:
         self._settings = configuration
 
-    def register_process(
-        self,
-        process_file: str,
-        entry: str,
-        configuration: dict | None,
-        results: dict | None,
-        process_index: int,
-        interpretations: dict | None = None,
-        overwrite: bool = True,
-    ) -> None:
-        """
-        Store the current process in the linked h5 file if any,
-        output data stored will be the one defined by the data_keys
-
-        :param process_file: where to store the processing information
-        :param entry: entry process
-        :param configuration: configuration of the process
-        :param results: result of the processing
-        :param process_index: index of the process
-        :param overwrite: if True then overwrite the process if already
-                               exists
-        :paramerpretations: for each result we can add a flag
-                                'interpretation'
-        """
-        assert process_file is not None
-        with FileLockerManager().get_lock(file_name=process_file):
-            try:
-                self._register_process(
-                    process_file=process_file,
-                    entry=entry,
-                    process=self,
-                    configuration=configuration,
-                    results=results,
-                    process_index=process_index,
-                    interpretations=interpretations,
-                    overwrite=overwrite,
-                )
-            except IOError as e:
-                _logger.error(e)
-
-    @staticmethod
-    def _register_process(
-        process_file: str,
-        process,
-        entry: str | None,
-        configuration: dict | None,
-        results: dict | None,
-        process_index: int,
-        interpretations: dict | None = None,
-        overwrite: bool = True,
-    ) -> None:
-        """
-        Store the current process in the linked h5 file if any,
-        output data stored will be the one defined by the data_keys
-
-        :param process: process to record
-        :param process_file: where to store the processing information
-        :param entry: entry process
-        :param configuration: configuration of the process
-        :param results: result of the processing
-        :param process_index: index of the process
-        :param overwrite: if True then overwrite the process if already
-                               exists
-        :paramerpretations: for each result we can add a flag
-                                'interpretation'
-        """
-        assert process_file is not None, "The process file should be defined"
-        if interpretations is None:
-            interpretations = {}
-        process_name = "tomwer_process_" + str(process_index)
-
-        def get_process_path():
-            return "/".join((entry or "entry", process_name))
-
-        # save it to the file (lock should be handled upper)
-        with HDF5File(process_file, mode="a") as h5f:
-            nx_process = h5f.require_group(get_process_path())
-            if "NX_class" not in nx_process.attrs:
-                nx_process.attrs["NX_class"] = "NXprocess"
-            if overwrite:
-                for key in (
-                    "program",
-                    "results",
-                    "version",
-                    "date",
-                    "sequence_index",
-                    "class_instance",
-                    "configuration",
-                ):
-                    if key in nx_process:
-                        del nx_process[key]
-
-            # write process information
-            nx_process["program"] = process.program_name()
-            nx_process["version"] = process.program_version()
-            nx_process["date"] = datetime.now().replace(microsecond=0).isoformat()
-            nx_process["sequence_index"] = numpy.int32(process_index)
-            _class = process.__class__
-            nx_process["class_instance"] = ".".join(
-                (_class.__module__, _class.__name__)
-            )
-
-        # dump result
-        if results is not None:
-            h5path = "/".join((get_process_path(), "results"))
-            dicttoh5(
-                results,
-                h5file=process_file,
-                h5path=h5path,
-                update_mode="modify",
-                mode="a",
-            )
-            for interpretation_key, interpretation in interpretations.items():
-                try:
-                    with HDF5File(process_file, mode="a") as h5f:
-                        node = h5f["/".join((get_process_path(), "results"))]
-                        node[interpretation_key].attrs[
-                            "interpretation"
-                        ] = interpretation
-                except KeyError:
-                    _logger.warning(
-                        "Invalid interpretation - no result store"
-                        " for %s" % interpretation_key
-                    )
-
-        # dump configuration
-        if configuration is not None:
-            h5path = "/".join((get_process_path(), "configuration"))
-            dicttoh5(
-                configuration,
-                h5file=process_file,
-                h5path=h5path,
-                update_mode="modify",
-                mode="a",
-            )
-
-            with HDF5File(process_file, mode="a") as h5f:
-                nx_process = h5f.require_group(get_process_path())
-                nx_process["configuration"].attrs["NX_class"] = "NXcollection"
-
-    @staticmethod
-    def _get_process_nodes(
-        root_node: h5py.Group, process, version: str = None, depth: int = 2
-    ) -> dict:
-        """
-
-        :param root_node:
-        :param version: version to look for (ignored for now)
-        :return: list of process nodes [as h5py.Group]. Key is node name,
-                 value is the process index
-        """
-
-        def is_process_node(node):
-            if node is None:
-                return False
-            return (
-                node.name.split("/")[-1].startswith("tomwer_process_")
-                and "NX_class" in node.attrs
-                and node.attrs["NX_class"] == "NXprocess"
-                and "program" in node
-                and (
-                    process is None
-                    or h5py_read_dataset(node["program"]) == process.program_name()
-                )
-                and "version" in node
-                and "sequence_index" in node
-            )
-
-        if isinstance(root_node, h5py.Dataset):
-            return {}
-        res = {}
-        if is_process_node(root_node):
-            res[root_node.name] = int(h5py_read_dataset(root_node["sequence_index"]))
-
-        if root_node is not None:
-            for _, node in root_node.items():
-                if depth >= 1:
-                    res.update(
-                        Task._get_process_nodes(
-                            process=process,
-                            root_node=node,
-                            depth=depth,
-                            version=version,
-                        )
-                    )
-        return res
-
-    @staticmethod
-    def get_most_recent_process(
-        root_node: h5py.Group, process, version: str = None
-    ) -> h5py.Group | None:
-        nodes = Task._get_process_nodes(
-            root_node=root_node, process=process, version=version, depth=1
-        )
-        nodes_with_time = []
-        nodes_time = []
-        import datetime
-
-        for node_name in nodes:
-            node = root_node[node_name]
-            assert isinstance(node, h5py.Group), "node should be a h5py.Group"
-            if "date" in node:
-                nodes_with_time.append(node)
-                nodes_time.append(
-                    datetime.datetime.fromisoformat(h5py_read_dataset(node["date"]))
-                )
-        if len(nodes_with_time) == 0:
-            return None
-        else:
-            st = numpy.argmax(nodes_time)
-            return nodes_with_time[st]
-
-    @staticmethod
-    def get_processes_frm_type(process_file: str, process_type, entry=None) -> list:
-        """
-
-        :param process_file: file to read
-        :param process_type: process type we are looking for
-        :return: list of _process_desc(sequence_index, configuration, results)
-        """
-        # retrieve process to load
-        with open_hdf5(process_file) as h5f:
-            if entry is None:
-                if len(h5f.keys()) > 0:
-                    root = h5f[list(h5f.keys())[0]]
-                else:
-                    _logger.warning("no process find")
-                    return []
-            else:
-                root = h5f[entry]
-
-            processes_to_load = {}
-            for process in root.keys():
-                nx_process = root[process]
-                if (
-                    h5py_read_dataset(nx_process["program"])
-                    == process_type.program_name()
-                ):
-                    processes_to_load[nx_process.name] = h5py_read_dataset(
-                        nx_process["sequence_index"]
-                    )
-
-        # load process
-        processes = []
-        for process_name, p_order in processes_to_load.items():
-            config = h5todict(process_file, "/".join((process_name, "configuration")))
-            results = h5todict(process_file, "/".join((process_name, "results")))
-            processes.append(
-                _process_desc(
-                    process_order=p_order, configuration=config, results=results
-                )
-            )
-        return processes
-
 
 class TaskWithProgress(_EwoksTaskWithProgress, BaseProcessInfo):
     """Class from which all tomwer process should inherit

tomwer/core/process/tests/test_dark_and_flat.py

@@ -146,7 +146,6 @@ class TestDarkRefEdf(unittest.TestCase):
         self.recons_params = DKRFRP()
         self.recons_params.overwrite_dark = True
         self.recons_params.overwrite_ref = True
-        self.assertFalse(os.path.exists(self.scan.process_file))
 
     def tearDown(self) -> None:
         shutil.rmtree(self.scan_folder)
@@ -169,8 +168,6 @@ class TestDarkRefEdf(unittest.TestCase):
         )
         for method, th_res in zip(method_to_test, th_results):
             with self.subTest(method=method):
-                if os.path.exists(self.scan.process_file):
-                    os.remove(self.scan.process_file)
                 process = DarkRefsTask(
                     inputs={
                         "dark_ref_params": self.recons_params,
@@ -204,8 +201,6 @@ class TestDarkRefEdf(unittest.TestCase):
         )
         for method, th_res in zip(method_to_test, th_results):
             with self.subTest(method=method):
-                if os.path.exists(self.scan.process_file):
-                    os.remove(self.scan.process_file)
                 self.recons_params.dark_calc_method = cMethod.NONE
                 self.recons_params.flat_calc_method = method
                 process = DarkRefsTask(
@@ -236,7 +231,6 @@ class TestDarkRefNx(unittest.TestCase):
             dst=self._file_path,
         )
         self.scan = NXtomoScan(scan=self._file_path, entry="entry0000")
-        self.assertFalse(os.path.exists(self.scan.process_file))
         self.recons_params = DKRFRP()
         self.recons_params.overwrite_dark = True
         self.recons_params.overwrite_ref = True

tomwer/core/process/tests/test_data_transfer.py

@@ -119,7 +119,6 @@ class TestBlissDataTransfer(unittest.TestCase):
             master_sample_file=None,
         )
         self.scan = scans[0]
-        self.assertTrue(os.path.exists(self.scan.process_file))
 
     def tearDown(self):
         shutil.rmtree(self.input_dir)

tomwer/core/process/tests/test_data_watcher.py

@@ -26,31 +26,14 @@ def test_data_watcher_io(tmp_path, det_method):
         scanID=str(scan_folder), nRadio=10, nRecons=1, nPagRecons=4, dim=10
     )
     data_watcher_process.setFolderObserved(str(tmp_path / "folder"))
-    data_watcher_process.clear_output_values()
     data_watcher_process.set_serialize_output_data(True)
     LoopThread.quitEvent.clear()
     data_watcher_process.start()
+    time.sleep(0.5)
 
-    # note: this quitEvent is nasty. But we don't want to spend time on the (EDF) data watcher which is not the future
+    data_watcher_process.stop()
+    data_watcher_process.waitForObservationFinished()
+    import gc
 
-    timeout = 8
-    while data_watcher_process.get_output_value("data") is None and timeout > 0:
-        _delta = 0.5
-        time.sleep(_delta)
-        timeout -= _delta
-
-    def quit_test():
-        data_watcher_process.stop()
-        data_watcher_process.waitForObservationFinished()
-        import gc
-
-        gc.collect()
-        LoopThread.quitEvent.clear()
-
-    if timeout <= 0:
-        quit_test()
-        raise TimeoutError("timeout expire")
-
-    out = data_watcher_process.get_output_value("data")
-    assert isinstance(out, dict)
-    quit_test()
+    gc.collect()
+    LoopThread.quitEvent.clear()

tomwer/core/scan/edfscan.py

@@ -75,8 +75,6 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
         self.add_reconstruction_path(self.path)
 
         self._dark = None
-        self._process_file = self.get_process_file_name(self)
-        self._init_index_process_file(overwrite_proc_file=overwrite_proc_file)
 
         if scan is not None and update:
             try:
@@ -129,15 +127,6 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
             )
         return EDFTomoScan(scan=identifier.folder)
 
-    @staticmethod
-    def get_process_file_name(scan):
-        if scan.path is not None:
-            basename = os.path.basename(scan.path)
-            basename = "_".join((basename, "tomwer_processes.h5"))
-            return os.path.join(scan.path, basename)
-        else:
-            return None
-
     def clear_caches(self):
         _tsEDFTomoScan.clear_caches(self)
         TomwerScanBase.clear_caches(self)

tomwer/core/scan/nxtomoscan.py

@@ -101,8 +101,6 @@ class NXtomoScan(_tsNXtomoScan, TomwerScanBase):
 
         self._reconstruction_urls = None
         self._projections_with_angles = None
-        self._process_file = self.get_process_file_name(self)
-        self._init_index_process_file(overwrite_proc_file=overwrite_proc_file)
         try:
             reduced_darks, metadata = self.load_reduced_darks(return_info=True)
         except (KeyError, OSError, ValueError):
@@ -159,16 +157,6 @@ class NXtomoScan(_tsNXtomoScan, TomwerScanBase):
             },
         )
 
-    @staticmethod
-    def get_process_file_name(scan):
-        if scan.path is not None:
-            basename, _ = os.path.splitext(scan.master_file)
-            basename = os.path.basename(basename)
-            basename = "_".join((basename, "tomwer_processes.h5"))
-            return os.path.join(scan.path, basename)
-        else:
-            return None
-
     @staticmethod
     def directory_contains_scan(directory, src_pattern=None, dest_pattern=None):
         """

tomwer/core/scan/scanbase.py

@@ -11,7 +11,6 @@ from typing import Iterable
 import numpy
 from silx.io.url import DataUrl
 from silx.utils.enum import Enum as _Enum
-from silx.io.utils import open as open_hdf5
 
 from tomoscan.identifier import VolumeIdentifier
 from tomoscan.normalization import IntensityNormalization
@@ -21,7 +20,6 @@ from tomoscan.esrf.volume.utils import guess_volumes
 
 from tomwer.core.tomwer_object import TomwerObject
 from tomwer.core.utils.ftseriesutils import orderFileByLastLastModification
-from tomwer.core.utils.locker import FileLockerManager
 from tomwer.io.utils.raw_and_processed_data import (
     to_raw_data_path,
     to_processed_data_path,
@@ -46,8 +44,6 @@ class TomwerScanBase(TomwerObject):
 
     _DICT_SA_DELTA_BETA_KEYS = "sa_delta_beta_params"
 
-    _DICT_PROCESS_INDEX_KEY = "next_process_index"
-
     _DICT_NORMALIZATION_KEY = "norm_params"
 
     VALID_RECONS_EXTENSION = ".edf", ".npy", ".npz", ".hdf5", ".tiff", ".jp2", ".vol"
@@ -67,9 +63,6 @@ class TomwerScanBase(TomwerObject):
         """Information regarding sa_delta_beta_params"""
         self._dark_ref_params = None
         """Information regarding dark - ref reconstruction"""
-        self._process_file = None
-        """file storing processes applied on the scan, with their configuration
-        and result"""
         self._cache_proj_urls = None
         """cache for the projection urls"""
         self._cache_radio_axis = {}
@@ -99,26 +92,11 @@ class TomwerScanBase(TomwerObject):
         """
         raise NotImplementedError("Base class")
 
-    def _init_index_process_file(self, overwrite_proc_file=False):
-        if (
-            not overwrite_proc_file
-            and self.process_file is not None
-            and os.path.exists(self.process_file)
-        ):
-            with open_hdf5(self.process_file) as h5s:
-                self._process_index = len(h5s.items())
-        else:
-            self._process_index = 0
-
     def clear_caches(self):
         self._cache_proj_urls = None
         self._notify_ffc_rsc_missing = True
         super().clear_caches()
 
-    @staticmethod
-    def get_process_file_name(scan):
-        raise NotImplementedError("Base class")
-
     def _flat_field_correction(
         self,
         data,
@@ -195,13 +173,6 @@ class TomwerScanBase(TomwerObject):
         div[div == 0] = 1
         return (data - dark) / div
 
-    def acquire_process_file_lock(self):
-        """create a FileLocker context manager to insure safe write to the
-        process file"""
-        if self.process_file is None:
-            raise ValueError("No processing file defined")
-        return FileLockerManager().get_lock(file_name=self.process_file)
-
     @property
     def reconstructions(self):
         """list of reconstruction files"""
@@ -269,12 +240,6 @@ class TomwerScanBase(TomwerObject):
         """return parent basename of the directory containing the acquisition"""
         raise NotImplementedError("Base class")
 
-    def pop_process_index(self) -> int:
-        """Return and lock the next free process index"""
-        process_index = self._process_index
-        self._process_index += 1
-        return process_index
-
     @functools.lru_cache(maxsize=3)
     def get_opposite_projections(self, mode) -> tuple:
         """
@@ -340,23 +305,6 @@ class TomwerScanBase(TomwerObject):
         """
         raise NotImplementedError("Base class")
 
-    @property
-    def process_file(self) -> str:
-        """
-
-        :return: file used to store the processes launch by tomwer
-        """
-        return self._process_file
-
-    @property
-    def process_file_url(self) -> DataUrl:
-        """
-
-        :return: DataUrl of the process file
-        """
-        entry = self.entry if hasattr(self, "entry") else "entry"
-        return DataUrl(file_path=self._process_file, data_path=entry, scheme="silx")
-
     def to_dict(self):
         res = {}
         # nabu reconstruction parameters
@@ -393,8 +341,6 @@ class TomwerScanBase(TomwerObject):
             res[self._DICT_NORMALIZATION_KEY] = None
         else:
             res[self._DICT_NORMALIZATION_KEY] = self.intensity_normalization.to_dict()
-        # process index
-        res[self._DICT_PROCESS_INDEX_KEY] = self._process_index
 
         return res
 
@@ -450,8 +396,6 @@ class TomwerScanBase(TomwerObject):
                 sa_delta_beta_params
             )
 
-        self._process_index = data[self._DICT_PROCESS_INDEX_KEY]
-
     def equal(self, other):
         """
 
@@ -720,31 +664,6 @@ class TomwerScanBase(TomwerObject):
         else:
             return None
 
-    def set_process_index_frm_tomwer_process_file(self):
-        """
-        Set the process_index to the last index find in the tomwer_process_file
-        + 1
-        """
-        from tomwer.core.process.task import Task
-
-        if os.path.exists(self.process_file):
-            with open_hdf5(self.process_file) as h5s:
-                if not hasattr(self, "entry"):
-                    entry = "entry"
-                else:
-                    entry = self.entry
-                if entry in h5s:
-                    node = h5s[entry]
-                    pn = Task._get_process_nodes(
-                        root_node=node, process=None, version=None
-                    )
-                    indexes = pn.values()
-                    if len(indexes) > 0:
-                        self._process_index = max(indexes) + 1
-                        logger.debug(
-                            f"set process_index from tomwer process file to {self._process_index}"
-                        )
-
     def get_nabu_dataset_info(self, binning=1, binning_z=1, proj_subsampling=1) -> dict:
         """
 

tomwer/gui/reconstruction/axis/CalculationWidget.py

@@ -180,10 +180,6 @@ class CalculationWidget(qt.QWidget):
         return axis_mode.AxisMode.from_value(self._qcbPosition.currentText())
 
     def setMode(self, mode: axis_mode.AxisMode):
-        if mode == self.getMode():
-            # when set mode the estimated cor might changed.
-            # that we want to avoid.
-            return
         with block_signals(self._qcbPosition):
             index = self._qcbPosition.findText(mode.value)
             if index >= 0:
@@ -211,7 +207,9 @@ class CalculationWidget(qt.QWidget):
             self._axis_params_changed()
 
     def _axis_params_changed(self, *args, **kwargs):
-        self.setMode(self._axis_params.mode)
+        if self._axis_params.mode != self.getMode():
+            # setMode will force to update visible side. So avoid to reset it if not necessary
+            self.setMode(self._axis_params.mode)
 
     def setScan(self, scan: TomwerScanBase | None):
         self._estimatedCorWidget.setPixelSize(pixel_size_m=scan.x_pixel_size)

tomwer/gui/reconstruction/tests/test_saaxis.py

@@ -140,7 +140,7 @@ class TestSAAxisWindow(TestCaseQt):
         conf = SAAxisParams.from_dict(conf)
         self.assertEqual(conf.slice_indexes, "middle")
         self.assertEqual(conf.file_format, "hdf5")
-        self.assertTrue(isinstance(conf.nabu_params, dict))
+        self.assertTrue(isinstance(conf.nabu_recons_params, dict))
         for key in (
             "preproc",
             "reconstruction",
@@ -150,7 +150,7 @@ class TestSAAxisWindow(TestCaseQt):
             "phase",
         ):
             with self.subTest(key=key):
-                self.assertTrue(key in conf.nabu_params)
+                self.assertTrue(key in conf.nabu_recons_params)
         self.assertEqual(conf._dry_run, False)
         self.assertEqual(conf.mode, ReconstructionMode.VERTICAL)
         self.assertEqual(conf.score_method, ScoreMethod.TV_INVERSE)

tomwer/gui/reconstruction/tests/test_sadeltabeta.py

@@ -72,7 +72,7 @@ class TestSADeltaBetaWindow(TestCaseQt):
         # the configuration should be convertible to
         conf = SADeltaBetaParams.from_dict(conf)
         self.assertEqual(conf.slice_indexes, "middle")
-        self.assertTrue(isinstance(conf.nabu_params, dict))
+        self.assertTrue(isinstance(conf.nabu_recons_params, dict))
         for key in (
             "preproc",
             "reconstruction",
@@ -82,7 +82,7 @@ class TestSADeltaBetaWindow(TestCaseQt):
             "phase",
         ):
             with self.subTest(key=key):
-                self.assertTrue(key in conf.nabu_params)
+                self.assertTrue(key in conf.nabu_recons_params)
         self.assertEqual(conf._dry_run, False)
         self.assertEqual(conf.score_method, ScoreMethod.TV_INVERSE)
         self.assertEqual(conf.output_dir, None)

tomwer/gui/stitching/config/axisparams.py

@@ -119,6 +119,8 @@ class StitcherAxisParams(qt.QWidget):
             for option in options:
                 clean_option = option.rstrip(" ").lstrip(" ")
                 opt_name, opt_value = clean_option.split("=")
+                if opt_value == "":
+                    continue
                 if opt_name == stitching_config.KEY_IMG_REG_METHOD:
                     self.setShiftSearchMethod(opt_value)
                 elif opt_name == stitching_config.KEY_WINDOW_SIZE: