tomwer 1.4.0rc5__py3-none-any.whl → 1.4.1__py3-none-any.whl

tomwer/core/process/control/datalistener/datalistener.py

@@ -9,7 +9,6 @@ import time
 
 from ewokscore.task import Task as EwoksTask
 from nxtomomill import converter as nxtomomill_converter
-from silx.io.utils import h5py_read_dataset
 from silx.io.utils import open as open_hdf5
 
 import tomwer.version
@@ -161,241 +160,10 @@ class DataListener(BaseProcessInfo):
             master_file, entry_master_file = mf_emf
             if master_file is not None and entry_master_file is not None:
                 scan = NXtomoScan(scan=master_file, entry=entry_master_file)
-
-                try:
-                    # register process.
-                    self.register_process(
-                        process_file=scan.process_file,
-                        entry=scan.entry,
-                        results={"output_file": scan.master_file, "entry": scan.entry},
-                        configuration={
-                            "sample_file": sample_file,
-                            "entry": entry,
-                            "file_proposal": proposal_file,
-                            "master_sample_file": master_sample_file,
-                        },
-                        process_index=scan.pop_process_index(),
-                        overwrite=True,
-                    )
-                except Exception as e:
-                    _logger.error(e)
-
                 self._signal_scan_ready(scan)
                 scans.append(scan)
         return tuple(scans)
 
-    @staticmethod
-    def get_proposal_file(process_file, entry):
-        """Return the proposal file of the experimentation if registered by the
-        data listener"""
-        if entry is None:
-            with open_hdf5(process_file) as h5f:
-                entries = BaseProcessInfo._get_process_nodes(
-                    root_node=h5f, process=DataListener
-                )
-                if len(entries) == 0:
-                    _logger.info(
-                        "unable to find a DarkRef process in %s" % process_file
-                    )
-                    return None
-                elif len(entries) > 1:
-                    raise ValueError("several entry found, entry should be " "specify")
-                else:
-                    entry = list(entries.keys())[0]
-                    _logger.info("take %s as default entry" % entry)
-
-        with open_hdf5(process_file) as h5f:
-            dl_nodes = BaseProcessInfo._get_process_nodes(
-                root_node=h5f[entry], process=DataListener
-            )
-            index_to_path = {}
-            for key, index in dl_nodes.items():
-                index_to_path[index] = key
-
-            if len(dl_nodes) == 0:
-                return None
-            # take the last processed dark ref
-            last_process_index = sorted(dl_nodes.values())[-1]
-            last_process_dl = index_to_path[last_process_index]
-            if (len(index_to_path)) > 1:
-                _logger.debug(
-                    "several processing found for data listener,"
-                    "take the last one: %s" % last_process_dl
-                )
-
-            if "configuration" in h5f[last_process_dl].keys():
-                results_node = h5f[last_process_dl]["configuration"]
-                if "file_proposal" in results_node.keys():
-                    try:
-                        fp = h5py_read_dataset(
-                            results_node["file_proposal"]
-                        )  # .decode("UTF-8")
-                    except AttributeError:
-                        return None
-                    else:
-                        return fp
-                else:
-                    return None
-            return None
-
-    # TODO the 3 static functions get_proposal_file, get_sample_file and
-    # get_source_scans are sharing a lot of source code and should
-    # be 'concatenate'
-    @staticmethod
-    def get_sample_file(process_file, entry):
-        """Return the proposal file of the experimentation if registred by the
-        data listener"""
-        if entry is None:
-            with open_hdf5(process_file) as h5f:
-                entries = BaseProcessInfo._get_process_nodes(
-                    root_node=h5f, process=DataListener
-                )
-                if len(entries) == 0:
-                    _logger.info(
-                        "unable to find a DarkRef process in %s" % process_file
-                    )
-                    return None
-                elif len(entries) > 1:
-                    raise ValueError("several entry found, entry should be " "specify")
-                else:
-                    entry = list(entries.keys())[0]
-                    _logger.info("take %s as default entry" % entry)
-
-        with open_hdf5(process_file) as h5f:
-            dl_nodes = BaseProcessInfo._get_process_nodes(
-                root_node=h5f[entry], process=DataListener
-            )
-            index_to_path = {}
-            for key, value in dl_nodes.items():
-                index_to_path[key] = key
-
-            if len(dl_nodes) == 0:
-                return None
-            # take the last processed dark ref
-            last_process_index = sorted(dl_nodes.keys())[-1]
-            last_process_dl = index_to_path[last_process_index]
-            if (len(index_to_path)) > 1:
-                _logger.debug(
-                    "several processing found for data listener,"
-                    "take the last one: %s" % last_process_dl
-                )
-
-            if "configuration" in h5f[last_process_dl].keys():
-                results_node = h5f[last_process_dl]["configuration"]
-                if "sample_file" in results_node.keys():
-                    return h5py_read_dataset(
-                        results_node["sample_file"]
-                    )  # .decode("UTF-8")
-                else:
-                    return None
-            return None
-
-    @staticmethod
-    def get_master_sample_file(process_file, entry):
-        """Return the proposal file of the experimentation if registered by the
-        data listener"""
-        if entry is None:
-            with open_hdf5(process_file) as h5f:
-                entries = BaseProcessInfo._get_process_nodes(
-                    root_node=h5f, process=DataListener
-                )
-                if len(entries) == 0:
-                    _logger.info(
-                        "unable to find a DarkRef process in %s" % process_file
-                    )
-                    return None
-                elif len(entries) > 1:
-                    raise ValueError("several entry found, entry should be " "specify")
-                else:
-                    entry = list(entries.keys())[0]
-                    _logger.info("take %s as default entry" % entry)
-
-        with open_hdf5(process_file) as h5f:
-            dl_nodes = BaseProcessInfo._get_process_nodes(
-                root_node=h5f[entry], process=DataListener
-            )
-            index_to_path = {}
-            for key, value in dl_nodes.items():
-                index_to_path[key] = key
-
-            if len(dl_nodes) == 0:
-                return None
-            # take the last processed dark ref
-            last_process_index = sorted(dl_nodes.keys())[-1]
-            last_process_dl = index_to_path[last_process_index]
-            if (len(index_to_path)) > 1:
-                _logger.debug(
-                    "several processing found for data listener,"
-                    "take the last one: %s" % last_process_dl
-                )
-
-            if "configuration" in h5f[last_process_dl].keys():
-                results_node = h5f[last_process_dl]["configuration"]
-                if "master_sample_file" in results_node.keys():
-                    return h5py_read_dataset(
-                        results_node["master_sample_file"]
-                    )  ##.decode("UTF-8")
-                else:
-                    return None
-            return None
-
-    @staticmethod
-    def get_source_scans(process_file, entry):
-        """Return the list of 'bliss scan directory' created for holding this
-        specific sequence data
-        """
-        if entry is None:
-            with open_hdf5(process_file) as h5f:
-                entries = BaseProcessInfo._get_process_nodes(
-                    root_node=h5f, process=DataListener
-                )
-                if len(entries) == 0:
-                    _logger.info(
-                        "unable to find a DarkRef process in %s" % process_file
-                    )
-                    return None
-                elif len(entries) > 1:
-                    raise ValueError("several entry found, entry should be " "specify")
-                else:
-                    entry = list(entries.keys())[0]
-                    _logger.info("take %s as default entry" % entry)
-
-        with open_hdf5(process_file) as h5f:
-            dl_nodes = BaseProcessInfo._get_process_nodes(
-                root_node=h5f[entry], process=DataListener
-            )
-            index_to_path = {}
-            for key, index in dl_nodes.items():
-                index_to_path[index] = key
-
-            if len(dl_nodes) == 0:
-                return {}
-            # take the last processed dark ref
-            last_process_index = sorted(dl_nodes.values())[-1]
-            last_process_dl = index_to_path[last_process_index]
-            if (len(index_to_path)) > 1:
-                _logger.debug(
-                    "several processing found for data listener,"
-                    "take the last one: %s" % last_process_dl
-                )
-
-            if "configuration" in h5f[last_process_dl].keys():
-                results_node = h5f[last_process_dl]["configuration"]
-                if "source_scans" in results_node.keys():
-                    tmp_res = h5py_read_dataset(results_node["source_scans"])
-
-                    def conv(my_str):
-                        if hasattr(my_str, "decode"):
-                            return my_str.decode("UTF-8")
-                        else:
-                            return my_str
-
-                    res = [conv(mystr) for mystr in tmp_res]
-                    return res
-                else:
-                    return None
-            return None
-
     def convert(self, bliss_file: str, entry: str) -> tuple:
         """
 

tomwer/core/process/control/datawatcher/datawatcher.py

@@ -409,11 +409,6 @@ class DataWatcher(_DataWatcher):
     def _signalScanReady(self, scan):
         assert isinstance(scan, TomwerScanBase)
         super()._signalScanReady(scan)
-        if self._serialize_output_data:
-            value = scan.to_dict()
-        else:
-            value = scan
-        self.register_output(key="data", value=value)
         self.scan_found_event.set()
 
     def _launchObservation(self):

tomwer/core/process/control/scantransfer.py

@@ -209,7 +209,6 @@ class ScanTransferTask(
                 callback_parameters=(scan, output_scan),
                 rights=777,
             )
-        self.register_output(key="data", value=outputdir)
         return output_scan
 
     def _get_hdf5_dst_scan(self, bliss_scan_folder_path):
@@ -248,7 +247,9 @@ class ScanTransferTask(
 
     def _process_hdf5_scan(self, scan) -> TomwerScanBase:
         assert isinstance(scan, NXtomoScan)
-        from tomwer.core.process.control.datalistener import DataListener
+        logger.warning(
+            "scan transfer for HDF5 is a prototype. Please check transfer is properly executed."
+        )
 
         files_sources = []
         files_dest = []
@@ -259,53 +260,6 @@ class ScanTransferTask(
             lambda file_: os.path.abspath(file_) == os.path.abspath(scan.master_file),
             associated_files,
         )
-        source_scans = [os.path.dirname(file_) for file_ in associated_files]
-        # manage scan folder
-        if source_scans is not None:
-            for source_scan in source_scans:
-                # files_sources.append(source_scan)
-                # files_dest.append(
-                #    self._get_hdf5_dst_scan(bliss_scan_folder_path=source_scan)
-                # )
-                # delete_opt.append(True)
-                # scan folder is no more managed. Now this is done by browsing VDS directly
-                pass
-
-        # manage proposal file
-        if os.path.exists(scan.process_file):
-            proposal_file = DataListener.get_proposal_file(
-                process_file=scan.process_file, entry=scan.entry
-            )
-        else:
-            proposal_file = None
-        if proposal_file is not None:
-            files_sources.append(proposal_file)
-            files_dest.append(self._get_hdf5_proposal_file_dst(proposal_file))
-            delete_opt.append(False)
-
-        # manage sample file
-        if os.path.exists(scan.process_file):
-            sample_file = DataListener.get_sample_file(
-                process_file=scan.process_file, entry=scan.entry
-            )
-        else:
-            sample_file = None
-        if sample_file is not None:
-            files_sources.append(sample_file)
-            files_dest.append(self._get_hdf5_sample_file_or_nx_dst(sample_file))
-            delete_opt.append(False)
-
-        # manage saving file
-        if os.path.exists(scan.process_file):
-            mst_sample_file = DataListener.get_master_sample_file(
-                process_file=scan.process_file, entry=scan.entry
-            )
-        else:
-            mst_sample_file = None
-        if mst_sample_file is not None:
-            files_sources.append(mst_sample_file)
-            files_dest.append(self._get_master_sample_file_dst(mst_sample_file))
-            delete_opt.append(False)
 
         # manage .nx file
         if os.path.exists(scan.master_file):
@@ -507,17 +461,6 @@ class ScanTransferTask(
     def __noticeTransfertSuccess(self, input_scan, output_scan):
         self._signalCopySucceed()
 
-        results = {"input_scan": str(input_scan), "output_scan": str(output_scan)}
-        entry = "entry"
-        if isinstance(output_scan, NXtomoScan):
-            entry = output_scan.entry
-        self.register_process(
-            process_file=output_scan.process_file,
-            entry=entry,
-            results=results,
-            process_index=output_scan.pop_process_index(),
-            configuration=None,
-        )
         logger.processSucceed(
             f"transfer succeed of {input_scan} to {output_scan}",
             extra={

tomwer/core/process/edit/darkflatpatch.py

@@ -69,14 +69,6 @@ class DarkFlatPatchTask(
             if isinstance(value, DataUrl):
                 config[key] = value.path()
 
-        self.register_process(
-            process_file=scan.process_file,
-            entry=scan.entry,
-            configuration=config,
-            results={},
-            process_index=scan.pop_process_index(),
-            overwrite=True,
-        )
         if self.get_input_value("serialize_output_data", True):
             self.outputs.data = scan.to_dict()
         else:

tomwer/core/process/edit/imagekeyeditor.py

@@ -115,14 +115,7 @@ class ImageKeyEditorTask(
             value = config["modifications"][key]
             config["modifications"][str(key)] = value
         config["modifications"] = new_modif
-        self.register_process(
-            process_file=scan.process_file,
-            entry=scan.entry,
-            configuration=config,
-            results={},
-            process_index=scan.pop_process_index(),
-            overwrite=True,
-        )
+
         if self.get_input_value("serialize_output_data", True):
             self.outputs.data = scan.to_dict()
         else:
@@ -175,17 +168,7 @@ class ImageKeyUpgraderTask(
 
         # apply modification using tomoscan
         change_image_key_control(scan=scan, config=configuration)
-        self.register_process(
-            process_file=scan.process_file,
-            entry=scan.entry,
-            configuration={
-                ImageKey.from_value(key).value: ImageKey.from_value(value).value
-                for key, value in operations.items()
-            },
-            results={},
-            process_index=scan.pop_process_index(),
-            overwrite=True,
-        )
+
         if self.get_input_value("serialize_output_data", True):
             self.outputs.data = scan.to_dict()
         else:

tomwer/core/process/reconstruction/axis/axis.py

@@ -251,19 +251,6 @@ class AxisTask(
 
     def _process_end(self, scan, cor, error=None):
         assert isinstance(scan, TomwerScanBase)
-        if scan.process_file is not None:
-            entry = "entry"
-            if isinstance(scan, NXtomoScan):
-                entry = scan.entry
-            self.register_process(
-                process_file=scan.process_file,
-                entry=entry,
-                results={"center_of_rotation": cor if cor is not None else "-"},
-                configuration=self._axis_params.to_dict(),
-                process_index=scan.pop_process_index(),
-                overwrite=True,
-            )
-
         try:
             extra = {
                 logconfig.DOC_TITLE: self._scheme_title,

tomwer/core/process/reconstruction/darkref/darkrefs.py

@@ -20,7 +20,6 @@ import tomwer.version
 from tomwer.core import settings
 from tomwer.core.utils.lbsram import is_low_on_memory
 from tomwer.core.process.task import Task
-from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import data_identifier_to_scan
@@ -200,47 +199,6 @@ class DarkRefsTask(
         else:
             self._recons_params.load_from_dict(properties)
 
-    @staticmethod
-    def get_darks_frm_process_file(
-        process_file, entry=None, as_url: bool = False
-    ) -> dict | None:
-        """
-
-        :param process_file: path to the process file
-        :param entry: entry to read in the process file if more than one
-        :param as_url: if true then an url will be used instead of a
-                            numpy.array
-        :return: dictionary with index in the sequence as key and numpy array
-                 as value (or url if as_url set to True)
-        """
-        deprecated_warning(
-            name="tomwer.core.process.reconstruction.darkref.DarkRefsTask.get_darks_frm_process_file",
-            type_="function",
-            reason="removed. Reduced dark / flat are expected to be stored on didicated files and not in tomwer_processes anymore",
-            since_version="1.2",
-        )
-        return None
-
-    @staticmethod
-    def get_flats_frm_process_file(
-        process_file, entry=None, as_url: bool = False
-    ) -> dict | None:
-        """
-
-        :param process_file:
-        :param entry: entry to read in the process file if more than one
-        :param as_url: if true then an url will be used instead of a
-                            numpy.array
-        :return:
-        """
-        deprecated_warning(
-            name="tomwer.core.process.reconstruction.darkref.DarkRefsTask.get_flats_frm_process_file",
-            type_="function",
-            reason="removed. Reduced dark / flat are expected to be stored on didicated files and not in tomwer_processes anymore",
-            since_version="1.2",
-        )
-        return None
-
     def run(self):
         if isinstance(self.inputs.data, str):
             try:
@@ -364,23 +322,6 @@ class DarkRefsTask(
 
             scan.save_reduced_flats(f_flats, flats_infos=o_flats_infos, overwrite=True)
 
-        if len(results) > 0:
-            # if some processing to be registered
-            if scan.process_file is not None and not (
-                isinstance(scan, NXtomoScan) and not self.__new_hdf5_entry_created
-            ):
-                entry = "entry"
-                if hasattr(scan, "entry"):
-                    entry = scan.entry
-                self.register_process(
-                    process_file=scan.process_file,
-                    entry=entry,
-                    configuration=self.recons_params.to_dict(),
-                    results=results,
-                    interpretations=interpretations,
-                    process_index=scan.pop_process_index(),
-                    overwrite=True,
-                )
         logger.processSucceed(f"Dark and flat reduction succeeded for {scan}")
         self.notify_to_state_to_managed(
             dataset=scan, state=DatasetState.SUCCEED, details=None

tomwer/core/process/reconstruction/nabu/nabuslices.py

@@ -54,8 +54,6 @@ from nabu.pipeline.fullfield.nabu_config import (
     nabu_config as nabu_fullfield_default_config,
 )
 from processview.core.superviseprocess import SuperviseProcess
-from silx.io.dictdump import h5todict
-from silx.io.utils import h5py_read_dataset
 from silx.utils.enum import Enum as _Enum
 
 from tomwer.core.process.task import Task
@@ -63,7 +61,6 @@ from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
-from tomwer.io.utils.h5pyutils import EntryReader
 from tomwer.utils import docstring
 
 from . import settings as nabu_settings
@@ -276,10 +273,6 @@ class NabuSlicesTask(
             raise ValueError(f"input type of {scan}: {type(scan)} is not managed" "")
         assert isinstance(configuration, dict), "configuration is expected to be a dict"
 
-        entry = "entry"
-        if isinstance(scan, NXtomoScan):
-            entry = scan.entry
-
         output_urls = []
         stderrs = []
         stdouts = []
@@ -362,7 +355,6 @@ class NabuSlicesTask(
                 VolumeFactory.from_identifier_to_vol_urls(rec_identifier)
             )
 
-        process_index = scan.pop_process_index()
         # update processes information / registration
         gc.collect()
 
@@ -395,15 +387,6 @@ class NabuSlicesTask(
             details=details,
         )
 
-        # register result
-        self.register_process(
-            process_file=scan.process_file,
-            entry=entry,
-            configuration=configuration,
-            results={},
-            process_index=process_index,
-            overwrite=True,
-        )
         if self.get_input_value("serialize_output_data", True):
             self.outputs.data = scan.to_dict()
         else:
@@ -433,69 +416,6 @@ class NabuSlicesTask(
     def dry_run(self):
         return self._dry_run
 
-    @staticmethod
-    def get_process_frm_process_file(process_file: str, entry: str) -> dict:
-        """
-        Read informations regarding the nabu process save in the tomwer_process.h5 file
-
-        :return: dictionary with the contain of the nabu process
-        """
-        if entry is None:
-            with open_hdf5(process_file) as h5f:
-                entries = NabuSlicesTask._get_process_nodes(
-                    root_node=h5f, process=NabuSlicesTask
-                )
-                if len(entries) == 0:
-                    _logger.info("unable to find a Axis process in %s" % process_file)
-                    return None
-                elif len(entries) > 1:
-                    raise ValueError("several entry found, entry should be " "specify")
-                else:
-                    entry = list(entries.keys())[0]
-                    _logger.info("take %s as default entry" % entry)
-
-        configuration_path = None
-        res = {}
-
-        with open_hdf5(process_file) as h5f:
-            nabu_nodes = NabuSlicesTask._get_process_nodes(
-                root_node=h5f[entry], process=NabuSlicesTask
-            )
-            index_to_path = {}
-            for key, index in nabu_nodes.items():
-                index_to_path[index] = key
-
-            if len(nabu_nodes) == 0:
-                return None
-            # take the last processed dark ref
-            last_process_index = sorted(list(nabu_nodes.values()))[-1]
-            last_process_dark = index_to_path[last_process_index]
-            if (len(index_to_path)) > 1:
-                _logger.debug(
-                    "several processing found for dark-ref, "
-                    "take the last one: %s" % last_process_dark
-                )
-
-            for key_name in (
-                "class_instance",
-                "date",
-                "program",
-                "sequence_index",
-                "version",
-            ):
-                if key_name in h5f[last_process_dark]:
-                    res[key_name] = h5py_read_dataset(h5f[last_process_dark][key_name])
-            if "configuration" in h5f[last_process_dark]:
-                configuration_path = "/".join(
-                    (h5f[last_process_dark].name, "configuration")
-                )
-
-        if configuration_path is not None:
-            res["configuration"] = h5todict(
-                h5file=process_file, path=configuration_path
-            )
-        return res
-
     def cancel(self):
         """
         stop current processing
@@ -504,14 +424,6 @@ class NabuSlicesTask(
         if self._current_processing is not None:
             self._current_processing.cancel()
 
-    @staticmethod
-    def retrieve_last_relative_cor(scan):
-        with EntryReader(scan.process_file_url) as h5f:
-            latest_nabu_node = Task.get_most_recent_process(h5f, NabuSlicesTask)
-            path = "configuration/reconstruction/rotation_axis_position"
-            if latest_nabu_node is not None and path in latest_nabu_node:
-                return h5py_read_dataset(latest_nabu_node[path])
-
 
 def interpret_tomwer_configuration(config: dict, scan: TomwerScanBase | None) -> tuple:
     """

tomwer/core/process/reconstruction/nabu/nabuvolume.py

@@ -14,7 +14,6 @@ from nabu.pipeline.fullfield.nabu_config import (
 from processview.core.manager.manager import ProcessManager, DatasetState
 from processview.core.superviseprocess import SuperviseProcess
 
-from silx.io.utils import h5py_read_dataset
 from silx.io.utils import open as open_hdf5
 from tomwer.core.utils.deprecation import deprecated_warning
 
@@ -30,7 +29,6 @@ from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.dictutils import concatenate_dict
 from tomwer.core.utils.scanutils import data_identifier_to_scan
 from tomwer.core.volume.volumefactory import VolumeFactory
-from tomwer.io.utils.h5pyutils import EntryReader
 from tomwer.utils import docstring
 from tomwer.io.utils import format_stderr_stdout
 from tomwer.core.process.reconstruction.output import (
@@ -548,14 +546,6 @@ class NabuVolumeTask(
     def dry_run(self):
         return self._dry_run
 
-    @staticmethod
-    def retrieve_last_relative_cor(scan):
-        with EntryReader(scan.process_file_url) as h5f:
-            latest_nabu_node = Task.get_most_recent_process(h5f, NabuVolumeTask)
-            path = "configuration/reconstruction/rotation_axis_position"
-            if latest_nabu_node is not None and path in latest_nabu_node:
-                return h5py_read_dataset(latest_nabu_node[path])
-
     def cancel(self):
         """
         stop current processing