tomwer 1.2.0a1__py3-none-any.whl → 1.2.0a3__py3-none-any.whl

tomwer/core/process/reconstruction/nabu/nabuvolume.py

@@ -39,8 +39,13 @@ from nabu.pipeline.config import generate_nabu_configfile
 from nabu.pipeline.fullfield.nabu_config import (
     nabu_config as nabu_fullfield_default_config,
 )
-from processview.core.manager.manager import ProcessManager
+from processview.core.manager.manager import ProcessManager, DatasetState
+from processview.core.superviseprocess import SuperviseProcess
+
 from silx.io.utils import h5py_read_dataset
+from silx.utils.deprecation import deprecated_warning
+
+from tomoscan.io import HDF5File
 
 from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.futureobject import FutureTomwerObject
@@ -52,6 +57,8 @@ from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import data_identifier_to_scan
 from tomwer.io.utils.h5pyutils import EntryReader
 from tomwer.utils import docstring
+from tomwer.core.utils import concatenate_dict
+from tomwer.io.utils import format_stderr_stdout
 
 from . import settings, utils
 from .nabucommon import ResultsLocalRun, ResultSlurmRun, _NabuBaseReconstructor
@@ -93,23 +100,23 @@ def run_volume_reconstruction(
     else:
         target = Target.SLURM
 
-    # beam shape is not directly used by nabu (uses ctf_geometry directly)
-    config.get("phase", {}).pop("beam_shape", None)
-
-    config_volume = copy.copy(config)
-    config_nabu_slices = copy.deepcopy(scan.nabu_recons_params)
-    if "tomwer_slices" in config_nabu_slices:
-        del config_nabu_slices["tomwer_slices"]
-
-    if "phase" in config_nabu_slices and "delta_beta" in config_nabu_slices["phase"]:
-        pag_dbs = config_nabu_slices["phase"]["delta_beta"]
-        if isinstance(pag_dbs, str):
-            pag_dbs = utils.retrieve_lst_of_value_from_str(pag_dbs, type_=float)
-        if len(pag_dbs) > 1:
-            raise ValueError(
-                "Several value of delta / beta found for volume reconstruction"
-            )
-    scan.clear_latest_vol_reconstructions()
+    # # beam shape is not directly used by nabu (uses ctf_geometry directly)
+    # config.get("phase", {}).pop("beam_shape", None)
+
+    # config_volume = copy.copy(config)
+    # config_nabu_slices = copy.deepcopy(scan.nabu_recons_params)
+    # if "tomwer_slices" in config_nabu_slices:
+    #     del config_nabu_slices["tomwer_slices"]
+
+    # if "phase" in config_nabu_slices and "delta_beta" in config_nabu_slices["phase"]:
+    #     pag_dbs = config_nabu_slices["phase"]["delta_beta"]
+    #     if isinstance(pag_dbs, str):
+    #         pag_dbs = utils.retrieve_lst_of_value_from_str(pag_dbs, type_=float)
+    #     if len(pag_dbs) > 1:
+    #         raise ValueError(
+    #             "Several value of delta / beta found for volume reconstruction"
+    #         )
+    # scan.clear_latest_vol_reconstructions()
 
     if process_id is not None:
         try:
@@ -121,8 +128,7 @@ def run_volume_reconstruction(
 
     volume_reconstructor = VolumeRunner(
         scan=scan,
-        config_volume=config_volume,
-        config_slices=config_nabu_slices,
+        config_nabu=config,
         cluster_config=cluster_config,
         dry_run=dry_run,
         target=target,
@@ -171,6 +177,7 @@ def run_volume_reconstruction(
             if res is not None
             else []
         )
+
         return succeed, stdouts, stderrs, configs, future_tomo_obj
 
 
@@ -185,8 +192,7 @@ class VolumeRunner(_NabuBaseReconstructor):
     def __init__(
         self,
         scan: TomwerScanBase,
-        config_volume: dict,
-        config_slices: dict,
+        config_nabu,
         cluster_config: Optional[dict],
         dry_run: bool,
         target: Target,
@@ -199,58 +205,44 @@ class VolumeRunner(_NabuBaseReconstructor):
             cluster_config=cluster_config,
             process_name=process_name,
         )
-        self._config_volume = config_volume
-        self._config_slices = config_slices
+        self._config = config_nabu
 
     @property
-    def config_volume(self):
-        return self._config_volume
-
-    @property
-    def config_slices(self):
-        return self._config_slices
+    def configuration(self):
+        return self._config
 
     @docstring(_NabuBaseReconstructor)
     def run(self) -> Iterable:
         dataset_params = self.scan.get_nabu_dataset_info()
-        if "dataset" in self._config_slices:
-            dataset_params.update(self._config_slices["dataset"])
-        self._config_slices["dataset"] = dataset_params
-        self._config_slices["resources"] = utils.get_nabu_resources_desc(
+        if "dataset" in self.configuration:
+            dataset_params.update(self.configuration["dataset"])
+        self.configuration["dataset"] = dataset_params
+        self.configuration["resources"] = utils.get_nabu_resources_desc(
             scan=self.scan, workers=1, method="local"
         )
 
         # force overwrite results
-        if "output" not in self.config_slices:
-            self._config_slices["output"] = {}
-        config_slices, extra_opts = self._treateOutputConfig(
-            self.config_slices, self.config_volume
-        )
+        if "output" not in self.configuration:
+            self.configuration["output"] = {}
+        config_slices, cfg_folder = self._treateOutputConfig(self.configuration)
+
+        # force overwriting results
         config_slices["output"].update({"overwrite_results": 1})
 
         # check and clamp `start_z` and `end_z`
-        if "reconstruction" in self.config_slices:
+        if "reconstruction" in self.configuration:
             for key in ("start_z", "end_z"):
-                value = self.config_slices["reconstruction"].get(key)
+                value = config_slices["reconstruction"].get(key)
                 if value is None:
                     continue
 
                 value = int(value)
-                if value < 0:
-                    _logger.warning(f"{key} < 0 found. Set it to 0 (vs value)")
-                    value = 0
                 if self.scan.dim_2 is not None and value >= self.scan.dim_2:
                     _logger.warning(
                         f"{key} > max_size (radio height: {self.scan.dim_2}). Set it to -1 (maximum)"
                     )
                     value = -1
-                self._config_slices["reconstruction"][key] = value
-
-        cfg_folder = os.path.join(
-            config_slices["output"]["location"], settings.NABU_CFG_FILE_FOLDER
-        )
-        if not os.path.exists(cfg_folder):
-            os.makedirs(cfg_folder)
+                config_slices["reconstruction"][key] = value
 
         name = (
             config_slices["output"]["file_prefix"] + settings.NABU_CONFIG_FILE_EXTENSION
@@ -260,6 +252,9 @@ class VolumeRunner(_NabuBaseReconstructor):
         conf_file = os.path.join(cfg_folder, name)
         _logger.info(f"{self.scan}: create {conf_file}")
 
+        # make sure output location exists
+        os.makedirs(config_slices["output"]["location"], exist_ok=True)
+
         # add some tomwer metadata and save the configuration
         # note: for now the section is ignored by nabu but shouldn't stay that way
         with utils.TomwerInfo(config_slices) as config_to_dump:
@@ -269,20 +264,19 @@ class VolumeRunner(_NabuBaseReconstructor):
                 config=config_to_dump,
                 options_level="advanced",
             )
-
-        return tuple(
-            [
-                self._process_config(
-                    config_to_dump=config_to_dump,
-                    config_file=conf_file,
-                    start_z=config_to_dump["reconstruction"]["start_z"],
-                    end_z=config_to_dump["reconstruction"]["end_z"],
-                    info="nabu volume reconstruction",
-                    file_format=config_slices["output"]["file_format"],
-                    process_name=self.process_name,
-                ),
-            ]
-        )
+            return tuple(
+                [
+                    self._process_config(
+                        config_to_dump=config_to_dump,
+                        config_file=conf_file,
+                        start_z=config_to_dump["reconstruction"]["start_z"],
+                        end_z=config_to_dump["reconstruction"]["end_z"],
+                        info="nabu volume reconstruction",
+                        file_format=config_slices["output"]["file_format"],
+                        process_name=self.process_name,
+                    )
+                ]
+            )
 
     @docstring(_NabuBaseReconstructor)
     def _get_futures_slurm_callback(self, config_to_dump) -> tuple:
@@ -316,26 +310,27 @@ class VolumeRunner(_NabuBaseReconstructor):
 
         return (CallBack(callback, self.scan),)
 
-    def _treateOutputConfig(self, config_s, config_v) -> tuple:
+    def _treateOutputConfig(self, config) -> tuple:
         """
 
         :return: (nabu config dict, nabu extra options)
         """
-        config_s = copy.deepcopy(config_s)
-        config_s = super()._treateOutputSliceConfig(config_s)
+        config = copy.deepcopy(config)
+        config, nabu_cfg_folder = super()._treateOutputSliceConfig(config)
+        os.makedirs(config["output"]["location"], exist_ok=True)
+
         # adapt config_s to specific volume treatment
-        if "postproc" in config_v:
-            config_s["postproc"] = config_v["postproc"]
+        if "postproc" in config:
+            config["postproc"] = config["postproc"]
 
-        extra_opts = config_v
-        if "start_z" in extra_opts:
-            config_s["reconstruction"]["start_z"] = extra_opts["start_z"]
-            del extra_opts["start_z"]
-        if "end_z" in extra_opts:
-            config_s["reconstruction"]["end_z"] = extra_opts["end_z"]
-            del extra_opts["end_z"]
+        if "start_z" in config:
+            config["reconstruction"]["start_z"] = config["start_z"]
+            del config["start_z"]
+        if "end_z" in config:
+            config["reconstruction"]["end_z"] = config["end_z"]
+            del config["end_z"]
 
-        return config_s, extra_opts
+        return config, nabu_cfg_folder
 
     @docstring(_NabuBaseReconstructor)
     def _get_file_basename_reconstruction(self, pag, db, ctf):
@@ -354,8 +349,16 @@ class VolumeRunner(_NabuBaseReconstructor):
         if isinstance(self.scan, HDF5TomoScan):
             basename, _ = os.path.splitext(self.scan.master_file)
             basename = os.path.basename(basename)
+            try:
+                # if there is more than one entry in the file append the entry name to the file basename
+                with HDF5File(self.scan.master_file, mode="r") as h5f:
+                    if len(h5f.keys()) > 1:
+                        basename = "_".join((basename, self.scan.entry.strip("/")))
+            except Exception:
+                pass
         else:
             basename = os.path.basename(self.scan.path)
+
         if pag:
             return "_".join((basename + "pag", "db" + str(db).zfill(4), "vol"))
         elif ctf:
@@ -364,15 +367,27 @@ class VolumeRunner(_NabuBaseReconstructor):
             return "_".join((basename, "vol"))
 
 
-class NabuVolume(
+class NabuVolumeTask(
     Task,
-    input_names=("data",),
-    optional_input_names=("nabu_params",),
-    output_names=("data", "volumes"),
+    SuperviseProcess,
+    input_names=("data", "nabu_params"),
+    output_names=("data", "volumes", "future_tomo_obj"),
+    optional_input_names=(
+        "dry_run",
+        "nabu_extra_params",  # some parameter that must update 'nabu_params' before launching the reconstruction. Such as z range...
+        "serialize_output_data",
+    ),
 ):
     def __init__(
-        self, varinfo=None, inputs=None, node_id=None, node_attrs=None, execinfo=None
+        self,
+        process_id=None,
+        varinfo=None,
+        inputs=None,
+        node_id=None,
+        node_attrs=None,
+        execinfo=None,
     ):
+        SuperviseProcess.__init__(self, process_id=process_id)
         Task.__init__(
             self,
             varinfo=varinfo,
@@ -382,15 +397,40 @@ class NabuVolume(
             execinfo=execinfo,
         )
         self._dry_run = inputs.get("dry_run", False)
+        self._current_processing = None
 
     def run(self):
         scan = data_identifier_to_scan(self.inputs.data)
         if scan is None:
             self.outputs.data = None
             return
-        nabu_params = self.inputs.nabu_params
-        if nabu_params is None:
-            nabu_params = scan.nabu_params
+
+        # update scan reconstruction parameters used
+        nabu_params = concatenate_dict(
+            copy.deepcopy(self.inputs.nabu_params),
+            self.get_input_value("nabu_extra_params", dict()),
+        )
+        scan.nabu_recons_params = nabu_params
+        scan.clear_latest_vol_reconstructions()
+
+        cluster_config = nabu_params.pop("cluster_config", None)
+        if cluster_config == {}:
+            cluster_config = None
+        elif isinstance(cluster_config, SlurmClusterConfiguration):
+            cluster_config = cluster_config.to_dict()
+
+        if "tomwer_slices" in nabu_params:
+            del nabu_params["tomwer_slices"]
+
+        if "phase" in nabu_params and "delta_beta" in nabu_params["phase"]:
+            pag_dbs = nabu_params["phase"]["delta_beta"]
+            if isinstance(pag_dbs, str):
+                pag_dbs = utils.retrieve_lst_of_value_from_str(pag_dbs, type_=float)
+            if len(pag_dbs) > 1:
+                raise ValueError(
+                    "Several value of delta / beta found for volume reconstruction"
+                )
+
         if isinstance(scan, TomwerScanBase):
             scan = scan
         elif isinstance(scan, dict):
@@ -398,30 +438,94 @@ class NabuVolume(
         else:
             raise ValueError(f"input type {scan} is not managed")
 
-        if scan.nabu_recons_params is None:
-            # if reconstruction parameters are not registered, register them
-            scan.nabu_recons_params = nabu_params
+        if cluster_config is None:
+            target = Target.LOCAL
+        else:
+            target = Target.SLURM
 
-        run_volume_reconstruction(
-            scan=scan,
-            config=self.get_configuration() or {},
-            dry_run=self.dry_run,
-        )
-        # register result
-        entry = "entry"
-        if isinstance(scan, HDF5TomoScan):
-            entry = scan.entry
-        with scan.acquire_process_file_lock():
-            self.register_process(
-                process_file=scan.process_file,
-                entry=entry,
-                configuration=self.get_configuration(),
-                results={},
-                process_index=scan.pop_process_index(),
-                overwrite=True,
+        state = None
+        details = None
+        try:
+            self._current_processing = VolumeRunner(
+                scan=scan,
+                config_nabu=nabu_params,
+                cluster_config=cluster_config,
+                dry_run=self._dry_run,
+                target=target,
+                process_name=self.program_name,
             )
-        self.outputs.data = scan
+            res = self._current_processing.run()[0]
+        except Exception as e:
+            _logger.error(f"Failed to process {scan}. Error is {e}")
+            state = DatasetState.FAILED
+            details = None
+            future_tomo_obj = None
+        else:
+            # tag latest reconstructions
+            if isinstance(res, ResultsLocalRun) and res.results_urls is not None:
+                scan.set_latest_vol_reconstructions(res.results_urls)
+            # create future if needed
+            if isinstance(res, ResultSlurmRun):
+                future_tomo_obj = FutureTomwerObject(
+                    tomo_obj=scan,
+                    futures=tuple(res.future_slurm_jobs),
+                    process_requester_id=self.process_id,
+                )
+
+            else:
+                future_tomo_obj = None
+
+            if self._cancelled:
+                state = DatasetState.CANCELLED
+                details = "cancelled by user"
+                _logger.info(f"Slices computation for {scan} cancelled")
+            else:
+                succeed = res.success
+                stdouts = (
+                    [
+                        res.std_out,
+                    ]
+                    if hasattr(res, "std_out")
+                    else []
+                )
+                stderrs = (
+                    [
+                        res.std_err,
+                    ]
+                    if hasattr(res, "std_err")
+                    else []
+                )
+
+                if not succeed:
+                    mess = f"Volume computed for {scan} failed."
+                    _logger.processFailed(mess)
+                    state = DatasetState.FAILED
+                else:
+                    mess = f"Volume computed for {scan}."
+                    _logger.processSucceed(mess)
+                    state = DatasetState.SUCCEED
+
+                # format stderr and stdout
+                elmts = [
+                    format_stderr_stdout(stderr=stderr, stdout=stdout)
+                    for stderr, stdout in zip(stderrs, stdouts)
+                ]
+                elmts.insert(0, mess)
+                details = "\n".join(elmts)
+        finally:
+            ProcessManager().notify_dataset_state(
+                dataset=scan,
+                process=self,
+                state=state,
+                details=details,
+            )
+
+        if self.get_input_value("serialize_output_data", True):
+            self.outputs.data = scan.to_dict()
+        else:
+            self.outputs.data = scan
         self.outputs.volumes = scan.latest_vol_reconstructions
+        self.outputs.future_tomo_obj = future_tomo_obj
 
     def set_configuration(self, configuration: dict) -> None:
         Task.set_configuration(self, configuration=configuration)
@@ -446,7 +550,35 @@ class NabuVolume(
     @staticmethod
     def retrieve_last_relative_cor(scan):
         with EntryReader(scan.process_file_url) as h5f:
-            latest_nabu_node = Task.get_most_recent_process(h5f, NabuVolume)
+            latest_nabu_node = Task.get_most_recent_process(h5f, NabuVolumeTask)
             path = "configuration/reconstruction/rotation_axis_position"
             if path in latest_nabu_node:
                 return h5py_read_dataset(latest_nabu_node[path])
+
+    def cancel(self):
+        """
+        stop current processing
+        """
+        self._cancelled = True
+        if self._current_processing is not None:
+            self._current_processing.cancel()
+
+
+class NabuVolume(NabuVolumeTask):
+    def __init__(
+        self,
+        process_id=None,
+        varinfo=None,
+        inputs=None,
+        node_id=None,
+        node_attrs=None,
+        execinfo=None,
+    ):
+        deprecated_warning(
+            name="tomwer.core.process.reconstruction.nabu.nabuvolume.NabuVolume",
+            type_="class",
+            reason="improve readibility",
+            since_version="1.2",
+            replacement="NabuVolumeTask",
+        )
+        super().__init__(process_id, varinfo, inputs, node_id, node_attrs, execinfo)

tomwer/core/process/reconstruction/nabu/utils.py

@@ -159,7 +159,7 @@ def get_recons_volume_identifier(
     if file_format in ("hdf5", "h5", "hdf"):
         if slice_index is not None:
             # case of a single hdf5 file
-            file_name = "_".join((file_prefix, str(slice_index).zfill(4)))
+            file_name = "_".join((file_prefix, str(slice_index).zfill(6)))
         else:
             file_name = file_prefix
         file_name = ".".join((file_name, file_format))
@@ -179,23 +179,13 @@ def get_recons_volume_identifier(
     elif file_format in ("vol", "raw"):
         if slice_index is not None:
             # case of a single hdf5 file
-            file_name = "_".join((file_prefix, str(slice_index).zfill(4)))
+            file_name = "_".join((file_prefix, str(slice_index).zfill(6)))
         else:
             file_name = file_prefix
-        # on 2022.3.x the location seems invalid for nabu. Add this second location to
-        # handle most of the case.
-        fix_nabu = os.path.join(file_prefix, file_name)
-
         file_name = ".".join((file_name, file_format))
         file_path = os.path.join(location, file_name)
 
-        file_name_fix_nabu = ".".join((fix_nabu, file_format))
-        file_path_fix_nabu = os.path.join(location, file_name_fix_nabu)
-
-        volumes = (
-            RawVolume(file_path=file_path_fix_nabu),
-            RawVolume(file_path=file_path),
-        )
+        volumes = (RawVolume(file_path=file_path),)
     elif file_format in ("jp2", "jp2k", "edf", "tiff"):
         if file_format in ("jp2k", "jp2"):
             constructor = JP2KVolume
@@ -205,31 +195,15 @@ def get_recons_volume_identifier(
             constructor = TIFFVolume
         else:
             raise NotImplementedError
-        output_extension = file_format
         basename = file_prefix
+        file_path = location
+        volumes = (
+            constructor(
+                folder=location,
+                volume_basename=basename,
+            ),
+        )
 
-        if slice_index is None and start_z is None and end_z is None:
-            # if this is a single slice
-            file_path = ".".join(
-                (
-                    os.path.join(location, file_prefix, file_prefix),
-                    output_extension,
-                )
-            )
-            # TODO: in the future we might want to provide data_slice according to z values
-            volumes = (
-                constructor(
-                    folder=os.path.join(location, file_prefix),
-                    volume_basename=basename,
-                ),
-            )
-        else:
-            volumes = (
-                constructor(
-                    folder=os.path.join(location, file_prefix),
-                    volume_basename=basename,
-                ),
-            )
     # case of the multitiff. Not handled by tomwer
     # elif file_format == "tiff":
     #     # for single frame tiff nabu uses another convention by saving it 'directly'.

tomwer/core/process/reconstruction/normalization/normalization.py

@@ -55,10 +55,14 @@ _logger = logging.getLogger(__name__)
 
 
 class SinoNormalizationTask(
-    Task, SuperviseProcess, input_names=("data",), output_names=("data",)
+    Task,
+    SuperviseProcess,
+    input_names=("data",),
+    output_names=("data",),
+    optional_input_names=("serialize_output_data",),
 ):
     """
-    Task to define the normalization to apply to a sinogram
+    Task to define the normalization to apply to a sinogram before reconstructing it with nabu
     """
 
     def __init__(
@@ -148,7 +152,10 @@ class SinoNormalizationTask(
         else:
             final_norm_info.update({"tomwer_processing_res_code": True})
         scan.intensity_normalization.set_extra_infos(final_norm_info)
-        self.outputs.data = scan
+        if self.get_input_value("serialize_output_data", True):
+            self.outputs.data = scan.to_dict()
+        else:
+            self.outputs.data = scan
 
     def _compute_from_manual_roi(self, scan):
         params = SinoNormalizationParams.from_dict(self.get_configuration())

tomwer/core/process/reconstruction/saaxis/__init__.py

@@ -1,2 +1,3 @@
 from .params import SAAxisParams  # noqa F401
 from .saaxis import SAAxisProcess  # noqa F401
+from .saaxis import SAAxisTask  # noqa F401