tomwer 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

tomwer/core/process/reconstruction/darkref/darkrefscopy.py

@@ -33,15 +33,19 @@ __date__ = "17/08/2021"
 
 
 import os
+import logging
+import tempfile
 from typing import Union
 from silx.io.url import DataUrl
 from silx.io.dictdump import dicttoh5
 from tomoscan.io import HDF5File
-import logging
+from tomoscan.esrf.scan.utils import cwd_context
+from tomoscan.framereducerbase import REDUCER_TARGET
 from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefs
 from tomwer.core.scan.scanbase import TomwerScanBase
-from tomoscan.esrf.scan.utils import get_data
+from tomwer.core.utils.scanutils import data_identifier_to_scan
 from tomoscan.esrf.scan.utils import copy_h5_dict_darks_to, copy_h5_dict_flats_to
+from silx.io.dictdump import h5todict
 
 logger = logging.getLogger(__name__)
 
@@ -76,11 +80,7 @@ class DarkRefsCopy(DarkRefs):
         self._mode_auto = inputs.get("mode_auto", True)
         if "savedir" in inputs:
             raise KeyError("savedir is not a valid key. Use save_dir")
-        self._savedir = inputs.get("save_dir", None)
-        if self._savedir is None:
-            raise KeyError(
-                "save_dir is expected (where the ref and dark file should be saved)"
-            )
+        self._savedir = inputs.get("save_dir", tempfile.mkdtemp())
         self._save_file = self.get_save_file(self._savedir)
         self._darks_url = self.get_darks_url(self._savedir)
         self._flats_url = self.get_flats_url(self._savedir)
@@ -127,10 +127,42 @@ class DarkRefsCopy(DarkRefs):
         self._processOnlyDkRf = False
         self._processOnlyCopy = value
 
+    @staticmethod
+    def get_reduced_frame_data(
+        url: DataUrl, reduced_target: REDUCER_TARGET, check_for_reduced_key: bool = True
+    ) -> tuple:
+        """
+        :param Data url: url expected to contain the dict with one key per serie of reduced frame
+        :param REDUCER_TARGET reduced_target: what we are looking for (used when check_for_reduced_key is set to True)
+        :param bool check_for_reduced_key: if True then if the dict contains the given key then return the value of this key instead
+                                           of the loaded dict. Used as a user friendly feature to be able to load dict when upper data path
+                                           is provided instead of the 'darks' or 'flats' one.
+        :return: data, metadata
+        :rtype: tuple of two dicts
+        """
+        reduced_target = REDUCER_TARGET.from_value(reduced_target)
+
+        with cwd_context(url.file_path()):
+            try:
+                reduced_info_dict = h5todict(
+                    h5file=url.file_path(),
+                    path=url.data_path(),
+                )
+            except Exception as e:
+                logger.error(e)
+                return None
+            else:
+                if check_for_reduced_key and reduced_target.value in reduced_info_dict:
+                    return reduced_info_dict[reduced_target.value]
+                else:
+                    return reduced_info_dict
+
     @staticmethod
     def save_flats_to_be_copied(save_dir, data: Union[DataUrl, dict]):
         if isinstance(data, DataUrl):
-            data = get_data(data)
+            data = DarkRefsCopy.get_reduced_frame_data(url=data, reduced_target="flats")
+        if data is None:
+            return
         flat_url = DarkRefsCopy.get_flats_url(save_dir)
         dicttoh5(
             data,
@@ -143,7 +175,9 @@ class DarkRefsCopy(DarkRefs):
     @staticmethod
     def save_darks_to_be_copied(save_dir, data: Union[DataUrl, dict]):
         if isinstance(data, DataUrl):
-            data = get_data(data)
+            data = DarkRefsCopy.get_reduced_frame_data(url=data, reduced_target="darks")
+        if data is None:
+            return
         dark_url = DarkRefsCopy.get_darks_url(save_dir)
         dicttoh5(
             data,
@@ -216,7 +250,7 @@ class DarkRefsCopy(DarkRefs):
         * if the scan has no ref files and refCopy has some register. Will
           create a copy of those, normalized from srCurrent (for flat field)
         """
-        scan = self.inputs.data
+        scan = data_identifier_to_scan(self.inputs.data)
         if not isinstance(scan, (type(None), TomwerScanBase, dict)):
             raise TypeError(
                 f"self.inputs.data is expected to be an instance "
@@ -322,9 +356,3 @@ class DarkRefsCopy(DarkRefs):
 
     def set_darkHST_prefix(self, prefix):
         self._darkHstPrefix = prefix
-
-    def clear_ref(self):
-        if self._flat_save_file is not None:
-            os.remove(self._flat_save_file)
-        if self._dark_save_file is not None:
-            os.remove(self._dark_save_file)

tomwer/core/process/reconstruction/darkref/params.py

@@ -36,6 +36,7 @@ from tomwer.core.process.reconstruction.darkref.settings import (
 )
 from tomwer.core.process.reconstruction.paramsbase import _ReconsParam
 from tomwer.core.utils.deprecation import deprecated
+from tomoscan.framereducerbase import ReduceMethod
 
 
 # TODO: those classes (when, method) should be linked / embedded in the DarkRef
@@ -47,21 +48,19 @@ class When(_Enum):
     after = (2,)
 
 
-@enum.unique
-class Method(_Enum):
-    none = (0,)
-    average = (1,)
-    median = 2
-    first = 10
-    last = 11
-
-    @classmethod
-    def from_value(cls, value):
-        if isinstance(value, str):
-            for m in Method:
-                if value.lower() == m.name.lower():
-                    return m
-        return super().from_value(value)
+def _from_value_reduce_method_backward_compatibility(value) -> ReduceMethod:
+    if value in ("none", 0, (0,)):
+        return ReduceMethod.NONE
+    elif value in ("average", 1, (1,)):
+        return ReduceMethod.MEAN
+    elif value in ("median", 2, (2,)):
+        return ReduceMethod.MEDIAN
+    elif value in ("first", 10, (10,)):
+        return ReduceMethod.FIRST
+    elif value in ("last", 11, (11,)):
+        return ReduceMethod.FIRST
+    else:
+        return ReduceMethod.from_value(value)
 
 
 class DKRFRP(_ReconsParam):
@@ -70,14 +69,14 @@ class DKRFRP(_ReconsParam):
     def __init__(self):
         _ReconsParam.__init__(self)
         self.__do_when = When.before
-        self.__dark_calc = Method.average
+        self.__dark_calc = ReduceMethod.MEAN
         self.__overwrite_dark = False
         self.__remove_dark = False
         self.__dark_pattern = "darkend[0-9]{3,4}"
-        self.__ref_calc = Method.median
+        self.__flat_calc = ReduceMethod.MEDIAN
         self.__overwrite_flat = False
-        self.__remove_ref = False
-        self.__ref_pattern = "ref*.*[0-9]{3,4}_[0-9]{3,4}"
+        self.__remove_flats = False
+        self.__flat_pattern = "ref*.*[0-9]{3,4}_[0-9]{3,4}"
         self.__dark_prefix = DARKHST_PREFIX
         self.__ref_prefix = REFHST_PREFIX
 
@@ -87,10 +86,10 @@ class DKRFRP(_ReconsParam):
             "DARKOVE": self.__class__.overwrite_dark,
             "DARKRMV": self.__class__.remove_dark,
             "DKFILE": self.__class__.dark_pattern,
-            "REFSCAL": self.__class__.ref_calc_method,
+            "REFSCAL": self.__class__.flat_calc_method,
             "REFSOVE": self.__class__.overwrite_flat,
-            "REFSRMV": self.__class__.remove_ref,
-            "RFFILE": self.__class__.ref_pattern,
+            "REFSRMV": self.__class__.remove_raw_flats,
+            "RFFILE": self.__class__.flat_pattern,
         }
 
     @property
@@ -114,11 +113,8 @@ class DKRFRP(_ReconsParam):
 
     @dark_calc_method.setter
     def dark_calc_method(self, method):
-        assert isinstance(method, (int, Method, str))
-        if isinstance(method, str):
-            _dark_calc = getattr(Method, method.lower())
-        else:
-            _dark_calc = Method(method)
+        assert isinstance(method, (int, ReduceMethod, str))
+        _dark_calc = _from_value_reduce_method_backward_compatibility(method)
 
         if self.__dark_calc != _dark_calc:
             self.__dark_calc = _dark_calc
@@ -163,19 +159,16 @@ class DKRFRP(_ReconsParam):
             self.changed()
 
     @property
-    def ref_calc_method(self):
+    def flat_calc_method(self):
         """Dark calculation method (None, Average, Median)"""
-        return self.__ref_calc
-
-    @ref_calc_method.setter
-    def ref_calc_method(self, method):
-        assert isinstance(method, (int, Method, str))
-        if isinstance(method, str):
-            _ref_calc = getattr(Method, method.lower())
-        else:
-            _ref_calc = Method(method)
-        if self.__ref_calc != _ref_calc:
-            self.__ref_calc = _ref_calc
+        return self.__flat_calc
+
+    @flat_calc_method.setter
+    def flat_calc_method(self, method):
+        assert isinstance(method, (int, ReduceMethod, str))
+        _ref_calc = _from_value_reduce_method_backward_compatibility(method)
+        if self.__flat_calc != _ref_calc:
+            self.__flat_calc = _ref_calc
             self.changed()
 
     @property
@@ -202,42 +195,44 @@ class DKRFRP(_ReconsParam):
             self.changed()
 
     @property
-    def remove_ref(self):
-        """Remove original ref files when done"""
-        return self.__remove_ref
+    def remove_raw_flats(self):
+        """Remove original flat / ref files when done (for EDF only)"""
+        return self.__remove_flats
 
-    @remove_ref.setter
-    def remove_ref(self, remove):
+    @remove_raw_flats.setter
+    def remove_raw_flats(self, remove):
         # TODO: float should be removed, but this is a legacy from fastomo3
         assert isinstance(remove, (int, bool, float))
-        _remove_ref = remove
-        if self.__remove_ref != _remove_ref:
-            self.__remove_ref = _remove_ref
+        _remove_flats = remove
+        if self.__remove_flats != _remove_flats:
+            self.__remove_flats = _remove_flats
             self.changed()
 
     @property
-    def ref_pattern(self):
-        """File pattern to detect references"""
-        return self.__ref_pattern
-
-    @ref_pattern.setter
-    def ref_pattern(self, pattern):
-        if pattern != self.__ref_pattern:
-            self.__ref_pattern = pattern
+    def flat_pattern(self):
+        """File pattern to detect EDF flats files"""
+        return self.__flat_pattern
+
+    @flat_pattern.setter
+    def flat_pattern(self, pattern):
+        if pattern != self.__flat_pattern:
+            self.__flat_pattern = pattern
             self.changed()
 
     @property
-    def ref_prefix(self):
+    def flat_prefix(self):
+        """flat prefix for EDF"""
         return self.__ref_prefix
 
-    @ref_prefix.setter
-    def ref_prefix(self, prefix):
+    @flat_prefix.setter
+    def flat_prefix(self, prefix):
         if prefix != self.__ref_prefix:
             self.__ref_prefix = prefix
             self.changed()
 
     @property
     def dark_prefix(self):
+        """dark prefix for EDF"""
         return self.__dark_prefix
 
     @dark_prefix.setter
@@ -247,7 +242,7 @@ class DKRFRP(_ReconsParam):
             self.changed()
 
     def _set_remove_opt(self, rm):
-        self.remove_ref = rm
+        self.remove_raw_flats = rm
         self.remove_dark = rm
 
     def _set_skip_if_exist(self, skip):
@@ -257,14 +252,14 @@ class DKRFRP(_ReconsParam):
     def to_dict(self):
         _dict = {
             "DOWHEN": self.do_when.name,
-            "DARKCAL": self.dark_calc_method.name.split(".")[-1].title(),
+            "DARKCAL": self.dark_calc_method.value,
             "DARKOVE": int(self.overwrite_dark),
             "DARKRMV": int(self.remove_dark),
             "DKFILE": self.dark_pattern,
-            "REFSCAL": self.ref_calc_method.name.split(".")[-1].title(),
+            "REFSCAL": self.flat_calc_method.value,
             "REFSOVE": int(self.overwrite_flat),
-            "REFSRMV": int(self.remove_ref),
-            "RFFILE": self.ref_pattern,
+            "REFSRMV": int(self.remove_raw_flats),
+            "RFFILE": self.flat_pattern,
         }
         _dict.update(self.unmanaged_params)
         return _dict
@@ -278,11 +273,11 @@ class DKRFRP(_ReconsParam):
     def load_from_dict(self, _dict):
         self._load_unmanaged_params(_dict=_dict)
         self.do_when = getattr(When, _dict["DOWHEN"])
-        self.dark_calc_method = getattr(Method, _dict["DARKCAL"].lower())
+        self.dark_calc_method = _dict["DARKCAL"]
         self.overwrite_dark = _dict["DARKOVE"]
         self.remove_dark = _dict["DARKRMV"]
         self.dark_pattern = _dict["DKFILE"]
-        self.ref_calc_method = getattr(Method, _dict["REFSCAL"].lower())
+        self.flat_calc_method = _dict["REFSCAL"]
         self.overwrite_flat = _dict["REFSOVE"]
-        self.remove_ref = _dict["REFSRMV"]
-        self.ref_pattern = _dict["RFFILE"]
+        self.remove_raw_flats = _dict["REFSRMV"]
+        self.flat_pattern = _dict["RFFILE"]

tomwer/core/process/reconstruction/lamino/tofu.py

@@ -813,7 +813,6 @@ class LaminoReconstruction(Task, input_names=("data",), output_names=("data",)):
                     blend=blend,
                     adjust_mean=adjust_mean,
                 ):
-
                     print("------------------------")
                     print("run stitching")
                     image_width = getDim1Dim2(scan.path)[0]
@@ -883,7 +882,7 @@ class LaminoReconstruction(Task, input_names=("data",), output_names=("data",)):
                 )
 
         if res is False:
-            logger.error("Reconstruction of", _scan.path, "failed")
+            logger.error(f"Reconstruction of {_scan.path} failed")
         if self._return_dict:
             self.outputs.data = _scan.to_dict()
         else:

tomwer/core/process/reconstruction/nabu/castvolume.py

@@ -32,7 +32,6 @@ __date__ = "15/12/2021"
 
 
 import os
-import inspect
 from processview.core.superviseprocess import SuperviseProcess
 from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.process.reconstruction.nabu import settings
@@ -54,6 +53,7 @@ from tomwer.core.futureobject import FutureTomwerObject
 from tomwer.core.utils.slurm import get_slurm_script_name
 from tomwer.core.volume.volumefactory import VolumeFactory
 from tomwer.core.volume.volumebase import TomwerVolumeBase
+from tomwer.core.utils.volumeutils import volume_identifier_to_volume
 from sluurp.job import SBatchScriptJob
 from sluurp.executor import submit as submit_to_slurm_cluster
 
@@ -97,7 +97,7 @@ class CastVolumeTask(
         self._cluster_config = None
 
     def run(self):
-        input_volume = self.inputs.volume
+        input_volume = volume_identifier_to_volume(self.inputs.volume)
         if not isinstance(input_volume, VolumeBase):
             raise TypeError(
                 f"input_volume is a {type(input_volume)} when {VolumeBase} expected"
@@ -165,36 +165,27 @@ class CastVolumeTask(
 
         # run volume cast locally
         if self._cluster_config is None:
-            cast_params = {
-                "input_volume": input_volume,
-                "output_volume": output_volume,
-                "output_data_type": output_data_type,
-                "data_min": data_min,
-                "data_max": data_max,
-                "scan": scan,
-                "rescale_min_percentile": rescale_min_percentile,
-                "rescale_max_percentile": rescale_max_percentile,
-            }
-            if "overwrite" in inspect.signature(_nabu_cast_volume).parameters:
-                cast_params["overwrite"] = overwrite
             try:
-                volume = _nabu_cast_volume(**cast_params)
+                _nabu_cast_volume(
+                    input_volume=input_volume,
+                    output_volume=output_volume,
+                    output_data_type=output_data_type,
+                    data_min=data_min,
+                    data_max=data_max,
+                    scan=scan,
+                    rescale_min_percentile=rescale_min_percentile,
+                    rescale_max_percentile=rescale_max_percentile,
+                    save=True,
+                    store=False,
+                )
             except Exception as e:
                 mess = f"volume cast of {str(input_volume)} failed. Reason is {str(e)}"
                 _logger.processFailed(mess)
                 state = DatasetState.FAILED
-
             else:
-                try:
-                    volume.save()
-                except Exception as e:
-                    mess = f"fail to save cast volume {str(output_volume)}. Reason is {str(e)}"
-                    _logger.processFailed(mess)
-                    state = DatasetState.FAILED
-                else:
-                    mess = f"volume cast of {str(input_volume)} succeed"
-                    _logger.processSucceed(mess)
-                    state = DatasetState.SUCCEED
+                mess = f"volume cast of {str(input_volume)} succeed"
+                _logger.processSucceed(mess)
+                state = DatasetState.SUCCEED
 
             ProcessManager().notify_dataset_state(
                 dataset=input_volume,
@@ -204,6 +195,10 @@ class CastVolumeTask(
             )
             self.outputs.future_tomo_obj = None
             self.outputs.volume = output_volume
+            if scan is not None:
+                scan.cast_volume = output_volume.get_identifier()
+            else:
+                input_volume.cast_volume = output_volume.get_identifier()
         # run volume cast remotly
         else:
 

tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -42,6 +42,7 @@ from tomwer.core.process.reconstruction.nabu.utils import (
     _NabuPhaseMethod,
     nabu_std_err_has_error,
 )
+from tomwer.core.utils.scanutils import format_output_location
 from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.hdf5scan import HDF5TomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
@@ -65,13 +66,20 @@ import numpy
 
 _logger = logging.getLogger(__name__)
 try:
-    from nabu.pipeline.fullfield.local_reconstruction import (  # noqa F401
-        ChunkedReconstructor,
+    from nabu.pipeline.fullfield.reconstruction import (  # noqa F401
+        FullFieldReconstructor,
     )
 except (ImportError, OSError) as e:
-    # import of cufft library can bring an OSError if cuda not install
-    _logger.error(e)
-    has_nabu = False
+    try:
+        from nabu.pipeline.fullfield.local_reconstruction import (  # noqa F401
+            ChunkedReconstructor,
+        )
+    except (ImportError, OSError):
+        # import of cufft library can bring an OSError if cuda not install
+        _logger.error(e)
+        has_nabu = False
+    else:
+        has_nabu = True
 else:
     has_nabu = True
 
@@ -445,28 +453,45 @@ class _NabuBaseReconstructor:
         """
         # handle phase
         pag = False
+        ctf = False
         db = None
         if "phase" in config:
-            if (
+            pag = (
                 "method" in config["phase"]
                 and config["phase"]["method"] == _NabuPhaseMethod.PAGANIN.value
-            ):
-                pag = True
+            )
+            ctf = (
+                "method" in config["phase"]
+                and config["phase"]["method"] == _NabuPhaseMethod.CTF.value
+            )
+            if pag or ctf:
                 if "delta_beta" in config["phase"]:
+                    if not numpy.isscalar(config["phase"]["delta_beta"]):
+                        if len(config["phase"]["delta_beta"]) > 1:
+                            raise ValueError(
+                                "expects at most one value for 'delta_beta'"
+                            )
+                        else:
+                            config["phase"]["delta_beta"] = config["phase"][
+                                "delta_beta"
+                            ][0]
                     db = round(float(config["phase"]["delta_beta"]))
         # handle output
         if "output" in config:
-            _file_name = self._get_file_basename_reconstruction(pag=pag, db=db)
+            _file_name = self._get_file_basename_reconstruction(pag=pag, db=db, ctf=ctf)
             config["output"]["file_prefix"] = _file_name
             location = config["output"].get("location", None)
             if location not in ("", None):
-                location = self.format_output_location(location, scan=self.scan)
+                location = format_output_location(location, scan=self.scan)
                 # if user specify the location
                 if not os.path.isdir(config["output"]["location"]):
                     os.makedirs(location)
             else:
                 # otherwise default location will be the data root level
                 location = self.scan.path
+            # add reconstruction path to the list. scan `reconstruction_paths` register all the existing path where
+            # reconstruction are saved in order to be able to browse them all
+            self.scan.add_reconstruction_path(location)
             config["output"]["location"] = location
         # handle preproc
         if "preproc" not in config:
@@ -545,37 +570,10 @@ class _NabuBaseReconstructor:
 
         return config
 
-    def _get_file_basename_reconstruction(self, pag, db):
+    def _get_file_basename_reconstruction(self, pag, db, ctf):
         """return created file base name"""
         raise NotImplementedError("Base class")
 
-    @staticmethod
-    def format_output_location(location, scan: TomwerScanBase):
-        """
-        format possible keys from the location like {scan_dir} or {scan_path}
-
-        :param location:
-        :param scan:
-        :return:
-        """
-        if scan is None:
-            _logger.warning("scan is !none, enable to format the nabu output location")
-
-        keywords = {
-            "scan_dir_name": scan.scan_dir_name(),
-            "scan_basename": scan.scan_basename(),
-            "scan_parent_dir_basename": scan.scan_parent_dir_basename(),
-        }
-        for keyword, value in keywords.items():
-            if value is None:
-                continue
-            try:
-                location = location.format(**{keyword: value})
-            except KeyError:
-                # then this mean scan_dir has not been provided
-                pass
-        return location
-
 
 def dump_normalization_array_for_nabu(
     scan: TomwerScanBase, output_file: str, array: Union[numpy.ndarray, float, int]

tomwer/core/process/reconstruction/nabu/nabuscores.py

@@ -31,14 +31,20 @@ __authors__ = [
 __license__ = "MIT"
 __date__ = "28/10/2021"
 
-
 try:
-    from nabu.pipeline.fullfield.local_reconstruction import (  # noqa F401
-        ChunkedReconstructor,
+    from nabu.pipeline.fullfield.reconstruction import (  # noqa F401
+        FullFieldReconstructor,
     )
 except (ImportError, OSError):
-    # import of cufft library can bring an OSError if cuda not install
-    has_nabu = False
+    try:
+        from nabu.pipeline.fullfield.local_reconstruction import (  # noqa F401
+            ChunkedReconstructor,
+        )
+    except (ImportError, OSError):
+        # import of cufft library can bring an OSError if cuda not install
+        has_nabu = False
+    else:
+        has_nabu = True
 else:
     has_nabu = True
 from typing import Iterable, Optional, Union
@@ -229,18 +235,24 @@ class _Reconstructor(_NabuBaseReconstructor):
         - create the output directory if does not exist
         """
         pag = False
+        ctf = False
         db = None
         if "phase" in _config:
-            if "method" in _config["phase"] and _config["phase"]["method"] not in (
-                "",
-                utils._NabuPhaseMethod.NONE.value,
-            ):
+            phase_method = _config["phase"].get("method", "").lower()
+            if phase_method in ("pag", "paganin"):
                 pag = True
-                if "delta_beta" in _config["phase"]:
-                    db = round(float(_config["phase"]["delta_beta"]))
+            elif phase_method in ("ctf",):
+                ctf = True
+
+            if "delta_beta" in _config["phase"]:
+                db = round(float(_config["phase"]["delta_beta"]))
         if "output" in _config:
             _file_name = SingleSliceRunner.get_file_basename_reconstruction(
-                scan=self.scan, slice_index=self.slice_index, pag=pag, db=db
+                scan=self.scan,
+                slice_index=self.slice_index,
+                pag=pag,
+                db=db,
+                ctf=ctf,
             )
             _config["output"]["file_prefix"] = "cor_{}_{}".format(_file_name, cor)
             if _config["output"]["location"] not in ("", None):
@@ -250,7 +262,10 @@ class _Reconstructor(_NabuBaseReconstructor):
             else:
                 # otherwise default location will be the data root level
                 _config["output"]["location"] = os.sep.join(
-                    self.scan.path, "saaxis_results"
+                    [
+                        self.scan.path,
+                        "saaxis_results",
+                    ]
                 )
         if "reconstruction" not in _config:
             _config["reconstruction"] = {}