nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl

nabu/resources/dataset_analyzer.py

@@ -1,13 +1,14 @@
 import os
-from bisect import bisect_left
 import numpy as np
-from silx.io import get_data
 from silx.io.url import DataUrl
+from silx.io import get_data
 from tomoscan.esrf.scan.edfscan import EDFTomoScan
 from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
-from ..utils import check_supported
+
+from ..utils import check_supported, indices_to_slices
+from ..io.reader import EDFStackReader, NXDarksFlats, NXTomoReader
 from ..io.utils import get_compacted_dataslices
-from .utils import is_hdf5_extension, get_values_from_file
+from .utils import get_values_from_file, is_hdf5_extension
 from .logger import LoggerOrPrint
 
 from ..pipeline.utils import nabu_env_settings
@@ -51,7 +52,7 @@ class DatasetAnalyzer:
             "output_dir": None,
             "exclude_projections": None,
             "hdf5_entry": None,
-            "nx_version": 1.0,
+            # "nx_version": 1.0,
         }
         # --
         advanced_options.update(extra_options)
@@ -59,16 +60,18 @@ class DatasetAnalyzer:
 
     # pylint: disable=E1136
     def _get_excluded_projections(self):
-        self._ignore_projections_indices = None
-        self._need_rebuild_tomoscan_object_to_exclude_projections = False
         excluded_projs = self.extra_options["exclude_projections"]
+        self._ignore_projections = None
         if excluded_projs is None:
             return
-        if excluded_projs["type"] == "indices":
-            projs_idx = get_values_from_file(excluded_projs["file"], any_size=True).astype(np.int32).tolist()
-            self._ignore_projections_indices = projs_idx
-        else:
-            self._need_rebuild_tomoscan_object_to_exclude_projections = True
+
+        if excluded_projs["type"] == "angular_range":
+            excluded_projs["type"] = "range"  # compat with tomoscan #pylint: disable=E1137
+            values = excluded_projs["range"]
+        for ignore_kind, dtype in {"indices": np.int32, "angles": np.float32}.items():
+            if excluded_projs["type"] == ignore_kind:
+                values = get_values_from_file(excluded_projs["file"], any_size=True).astype(dtype).tolist()
+        self._ignore_projections = {"kind": excluded_projs["type"], "values": values}  # pylint: disable=E0606
 
     def _init_dataset_scan(self, **kwargs):
         if self._scanner is None:
@@ -83,40 +86,11 @@ class DatasetAnalyzer:
             kwargs["n_frames"] = 1
 
         self.dataset_scanner = self._scanner(  # pylint: disable=E1102
-            self.location, ignore_projections=self._ignore_projections_indices, **kwargs
+            self.location, ignore_projections=self._ignore_projections, **kwargs
         )
-        self.projections = self.dataset_scanner.projections
-
-        # ---
-        if self._need_rebuild_tomoscan_object_to_exclude_projections:
-            # pylint: disable=E1136
-            exclude_projs = self.extra_options["exclude_projections"]
-            rot_angles_deg = np.rad2deg(self.rotation_angles)
-            self._rotation_angles = None  # prevent caching
-            # tomoscan only supports ignore_projections=<list of integers>
-            # However this is cumbersome to use, it's more convenient to use angular range or list of angles
-            # But having angles instead of indices implies to already have information on current scan angular range
-            ignore_projections_indices = []
-            if exclude_projs["type"] == "angular_range":
-                exclude_angle_min, exclude_angle_max = exclude_projs["range"]
-                projections_indices = np.array(sorted(self.dataset_scanner.projections.keys()))
-                for proj_idx, angle in zip(projections_indices, rot_angles_deg):
-                    if exclude_angle_min <= angle and angle <= exclude_angle_max:
-                        ignore_projections_indices.append(proj_idx)
-            elif exclude_projs["type"] == "angles":
-                excluded_angles = get_values_from_file(exclude_projs["file"], any_size=True).astype(np.float32).tolist()
-                for excluded_angle in excluded_angles:
-                    proj_idx = bisect_left(rot_angles_deg, excluded_angle)
-                    if proj_idx < rot_angles_deg.size:
-                        ignore_projections_indices.append(proj_idx)
-            # Rebuild the dataset_scanner instance
-            self._ignore_projections_indices = ignore_projections_indices
-            self.dataset_scanner = self._scanner(  # pylint: disable=E1102
-                self.location, ignore_projections=self._ignore_projections_indices, **kwargs
-            )
-        # ---
-        if self._ignore_projections_indices is not None:
-            self.logger.info("Excluding projections: %s" % str(self._ignore_projections_indices))
+
+        if self._ignore_projections is not None:
+            self.logger.info("Excluding projections: %s" % str(self._ignore_projections))
 
         if nabu_env_settings.skip_tomoscan_checks:
             self.logger.warning(
@@ -124,9 +98,8 @@ class DatasetAnalyzer:
             )
             self.dataset_scanner.set_check_behavior(run_check=False, raise_error=False)
 
-        self.projections = self.dataset_scanner.projections
-        self.flats = self.dataset_scanner.flats
-        self.darks = self.dataset_scanner.darks
+        self.raw_flats = self.dataset_scanner.flats
+        self.raw_darks = self.dataset_scanner.darks
         self.n_angles = len(self.dataset_scanner.projections)
         self.radio_dims = (self.dataset_scanner.dim_1, self.dataset_scanner.dim_2)
         self._radio_dims_notbinned = self.radio_dims  # COMPAT
@@ -146,7 +119,10 @@ class DatasetAnalyzer:
         self._pixel_size = None
         self._distance = None
         self._flats_srcurrent = None
+        self._projections = None
         self._projections_srcurrent = None
+        self._reduced_flats = None
+        self._reduced_darks = None
 
     @property
     def energy(self):
@@ -223,15 +199,19 @@ class DatasetAnalyzer:
     def detector_tilt(self, tilt):
         self._detector_tilt = tilt
 
-    def _get_srcurrent(self, indices):
-        srcurrent = self.dataset_scanner.electric_current
-        if srcurrent is None or len(srcurrent) == 0:
-            return None
-        srcurrent_all = np.array(srcurrent)
-        if np.any(indices >= len(srcurrent_all)):
-            self.logger.error("Something wrong with SRCurrent: not enough values!")
-            return None
-        return srcurrent_all[indices].astype("f")
+    def _get_srcurrent(self, frame_type):
+        # To be implemented by inheriting class
+        return None
+
+    @property
+    def projections(self):
+        if self._projections is None:
+            self._projections = self.dataset_scanner.projections
+        return self._projections
+
+    @projections.setter
+    def projections(self, val):
+        raise ValueError
 
     @property
     def projections_srcurrent(self):
@@ -239,8 +219,7 @@ class DatasetAnalyzer:
         Return the synchrotron electric current for each projection.
         """
         if self._projections_srcurrent is None:
-            projections_indices = np.array(sorted(self.projections.keys()))
-            self._projections_srcurrent = self._get_srcurrent(projections_indices)
+            self._projections_srcurrent = self._get_srcurrent("radios")  # pylint: disable=E1128
         return self._projections_srcurrent
 
     @projections_srcurrent.setter
@@ -253,8 +232,7 @@ class DatasetAnalyzer:
         Return the synchrotron electric current for each flat image.
         """
         if self._flats_srcurrent is None:
-            flats_indices = np.array(sorted(self.flats.keys()))
-            self._flats_srcurrent = self._get_srcurrent(flats_indices)
+            self._flats_srcurrent = self._get_srcurrent("flats")  # pylint: disable=E1128
         return self._flats_srcurrent
 
     @flats_srcurrent.setter
@@ -268,6 +246,32 @@ class DatasetAnalyzer:
         if getattr(self, name, None) is None:
             raise ValueError(error_msg or str("No information on %s was found in the dataset" % name))
 
+    @property
+    def flats(self):
+        """
+        Return the REDUCED flat-field images. Either by reducing (median) the raw flats, or a user-defined reduced flats.
+        """
+        if self._reduced_flats is None:
+            self._reduced_flats = self.get_reduced_flats()
+        return self._reduced_flats
+
+    @flats.setter
+    def flats(self, val):
+        self._reduced_flats = val
+
+    @property
+    def darks(self):
+        """
+        Return the REDUCED flat-field images. Either by reducing (mean) the raw darks, or a user-defined reduced darks.
+        """
+        if self._reduced_darks is None:
+            self._reduced_darks = self.get_reduced_darks()
+        return self._reduced_darks
+
+    @darks.setter
+    def darks(self, val):
+        self._reduced_darks = val
+
 
 class EDFDatasetAnalyzer(DatasetAnalyzer):
     """
@@ -278,23 +282,7 @@ class EDFDatasetAnalyzer(DatasetAnalyzer):
     kind = "edf"
 
     def _finish_init(self):
-        self.remove_unused_radios()
-
-    def remove_unused_radios(self):
-        """
-        Remove "unused" radios.
-        This is used for legacy ESRF scans.
-        """
-        # Extraneous projections are assumed to be on the end
-        projs_indices = sorted(self.projections.keys())
-        used_radios_range = range(projs_indices[0], len(self.projections))
-        radios_not_used = []
-        for idx in self.projections.keys():
-            if idx not in used_radios_range:
-                radios_not_used.append(idx)
-        for idx in radios_not_used:
-            self.projections.pop(idx)
-        return radios_not_used
+        pass
 
     def _get_flats_darks(self):
         return
@@ -311,13 +299,34 @@ class EDFDatasetAnalyzer(DatasetAnalyzer):
         return None
 
     def _get_rotation_angles(self):
-        if self._rotation_angles is None:
-            scan_range = self.dataset_scanner.scan_range
-            if scan_range is not None:
-                fullturn = abs(scan_range - 360) < abs(scan_range - 180)
-                angles = np.linspace(0, scan_range, num=len(self.projections), endpoint=fullturn, dtype="f")
-                self._rotation_angles = np.deg2rad(angles)
-        return self._rotation_angles
+        return np.deg2rad(self.dataset_scanner.rotation_angle())
+
+    def get_reduced_flats(self, **reader_kwargs):
+        if self.raw_flats in [None, {}]:
+            raise FileNotFoundError("No reduced flat ('refHST') found in %s" % self.location)
+        # A few notes:
+        # (1) In principle we could do the reduction (mean/median) from raw frames (ref_xxxx_yyyy)
+        #   but for legacy datasets it's always already done (by fasttomo3), and EDF support is supposed to be dropped on our side
+        # (2) We use EDFStackReader class to handle the possible additional data modifications
+        #  (eg. subsampling, binning, distortion correction...)
+        # (3) The following spawns one reader instance per file, which is not elegant,
+        #   but in principle there are typically 1-2 reduced flats in a scan
+        readers = {k: EDFStackReader([self.raw_flats[k].file_path()], **reader_kwargs) for k in self.raw_flats.keys()}
+        return {k: readers[k].load_data()[0] for k in self.raw_flats.keys()}
+
+    def get_reduced_darks(self, **reader_kwargs):
+        # See notes in get_reduced_flats() above
+        if self.raw_darks in [None, {}]:
+            raise FileNotFoundError("No reduced dark ('darkend.edf' or 'dark.edf') found in %s" % self.location)
+        readers = {k: EDFStackReader([self.raw_darks[k].file_path()], **reader_kwargs) for k in self.raw_darks.keys()}
+        return {k: readers[k].load_data()[0] for k in self.raw_darks.keys()}
+
+    @property
+    def files(self):
+        return sorted([u.file_path() for u in self.dataset_scanner.projections.values()])
+
+    def get_reader(self, **kwargs):
+        return EDFStackReader(self.files, **kwargs)
 
 
 class HDF5DatasetAnalyzer(DatasetAnalyzer):
@@ -326,7 +335,10 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
     """
 
     _scanner = NXtomoScan
-    kind = "hdf5"
+    kind = "nx"
+    # We could import the 1000+ LoC  nxtomo.nxobject.nxdetector.ImageKey... or we can do this
+    _image_key_value = {"flats": 1, "darks": 2, "radios": 0}
+    #
 
     @property
     def z_translation(self):
@@ -353,10 +365,10 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
     def _get_dataset_hdf5_url(self):
         if len(self.projections) > 0:
             frames_to_take = self.projections
-        elif len(self.flats) > 0:
-            frames_to_take = self.flats
-        elif len(self.darks) > 0:
-            frames_to_take = self.darks
+        elif len(self.raw_flats) > 0:
+            frames_to_take = self.raw_flats
+        elif len(self.raw_darks) > 0:
+            frames_to_take = self.raw_darks
         else:
             raise ValueError("No projections, no flats and no darks ?!")
         first_proj_idx = sorted(frames_to_take.keys())[0]
@@ -397,8 +409,13 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
         slices: list of slice
             A list where each item is a slice.
         """
-        check_supported(what, ["projections", "flats", "darks"], "image type")
-        images = getattr(self, what)  # dict
+        name_to_attr = {
+            "projections": self.projections,
+            "flats": self.raw_flats,
+            "darks": self.raw_darks,
+        }
+        check_supported(what, name_to_attr.keys(), "image type")
+        images = name_to_attr[what]  # dict
         # we can't directly use set() on slice() object (unhashable). Use tuples
         slices = set()
         for du in get_compacted_dataslices(images).values():
@@ -410,6 +427,39 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
         slices_list = [slice(item[0], item[1]) if item is not None else None for item in list(slices)]
         return slices_list
 
+    def _select_according_to_frame_type(self, data, frame_type):
+        if data is None:
+            return None
+        return data[self.dataset_scanner.image_key_control == self._image_key_value[frame_type]]
+
+    def get_reduced_flats(self, method="median", force_reload=False, **reader_kwargs):
+        dkrf_reader = NXDarksFlats(
+            self.dataset_hdf5_url.file_path(), data_path=self.dataset_hdf5_url.data_path(), **reader_kwargs
+        )
+        return dkrf_reader.get_reduced_flats(method=method, force_reload=force_reload, as_dict=True)
+
+    def get_reduced_darks(self, method="mean", force_reload=False, **reader_kwargs):
+        dkrf_reader = NXDarksFlats(
+            self.dataset_hdf5_url.file_path(), data_path=self.dataset_hdf5_url.data_path(), **reader_kwargs
+        )
+        return dkrf_reader.get_reduced_darks(method=method, force_reload=force_reload, as_dict=True)
+
+    def _get_srcurrent(self, frame_type):
+        return self._select_according_to_frame_type(self.dataset_scanner.electric_current, frame_type)
+
+    def frames_slices(self, frame_type):
+        """
+        Return a list of slice objects corresponding to the data corresponding to "frame_type".
+        For example, if the dataset flats are located at indices [1, 2, ..., 99], then
+        frame_slices("flats") will return [slice(0, 100)].
+        """
+        return indices_to_slices(
+            np.where(self.dataset_scanner.image_key_control == self._image_key_value[frame_type])[0]
+        )
+
+    def get_reader(self, **kwargs):
+        return NXTomoReader(self.dataset_hdf5_url.file_path(), data_path=self.dataset_hdf5_url.data_path(), **kwargs)
+
 
 def analyze_dataset(dataset_path, extra_options=None, logger=None):
     if not (os.path.isdir(dataset_path)):

nabu/resources/gpu.py

@@ -1,6 +1,7 @@
 """
 gpu.py: general-purpose utilities for GPU
 """
+
 from ..utils import check_supported
 
 try:

nabu/resources/nxflatfield.py

@@ -1,14 +1,16 @@
 import os
 import numpy as np
+from nxtomo.io import HDF5File
 from silx.io.url import DataUrl
-from tomoscan.io import HDF5File
+from silx.io import get_data
+from tomoscan.framereducer.reducedframesinfos import ReducedFramesInfos
 from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
 from ..utils import check_supported, is_writeable
 
 
-def get_frame_possible_urls(dataset_info, user_dir, output_dir, frame_type):
+def get_frame_possible_urls(dataset_info, user_dir, output_dir):
     """
-    Return a list with the possible location of reduced dark/flat frames.
+    Return a dict with the possible location of reduced dark/flat frames.
 
     Parameters
     ----------
@@ -18,19 +20,21 @@ def get_frame_possible_urls(dataset_info, user_dir, output_dir, frame_type):
         User-provided directory location for the reduced frames.
     output_dir: str or None
         Output processing directory
-    frame_type: str
-        Frame type, can be "flats" or "darks".
     """
-    check_supported(frame_type, ["flats", "darks"], "frame type")
 
+    frame_types = ["flats", "darks"]
     h5scan = dataset_info.dataset_scanner  # tomoscan object
-    if frame_type == "flats":
-        dataurl_default_template = h5scan.REDUCED_FLATS_DATAURLS[0]
-    else:
-        dataurl_default_template = h5scan.REDUCED_DARKS_DATAURLS[0]
 
-    def make_dataurl(dirname):
-        # The template formatting should be done by tomoscan in principle, but this complicates logging.
+    def make_dataurl(dirname, frame_type):
+        """
+        The template formatting should be done by tomoscan in principle, but this complicates logging.
+        """
+
+        if frame_type == "flats":
+            dataurl_default_template = h5scan.REDUCED_FLATS_DATAURLS[0]
+        else:
+            dataurl_default_template = h5scan.REDUCED_DARKS_DATAURLS[0]
+
         rel_file_path = dataurl_default_template.file_path().format(
             scan_prefix=dataset_info.dataset_scanner.get_dataset_basename()
         )
@@ -44,18 +48,73 @@ def get_frame_possible_urls(dataset_info, user_dir, output_dir, frame_type):
     urls = {"user": None, "dataset": None, "output": None}
 
     if user_dir is not None:
-        urls["user"] = make_dataurl(user_dir)
+        urls["user"] = {frame_type: make_dataurl(user_dir, frame_type) for frame_type in frame_types}
 
     # tomoscan.esrf.scan.hdf5scan.REDUCED_{DARKS|FLATS}_DATAURLS.file_path() is a relative path
     # Create a absolute path instead
-    urls["dataset"] = make_dataurl(os.path.dirname(h5scan.master_file))
+    urls["dataset"] = {
+        frame_type: make_dataurl(os.path.dirname(h5scan.master_file), frame_type) for frame_type in frame_types
+    }
 
     if output_dir is not None:
-        urls["output"] = make_dataurl(output_dir)
+        urls["output"] = {frame_type: make_dataurl(output_dir, frame_type) for frame_type in frame_types}
 
     return urls
 
 
+def save_reduced_frames(dataset_info, reduced_frames_arrays, reduced_frames_urls):
+    reduce_func = {"flats": np.median, "darks": np.mean}  # TODO configurable ?
+
+    # Get "where to write". tomoscan expects a DataUrl
+    darks_flats_dir_url = reduced_frames_urls.get("user", None)
+    if darks_flats_dir_url is not None:
+        output_url = darks_flats_dir_url
+    elif is_writeable(os.path.dirname(reduced_frames_urls["dataset"]["flats"].file_path())):
+        output_url = reduced_frames_urls["dataset"]
+    else:
+        output_url = reduced_frames_urls["output"]
+
+    # Get the "ReducedFrameInfos" data structure expected by tomoscan
+    def _get_additional_info(frame_type):
+        electric_current = dataset_info.dataset_scanner.electric_current
+        count_time = dataset_info.dataset_scanner.count_time
+        if electric_current is not None:
+            electric_current = {
+                sl.start: reduce_func[frame_type](electric_current[sl]) for sl in dataset_info.frames_slices(frame_type)
+            }
+            electric_current = [electric_current[k] for k in sorted(electric_current.keys())]
+        if count_time is not None:
+            count_time = {
+                sl.start: reduce_func[frame_type](count_time[sl]) for sl in dataset_info.frames_slices(frame_type)
+            }
+            count_time = [count_time[k] for k in sorted(count_time.keys())]
+        info = ReducedFramesInfos()
+        info.count_time = count_time
+        info.machine_electric_current = electric_current
+        return info
+
+    flats_info = _get_additional_info("flats")
+    darks_info = _get_additional_info("darks")
+
+    # Call tomoscan to save the reduced frames
+    dataset_info.dataset_scanner.save_reduced_darks(
+        reduced_frames_arrays["darks"],
+        output_urls=[output_url["darks"]],
+        darks_infos=darks_info,
+        metadata_output_urls=[get_metadata_url(output_url["darks"], "darks")],
+        overwrite=True,
+    )
+    dataset_info.dataset_scanner.save_reduced_flats(
+        reduced_frames_arrays["flats"],
+        output_urls=[output_url["flats"]],
+        flats_infos=flats_info,
+        metadata_output_urls=[get_metadata_url(output_url["flats"], "flats")],
+        overwrite=True,
+    )
+    dataset_info.logger.info("Saved reduced darks/flats to %s" % output_url["flats"].file_path())
+    return output_url, flats_info, darks_info
+
+
 def get_metadata_url(url, frame_type):
     """
     Return the url of the metadata stored alongside flats/darks
@@ -79,12 +138,15 @@ def tomoscan_load_reduced_frames(dataset_info, frame_type, url):
     )
 
 
-def tomoscan_save_reduced_frames(dataset_info, frame_type, url, frames, info):
-    tomoscan_method = getattr(dataset_info.dataset_scanner, "save_reduced_%s" % frame_type)
-    kwargs = {"%s_infos" % frame_type: info, "overwrite": True}
-    return tomoscan_method(
-        frames, output_urls=[url], metadata_output_urls=[get_metadata_url(url, frame_type)], **kwargs
-    )
+def data_url_exists(data_url):
+    """
+    Return true iff the file exists and the data URL is valid (i.e data/group is actually in the file)
+    """
+    if not (os.path.isfile(data_url.file_path())):
+        return False
+    with HDF5File(data_url.file_path(), "r") as f:
+        path_exists = f.get(data_url.data_path(), default=None) is not None
+    return path_exists
 
 
 # pylint: disable=E1136
@@ -107,111 +169,58 @@ def update_dataset_info_flats_darks(dataset_info, flatfield_mode, output_dir=Non
     """
     if flatfield_mode is False:
         return
-    logger = dataset_info.logger
+
     frames_types = ["darks", "flats"]
+    reduced_frames_urls = get_frame_possible_urls(dataset_info, darks_flats_dir, output_dir)
+
+    def _compute_and_save_reduced_frames():
+        try:
+            dataset_info.flats = dataset_info.get_reduced_flats()
+            dataset_info.darks = dataset_info.get_reduced_darks()
+        except FileNotFoundError:
+            msg = "Could not find any flats and/or darks"
+            raise FileNotFoundError(msg)
+        _, flats_info, darks_info = save_reduced_frames(
+            dataset_info, {"darks": dataset_info.darks, "flats": dataset_info.flats}, reduced_frames_urls
+        )
+        dataset_info.flats_srcurrent = flats_info.machine_electric_current
 
-    reduced_frames_urls = {}
-    for frame_type in frames_types:
-        reduced_frames_urls[frame_type] = get_frame_possible_urls(dataset_info, darks_flats_dir, output_dir, frame_type)
-
-    reduced_frames = dict.fromkeys(frames_types, None)
-
-    #
-    # Try to load frames
-    #
-    def load_reduced_frame(url, frame_type, frames_loaded, reduced_frames):
-        if frames_loaded[frame_type]:
-            return
-        frames, info = tomoscan_load_reduced_frames(dataset_info, frame_type, url)
-        if frames not in (None, {}):
-            dataset_info.logger.info("Loaded %s from %s" % (frame_type, url.file_path()))
-            frames_loaded[frame_type] = True
-            reduced_frames[frame_type] = frames, info
-        else:
-            msg = "Could not load %s from %s" % (frame_type, url.file_path())
-            logger.error(msg)
-
-    frames_loaded = dict.fromkeys(frames_types, False)
-    if flatfield_mode != "force-compute":
-        for load_from in ["user", "dataset", "output"]:  # in that order
-            for frame_type in frames_types:
-                url = reduced_frames_urls[frame_type][load_from]
-                if url is None:
-                    continue  # cannot load from this source (eg. undefined folder)
-                load_reduced_frame(url, frame_type, frames_loaded, reduced_frames)
-            if all(frames_loaded.values()):
-                break
-
-    if not all(frames_loaded.values()) and flatfield_mode == "force-load":
-        raise ValueError("Could not load darks/flats (using 'force-load')")
+    if flatfield_mode == "force-compute":
+        _compute_and_save_reduced_frames()
+        return
 
-    #
-    # COMPAT. Keep DataUrl - won't be needed in future versions when pipeline will use FlatField
-    # instead of FlatFieldDataUrl
-    frames_urls = reduced_frames.copy()
-    #
-
-    # Compute reduced frames, if needed
-    #
-    if reduced_frames["flats"] is None:
-        reduced_frames["flats"] = dataset_info.dataset_scanner.compute_reduced_flats(return_info=True)
-    if reduced_frames["darks"] is None:
-        reduced_frames["darks"] = dataset_info.dataset_scanner.compute_reduced_darks(return_info=True)
-
-    if reduced_frames["darks"][0] == {} or reduced_frames["flats"][0] == {}:
-        raise ValueError(
-            "Could not get any reduced flat/dark. This probably means that no already-reduced flats/darks were found and that the dataset itself does not have any flat/dark"
-        )
+    def _can_load_from(folder_type):
+        if reduced_frames_urls.get(folder_type, None) is None:
+            return False
+        return all([data_url_exists(reduced_frames_urls[folder_type][frame_type]) for frame_type in frames_types])
+
+    where_to_load_from = None
+    if reduced_frames_urls["user"] is not None and _can_load_from("user"):
+        where_to_load_from = "user"
+    elif _can_load_from("dataset"):
+        where_to_load_from = "dataset"
+    elif _can_load_from("output"):
+        where_to_load_from = "output"
 
-    #
-    # Save reduced frames
-    #
-
-    def save_reduced_frame(url, frame_type, frames_saved):
-        frames, info = reduced_frames[frame_type]
-        tomoscan_save_reduced_frames(dataset_info, frame_type, url, frames, info)
-        dataset_info.logger.info("Saved reduced %s to %s" % (frame_type, url.file_path()))
-        frames_saved[frame_type] = True
-
-    frames_saved = dict.fromkeys(frames_types, False)
-    if not all(frames_loaded.values()):
-        for save_to in ["user", "dataset", "output"]:  # in that order
-            for frame_type in frames_types:
-                if frames_loaded[frame_type]:
-                    continue  # already loaded
-                url = reduced_frames_urls[frame_type][save_to]
-                if url is None:
-                    continue  # cannot load from this source (eg. undefined folder)
-                if not is_writeable(os.path.dirname(url.file_path())):
-                    continue
-                save_reduced_frame(url, frame_type, frames_saved)
-                # COMPAT.
-                if frames_urls[frame_type] is None:
-                    frames_urls[frame_type] = tomoscan_load_reduced_frames(dataset_info, frame_type, url)
-                #
-            if all(frames_saved.values()):
-                break
-
-    dataset_info.flats = frames_urls["flats"][0]  # reduced_frames["flats"] # in future versions
-    dataset_info.flats_srcurrent = frames_urls["flats"][1].machine_electric_current
-    # This is an extra check to avoid having more than 1 (reduced) dark.
-    # FlatField only works with exactly 1 (reduced) dark (having more than 1 series of darks makes little sense)
-    # This is normally prevented by tomoscan HDF5FramesReducer, but let's add this extra check
-    darks_ = frames_urls["darks"][0]  # reduced_frames["darks"] # in future versions
-    if len(darks_) > 1:
-        dark_idx = sorted(darks_.keys())[0]
-        dataset_info.logger.error("Found more that one series of darks. Keeping only the first one")
-        darks_ = {dark_idx: darks_[dark_idx]}
-    #
-    dataset_info.darks = darks_
-    dataset_info.darks_srcurrent = frames_urls["darks"][1].machine_electric_current
-
-
-# tomoscan "compute_reduced_XX" is quite slow. If needed, here is an alternative implementation
-def my_reduce_flats(di):
-    res = {}
-    with HDF5File(di.dataset_hdf5_url.file_path(), "r") as f:
-        for data_slice in di.get_data_slices("flats"):
-            data = f[di.dataset_hdf5_url.data_path()][data_slice.start : data_slice.stop]
-            res[data_slice.start] = np.median(data, axis=0)
-    return res
+    if where_to_load_from == None and flatfield_mode == "force-load":
+        raise ValueError("Could not load darks/flats (using 'force-load')")
+
+    if where_to_load_from is not None:
+        reduced_frames_with_info = {}
+        for frame_type in frames_types:
+            reduced_frames_with_info[frame_type] = tomoscan_load_reduced_frames(
+                dataset_info, frame_type, reduced_frames_urls[where_to_load_from][frame_type]
+            )
+            dataset_info.logger.info(
+                "Loaded %s from %s" % (frame_type, reduced_frames_urls[where_to_load_from][frame_type].file_path())
+            )
+            red_frames_dict, red_frames_info = reduced_frames_with_info[frame_type]
+            setattr(
+                dataset_info,
+                frame_type,
+                {k: get_data(red_frames_dict[k]) for k in red_frames_dict.keys()},
+            )
+            if frame_type == "flats":
+                dataset_info.flats_srcurrent = red_frames_info.machine_electric_current
+    else:
+        _compute_and_save_reduced_frames()