nabu 2024.2.13__py3-none-any.whl → 2025.1.0__py3-none-any.whl

nabu/resources/dataset_analyzer.py

@@ -1,3 +1,4 @@
+from enum import Enum
 import os
 import numpy as np
 from silx.io.url import DataUrl
@@ -7,7 +8,7 @@ from tomoscan.esrf.scan.edfscan import EDFTomoScan
 from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
 from packaging.version import parse as parse_version
 
-from ..utils import check_supported, indices_to_slices
+from ..utils import BaseClassError, check_supported, indices_to_slices, is_scalar, search_sorted
 from ..io.reader import EDFStackReader, NXDarksFlats, NXTomoReader
 from ..io.utils import get_compacted_dataslices
 from .utils import get_values_from_file, is_hdf5_extension
@@ -16,6 +17,33 @@ from .logger import LoggerOrPrint
 from ..pipeline.utils import nabu_env_settings
 
 
+# We could import the 1000+ LoC  nxtomo.nxobject.nxdetector.ImageKey... or we can do this
+class ImageKey(Enum):
+    ALIGNMENT = -1
+    PROJECTION = 0
+    FLAT_FIELD = 1
+    DARK_FIELD = 2
+    INVALID = 3
+
+
+# ---
+
+_image_type = {
+    "projections": ImageKey.PROJECTION.value,
+    "projection": ImageKey.PROJECTION.value,
+    "radios": ImageKey.PROJECTION.value,
+    "radio": ImageKey.PROJECTION.value,
+    "flats": ImageKey.FLAT_FIELD.value,
+    "flat": ImageKey.FLAT_FIELD.value,
+    "darks": ImageKey.DARK_FIELD.value,
+    "dark": ImageKey.DARK_FIELD.value,
+    "static": ImageKey.ALIGNMENT.value,
+    "alignment": ImageKey.ALIGNMENT.value,
+    "return": ImageKey.ALIGNMENT.value,
+    "invalid": ImageKey.INVALID.value,
+}
+
+
 class DatasetAnalyzer:
     _scanner = None
     kind = "none"
@@ -277,6 +305,32 @@ class DatasetAnalyzer:
     def darks(self, val):
         self._reduced_darks = val
 
+    @property
+    def scan_basename(self):
+        raise BaseClassError
+
+    @property
+    def scan_dirname(self):
+        raise BaseClassError
+
+    def get_alignment_projections(self, image_sub_region=None):
+        raise NotImplementedError
+
+    @property
+    def all_angles(self):
+        raise NotImplementedError
+
+    def get_frame(self, idx): ...
+
+    @property
+    def is_360(self):
+        """
+        Return True iff the scan is 360 degrees (regardless of half-tomo mode)
+        """
+        angles = self.rotation_angles
+        delta_theta = abs(angles.max() - angles.min())
+        return abs(delta_theta - 2 * np.pi) < abs(delta_theta - np.pi)
+
 
 class EDFDatasetAnalyzer(DatasetAnalyzer):
     """
@@ -316,15 +370,15 @@ class EDFDatasetAnalyzer(DatasetAnalyzer):
         #  (eg. subsampling, binning, distortion correction...)
         # (3) The following spawns one reader instance per file, which is not elegant,
         #   but in principle there are typically 1-2 reduced flats in a scan
-        readers = {k: EDFStackReader([self.raw_flats[k].file_path()], **reader_kwargs) for k in self.raw_flats.keys()}
-        return {k: readers[k].load_data()[0] for k in self.raw_flats.keys()}
+        readers = {k: EDFStackReader([self.raw_flats[k].file_path()], **reader_kwargs) for k in self.raw_flats}
+        return {k: readers[k].load_data()[0] for k in self.raw_flats}
 
     def get_reduced_darks(self, **reader_kwargs):
         # See notes in get_reduced_flats() above
         if self.raw_darks in [None, {}]:
             raise FileNotFoundError("No reduced dark ('darkend.edf' or 'dark.edf') found in %s" % self.location)
-        readers = {k: EDFStackReader([self.raw_darks[k].file_path()], **reader_kwargs) for k in self.raw_darks.keys()}
-        return {k: readers[k].load_data()[0] for k in self.raw_darks.keys()}
+        readers = {k: EDFStackReader([self.raw_darks[k].file_path()], **reader_kwargs) for k in self.raw_darks}
+        return {k: readers[k].load_data()[0] for k in self.raw_darks}
 
     @property
     def files(self):
@@ -333,6 +387,20 @@ class EDFDatasetAnalyzer(DatasetAnalyzer):
     def get_reader(self, **kwargs):
         return EDFStackReader(self.files, **kwargs)
 
+    @property
+    def scan_basename(self):
+        # os.path.basename(self.dataset_scanner.path)
+        return self.dataset_scanner.get_dataset_basename()
+
+    @property
+    def scan_dirname(self):
+        return self.dataset_scanner.path
+
+    def get_excluded_projections_indices(self, including_other_frames_types=True):
+        if not (including_other_frames_types):
+            raise NotImplementedError
+        return self.dataset_scanner.get_ignored_projection_indices()
+
 
 class HDF5DatasetAnalyzer(DatasetAnalyzer):
     """
@@ -341,9 +409,6 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
 
     _scanner = NXtomoScan
     kind = "nx"
-    # We could import the 1000+ LoC  nxtomo.nxobject.nxdetector.ImageKey... or we can do this
-    _image_key_value = {"flats": 1, "darks": 2, "radios": 0}
-    #
 
     @property
     def z_translation(self):
@@ -425,7 +490,8 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
         slices = set()
         for du in get_compacted_dataslices(images).values():
             if du.data_slice() is not None:
-                s = (du.data_slice().start, du.data_slice().stop)
+                # note: du.data_slice is a uint in recent tomoscan version
+                s = (int(du.data_slice().start), int(du.data_slice().stop))
             else:
                 s = None
             slices.add(s)
@@ -435,7 +501,7 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
     def _select_according_to_frame_type(self, data, frame_type):
         if data is None:
             return None
-        return data[self.dataset_scanner.image_key_control == self._image_key_value[frame_type]]
+        return data[self.dataset_scanner.image_key_control == _image_type[frame_type]]
 
     def get_reduced_flats(self, method="median", force_reload=False, **reader_kwargs):
         dkrf_reader = NXDarksFlats(
@@ -458,25 +524,132 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
         For example, if the dataset flats are located at indices [1, 2, ..., 99], then
         frame_slices("flats") will return [slice(0, 100)].
         """
-        return indices_to_slices(
-            np.where(self.dataset_scanner.image_key_control == self._image_key_value[frame_type])[0]
-        )
+        return indices_to_slices(np.where(self.dataset_scanner.image_key_control == _image_type[frame_type])[0])
 
     def get_reader(self, **kwargs):
         return NXTomoReader(self.dataset_hdf5_url.file_path(), data_path=self.dataset_hdf5_url.data_path(), **kwargs)
 
+    @property
+    def scan_basename(self):
+        # os.path.splitext(os.path.basename(self.dataset_hdf5_url.file_path()))[0]
+        return self.dataset_scanner.get_dataset_basename()
 
-def analyze_dataset(dataset_path, extra_options=None, logger=None):
-    if not (os.path.isdir(dataset_path)):
-        if not (os.path.isfile(dataset_path)):
-            raise ValueError("Error: %s no such file or directory" % dataset_path)
-        if not (is_hdf5_extension(os.path.splitext(dataset_path)[-1].replace(".", ""))):
-            raise ValueError("Error: expected a HDF5 file")
-        dataset_analyzer_class = HDF5DatasetAnalyzer
-    else:  # directory -> assuming EDF
-        dataset_analyzer_class = EDFDatasetAnalyzer
-    dataset_structure = dataset_analyzer_class(dataset_path, extra_options=extra_options, logger=logger)
-    return dataset_structure
+    @property
+    def scan_dirname(self):
+        # os.path.dirname(di.dataset_hdf5_url.file_path())
+        return self.dataset_scanner.path
+
+    def get_alignment_projections(self, image_sub_region=None):
+        """
+        Get the extra projections (if any) that are used as "reference projections" for alignment.
+        For certain scan, when completing a (half) turn, sometimes extra projections are acquired for alignment purpose.
+
+        Returns
+        -------
+        projs: numpy.ndarray
+            Array with shape (n_projections, n_y, n_x)
+        angles: numpy.ndarray
+            Corresponding angles in degrees
+        indices:
+            Indices of projections
+        """
+        sub_region = None
+        if image_sub_region is not None:
+            sub_region = (None,) + image_sub_region
+        reader = self.get_reader(image_key=ImageKey.ALIGNMENT.value, sub_region=sub_region)
+        projs = reader.load_data()
+        indices = reader.get_frames_indices()
+        angles = get_angle_at_index(self.all_angles, indices)
+        return projs, angles, indices
+
+    @property
+    def all_angles(self):
+        return np.array(self.dataset_scanner.rotation_angle)
+
+    def get_index_from_angle(self, angle, image_key=0, return_found_angle=False):
+        """
+        Return the index of the image taken at rotation angle 'angle'.
+        By default look at the projections, i.e image_key = 0
+        """
+        all_angles = self.all_angles
+        all_indices = np.arange(len(all_angles))
+        all_image_key = self.dataset_scanner.image_key_control
+
+        idx2 = np.where(all_image_key == image_key)[0]
+        angles = all_angles[idx2]
+        idx_angles_sorted = np.argsort(angles)
+        angles_sorted = angles[idx_angles_sorted]
+
+        pos = search_sorted(angles_sorted, angle)
+        # this gives a position in "idx2", but we need the position in "all_indices"
+        idx = all_indices[idx2[idx_angles_sorted[pos]]]
+        if return_found_angle:
+            return idx, angles_sorted[pos]
+        return idx
+
+    def get_image_at_angle(self, angle_deg, image_type="projection", sub_region=None, return_angle_and_index=False):
+        image_key = _image_type[image_type]
+        idx, angle_found = self.get_index_from_angle(angle_deg, image_key=image_key, return_found_angle=True)
+
+        # Option 1:
+        if sub_region is None:
+            sub_region = (None, None)
+        # Convert absolute index to index of image_key
+        idx2 = np.searchsorted(np.where(self.dataset_scanner.image_key_control == image_key)[0], idx)
+        sub_region = (slice(idx2, idx2 + 1),) + sub_region
+        reader = self.get_reader(image_key=image_key, sub_region=sub_region)
+        img = reader.load_data()[0]
+        if return_angle_and_index:
+            return img, angle_found, idx
+        return img
+
+        # Option 2:
+        # return self.get_frame(idx)
+        # something like:
+        # [fr for fr in self.dataset_scanner.frames if fr.image_key.value == 0 and fr.rotation_angle == 180 and fr._is_control_frame is False]
+
+    def get_frame(self, idx):
+        return get_data(self.dataset_scanner.frames[idx].url)
+
+    def get_frames_indices(self, frame_type):
+        return self._select_according_to_frame_type(np.arange(self.dataset_scanner.image_key_control.size), frame_type)
+
+    def index_to_proj_number(self, proj_index):
+        """
+        Return the projection *number*, from its frame *index*.
+
+        For example if there are 11 flats before projections,
+        then projections will have indices [11, 12, .....] (possibly not contiguous)
+        while their number is [0, 1, ..., ] (contiguous, starts from 0)
+        """
+        all_projs_indices = self.get_frames_indices("projection")
+        return search_sorted(all_projs_indices, proj_index)
+
+    def get_excluded_projections_indices(self, including_other_frames_types=True):
+        # Get indices of ALL projections (even excluded ones)
+        # the index accounts for flats/darks !
+        # Get indices of excluded projs (again, accounting for flats/darks)
+        ignored_projs_indices = self.dataset_scanner.get_ignored_projection_indices()
+        ignored_projs_indices = [
+            idx for idx in ignored_projs_indices if self.dataset_scanner.frames[idx].is_control is False
+        ]
+        if including_other_frames_types:
+            return ignored_projs_indices
+        # Get indices of excluded projs, now relative to the pure projections stack
+        ignored_projs_indices_rel = [
+            self.index_to_proj_number(ignored_proj_idx_abs) for ignored_proj_idx_abs in ignored_projs_indices
+        ]
+        return ignored_projs_indices_rel
+
+
+def get_angle_at_index(all_angles, index):
+    """
+    Return the rotation angle corresponding to image index 'index'
+    """
+    if is_scalar(index):
+        return all_angles[index]
+    else:
+        return all_angles[np.array(index)]
 
 
 def get_radio_pair(dataset_info, radio_angles: tuple, return_indices=False):
@@ -522,3 +695,16 @@ def get_radio_pair(dataset_info, radio_angles: tuple, return_indices=False):
         return radios, radios_indices
     else:
         return radios
+
+
+def analyze_dataset(dataset_path, extra_options=None, logger=None):
+    if not (os.path.isdir(dataset_path)):
+        if not (os.path.isfile(dataset_path)):
+            raise ValueError("Error: %s no such file or directory" % dataset_path)
+        if not (is_hdf5_extension(os.path.splitext(dataset_path)[-1].replace(".", ""))):
+            raise ValueError("Error: expected a HDF5 file")
+        dataset_analyzer_class = HDF5DatasetAnalyzer
+    else:  # directory -> assuming EDF
+        dataset_analyzer_class = EDFDatasetAnalyzer
+    dataset_structure = dataset_analyzer_class(dataset_path, extra_options=extra_options, logger=logger)
+    return dataset_structure

nabu/resources/gpu.py

@@ -131,7 +131,7 @@ def pick_gpus_auto(cuda_gpus, opencl_platforms, n_gpus):
         return (gpu1["device_id"] == gpu2["device_id"]) and (gpu1["name"] == gpu2["name"])
 
     def is_in_gpus(avail_gpus, query_gpu):
-        for gpu in avail_gpus:
+        for gpu in avail_gpus:  # noqa: SIM110
             if gpu_equal(gpu, query_gpu):
                 return True
         return False
@@ -142,8 +142,8 @@ def pick_gpus_auto(cuda_gpus, opencl_platforms, n_gpus):
     chosen_gpus = list(cuda_gpus.values())
     if len(chosen_gpus) >= n_gpus:
         return chosen_gpus
-    for platform, gpus in opencl_platforms.items():
-        for gpu_id, gpu in gpus.items():
+    for platform, gpus in opencl_platforms.items():  # noqa: PERF102
+        for gpu_id, gpu in gpus.items():  # noqa: PERF102
             if not (is_in_gpus(chosen_gpus, gpu)):
                 # TODO prioritize some OpenCL implementations ?
                 chosen_gpus.append(gpu)
@@ -166,5 +166,5 @@ def pick_gpus_nvidia(cuda_gpus, n_gpus):
     gpus_cc_sorted = sorted(gpus_cc, key=lambda x: x[1], reverse=True)
     res = []
     for i in range(n_gpus):
-        res.append(cuda_gpus[gpus_cc_sorted[i][0]])
+        res.append(cuda_gpus[gpus_cc_sorted[i][0]])  # noqa: PERF401
     return res

nabu/resources/logger.py

@@ -2,7 +2,7 @@ import logging
 import logging.config
 
 
-class Logger(object):
+class Logger:
     def __init__(self, loggername, level="DEBUG", logfile="logger.log", console=True):
         """
         Configure a Logger object.
@@ -30,7 +30,7 @@ class Logger(object):
 
     def _configure_logger(self):
         conf = self._get_default_config_dict()
-        for handler in conf["handlers"].keys():
+        for handler in conf["handlers"]:
             conf["handlers"][handler]["level"] = self.level.upper()
         conf["loggers"][self.loggername]["level"] = self.level.upper()
         if not (self.console):
@@ -103,7 +103,7 @@ def LoggerOrPrint(logger):
     return logger
 
 
-class PrinterLogger(object):
+class PrinterLogger:
     def __init__(self):
         methods = [
             "debug",
@@ -122,7 +122,7 @@ LogLevel = {
     "notset": logging.NOTSET,
     "debug": logging.DEBUG,
     "info": logging.INFO,
-    "warn": logging.WARN,
+    "warn": logging.WARNING,
     "warning": logging.WARNING,
     "error": logging.ERROR,
     "critical": logging.CRITICAL,

nabu/resources/nxflatfield.py

@@ -6,6 +6,8 @@ from silx.io import get_data
 from tomoscan.framereducer.reducedframesinfos import ReducedFramesInfos
 from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
 from ..utils import check_supported, is_writeable
+from ..preproc.flatfield import PCAFlatsDecomposer
+from ..io.reader import NXDarksFlats
 
 
 def get_frame_possible_urls(dataset_info, user_dir, output_dir):
@@ -22,7 +24,7 @@ def get_frame_possible_urls(dataset_info, user_dir, output_dir):
         Output processing directory
     """
 
-    frame_types = ["flats", "darks"]
+    frame_types = ["flats", "darks", "pcaflats"]
     h5scan = dataset_info.dataset_scanner  # tomoscan object
 
     def make_dataurl(dirname, frame_type):
@@ -32,8 +34,10 @@ def get_frame_possible_urls(dataset_info, user_dir, output_dir):
 
         if frame_type == "flats":
             dataurl_default_template = h5scan.REDUCED_FLATS_DATAURLS[0]
-        else:
+        elif frame_type == "darks":
             dataurl_default_template = h5scan.REDUCED_DARKS_DATAURLS[0]
+        elif frame_type == "pcaflats":
+            dataurl_default_template = h5scan.PCA_FLATS_DATAURLS[0]
 
         rel_file_path = dataurl_default_template.file_path().format(scan_prefix=h5scan.get_dataset_basename())
         return DataUrl(
@@ -67,7 +71,7 @@ def save_reduced_frames(dataset_info, reduced_frames_arrays, reduced_frames_urls
     darks_flats_dir_url = reduced_frames_urls.get("user", None)
     if darks_flats_dir_url is not None:
         output_url = darks_flats_dir_url
-    elif is_writeable(os.path.dirname(reduced_frames_urls["dataset"]["flats"].file_path())):
+    elif is_writeable(os.path.abspath(os.path.dirname(reduced_frames_urls["dataset"]["flats"].file_path()))):
         output_url = reduced_frames_urls["dataset"]
     else:
         output_url = reduced_frames_urls["output"]
@@ -149,8 +153,94 @@ def data_url_exists(data_url):
     return group_exists
 
 
+def _compute_and_save_reduced_frames(flatfield_mode, dataset_info, reduced_frames_urls):
+    if flatfield_mode == "pca":
+        dfreader = NXDarksFlats(dataset_info.location)
+        darks = np.concatenate([d for d in dfreader.get_raw_darks()], axis=0)
+        flats = np.concatenate([f for f in dfreader.get_raw_flats()], axis=0)
+        pcaflats_darks = PCAFlatsDecomposer(flats, darks)
+
+        # Get "where to write". tomoscan expects a DataUrl
+        pcaflats_dir_url = reduced_frames_urls.get("user", None)
+        if pcaflats_dir_url is not None:
+            output_url = pcaflats_dir_url
+        elif is_writeable(os.path.dirname(reduced_frames_urls["dataset"]["flats"].file_path())):
+            output_url = reduced_frames_urls["dataset"]
+        else:
+            output_url = reduced_frames_urls["output"]
+        pcaflats_darks.save_decomposition(
+            path=output_url["pcaflats"].file_path(), entry=output_url["pcaflats"].data_path().strip("/").split("/")[0]
+        )
+        dataset_info.logger.info("PCA flats computed and written at %s" % (output_url["pcaflats"].file_path()))
+
+        # Update dataset_info with pca flats and dark
+        dataset_info.darks = {0: pcaflats_darks.dark}
+        flats = {0: pcaflats_darks.mean}
+        for k in range(len(pcaflats_darks.components)):
+            flats.update({k + 1: pcaflats_darks.components[k]})
+        dataset_info.flats = flats
+    else:
+        try:
+            dataset_info.flats = dataset_info.get_reduced_flats()
+            dataset_info.darks = dataset_info.get_reduced_darks()
+        except FileNotFoundError:
+            msg = "Could not find any flats and/or darks"
+            raise FileNotFoundError(msg)
+        _, flats_info, darks_info = save_reduced_frames(
+            dataset_info, {"darks": dataset_info.darks, "flats": dataset_info.flats}, reduced_frames_urls
+        )
+        dataset_info.flats_srcurrent = flats_info.machine_electric_current
+
+
+def _load_existing_flatfields(dataset_info, reduced_frames_urls, frames_types, where_to_load_from):
+    if "pcaflats" not in frames_types:
+        reduced_frames_with_info = {}
+        for frame_type in frames_types:
+            reduced_frames_with_info[frame_type] = tomoscan_load_reduced_frames(
+                dataset_info, frame_type, reduced_frames_urls[where_to_load_from][frame_type]
+            )
+            dataset_info.logger.info(
+                "Loaded %s from %s" % (frame_type, reduced_frames_urls[where_to_load_from][frame_type].file_path())
+            )
+            red_frames_dict, red_frames_info = reduced_frames_with_info[frame_type]
+            setattr(
+                dataset_info,
+                frame_type,
+                {k: get_data(red_frames_dict[k]) for k in red_frames_dict},
+            )
+            if frame_type == "flats":
+                dataset_info.flats_srcurrent = red_frames_info.machine_electric_current
+    else:
+        df_path = reduced_frames_urls[where_to_load_from]["pcaflats"].file_path()
+        entry = reduced_frames_urls[where_to_load_from]["pcaflats"].data_path()
+
+        # Update dark
+        dark_url = DataUrl(f"silx://{df_path}?{entry}/dark")
+        dark = get_data(dark_url)
+        setattr(
+            dataset_info,
+            "dark",
+            {0: dark},
+        )
+        # Update flats with principal compenents
+        # Take mean as first comp., mask as second, flats thereafter
+        flats_url = DataUrl(f"silx://{df_path}?{entry}/p_components")
+        mean_url = DataUrl(f"silx://{df_path}?{entry}/p_mean")
+        flats = get_data(flats_url)
+        mean = get_data(mean_url)
+        flats = np.concatenate([mean[np.newaxis], flats], axis=0)
+        setattr(
+            dataset_info,
+            "flats",
+            {k: flats[k] for k in range(len(flats))},
+        )
+        dataset_info.logger.info("Loaded %s from %s" % ("PCA darks/flats", df_path))
+
+
 # pylint: disable=E1136
-def update_dataset_info_flats_darks(dataset_info, flatfield_mode, output_dir=None, darks_flats_dir=None):
+def update_dataset_info_flats_darks(
+    dataset_info, flatfield_mode, loading_mode="load_if_present", output_dir=None, darks_flats_dir=None
+):
     """
     Update a DatasetAnalyzer object with reduced flats/darks (hereafter "reduced frames").
 
@@ -170,23 +260,14 @@ def update_dataset_info_flats_darks(dataset_info, flatfield_mode, output_dir=Non
     if flatfield_mode is False:
         return
 
-    frames_types = ["darks", "flats"]
+    if flatfield_mode == "pca":
+        frames_types = ["pcaflats"]
+    else:
+        frames_types = ["darks", "flats"]
     reduced_frames_urls = get_frame_possible_urls(dataset_info, darks_flats_dir, output_dir)
 
-    def _compute_and_save_reduced_frames():
-        try:
-            dataset_info.flats = dataset_info.get_reduced_flats()
-            dataset_info.darks = dataset_info.get_reduced_darks()
-        except FileNotFoundError:
-            msg = "Could not find any flats and/or darks"
-            raise FileNotFoundError(msg)
-        _, flats_info, darks_info = save_reduced_frames(
-            dataset_info, {"darks": dataset_info.darks, "flats": dataset_info.flats}, reduced_frames_urls
-        )
-        dataset_info.flats_srcurrent = flats_info.machine_electric_current
-
-    if flatfield_mode == "force-compute":
-        _compute_and_save_reduced_frames()
+    if loading_mode == "force-compute":
+        _compute_and_save_reduced_frames(flatfield_mode, dataset_info, reduced_frames_urls)
         return
 
     def _can_load_from(folder_type):
@@ -202,25 +283,10 @@ def update_dataset_info_flats_darks(dataset_info, flatfield_mode, output_dir=Non
     elif _can_load_from("output"):
         where_to_load_from = "output"
 
-    if where_to_load_from == None and flatfield_mode == "force-load":
+    if where_to_load_from is None and flatfield_mode == "force-load":
         raise ValueError("Could not load darks/flats (using 'force-load')")
 
     if where_to_load_from is not None:
-        reduced_frames_with_info = {}
-        for frame_type in frames_types:
-            reduced_frames_with_info[frame_type] = tomoscan_load_reduced_frames(
-                dataset_info, frame_type, reduced_frames_urls[where_to_load_from][frame_type]
-            )
-            dataset_info.logger.info(
-                "Loaded %s from %s" % (frame_type, reduced_frames_urls[where_to_load_from][frame_type].file_path())
-            )
-            red_frames_dict, red_frames_info = reduced_frames_with_info[frame_type]
-            setattr(
-                dataset_info,
-                frame_type,
-                {k: get_data(red_frames_dict[k]) for k in red_frames_dict.keys()},
-            )
-            if frame_type == "flats":
-                dataset_info.flats_srcurrent = red_frames_info.machine_electric_current
+        _load_existing_flatfields(dataset_info, reduced_frames_urls, frames_types, where_to_load_from)
     else:
-        _compute_and_save_reduced_frames()
+        _compute_and_save_reduced_frames(flatfield_mode, dataset_info, reduced_frames_urls)

nabu/resources/tests/test_dataset_analyzer.py

@@ -0,0 +1,37 @@
+import pytest
+import numpy as np
+from nabu.testutils import get_dummy_nxtomo_info
+from nabu.resources.dataset_analyzer import analyze_dataset
+
+
+@pytest.fixture(scope="class")
+def bootstrap_nx(request):
+    cls = request.cls
+    cls.nx_fname, cls.data_desc, cls.image_key, cls.projs_vals, cls.darks_vals, cls.flats1_vals, cls.flats2_vals = (
+        get_dummy_nxtomo_info()
+    )
+
+
+@pytest.mark.usefixtures("bootstrap_nx")
+class TestNXDataset:
+
+    def test_exclude_projs_angular_range(self):
+        dataset_info_with_all_projs = analyze_dataset(self.nx_fname)
+
+        # Test exclude angular range - angles min and max in degrees
+        angular_ranges_to_test = [(0, 15), (5, 6), (50, 58.5)]
+        for angular_range in angular_ranges_to_test:
+            angle_min, angle_max = angular_range
+            dataset_info = analyze_dataset(
+                self.nx_fname,
+                extra_options={"exclude_projections": {"type": "angular_range", "range": [angle_min, angle_max]}},
+            )
+            excluded_projs_indices = dataset_info.get_excluded_projections_indices()
+            # Check that get_excluded_projections_indices() angles are correct
+            for excluded_proj_index in excluded_projs_indices:
+                frame_angle_deg = dataset_info.dataset_scanner.frames[excluded_proj_index].rotation_angle
+                assert angle_min <= frame_angle_deg and frame_angle_deg <= angle_max
+
+            assert set(dataset_info_with_all_projs.projections.keys()) - set(dataset_info.projections.keys()) == set(
+                excluded_projs_indices
+            )

nabu/resources/tests/test_extract.py

@@ -0,0 +1,11 @@
+from nabu.utils import list_match_queries
+
+
+def test_list_match_queries():
+
+    # entry0000 .... entry0099
+    avail = ["entry%04d" % i for i in range(100)]
+    assert list_match_queries(avail, "entry0000") == ["entry0000"]
+    assert list_match_queries(avail, ["entry0001"]) == ["entry0001"]
+    assert list_match_queries(avail, ["entry000?"]) == ["entry%04d" % i for i in range(10)]
+    assert list_match_queries(avail, ["entry*"]) == avail

nabu/resources/tests/test_nxflatfield.py

@@ -80,11 +80,11 @@ class TestNXFlatField:
         output_dir = self.params.get("output_dir", None)
         if output_dir is not None:
             output_dir = output_dir.format(tempdir=self.tempdir)
-        update_dataset_info_flats_darks(dataset_info, True, output_dir=output_dir)
+        update_dataset_info_flats_darks(dataset_info, True, loading_mode="load_if_present", output_dir=output_dir)
         # After reduction (median/mean), the flats/darks are located in another file.
         # median(series_1) goes to entry/flats/idx1, mean(series_2) goes to entry/flats/idx2, etc.
-        assert set(dataset_info.flats.keys()) == set(s.start for s in self.params["flats_pos"])
-        assert set(dataset_info.darks.keys()) == set(s.start for s in self.params["darks_pos"])
+        assert set(dataset_info.flats.keys()) == set(s.start for s in self.params["flats_pos"])  # noqa: C401
+        assert set(dataset_info.darks.keys()) == set(s.start for s in self.params["darks_pos"])  # noqa: C401
 
         # Check that the computations were correct
         # Loads the entire volume in memory ! So keep the data volume small for the tests
@@ -97,8 +97,8 @@ class TestNXFlatField:
             expected_darks[s.start] = self._reduction_func["darks"](data_volume[s.start : s.stop], axis=0)
 
         flats = dataset_info.flats
-        for idx in flats.keys():
+        for idx in flats:
             assert np.allclose(flats[idx], expected_flats[idx])
         darks = dataset_info.darks
-        for idx in darks.keys():
+        for idx in darks:
             assert np.allclose(darks[idx], expected_darks[idx])