nabu 2024.2.13__py3-none-any.whl → 2025.1.0__py3-none-any.whl

nabu/pipeline/fullfield/chunked_cuda.py

@@ -75,8 +75,11 @@ class CudaChunkedPipeline(ChunkedPipeline):
         # Decide when to transfer data to GPU. Normally it's right after reading the data,
         # But sometimes a part of the processing is done on CPU.
         self._when_to_transfer_radios_on_gpu = "read_data"
-        if self.flatfield is not None and self.flatfield.distortion_correction is not None:
-            self._when_to_transfer_radios_on_gpu = "flatfield"
+        if self.flatfield is not None:
+            use_flats_distortion = getattr(self.flatfield, "distortion_correction", None) is not None
+            use_pca_flats = self.processing_options["flatfield"]["method"].lower() == "pca"
+            if use_flats_distortion or use_pca_flats:
+                self._when_to_transfer_radios_on_gpu = "flatfield"
 
     def _init_cuda(self, cuda_options):
         if not (__has_pycuda__):

nabu/pipeline/fullfield/computations.py

@@ -128,7 +128,10 @@ def estimate_required_memory(
         if process_config.rec_params["method"] == "cone":
             # In cone-beam reconstruction, need both sinograms and reconstruction inside GPU.
             # That's big!
-            total_memory_needed += 2 * data_volume_size
+            mult_factor = 2
+            if rec_config["crop_filtered_data"] is False:
+                mult_factor = 4
+            total_memory_needed += mult_factor * data_volume_size
 
     if debug:
         print(

nabu/pipeline/fullfield/dataset_validator.py

@@ -1,4 +1,3 @@
-import os
 from ..dataset_validator import DatasetValidatorBase
 
 

nabu/pipeline/fullfield/get_double_flatfield.py

@@ -0,0 +1,147 @@
+"""
+Double-flatfield:
+  - Compute the average of all projections, which gives one resulting image
+  - Apply some filter to this image (DFF)
+  - Subtract or divide this image from all the projections
+"""
+
+from os import path
+from silx.io.url import DataUrl
+from silx.io.dictdump import h5todict
+
+from nabu.io.utils import get_first_hdf5_entry
+
+from ...utils import is_writeable
+from ...app.double_flatfield import DoubleFlatFieldChunks
+from ...resources.nxflatfield import data_url_exists
+
+rel_file_path_template = "{scan_name}_dff.h5"
+data_path_template = "{entry}/double_flatfield"
+
+
+def get_possible_dff_urls(dataset_info, user_dir, output_dir):
+    """
+    See nabu.resources.nxflatfield.get_frame_possible_urls
+    """
+    entry = dataset_info.hdf5_entry or ""
+
+    def make_dataurl(dirname):
+        file_path = path.join(
+            dirname,
+            rel_file_path_template.format(scan_name=dataset_info.scan_basename),
+        )
+        return DataUrl(
+            file_path=file_path,
+            data_path=data_path_template.format(entry=entry),
+            scheme="silx",
+        )
+
+    urls = {"user": None, "dataset": None, "output": None}
+
+    if user_dir is not None:
+        urls["user"] = make_dataurl(user_dir)
+    urls["dataset"] = make_dataurl(dataset_info.scan_dirname)
+    if output_dir is not None:
+        urls["output"] = make_dataurl(output_dir)
+
+    return urls
+
+
+def compute_and_save_dff(dataset_info, possible_urls, dff_options):
+    if possible_urls["user"] is not None:
+        dff_output_file = possible_urls["user"].file_path()
+    elif is_writeable(path.dirname(possible_urls["dataset"].file_path())):
+        dff_output_file = possible_urls["dataset"].file_path()
+    else:
+        dff_output_file = possible_urls["output"].file_path()
+
+    dataset_info.logger.info("Computing double flatfield")
+    dff = DoubleFlatFieldChunks(
+        None,
+        dff_output_file,
+        dataset_info=dataset_info,
+        chunk_size=dff_options.get("chunk_size", 100),
+        sigma=dff_options.get("dff_sigma", None),
+        do_flatfield=dff_options.get("do_flatfield", True),
+        logger=dataset_info.logger,
+    )
+    dff_image = dff.compute_double_flatfield()
+    return dff.write_double_flatfield(dff_image)
+
+
+def check_existing_dff(dff_url, dff_options, logger):
+    # Check that the DFF exists at the given DataUrl, and that its configuration matches the wanted config
+    # Return the DFF file path
+    if not (data_url_exists(dff_url)):
+        raise ValueError("DFF file not found:", dff_url)
+
+    fname = dff_url.file_path()
+    entry = get_first_hdf5_entry(fname)
+    dff_file_options = h5todict(fname, path=entry + "/double_flatfield/configuration", asarray=False)
+
+    ff_file = dff_file_options.get("do_flatfield", True)
+    ff_user = dff_options.get("do_flatfield", True)
+    # Use "==" instead of "is" here, as h5todict() will return something like numpy.True_ instead of True
+    if ff_file != ff_user:
+        msg = "DFF was computed with flatfield=%s, but you asked flatfield=%s" % (ff_file, ff_user)
+        logger.error(msg)
+        return False
+
+    # Use this because h5todict() returns str("None") instead of None
+    def _correct_none(x):
+        if x in [None, "None"]:
+            return None
+        return x
+
+    sigma_file = _correct_none(dff_file_options.get("dff_sigma", None))
+    sigma_user = _correct_none(dff_options.get("dff_sigma", None))
+    if sigma_file != sigma_user:
+        msg = "DFF was computed with dff_sigma=%s, but you asked dff_sigma=%s" % (sigma_file, sigma_user)
+        logger.error(msg)
+        return False
+
+    return fname
+
+
+# pylint: disable=E1136
+def get_double_flatfield(dataset_info, mode, output_dir=None, darks_flats_dir=None, dff_options=None):
+    """
+    See nabu.resources.nxflatfield.update_dataset_info_flats_darks for the logic
+    """
+    if mode is False:
+        return
+    dff_options = dff_options or {}
+
+    possible_urls = get_possible_dff_urls(dataset_info, darks_flats_dir, output_dir)
+
+    if mode == "force-compute":
+        return compute_and_save_dff(dataset_info, possible_urls, dff_options)
+
+    def _can_load_from(folder_type):
+        if possible_urls.get(folder_type, None) is None:
+            return False
+        return data_url_exists(possible_urls[folder_type])
+
+    where_to_load_from = None
+    if possible_urls["user"] is not None and _can_load_from("user"):
+        where_to_load_from = "user"
+    elif _can_load_from("dataset"):
+        where_to_load_from = "dataset"
+    elif _can_load_from("output"):
+        where_to_load_from = "output"
+
+    if where_to_load_from is None:
+        if mode == "force-load":
+            raise ValueError("Could not load double-flatfield file (using 'force-load')")
+        else:
+            return compute_and_save_dff(dataset_info, possible_urls, dff_options)
+
+    fname = check_existing_dff(possible_urls[where_to_load_from], dff_options, dataset_info.logger)
+    if fname is False:
+        if mode == "force-load":
+            raise ValueError("Could not load double-flatfield file (using 'force-load'): wrong configuration")
+        return compute_and_save_dff(dataset_info, possible_urls, dff_options)
+    return fname
+
+    # One possible corner case: if mode == "force-load" and darks_flats_dir is not None (but the actual folder is empty)
+    # then nabu will load a DFF found elsewhere (if any). We might want to raise an error instead.

nabu/pipeline/fullfield/nabu_config.py

@@ -1,4 +1,5 @@
-from ..config_validators import *
+# ruff: noqa: F405
+from ..config_validators import *  # noqa: F403
 
 nabu_config = {
     "dataset": {
@@ -15,14 +16,14 @@ nabu_config = {
             "type": "advanced",
         },
         "nexus_version": {
-            "default": "1.4",
-            "help": "Nexus version to use when browsing the HDF5 dataset. Default is 1.0.",
-            "validator": float_validator,
+            "default": "",
+            "help": "Specify a Nexus version to use when browsing the HDF5 dataset.",
+            "validator": optional_float_validator,
             "type": "advanced",
         },
         "darks_flats_dir": {
             "default": "",
-            "help": "Path to a directory where XXX_flats.h5 and XXX_darks.h5 are to be found, where 'XXX' denotes the dataset basename. If these files are found, then reduced flats/darks will be loaded from them. Otherwise, reduced flats/darks will be saved to there once computed, either in the .nx directory, or in the output directory. Mind that the HDF5 entry corresponds to the one of the dataset.",
+            "help": "Path to a directory where XXX_flats.h5 and XXX_darks.h5 are to be found, where 'XXX' denotes the dataset basename. If these files are found, then reduced flats/darks will be loaded from them. Otherwise, reduced flats/darks will be saved there once computed, either in the .nx directory, or in the output directory. Mind that the HDF5 entry corresponds to the one of the dataset.",
             "validator": optional_directory_location_validator,
             "type": "optional",
         },
@@ -40,7 +41,7 @@ nabu_config = {
         },
         "projections_subsampling": {
             "default": "1",
-            "help": "Projections subsampling factor: take one projection out of 'projection_subsampling'. The format can be an integer (take 1 projection out of N), or N:M (take 1 projection out of N, start with the projection number M)\nFor example: 2 (or 2:0) to reconstruct from even projections, 2:1 to reconstruct from odd projections.",
+            "help": "Projections subsampling factor: take one projection out of 'projections_subsampling'. The format can be an integer (take 1 projection out of N), or N:M (take 1 projection out of N, start with the projection number M)\nFor example: 2 (or 2:0) to reconstruct from even projections, 2:1 to reconstruct from odd projections.",
             "validator": projections_subsampling_validator,
             "type": "advanced",
         },
@@ -60,13 +61,19 @@ nabu_config = {
     "preproc": {
         "flatfield": {
             "default": "1",
-            "help": "How to perform flat-field normalization. The parameter value can be:\n - 1 or True: enabled.\n - 0 or False: disabled\n - forced or force-load: perform flatfield regardless of the dataset by attempting to load darks/flats\n - force-compute: perform flatfield, ignore all .h5 files containing already computed darks/flats.",
-            "validator": flatfield_enabled_validator,
+            "help": "How to perform flat-field normalization. The parameter value can be:\n - 1 or True: enabled.\n - 0 or False: disabled\n - pca: perform a normalization via Principal Component Analysis decomposition PCA-flat-field normalization",
+            "validator": flatfield_validator,
             "type": "required",
         },
+        "flatfield_loading_mode": {
+            "default": "load_if_present",
+            "help": "How to load/compute flat-field. This parameter can be:\n - load_if_present (default) or empty string: Use the existing flatfield files, if existing.\n - force-load: perform flatfield regardless of the dataset by attempting to load darks/flats\n - force-compute: perform flatfield, ignore all .h5 files containing already computed darks/flats.",
+            "validator": flatfield_loading_mode_validator,
+            "type": "optional",
+        },
         "flat_distortion_correction_enabled": {
             "default": "0",
-            "help": "Whether to correct for flat distortion. If activated, each radio is correlated with its corresponding flat, in order to determine and correct the flat distortion.",
+            "help": "Whether to correct for flat distortion. If activated, each radiograph is correlated with its corresponding flat, in order to determine and correct the flat distortion.",
             "validator": boolean_validator,
             "type": "advanced",
         },
@@ -109,10 +116,10 @@ nabu_config = {
             "validator": generic_options_validator,
             "type": "advanced",
         },
-        "double_flatfield_enabled": {
+        "double_flatfield": {
             "default": "0",
-            "help": "Whether to enable the 'double flat-field' filetering for correcting rings artefacts.",
-            "validator": boolean_validator,
+            "help": "Whether to perform 'double flat-field' filtering (this can help to remove rings artefacts). Possible values:\n - 1 or True: enabled.\n - 0 or False: disabled\n - force-load: use an existing DFF file regardless of the dataset\n - force-compute: re-compute the DFF, ignore all existing .h5 files containing already computed DFF",
+            "validator": flatfield_validator,
             "type": "optional",
         },
         "dff_sigma": {
@@ -171,7 +178,7 @@ nabu_config = {
         },
         "rotate_projections_center": {
             "default": "",
-            "help": "Center of rotation when 'tilt_correction' is non-empty. By default the center of rotation is the middle of each radio, i.e ((Nx-1)/2.0, (Ny-1)/2.0).",
+            "help": "Center of rotation when 'tilt_correction' is non-empty. By default the center of rotation is the middle of each radiograph, i.e ((Nx-1)/2.0, (Ny-1)/2.0).",
             "validator": optional_tuple_of_floats_validator,
             "type": "advanced",
         },
@@ -271,7 +278,7 @@ nabu_config = {
         },
         "cor_slice": {
             "default": "",
-            "help": "Which slice to use for estimating the Center of Rotation (CoR). This parameter can be an integer or 'top', 'middle', 'bottom'.\nIf provided, the CoR will be estimated from the correspondig sinogram, and 'cor_options' can contain the parameter 'subsampling'.",
+            "help": "Which slice to use for estimating the Center of Rotation (CoR). This parameter can be an integer or 'top', 'middle', 'bottom'.\nIf provided, the CoR will be estimated from the corresponding sinogram, and 'cor_options' can contain the parameter 'subsampling'.",
             "validator": cor_slice_validator,
             "type": "advanced",
         },
@@ -401,6 +408,12 @@ nabu_config = {
             "validator": nonnegative_integer_validator,
             "type": "advanced",
         },
+        "crop_filtered_data": {
+            "default": "1",
+            "help": "Whether to crop the data after the filtering step in FBP/FDK. This parameter should be always 1 unless you know what you are doing.",
+            "validator": boolean_validator,
+            "type": "advanced",
+        },
         "optim_algorithm": {
             "default": "chambolle-pock",
             "help": "Optimization algorithm for iterative methods",
@@ -472,7 +485,7 @@ nabu_config = {
     },
     "postproc": {
         "output_histogram": {
-            "default": "0",
+            "default": "1",
             "help": "Whether to compute a histogram of the volume.",
             "validator": boolean_validator,
             "type": "optional",
@@ -537,7 +550,7 @@ nabu_config = {
     "pipeline": {
         "save_steps": {
             "default": "",
-            "help": "Save intermediate results. This is a list of comma-separated processing steps, for ex: flatfield, phase, sinogram.\nEach step generates a HDF5 file in the form name_file_prefix.hdf5 (ex. 'sinogram_file_prefix.hdf5')",
+            "help": "Save intermediate results. This is a list of comma-separated processing steps, for ex: flatfield, phase, sinogram.\nEach step generates a HDF5 file in the form name_file_prefix.hdf5 (e.g. 'sinogram_file_prefix.hdf5')",
             "validator": optional_string_validator,
             "type": "optional",
         },
@@ -549,7 +562,7 @@ nabu_config = {
         },
         "steps_file": {
             "default": "",
-            "help": "File where the intermediate processing steps are written. By default it is empty, and intermediate processing steps are written in the same directory as the reconstructions, with a file prefix, ex. sinogram_mydataset.hdf5.",
+            "help": "File where the intermediate processing steps are written. By default it is empty, and intermediate processing steps are written in the same directory as the reconstructions, with a file prefix, e.g. sinogram_mydataset.hdf5.",
             "validator": optional_output_file_path_validator,
             "type": "advanced",
         },
@@ -605,6 +618,12 @@ renamed_keys = {
         "since": "2021.2.0",
         "message": "Option 'flatfield_enabled' has been renamed 'flatfield' in [preproc]",
     },
+    "double_flatfield_enabled": {
+        "section": "preproc",
+        "new_name": "double_flatfield",
+        "since": "2025.1.0",
+        "message": "Option 'double_flatfield_enabled' has been renamed 'double_flatfield' in [preproc]",
+    },
     "rotate_projections": {
         "section": "preproc",
         "new_name": "",

nabu/pipeline/fullfield/processconfig.py

@@ -1,9 +1,10 @@
 import os
 import posixpath
 import numpy as np
+from .get_double_flatfield import get_double_flatfield
 from silx.io import get_data
 from silx.io.url import DataUrl
-from ...utils import copy_dict_items, compare_dicts
+from ...utils import copy_dict_items, compare_dicts, deprecation_warning
 from ...io.utils import hdf5_entry_exists, get_h5_value
 from ...io.reader import import_h5_to_dict
 from ...resources.utils import extract_parameters, get_values_from_file
@@ -11,6 +12,7 @@ from ...resources.nxflatfield import update_dataset_info_flats_darks
 from ...resources.utils import get_quantities_and_units
 from ..estimators import estimate_cor
 from ..processconfig import ProcessConfigBase
+from ..config_validators import convert_to_bool
 from .nabu_config import nabu_config, renamed_keys
 from .dataset_validator import FullFieldDatasetValidator
 from nxtomo.nxobject.nxdetector import ImageKey
@@ -32,6 +34,7 @@ class ProcessConfig(ProcessConfigBase):
 
     (2) update_dataset_info_with_user_config
       - Update flats/darks
+      - Double-flat-field
       - CoR  (value or estimation method) # no estimation yet
       - rotation angles
       - translations files
@@ -73,10 +76,11 @@ class ProcessConfig(ProcessConfigBase):
         Update the 'dataset_info' (DatasetAnalyzer class instance) data structure with options from user configuration.
         """
         self.logger.debug("Updating dataset information with user configuration")
-        if self.dataset_info.kind == "nx":
+        if self.dataset_info.kind == "nx" and self.nabu_config["preproc"]["flatfield"]:
             update_dataset_info_flats_darks(
                 self.dataset_info,
                 self.nabu_config["preproc"]["flatfield"],
+                loading_mode=self.nabu_config["preproc"]["flatfield_loading_mode"],
                 output_dir=self.nabu_config["output"]["location"],
                 darks_flats_dir=self.nabu_config["dataset"]["darks_flats_dir"],
             )
@@ -89,12 +93,29 @@ class ProcessConfig(ProcessConfigBase):
         self.subsampling_factor = subsampling_factor or 1
         self.subsampling_start = subsampling_start or 0
 
+        self._get_double_flatfield()
         self._update_dataset_with_user_overwrites()
         self._get_rotation_axis_position()
         self._update_rotation_angles()
         self._get_translation_file("reconstruction", "translation_movements_file", "translations")
         self._get_user_sino_normalization()
 
+    def _get_double_flatfield(self):
+        self._dff_file = None
+        dff_mode = self.nabu_config["preproc"]["double_flatfield"]
+        if not (dff_mode):
+            return
+        self._dff_file = get_double_flatfield(
+            self.dataset_info,
+            dff_mode,
+            output_dir=self.nabu_config["output"]["location"],
+            darks_flats_dir=self.nabu_config["dataset"]["darks_flats_dir"],
+            dff_options={
+                "dff_sigma": self.nabu_config["preproc"]["dff_sigma"],
+                "do_flatfield": (self.nabu_config["preproc"]["flatfield"] is not False),
+            },
+        )
+
     def _update_dataset_with_user_overwrites(self):
         user_overwrites = self.nabu_config["dataset"]["overwrite_metadata"].strip()
         if user_overwrites in ("", None):
@@ -406,8 +427,10 @@ class ProcessConfig(ProcessConfigBase):
         # Flat-field
         #
         if nabu_config["preproc"]["flatfield"]:
+            ff_method = "pca" if nabu_config["preproc"]["flatfield"] == "pca" else "default"
             tasks.append("flatfield")
             options["flatfield"] = {
+                "method": ff_method,
                 #  Data reader handles binning/subsampling by itself,
                 # but FlatField needs "real" indices (after binning/subsampling)
                 "projs_indices": self.projs_indices(subsampling=False),
@@ -415,7 +438,7 @@ class ProcessConfig(ProcessConfigBase):
                 "do_flat_distortion": nabu_config["preproc"]["flat_distortion_correction_enabled"],
                 "flat_distortion_params": extract_parameters(nabu_config["preproc"]["flat_distortion_params"]),
             }
-            normalize_srcurrent = nabu_config["preproc"]["normalize_srcurrent"]
+            normalize_srcurrent = nabu_config["preproc"]["normalize_srcurrent"] and ff_method == "default"
             radios_srcurrent = None
             flats_srcurrent = None
             if normalize_srcurrent:
@@ -439,6 +462,7 @@ class ProcessConfig(ProcessConfigBase):
             if len(dataset_info.darks) > 1:
                 self.logger.warning("Cannot do flat-field with more than one reduced dark. Taking the first one.")
                 dataset_info.darks = dataset_info.darks[sorted(dataset_info.darks.keys())[0]]
+
         #
         # Spikes filter
         #
@@ -451,11 +475,19 @@ class ProcessConfig(ProcessConfigBase):
         #
         # Double flat field
         #
-        if nabu_config["preproc"]["double_flatfield_enabled"]:
+        # ---- COMPAT  ----
+        if convert_to_bool(nabu_config["preproc"].get("double_flatfield_enabled", False))[0]:
+            deprecation_warning(
+                "'double_flatfield_enabled' has been renamed to 'double_flatfield'. Please update your configuration file"
+            )
+            nabu_config["preproc"]["double_flatfield"] = True
+
+        # -------------
+        if nabu_config["preproc"]["double_flatfield"]:
             tasks.append("double_flatfield")
             options["double_flatfield"] = {
                 "sigma": nabu_config["preproc"]["dff_sigma"],
-                "processes_file": nabu_config["preproc"]["processes_file"],
+                "processes_file": self._dff_file or nabu_config["preproc"]["processes_file"],
                 "log_min_clip": nabu_config["preproc"]["log_min_clip"],
                 "log_max_clip": nabu_config["preproc"]["log_max_clip"],
             }
@@ -558,6 +590,7 @@ class ProcessConfig(ProcessConfigBase):
                 self.rec_params,
                 [
                     "method",
+                    "iterations",
                     "implementation",
                     "fbp_filter_type",
                     "fbp_filter_cutoff",
@@ -575,6 +608,7 @@ class ProcessConfig(ProcessConfigBase):
                     "sample_detector_dist",
                     "hbp_legs",
                     "hbp_reduction_steps",
+                    "crop_filtered_data",
                 ],
             )
             rec_options = options["reconstruction"]
@@ -593,8 +627,6 @@ class ProcessConfig(ProcessConfigBase):
                 voxel_size,
             )  # pix size is in microns in dataset_info
 
-            rec_options["iterations"] = nabu_config["reconstruction"]["iterations"]
-
             # x/y/z position information
             def get_mean_pos(position_array):
                 if position_array is None:
@@ -616,6 +648,8 @@ class ProcessConfig(ProcessConfigBase):
                 rec_options["position"] = mean_positions_xyz
             if rec_options["method"] == "cone" and rec_options["sample_detector_dist"] is None:
                 rec_options["sample_detector_dist"] = self.dataset_info.distance  # was checked to be not None earlier
+            if rec_options["method"].lower() == "mlem" and rec_options["implementation"] in [None, ""]:
+                rec_options["implementation"] = "corrct"
 
             # New key
             rec_options["cor_estimated_auto"] = isinstance(nabu_config["reconstruction"]["rotation_axis_position"], str)

nabu/pipeline/fullfield/reconstruction.py

@@ -7,7 +7,7 @@ from silx.io import get_data
 from silx.io.url import DataUrl
 from tomoscan.esrf.volume.singleframebase import VolumeSingleFrameBase
 from ... import version as nabu_version
-from ...utils import check_supported, subdivide_into_overlapping_segment
+from ...utils import check_supported, first_generator_item, subdivide_into_overlapping_segment
 from ...resources.logger import LoggerOrPrint
 from ...resources.utils import is_hdf5_extension
 from ...io.writer import merge_hdf5_files, NXProcessWriter
@@ -61,7 +61,7 @@ class FullFieldReconstructor:
             Dictionary with cuda options passed to `nabu.cuda.processing.CudaProcessing`
 
 
-        Other parameters
+        Other Parameters
         -----------------
         Advanced options can be passed in the 'extra_options' dictionary. These can be:
 
@@ -120,7 +120,7 @@ class FullFieldReconstructor:
         vm = virtual_memory()
         self.resources["mem_avail_GB"] = vm.available / 1e9
         # Account for other memory constraints. There might be a better way
-        slurm_mem_constraint_MB = int(environ.get("SLURM_MEM_PER_NODE", 0))
+        slurm_mem_constraint_MB = int(environ.get("SLURM_MEM_PER_NODE", 0))  # noqa: PLW1508
         if slurm_mem_constraint_MB > 0:
             self.resources["mem_avail_GB"] = slurm_mem_constraint_MB / 1e3
         #
@@ -261,6 +261,9 @@ class FullFieldReconstructor:
         if (self.process_config.dataset_info.detector_tilt or 0) > 15:
             force_grouped_mode = True
             msg = "Radios rotation with a large angle needs to process full radios"
+        if self.process_config.processing_options.get("flatfield", {}).get("method", "default") == "pca":
+            force_grouped_mode = True
+            msg = "PCA-Flatfield normalization needs to process full radios"
         if self.process_config.resume_from_step == "sinogram" and force_grouped_mode:
             self.logger.warning("Cannot use grouped-radios processing when resuming from sinogram")
             force_grouped_mode = False
@@ -312,7 +315,7 @@ class FullFieldReconstructor:
         sigma = opts["unsharp_sigma"]
         # nabu uses cutoff = 4
         cutoff = 4
-        gaussian_kernel_size = int(ceil(2 * cutoff * sigma + 1))
+        gaussian_kernel_size = ceil(2 * cutoff * sigma + 1)
         self.logger.debug("Unsharp mask margin: %d pixels" % gaussian_kernel_size)
         return (gaussian_kernel_size, gaussian_kernel_size)
 
@@ -360,8 +363,8 @@ class FullFieldReconstructor:
         d2 = rec_cfg["sample_detector_dist"]
         n_z, _ = self.process_config.radio_shape(binning=True)
 
-        delta_z = self.process_config.rec_delta_z  # accounts_for_binning
-        overlap = ceil(delta_z * d2 / (d1 + d2))  # sqrt(2) missing ?
+        # delta_z = self.process_config.rec_delta_z  # accounts_for_binning
+        # overlap = ceil(delta_z * d2 / (d1 + d2))  # sqrt(2) missing ?
 
         max_overlap = ceil(n_z * d2 / (d1 + d2))  # sqrt(2) missing ?
 
@@ -483,7 +486,7 @@ class FullFieldReconstructor:
                 {
                     "sub_region": (
                         (0, self.n_angles),
-                        (curr_z_min - margin_up, curr_z_max + margin_down),
+                        (int(curr_z_min - margin_up), int(curr_z_max + margin_down)),
                         (0, self.chunk_shape[-1]),
                     ),
                     "margin": ((margin_up, margin_down), (0, 0)),
@@ -690,7 +693,7 @@ class FullFieldReconstructor:
         # Prevent issue when out_dir is empty, which happens only if dataset/location is a relative path.
         # TODO this should be prevented earlier
         if out_dir is None or len(out_dir.strip()) == 0:
-            out_dir = dirname(dirname(self.results[list(self.results.keys())[0]]))
+            out_dir = dirname(dirname(self.results[first_generator_item(self.results.keys())]))
         #
         if output_file is None:
             output_file = join(out_dir, prefix + out_cfg["file_prefix"]) + ".hdf5"
@@ -771,7 +774,8 @@ class FullFieldReconstructor:
         out_cfg = self.process_config.nabu_config["output"]
         if output_file is None:
             output_file = (
-                join(dirname(list(self._histograms.values())[0]), out_cfg["file_prefix"] + "_histogram") + ".hdf5"
+                join(dirname(first_generator_item(self._histograms.values())), out_cfg["file_prefix"] + "_histogram")
+                + ".hdf5"
             )
         local_files = self.get_relative_files(files=list(self._histograms.values()))
         #
@@ -823,7 +827,7 @@ class FullFieldReconstructor:
     def merge_data_dumps(self, axis=1):
         # Collect in a dict where keys are step names (instead of task keys)
         dumps = {}
-        for task_key, data_dumps in self._data_dumps.items():
+        for task_key, data_dumps in self._data_dumps.items():  # noqa: PERF102
             for step_name, fname in data_dumps.items():
                 fname = join(basename(dirname(fname)), basename(fname))
                 if step_name not in dumps:

nabu/pipeline/helical/dataset_validator.py

@@ -1,4 +1,4 @@
-from ..fullfield.dataset_validator import *
+from ..fullfield.dataset_validator import FullFieldDatasetValidator
 from ...utils import copy_dict_items
 
 
@@ -10,10 +10,9 @@ class HelicalDatasetValidator(FullFieldDatasetValidator):
 
     def _check_slice_indices(self):
         """Slice indices can be far beyond what fullfield pipeline accepts, no check here, but
-        Nabu expects that rec_region is initialised here"""
+        Nabu expects that rec_region is initialised here
+        """
 
         what = ["start_x", "end_x", "start_y", "end_y", "start_z", "end_z"]
 
         self.rec_region = copy_dict_items(self.rec_params, what)
-
-        return

nabu/pipeline/helical/fbp.py

@@ -1,6 +1,6 @@
-from ...reconstruction.fbp import *
+import numpy as np
+from ...reconstruction.fbp import Backprojector
 from .filtering import HelicalSinoFilter
-from ...utils import convert_index
 
 
 class BackprojectorHelical(Backprojector):
@@ -25,9 +25,9 @@ class BackprojectorHelical(Backprojector):
 
     def set_custom_angles_and_axis_corrections(self, angles_rad, x_per_proj):
         """To arbitrarily change angles
-        Parameters
-        ==========
 
+        Parameters
+        ----------
         angles_rad: array of floats
            one angle per each projection in radians
 

nabu/pipeline/helical/filtering.py

@@ -68,26 +68,26 @@ class HelicalSinoFilter(CudaSinoFilter):
             "padding",
             filename=get_cuda_srcfile("helical_padding.cu"),
             signature="PPfiiiii",
-            options=[str("-DMIRROR_EDGES")],
+            options=["-DMIRROR_EDGES"],
         )
         self._pad_mirror_constant_kernel = self.cuda.kernel(
             "padding",
             filename=get_cuda_srcfile("helical_padding.cu"),
             signature="PPfiiiiiff",
-            options=[str("-DMIRROR_CONSTANT")],
+            options=["-DMIRROR_CONSTANT"],
         )
 
         self._pad_mirror_edges_variable_rot_pos_kernel = self.cuda.kernel(
             "padding",
             filename=get_cuda_srcfile("helical_padding.cu"),
             signature="PPPiiiii",
-            options=[str("-DMIRROR_EDGES_VARIABLE_ROT_POS")],
+            options=["-DMIRROR_EDGES_VARIABLE_ROT_POS"],
         )
         self._pad_mirror_constant_variable_rot_pos_kernel = self.cuda.kernel(
             "padding",
             filename=get_cuda_srcfile("helical_padding.cu"),
             signature="PPPiiiiiff",
-            options=[str("-DMIRROR_CONSTANT_VARIABLE_ROT_POS")],
+            options=["-DMIRROR_CONSTANT_VARIABLE_ROT_POS"],
         )
 
         self.d_mirror_indexes = self.cuda.allocate_array(
@@ -179,6 +179,7 @@ class HelicalSinoFilter(CudaSinoFilter):
         """
         Perform the sinogram siltering.
         redefined here to use also mirror data
+
         Parameters
         ----------
         sino: numpy.ndarray or pycuda.gpuarray.GPUArray