nabu 2024.1.1__py3-none-any.whl → 2024.1.3__py3-none-any.whl

build/lib/nabu/pipeline/helical/helical_chunked_regridded.py

@@ -1,1670 +0,0 @@
-# pylint: skip-file
-
-from os import path
-import numpy as np
-import math
-from silx.image.tomography import get_next_power
-from scipy import ndimage as nd
-import h5py
-import silx.io
-import copy
-from silx.io.url import DataUrl
-from ...resources.logger import LoggerOrPrint
-from ...resources.utils import is_hdf5_extension
-from ...io.reader_helical import ChunkReaderHelical, get_hdf5_dataset_shape
-from ...preproc.flatfield_variable_region import FlatFieldDataVariableRegionUrls as FlatFieldDataHelicalUrls
-from ...preproc.distortion import DistortionCorrection
-from ...preproc.shift import VerticalShift
-from ...preproc.double_flatfield_variable_region import DoubleFlatFieldVariableRegion as DoubleFlatFieldHelical
-from ...preproc.phase import PaganinPhaseRetrieval
-from ...reconstruction.sinogram import SinoBuilder
-from ...processing.unsharp import UnsharpMask
-from ...processing.histogram import PartialHistogram, hist_as_2Darray
-from ..utils import use_options, pipeline_step
-
-from ..detector_distortion_provider import DetectorDistortionProvider
-
-from .utils import (
-    WriterConfiguratorHelical as WriterConfigurator,
-)  # .utils is the same as ..utils but internally we retouch the key associated to "tiffwriter" of Writers to
-
-# point to our class which can write tiff with names indexed by the z height above the sample stage in millimiters
-
-from numpy.lib.stride_tricks import sliding_window_view
-from ...misc.binning import get_binning_function
-from .helical_utils import find_mirror_indexes
-
-
-try:
-    import nabuxx
-
-    GriddedAccumulator = nabuxx.gridded_accumulator.GriddedAccumulator
-    CCDFilter = nabuxx.ccd.CCDFilter
-    Log = nabuxx.ccd.LogFilter
-    cxx_paganin = nabuxx.paganin
-except:
-    logger_tmp = LoggerOrPrint(None)
-    logger_tmp.info(
-        "Nabuxx not available. Loading python implementation for gridded_accumulator, Log, CCDFilter, paganin"
-    )
-    from . import gridded_accumulator
-
-    GriddedAccumulator = gridded_accumulator.GriddedAccumulator
-    from ...preproc.ccd import Log, CCDFilter
-
-    cxx_paganin = None
-
-
-# For now we don't have a plain python/numpy backend for reconstruction
-Backprojector = None
-
-
-class HelicalChunkedRegriddedPipeline:
-    """
-    Pipeline for "helical" full or half field tomography.
-    Data is processed by chunks. A chunk consists in K+-1 contiguous lines of all the radios
-    which are read at variable height following the translations
-    """
-
-    extra_marge_granularity = 4
-    """ This offers extra reading space to be able to read the redundant part 
-    which might be sligtly larger and or require extra border for interpolation
-    """
-
-    FlatFieldClass = FlatFieldDataHelicalUrls
-    DoubleFlatFieldClass = DoubleFlatFieldHelical
-
-    CCDFilterClass = CCDFilter
-    MLogClass = Log
-
-    PaganinPhaseRetrievalClass = PaganinPhaseRetrieval
-    UnsharpMaskClass = UnsharpMask
-    VerticalShiftClass = VerticalShift
-    SinoBuilderClass = SinoBuilder
-
-    FBPClass = Backprojector
-    HBPClass = None
-    HistogramClass = PartialHistogram
-    regular_accumulator = None
-
-    def __init__(
-        self,
-        process_config,
-        sub_region,
-        logger=None,
-        extra_options=None,
-        phase_margin=None,
-        reading_granularity=10,
-        span_info=None,
-        diag_zpro_run=0,
-    ):
-        """
-        Initialize a "HelicalChunked" pipeline.
-
-        Parameters
-        ----------
-        process_config: `nabu.resources.processcinfig.ProcessConfig`
-            Process configuration.
-        sub_region: tuple
-            Sub-region to process in the volume for this worker, in the format
-            `(start_x, end_x, start_z, end_z)`.
-        logger: `nabu.app.logger.Logger`, optional
-            Logger class
-        extra_options: dict, optional
-            Advanced extra options.
-        phase_margin: tuple, optional
-            Margin to use when performing phase retrieval, in the form ((up, down), (left, right)).
-            See also the documentation of PaganinPhaseRetrieval.
-            If not provided, no margin is applied.
-        reading_granularity: int
-            The data angular span which needs to be read for a reconstruction is read step by step,
-            reading each time a maximum of reading_granularity radios, and doing the preprocessing
-            till phase retrieval for each of these  angular groups
-
-        Notes
-        ------
-        Using a `phase_margin` results in a lesser number of reconstructed slices.
-        More specifically, if `phase_margin = (V, H)`, then there will be `chunk_size - 2*V`
-        reconstructed slices (if the sub-region is in the middle of the volume)
-        or `chunk_size - V` reconstructed slices (if the sub-region is on top or bottom
-        of the volume).
-        """
-
-        self.span_info = span_info
-        self.reading_granularity = reading_granularity
-
-        self.logger = LoggerOrPrint(logger)
-
-        self._set_params(process_config, sub_region, extra_options, phase_margin, diag_zpro_run)
-
-        self._init_pipeline()
-
-    def _set_params(self, process_config, sub_region, extra_options, phase_margin, diag_zpro_run):
-        self.diag_zpro_run = diag_zpro_run
-        self.process_config = process_config
-        self.dataset_info = self.process_config.dataset_info
-
-        self.processing_steps = self.process_config.processing_steps.copy()
-        self.processing_options = self.process_config.processing_options
-
-        sub_region = self._check_subregion(sub_region)
-
-        self.chunk_size = sub_region[-1] - sub_region[-2]
-        self.radios_buffer = None
-
-        self._set_detector_distortion_correction()
-
-        self.set_subregion(sub_region)
-
-        self._set_phase_margin(phase_margin)
-        self._set_extra_options(extra_options)
-        self._callbacks = {}
-        self._steps_name2component = {}
-        self._steps_component2name = {}
-        self._data_dump = {}
-        self._resume_from_step = None
-
-    @staticmethod
-    def _check_subregion(sub_region):
-        if len(sub_region) < 4:
-            assert len(sub_region) == 2, " at least start_z and end_z are required in subregion"
-            sub_region = (None, None) + sub_region
-        if None in sub_region[-2:]:
-            raise ValueError("Cannot set z_min or z_max to None")
-        return sub_region
-
-    def _set_extra_options(self, extra_options):
-        if extra_options is None:
-            extra_options = {}
-        advanced_options = {}
-        advanced_options.update(extra_options)
-        self.extra_options = advanced_options
-
-    def _set_phase_margin(self, phase_margin):
-        if phase_margin is None:
-            phase_margin = ((0, 0), (0, 0))
-        self._phase_margin_up = phase_margin[0][0]
-        self._phase_margin_down = phase_margin[0][1]
-        self._phase_margin_left = phase_margin[1][0]
-        self._phase_margin_right = phase_margin[1][1]
-
-    def set_subregion(self, sub_region):
-        """
-        Set a sub-region to process.
-
-        Parameters
-        ----------
-        sub_region: tuple
-            Sub-region to process in the volume, in the format
-            `(start_x, end_x, start_z, end_z)` or `(start_z, end_z)`.
-        """
-        sub_region = self._check_subregion(sub_region)
-        dz = sub_region[-1] - sub_region[-2]
-        if dz != self.chunk_size:
-            raise ValueError(
-                "Class was initialized for chunk_size = %d but provided sub_region has chunk_size = %d"
-                % (self.chunk_size, dz)
-            )
-        self.sub_region = sub_region
-        self.z_min = sub_region[-2]
-        self.z_max = sub_region[-1]
-
-    def _compute_phase_kernel_margin(self):
-        """
-        Get the "margin" to pass to classes like PaganinPhaseRetrieval.
-        In order to have a good accuracy for filter-based phase retrieval methods,
-        we need to load extra data around the edges of each image. Otherwise,
-        a default padding type is applied.
-        """
-        if not (self.use_radio_processing_margin):
-            self._phase_margin = None
-            return
-        up_margin = self._phase_margin_up
-        down_margin = self._phase_margin_down
-        # Horizontal margin is not implemented
-        left_margin, right_margin = (0, 0)
-        self._phase_margin = ((up_margin, down_margin), (left_margin, right_margin))
-
-    @property
-    def use_radio_processing_margin(self):
-        return ("phase" in self.processing_steps) or ("unsharp_mask" in self.processing_steps)
-
-    def _get_phase_margin(self):
-        if not (self.use_radio_processing_margin):
-            return ((0, 0), (0, 0))
-        return self._phase_margin
-
-    @property
-    def phase_margin(self):
-        """
-        Return the margin for phase retrieval in the form ((up, down), (left, right))
-        """
-        return self._get_phase_margin()
-
-    def _get_process_name(self, kind="reconstruction"):
-        # In the future, might be something like "reconstruction-<ID>"
-        if kind == "reconstruction":
-            return "reconstruction"
-        elif kind == "histogram":
-            return "histogram"
-        return kind
-
-    def _configure_dump(self, step_name):
-        if step_name not in self.processing_steps:
-            if step_name == "sinogram" and self.process_config._dump_sinogram:
-                fname_full = self.process_config._dump_sinogram_file
-            else:
-                return
-        else:
-            if not self.processing_options[step_name].get("save", False):
-                return
-            fname_full = self.processing_options[step_name]["save_steps_file"]
-
-        fname, ext = path.splitext(fname_full)
-        dirname, file_prefix = path.split(fname)
-        output_dir = path.join(dirname, file_prefix)
-        file_prefix += str("_%06d" % self._get_image_start_index())
-
-        self.logger.info("omitting config in  data_dump because of too slow nexus writer ")
-
-        self._data_dump[step_name] = WriterConfigurator(
-            output_dir,
-            file_prefix,
-            file_format="hdf5",
-            overwrite=True,
-            logger=self.logger,
-            nx_info={
-                "process_name": step_name,
-                "processing_index": 0,  # TODO
-                #                "config": {"processing_options": self.processing_options, "nabu_config": self.process_config.nabu_config},
-                "config": None,
-                "entry": getattr(self.dataset_info.dataset_scanner, "entry", None),
-            },
-        )
-
-    def _configure_data_dumps(self):
-        self.process_config._configure_save_steps()
-        for step_name in self.processing_steps:
-            self._configure_dump(step_name)
-        # sinogram is a special keyword: not in processing_steps, but guaranteed to be before sinogram generation
-
-        if self.process_config._dump_sinogram:
-            self._configure_dump("sinogram")
-
-    #
-    # Callbacks
-    #
-
-    def register_callback(self, step_name, callback):
-        """
-        Register a callback for a pipeline processing step.
-
-        Parameters
-        ----------
-        step_name: str
-            processing step name
-        callback: callable
-            A function. It will be executed once the processing step `step_name`
-            is finished. The function takes only one argument: the class instance.
-        """
-        if step_name not in self.processing_steps:
-            raise ValueError("'%s' is not in processing steps %s" % (step_name, self.processing_steps))
-        if step_name in self._callbacks:
-            self._callbacks[step_name].append(callback)
-        else:
-            self._callbacks[step_name] = [callback]
-
-    #
-    # Overwritten in inheriting classes
-    #
-
-    def _get_shape(self, step_name):
-        """
-        Get the shape to provide to the class corresponding to step_name.
-        """
-        if step_name == "flatfield":
-            shape = self.radios_subset.shape
-        elif step_name == "double_flatfield":
-            shape = self.radios_subset.shape
-        elif step_name == "phase":
-            shape = self.radios_subset.shape[1:]
-        elif step_name == "ccd_correction":
-            shape = self.gridded_radios.shape[1:]
-        elif step_name == "unsharp_mask":
-            shape = self.radios_subset.shape[1:]
-        elif step_name == "take_log":
-            shape = self.radios.shape
-        elif step_name == "radios_movements":
-            shape = self.radios.shape
-        elif step_name == "sino_normalization":
-            shape = self.radios.shape
-        elif step_name == "sino_normalization_slim":
-            shape = self.radios.shape[:1] + (1,) + self.radios.shape[2:]
-        elif step_name == "one_sino_slim":
-            shape = self.radios.shape[:1] + self.radios.shape[2:]
-        elif step_name == "build_sino":
-            shape = self.radios.shape[:1] + (1,) + self.radios.shape[2:]
-        elif step_name == "reconstruction":
-            shape = self.sino_builder.output_shape[1:]
-        else:
-            raise ValueError("Unknown processing step %s" % step_name)
-        self.logger.debug("Data shape for %s is %s" % (step_name, str(shape)))
-        return shape
-
-    def _allocate_array(self, shape, dtype, name=None):
-        """this function can be redefined in the  derived class which is dedicated to gpu
-        and will return gpu garrays
-        """
-        return _cpu_allocate_array(shape, dtype, name=name)
-
-    def _cpu_allocate_array(self, shape, dtype, name=None):
-        """For objects used in the pre-gpu part. They will be always on CPU even in the derived class"""
-        result = np.zeros(shape, dtype=dtype)
-        return result
-
-    def _allocate_sinobuilder_output(self):
-        return self._cpu_allocate_array(self.sino_builder.output_shape, "f", name="sinos")
-
-    def _allocate_recs(self, ny, nx):
-        self.n_slices = self.gridded_radios.shape[1]
-        if self.use_radio_processing_margin:
-            self.n_slices -= sum(self.phase_margin[0])
-        self.recs = self._allocate_array((1, ny, nx), "f", name="recs")
-        self.recs_stack = self._cpu_allocate_array((self.n_slices, ny, nx), "f", name="recs_stack")
-
-    def _reset_memory(self):
-        pass
-
-    def _get_read_dump_subregion(self):
-        read_opts = self.processing_options["read_chunk"]
-        if read_opts.get("process_file", None) is None:
-            return None
-        dump_start_z, dump_end_z = read_opts["dump_start_z"], read_opts["dump_end_z"]
-        relative_start_z = self.z_min - dump_start_z
-        relative_end_z = relative_start_z + self.chunk_size
-        # (n_angles, n_z, n_x)
-        subregion = (None, None, relative_start_z, relative_end_z, None, None)
-        return subregion
-
-    def _check_resume_from_step(self):
-        if self._resume_from_step is None:
-            return
-        read_opts = self.processing_options["read_chunk"]
-        expected_radios_shape = get_hdf5_dataset_shape(
-            read_opts["process_file"],
-            read_opts["process_h5_path"],
-            sub_region=self._get_read_dump_subregion(),
-        )
-        # TODO check
-
-    def _init_reader_finalize(self):
-        """
-        Method called after _init_reader
-        """
-        self._check_resume_from_step()
-
-        self._compute_phase_kernel_margin()
-
-        self._allocate_reduced_gridded_and_subset_radios()
-
-    def _allocate_reduced_gridded_and_subset_radios(self):
-        shp_h = self.chunk_reader.data.shape[-1]
-
-        sliding_window_size = self.chunk_size
-        if sliding_window_size % 2 == 0:
-            sliding_window_size += 1
-        sliding_window_radius = (sliding_window_size - 1) // 2
-
-        if sliding_window_radius == 0:
-            n_projs_max = (self.span_info.sunshine_ends - self.span_info.sunshine_starts).max()
-        else:
-            padded_starts = self.span_info.sunshine_starts
-            padded_ends = self.span_info.sunshine_ends
-
-            padded_starts = np.concatenate(
-                [[padded_starts[0]] * sliding_window_radius, padded_starts, [padded_starts[-1]] * sliding_window_radius]
-            )
-            starts = sliding_window_view(padded_starts, sliding_window_size).min(axis=-1)
-
-            padded_ends = np.concatenate(
-                [[padded_ends[0]] * sliding_window_radius, padded_ends, [padded_ends[-1]] * sliding_window_radius]
-            )
-
-            ends = sliding_window_view(padded_ends, sliding_window_size).max(axis=-1)
-
-            n_projs_max = (ends - starts).max()
-
-        ((up_margin, down_margin), (left_margin, right_margin)) = self.phase_margin
-
-        (start_x, end_x, start_z, end_z) = self.sub_region
-
-        ## and now the gridded ones
-
-        my_angle_step = abs(np.diff(self.span_info.projection_angles_deg).mean())
-        self.n_gridded_angles = int(round(360.0 / my_angle_step))
-
-        self.my_angles_rad = np.arange(self.n_gridded_angles) * 2 * np.pi / self.n_gridded_angles
-
-        my_angles_deg = np.rad2deg(self.my_angles_rad)
-
-        self.mirror_angle_relative_indexes = find_mirror_indexes(my_angles_deg)
-
-        if "read_chunk" not in self.processing_steps:
-            raise ValueError("Cannot proceed without reading data")
-
-        r_shp_v, r_shp_h = self.whole_radio_shape
-
-        (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
-
-        subradio_shape = subr_end_z - subr_start_z, r_shp_h
-
-        ### these radios are for diagnostic of the translations ( they will be optionally written, for being further used
-        ##  by correlation techniques ). Two radios for the first two pass over the first gridded angles
-        if self.diag_zpro_run:
-            # 2 for the redundancy, 2 for +180 mirror
-            ndiag = 2 * 2 * self.diag_zpro_run
-        else:
-            ndiag = 2 * 2
-
-        self.diagnostic_searched_angles_rad_clipped = (
-            (0.5 + np.arange(ndiag // 2)) * (2 * np.pi / (ndiag // 2))
-        ).astype("f")
-
-        self.diagnostic_radios = np.zeros((ndiag,) + subradio_shape, np.float32)
-        self.diagnostic_weights = np.zeros((ndiag,) + subradio_shape, np.float32)
-        self.diagnostic_proj_angle = np.zeros([ndiag], "f")
-        self.diagnostic_zpix_transl = np.zeros([ndiag], "f")
-        self.diagnostic_zmm_transl = np.zeros([ndiag], "f")
-
-        self.diagnostic = {
-            "radios": self.diagnostic_radios,
-            "weights": self.diagnostic_weights,
-            "angles": self.diagnostic_proj_angle,
-            "zpix_transl": self.diagnostic_zpix_transl,
-            "zmm_trans": self.diagnostic_zmm_transl,
-            "pixel_size_mm": self.span_info.pix_size_mm,
-            "searched_rad": self.diagnostic_searched_angles_rad_clipped,
-        }
-        ## -------
-        if self.diag_zpro_run == 0:
-            self.gridded_radios = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
-            self.gridded_cumulated_weights = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
-        else:
-            # only diagnostic will be cumulated. No need to keep the full size for diagnostic runs.
-            # The gridder is initialised passing also the two buffer below,
-            # and the two first dimensions are used to allocate auxiliaries,
-            # so we shorten only the last dimension, but this is already a good cut
-            self.gridded_radios = np.zeros((self.n_gridded_angles,) + (subradio_shape[0], 2), np.float32)
-            self.gridded_cumulated_weights = np.zeros((self.n_gridded_angles,) + (subradio_shape[0], 2), np.float32)
-
-        self.radios_subset = np.zeros((self.reading_granularity,) + subradio_shape, np.float32)
-        self.radios_weights_subset = np.zeros((self.reading_granularity,) + subradio_shape, np.float32)
-
-        if not self.diag_zpro_run:
-            self.radios = np.zeros(
-                (self.n_gridded_angles,) + ((end_z - down_margin) - (start_z + up_margin), shp_h), np.float32
-            )
-        else:
-            # place holder
-            self.radios = np.zeros((self.n_gridded_angles,) + (1, 1), np.float32)
-
-        self.radios_weights = np.zeros_like(self.radios)
-
-        self.radios_slim = self._allocate_array(self._get_shape("one_sino_slim"), "f", name="radios_slim")
-
-    def _process_finalize(self):
-        """
-        Method called once the pipeline has been executed
-        """
-        pass
-
-    def _get_slice_start_index(self):
-        return self.z_min + self._phase_margin_up
-
-    _get_image_start_index = _get_slice_start_index
-
-    #
-    # Pipeline initialization
-    #
-
-    def _reset_diagnostics(self):
-        self.diagnostic_radios[:] = 0
-        self.diagnostic_weights[:] = 0
-        self.diagnostic_zpix_transl[:] = 0
-        self.diagnostic_zmm_transl[:] = 0
-        self.diagnostic_proj_angle[:] = np.nan
-
-    def _init_pipeline(self):
-        self._get_size_of_a_raw_radio()
-
-        self._init_reader()
-
-        self._init_flatfield()
-
-        self._init_double_flatfield()
-        self._init_weights_field()
-
-        self._init_ccd_corrections()
-        self._init_phase()
-        self._init_unsharp()
-        self._init_mlog()
-        self._init_sino_normalization()
-        self._init_sino_builder()
-        self._prepare_reconstruction()
-        self._init_reconstruction()
-        self._init_histogram()
-        self._init_writer()
-        self._configure_data_dumps()
-
-        self._configure_regular_accumulator()
-
-    def _set_detector_distortion_correction(self):
-        if self.process_config.nabu_config["preproc"]["detector_distortion_correction"] is None:
-            self.detector_corrector = None
-        else:
-            self.detector_corrector = DetectorDistortionProvider(
-                detector_full_shape_vh=self.process_config.dataset_info.radio_dims[::-1],
-                correction_type=self.process_config.nabu_config["preproc"]["detector_distortion_correction"],
-                options=self.process_config.nabu_config["preproc"]["detector_distortion_correction_options"],
-            )
-
-    def _configure_regular_accumulator(self):
-        ##
-        # keeping these freshly numpyed objects referenced by self
-        # ensures that their buffer info, conserved by c++ implementation of  GriddedAccumulator
-        # will always point to existing data, which could otherwise be garbage collected by python
-        #
-
-        if self.process_config.nabu_config["preproc"]["normalize_srcurrent"]:
-            self.radios_srcurrent = np.array(self.dataset_info.projections_srcurrent, "f")
-            self.flats_srcurrent = np.array(self.dataset_info.flats_srcurrent, "f")
-        else:
-            self.radios_srcurrent = None
-            self.flats_srcurrent = None
-
-        self.regular_accumulator = GriddedAccumulator(
-            gridded_radios=self.gridded_radios,
-            gridded_weights=self.gridded_cumulated_weights,
-            diagnostic_radios=self.diagnostic_radios,
-            diagnostic_weights=self.diagnostic_weights,
-            diagnostic_angles=self.diagnostic_proj_angle,
-            diagnostic_zpix_transl=self.diagnostic_zpix_transl,
-            diagnostic_searched_angles_rad_clipped=self.diagnostic_searched_angles_rad_clipped,
-            dark=self.flatfield.get_dark(),
-            flat_indexes=self.flatfield._sorted_flat_indices,
-            flats=self.flatfield.flats_stack,
-            weights=self.weights_field.data,
-            double_flat=self.double_flatfield.data,
-            diag_zpro_run=self.diag_zpro_run,
-            radios_srcurrent=self.radios_srcurrent,
-            flats_srcurrent=self.flats_srcurrent,
-        )
-
-        return
-
-    def _get_size_of_a_raw_radio(self):
-        """Once for all we find the shape of a radio.
-        This information will be used in other parts of the code when allocating
-        bunch of data holders
-        """
-        if "read_chunk" not in self.processing_steps:
-            raise ValueError("Cannot proceed without reading data")
-
-        options = self.processing_options["read_chunk"]
-
-        here_a_file = next(iter(options["files"].values()))
-        here_a_radio = silx.io.get_data(here_a_file)
-
-        binning_x, binning_z = self._get_binning()
-        if (binning_z, binning_x) != (1, 1):
-            binning_function = get_binning_function((binning_z, binning_x))
-            here_a_radio = binning_function(here_a_radio)
-
-        self.whole_radio_shape = here_a_radio.shape
-        return self.whole_radio_shape
-
-    @use_options("read_chunk", "chunk_reader")
-    def _init_reader(self):
-        if "read_chunk" not in self.processing_steps:
-            raise ValueError("Cannot proceed without reading data")
-        options = self.processing_options["read_chunk"]
-
-        assert options.get("process_file", None) is None, "Resume not yet implemented in helical pipeline"
-
-        # dummy initialisation, it will be _set_subregion'ed and set_data_buffer'ed in the loops
-        self.chunk_reader = ChunkReaderHelical(
-            options["files"],
-            sub_region=None,  # setting of subregion will be already done by calls to set_subregion
-            detector_corrector=self.detector_corrector,
-            convert_float=True,
-            binning=options["binning"],
-            dataset_subsampling=options["dataset_subsampling"],
-            data_buffer=None,
-            pre_allocate=True,
-        )
-
-        self._init_reader_finalize()
-
-    @use_options("flatfield", "flatfield")
-    def _init_flatfield(self, shape=None):
-        if shape is None:
-            shape = self._get_shape("flatfield")
-        options = self.processing_options["flatfield"]
-
-        distortion_correction = None
-        if options["do_flat_distortion"]:
-            self.logger.info("Flats distortion correction will be applied")
-            estimation_kwargs = {}
-            estimation_kwargs.update(options["flat_distortion_params"])
-            estimation_kwargs["logger"] = self.logger
-            distortion_correction = DistortionCorrection(
-                estimation_method="fft-correlation", estimation_kwargs=estimation_kwargs, correction_method="interpn"
-            )
-
-        self.flatfield = self.FlatFieldClass(
-            shape,
-            flats=self.dataset_info.flats,
-            darks=self.dataset_info.darks,
-            radios_indices=options["projs_indices"],
-            interpolation="linear",
-            distortion_correction=distortion_correction,
-            radios_srcurrent=options["radios_srcurrent"],
-            flats_srcurrent=options["flats_srcurrent"],
-            detector_corrector=self.detector_corrector,
-            ## every flat will be read at a different heigth
-            ### sub_region=self.sub_region,
-            binning=options["binning"],
-            convert_float=True,
-        )
-
-    def _get_binning(self):
-        options = self.processing_options["read_chunk"]
-        binning = options["binning"]
-        if binning is None:
-            return 1, 1
-        else:
-            return binning
-
-    def _init_double_flatfield(self):
-        options = self.processing_options["double_flatfield"]
-
-        binning_x, binning_z = self._get_binning()
-
-        result_url = None
-
-        self.double_flatfield = None
-
-        if options["processes_file"] not in (None, ""):
-            file_path = options["processes_file"]
-            data_path = (self.dataset_info.hdf5_entry or "entry") + "/double_flatfield/results/data"
-
-            if path.exists(file_path) and (data_path in h5py.File(file_path, "r")):
-                result_url = DataUrl(file_path=file_path, data_path=data_path)
-                self.logger.info("Loading double flatfield from %s" % result_url.file_path())
-
-                self.double_flatfield = self.DoubleFlatFieldClass(
-                    self._get_shape("double_flatfield"),
-                    result_url=result_url,
-                    binning_x=binning_x,
-                    binning_z=binning_z,
-                    detector_corrector=self.detector_corrector,
-                )
-
-    def _init_weights_field(self):
-        options = self.processing_options["double_flatfield"]
-        result_url = None
-
-        binning_x, binning_z = self.chunk_reader.get_binning()
-
-        self.weights_field = None
-
-        if options["processes_file"] not in (None, ""):
-            file_path = options["processes_file"]
-            data_path = (self.dataset_info.hdf5_entry or "entry") + "/weights_field/results/data"
-
-            if path.exists(file_path) and (data_path in h5py.File(file_path, "r")):
-                result_url = DataUrl(file_path=file_path, data_path=data_path)
-                self.logger.info("Loading weights_field from %s" % result_url.file_path())
-
-                self.weights_field = self.DoubleFlatFieldClass(
-                    self._get_shape("double_flatfield"), result_url=result_url, binning_x=binning_x, binning_z=binning_z
-                )
-
-    def _init_ccd_corrections(self):
-        if "ccd_correction" not in self.processing_steps:
-            return
-
-        options = self.processing_options["ccd_correction"]
-
-        median_clip_thresh = options["median_clip_thresh"]
-
-        self.ccd_correction = self.CCDFilterClass(
-            self._get_shape("ccd_correction"), median_clip_thresh=median_clip_thresh
-        )
-
-    @use_options("phase", "phase_retrieval")
-    def _init_phase(self):
-        options = self.processing_options["phase"]
-        # If unsharp mask follows phase retrieval, then it should be done
-        # before cropping to the "inner part".
-        # Otherwise, crop the data just after phase retrieval.
-        if "unsharp_mask" in self.processing_steps:
-            margin = None
-        else:
-            margin = self._phase_margin
-        self.phase_retrieval = self.PaganinPhaseRetrievalClass(
-            self._get_shape("phase"),
-            distance=options["distance_m"],
-            energy=options["energy_kev"],
-            delta_beta=options["delta_beta"],
-            pixel_size=options["pixel_size_m"],
-            padding=options["padding_type"],
-            margin=margin,
-            fft_num_threads=True,  # TODO tune in advanced params of nabu config file
-        )
-        if self.phase_retrieval.use_fftw:
-            self.logger.debug(
-                "PaganinPhaseRetrieval using FFTW with %d threads" % self.phase_retrieval.fftw.num_threads
-            )
-
-    ##@use_options("unsharp_mask", "unsharp_mask")
-
-    def _init_unsharp(self):
-        if "unsharp_mask" not in self.processing_steps:
-            self.unsharp_mask = None
-            self.unsharp_sigma = 0.0
-            self.unsharp_coeff = 0.0
-            self.unsharp_method = "log"
-        else:
-            options = self.processing_options["unsharp_mask"]
-            self.unsharp_sigma = options["unsharp_sigma"]
-            self.unsharp_coeff = options["unsharp_coeff"]
-            self.unsharp_method = options["unsharp_method"]
-
-            self.unsharp_mask = self.UnsharpMaskClass(
-                self._get_shape("unsharp_mask"),
-                options["unsharp_sigma"],
-                options["unsharp_coeff"],
-                mode="reflect",
-                method=options["unsharp_method"],
-            )
-
-    def _init_mlog(self):
-        options = self.processing_options["take_log"]
-
-        self.mlog = self.MLogClass(
-            self._get_shape("take_log"), clip_min=options["log_min_clip"], clip_max=options["log_max_clip"]
-        )
-
-    @use_options("sino_normalization", "sino_normalization")
-    def _init_sino_normalization(self):
-        options = self.processing_options["sino_normalization"]
-        self.sino_normalization = self.SinoNormalizationClass(
-            kind=options["method"],
-            radios_shape=self._get_shape("sino_normalization_slim"),
-        )
-
-    def _init_sino_builder(self):
-        options = self.processing_options["reconstruction"]  ## build_sino class disappeared disappeared
-
-        self.sino_builder = self.SinoBuilderClass(
-            radios_shape=self._get_shape("build_sino"),
-            rot_center=options["rotation_axis_position"],
-            halftomo=False,
-        )
-        self._sinobuilder_copy = False
-        self._sinobuilder_output = None
-        self.sinos = None
-
-    # this should be renamed, as it could be confused with _init_reconstruction. What about _get_reconstruction_array ?
-    @use_options("reconstruction", "reconstruction")
-    def _prepare_reconstruction(self):
-        options = self.processing_options["reconstruction"]
-        x_s, x_e = options["start_x"], options["end_x"]
-        y_s, y_e = options["start_y"], options["end_y"]
-
-        if not self.diag_zpro_run:
-            self._rec_roi = (x_s, x_e + 1, y_s, y_e + 1)
-            self._allocate_recs(y_e - y_s + 1, x_e - x_s + 1)
-        else:
-            ## Dummy 1x1 place holder
-            self._rec_roi = (x_s, x_s + 1, y_s, y_s + 1)
-            self._allocate_recs(y_s - y_s + 1, x_s - x_s + 1)
-
-    @use_options("reconstruction", "reconstruction")
-    def _init_reconstruction(self):
-        options = self.processing_options["reconstruction"]
-
-        if self.sino_builder is None:
-            raise ValueError("Reconstruction cannot be done without build_sino")
-
-        if self.FBPClass is None:
-            raise ValueError("No usable FBP  module was found")
-
-        rot_center = options["rotation_axis_position"]
-
-        start_y, end_y, start_x, end_x = self._rec_roi
-
-        if self.HBPClass is not None and self.process_config.nabu_config["reconstruction"]["use_hbp"]:
-            fan_source_distance_meters = self.process_config.nabu_config["reconstruction"]["fan_source_distance_meters"]
-
-            self.reconstruction_hbp = self.HBPClass(
-                self._get_shape("one_sino_slim"),
-                slice_shape=(end_y - start_y, end_x - start_x),
-                angles=self.my_angles_rad,
-                rot_center=rot_center,
-                extra_options={"axis_correction": np.zeros(self.radios.shape[0], "f")},
-                axis_source_meters=fan_source_distance_meters,
-                voxel_size_microns=options["voxel_size_cm"][0] * 1.0e4,
-                scale_factor=2.0 / options["voxel_size_cm"][0],
-                clip_outer_circle=options["clip_outer_circle"],
-            )
-
-        else:
-            self.reconstruction_hbp = None
-
-        self.reconstruction = self.FBPClass(
-            self._get_shape("reconstruction"),
-            angles=np.zeros(self.radios.shape[0], "f"),
-            rot_center=rot_center,
-            filter_name=options["fbp_filter_type"],
-            slice_roi=self._rec_roi,
-            # slice_shape =   ( end_y-start_y,  end_x- start_x ),
-            scale_factor=2.0 / options["voxel_size_cm"][0],
-            padding_mode=options["padding_type"],
-            extra_options={
-                "scale_factor": 2.0 / options["voxel_size_cm"][0],
-                "axis_correction": np.zeros(self.radios.shape[0], "f"),
-                "clip_outer_circle": options["clip_outer_circle"],
-            },  #  "padding_mode": options["padding_type"],         },
-        )
-
-        my_options = self.process_config.nabu_config["reconstruction"]
-        if my_options["axis_to_the_center"]:
-            x_s, x_ep1, y_s, y_ep1 = self._rec_roi
-            off_x = -int(round((x_s + x_ep1 - 1) / 2.0 - rot_center))
-            off_y = -int(round((y_s + y_ep1 - 1) / 2.0 - rot_center))
-            self.reconstruction.offsets = {"x": off_x, "y": off_y}
-
-        if options["fbp_filter_type"] is None:
-            self.reconstruction.fbp = self.reconstruction.backproj
-
-    @use_options("histogram", "histogram")
-    def _init_histogram(self):
-        options = self.processing_options["histogram"]
-        self.histogram = self.HistogramClass(method="fixed_bins_number", num_bins=options["histogram_bins"])
-        self.histo_stack = []
-
-    @use_options("save", "writer")
-    def _init_writer(self, chunk_info=None):
-        options = self.processing_options["save"]
-        file_prefix = options["file_prefix"]
-        output_dir = path.join(options["location"], file_prefix)
-        nx_info = None
-        self._hdf5_output = is_hdf5_extension(options["file_format"])
-
-        if chunk_info is not None:
-            d_v, d_h = self.process_config.dataset_info.radio_dims[::-1]
-            h_rels = self._get_slice_start_index() + np.arange(chunk_info.span_v[1] - chunk_info.span_v[0])
-            fact_mm = self.process_config.dataset_info.pixel_size * 1.0e-3
-            heights_mm = (
-                fact_mm * (-self.span_info.z_pix_per_proj[0] + (d_v - 1) / 2 - h_rels) - self.span_info.z_offset_mm
-            )
-        else:
-            heights_mm = None
-
-        if self._hdf5_output:
-            fname_start_index = None
-            file_prefix += str("_%06d" % self._get_slice_start_index())
-            entry = getattr(self.dataset_info.dataset_scanner, "entry", None)
-            nx_info = {
-                "process_name": self._get_process_name(),
-                "processing_index": 0,
-                "config": {
-                    "processing_options": self.processing_options,
-                    "nabu_config": self.process_config.nabu_config,
-                },
-                "entry": entry,
-            }
-            self._histogram_processing_index = nx_info["processing_index"] + 1
-        elif options["file_format"] in ["tif", "tiff", "edf"]:
-            fname_start_index = self._get_slice_start_index()
-            self._histogram_processing_index = 1
-
-        self._writer_configurator = WriterConfigurator(
-            output_dir,
-            file_prefix,
-            file_format=options["file_format"],
-            overwrite=options["overwrite"],
-            start_index=fname_start_index,
-            heights_above_stage_mm=heights_mm,
-            logger=self.logger,
-            nx_info=nx_info,
-            write_histogram=("histogram" in self.processing_steps),
-            histogram_entry=getattr(self.dataset_info.dataset_scanner, "entry", "entry"),
-        )
-        self.writer = self._writer_configurator.writer
-        self._writer_exec_args = self._writer_configurator._writer_exec_args
-        self._writer_exec_kwargs = self._writer_configurator._writer_exec_kwargs
-        self.histogram_writer = self._writer_configurator.get_histogram_writer()
-
-    def _apply_expand_fact(self, t):
-        if t is not None:
-            t = t * self.chunk_reader.dataset_subsampling
-        return t
-
-    def _expand_slice(self, subchunk_slice):
-        start, stop, step = subchunk_slice.start, subchunk_slice.stop, subchunk_slice.step
-        if step is None:
-            step = 1
-
-        start, stop, step = list(map(self._apply_expand_fact, [start, stop, step]))
-        result_slice = slice(start, stop, step)
-        return result_slice
-
-    def _read_data_and_apply_flats(self, sub_total_prange_slice, subchunk_slice, chunk_info):
-        my_integer_shifts_v = chunk_info.integer_shift_v[subchunk_slice]
-        fract_complement_shifts_v = chunk_info.fract_complement_to_integer_shift_v[subchunk_slice]
-        x_shifts_list = chunk_info.x_pix_per_proj[subchunk_slice]
-        (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
-        subr_start_z_list = subr_start_z - my_integer_shifts_v
-        subr_end_z_list = subr_end_z - my_integer_shifts_v + 1
-
-        self._reset_reader_subregion((None, None, subr_start_z_list.min(), subr_end_z_list.max()))
-
-        dtasrc_start_x, dtasrc_end_x, dtasrc_start_z, dtasrc_end_z = self.trimmed_floating_subregion
-
-        if self.diag_zpro_run:
-            searched_angles = self.diagnostic_searched_angles_rad_clipped
-
-            these_angles = chunk_info.angles_rad[subchunk_slice]
-            if len(these_angles) > 1:
-                # these_angles are the projection angles
-                # if no diagnostic angle falls close to them we skip to the next angular subchunk
-                # (here slice refers to angular slicing)
-                # We like hdf5 but we that is not a reason to read them all the time, so we spare time
-
-                a_step = abs(these_angles[1:] - these_angles[:-1]).mean()
-                distance = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None]).min()
-                distance_l = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None] - a_step).min()
-                distance_h = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None] + a_step).min()
-                distance = np.array([distance, distance_h, distance_l]).min()
-
-                if distance > 2 * a_step:
-                    return
-
-        self.chunk_reader.load_data(overwrite=True, sub_total_prange_slice=sub_total_prange_slice)
-        if self.chunk_reader.dataset_subsampling > 1:
-            radios_angular_range_slicing = self._expand_slice(sub_total_prange_slice)
-        else:
-            radios_angular_range_slicing = sub_total_prange_slice
-        my_subsampled_indexes = self.chunk_reader._sorted_files_indices[radios_angular_range_slicing]
-        data_raw = self.chunk_reader.data[: len(my_subsampled_indexes)]
-
-        self.regular_accumulator.extract_preprocess_with_flats(
-            subchunk_slice,
-            my_subsampled_indexes,  # these are indexes pointing within the global domain sequence which is composed of darks flats radios
-            chunk_info,
-            np.array((subr_start_z, subr_end_z), "i"),
-            np.array((dtasrc_start_z, dtasrc_end_z), "i"),
-            data_raw,
-            radios_angular_range_slicing  # my_subsampled_indexes  is important in order to compare the
-            # radios positions with respect to the flat position, and these position
-            # are given as the sequential acquisition number which counts everything ( flats, darks, radios )
-            # Insteqd, in order to access array which spans only the radios, we need to have an idea of where we are.
-            # this is provided by radios_angular_range_slicing which addresses the radios domain
-        )
-
-    def binning_expanded(self, region):
-        binning_x, binning_z = self.chunk_reader.get_binning()
-        binnings = [binning_x] * 2 + [binning_z] * 2
-        res = [None if tok is None else tok * fact for tok, fact in zip(region, binnings)]
-        return res
-
-    def _reset_reader_subregion(self, floating_subregion):
-        if self._resume_from_step is None:
-            binning_x, binning_z = self.chunk_reader.get_binning()
-
-            start_x, end_x, start_z, end_z = floating_subregion
-            trimmed_start_z = max(0, start_z)
-            trimmed_end_z = min(self.whole_radio_shape[0], end_z)
-
-            my_buffer_height = trimmed_end_z - trimmed_start_z
-
-            if self.radios_buffer is None or my_buffer_height > self.safe_buffer_height:
-                self.safe_buffer_height = end_z - start_z
-                assert (
-                    self.safe_buffer_height >= my_buffer_height
-                ), "This should always be true, if not contact the developer"
-                self.radios_buffer = None
-                self.radios_buffer = np.zeros(
-                    (self.reading_granularity + self.extra_marge_granularity,)
-                    + (self.safe_buffer_height, self.whole_radio_shape[1]),
-                    np.float32,
-                )
-
-            self.trimmed_floating_subregion = start_x, end_x, trimmed_start_z, trimmed_end_z
-
-            self.chunk_reader._set_subregion(self.binning_expanded(self.trimmed_floating_subregion))
-            self.chunk_reader._init_reader()
-            self.chunk_reader._loaded = False
-
-            self.chunk_reader.set_data_buffer(self.radios_buffer[:, :my_buffer_height, :], pre_allocate=False)
-
-        else:
-            message = "Resume not yet implemented in helical pipeline"
-            raise RuntimeError(message)
-
-    def _ccd_corrections(self, radios=None):
-        if radios is None:
-            radios = self.gridded_radios
-        if hasattr(self.ccd_correction, "median_clip_correction_multiple_images"):
-            self.ccd_correction.median_clip_correction_multiple_images(radios)
-        else:
-            _tmp_radio = self._cpu_allocate_array(radios.shape[1:], "f", name="tmp_ccdcorr_radio")
-            for i in range(radios.shape[0]):
-                self.ccd_correction.median_clip_correction(radios[i], output=_tmp_radio)
-                radios[i][:] = _tmp_radio[:]
-
-    def _retrieve_phase(self):
-        if "unsharp_mask" in self.processing_steps:
-            for i in range(self.gridded_radios.shape[0]):
-                self.gridded_radios[i] = self.phase_retrieval.apply_filter(self.gridded_radios[i])
-        else:
-            for i in range(self.gridded_radios.shape[0]):
-                self.radios[i] = self.phase_retrieval.apply_filter(self.gridded_radios[i])
-
-    def _nophase_put_to_radios(self, target, source):
-        ((up_margin, down_margin), (left_margin, right_margin)) = self.phase_margin
-
-        zslice = slice(up_margin or None, -down_margin or None)
-        xslice = slice(left_margin or None, -right_margin or None)
-
-        for i in range(target.shape[0]):
-            target[i] = source[i][zslice, xslice]
-
-    def _apply_unsharp():
-        ((up_margin, down_margin), (left_margin, right_margin)) = self._phase_margin
-
-        zslice = slice(up_margin or None, -down_margin or None)
-        xslice = slice(left_margin or None, -right_margin or None)
-
-        for i in range(self.radios.shape[0]):
-            self.radios[i] = self.unsharp_mask.unsharp(self.gridded_radios[i])[zslice, xslice]
-
-    def _take_log(self):
-        self.mlog.take_logarithm(self.radios)
-
-    @pipeline_step("sino_normalization", "Normalizing sinograms")
-    def _normalize_sinos(self, radios=None):
-        if radios is None:
-            radios = self.radios
-        sinos = radios.transpose((1, 0, 2))
-        self.sino_normalization.normalize(sinos)
-
-    def _dump_sinogram(self, radios=None):
-        if radios is None:
-            radios = self.radios
-        self._dump_data_to_file("sinogram", data=radios)
-
-    @pipeline_step("sino_builder", "Building sinograms")
-    def _build_sino(self):
-        self.sinos = self.radios_slim
-
-    def _filter(self):
-        rot_center = self.processing_options["reconstruction"]["rotation_axis_position"]
-        self.reconstruction.sino_filter.filter_sino(
-            self.radios_slim,
-            mirror_indexes=self.mirror_angle_relative_indexes,
-            rot_center=rot_center,
-            output=self.radios_slim,
-        )
-
-    def _build_sino(self):
-        self.sinos = self.radios_slim
-
-    def _reconstruct(self, sinos=None, chunk_info=None, i_slice=0):
-        if sinos is None:
-            sinos = self.sinos
-
-        use_hbp = self.process_config.nabu_config["reconstruction"]["use_hbp"]
-
-        if not use_hbp:
-            if i_slice == 0:
-                self.reconstruction.set_custom_angles_and_axis_corrections(
-                    self.my_angles_rad, np.zeros_like(self.my_angles_rad)
-                )
-
-            self.reconstruction.backprojection(sinos, output=self.recs[0])
-
-            self.recs[0].get(self.recs_stack[i_slice])
-        else:
-            if self.reconstruction_hbp is None:
-                raise ValueError("You requested the hierchical backprojector but the module could not be imported")
-            self.reconstruction_hbp.backprojection(sinos, output=self.recs_stack[i_slice])
-
-    def _compute_histogram(self, data=None, i_slice=None, num_slices=None):
-        if self.histogram is None:
-            return
-
-        if data is None:
-            data = self.recs
-
-        my_histo = self.histogram.compute_histogram(data.ravel())
-        self.histo_stack.append(my_histo)
-
-        if i_slice == num_slices - 1:
-            self.recs_histogram = self.histogram.merge_histograms(self.histo_stack)
-            self.histo_stack.clear()
-
-    def _write_data(self, data=None):
-        if data is None:
-            data = self.recs_stack
-        my_kw_args = copy.copy(self._writer_exec_kwargs)
-        if "config" in my_kw_args:
-            self.logger.info(
-                "omitting config in  writer because of too slow nexus writer. Just writing the diagnostics, if any "
-            )
-
-        # diagnostic are saved here, with the Nabu mechanism for config
-        self.diagnostic_zpix_transl[:] = np.interp(
-            self.diagnostic_proj_angle,
-            np.deg2rad(self.span_info.projection_angles_deg_internally),
-            self.span_info.z_pix_per_proj,
-        )
-        self.diagnostic_zmm_transl[:] = self.diagnostic_zpix_transl * self.span_info.pix_size_mm
-
-        my_kw_args["config"] = self.diagnostic
-
-        self.writer.write(data, *self._writer_exec_args, **my_kw_args)
-        self.logger.info("Wrote %s" % self.writer.get_filename())
-        self._write_histogram()
-
-    def _write_histogram(self):
-        if "histogram" not in self.processing_steps:
-            return
-        self.logger.info("Saving histogram")
-        self.histogram_writer.write(
-            hist_as_2Darray(self.recs_histogram),
-            self._get_process_name(kind="histogram"),
-            processing_index=self._histogram_processing_index,
-            config={
-                "file": path.basename(self.writer.get_filename()),
-                "bins": self.processing_options["histogram"]["histogram_bins"],
-            },
-        )
-
-    def _dump_data_to_file(self, step_name, data=None):
-        if step_name not in self._data_dump:
-            return
-        self.logger.info(f"DUMP step_name={step_name}")
-        if data is None:
-            data = self.radios
-        writer = self._data_dump[step_name]
-        self.logger.info("Dumping data to %s" % writer.fname)
-        writer.write_data(data)
-
-    def balance_weights(self):
-        options = self.processing_options["reconstruction"]
-
-        rot_center = options["rotation_axis_position"]
-
-        self.radios_weights[:] = rebalance(self.radios_weights, self.my_angles_rad, rot_center)
-
-        # When standard scans are incomplete, due to motors errors, some angular range
-        #  is missing short of 360 degrees.
-        # The weight accounting correctly deal with it, but still the padding
-        # procedure with theta+180 data may fall on empty data
-        # and this may cause problems, coming from the ramp filter,
-        # in half tomo.
-        # To correct this we complete with what we have at hand from the nearest
-        # non empty data
-        #
-        to_be_filled = []
-        for i in range(len(self.radios_weights) - 1, 0, -1):
-            if self.radios_weights[i].sum():
-                break
-            to_be_filled.append(i)
-        for i in to_be_filled:
-            self.radios[i] = self.radios[to_be_filled[-1] - 1]
-
-    def _post_primary_data_reduction(self, i_slice):
-        """This will be used in the derived class to transfer data to gpu"""
-        self.radios_slim[:] = self.radios[:, i_slice, :]
-
-    def process_chunk(self, sub_region=None):
-        self._private_process_chunk(sub_region=sub_region)
-        self._process_finalize()
-
-    def _private_process_chunk(self, sub_region=None):
-        assert sub_region is not None, "sub_region argument is mandatory in helical pipeline"
-
-        # Every chunk has its diagnostic, that is good to follow the trends in helical scans
-        # or zstages
-        self._reset_diagnostics()
-
-        self.set_subregion(sub_region)
-
-        (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
-
-        span_v = subr_start_z + self._phase_margin_up, subr_end_z - self._phase_margin_down
-
-        chunk_info = self.span_info.get_chunk_info(span_v)
-
-        self._reset_memory()
-        self._init_writer(chunk_info)
-
-        self._configure_data_dumps()
-        proj_num_start, proj_num_end = chunk_info.angle_index_span
-
-        n_granularity = self.reading_granularity
-        pnum_start_list = list(np.arange(proj_num_start, proj_num_end, n_granularity))
-        pnum_end_list = pnum_start_list[1:] + [proj_num_end]
-
-        my_first_pnum = proj_num_start
-
-        if self.diag_zpro_run == 0:
-            # It may seem anodine, but setting a huge vector to zero
-            # takes time.
-            # In diagnostic collection mode we can spare it. On the other hand nothing has would be allocated for the data
-            # in such case
-            self.gridded_cumulated_weights[:] = 0
-            self.gridded_radios[:] = 0
-
-        for pnum_start, pnum_end in zip(pnum_start_list, pnum_end_list):
-            start_in_chunk = pnum_start - my_first_pnum
-            end_in_chunk = pnum_end - my_first_pnum
-
-            self._read_data_and_apply_flats(
-                slice(pnum_start, pnum_end), slice(start_in_chunk, end_in_chunk), chunk_info
-            )
-
-        if not self.diag_zpro_run:
-            # when we collect diagnostics  we dont collect all the data
-            # so there would be nothing to process here
-
-            self.gridded_radios[:] /= self.gridded_cumulated_weights
-
-            self.correct_for_missing_angles()
-
-            linea = self.gridded_cumulated_weights.sum(axis=(1, 2))
-            i_zero_list = np.where(linea == 0)[0]
-            for i_zero in i_zero_list:
-                if i_zero > linea.shape[0] // 2:
-                    direction = -1
-                else:
-                    direction = 1
-                i = i_zero
-                while ((i >= 0 and direction == -1) or ((i < linea.shape[0] - 1) and direction == 1)) and linea[i] == 0:
-                    i += direction
-                if linea[i]:
-                    self.gridded_radios[i_zero] = self.gridded_radios[i]
-                    self.gridded_cumulated_weights[i_zero] = self.gridded_cumulated_weights[i]
-
-            if "flatfield" in self._data_dump:
-                paganin_margin = self._phase_margin_up
-                if paganin_margin:
-                    data_to_dump = self.gridded_radios[:, paganin_margin:-paganin_margin, :]
-                else:
-                    data_to_dump = self.gridded_radios
-                self._dump_data_to_file("flatfield", data_to_dump)
-                if self.process_config.nabu_config["pipeline"]["skip_after_flatfield_dump"]:
-                    return
-
-            if "ccd_correction" in self.processing_steps:
-                self._ccd_corrections()
-
-            if cxx_paganin is None:
-                if ("phase" in self.processing_steps) or ("unsharp_mask" in self.processing_steps):
-                    self._retrieve_phase()
-                    if "unsharp_mask" in self.processing_steps:
-                        self._apply_unsharp()
-                else:
-                    self._nophase_put_to_radios(self.radios, self.gridded_radios)
-            else:
-                if "phase" in self.processing_steps:
-                    pr = self.phase_retrieval
-                    paganin_l_micron = math.sqrt(pr.wavelength_micron * pr.distance_micron * pr.delta_beta * math.pi)
-                    cxx_paganin.paganin_pyhst(
-                        data_raw=self.gridded_radios,
-                        output=self.radios,
-                        num_of_threads=-1,
-                        paganin_marge=self._phase_margin_up,
-                        paganin_l_micron=paganin_l_micron / pr.pixel_size_micron,
-                        image_pixel_size_y=1.0,
-                        image_pixel_size_x=1.0,
-                        unsharp_sigma=self.unsharp_sigma,
-                        unsharp_coeff=self.unsharp_coeff,
-                        unsharp_LoG=int((self.unsharp_method == "log")),
-                    )
-                else:
-                    self._nophase_put_to_radios(self.radios, self.gridded_radios)
-
-            self.logger.info(" LOG ")
-            self._nophase_put_to_radios(self.radios_weights, self.gridded_cumulated_weights)
-
-            # print( " processing steps ", self.processing_steps )
-            # ['read_chunk', 'flatfield', 'double_flatfield', 'take_log', 'reconstruction', 'save']
-
-            if "take_log" in self.processing_steps:
-                self._take_log()
-
-            self.logger.info(" BALANCE ")
-
-            self.balance_weights()
-
-            num_slices = self.radios.shape[1]
-
-            self.logger.info(" NORMALIZE")
-            self._normalize_sinos()
-            self._dump_sinogram()
-
-        if "reconstruction" in self.processing_steps:
-            if not self.diag_zpro_run:
-                # otherwise, when collecting diagnostic, we are not interested in the remaining steps
-                # on the other hand there would be nothing to process because only diagnostics have been collected
-                for i_slice in range(num_slices):
-                    self._post_primary_data_reduction(i_slice)  # charge on self.radios_slim
-
-                    self._filter()
-
-                    self.apply_weights(i_slice)
-
-                    self._build_sino()
-
-                    self._reconstruct(chunk_info=chunk_info, i_slice=i_slice)
-
-                    self._compute_histogram(i_slice=i_slice, num_slices=num_slices)
-
-            self._write_data()
-
-    def apply_weights(self, i_slice):
-        """radios_slim is on gpu"""
-        n_provided_angles = self.radios_slim.shape[0]
-
-        for first_angle_index in range(0, n_provided_angles, self.num_weight_radios_per_app):
-            end_angle_index = min(n_provided_angles, first_angle_index + self.num_weight_radios_per_app)
-            self._d_radios_weights[: end_angle_index - first_angle_index].set(
-                self.radios_weights[first_angle_index:end_angle_index, i_slice]
-            )
-            self.radios_slim[first_angle_index:end_angle_index] *= self._d_radios_weights[
-                : end_angle_index - first_angle_index
-            ]
-
-    def correct_for_missing_angles(self):
-        """For non helical scan, the rotation is often incomplete ( < 360)
-        here we complement the missing angles
-        """
-        linea = self.gridded_cumulated_weights.sum(axis=(1, 2))
-        i_zero_list = np.where(linea == 0)[0]
-        for i_zero in i_zero_list:
-            if i_zero > linea.shape[0] // 2:
-                direction = -1
-            else:
-                direction = 1
-            i = i_zero
-            while ((i >= 0 and direction == -1) or ((i < linea.shape[0] - 1) and direction == 1)) and linea[i] == 0:
-                i += direction
-            if linea[i]:
-                self.gridded_radios[i_zero] = self.gridded_radios[i]
-                self.gridded_cumulated_weights[i_zero] = self.gridded_cumulated_weights[i]
-
-    @classmethod
-    def estimate_required_memory(
-        cls, process_config, reading_granularity=None, chunk_size=None, margin_v=0, span_info=None, diag_zpro_run=0
-    ):
-        """
-        Estimate the memory (RAM) needed for a reconstruction.
-
-        Parameters
-        -----------
-        process_config: `ProcessConfig` object
-            Data structure with the processing configuration
-        chunk_size: int, optional
-            Size of a "radios chunk", i.e "delta z". A radios chunk is a 3D array of shape (n_angles, chunk_size, n_x)
-            If set to None, then chunk_size = n_z
-
-        Notes
-        -----
-        It seems that Cuda does not allow allocating and/or transferring more than 16384 MiB (17.18 GB).
-        If warn_from_GB is not None, then the result is in the form (estimated_memory_GB, warning)
-        where warning is a boolean indicating wheher memory allocation/transfer might be problematic.
-        """
-        dataset = process_config.dataset_info
-        nabu_config = process_config.nabu_config
-        processing_steps = process_config.processing_steps
-        Nx, Ny = dataset.radio_dims
-
-        total_memory_needed = 0
-
-        # Read data
-        # ----------
-
-        # gridded part
-        tmp_angles_deg = np.rad2deg(process_config.processing_options["reconstruction"]["angles"])
-        tmp_my_angle_step = abs(np.diff(tmp_angles_deg).mean())
-
-        my_angle_step = abs(np.diff(span_info.projection_angles_deg).mean())
-        n_gridded_angles = int(round(360.0 / my_angle_step))
-
-        binning_z = nabu_config["dataset"]["binning_z"]
-        projections_subsampling = nabu_config["dataset"]["projections_subsampling"]
-
-        if not diag_zpro_run:
-            # the gridded target
-            total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
-
-            # the gridded weights
-            total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
-
-        # the read grain
-        total_memory_needed += (
-            (reading_granularity + cls.extra_marge_granularity) * (2 * margin_v + chunk_size + 2) * Nx * 4
-        )
-
-        total_memory_needed += (
-            (reading_granularity + cls.extra_marge_granularity) * (2 * margin_v + chunk_size + 2) * Nx * 4
-        )
-
-        # the preprocessed radios, their weigth and the buffer used for balancing ( total of three buffer of the same size plus mask plus temporary)
-        total_memory_needed += 5 * (Nx * (chunk_size) * n_gridded_angles) * 4
-
-        if "flatfield" in processing_steps:
-            # Flat-field is done in-place, but still need to load darks/flats
-            n_darks = len(dataset.darks)
-            n_flats = len(dataset.flats)
-            darks_size = n_darks * Nx * (2 * margin_v + chunk_size) * 2  # uint16
-            flats_size = n_flats * Nx * (2 * margin_v + chunk_size) * 4  # f32
-            total_memory_needed += darks_size + flats_size
-
-        if "ccd_correction" in processing_steps:
-            total_memory_needed += Nx * (2 * margin_v + chunk_size) * 4
-
-        # Phase retrieval
-        # ---------------
-        if "phase" in processing_steps:
-            # Phase retrieval is done image-wise, so near in-place, but needs to
-            # allocate some images, fft plans, and so on
-            Nx_p = get_next_power(2 * Nx)
-            Ny_p = get_next_power(2 * (2 * margin_v + chunk_size))
-            img_size_real = 2 * 4 * Nx_p * Ny_p
-            img_size_cplx = 2 * 8 * ((Nx_p * Ny_p) // 2 + 1)
-            total_memory_needed += 2 * img_size_real + 3 * img_size_cplx
-
-        # Reconstruction
-        # ---------------
-        reconstructed_volume_size = 0
-        if "reconstruction" in processing_steps and not diag_zpro_run:
-            ##  radios_slim is used to process one slice at once, It will be on the gpu
-            ##  and cannot be reduced further, therefore no need to estimate it.
-            ##  Either it passes or it does not.
-            #### if radios_and_sinos:
-            ####     togtal_memory_needed += data_volume_size  # radios + sinos
-
-            rec_config = process_config.processing_options["reconstruction"]
-
-            Nx_rec = rec_config["end_x"] - rec_config["start_x"] + 1
-            Ny_rec = rec_config["end_y"] - rec_config["start_y"] + 1
-
-            Nz_rec = chunk_size // binning_z
-
-            ## the volume is used to reconstruct for each chunk
-            reconstructed_volume_size = Nx_rec * Ny_rec * Nz_rec * 4  # float32
-            total_memory_needed += reconstructed_volume_size
-
-        return total_memory_needed
-
-
-# target_central_slicer, source_central_slicer = overlap_logic(   subr_start_z, subr_end_z, dtasrc_start_z, dtasrcs_end_z  )
-def overlap_logic(subr_start_z, subr_end_z, dtasrc_start_z, dtasrc_end_z):
-    """determines the useful lines which can be transferred from the dtasrc_start_z:dtasrc_end_z
-    range targeting the range  subr_start_z: subr_end_z ..................
-
-    """
-
-    t_h = subr_end_z - subr_start_z
-    s_h = dtasrc_end_z - dtasrc_start_z
-
-    my_start = max(0, dtasrc_start_z - subr_start_z)
-    my_end = min(t_h, dtasrc_end_z - subr_start_z)
-
-    if my_start >= my_end:
-        return None, None
-
-    target_central_slicer = slice(my_start, my_end)
-
-    my_start = max(0, subr_start_z - dtasrc_start_z)
-    my_end = min(s_h, subr_end_z - dtasrc_start_z)
-
-    assert my_start < my_end, "Overlap_logic logic error"
-
-    dtasrc_central_slicer = slice(my_start, my_end)
-
-    return target_central_slicer, dtasrc_central_slicer
-
-
-def padding_logic(subr_start_z, subr_end_z, dtasrc_start_z, dtasrc_end_z):
-    """.......... and the missing ranges which possibly could be obtained by extension padding"""
-    t_h = subr_end_z - subr_start_z
-    s_h = dtasrc_end_z - dtasrc_start_z
-
-    if dtasrc_start_z <= subr_start_z:
-        target_lower_padding = None
-    else:
-        target_lower_padding = slice(0, dtasrc_start_z - subr_start_z)
-
-    if dtasrc_end_z >= subr_end_z:
-        target_upper_padding = None
-    else:
-        target_upper_padding = slice(dtasrc_end_z - subr_end_z, None)
-
-    return target_lower_padding, target_upper_padding
-
-
-def _fill_in_chunk_by_shift_crop_data(
-    data_target,
-    data_read,
-    fract_shit,
-    my_subr_start_z,
-    my_subr_end_z,
-    dtasrc_start_z,
-    dtasrc_end_z,
-    x_shift=0.0,
-    extension_padding=True,
-):
-    """given a freshly read cube of data, it dispatches every slice to its proper vertical  position and proper radio by shifting, cropping, and extending if necessary"""
-    data_read_precisely_shifted = nd.interpolation.shift(data_read, (-fract_shit, x_shift), order=1, mode="nearest")[
-        :-1
-    ]
-
-    target_central_slicer, dtasrc_central_slicer = overlap_logic(
-        my_subr_start_z, my_subr_end_z - 1, dtasrc_start_z, dtasrc_end_z - 1
-    )
-
-    if None not in [target_central_slicer, dtasrc_central_slicer]:
-        data_target[target_central_slicer] = data_read_precisely_shifted[dtasrc_central_slicer]
-
-    target_lower_slicer, target_upper_slicer = padding_logic(
-        my_subr_start_z, my_subr_end_z - 1, dtasrc_start_z, dtasrc_end_z - 1
-    )
-
-    if extension_padding:
-        if target_lower_slicer is not None:
-            data_target[target_lower_slicer] = data_read_precisely_shifted[0]
-        if target_upper_slicer is not None:
-            data_target[target_upper_slicer] = data_read_precisely_shifted[-1]
-    else:
-        if target_lower_slicer is not None:
-            data_target[target_lower_slicer] = 1.0e-6
-        if target_upper_slicer is not None:
-            data_target[target_upper_slicer] = 1.0e-6
-
-
-def shift(arr, shift, fill_value=0.0):
-    """trivial horizontal shift.
-    Contrarily to scipy.ndimage.interpolation.shift, this shift does not cut the tails abruptly, but by interpolation
-    """
-    result = np.zeros_like(arr)
-
-    num1 = int(math.floor(shift))
-    num2 = num1 + 1
-    partition = shift - num1
-
-    for num, factor in zip([num1, num2], [(1 - partition), partition]):
-        if num > 0:
-            result[:, :num] += fill_value * factor
-            result[:, num:] += arr[:, :-num] * factor
-        elif num < 0:
-            result[:, num:] += fill_value * factor
-            result[:, :num] += arr[:, -num:] * factor
-        else:
-            result[:] += arr * factor
-
-    return result
-
-
-def rebalance(radios_weights, angles, ax_pos):
-    """rebalance the  weights,  within groups of equivalent (up to multiple of 180), data pixels"""
-    balanced = np.zeros_like(radios_weights)
-    n_span = int(math.ceil(angles[-1] - angles[0]) / np.pi)
-    center = (radios_weights.shape[-1] - 1) / 2
-    nloop = balanced.shape[0]
-
-    for i in range(nloop):
-        w_res = balanced[i]
-        angle = angles[i]
-
-        for i_half_turn in range(-n_span - 1, n_span + 2):
-            if i_half_turn == 0:
-                w_res[:] += radios_weights[i]
-                continue
-
-            shifted_angle = angle + i_half_turn * np.pi
-
-            insertion_index = np.searchsorted(angles, shifted_angle)
-
-            if insertion_index in [0, angles.shape[0]]:
-                if insertion_index == 0:
-                    continue
-                else:
-                    if shifted_angle > 2 * np.pi:
-                        continue
-                    myimage = radios_weights[-1]
-            else:
-                partition = (shifted_angle - angles[insertion_index - 1]) / (
-                    angles[insertion_index] - angles[insertion_index - 1]
-                )
-                myimage = (1.0 - partition) * radios_weights[insertion_index - 1] + partition * radios_weights[
-                    insertion_index
-                ]
-
-            if i_half_turn % 2 == 0:
-                w_res[:] += myimage
-            else:
-                myimage = np.fliplr(myimage)
-                w_res[:] += shift(myimage, (2 * ax_pos - 2 * center))
-
-    mask = np.equal(0, radios_weights)
-    balanced[:] = radios_weights / balanced
-    balanced[mask] = 0
-    return balanced