nabu 2024.1.0rc3__py3-none-any.whl → 2024.1.1__py3-none-any.whl

build/lib/nabu/pipeline/helical/helical_chunked_regridded.py

@@ -0,0 +1,1670 @@
+# pylint: skip-file
+
+from os import path
+import numpy as np
+import math
+from silx.image.tomography import get_next_power
+from scipy import ndimage as nd
+import h5py
+import silx.io
+import copy
+from silx.io.url import DataUrl
+from ...resources.logger import LoggerOrPrint
+from ...resources.utils import is_hdf5_extension
+from ...io.reader_helical import ChunkReaderHelical, get_hdf5_dataset_shape
+from ...preproc.flatfield_variable_region import FlatFieldDataVariableRegionUrls as FlatFieldDataHelicalUrls
+from ...preproc.distortion import DistortionCorrection
+from ...preproc.shift import VerticalShift
+from ...preproc.double_flatfield_variable_region import DoubleFlatFieldVariableRegion as DoubleFlatFieldHelical
+from ...preproc.phase import PaganinPhaseRetrieval
+from ...reconstruction.sinogram import SinoBuilder
+from ...processing.unsharp import UnsharpMask
+from ...processing.histogram import PartialHistogram, hist_as_2Darray
+from ..utils import use_options, pipeline_step
+
+from ..detector_distortion_provider import DetectorDistortionProvider
+
+from .utils import (
+    WriterConfiguratorHelical as WriterConfigurator,
+)  # .utils is the same as ..utils but internally we retouch the key associated to "tiffwriter" of Writers to
+
+# point to our class which can write tiff with names indexed by the z height above the sample stage in millimiters
+
+from numpy.lib.stride_tricks import sliding_window_view
+from ...misc.binning import get_binning_function
+from .helical_utils import find_mirror_indexes
+
+
+try:
+    import nabuxx
+
+    GriddedAccumulator = nabuxx.gridded_accumulator.GriddedAccumulator
+    CCDFilter = nabuxx.ccd.CCDFilter
+    Log = nabuxx.ccd.LogFilter
+    cxx_paganin = nabuxx.paganin
+except:
+    logger_tmp = LoggerOrPrint(None)
+    logger_tmp.info(
+        "Nabuxx not available. Loading python implementation for gridded_accumulator, Log, CCDFilter, paganin"
+    )
+    from . import gridded_accumulator
+
+    GriddedAccumulator = gridded_accumulator.GriddedAccumulator
+    from ...preproc.ccd import Log, CCDFilter
+
+    cxx_paganin = None
+
+
+# For now we don't have a plain python/numpy backend for reconstruction
+Backprojector = None
+
+
+class HelicalChunkedRegriddedPipeline:
+    """
+    Pipeline for "helical" full or half field tomography.
+    Data is processed by chunks. A chunk consists in K+-1 contiguous lines of all the radios
+    which are read at variable height following the translations
+    """
+
+    extra_marge_granularity = 4
+    """ This offers extra reading space to be able to read the redundant part 
+    which might be sligtly larger and or require extra border for interpolation
+    """
+
+    FlatFieldClass = FlatFieldDataHelicalUrls
+    DoubleFlatFieldClass = DoubleFlatFieldHelical
+
+    CCDFilterClass = CCDFilter
+    MLogClass = Log
+
+    PaganinPhaseRetrievalClass = PaganinPhaseRetrieval
+    UnsharpMaskClass = UnsharpMask
+    VerticalShiftClass = VerticalShift
+    SinoBuilderClass = SinoBuilder
+
+    FBPClass = Backprojector
+    HBPClass = None
+    HistogramClass = PartialHistogram
+    regular_accumulator = None
+
+    def __init__(
+        self,
+        process_config,
+        sub_region,
+        logger=None,
+        extra_options=None,
+        phase_margin=None,
+        reading_granularity=10,
+        span_info=None,
+        diag_zpro_run=0,
+    ):
+        """
+        Initialize a "HelicalChunked" pipeline.
+
+        Parameters
+        ----------
+        process_config: `nabu.resources.processcinfig.ProcessConfig`
+            Process configuration.
+        sub_region: tuple
+            Sub-region to process in the volume for this worker, in the format
+            `(start_x, end_x, start_z, end_z)`.
+        logger: `nabu.app.logger.Logger`, optional
+            Logger class
+        extra_options: dict, optional
+            Advanced extra options.
+        phase_margin: tuple, optional
+            Margin to use when performing phase retrieval, in the form ((up, down), (left, right)).
+            See also the documentation of PaganinPhaseRetrieval.
+            If not provided, no margin is applied.
+        reading_granularity: int
+            The data angular span which needs to be read for a reconstruction is read step by step,
+            reading each time a maximum of reading_granularity radios, and doing the preprocessing
+            till phase retrieval for each of these  angular groups
+
+        Notes
+        ------
+        Using a `phase_margin` results in a lesser number of reconstructed slices.
+        More specifically, if `phase_margin = (V, H)`, then there will be `chunk_size - 2*V`
+        reconstructed slices (if the sub-region is in the middle of the volume)
+        or `chunk_size - V` reconstructed slices (if the sub-region is on top or bottom
+        of the volume).
+        """
+
+        self.span_info = span_info
+        self.reading_granularity = reading_granularity
+
+        self.logger = LoggerOrPrint(logger)
+
+        self._set_params(process_config, sub_region, extra_options, phase_margin, diag_zpro_run)
+
+        self._init_pipeline()
+
+    def _set_params(self, process_config, sub_region, extra_options, phase_margin, diag_zpro_run):
+        self.diag_zpro_run = diag_zpro_run
+        self.process_config = process_config
+        self.dataset_info = self.process_config.dataset_info
+
+        self.processing_steps = self.process_config.processing_steps.copy()
+        self.processing_options = self.process_config.processing_options
+
+        sub_region = self._check_subregion(sub_region)
+
+        self.chunk_size = sub_region[-1] - sub_region[-2]
+        self.radios_buffer = None
+
+        self._set_detector_distortion_correction()
+
+        self.set_subregion(sub_region)
+
+        self._set_phase_margin(phase_margin)
+        self._set_extra_options(extra_options)
+        self._callbacks = {}
+        self._steps_name2component = {}
+        self._steps_component2name = {}
+        self._data_dump = {}
+        self._resume_from_step = None
+
+    @staticmethod
+    def _check_subregion(sub_region):
+        if len(sub_region) < 4:
+            assert len(sub_region) == 2, " at least start_z and end_z are required in subregion"
+            sub_region = (None, None) + sub_region
+        if None in sub_region[-2:]:
+            raise ValueError("Cannot set z_min or z_max to None")
+        return sub_region
+
+    def _set_extra_options(self, extra_options):
+        if extra_options is None:
+            extra_options = {}
+        advanced_options = {}
+        advanced_options.update(extra_options)
+        self.extra_options = advanced_options
+
+    def _set_phase_margin(self, phase_margin):
+        if phase_margin is None:
+            phase_margin = ((0, 0), (0, 0))
+        self._phase_margin_up = phase_margin[0][0]
+        self._phase_margin_down = phase_margin[0][1]
+        self._phase_margin_left = phase_margin[1][0]
+        self._phase_margin_right = phase_margin[1][1]
+
+    def set_subregion(self, sub_region):
+        """
+        Set a sub-region to process.
+
+        Parameters
+        ----------
+        sub_region: tuple
+            Sub-region to process in the volume, in the format
+            `(start_x, end_x, start_z, end_z)` or `(start_z, end_z)`.
+        """
+        sub_region = self._check_subregion(sub_region)
+        dz = sub_region[-1] - sub_region[-2]
+        if dz != self.chunk_size:
+            raise ValueError(
+                "Class was initialized for chunk_size = %d but provided sub_region has chunk_size = %d"
+                % (self.chunk_size, dz)
+            )
+        self.sub_region = sub_region
+        self.z_min = sub_region[-2]
+        self.z_max = sub_region[-1]
+
+    def _compute_phase_kernel_margin(self):
+        """
+        Get the "margin" to pass to classes like PaganinPhaseRetrieval.
+        In order to have a good accuracy for filter-based phase retrieval methods,
+        we need to load extra data around the edges of each image. Otherwise,
+        a default padding type is applied.
+        """
+        if not (self.use_radio_processing_margin):
+            self._phase_margin = None
+            return
+        up_margin = self._phase_margin_up
+        down_margin = self._phase_margin_down
+        # Horizontal margin is not implemented
+        left_margin, right_margin = (0, 0)
+        self._phase_margin = ((up_margin, down_margin), (left_margin, right_margin))
+
+    @property
+    def use_radio_processing_margin(self):
+        return ("phase" in self.processing_steps) or ("unsharp_mask" in self.processing_steps)
+
+    def _get_phase_margin(self):
+        if not (self.use_radio_processing_margin):
+            return ((0, 0), (0, 0))
+        return self._phase_margin
+
+    @property
+    def phase_margin(self):
+        """
+        Return the margin for phase retrieval in the form ((up, down), (left, right))
+        """
+        return self._get_phase_margin()
+
+    def _get_process_name(self, kind="reconstruction"):
+        # In the future, might be something like "reconstruction-<ID>"
+        if kind == "reconstruction":
+            return "reconstruction"
+        elif kind == "histogram":
+            return "histogram"
+        return kind
+
+    def _configure_dump(self, step_name):
+        if step_name not in self.processing_steps:
+            if step_name == "sinogram" and self.process_config._dump_sinogram:
+                fname_full = self.process_config._dump_sinogram_file
+            else:
+                return
+        else:
+            if not self.processing_options[step_name].get("save", False):
+                return
+            fname_full = self.processing_options[step_name]["save_steps_file"]
+
+        fname, ext = path.splitext(fname_full)
+        dirname, file_prefix = path.split(fname)
+        output_dir = path.join(dirname, file_prefix)
+        file_prefix += str("_%06d" % self._get_image_start_index())
+
+        self.logger.info("omitting config in  data_dump because of too slow nexus writer ")
+
+        self._data_dump[step_name] = WriterConfigurator(
+            output_dir,
+            file_prefix,
+            file_format="hdf5",
+            overwrite=True,
+            logger=self.logger,
+            nx_info={
+                "process_name": step_name,
+                "processing_index": 0,  # TODO
+                #                "config": {"processing_options": self.processing_options, "nabu_config": self.process_config.nabu_config},
+                "config": None,
+                "entry": getattr(self.dataset_info.dataset_scanner, "entry", None),
+            },
+        )
+
+    def _configure_data_dumps(self):
+        self.process_config._configure_save_steps()
+        for step_name in self.processing_steps:
+            self._configure_dump(step_name)
+        # sinogram is a special keyword: not in processing_steps, but guaranteed to be before sinogram generation
+
+        if self.process_config._dump_sinogram:
+            self._configure_dump("sinogram")
+
+    #
+    # Callbacks
+    #
+
+    def register_callback(self, step_name, callback):
+        """
+        Register a callback for a pipeline processing step.
+
+        Parameters
+        ----------
+        step_name: str
+            processing step name
+        callback: callable
+            A function. It will be executed once the processing step `step_name`
+            is finished. The function takes only one argument: the class instance.
+        """
+        if step_name not in self.processing_steps:
+            raise ValueError("'%s' is not in processing steps %s" % (step_name, self.processing_steps))
+        if step_name in self._callbacks:
+            self._callbacks[step_name].append(callback)
+        else:
+            self._callbacks[step_name] = [callback]
+
+    #
+    # Overwritten in inheriting classes
+    #
+
+    def _get_shape(self, step_name):
+        """
+        Get the shape to provide to the class corresponding to step_name.
+        """
+        if step_name == "flatfield":
+            shape = self.radios_subset.shape
+        elif step_name == "double_flatfield":
+            shape = self.radios_subset.shape
+        elif step_name == "phase":
+            shape = self.radios_subset.shape[1:]
+        elif step_name == "ccd_correction":
+            shape = self.gridded_radios.shape[1:]
+        elif step_name == "unsharp_mask":
+            shape = self.radios_subset.shape[1:]
+        elif step_name == "take_log":
+            shape = self.radios.shape
+        elif step_name == "radios_movements":
+            shape = self.radios.shape
+        elif step_name == "sino_normalization":
+            shape = self.radios.shape
+        elif step_name == "sino_normalization_slim":
+            shape = self.radios.shape[:1] + (1,) + self.radios.shape[2:]
+        elif step_name == "one_sino_slim":
+            shape = self.radios.shape[:1] + self.radios.shape[2:]
+        elif step_name == "build_sino":
+            shape = self.radios.shape[:1] + (1,) + self.radios.shape[2:]
+        elif step_name == "reconstruction":
+            shape = self.sino_builder.output_shape[1:]
+        else:
+            raise ValueError("Unknown processing step %s" % step_name)
+        self.logger.debug("Data shape for %s is %s" % (step_name, str(shape)))
+        return shape
+
+    def _allocate_array(self, shape, dtype, name=None):
+        """this function can be redefined in the  derived class which is dedicated to gpu
+        and will return gpu garrays
+        """
+        return _cpu_allocate_array(shape, dtype, name=name)
+
+    def _cpu_allocate_array(self, shape, dtype, name=None):
+        """For objects used in the pre-gpu part. They will be always on CPU even in the derived class"""
+        result = np.zeros(shape, dtype=dtype)
+        return result
+
+    def _allocate_sinobuilder_output(self):
+        return self._cpu_allocate_array(self.sino_builder.output_shape, "f", name="sinos")
+
+    def _allocate_recs(self, ny, nx):
+        self.n_slices = self.gridded_radios.shape[1]
+        if self.use_radio_processing_margin:
+            self.n_slices -= sum(self.phase_margin[0])
+        self.recs = self._allocate_array((1, ny, nx), "f", name="recs")
+        self.recs_stack = self._cpu_allocate_array((self.n_slices, ny, nx), "f", name="recs_stack")
+
+    def _reset_memory(self):
+        pass
+
+    def _get_read_dump_subregion(self):
+        read_opts = self.processing_options["read_chunk"]
+        if read_opts.get("process_file", None) is None:
+            return None
+        dump_start_z, dump_end_z = read_opts["dump_start_z"], read_opts["dump_end_z"]
+        relative_start_z = self.z_min - dump_start_z
+        relative_end_z = relative_start_z + self.chunk_size
+        # (n_angles, n_z, n_x)
+        subregion = (None, None, relative_start_z, relative_end_z, None, None)
+        return subregion
+
+    def _check_resume_from_step(self):
+        if self._resume_from_step is None:
+            return
+        read_opts = self.processing_options["read_chunk"]
+        expected_radios_shape = get_hdf5_dataset_shape(
+            read_opts["process_file"],
+            read_opts["process_h5_path"],
+            sub_region=self._get_read_dump_subregion(),
+        )
+        # TODO check
+
+    def _init_reader_finalize(self):
+        """
+        Method called after _init_reader
+        """
+        self._check_resume_from_step()
+
+        self._compute_phase_kernel_margin()
+
+        self._allocate_reduced_gridded_and_subset_radios()
+
+    def _allocate_reduced_gridded_and_subset_radios(self):
+        shp_h = self.chunk_reader.data.shape[-1]
+
+        sliding_window_size = self.chunk_size
+        if sliding_window_size % 2 == 0:
+            sliding_window_size += 1
+        sliding_window_radius = (sliding_window_size - 1) // 2
+
+        if sliding_window_radius == 0:
+            n_projs_max = (self.span_info.sunshine_ends - self.span_info.sunshine_starts).max()
+        else:
+            padded_starts = self.span_info.sunshine_starts
+            padded_ends = self.span_info.sunshine_ends
+
+            padded_starts = np.concatenate(
+                [[padded_starts[0]] * sliding_window_radius, padded_starts, [padded_starts[-1]] * sliding_window_radius]
+            )
+            starts = sliding_window_view(padded_starts, sliding_window_size).min(axis=-1)
+
+            padded_ends = np.concatenate(
+                [[padded_ends[0]] * sliding_window_radius, padded_ends, [padded_ends[-1]] * sliding_window_radius]
+            )
+
+            ends = sliding_window_view(padded_ends, sliding_window_size).max(axis=-1)
+
+            n_projs_max = (ends - starts).max()
+
+        ((up_margin, down_margin), (left_margin, right_margin)) = self.phase_margin
+
+        (start_x, end_x, start_z, end_z) = self.sub_region
+
+        ## and now the gridded ones
+
+        my_angle_step = abs(np.diff(self.span_info.projection_angles_deg).mean())
+        self.n_gridded_angles = int(round(360.0 / my_angle_step))
+
+        self.my_angles_rad = np.arange(self.n_gridded_angles) * 2 * np.pi / self.n_gridded_angles
+
+        my_angles_deg = np.rad2deg(self.my_angles_rad)
+
+        self.mirror_angle_relative_indexes = find_mirror_indexes(my_angles_deg)
+
+        if "read_chunk" not in self.processing_steps:
+            raise ValueError("Cannot proceed without reading data")
+
+        r_shp_v, r_shp_h = self.whole_radio_shape
+
+        (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
+
+        subradio_shape = subr_end_z - subr_start_z, r_shp_h
+
+        ### these radios are for diagnostic of the translations ( they will be optionally written, for being further used
+        ##  by correlation techniques ). Two radios for the first two pass over the first gridded angles
+        if self.diag_zpro_run:
+            # 2 for the redundancy, 2 for +180 mirror
+            ndiag = 2 * 2 * self.diag_zpro_run
+        else:
+            ndiag = 2 * 2
+
+        self.diagnostic_searched_angles_rad_clipped = (
+            (0.5 + np.arange(ndiag // 2)) * (2 * np.pi / (ndiag // 2))
+        ).astype("f")
+
+        self.diagnostic_radios = np.zeros((ndiag,) + subradio_shape, np.float32)
+        self.diagnostic_weights = np.zeros((ndiag,) + subradio_shape, np.float32)
+        self.diagnostic_proj_angle = np.zeros([ndiag], "f")
+        self.diagnostic_zpix_transl = np.zeros([ndiag], "f")
+        self.diagnostic_zmm_transl = np.zeros([ndiag], "f")
+
+        self.diagnostic = {
+            "radios": self.diagnostic_radios,
+            "weights": self.diagnostic_weights,
+            "angles": self.diagnostic_proj_angle,
+            "zpix_transl": self.diagnostic_zpix_transl,
+            "zmm_trans": self.diagnostic_zmm_transl,
+            "pixel_size_mm": self.span_info.pix_size_mm,
+            "searched_rad": self.diagnostic_searched_angles_rad_clipped,
+        }
+        ## -------
+        if self.diag_zpro_run == 0:
+            self.gridded_radios = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
+            self.gridded_cumulated_weights = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
+        else:
+            # only diagnostic will be cumulated. No need to keep the full size for diagnostic runs.
+            # The gridder is initialised passing also the two buffer below,
+            # and the two first dimensions are used to allocate auxiliaries,
+            # so we shorten only the last dimension, but this is already a good cut
+            self.gridded_radios = np.zeros((self.n_gridded_angles,) + (subradio_shape[0], 2), np.float32)
+            self.gridded_cumulated_weights = np.zeros((self.n_gridded_angles,) + (subradio_shape[0], 2), np.float32)
+
+        self.radios_subset = np.zeros((self.reading_granularity,) + subradio_shape, np.float32)
+        self.radios_weights_subset = np.zeros((self.reading_granularity,) + subradio_shape, np.float32)
+
+        if not self.diag_zpro_run:
+            self.radios = np.zeros(
+                (self.n_gridded_angles,) + ((end_z - down_margin) - (start_z + up_margin), shp_h), np.float32
+            )
+        else:
+            # place holder
+            self.radios = np.zeros((self.n_gridded_angles,) + (1, 1), np.float32)
+
+        self.radios_weights = np.zeros_like(self.radios)
+
+        self.radios_slim = self._allocate_array(self._get_shape("one_sino_slim"), "f", name="radios_slim")
+
+    def _process_finalize(self):
+        """
+        Method called once the pipeline has been executed
+        """
+        pass
+
+    def _get_slice_start_index(self):
+        return self.z_min + self._phase_margin_up
+
+    _get_image_start_index = _get_slice_start_index
+
+    #
+    # Pipeline initialization
+    #
+
+    def _reset_diagnostics(self):
+        self.diagnostic_radios[:] = 0
+        self.diagnostic_weights[:] = 0
+        self.diagnostic_zpix_transl[:] = 0
+        self.diagnostic_zmm_transl[:] = 0
+        self.diagnostic_proj_angle[:] = np.nan
+
+    def _init_pipeline(self):
+        self._get_size_of_a_raw_radio()
+
+        self._init_reader()
+
+        self._init_flatfield()
+
+        self._init_double_flatfield()
+        self._init_weights_field()
+
+        self._init_ccd_corrections()
+        self._init_phase()
+        self._init_unsharp()
+        self._init_mlog()
+        self._init_sino_normalization()
+        self._init_sino_builder()
+        self._prepare_reconstruction()
+        self._init_reconstruction()
+        self._init_histogram()
+        self._init_writer()
+        self._configure_data_dumps()
+
+        self._configure_regular_accumulator()
+
+    def _set_detector_distortion_correction(self):
+        if self.process_config.nabu_config["preproc"]["detector_distortion_correction"] is None:
+            self.detector_corrector = None
+        else:
+            self.detector_corrector = DetectorDistortionProvider(
+                detector_full_shape_vh=self.process_config.dataset_info.radio_dims[::-1],
+                correction_type=self.process_config.nabu_config["preproc"]["detector_distortion_correction"],
+                options=self.process_config.nabu_config["preproc"]["detector_distortion_correction_options"],
+            )
+
+    def _configure_regular_accumulator(self):
+        ##
+        # keeping these freshly numpyed objects referenced by self
+        # ensures that their buffer info, conserved by c++ implementation of  GriddedAccumulator
+        # will always point to existing data, which could otherwise be garbage collected by python
+        #
+
+        if self.process_config.nabu_config["preproc"]["normalize_srcurrent"]:
+            self.radios_srcurrent = np.array(self.dataset_info.projections_srcurrent, "f")
+            self.flats_srcurrent = np.array(self.dataset_info.flats_srcurrent, "f")
+        else:
+            self.radios_srcurrent = None
+            self.flats_srcurrent = None
+
+        self.regular_accumulator = GriddedAccumulator(
+            gridded_radios=self.gridded_radios,
+            gridded_weights=self.gridded_cumulated_weights,
+            diagnostic_radios=self.diagnostic_radios,
+            diagnostic_weights=self.diagnostic_weights,
+            diagnostic_angles=self.diagnostic_proj_angle,
+            diagnostic_zpix_transl=self.diagnostic_zpix_transl,
+            diagnostic_searched_angles_rad_clipped=self.diagnostic_searched_angles_rad_clipped,
+            dark=self.flatfield.get_dark(),
+            flat_indexes=self.flatfield._sorted_flat_indices,
+            flats=self.flatfield.flats_stack,
+            weights=self.weights_field.data,
+            double_flat=self.double_flatfield.data,
+            diag_zpro_run=self.diag_zpro_run,
+            radios_srcurrent=self.radios_srcurrent,
+            flats_srcurrent=self.flats_srcurrent,
+        )
+
+        return
+
+    def _get_size_of_a_raw_radio(self):
+        """Once for all we find the shape of a radio.
+        This information will be used in other parts of the code when allocating
+        bunch of data holders
+        """
+        if "read_chunk" not in self.processing_steps:
+            raise ValueError("Cannot proceed without reading data")
+
+        options = self.processing_options["read_chunk"]
+
+        here_a_file = next(iter(options["files"].values()))
+        here_a_radio = silx.io.get_data(here_a_file)
+
+        binning_x, binning_z = self._get_binning()
+        if (binning_z, binning_x) != (1, 1):
+            binning_function = get_binning_function((binning_z, binning_x))
+            here_a_radio = binning_function(here_a_radio)
+
+        self.whole_radio_shape = here_a_radio.shape
+        return self.whole_radio_shape
+
+    @use_options("read_chunk", "chunk_reader")
+    def _init_reader(self):
+        if "read_chunk" not in self.processing_steps:
+            raise ValueError("Cannot proceed without reading data")
+        options = self.processing_options["read_chunk"]
+
+        assert options.get("process_file", None) is None, "Resume not yet implemented in helical pipeline"
+
+        # dummy initialisation, it will be _set_subregion'ed and set_data_buffer'ed in the loops
+        self.chunk_reader = ChunkReaderHelical(
+            options["files"],
+            sub_region=None,  # setting of subregion will be already done by calls to set_subregion
+            detector_corrector=self.detector_corrector,
+            convert_float=True,
+            binning=options["binning"],
+            dataset_subsampling=options["dataset_subsampling"],
+            data_buffer=None,
+            pre_allocate=True,
+        )
+
+        self._init_reader_finalize()
+
+    @use_options("flatfield", "flatfield")
+    def _init_flatfield(self, shape=None):
+        if shape is None:
+            shape = self._get_shape("flatfield")
+        options = self.processing_options["flatfield"]
+
+        distortion_correction = None
+        if options["do_flat_distortion"]:
+            self.logger.info("Flats distortion correction will be applied")
+            estimation_kwargs = {}
+            estimation_kwargs.update(options["flat_distortion_params"])
+            estimation_kwargs["logger"] = self.logger
+            distortion_correction = DistortionCorrection(
+                estimation_method="fft-correlation", estimation_kwargs=estimation_kwargs, correction_method="interpn"
+            )
+
+        self.flatfield = self.FlatFieldClass(
+            shape,
+            flats=self.dataset_info.flats,
+            darks=self.dataset_info.darks,
+            radios_indices=options["projs_indices"],
+            interpolation="linear",
+            distortion_correction=distortion_correction,
+            radios_srcurrent=options["radios_srcurrent"],
+            flats_srcurrent=options["flats_srcurrent"],
+            detector_corrector=self.detector_corrector,
+            ## every flat will be read at a different heigth
+            ### sub_region=self.sub_region,
+            binning=options["binning"],
+            convert_float=True,
+        )
+
+    def _get_binning(self):
+        options = self.processing_options["read_chunk"]
+        binning = options["binning"]
+        if binning is None:
+            return 1, 1
+        else:
+            return binning
+
+    def _init_double_flatfield(self):
+        options = self.processing_options["double_flatfield"]
+
+        binning_x, binning_z = self._get_binning()
+
+        result_url = None
+
+        self.double_flatfield = None
+
+        if options["processes_file"] not in (None, ""):
+            file_path = options["processes_file"]
+            data_path = (self.dataset_info.hdf5_entry or "entry") + "/double_flatfield/results/data"
+
+            if path.exists(file_path) and (data_path in h5py.File(file_path, "r")):
+                result_url = DataUrl(file_path=file_path, data_path=data_path)
+                self.logger.info("Loading double flatfield from %s" % result_url.file_path())
+
+                self.double_flatfield = self.DoubleFlatFieldClass(
+                    self._get_shape("double_flatfield"),
+                    result_url=result_url,
+                    binning_x=binning_x,
+                    binning_z=binning_z,
+                    detector_corrector=self.detector_corrector,
+                )
+
+    def _init_weights_field(self):
+        options = self.processing_options["double_flatfield"]
+        result_url = None
+
+        binning_x, binning_z = self.chunk_reader.get_binning()
+
+        self.weights_field = None
+
+        if options["processes_file"] not in (None, ""):
+            file_path = options["processes_file"]
+            data_path = (self.dataset_info.hdf5_entry or "entry") + "/weights_field/results/data"
+
+            if path.exists(file_path) and (data_path in h5py.File(file_path, "r")):
+                result_url = DataUrl(file_path=file_path, data_path=data_path)
+                self.logger.info("Loading weights_field from %s" % result_url.file_path())
+
+                self.weights_field = self.DoubleFlatFieldClass(
+                    self._get_shape("double_flatfield"), result_url=result_url, binning_x=binning_x, binning_z=binning_z
+                )
+
+    def _init_ccd_corrections(self):
+        if "ccd_correction" not in self.processing_steps:
+            return
+
+        options = self.processing_options["ccd_correction"]
+
+        median_clip_thresh = options["median_clip_thresh"]
+
+        self.ccd_correction = self.CCDFilterClass(
+            self._get_shape("ccd_correction"), median_clip_thresh=median_clip_thresh
+        )
+
+    @use_options("phase", "phase_retrieval")
+    def _init_phase(self):
+        options = self.processing_options["phase"]
+        # If unsharp mask follows phase retrieval, then it should be done
+        # before cropping to the "inner part".
+        # Otherwise, crop the data just after phase retrieval.
+        if "unsharp_mask" in self.processing_steps:
+            margin = None
+        else:
+            margin = self._phase_margin
+        self.phase_retrieval = self.PaganinPhaseRetrievalClass(
+            self._get_shape("phase"),
+            distance=options["distance_m"],
+            energy=options["energy_kev"],
+            delta_beta=options["delta_beta"],
+            pixel_size=options["pixel_size_m"],
+            padding=options["padding_type"],
+            margin=margin,
+            fft_num_threads=True,  # TODO tune in advanced params of nabu config file
+        )
+        if self.phase_retrieval.use_fftw:
+            self.logger.debug(
+                "PaganinPhaseRetrieval using FFTW with %d threads" % self.phase_retrieval.fftw.num_threads
+            )
+
+    ##@use_options("unsharp_mask", "unsharp_mask")
+
+    def _init_unsharp(self):
+        if "unsharp_mask" not in self.processing_steps:
+            self.unsharp_mask = None
+            self.unsharp_sigma = 0.0
+            self.unsharp_coeff = 0.0
+            self.unsharp_method = "log"
+        else:
+            options = self.processing_options["unsharp_mask"]
+            self.unsharp_sigma = options["unsharp_sigma"]
+            self.unsharp_coeff = options["unsharp_coeff"]
+            self.unsharp_method = options["unsharp_method"]
+
+            self.unsharp_mask = self.UnsharpMaskClass(
+                self._get_shape("unsharp_mask"),
+                options["unsharp_sigma"],
+                options["unsharp_coeff"],
+                mode="reflect",
+                method=options["unsharp_method"],
+            )
+
+    def _init_mlog(self):
+        options = self.processing_options["take_log"]
+
+        self.mlog = self.MLogClass(
+            self._get_shape("take_log"), clip_min=options["log_min_clip"], clip_max=options["log_max_clip"]
+        )
+
+    @use_options("sino_normalization", "sino_normalization")
+    def _init_sino_normalization(self):
+        options = self.processing_options["sino_normalization"]
+        self.sino_normalization = self.SinoNormalizationClass(
+            kind=options["method"],
+            radios_shape=self._get_shape("sino_normalization_slim"),
+        )
+
+    def _init_sino_builder(self):
+        options = self.processing_options["reconstruction"]  ## build_sino class disappeared disappeared
+
+        self.sino_builder = self.SinoBuilderClass(
+            radios_shape=self._get_shape("build_sino"),
+            rot_center=options["rotation_axis_position"],
+            halftomo=False,
+        )
+        self._sinobuilder_copy = False
+        self._sinobuilder_output = None
+        self.sinos = None
+
+    # this should be renamed, as it could be confused with _init_reconstruction. What about _get_reconstruction_array ?
+    @use_options("reconstruction", "reconstruction")
+    def _prepare_reconstruction(self):
+        options = self.processing_options["reconstruction"]
+        x_s, x_e = options["start_x"], options["end_x"]
+        y_s, y_e = options["start_y"], options["end_y"]
+
+        if not self.diag_zpro_run:
+            self._rec_roi = (x_s, x_e + 1, y_s, y_e + 1)
+            self._allocate_recs(y_e - y_s + 1, x_e - x_s + 1)
+        else:
+            ## Dummy 1x1 place holder
+            self._rec_roi = (x_s, x_s + 1, y_s, y_s + 1)
+            self._allocate_recs(y_s - y_s + 1, x_s - x_s + 1)
+
+    @use_options("reconstruction", "reconstruction")
+    def _init_reconstruction(self):
+        options = self.processing_options["reconstruction"]
+
+        if self.sino_builder is None:
+            raise ValueError("Reconstruction cannot be done without build_sino")
+
+        if self.FBPClass is None:
+            raise ValueError("No usable FBP  module was found")
+
+        rot_center = options["rotation_axis_position"]
+
+        start_y, end_y, start_x, end_x = self._rec_roi
+
+        if self.HBPClass is not None and self.process_config.nabu_config["reconstruction"]["use_hbp"]:
+            fan_source_distance_meters = self.process_config.nabu_config["reconstruction"]["fan_source_distance_meters"]
+
+            self.reconstruction_hbp = self.HBPClass(
+                self._get_shape("one_sino_slim"),
+                slice_shape=(end_y - start_y, end_x - start_x),
+                angles=self.my_angles_rad,
+                rot_center=rot_center,
+                extra_options={"axis_correction": np.zeros(self.radios.shape[0], "f")},
+                axis_source_meters=fan_source_distance_meters,
+                voxel_size_microns=options["voxel_size_cm"][0] * 1.0e4,
+                scale_factor=2.0 / options["voxel_size_cm"][0],
+                clip_outer_circle=options["clip_outer_circle"],
+            )
+
+        else:
+            self.reconstruction_hbp = None
+
+        self.reconstruction = self.FBPClass(
+            self._get_shape("reconstruction"),
+            angles=np.zeros(self.radios.shape[0], "f"),
+            rot_center=rot_center,
+            filter_name=options["fbp_filter_type"],
+            slice_roi=self._rec_roi,
+            # slice_shape =   ( end_y-start_y,  end_x- start_x ),
+            scale_factor=2.0 / options["voxel_size_cm"][0],
+            padding_mode=options["padding_type"],
+            extra_options={
+                "scale_factor": 2.0 / options["voxel_size_cm"][0],
+                "axis_correction": np.zeros(self.radios.shape[0], "f"),
+                "clip_outer_circle": options["clip_outer_circle"],
+            },  #  "padding_mode": options["padding_type"],         },
+        )
+
+        my_options = self.process_config.nabu_config["reconstruction"]
+        if my_options["axis_to_the_center"]:
+            x_s, x_ep1, y_s, y_ep1 = self._rec_roi
+            off_x = -int(round((x_s + x_ep1 - 1) / 2.0 - rot_center))
+            off_y = -int(round((y_s + y_ep1 - 1) / 2.0 - rot_center))
+            self.reconstruction.offsets = {"x": off_x, "y": off_y}
+
+        if options["fbp_filter_type"] is None:
+            self.reconstruction.fbp = self.reconstruction.backproj
+
+    @use_options("histogram", "histogram")
+    def _init_histogram(self):
+        options = self.processing_options["histogram"]
+        self.histogram = self.HistogramClass(method="fixed_bins_number", num_bins=options["histogram_bins"])
+        self.histo_stack = []
+
+    @use_options("save", "writer")
+    def _init_writer(self, chunk_info=None):
+        options = self.processing_options["save"]
+        file_prefix = options["file_prefix"]
+        output_dir = path.join(options["location"], file_prefix)
+        nx_info = None
+        self._hdf5_output = is_hdf5_extension(options["file_format"])
+
+        if chunk_info is not None:
+            d_v, d_h = self.process_config.dataset_info.radio_dims[::-1]
+            h_rels = self._get_slice_start_index() + np.arange(chunk_info.span_v[1] - chunk_info.span_v[0])
+            fact_mm = self.process_config.dataset_info.pixel_size * 1.0e-3
+            heights_mm = (
+                fact_mm * (-self.span_info.z_pix_per_proj[0] + (d_v - 1) / 2 - h_rels) - self.span_info.z_offset_mm
+            )
+        else:
+            heights_mm = None
+
+        if self._hdf5_output:
+            fname_start_index = None
+            file_prefix += str("_%06d" % self._get_slice_start_index())
+            entry = getattr(self.dataset_info.dataset_scanner, "entry", None)
+            nx_info = {
+                "process_name": self._get_process_name(),
+                "processing_index": 0,
+                "config": {
+                    "processing_options": self.processing_options,
+                    "nabu_config": self.process_config.nabu_config,
+                },
+                "entry": entry,
+            }
+            self._histogram_processing_index = nx_info["processing_index"] + 1
+        elif options["file_format"] in ["tif", "tiff", "edf"]:
+            fname_start_index = self._get_slice_start_index()
+            self._histogram_processing_index = 1
+
+        self._writer_configurator = WriterConfigurator(
+            output_dir,
+            file_prefix,
+            file_format=options["file_format"],
+            overwrite=options["overwrite"],
+            start_index=fname_start_index,
+            heights_above_stage_mm=heights_mm,
+            logger=self.logger,
+            nx_info=nx_info,
+            write_histogram=("histogram" in self.processing_steps),
+            histogram_entry=getattr(self.dataset_info.dataset_scanner, "entry", "entry"),
+        )
+        self.writer = self._writer_configurator.writer
+        self._writer_exec_args = self._writer_configurator._writer_exec_args
+        self._writer_exec_kwargs = self._writer_configurator._writer_exec_kwargs
+        self.histogram_writer = self._writer_configurator.get_histogram_writer()
+
+    def _apply_expand_fact(self, t):
+        if t is not None:
+            t = t * self.chunk_reader.dataset_subsampling
+        return t
+
+    def _expand_slice(self, subchunk_slice):
+        start, stop, step = subchunk_slice.start, subchunk_slice.stop, subchunk_slice.step
+        if step is None:
+            step = 1
+
+        start, stop, step = list(map(self._apply_expand_fact, [start, stop, step]))
+        result_slice = slice(start, stop, step)
+        return result_slice
+
+    def _read_data_and_apply_flats(self, sub_total_prange_slice, subchunk_slice, chunk_info):
+        my_integer_shifts_v = chunk_info.integer_shift_v[subchunk_slice]
+        fract_complement_shifts_v = chunk_info.fract_complement_to_integer_shift_v[subchunk_slice]
+        x_shifts_list = chunk_info.x_pix_per_proj[subchunk_slice]
+        (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
+        subr_start_z_list = subr_start_z - my_integer_shifts_v
+        subr_end_z_list = subr_end_z - my_integer_shifts_v + 1
+
+        self._reset_reader_subregion((None, None, subr_start_z_list.min(), subr_end_z_list.max()))
+
+        dtasrc_start_x, dtasrc_end_x, dtasrc_start_z, dtasrc_end_z = self.trimmed_floating_subregion
+
+        if self.diag_zpro_run:
+            searched_angles = self.diagnostic_searched_angles_rad_clipped
+
+            these_angles = chunk_info.angles_rad[subchunk_slice]
+            if len(these_angles) > 1:
+                # these_angles are the projection angles
+                # if no diagnostic angle falls close to them we skip to the next angular subchunk
+                # (here slice refers to angular slicing)
+                # We like hdf5 but we that is not a reason to read them all the time, so we spare time
+
+                a_step = abs(these_angles[1:] - these_angles[:-1]).mean()
+                distance = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None]).min()
+                distance_l = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None] - a_step).min()
+                distance_h = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None] + a_step).min()
+                distance = np.array([distance, distance_h, distance_l]).min()
+
+                if distance > 2 * a_step:
+                    return
+
+        self.chunk_reader.load_data(overwrite=True, sub_total_prange_slice=sub_total_prange_slice)
+        if self.chunk_reader.dataset_subsampling > 1:
+            radios_angular_range_slicing = self._expand_slice(sub_total_prange_slice)
+        else:
+            radios_angular_range_slicing = sub_total_prange_slice
+        my_subsampled_indexes = self.chunk_reader._sorted_files_indices[radios_angular_range_slicing]
+        data_raw = self.chunk_reader.data[: len(my_subsampled_indexes)]
+
+        self.regular_accumulator.extract_preprocess_with_flats(
+            subchunk_slice,
+            my_subsampled_indexes,  # these are indexes pointing within the global domain sequence which is composed of darks flats radios
+            chunk_info,
+            np.array((subr_start_z, subr_end_z), "i"),
+            np.array((dtasrc_start_z, dtasrc_end_z), "i"),
+            data_raw,
+            radios_angular_range_slicing  # my_subsampled_indexes  is important in order to compare the
+            # radios positions with respect to the flat position, and these position
+            # are given as the sequential acquisition number which counts everything ( flats, darks, radios )
+            # Insteqd, in order to access array which spans only the radios, we need to have an idea of where we are.
+            # this is provided by radios_angular_range_slicing which addresses the radios domain
+        )
+
+    def binning_expanded(self, region):
+        binning_x, binning_z = self.chunk_reader.get_binning()
+        binnings = [binning_x] * 2 + [binning_z] * 2
+        res = [None if tok is None else tok * fact for tok, fact in zip(region, binnings)]
+        return res
+
+    def _reset_reader_subregion(self, floating_subregion):
+        if self._resume_from_step is None:
+            binning_x, binning_z = self.chunk_reader.get_binning()
+
+            start_x, end_x, start_z, end_z = floating_subregion
+            trimmed_start_z = max(0, start_z)
+            trimmed_end_z = min(self.whole_radio_shape[0], end_z)
+
+            my_buffer_height = trimmed_end_z - trimmed_start_z
+
+            if self.radios_buffer is None or my_buffer_height > self.safe_buffer_height:
+                self.safe_buffer_height = end_z - start_z
+                assert (
+                    self.safe_buffer_height >= my_buffer_height
+                ), "This should always be true, if not contact the developer"
+                self.radios_buffer = None
+                self.radios_buffer = np.zeros(
+                    (self.reading_granularity + self.extra_marge_granularity,)
+                    + (self.safe_buffer_height, self.whole_radio_shape[1]),
+                    np.float32,
+                )
+
+            self.trimmed_floating_subregion = start_x, end_x, trimmed_start_z, trimmed_end_z
+
+            self.chunk_reader._set_subregion(self.binning_expanded(self.trimmed_floating_subregion))
+            self.chunk_reader._init_reader()
+            self.chunk_reader._loaded = False
+
+            self.chunk_reader.set_data_buffer(self.radios_buffer[:, :my_buffer_height, :], pre_allocate=False)
+
+        else:
+            message = "Resume not yet implemented in helical pipeline"
+            raise RuntimeError(message)
+
+    def _ccd_corrections(self, radios=None):
+        if radios is None:
+            radios = self.gridded_radios
+        if hasattr(self.ccd_correction, "median_clip_correction_multiple_images"):
+            self.ccd_correction.median_clip_correction_multiple_images(radios)
+        else:
+            _tmp_radio = self._cpu_allocate_array(radios.shape[1:], "f", name="tmp_ccdcorr_radio")
+            for i in range(radios.shape[0]):
+                self.ccd_correction.median_clip_correction(radios[i], output=_tmp_radio)
+                radios[i][:] = _tmp_radio[:]
+
+    def _retrieve_phase(self):
+        if "unsharp_mask" in self.processing_steps:
+            for i in range(self.gridded_radios.shape[0]):
+                self.gridded_radios[i] = self.phase_retrieval.apply_filter(self.gridded_radios[i])
+        else:
+            for i in range(self.gridded_radios.shape[0]):
+                self.radios[i] = self.phase_retrieval.apply_filter(self.gridded_radios[i])
+
+    def _nophase_put_to_radios(self, target, source):
+        ((up_margin, down_margin), (left_margin, right_margin)) = self.phase_margin
+
+        zslice = slice(up_margin or None, -down_margin or None)
+        xslice = slice(left_margin or None, -right_margin or None)
+
+        for i in range(target.shape[0]):
+            target[i] = source[i][zslice, xslice]
+
+    def _apply_unsharp():
+        ((up_margin, down_margin), (left_margin, right_margin)) = self._phase_margin
+
+        zslice = slice(up_margin or None, -down_margin or None)
+        xslice = slice(left_margin or None, -right_margin or None)
+
+        for i in range(self.radios.shape[0]):
+            self.radios[i] = self.unsharp_mask.unsharp(self.gridded_radios[i])[zslice, xslice]
+
+    def _take_log(self):
+        self.mlog.take_logarithm(self.radios)
+
+    @pipeline_step("sino_normalization", "Normalizing sinograms")
+    def _normalize_sinos(self, radios=None):
+        if radios is None:
+            radios = self.radios
+        sinos = radios.transpose((1, 0, 2))
+        self.sino_normalization.normalize(sinos)
+
+    def _dump_sinogram(self, radios=None):
+        if radios is None:
+            radios = self.radios
+        self._dump_data_to_file("sinogram", data=radios)
+
+    @pipeline_step("sino_builder", "Building sinograms")
+    def _build_sino(self):
+        self.sinos = self.radios_slim
+
+    def _filter(self):
+        rot_center = self.processing_options["reconstruction"]["rotation_axis_position"]
+        self.reconstruction.sino_filter.filter_sino(
+            self.radios_slim,
+            mirror_indexes=self.mirror_angle_relative_indexes,
+            rot_center=rot_center,
+            output=self.radios_slim,
+        )
+
+    def _build_sino(self):
+        self.sinos = self.radios_slim
+
+    def _reconstruct(self, sinos=None, chunk_info=None, i_slice=0):
+        if sinos is None:
+            sinos = self.sinos
+
+        use_hbp = self.process_config.nabu_config["reconstruction"]["use_hbp"]
+
+        if not use_hbp:
+            if i_slice == 0:
+                self.reconstruction.set_custom_angles_and_axis_corrections(
+                    self.my_angles_rad, np.zeros_like(self.my_angles_rad)
+                )
+
+            self.reconstruction.backprojection(sinos, output=self.recs[0])
+
+            self.recs[0].get(self.recs_stack[i_slice])
+        else:
+            if self.reconstruction_hbp is None:
+                raise ValueError("You requested the hierchical backprojector but the module could not be imported")
+            self.reconstruction_hbp.backprojection(sinos, output=self.recs_stack[i_slice])
+
+    def _compute_histogram(self, data=None, i_slice=None, num_slices=None):
+        if self.histogram is None:
+            return
+
+        if data is None:
+            data = self.recs
+
+        my_histo = self.histogram.compute_histogram(data.ravel())
+        self.histo_stack.append(my_histo)
+
+        if i_slice == num_slices - 1:
+            self.recs_histogram = self.histogram.merge_histograms(self.histo_stack)
+            self.histo_stack.clear()
+
+    def _write_data(self, data=None):
+        if data is None:
+            data = self.recs_stack
+        my_kw_args = copy.copy(self._writer_exec_kwargs)
+        if "config" in my_kw_args:
+            self.logger.info(
+                "omitting config in  writer because of too slow nexus writer. Just writing the diagnostics, if any "
+            )
+
+        # diagnostic are saved here, with the Nabu mechanism for config
+        self.diagnostic_zpix_transl[:] = np.interp(
+            self.diagnostic_proj_angle,
+            np.deg2rad(self.span_info.projection_angles_deg_internally),
+            self.span_info.z_pix_per_proj,
+        )
+        self.diagnostic_zmm_transl[:] = self.diagnostic_zpix_transl * self.span_info.pix_size_mm
+
+        my_kw_args["config"] = self.diagnostic
+
+        self.writer.write(data, *self._writer_exec_args, **my_kw_args)
+        self.logger.info("Wrote %s" % self.writer.get_filename())
+        self._write_histogram()
+
+    def _write_histogram(self):
+        if "histogram" not in self.processing_steps:
+            return
+        self.logger.info("Saving histogram")
+        self.histogram_writer.write(
+            hist_as_2Darray(self.recs_histogram),
+            self._get_process_name(kind="histogram"),
+            processing_index=self._histogram_processing_index,
+            config={
+                "file": path.basename(self.writer.get_filename()),
+                "bins": self.processing_options["histogram"]["histogram_bins"],
+            },
+        )
+
+    def _dump_data_to_file(self, step_name, data=None):
+        if step_name not in self._data_dump:
+            return
+        self.logger.info(f"DUMP step_name={step_name}")
+        if data is None:
+            data = self.radios
+        writer = self._data_dump[step_name]
+        self.logger.info("Dumping data to %s" % writer.fname)
+        writer.write_data(data)
+
+    def balance_weights(self):
+        options = self.processing_options["reconstruction"]
+
+        rot_center = options["rotation_axis_position"]
+
+        self.radios_weights[:] = rebalance(self.radios_weights, self.my_angles_rad, rot_center)
+
+        # When standard scans are incomplete, due to motors errors, some angular range
+        #  is missing short of 360 degrees.
+        # The weight accounting correctly deal with it, but still the padding
+        # procedure with theta+180 data may fall on empty data
+        # and this may cause problems, coming from the ramp filter,
+        # in half tomo.
+        # To correct this we complete with what we have at hand from the nearest
+        # non empty data
+        #
+        to_be_filled = []
+        for i in range(len(self.radios_weights) - 1, 0, -1):
+            if self.radios_weights[i].sum():
+                break
+            to_be_filled.append(i)
+        for i in to_be_filled:
+            self.radios[i] = self.radios[to_be_filled[-1] - 1]
+
+    def _post_primary_data_reduction(self, i_slice):
+        """This will be used in the derived class to transfer data to gpu"""
+        self.radios_slim[:] = self.radios[:, i_slice, :]
+
+    def process_chunk(self, sub_region=None):
+        self._private_process_chunk(sub_region=sub_region)
+        self._process_finalize()
+
+    def _private_process_chunk(self, sub_region=None):
+        assert sub_region is not None, "sub_region argument is mandatory in helical pipeline"
+
+        # Every chunk has its diagnostic, that is good to follow the trends in helical scans
+        # or zstages
+        self._reset_diagnostics()
+
+        self.set_subregion(sub_region)
+
+        (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
+
+        span_v = subr_start_z + self._phase_margin_up, subr_end_z - self._phase_margin_down
+
+        chunk_info = self.span_info.get_chunk_info(span_v)
+
+        self._reset_memory()
+        self._init_writer(chunk_info)
+
+        self._configure_data_dumps()
+        proj_num_start, proj_num_end = chunk_info.angle_index_span
+
+        n_granularity = self.reading_granularity
+        pnum_start_list = list(np.arange(proj_num_start, proj_num_end, n_granularity))
+        pnum_end_list = pnum_start_list[1:] + [proj_num_end]
+
+        my_first_pnum = proj_num_start
+
+        if self.diag_zpro_run == 0:
+            # It may seem anodine, but setting a huge vector to zero
+            # takes time.
+            # In diagnostic collection mode we can spare it. On the other hand nothing has would be allocated for the data
+            # in such case
+            self.gridded_cumulated_weights[:] = 0
+            self.gridded_radios[:] = 0
+
+        for pnum_start, pnum_end in zip(pnum_start_list, pnum_end_list):
+            start_in_chunk = pnum_start - my_first_pnum
+            end_in_chunk = pnum_end - my_first_pnum
+
+            self._read_data_and_apply_flats(
+                slice(pnum_start, pnum_end), slice(start_in_chunk, end_in_chunk), chunk_info
+            )
+
+        if not self.diag_zpro_run:
+            # when we collect diagnostics  we dont collect all the data
+            # so there would be nothing to process here
+
+            self.gridded_radios[:] /= self.gridded_cumulated_weights
+
+            self.correct_for_missing_angles()
+
+            linea = self.gridded_cumulated_weights.sum(axis=(1, 2))
+            i_zero_list = np.where(linea == 0)[0]
+            for i_zero in i_zero_list:
+                if i_zero > linea.shape[0] // 2:
+                    direction = -1
+                else:
+                    direction = 1
+                i = i_zero
+                while ((i >= 0 and direction == -1) or ((i < linea.shape[0] - 1) and direction == 1)) and linea[i] == 0:
+                    i += direction
+                if linea[i]:
+                    self.gridded_radios[i_zero] = self.gridded_radios[i]
+                    self.gridded_cumulated_weights[i_zero] = self.gridded_cumulated_weights[i]
+
+            if "flatfield" in self._data_dump:
+                paganin_margin = self._phase_margin_up
+                if paganin_margin:
+                    data_to_dump = self.gridded_radios[:, paganin_margin:-paganin_margin, :]
+                else:
+                    data_to_dump = self.gridded_radios
+                self._dump_data_to_file("flatfield", data_to_dump)
+                if self.process_config.nabu_config["pipeline"]["skip_after_flatfield_dump"]:
+                    return
+
+            if "ccd_correction" in self.processing_steps:
+                self._ccd_corrections()
+
+            if cxx_paganin is None:
+                if ("phase" in self.processing_steps) or ("unsharp_mask" in self.processing_steps):
+                    self._retrieve_phase()
+                    if "unsharp_mask" in self.processing_steps:
+                        self._apply_unsharp()
+                else:
+                    self._nophase_put_to_radios(self.radios, self.gridded_radios)
+            else:
+                if "phase" in self.processing_steps:
+                    pr = self.phase_retrieval
+                    paganin_l_micron = math.sqrt(pr.wavelength_micron * pr.distance_micron * pr.delta_beta * math.pi)
+                    cxx_paganin.paganin_pyhst(
+                        data_raw=self.gridded_radios,
+                        output=self.radios,
+                        num_of_threads=-1,
+                        paganin_marge=self._phase_margin_up,
+                        paganin_l_micron=paganin_l_micron / pr.pixel_size_micron,
+                        image_pixel_size_y=1.0,
+                        image_pixel_size_x=1.0,
+                        unsharp_sigma=self.unsharp_sigma,
+                        unsharp_coeff=self.unsharp_coeff,
+                        unsharp_LoG=int((self.unsharp_method == "log")),
+                    )
+                else:
+                    self._nophase_put_to_radios(self.radios, self.gridded_radios)
+
+            self.logger.info(" LOG ")
+            self._nophase_put_to_radios(self.radios_weights, self.gridded_cumulated_weights)
+
+            # print( " processing steps ", self.processing_steps )
+            # ['read_chunk', 'flatfield', 'double_flatfield', 'take_log', 'reconstruction', 'save']
+
+            if "take_log" in self.processing_steps:
+                self._take_log()
+
+            self.logger.info(" BALANCE ")
+
+            self.balance_weights()
+
+            num_slices = self.radios.shape[1]
+
+            self.logger.info(" NORMALIZE")
+            self._normalize_sinos()
+            self._dump_sinogram()
+
+        if "reconstruction" in self.processing_steps:
+            if not self.diag_zpro_run:
+                # otherwise, when collecting diagnostic, we are not interested in the remaining steps
+                # on the other hand there would be nothing to process because only diagnostics have been collected
+                for i_slice in range(num_slices):
+                    self._post_primary_data_reduction(i_slice)  # charge on self.radios_slim
+
+                    self._filter()
+
+                    self.apply_weights(i_slice)
+
+                    self._build_sino()
+
+                    self._reconstruct(chunk_info=chunk_info, i_slice=i_slice)
+
+                    self._compute_histogram(i_slice=i_slice, num_slices=num_slices)
+
+            self._write_data()
+
+    def apply_weights(self, i_slice):
+        """radios_slim is on gpu"""
+        n_provided_angles = self.radios_slim.shape[0]
+
+        for first_angle_index in range(0, n_provided_angles, self.num_weight_radios_per_app):
+            end_angle_index = min(n_provided_angles, first_angle_index + self.num_weight_radios_per_app)
+            self._d_radios_weights[: end_angle_index - first_angle_index].set(
+                self.radios_weights[first_angle_index:end_angle_index, i_slice]
+            )
+            self.radios_slim[first_angle_index:end_angle_index] *= self._d_radios_weights[
+                : end_angle_index - first_angle_index
+            ]
+
+    def correct_for_missing_angles(self):
+        """For non helical scan, the rotation is often incomplete ( < 360)
+        here we complement the missing angles
+        """
+        linea = self.gridded_cumulated_weights.sum(axis=(1, 2))
+        i_zero_list = np.where(linea == 0)[0]
+        for i_zero in i_zero_list:
+            if i_zero > linea.shape[0] // 2:
+                direction = -1
+            else:
+                direction = 1
+            i = i_zero
+            while ((i >= 0 and direction == -1) or ((i < linea.shape[0] - 1) and direction == 1)) and linea[i] == 0:
+                i += direction
+            if linea[i]:
+                self.gridded_radios[i_zero] = self.gridded_radios[i]
+                self.gridded_cumulated_weights[i_zero] = self.gridded_cumulated_weights[i]
+
+    @classmethod
+    def estimate_required_memory(
+        cls, process_config, reading_granularity=None, chunk_size=None, margin_v=0, span_info=None, diag_zpro_run=0
+    ):
+        """
+        Estimate the memory (RAM) needed for a reconstruction.
+
+        Parameters
+        -----------
+        process_config: `ProcessConfig` object
+            Data structure with the processing configuration
+        chunk_size: int, optional
+            Size of a "radios chunk", i.e "delta z". A radios chunk is a 3D array of shape (n_angles, chunk_size, n_x)
+            If set to None, then chunk_size = n_z
+
+        Notes
+        -----
+        It seems that Cuda does not allow allocating and/or transferring more than 16384 MiB (17.18 GB).
+        If warn_from_GB is not None, then the result is in the form (estimated_memory_GB, warning)
+        where warning is a boolean indicating wheher memory allocation/transfer might be problematic.
+        """
+        dataset = process_config.dataset_info
+        nabu_config = process_config.nabu_config
+        processing_steps = process_config.processing_steps
+        Nx, Ny = dataset.radio_dims
+
+        total_memory_needed = 0
+
+        # Read data
+        # ----------
+
+        # gridded part
+        tmp_angles_deg = np.rad2deg(process_config.processing_options["reconstruction"]["angles"])
+        tmp_my_angle_step = abs(np.diff(tmp_angles_deg).mean())
+
+        my_angle_step = abs(np.diff(span_info.projection_angles_deg).mean())
+        n_gridded_angles = int(round(360.0 / my_angle_step))
+
+        binning_z = nabu_config["dataset"]["binning_z"]
+        projections_subsampling = nabu_config["dataset"]["projections_subsampling"]
+
+        if not diag_zpro_run:
+            # the gridded target
+            total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
+
+            # the gridded weights
+            total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
+
+        # the read grain
+        total_memory_needed += (
+            (reading_granularity + cls.extra_marge_granularity) * (2 * margin_v + chunk_size + 2) * Nx * 4
+        )
+
+        total_memory_needed += (
+            (reading_granularity + cls.extra_marge_granularity) * (2 * margin_v + chunk_size + 2) * Nx * 4
+        )
+
+        # the preprocessed radios, their weigth and the buffer used for balancing ( total of three buffer of the same size plus mask plus temporary)
+        total_memory_needed += 5 * (Nx * (chunk_size) * n_gridded_angles) * 4
+
+        if "flatfield" in processing_steps:
+            # Flat-field is done in-place, but still need to load darks/flats
+            n_darks = len(dataset.darks)
+            n_flats = len(dataset.flats)
+            darks_size = n_darks * Nx * (2 * margin_v + chunk_size) * 2  # uint16
+            flats_size = n_flats * Nx * (2 * margin_v + chunk_size) * 4  # f32
+            total_memory_needed += darks_size + flats_size
+
+        if "ccd_correction" in processing_steps:
+            total_memory_needed += Nx * (2 * margin_v + chunk_size) * 4
+
+        # Phase retrieval
+        # ---------------
+        if "phase" in processing_steps:
+            # Phase retrieval is done image-wise, so near in-place, but needs to
+            # allocate some images, fft plans, and so on
+            Nx_p = get_next_power(2 * Nx)
+            Ny_p = get_next_power(2 * (2 * margin_v + chunk_size))
+            img_size_real = 2 * 4 * Nx_p * Ny_p
+            img_size_cplx = 2 * 8 * ((Nx_p * Ny_p) // 2 + 1)
+            total_memory_needed += 2 * img_size_real + 3 * img_size_cplx
+
+        # Reconstruction
+        # ---------------
+        reconstructed_volume_size = 0
+        if "reconstruction" in processing_steps and not diag_zpro_run:
+            ##  radios_slim is used to process one slice at once, It will be on the gpu
+            ##  and cannot be reduced further, therefore no need to estimate it.
+            ##  Either it passes or it does not.
+            #### if radios_and_sinos:
+            ####     togtal_memory_needed += data_volume_size  # radios + sinos
+
+            rec_config = process_config.processing_options["reconstruction"]
+
+            Nx_rec = rec_config["end_x"] - rec_config["start_x"] + 1
+            Ny_rec = rec_config["end_y"] - rec_config["start_y"] + 1
+
+            Nz_rec = chunk_size // binning_z
+
+            ## the volume is used to reconstruct for each chunk
+            reconstructed_volume_size = Nx_rec * Ny_rec * Nz_rec * 4  # float32
+            total_memory_needed += reconstructed_volume_size
+
+        return total_memory_needed
+
+
+# target_central_slicer, source_central_slicer = overlap_logic(   subr_start_z, subr_end_z, dtasrc_start_z, dtasrcs_end_z  )
+def overlap_logic(subr_start_z, subr_end_z, dtasrc_start_z, dtasrc_end_z):
+    """determines the useful lines which can be transferred from the dtasrc_start_z:dtasrc_end_z
+    range targeting the range  subr_start_z: subr_end_z ..................
+
+    """
+
+    t_h = subr_end_z - subr_start_z
+    s_h = dtasrc_end_z - dtasrc_start_z
+
+    my_start = max(0, dtasrc_start_z - subr_start_z)
+    my_end = min(t_h, dtasrc_end_z - subr_start_z)
+
+    if my_start >= my_end:
+        return None, None
+
+    target_central_slicer = slice(my_start, my_end)
+
+    my_start = max(0, subr_start_z - dtasrc_start_z)
+    my_end = min(s_h, subr_end_z - dtasrc_start_z)
+
+    assert my_start < my_end, "Overlap_logic logic error"
+
+    dtasrc_central_slicer = slice(my_start, my_end)
+
+    return target_central_slicer, dtasrc_central_slicer
+
+
+def padding_logic(subr_start_z, subr_end_z, dtasrc_start_z, dtasrc_end_z):
+    """.......... and the missing ranges which possibly could be obtained by extension padding"""
+    t_h = subr_end_z - subr_start_z
+    s_h = dtasrc_end_z - dtasrc_start_z
+
+    if dtasrc_start_z <= subr_start_z:
+        target_lower_padding = None
+    else:
+        target_lower_padding = slice(0, dtasrc_start_z - subr_start_z)
+
+    if dtasrc_end_z >= subr_end_z:
+        target_upper_padding = None
+    else:
+        target_upper_padding = slice(dtasrc_end_z - subr_end_z, None)
+
+    return target_lower_padding, target_upper_padding
+
+
+def _fill_in_chunk_by_shift_crop_data(
+    data_target,
+    data_read,
+    fract_shit,
+    my_subr_start_z,
+    my_subr_end_z,
+    dtasrc_start_z,
+    dtasrc_end_z,
+    x_shift=0.0,
+    extension_padding=True,
+):
+    """given a freshly read cube of data, it dispatches every slice to its proper vertical  position and proper radio by shifting, cropping, and extending if necessary"""
+    data_read_precisely_shifted = nd.interpolation.shift(data_read, (-fract_shit, x_shift), order=1, mode="nearest")[
+        :-1
+    ]
+
+    target_central_slicer, dtasrc_central_slicer = overlap_logic(
+        my_subr_start_z, my_subr_end_z - 1, dtasrc_start_z, dtasrc_end_z - 1
+    )
+
+    if None not in [target_central_slicer, dtasrc_central_slicer]:
+        data_target[target_central_slicer] = data_read_precisely_shifted[dtasrc_central_slicer]
+
+    target_lower_slicer, target_upper_slicer = padding_logic(
+        my_subr_start_z, my_subr_end_z - 1, dtasrc_start_z, dtasrc_end_z - 1
+    )
+
+    if extension_padding:
+        if target_lower_slicer is not None:
+            data_target[target_lower_slicer] = data_read_precisely_shifted[0]
+        if target_upper_slicer is not None:
+            data_target[target_upper_slicer] = data_read_precisely_shifted[-1]
+    else:
+        if target_lower_slicer is not None:
+            data_target[target_lower_slicer] = 1.0e-6
+        if target_upper_slicer is not None:
+            data_target[target_upper_slicer] = 1.0e-6
+
+
+def shift(arr, shift, fill_value=0.0):
+    """trivial horizontal shift.
+    Contrarily to scipy.ndimage.interpolation.shift, this shift does not cut the tails abruptly, but by interpolation
+    """
+    result = np.zeros_like(arr)
+
+    num1 = int(math.floor(shift))
+    num2 = num1 + 1
+    partition = shift - num1
+
+    for num, factor in zip([num1, num2], [(1 - partition), partition]):
+        if num > 0:
+            result[:, :num] += fill_value * factor
+            result[:, num:] += arr[:, :-num] * factor
+        elif num < 0:
+            result[:, num:] += fill_value * factor
+            result[:, :num] += arr[:, -num:] * factor
+        else:
+            result[:] += arr * factor
+
+    return result
+
+
+def rebalance(radios_weights, angles, ax_pos):
+    """rebalance the  weights,  within groups of equivalent (up to multiple of 180), data pixels"""
+    balanced = np.zeros_like(radios_weights)
+    n_span = int(math.ceil(angles[-1] - angles[0]) / np.pi)
+    center = (radios_weights.shape[-1] - 1) / 2
+    nloop = balanced.shape[0]
+
+    for i in range(nloop):
+        w_res = balanced[i]
+        angle = angles[i]
+
+        for i_half_turn in range(-n_span - 1, n_span + 2):
+            if i_half_turn == 0:
+                w_res[:] += radios_weights[i]
+                continue
+
+            shifted_angle = angle + i_half_turn * np.pi
+
+            insertion_index = np.searchsorted(angles, shifted_angle)
+
+            if insertion_index in [0, angles.shape[0]]:
+                if insertion_index == 0:
+                    continue
+                else:
+                    if shifted_angle > 2 * np.pi:
+                        continue
+                    myimage = radios_weights[-1]
+            else:
+                partition = (shifted_angle - angles[insertion_index - 1]) / (
+                    angles[insertion_index] - angles[insertion_index - 1]
+                )
+                myimage = (1.0 - partition) * radios_weights[insertion_index - 1] + partition * radios_weights[
+                    insertion_index
+                ]
+
+            if i_half_turn % 2 == 0:
+                w_res[:] += myimage
+            else:
+                myimage = np.fliplr(myimage)
+                w_res[:] += shift(myimage, (2 * ax_pos - 2 * center))
+
+    mask = np.equal(0, radios_weights)
+    balanced[:] = radios_weights / balanced
+    balanced[mask] = 0
+    return balanced