nabu 2023.2.1__py3-none-any.whl → 2024.1.0rc3__py3-none-any.whl

nabu/pipeline/helical/helical_chunked_regridded.py

@@ -1,7 +1,6 @@
 # pylint: skip-file
 
 from os import path
-from time import time
 import numpy as np
 import math
 from silx.image.tomography import get_next_power
@@ -11,19 +10,18 @@ import silx.io
 import copy
 from silx.io.url import DataUrl
 from ...resources.logger import LoggerOrPrint
-from ...resources.utils import is_hdf5_extension, extract_parameters
+from ...resources.utils import is_hdf5_extension
 from ...io.reader_helical import ChunkReaderHelical, get_hdf5_dataset_shape
 from ...preproc.flatfield_variable_region import FlatFieldDataVariableRegionUrls as FlatFieldDataHelicalUrls
 from ...preproc.distortion import DistortionCorrection
 from ...preproc.shift import VerticalShift
 from ...preproc.double_flatfield_variable_region import DoubleFlatFieldVariableRegion as DoubleFlatFieldHelical
 from ...preproc.phase import PaganinPhaseRetrieval
-from ...reconstruction.sinogram import SinoBuilder, SinoNormalization
-from ...misc.unsharp import UnsharpMask
-from ...misc.histogram import PartialHistogram, hist_as_2Darray
+from ...reconstruction.sinogram import SinoBuilder
+from ...processing.unsharp import UnsharpMask
+from ...processing.histogram import PartialHistogram, hist_as_2Darray
 from ..utils import use_options, pipeline_step
 
-from ...resources.utils import extract_parameters
 from ..detector_distortion_provider import DetectorDistortionProvider
 
 from .utils import (
@@ -36,9 +34,26 @@ from numpy.lib.stride_tricks import sliding_window_view
 from ...misc.binning import get_binning_function
 from .helical_utils import find_mirror_indexes
 
-from ...preproc.ccd import Log, CCDFilter
 
-from . import gridded_accumulator
+try:
+    import nabuxx
+
+    GriddedAccumulator = nabuxx.gridded_accumulator.GriddedAccumulator
+    CCDFilter = nabuxx.ccd.CCDFilter
+    Log = nabuxx.ccd.LogFilter
+    cxx_paganin = nabuxx.paganin
+except:
+    logger_tmp = LoggerOrPrint(None)
+    logger_tmp.info(
+        "Nabuxx not available. Loading python implementation for gridded_accumulator, Log, CCDFilter, paganin"
+    )
+    from . import gridded_accumulator
+
+    GriddedAccumulator = gridded_accumulator.GriddedAccumulator
+    from ...preproc.ccd import Log, CCDFilter
+
+    cxx_paganin = None
+
 
 # For now we don't have a plain python/numpy backend for reconstruction
 Backprojector = None
@@ -81,6 +96,7 @@ class HelicalChunkedRegriddedPipeline:
         phase_margin=None,
         reading_granularity=10,
         span_info=None,
+        diag_zpro_run=0,
     ):
         """
         Initialize a "HelicalChunked" pipeline.
@@ -119,11 +135,12 @@ class HelicalChunkedRegriddedPipeline:
 
         self.logger = LoggerOrPrint(logger)
 
-        self._set_params(process_config, sub_region, extra_options, phase_margin)
+        self._set_params(process_config, sub_region, extra_options, phase_margin, diag_zpro_run)
 
         self._init_pipeline()
 
-    def _set_params(self, process_config, sub_region, extra_options, phase_margin):
+    def _set_params(self, process_config, sub_region, extra_options, phase_margin, diag_zpro_run):
+        self.diag_zpro_run = diag_zpro_run
         self.process_config = process_config
         self.dataset_info = self.process_config.dataset_info
 
@@ -443,23 +460,54 @@ class HelicalChunkedRegriddedPipeline:
 
         ### these radios are for diagnostic of the translations ( they will be optionally written, for being further used
         ##  by correlation techniques ). Two radios for the first two pass over the first gridded angles
-        self.diagnostic_radios = np.zeros((2,) + subradio_shape, np.float32)
-        self.diagnostic_weights = np.zeros((2,) + subradio_shape, np.float32)
-        self.diagnostic_proj_angle = np.zeros([2], "f")
+        if self.diag_zpro_run:
+            # 2 for the redundancy, 2 for +180 mirror
+            ndiag = 2 * 2 * self.diag_zpro_run
+        else:
+            ndiag = 2 * 2
+
+        self.diagnostic_searched_angles_rad_clipped = (
+            (0.5 + np.arange(ndiag // 2)) * (2 * np.pi / (ndiag // 2))
+        ).astype("f")
+
+        self.diagnostic_radios = np.zeros((ndiag,) + subradio_shape, np.float32)
+        self.diagnostic_weights = np.zeros((ndiag,) + subradio_shape, np.float32)
+        self.diagnostic_proj_angle = np.zeros([ndiag], "f")
+        self.diagnostic_zpix_transl = np.zeros([ndiag], "f")
+        self.diagnostic_zmm_transl = np.zeros([ndiag], "f")
+
         self.diagnostic = {
             "radios": self.diagnostic_radios,
             "weights": self.diagnostic_weights,
             "angles": self.diagnostic_proj_angle,
+            "zpix_transl": self.diagnostic_zpix_transl,
+            "zmm_trans": self.diagnostic_zmm_transl,
+            "pixel_size_mm": self.span_info.pix_size_mm,
+            "searched_rad": self.diagnostic_searched_angles_rad_clipped,
         }
         ## -------
-        self.gridded_radios = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
-        self.gridded_cumulated_weights = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
+        if self.diag_zpro_run == 0:
+            self.gridded_radios = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
+            self.gridded_cumulated_weights = np.zeros((self.n_gridded_angles,) + subradio_shape, np.float32)
+        else:
+            # only diagnostic will be cumulated. No need to keep the full size for diagnostic runs.
+            # The gridder is initialised passing also the two buffer below,
+            # and the two first dimensions are used to allocate auxiliaries,
+            # so we shorten only the last dimension, but this is already a good cut
+            self.gridded_radios = np.zeros((self.n_gridded_angles,) + (subradio_shape[0], 2), np.float32)
+            self.gridded_cumulated_weights = np.zeros((self.n_gridded_angles,) + (subradio_shape[0], 2), np.float32)
+
         self.radios_subset = np.zeros((self.reading_granularity,) + subradio_shape, np.float32)
         self.radios_weights_subset = np.zeros((self.reading_granularity,) + subradio_shape, np.float32)
 
-        self.radios = np.zeros(
-            (self.n_gridded_angles,) + ((end_z - down_margin) - (start_z + up_margin), shp_h), np.float32
-        )
+        if not self.diag_zpro_run:
+            self.radios = np.zeros(
+                (self.n_gridded_angles,) + ((end_z - down_margin) - (start_z + up_margin), shp_h), np.float32
+            )
+        else:
+            # place holder
+            self.radios = np.zeros((self.n_gridded_angles,) + (1, 1), np.float32)
+
         self.radios_weights = np.zeros_like(self.radios)
 
         self.radios_slim = self._allocate_array(self._get_shape("one_sino_slim"), "f", name="radios_slim")
@@ -479,6 +527,13 @@ class HelicalChunkedRegriddedPipeline:
     # Pipeline initialization
     #
 
+    def _reset_diagnostics(self):
+        self.diagnostic_radios[:] = 0
+        self.diagnostic_weights[:] = 0
+        self.diagnostic_zpix_transl[:] = 0
+        self.diagnostic_zmm_transl[:] = 0
+        self.diagnostic_proj_angle[:] = np.nan
+
     def _init_pipeline(self):
         self._get_size_of_a_raw_radio()
 
@@ -514,19 +569,35 @@ class HelicalChunkedRegriddedPipeline:
             )
 
     def _configure_regular_accumulator(self):
-        accumulator_cls = gridded_accumulator.GriddedAccumulator
+        ##
+        # keeping these freshly numpyed objects referenced by self
+        # ensures that their buffer info, conserved by c++ implementation of  GriddedAccumulator
+        # will always point to existing data, which could otherwise be garbage collected by python
+        #
+
+        if self.process_config.nabu_config["preproc"]["normalize_srcurrent"]:
+            self.radios_srcurrent = np.array(self.dataset_info.projections_srcurrent, "f")
+            self.flats_srcurrent = np.array(self.dataset_info.flats_srcurrent, "f")
+        else:
+            self.radios_srcurrent = None
+            self.flats_srcurrent = None
 
-        self.regular_accumulator = accumulator_cls(
+        self.regular_accumulator = GriddedAccumulator(
             gridded_radios=self.gridded_radios,
             gridded_weights=self.gridded_cumulated_weights,
             diagnostic_radios=self.diagnostic_radios,
             diagnostic_weights=self.diagnostic_weights,
             diagnostic_angles=self.diagnostic_proj_angle,
+            diagnostic_zpix_transl=self.diagnostic_zpix_transl,
+            diagnostic_searched_angles_rad_clipped=self.diagnostic_searched_angles_rad_clipped,
             dark=self.flatfield.get_dark(),
             flat_indexes=self.flatfield._sorted_flat_indices,
             flats=self.flatfield.flats_stack,
             weights=self.weights_field.data,
             double_flat=self.double_flatfield.data,
+            diag_zpro_run=self.diag_zpro_run,
+            radios_srcurrent=self.radios_srcurrent,
+            flats_srcurrent=self.flats_srcurrent,
         )
 
         return
@@ -689,7 +760,7 @@ class HelicalChunkedRegriddedPipeline:
             pixel_size=options["pixel_size_m"],
             padding=options["padding_type"],
             margin=margin,
-            fftw_num_threads=True,  # TODO tune in advanced params of nabu config file
+            fft_num_threads=True,  # TODO tune in advanced params of nabu config file
         )
         if self.phase_retrieval.use_fftw:
             self.logger.debug(
@@ -752,8 +823,13 @@ class HelicalChunkedRegriddedPipeline:
         x_s, x_e = options["start_x"], options["end_x"]
         y_s, y_e = options["start_y"], options["end_y"]
 
-        self._rec_roi = (x_s, x_e + 1, y_s, y_e + 1)
-        self._allocate_recs(y_e - y_s + 1, x_e - x_s + 1)
+        if not self.diag_zpro_run:
+            self._rec_roi = (x_s, x_e + 1, y_s, y_e + 1)
+            self._allocate_recs(y_e - y_s + 1, x_e - x_s + 1)
+        else:
+            ## Dummy 1x1 place holder
+            self._rec_roi = (x_s, x_s + 1, y_s, y_s + 1)
+            self._allocate_recs(y_s - y_s + 1, x_s - x_s + 1)
 
     @use_options("reconstruction", "reconstruction")
     def _init_reconstruction(self):
@@ -769,7 +845,7 @@ class HelicalChunkedRegriddedPipeline:
 
         start_y, end_y, start_x, end_x = self._rec_roi
 
-        if self.HBPClass is not None:
+        if self.HBPClass is not None and self.process_config.nabu_config["reconstruction"]["use_hbp"]:
             fan_source_distance_meters = self.process_config.nabu_config["reconstruction"]["fan_source_distance_meters"]
 
             self.reconstruction_hbp = self.HBPClass(
@@ -780,7 +856,8 @@ class HelicalChunkedRegriddedPipeline:
                 extra_options={"axis_correction": np.zeros(self.radios.shape[0], "f")},
                 axis_source_meters=fan_source_distance_meters,
                 voxel_size_microns=options["voxel_size_cm"][0] * 1.0e4,
-                scale_factor=1.0 / options["voxel_size_cm"][0],
+                scale_factor=2.0 / options["voxel_size_cm"][0],
+                clip_outer_circle=options["clip_outer_circle"],
             )
 
         else:
@@ -793,10 +870,10 @@ class HelicalChunkedRegriddedPipeline:
             filter_name=options["fbp_filter_type"],
             slice_roi=self._rec_roi,
             # slice_shape =   ( end_y-start_y,  end_x- start_x ),
-            scale_factor=1.0 / options["voxel_size_cm"][0],
+            scale_factor=2.0 / options["voxel_size_cm"][0],
             padding_mode=options["padding_type"],
             extra_options={
-                "scale_factor": 1.0 / options["voxel_size_cm"][0],
+                "scale_factor": 2.0 / options["voxel_size_cm"][0],
                 "axis_correction": np.zeros(self.radios.shape[0], "f"),
                 "clip_outer_circle": options["clip_outer_circle"],
             },  #  "padding_mode": options["padding_type"],         },
@@ -850,9 +927,10 @@ class HelicalChunkedRegriddedPipeline:
                 "entry": entry,
             }
             self._histogram_processing_index = nx_info["processing_index"] + 1
-        elif options["file_format"] in ["tif", "tiff"]:
+        elif options["file_format"] in ["tif", "tiff", "edf"]:
             fname_start_index = self._get_slice_start_index()
             self._histogram_processing_index = 1
+
         self._writer_configurator = WriterConfigurator(
             output_dir,
             file_prefix,
@@ -884,94 +962,57 @@ class HelicalChunkedRegriddedPipeline:
         result_slice = slice(start, stop, step)
         return result_slice
 
-    def _extract_preprocess_with_flats(self, sub_total_prange_slice, subchunk_slice, chunk_info, output):
-        """Read, and apply dark+ff to, a small angular domain corresponding to the slice argument sub_total_prange_slice
-        without refilling the holes.
-
-        """
-
-        if self.chunk_reader.dataset_subsampling > 1:
-            subsampling_file_slice = self._expand_slice(sub_total_prange_slice)
-        else:
-            subsampling_file_slice = sub_total_prange_slice
-
+    def _read_data_and_apply_flats(self, sub_total_prange_slice, subchunk_slice, chunk_info):
         my_integer_shifts_v = chunk_info.integer_shift_v[subchunk_slice]
         fract_complement_shifts_v = chunk_info.fract_complement_to_integer_shift_v[subchunk_slice]
-
         x_shifts_list = chunk_info.x_pix_per_proj[subchunk_slice]
-
         (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
         subr_start_z_list = subr_start_z - my_integer_shifts_v
         subr_end_z_list = subr_end_z - my_integer_shifts_v + 1
-        floating_start_z = subr_start_z_list.min()
-        floating_end_z = subr_end_z_list.max()
-
-        floating_subregion = None, None, floating_start_z, floating_end_z
-
-        self._reset_reader_subregion(floating_subregion)
-
-        self.chunk_reader.load_data(overwrite=True, sub_total_prange_slice=sub_total_prange_slice)
-        my_indexes = self.chunk_reader._sorted_files_indices[subsampling_file_slice]
-        data_raw = self.chunk_reader.data[: len(my_indexes)]
 
-        if (self.flatfield is not None) or (self.double_flatfield is not None):
-            sub_regions_per_radio = [self.trimmed_floating_subregion] * len(my_indexes)
-
-            if self.flatfield is not None:
-                self.flatfield.normalize_radios(data_raw, my_indexes, sub_regions_per_radio)
+        self._reset_reader_subregion((None, None, subr_start_z_list.min(), subr_end_z_list.max()))
 
-            if self.double_flatfield is not None:
-                self.double_flatfield.apply_double_flatfield_for_sub_regions(data_raw, sub_regions_per_radio)
+        dtasrc_start_x, dtasrc_end_x, dtasrc_start_z, dtasrc_end_z = self.trimmed_floating_subregion
 
-        source_start_x, source_end_x, source_start_z, sources_end_z = self.trimmed_floating_subregion
+        if self.diag_zpro_run:
+            searched_angles = self.diagnostic_searched_angles_rad_clipped
 
-        if self.weights_field is not None:
-            data_weight = self.weights_field.data[source_start_z:sources_end_z]
-        else:
-            data_weight = None
-
-        for data_read, list_subr_start_z, list_subr_end_z, fract_shit, x_shift, data_target in zip(
-            data_raw, subr_start_z_list, subr_end_z_list, fract_complement_shifts_v, x_shifts_list, output
-        ):
-            _fill_in_chunk_by_shift_crop_data(
-                data_target,
-                data_read,
-                fract_shit,
-                list_subr_start_z,
-                list_subr_end_z,
-                source_start_z,
-                sources_end_z,
-                x_shift=x_shift,
-            )
-
-    def _read_data_and_apply_flats(self, sub_total_prange_slice, subchunk_slice, chunk_info):
-        my_integer_shifts_v = chunk_info.integer_shift_v[subchunk_slice]
-        fract_complement_shifts_v = chunk_info.fract_complement_to_integer_shift_v[subchunk_slice]
-        x_shifts_list = chunk_info.x_pix_per_proj[subchunk_slice]
-        (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
-        subr_start_z_list = subr_start_z - my_integer_shifts_v
-        subr_end_z_list = subr_end_z - my_integer_shifts_v + 1
+            these_angles = chunk_info.angles_rad[subchunk_slice]
+            if len(these_angles) > 1:
+                # these_angles are the projection angles
+                # if no diagnostic angle falls close to them we skip to the next angular subchunk
+                # (here slice refers to angular slicing)
+                # We like hdf5 but we that is not a reason to read them all the time, so we spare time
 
-        self._reset_reader_subregion((None, None, subr_start_z_list.min(), subr_end_z_list.max()))
+                a_step = abs(these_angles[1:] - these_angles[:-1]).mean()
+                distance = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None]).min()
+                distance_l = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None] - a_step).min()
+                distance_h = abs(np.mod(these_angles, np.pi * 2) - searched_angles[:, None] + a_step).min()
+                distance = np.array([distance, distance_h, distance_l]).min()
 
-        dtasrc_start_x, dtasrc_end_x, dtasrc_start_z, dtasrc_end_z = self.trimmed_floating_subregion
+                if distance > 2 * a_step:
+                    return
 
         self.chunk_reader.load_data(overwrite=True, sub_total_prange_slice=sub_total_prange_slice)
-
         if self.chunk_reader.dataset_subsampling > 1:
-            subsampling_file_slice = self._expand_slice(sub_total_prange_slice)
+            radios_angular_range_slicing = self._expand_slice(sub_total_prange_slice)
         else:
-            subsampling_file_slice = sub_total_prange_slice
-        my_subsampled_indexes = self.chunk_reader._sorted_files_indices[subsampling_file_slice]
+            radios_angular_range_slicing = sub_total_prange_slice
+        my_subsampled_indexes = self.chunk_reader._sorted_files_indices[radios_angular_range_slicing]
         data_raw = self.chunk_reader.data[: len(my_subsampled_indexes)]
 
         self.regular_accumulator.extract_preprocess_with_flats(
             subchunk_slice,
-            my_subsampled_indexes,
+            my_subsampled_indexes,  # these are indexes pointing within the global domain sequence which is composed of darks flats radios
             chunk_info,
             np.array((subr_start_z, subr_end_z), "i"),
             np.array((dtasrc_start_z, dtasrc_end_z), "i"),
             data_raw,
+            radios_angular_range_slicing  # my_subsampled_indexes  is important in order to compare the
+            # radios positions with respect to the flat position, and these position
+            # are given as the sequential acquisition number which counts everything ( flats, darks, radios )
+            # Insteqd, in order to access array which spans only the radios, we need to have an idea of where we are.
+            # this is provided by radios_angular_range_slicing which addresses the radios domain
         )
 
     def binning_expanded(self, region):
@@ -1116,14 +1157,24 @@ class HelicalChunkedRegriddedPipeline:
             self.recs_histogram = self.histogram.merge_histograms(self.histo_stack)
             self.histo_stack.clear()
 
-    def _write_data(self, data=None, counter=[0]):
+    def _write_data(self, data=None):
         if data is None:
             data = self.recs_stack
         my_kw_args = copy.copy(self._writer_exec_kwargs)
         if "config" in my_kw_args:
-            self.logger.info("omitting config in  writer because of too slow nexus writer ")
-            my_kw_args["config"] = {"test": counter[0]}
-            counter[0] += 1
+            self.logger.info(
+                "omitting config in  writer because of too slow nexus writer. Just writing the diagnostics, if any "
+            )
+
+        # diagnostic are saved here, with the Nabu mechanism for config
+        self.diagnostic_zpix_transl[:] = np.interp(
+            self.diagnostic_proj_angle,
+            np.deg2rad(self.span_info.projection_angles_deg_internally),
+            self.span_info.z_pix_per_proj,
+        )
+        self.diagnostic_zmm_transl[:] = self.diagnostic_zpix_transl * self.span_info.pix_size_mm
+
+        my_kw_args["config"] = self.diagnostic
 
         self.writer.write(data, *self._writer_exec_args, **my_kw_args)
         self.logger.info("Wrote %s" % self.writer.get_filename())
@@ -1181,12 +1232,6 @@ class HelicalChunkedRegriddedPipeline:
         """This will be used in the derived class to transfer data to gpu"""
         self.radios_slim[:] = self.radios[:, i_slice, :]
 
-    def reset_translation_diagnostics_accumulators(self):
-        self.diagnostic_radios[:] = 0
-        self.diagnostic_weights[:] = 0
-        self.diagnostic_proj_angle[1] = (2**30) * 3.14
-        self.diagnostic_proj_angle[0] = (2**30 - 1) * 3.14
-
     def process_chunk(self, sub_region=None):
         self._private_process_chunk(sub_region=sub_region)
         self._process_finalize()
@@ -1194,11 +1239,11 @@ class HelicalChunkedRegriddedPipeline:
     def _private_process_chunk(self, sub_region=None):
         assert sub_region is not None, "sub_region argument is mandatory in helical pipeline"
 
-        self.set_subregion(sub_region)
-        self.reset_translation_diagnostics_accumulators()
-        # self._allocate_reduced_radios()
+        # Every chunk has its diagnostic, that is good to follow the trends in helical scans
+        # or zstages
+        self._reset_diagnostics()
 
-        # self._allocate_reduced_gridded_and_subset_radios()
+        self.set_subregion(sub_region)
 
         (subr_start_x, subr_end_x, subr_start_z, subr_end_z) = self.sub_region
 
@@ -1218,8 +1263,13 @@ class HelicalChunkedRegriddedPipeline:
 
         my_first_pnum = proj_num_start
 
-        self.gridded_cumulated_weights[:] = 0
-        self.gridded_radios[:] = 0
+        if self.diag_zpro_run == 0:
+            # It may seem anodine, but setting a huge vector to zero
+            # takes time.
+            # In diagnostic collection mode we can spare it. On the other hand nothing has would be allocated for the data
+            # in such case
+            self.gridded_cumulated_weights[:] = 0
+            self.gridded_radios[:] = 0
 
         for pnum_start, pnum_end in zip(pnum_start_list, pnum_end_list):
             start_in_chunk = pnum_start - my_first_pnum
@@ -1229,60 +1279,102 @@ class HelicalChunkedRegriddedPipeline:
                 slice(pnum_start, pnum_end), slice(start_in_chunk, end_in_chunk), chunk_info
             )
 
-        self.gridded_radios[:] /= self.gridded_cumulated_weights
+        if not self.diag_zpro_run:
+            # when we collect diagnostics  we dont collect all the data
+            # so there would be nothing to process here
 
-        if "flatfield" in self._data_dump:
-            paganin_margin = self._phase_margin_up
-            if paganin_margin:
-                data_to_dump = self.gridded_radios[:, paganin_margin:-paganin_margin, :]
-            else:
-                data_to_dump = self.gridded_radios
-            self._dump_data_to_file("flatfield", data_to_dump)
-            if self.process_config.nabu_config["pipeline"]["skip_after_flatfield_dump"]:
-                return
+            self.gridded_radios[:] /= self.gridded_cumulated_weights
 
-        if "ccd_correction" in self.processing_steps:
-            self._ccd_corrections()
+            self.correct_for_missing_angles()
 
-        if ("phase" in self.processing_steps) or ("unsharp_mask" in self.processing_steps):
-            self._retrieve_phase()
-            if "unsharp_mask" in self.processing_steps:
-                self._apply_unsharp()
-        else:
-            self._nophase_put_to_radios(self.radios, self.gridded_radios)
+            linea = self.gridded_cumulated_weights.sum(axis=(1, 2))
+            i_zero_list = np.where(linea == 0)[0]
+            for i_zero in i_zero_list:
+                if i_zero > linea.shape[0] // 2:
+                    direction = -1
+                else:
+                    direction = 1
+                i = i_zero
+                while ((i >= 0 and direction == -1) or ((i < linea.shape[0] - 1) and direction == 1)) and linea[i] == 0:
+                    i += direction
+                if linea[i]:
+                    self.gridded_radios[i_zero] = self.gridded_radios[i]
+                    self.gridded_cumulated_weights[i_zero] = self.gridded_cumulated_weights[i]
+
+            if "flatfield" in self._data_dump:
+                paganin_margin = self._phase_margin_up
+                if paganin_margin:
+                    data_to_dump = self.gridded_radios[:, paganin_margin:-paganin_margin, :]
+                else:
+                    data_to_dump = self.gridded_radios
+                self._dump_data_to_file("flatfield", data_to_dump)
+                if self.process_config.nabu_config["pipeline"]["skip_after_flatfield_dump"]:
+                    return
+
+            if "ccd_correction" in self.processing_steps:
+                self._ccd_corrections()
+
+            if cxx_paganin is None:
+                if ("phase" in self.processing_steps) or ("unsharp_mask" in self.processing_steps):
+                    self._retrieve_phase()
+                    if "unsharp_mask" in self.processing_steps:
+                        self._apply_unsharp()
+                else:
+                    self._nophase_put_to_radios(self.radios, self.gridded_radios)
+            else:
+                if "phase" in self.processing_steps:
+                    pr = self.phase_retrieval
+                    paganin_l_micron = math.sqrt(pr.wavelength_micron * pr.distance_micron * pr.delta_beta * math.pi)
+                    cxx_paganin.paganin_pyhst(
+                        data_raw=self.gridded_radios,
+                        output=self.radios,
+                        num_of_threads=-1,
+                        paganin_marge=self._phase_margin_up,
+                        paganin_l_micron=paganin_l_micron / pr.pixel_size_micron,
+                        image_pixel_size_y=1.0,
+                        image_pixel_size_x=1.0,
+                        unsharp_sigma=self.unsharp_sigma,
+                        unsharp_coeff=self.unsharp_coeff,
+                        unsharp_LoG=int((self.unsharp_method == "log")),
+                    )
+                else:
+                    self._nophase_put_to_radios(self.radios, self.gridded_radios)
 
-        self.logger.info(" LOG ")
-        self._nophase_put_to_radios(self.radios_weights, self.gridded_cumulated_weights)
+            self.logger.info(" LOG ")
+            self._nophase_put_to_radios(self.radios_weights, self.gridded_cumulated_weights)
 
-        # print( " processing steps ", self.processing_steps )
-        # ['read_chunk', 'flatfield', 'double_flatfield', 'take_log', 'reconstruction', 'save']
+            # print( " processing steps ", self.processing_steps )
+            # ['read_chunk', 'flatfield', 'double_flatfield', 'take_log', 'reconstruction', 'save']
 
-        if "take_log" in self.processing_steps:
-            self._take_log()
+            if "take_log" in self.processing_steps:
+                self._take_log()
 
-        self.logger.info(" BALANCE ")
+            self.logger.info(" BALANCE ")
 
-        self.balance_weights()
+            self.balance_weights()
 
-        num_slices = self.radios.shape[1]
+            num_slices = self.radios.shape[1]
 
-        self.logger.info(" NORMALIZE")
-        self._normalize_sinos()
-        self._dump_sinogram()
+            self.logger.info(" NORMALIZE")
+            self._normalize_sinos()
+            self._dump_sinogram()
 
         if "reconstruction" in self.processing_steps:
-            for i_slice in range(num_slices):
-                self._post_primary_data_reduction(i_slice)  # charge on self.radios_slim
+            if not self.diag_zpro_run:
+                # otherwise, when collecting diagnostic, we are not interested in the remaining steps
+                # on the other hand there would be nothing to process because only diagnostics have been collected
+                for i_slice in range(num_slices):
+                    self._post_primary_data_reduction(i_slice)  # charge on self.radios_slim
 
-                self._filter()
+                    self._filter()
 
-                self.apply_weights(i_slice)
+                    self.apply_weights(i_slice)
 
-                self._build_sino()
+                    self._build_sino()
 
-                self._reconstruct(chunk_info=chunk_info, i_slice=i_slice)
+                    self._reconstruct(chunk_info=chunk_info, i_slice=i_slice)
 
-                self._compute_histogram(i_slice=i_slice, num_slices=num_slices)
+                    self._compute_histogram(i_slice=i_slice, num_slices=num_slices)
 
             self._write_data()
 
@@ -1299,9 +1391,27 @@ class HelicalChunkedRegriddedPipeline:
                 : end_angle_index - first_angle_index
             ]
 
+    def correct_for_missing_angles(self):
+        """For non helical scan, the rotation is often incomplete ( < 360)
+        here we complement the missing angles
+        """
+        linea = self.gridded_cumulated_weights.sum(axis=(1, 2))
+        i_zero_list = np.where(linea == 0)[0]
+        for i_zero in i_zero_list:
+            if i_zero > linea.shape[0] // 2:
+                direction = -1
+            else:
+                direction = 1
+            i = i_zero
+            while ((i >= 0 and direction == -1) or ((i < linea.shape[0] - 1) and direction == 1)) and linea[i] == 0:
+                i += direction
+            if linea[i]:
+                self.gridded_radios[i_zero] = self.gridded_radios[i]
+                self.gridded_cumulated_weights[i_zero] = self.gridded_cumulated_weights[i]
+
     @classmethod
     def estimate_required_memory(
-        cls, process_config, reading_granularity=None, chunk_size=None, margin_v=0, span_info=None
+        cls, process_config, reading_granularity=None, chunk_size=None, margin_v=0, span_info=None, diag_zpro_run=0
     ):
         """
         Estimate the memory (RAM) needed for a reconstruction.
@@ -1340,11 +1450,12 @@ class HelicalChunkedRegriddedPipeline:
         binning_z = nabu_config["dataset"]["binning_z"]
         projections_subsampling = nabu_config["dataset"]["projections_subsampling"]
 
-        # the gridded target
-        total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
+        if not diag_zpro_run:
+            # the gridded target
+            total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
 
-        # the gridded weights
-        total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
+            # the gridded weights
+            total_memory_needed += Nx * (2 * margin_v + chunk_size) * n_gridded_angles * 4
 
         # the read grain
         total_memory_needed += (
@@ -1383,7 +1494,7 @@ class HelicalChunkedRegriddedPipeline:
         # Reconstruction
         # ---------------
         reconstructed_volume_size = 0
-        if "reconstruction" in processing_steps:
+        if "reconstruction" in processing_steps and not diag_zpro_run:
             ##  radios_slim is used to process one slice at once, It will be on the gpu
             ##  and cannot be reduced further, therefore no need to estimate it.
             ##  Either it passes or it does not.