nabu 2024.2.14__py3-none-any.whl → 2025.1.0__py3-none-any.whl

nabu/pipeline/datadump.py

@@ -126,7 +126,7 @@ class DataDumpManager:
         read_opts = self.processing_options["read_chunk"]
         if read_opts.get("process_file", None) is None:
             return None
-        dump_start_z, dump_end_z = read_opts["dump_start_z"], read_opts["dump_end_z"]
+        dump_start_z, dump_end_z = read_opts["dump_start_z"], read_opts["dump_end_z"]  # noqa: F841
         relative_start_z = self.z_min - dump_start_z
         relative_end_z = relative_start_z + self.delta_z
         # When using binning, every step after "read" results in smaller-sized data.
@@ -139,7 +139,7 @@ class DataDumpManager:
 
     def _check_resume_from_step(self):
         read_opts = self.processing_options["read_chunk"]
-        expected_radios_shape = get_hdf5_dataset_shape(
+        expected_radios_shape = get_hdf5_dataset_shape(  # noqa: F841
             read_opts["process_file"],
             read_opts["process_h5_path"],
             sub_region=self.get_read_dump_subregion(),
@@ -151,7 +151,7 @@ class DataDumpManager:
             return
         writer = self.data_dump[step_name]
         self.logger.info("Dumping data to %s" % writer.fname)
-        if __has_pycuda__:
+        if __has_pycuda__:  # noqa: SIM102
             if isinstance(data, garray.GPUArray):
                 data = data.get()
 

nabu/pipeline/estimators.py

@@ -9,6 +9,7 @@ import scipy.fft  #  pylint: disable=E0611
 from silx.io import get_data
 import math
 from scipy import ndimage as nd
+
 from ..preproc.flatfield import FlatField
 from ..estimation.cor import (
     CenterOfRotation,
@@ -17,8 +18,10 @@ from ..estimation.cor import (
     CenterOfRotationGrowingWindow,
     CenterOfRotationOctaveAccurate,
 )
+from .. import version as nabu_version
 from ..estimation.cor_sino import SinoCorInterface, CenterOfRotationFourierAngles, CenterOfRotationVo
 from ..estimation.tilt import CameraTilt
+from ..estimation.motion import MotionEstimation
 from ..estimation.utils import is_fullturn_scan
 from ..resources.logger import LoggerOrPrint
 from ..resources.utils import extract_parameters
@@ -298,7 +301,7 @@ class SinoCORFinder(CORFinderBase):
         """
         Initialize a SinoCORFinder object.
 
-        Other parameters
+        Other Parameters
         ----------------
         The following keys can be set in cor_options.
 
@@ -341,7 +344,7 @@ class SinoCORFinder(CORFinderBase):
                 self.angles = self.dataset_info.rotation_angles[::subsampling]
                 self.subsampling = subsampling
         else:  # Angular step
-            raise NotImplementedError()
+            raise NotImplementedError
 
     def _load_raw_sinogram(self):
         if self.slice_idx is None:
@@ -441,7 +444,7 @@ class CompositeCORFinder(CORFinderBase):
 
     search_methods = {
         "composite-coarse-to-fine": {
-            "class": CenterOfRotation,  # Hack. Not used. Everything is done in the find_cor() func.
+            "class": CenterOfRotation,  # Not used. Everything is done in the find_cor() func.
         }
     }
     _default_cor_options = {"low_pass": 0.4, "high_pass": 10, "side": "near", "near_pos": 0, "near_width": 40}
@@ -542,7 +545,7 @@ class CompositeCORFinder(CORFinderBase):
         else:
             my_flats = None
 
-        if my_flats is not None and len(list(my_flats.keys())):
+        if my_flats is not None and len(list(my_flats.keys())) > 0:
             self.use_flat = True
             self.flatfield = FlatField(
                 (len(self.absolute_indices), self.sy, self.sx),
@@ -750,15 +753,15 @@ class CompositeCORFinder(CORFinderBase):
                 my_blurred_radio1 = np.fliplr(blurred_radio1)
                 my_blurred_radio2 = np.fliplr(blurred_radio2)
 
-            common_left = np.fliplr(my_radio1[:, ovsd_sx - my_z :])[:, : -int(math.ceil(self.ovs * self.high_pass * 2))]
+            common_left = np.fliplr(my_radio1[:, ovsd_sx - my_z :])[:, : -math.ceil(self.ovs * self.high_pass * 2)]
             # adopt a 'safe' margin considering high_pass value (possibly float)
-            common_right = my_radio2[:, ovsd_sx - my_z : -int(math.ceil(self.ovs * self.high_pass * 2))]
+            common_right = my_radio2[:, ovsd_sx - my_z : -math.ceil(self.ovs * self.high_pass * 2)]
 
             common_blurred_left = np.fliplr(my_blurred_radio1[:, ovsd_sx - my_z :])[
-                :, : -int(math.ceil(self.ovs * self.high_pass * 2))
+                :, : -math.ceil(self.ovs * self.high_pass * 2)
             ]
             # adopt a 'safe' margin considering high_pass value (possibly float)
-            common_blurred_right = my_blurred_radio2[:, ovsd_sx - my_z : -int(math.ceil(self.ovs * self.high_pass * 2))]
+            common_blurred_right = my_blurred_radio2[:, ovsd_sx - my_z : -math.ceil(self.ovs * self.high_pass * 2)]
 
             if common_right.size == 0:
                 continue
@@ -786,7 +789,7 @@ class CompositeCORFinder(CORFinderBase):
         elif self.norm_order == 1:
             return self.error_metric_l1(common_right, common_left, common_blurred_right, common_blurred_left)
         else:
-            assert False, "this cannot happen"
+            raise RuntimeError("this cannot happen")
 
     def error_metric_l2(self, common_right, common_left):
         common = common_right - common_left
@@ -818,10 +821,10 @@ def oversample(radio, ovs_s):
     # Pre-initialisation: The original data falls exactly on the following strided positions in the new data array.
     result[:: ovs_s[0], :: ovs_s[1]] = radio
 
-    for k in range(0, ovs_s[0]):
+    for k in range(ovs_s[0]):
         # interpolation coefficient for axis 0
         g = k / ovs_s[0]
-        for i in range(0, ovs_s[1]):
+        for i in range(ovs_s[1]):
             if i == 0 and k == 0:
                 # this case subset was already exactly matched from before the present double loop,
                 # in the pre-initialisation line.
@@ -989,3 +992,260 @@ class DetectorTiltEstimator:
 
 # alias
 TiltFinder = DetectorTiltEstimator
+
+
+def estimate_translations(dataset_info, do_flatfield=True): ...
+
+
+class TranslationsEstimator:
+
+    _default_extra_options = {
+        "window_size": 300,
+    }
+
+    def __init__(
+        self,
+        dataset_info,
+        do_flatfield=True,
+        rot_center=None,
+        halftomo_side=None,
+        angular_subsampling=10,
+        deg_xy=2,
+        deg_z=2,
+        shifts_estimator="phase_cross_correlation",
+        radios_filter=None,
+        extra_options=None,
+    ):
+        self._configure_extra_options(extra_options)
+        self.logger = LoggerOrPrint(dataset_info.logger)
+        self.dataset_info = dataset_info
+        self.angular_subsampling = angular_subsampling
+        self.do_360 = self.dataset_info.is_360
+        self.do_flatfield = do_flatfield
+        self.radios_filter = radios_filter
+        self.radios = None
+        self._deg_xy = deg_xy
+        self._deg_z = deg_z
+        self._shifts_estimator = shifts_estimator
+        self._shifts_estimator_kwargs = {}
+        self._cor = rot_center
+        self._configure_halftomo(halftomo_side)
+        self._estimate_cor = self._cor is None
+        self.sample_shifts_xy = None
+        self.sample_shifts_z = None
+
+    def _configure_extra_options(self, extra_options):
+        self.extra_options = self._default_extra_options.copy()
+        self.extra_options.update(extra_options or {})
+
+    def _configure_halftomo(self, halftomo_side):
+        if halftomo_side is False:
+            # Force disable halftomo
+            self.halftomo_side = False
+            return
+        self._start_x = None
+        self._end_x = None
+        if (halftomo_side is not None) and not (self.do_360):
+            raise ValueError(
+                "Expected 360° dataset for half-tomography, but this dataset does not look like a 360° dataset"
+            )
+        if halftomo_side is None:
+            if self.dataset_info.is_halftomo:
+                halftomo_side = "right"
+            else:
+                self.halftomo_side = False
+                return
+        self.halftomo_side = halftomo_side
+        window_size = self.extra_options["window_size"]
+        if self._cor is not None:
+            # In this case we look for shifts around the CoR
+            self._start_x = int(self._cor - window_size / 2)
+            self._end_x = int(self._cor + window_size / 2)
+        elif halftomo_side == "right":
+            self._start_x = -window_size
+            self._end_x = None
+        elif halftomo_side == "left":
+            self._start_x = 0
+            self._end_x = window_size
+        elif is_scalar(halftomo_side):
+            # Expect approximate location of CoR, relative to left-most column
+            self._start_x = int(halftomo_side - window_size / 2)
+            self._end_x = int(halftomo_side + window_size / 2)
+        else:
+            raise ValueError(
+                f"Expected 'halftomo_side' to be either 'left', 'right', or an integer (got {halftomo_side})"
+            )
+        self.logger.debug(f"[MotionEstimation] Half-tomo looking at [{self._start_x}:{self._end_x}]")
+        # For half-tomo, skimage.registration.phase_cross_correlation might look a bit too far away
+        if (
+            self._shifts_estimator == "phase_cross_correlation"
+            and self._shifts_estimator_kwargs.get("overlap_ratio", 0.3) >= 0.3
+        ):
+            self._shifts_estimator_kwargs.update({"overlap_ratio": 0.2})
+        #
+
+    def _load_data(self):
+        self.logger.debug("[MotionEstimation] reading data")
+        if self.do_360:
+            """
+            In this case we compare pair of opposite projections.
+            If rotation angles are arbitrary, we should do something like
+              for angle in dataset_info.rotation_angles:
+                  img, angle_deg, idx = dataset_info.get_image_at_angle(
+                      np.degrees(angle)+180, return_angle_and_index=True
+                  )
+            Most of the time (always ?), the dataset was acquired with a circular trajectory,
+            so we can use angles:
+                dataset_info.rotation_angles[::self.angular_subsampling]
+            which amounts to reading one radio out of "angular_subsampling"
+            """
+
+            # TODO account for more general rotation angles. The following will only work for circular trajectory and ordered angles
+            self._reader = self.dataset_info.get_reader(
+                sub_region=(slice(None, None, self.angular_subsampling), slice(None), slice(None))
+            )
+            self.radios = self._reader.load_data()
+            self.angles = self.dataset_info.rotation_angles[:: self.angular_subsampling]
+            self._radios_idx = self._reader.get_frames_indices()
+            self.logger.debug("[MotionEstimation] This is a 360° scan, will use pairs of opposite projections")
+        else:
+            """
+            In this case we use the "return projections", i.e special projections acquired at several angles
+            (eg. [180, 90, 0]) before ending the scan
+            """
+            return_projs, return_angles_deg, return_idx = self.dataset_info.get_alignment_projections()
+            self._angles_return = np.radians(return_angles_deg)
+            self._radios_return = return_projs
+            self._radios_idx_return = return_idx
+
+            projs = []
+            angles_rad = []
+            projs_idx = []
+            for angle_deg in return_angles_deg:
+                proj, rot_angle_deg, proj_idx = self.dataset_info.get_image_at_angle(
+                    angle_deg, image_type="projection", return_angle_and_index=True
+                )
+                projs.append(proj)
+                angles_rad.append(np.radians(rot_angle_deg))
+                projs_idx.append(proj_idx)
+            self._radios_outwards = np.array(projs)
+            self._angles_outward = np.array(angles_rad)
+            self._radios_idx_outwards = np.array(projs_idx)
+            self.logger.debug("[MotionEstimation] This is a 180° scan, will use 'return projections'")
+
+    def _apply_flatfield(self):
+        if not (self.do_flatfield):
+            return
+        self.logger.debug("[MotionEstimation] flatfield")
+        if self.do_360:
+            self._flatfield = FlatField(
+                self.radios.shape,
+                flats=self.dataset_info.flats,
+                darks=self.dataset_info.darks,
+                radios_indices=self._radios_idx,
+            )
+            self._flatfield.normalize_radios(self.radios)
+        else:
+            # 180 + return projs
+            self._flatfield_outwards = FlatField(
+                self._radios_outwards.shape,
+                flats=self.dataset_info.flats,
+                darks=self.dataset_info.darks,
+                radios_indices=self._radios_idx_outwards,
+            )
+            self._flatfield_outwards.normalize_radios(self._radios_outwards)
+            self._flatfield_return = FlatField(
+                self._radios_return.shape,
+                flats=self.dataset_info.flats,
+                darks=self.dataset_info.darks,
+                radios_indices=self._radios_idx_return,
+            )
+            self._flatfield_outwards.normalize_radios(self._radios_return)
+
+    def estimate_motion(self):
+        self._load_data()
+        self._apply_flatfield()
+        if self.radios_filter is not None:
+            self.logger.debug("[MotionEstimation] applying radios filter")
+            self.radios_filter(self.radios)
+
+        n_projs_tot = self.dataset_info.n_angles
+        if self.do_360:
+            n_a = self.radios.shape[0]
+            # See notes above - this works only for circular trajectory / ordered angles
+            projs_stack1 = self.radios[: n_a // 2]
+            projs_stack2 = self.radios[n_a // 2 :]
+            angles1 = self.angles[: n_a // 2]
+            angles2 = self.angles[n_a // 2 :]
+            indices1 = (self._radios_idx - self._radios_idx[0])[: n_a // 2]
+            indices2 = (self._radios_idx - self._radios_idx[0])[n_a // 2 :]
+        else:
+            projs_stack1 = self._radios_outwards
+            projs_stack2 = self._radios_return
+            angles1 = self._angles_outward
+            angles2 = self._angles_return
+            indices1 = self._radios_idx_outwards - self._radios_idx_outwards.min()
+            indices2 = self._radios_idx_return - self._radios_idx_outwards.min()
+
+        if self._start_x is not None:
+            # Compute Motion Estimation on subset of images (eg. for half-tomo)
+            projs_stack1 = projs_stack1[..., self._start_x : self._end_x]
+            projs_stack2 = projs_stack2[..., self._start_x : self._end_x]
+
+        self.motion_estimator = MotionEstimation(
+            projs_stack1,
+            projs_stack2,
+            angles1,
+            angles2,
+            indices1,
+            indices2,
+            n_projs_tot,
+            shifts_estimator=self._shifts_estimator,
+            shifts_estimator_kwargs=self._shifts_estimator_kwargs,
+        )
+
+        self.logger.debug("[MotionEstimation] estimating shifts")
+
+        estimated_shifts_v = self.motion_estimator.estimate_vertical_motion(degree=self._deg_z)
+        estimated_shifts_h, cor = self.motion_estimator.estimate_horizontal_motion(degree=self._deg_xy, cor=self._cor)
+        if self._start_x is not None:
+            cor += (self._start_x % self.radios.shape[-1]) + (projs_stack1.shape[-1] - 1) / 2.0
+
+        self.sample_shifts_xy = estimated_shifts_h
+        self.sample_shifts_z = estimated_shifts_v
+        if self._cor is None:
+            self.logger.info(
+                "[MotionEstimation] Estimated center of rotation (relative to left-most pixel): %.2f" % cor
+            )
+        return estimated_shifts_h, estimated_shifts_v, cor
+
+    def generate_translations_movements_file(self, filename, fmt="%.3f", only=None):
+        if self.sample_shifts_xy is None:
+            raise RuntimeError("Need to run estimate_motion() first")
+
+        angles = self.dataset_info.rotation_angles
+        cor = self._cor or 0
+        txy_est_all_angles = self.motion_estimator.apply_fit_horiz(angles=angles)
+        tz_est_all_angles = self.motion_estimator.apply_fit_vertic(angles=angles)
+        estimated_shifts_vu_all_angles = self.motion_estimator.convert_sample_motion_to_detector_shifts(
+            txy_est_all_angles, tz_est_all_angles, angles, cor=cor
+        )
+        estimated_shifts_vu_all_angles[:, 1] -= cor
+        correct_shifts_uv = -estimated_shifts_vu_all_angles[:, ::-1]
+
+        if only is not None:
+            if only == "horizontal":
+                correct_shifts_uv[:, 1] = 0
+            elif only == "vertical":
+                correct_shifts_uv[:, 0] = 0
+            else:
+                raise ValueError("Expected 'only' to be either None, 'horizontal' or 'vertical'")
+
+        header = f"Generated by nabu {nabu_version} : {str(self)}"
+        np.savetxt(filename, correct_shifts_uv, fmt=fmt, header=header)
+
+    def __str__(self):
+        ret = f"{self.__class__.__name__}(do_flatfield={self.do_flatfield}, rot_center={self._cor}, angular_subsampling={self.angular_subsampling})"
+        if self.sample_shifts_xy is not None:
+            ret += f", shifts_estimator={self.motion_estimator.shifts_estimator}"
+        return ret

nabu/pipeline/fullfield/chunked.py

@@ -1,6 +1,5 @@
 from os import path
 from time import time
-from math import ceil
 import numpy as np
 from silx.io.url import DataUrl
 
@@ -10,7 +9,7 @@ from ...resources.utils import extract_parameters
 from ...misc.binning import binning as image_binning
 from ...io.reader import EDFStackReader, HDF5Loader, NXTomoReader
 from ...preproc.ccd import Log, CCDFilter
-from ...preproc.flatfield import FlatField
+from ...preproc.flatfield import FlatField, PCAFlatsNormalizer
 from ...preproc.distortion import DistortionCorrection
 from ...preproc.shift import VerticalShift
 from ...preproc.double_flatfield import DoubleFlatField
@@ -18,7 +17,7 @@ from ...preproc.phase import PaganinPhaseRetrieval
 from ...preproc.ctf import CTFPhaseRetrieval, GeoPars
 from ...reconstruction.sinogram import SinoNormalization
 from ...reconstruction.filtering import SinoFilter
-from ...reconstruction.mlem import __have_corrct__, MLEMReconstructor
+from ...reconstruction.mlem import MLEMReconstructor
 from ...processing.rotation import Rotation
 from ...reconstruction.rings import MunchDeringer, SinoMeanDeringer, VoDeringer
 from ...processing.unsharp import UnsharpMask
@@ -45,6 +44,7 @@ class ChunkedPipeline:
 
     backend = "numpy"
     FlatFieldClass = FlatField
+    PCAFlatFieldClass = PCAFlatsNormalizer
     DoubleFlatFieldClass = DoubleFlatField
     CCDCorrectionClass = CCDFilter
     PaganinPhaseRetrievalClass = PaganinPhaseRetrieval
@@ -99,7 +99,6 @@ class ChunkedPipeline:
 
         Notes
         ------
-
         Using `margin` results in a lesser number of reconstructed slices.
         More specifically, if `margin = (V, H)`, then there will be `delta_z - 2*V`
         reconstructed slices (if the sub-region is in the middle of the volume)
@@ -127,10 +126,10 @@ class ChunkedPipeline:
         if len(chunk_shape) != 3:
             raise ValueError("Expected chunk_shape to be a tuple of length 3 in the form (n_z, n_y, n_x)")
         self.chunk_shape = tuple(int(c) for c in chunk_shape)  # cast to int, as numpy.int64 can make pycuda crash
-        # TODO: sanity check (eg. compare to size of radios in dataset_info) ?
+        ss_start = getattr(self.process_config, "subsampling_start", 0)
         # (n_a, n_z, n_x)
         self.radios_shape = (
-            ceil(self.chunk_shape[0] / self.process_config.subsampling_factor),
+            np.arange(self.chunk_shape[0])[ss_start :: self.process_config.subsampling_factor].size,
             self.chunk_shape[1] // self.process_config.binning[1],
             self.chunk_shape[2] // self.process_config.binning[0],
         )
@@ -175,7 +174,7 @@ class ChunkedPipeline:
             Data volume sub-region, in the form ((start_a, end_a), (start_z, end_z), (start_x, end_x))
             where the data volume has a layout (angles, Z, X)
         """
-        n_angles = self.dataset_info.n_angles
+        # n_angles = self.dataset_info.n_angles
         n_x, n_z = self.dataset_info.radio_dims
         c_a, c_z, c_x = self.chunk_shape
         if sub_region is None:
@@ -190,7 +189,7 @@ class ChunkedPipeline:
             # check sub-region
             for i, start_end in enumerate(sub_region):
                 start, end = start_end
-                if start is not None and end is not None:
+                if start is not None and end is not None:  # noqa: SIM102
                     if end - start != self.chunk_shape[i]:
                         raise ValueError(
                             "Invalid (start, end)=(%d, %d) for sub-region (dimension %d): chunk shape is %s, but %d-%d=%d != %d"
@@ -340,13 +339,28 @@ class ChunkedPipeline:
             subs_z = None
             subs_x = None
             angular_sub_region = slice(*(self.sub_region[0]))
+
+            # exclude(subsample(.)) != subsample(exclude(.))
+            # Here we want the latter: first exclude the user-defined angular range, and then subsample the remaining indices
+            if len(self.dataset_info.get_excluded_projections_indices()) > 0:
+                angular_sub_region = np.array(
+                    [
+                        self.dataset_info.index_to_proj_number(i)
+                        for i in sorted(list(self.dataset_info.projections.keys()))
+                    ]
+                )
             if self.process_config.subsampling_factor:
                 subs_angles = self.process_config.subsampling_factor
-                angular_sub_region = slice(
-                    getattr(self.process_config, "subsampling_start", 0) + self.sub_region[0][0],
-                    self.sub_region[0][1],
-                    subs_angles,
-                )
+                start = getattr(self.process_config, "subsampling_start", 0) + self.sub_region[0][0]
+                if isinstance(angular_sub_region, slice):
+                    angular_sub_region = slice(
+                        start,
+                        self.sub_region[0][1],
+                        subs_angles,
+                    )
+                else:
+                    angular_sub_region = angular_sub_region[start::subs_angles]
+
             reader_sub_region = (
                 angular_sub_region,
                 slice(*(self.sub_region[1]) + ((subs_z,) if subs_z else ())),
@@ -363,7 +377,7 @@ class ChunkedPipeline:
             if self.dataset_info.kind == "nx":
                 self.chunk_reader = NXTomoReader(
                     self.dataset_info.dataset_hdf5_url.file_path(),
-                    self.dataset_info.dataset_hdf5_url.data_path(),
+                    data_path=self.dataset_info.dataset_hdf5_url.data_path(),
                     sub_region=reader_sub_region,
                     image_key=0,
                     **other_reader_kwargs,
@@ -394,50 +408,68 @@ class ChunkedPipeline:
 
     @use_options("flatfield", "flatfield")
     def _init_flatfield(self):
-        self._ff_options = self.processing_options["flatfield"].copy()
-
-        # This won't work when resuming from a step (i.e before FF), because we rely on H5Loader()
-        # which re-compacts the data. When data is re-compacted, we have to know the original radios positions.
-        # These positions can be saved in the "file_dump" metadata, but it is not loaded for now
-        # (the process_config object is re-built from scratch every time)
-        self._ff_options["projs_indices"] = self.chunk_reader.get_frames_indices()
-
-        if self._ff_options.get("normalize_srcurrent", False):
-            a_start_idx, a_end_idx = self.sub_region[0]
-            subs = self.process_config.subsampling_factor
-            self._ff_options["radios_srcurrent"] = self._ff_options["radios_srcurrent"][a_start_idx:a_end_idx:subs]
-
-        distortion_correction = None
-        if self._ff_options["do_flat_distortion"]:
-            self.logger.info("Flats distortion correction will be applied")
-            self.FlatFieldClass = FlatField  # no GPU implementation available, force this backend
-            estimation_kwargs = {}
-            estimation_kwargs.update(self._ff_options["flat_distortion_params"])
-            estimation_kwargs["logger"] = self.logger
-            distortion_correction = DistortionCorrection(
-                estimation_method="fft-correlation", estimation_kwargs=estimation_kwargs, correction_method="interpn"
-            )
+        if self.processing_options["flatfield"]:
+            self._ff_options = self.processing_options["flatfield"].copy()
+
+            # This won't work when resuming from a step (i.e before FF), because we rely on H5Loader()
+            # which re-compacts the data. When data is re-compacted, we have to know the original radios positions.
+            # These positions can be saved in the "file_dump" metadata, but it is not loaded for now
+            # (the process_config object is re-built from scratch every time)
+            self._ff_options["projs_indices"] = self.chunk_reader.get_frames_indices()
+
+            if self._ff_options.get("normalize_srcurrent", False):
+                a_start_idx, a_end_idx = self.sub_region[0]
+                subs = self.process_config.subsampling_factor
+                self._ff_options["radios_srcurrent"] = self._ff_options["radios_srcurrent"][a_start_idx:a_end_idx:subs]
+
+            distortion_correction = None
+            if self._ff_options["do_flat_distortion"]:
+                self.logger.info("Flats distortion correction will be applied")
+                self.FlatFieldClass = FlatField  # no GPU implementation available, force this backend
+                estimation_kwargs = {}
+                estimation_kwargs.update(self._ff_options["flat_distortion_params"])
+                estimation_kwargs["logger"] = self.logger
+                distortion_correction = DistortionCorrection(
+                    estimation_method="fft-correlation",
+                    estimation_kwargs=estimation_kwargs,
+                    correction_method="interpn",
+                )
 
-        # Reduced darks/flats are loaded, but we have to crop them on the current sub-region
-        # and possibly do apply some pre-processing (binning, distortion correction, ...)
-        darks_flats = load_darks_flats(
-            self.dataset_info,
-            self.sub_region[1:],
-            processing_func=self._ff_processing_function,
-            processing_func_args=self._ff_processing_function_args,
-        )
+            if self.processing_options["flatfield"]["method"].lower() != "pca":
+                # Reduced darks/flats are loaded, but we have to crop them on the current sub-region
+                # and possibly do apply some pre-processing (binning, distortion correction, ...)
+                darks_flats = load_darks_flats(
+                    self.dataset_info,
+                    self.sub_region[1:],
+                    processing_func=self._ff_processing_function,
+                    processing_func_args=self._ff_processing_function_args,
+                )
 
-        # FlatField parameter "radios_indices" must account for subsampling
-        self.flatfield = self.FlatFieldClass(
-            self.radios_shape,
-            flats=darks_flats["flats"],
-            darks=darks_flats["darks"],
-            radios_indices=self._ff_options["projs_indices"],
-            interpolation="linear",
-            distortion_correction=distortion_correction,
-            radios_srcurrent=self._ff_options["radios_srcurrent"],
-            flats_srcurrent=self._ff_options["flats_srcurrent"],
-        )
+                # FlatField parameter "radios_indices" must account for subsampling
+                self.flatfield = self.FlatFieldClass(
+                    self.radios_shape,
+                    flats=darks_flats["flats"],
+                    darks=darks_flats["darks"],
+                    radios_indices=self._ff_options["projs_indices"],
+                    interpolation="linear",
+                    distortion_correction=distortion_correction,
+                    radios_srcurrent=self._ff_options["radios_srcurrent"],
+                    flats_srcurrent=self._ff_options["flats_srcurrent"],
+                )
+            else:
+                flats = self.process_config.dataset_info.flats
+                darks = self.process_config.dataset_info.darks
+                if len(darks) != 1:
+                    raise ValueError(f"There should be only one reduced dark. Found {len(darks)}.")
+                else:
+                    dark_key = list(darks.keys())[0]
+                nb_pca_components = len(flats) - 1
+                img_subregion = tuple(slice(*sr) for sr in self.sub_region[1:])
+                self.flatfield = self.PCAFlatFieldClass(
+                    np.array([flats[k][img_subregion] for k in range(1, nb_pca_components)]),
+                    darks[dark_key][img_subregion],
+                    flats[0][img_subregion],  # Mean
+                )
 
     @use_options("double_flatfield", "double_flatfield")
     def _init_double_flatfield(self):
@@ -630,13 +662,14 @@ class ChunkedPipeline:
                     "clip_outer_circle": options["clip_outer_circle"],
                     "outer_circle_value": options["outer_circle_value"],
                     "filter_cutoff": options["fbp_filter_cutoff"],
+                    "crop_filtered_data": options["crop_filtered_data"],
                 },
             )
 
         if options["method"] == "mlem" and options["implementation"] in (None, "corrct"):
             self.reconstruction = self.MLEMClass(  # pylint: disable=E1102
                 (self.radios_shape[1],) + self.sino_shape,
-                angles_rad=-options["angles"],  # WARNING: mind the sign...
+                angles_rad=options["angles"],
                 shifts_uv=self.dataset_info.translations,  # In config file, one line per proj, each line is (tu,tv). Corrct expects one col per proj and (tv,tu).
                 cor=options["rotation_axis_position"],
                 n_iterations=options["iterations"],
@@ -647,9 +680,19 @@ class ChunkedPipeline:
                     "v_max_for_v_shifts": None,
                     "v_min_for_u_shifts": 0,
                     "v_max_for_u_shifts": None,
+                    "scale_factor": 1.0 / options["voxel_size_cm"][0],
+                    "clip_outer_circle": options["clip_outer_circle"],
+                    "outer_circle_value": options["outer_circle_value"],
+                    "filter_cutoff": options["fbp_filter_cutoff"],
+                    "crop_filtered_data": options["crop_filtered_data"],
                 },
             )
 
+        if options.get("crop_filtered_data", True) is False:
+            self.logger.warning(
+                "Using [reconstruction] crop_filtered_data = False. This will use a large amount of memory."
+            )
+
         self._allocate_recs(*self.process_config.rec_shape, n_slices=n_slices)
         n_a, _, n_x = self.radios_cropped_shape
         self._tmp_sino = self._allocate_array((n_a, n_x), "f", name="tmp_sino")
@@ -806,15 +849,7 @@ class ChunkedPipeline:
         """
         This reconstructs the entire sinograms stack at once
         """
-
-        n_angles, n_z, n_x = self.radios.shape
-
-        # FIXME
-        # can't do a discontiguous single copy...
-        sinos_contig = self._allocate_array((n_z, n_angles, n_x), np.float32, "sinos_cone")
-        for i in range(n_z):
-            sinos_contig[i] = self.radios[:, i, :]
-        # ---
+        sinos_discontig = self.radios.transpose(axes=(1, 0, 2))  # view
 
         # In principle radios are not cropped at this stage,
         # so self.sub_region[2][0] can be used instead of self.get_slice_start_index() instead of self.sub_region[2][0]
@@ -822,7 +857,8 @@ class ChunkedPipeline:
         n_z_tot = self.process_config.radio_shape(binning=True)[0]
 
         self.reconstruction.reconstruct(  # pylint: disable=E1101
-            sinos_contig,
+            # sinos_contig,
+            sinos_discontig,
             output=self.recs,
             relative_z_position=((z_min + z_max) / self.process_config.binning_z / 2) - n_z_tot / 2,
         )