nabu 2025.1.0.dev5__tar.gz → 2025.1.0.dev12__tar.gz

{nabu-2025.1.0.dev5/nabu.egg-info → nabu-2025.1.0.dev12}/PKG-INFO

@@ -1,31 +1,10 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: nabu
-Version: 2025.1.0.dev5
+Version: 2025.1.0.dev12
 Summary: Nabu - Tomography software
 Author-email: Pierre Paleo <pierre.paleo@esrf.fr>, Henri Payno <henri.payno@esrf.fr>, Alessandro Mirone <mirone@esrf.fr>, Jérôme Lesaint <jerome.lesaint@esrf.fr>
 Maintainer-email: Pierre Paleo <pierre.paleo@esrf.fr>
-License: MIT License
-        
-        Copyright (c) 2020-2024 ESRF
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-        
+License-Expression: MIT
 Project-URL: Homepage, https://gitlab.esrf.fr/tomotools/nabu
 Project-URL: Documentation, http://www.silx.org/pub/nabu/doc
 Project-URL: Repository, https://gitlab.esrf.fr/tomotools/nabu/-/releases
@@ -35,25 +14,26 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Environment :: Console
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: Unix
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: POSIX
 Classifier: Topic :: Scientific/Engineering :: Physics
 Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
-Requires-Python: >=3.7
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy>1.9.0
 Requires-Dist: scipy
 Requires-Dist: h5py>=3.0
 Requires-Dist: silx>=0.15.0
-Requires-Dist: tomoscan>=2.1.0
+Requires-Dist: tomoscan>=2.1.5
 Requires-Dist: psutil
 Requires-Dist: pytest
 Requires-Dist: tifffile
@@ -77,6 +57,7 @@ Requires-Dist: sphinx; extra == "doc"
 Requires-Dist: cloud_sptheme; extra == "doc"
 Requires-Dist: myst-parser; extra == "doc"
 Requires-Dist: nbsphinx; extra == "doc"
+Dynamic: license-file
 
 # Nabu
 

nabu-2025.1.0.dev12/doc/doc_config.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+from nabu.resources.nabu_config import nabu_config
+
+
+def generate(file_):
+    def write(content):
+        print(content, file=file_)
+    for section, values in nabu_config.items():
+        if section == "about":
+            continue
+        write("## %s\n" % section)
+        for key, val in values.items():
+            if val["type"] == "unsupported":
+                continue
+            write(val["help"] + "\n")
+            write(
+                "```ini\n%s = %s\n```"
+                % (key, val["default"])
+            )
+
+
+
+if __name__ == "__main__":
+
+    import sys, os
+    print(os.path.abspath(__file__))
+    exit(0)
+
+    fname = "/tmp/test.md"
+    with open(fname, "w") as f:
+        generate(f)

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2025.1.0-dev5"
+__version__ = "2025.1.0-dev12"
 __nabu_modules__ = [
     "app",
     "cuda",

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/app/double_flatfield.py

@@ -11,14 +11,26 @@ from .utils import parse_params_values
 
 class DoubleFlatFieldChunks:
     def __init__(
-        self, dataset_path, output_file, chunk_size=100, sigma=None, do_flatfield=True, h5_entry=None, logger=None
+        self,
+        dataset_path,
+        output_file,
+        dataset_info=None,
+        chunk_size=100,
+        sigma=None,
+        do_flatfield=True,
+        h5_entry=None,
+        logger=None,
     ):
         self.logger = LoggerOrPrint(logger)
-        self.dataset_info = analyze_dataset(dataset_path, extra_options={"hdf5_entry": h5_entry}, logger=logger)
-        self.chunk_size = min(chunk_size, self.dataset_info.radio_dims[-1])
         self.do_flatfield = bool(do_flatfield)
-        if self.do_flatfield:
-            update_dataset_info_flats_darks(self.dataset_info, flatfield_mode=True)
+        if dataset_info is not None:
+            self.dataset_info = dataset_info
+        else:
+            self.dataset_info = analyze_dataset(dataset_path, extra_options={"hdf5_entry": h5_entry}, logger=logger)
+            if self.do_flatfield:
+                update_dataset_info_flats_darks(self.dataset_info, flatfield_mode=True)
+
+        self.chunk_size = min(chunk_size, self.dataset_info.radio_dims[-1])
         self.output_file = output_file
         self.sigma = sigma if sigma is not None and abs(sigma) > 1e-5 else None
 
@@ -103,6 +115,7 @@ class DoubleFlatFieldChunks:
         )
         writer.write(arr, "double_flatfield", config=self._get_config())
         self.logger.info("Wrote %s" % writer.fname)
+        return writer.fname
 
 
 def dff_cli():

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/app/reconstruct_helical.py

@@ -67,11 +67,11 @@ def main_helical():
 
         if proc.nabu_config["reconstruction"]["auto_size"]:
             if 2 * rot_center > Nx:
-                w = int(round(2 * rot_center))
+                w = round(2 * rot_center)
             else:
-                w = int(round(2 * Nx - 2 * rot_center))
-            rec_config["start_x"] = int(round(rot_center - w / 2))
-            rec_config["end_x"] = int(round(rot_center + w / 2))
+                w = round(2 * Nx - 2 * rot_center)
+            rec_config["start_x"] = round(rot_center - w / 2)
+            rec_config["end_x"] = round(rot_center + w / 2)
 
             rec_config["start_y"] = rec_config["start_x"]
             rec_config["end_y"] = rec_config["end_x"]

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/app/stitching.py

@@ -67,12 +67,17 @@ def main():
         futures = {}
         # 2.1 launch jobs
         slurm_job_progress_bars: dict = {}
+
+        # set job name
+        final_output_object_identifier = stitching_config.get_output_object().get_identifier().to_str()
+        stitching_config.slurm_config.job_name = f"stitching-{final_output_object_identifier}"
+
         for i_job, (job, sub_config) in enumerate(
             split_stitching_configuration_to_slurm_job(stitching_config, yield_configuration=True)
         ):
             _logger.info(f"submit job nb {i_job}: handles {sub_config.slices}")
-            output_volume = sub_config.get_output_object().get_identifier().to_str()
-            futures[output_volume] = submit(job, timeout=999999)
+            output_object = sub_config.get_output_object().get_identifier().to_str()
+            futures[output_object] = submit(job, timeout=999999)
             # note on total=100: we only consider percentage in this case (providing advancement from slurm jobs)
             slurm_job_progress_bars[job] = tqdm(
                 total=100,

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/cuda/src/backproj.cu

@@ -18,8 +18,8 @@ inline __device__ int is_in_circle(int x, int y, float center_x, float center_y,
   This will return arr[y][x] where y is an int (exact access) and x is a float (linear interp horizontally)
 */
 static inline __device__ float linear_interpolation(float* arr, int Nx, float x, int y) {
-    // check commented to gain a bit of speed - the check was done before function call
-    // if (x < 0 || x >= Nx) return 0.0f; // texture address mode CLAMP_TO_EDGE
+    // if (x < 0 || x > Nx-1) return 0.0f; // texture address mode BORDER (CLAMP_TO_EDGE continues with edge)
+    if (x <= -0.5f || x >= Nx - 0.5f) return 0.0f; // texture address mode BORDER (CLAMP_TO_EDGE continues with edge)
     int xm = (int) floorf(x);
     int xp = (int) ceilf(x);
     if ((xm == xp) || (xp >= Nx)) return arr[y*Nx+xm];
@@ -127,15 +127,15 @@ __global__ void backproj(
         #endif
 
         #ifdef USE_TEXTURES
-        if (h1 >= 0 && h1 < num_bins) sum1 += tex2D(tex_projections, h1 + 0.5f, proj + 0.5f);
-        if (h2 >= 0 && h2 < num_bins) sum2 += tex2D(tex_projections, h2 + 0.5f, proj + 0.5f);
-        if (h3 >= 0 && h3 < num_bins) sum3 += tex2D(tex_projections, h3 + 0.5f, proj + 0.5f);
-        if (h4 >= 0 && h4 < num_bins) sum4 += tex2D(tex_projections, h4 + 0.5f, proj + 0.5f);
+        sum1 += tex2D(tex_projections, h1 + 0.5f, proj + 0.5f);
+        sum2 += tex2D(tex_projections, h2 + 0.5f, proj + 0.5f);
+        sum3 += tex2D(tex_projections, h3 + 0.5f, proj + 0.5f);
+        sum4 += tex2D(tex_projections, h4 + 0.5f, proj + 0.5f);
         #else
-        if (h1 >= 0 && h1 < num_bins) sum1 += linear_interpolation(d_sino, num_bins, h1, proj);
-        if (h2 >= 0 && h2 < num_bins) sum2 += linear_interpolation(d_sino, num_bins, h2, proj);
-        if (h3 >= 0 && h3 < num_bins) sum3 += linear_interpolation(d_sino, num_bins, h3, proj);
-        if (h4 >= 0 && h4 < num_bins) sum4 += linear_interpolation(d_sino, num_bins, h4, proj);
+        sum1 += linear_interpolation(d_sino, num_bins, h1, proj);
+        sum2 += linear_interpolation(d_sino, num_bins, h2, proj);
+        sum3 += linear_interpolation(d_sino, num_bins, h3, proj);
+        sum4 += linear_interpolation(d_sino, num_bins, h4, proj);
         #endif
     }
 

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/cuda/src/cone.cu

@@ -80,7 +80,11 @@ __global__ void devFDK_preweight(void* D_projData, unsigned int projPitch, unsig
 
         const float fWeight = fW / fRayLength;
 
+        #ifndef RADIOS_LAYOUT
         projData[(detectorV*iProjAngles+angle)*projPitch+detectorU] *= fWeight;
+        #else
+        projData[(angle*iProjV+detectorV)*projPitch+detectorU] *= fWeight;
+        #endif
 
         fV += fDetVSize;
     }

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/cuda/utils.py

@@ -197,7 +197,7 @@ def cuarray_shape_dtype(cuarray):
 
 
 def get_shape_dtype(arr):
-    if isinstance(arr, garray.GPUArray | np.ndarray):
+    if isinstance(arr, (garray.GPUArray, np.ndarray)):
         return arr.shape, arr.dtype
     elif isinstance(arr, cuda.Array):
         return cuarray_shape_dtype(arr)

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/estimation/cor.py

@@ -640,8 +640,8 @@ class CenterOfRotationAdaptiveSearch(CenterOfRotation):
                 * self.sigma_fraction
             )
 
-            M1 = int(round(cor_position + img_1.shape[1] // 2)) - int(round(tmp_sigma))
-            M2 = int(round(cor_position + img_1.shape[1] // 2)) + int(round(tmp_sigma))
+            M1 = round(cor_position + img_1.shape[1] // 2) - round(tmp_sigma)
+            M2 = round(cor_position + img_1.shape[1] // 2) + round(tmp_sigma)
 
             piece_1 = img_filtered_1[:, M1:M2]
             piece_2 = img_filtered_2[:, img_1.shape[1] - M2 : img_1.shape[1] - M1]
@@ -678,7 +678,7 @@ class CenterOfRotationAdaptiveSearch(CenterOfRotation):
                     filtered_found_centers.append(found_centers[i])
                     continue
 
-        if len(filtered_found_centers):
+        if len(filtered_found_centers) > 0:
             found_centers = filtered_found_centers
 
         min_choice = min(found_centers)

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/io/cast_volume.py

@@ -1,4 +1,6 @@
 import os
+
+from tomoscan.esrf.volume.singleframebase import VolumeSingleFrameBase
 from nabu.misc.utils import rescale_data
 from nabu.pipeline.params import files_formats
 from tomoscan.volumebase import VolumeBase
@@ -176,6 +178,17 @@ def cast_volume(
     if not isinstance(output_data_type, numpy.dtype):
         raise TypeError(f"output_data_type is expected to be a {numpy.dtype}. {type(output_data_type)} provided")
 
+    # Make sure the output volume has the same "start_index" as input volume, if relevant
+    if isinstance(input_volume, VolumeSingleFrameBase) and isinstance(output_volume, VolumeSingleFrameBase):
+        try:
+            first_file_name = next(input_volume.browse_data_files())
+            start_idx = int(first_file_name.split(".")[0].split("_")[-1])
+        except (StopIteration, ValueError, TypeError):
+            # StopIteration: Input volume has no file - should not happen
+            # ValueError / TypeError: fail to convert to int, something wrong when extracting slice number (non-default file name scheme)
+            start_idx = 0
+        output_volume.start_index = start_idx
+
     # start processing
     #  check for data_min and data_max
     if data_min is None or data_max is None:

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/io/reader.py

@@ -688,8 +688,9 @@ class NXTomoReader(VolReaderBase):
             user_selection_dim0 = self.sub_region[0]
             indices = np.arange(self.data_shape_total[0])
             data_selection_indices_axis0 = np.hstack(
-                [indices[image_key_slice][user_selection_dim0] for image_key_slice in self._image_key_slices]
-            )
+                [indices[image_key_slice] for image_key_slice in self._image_key_slices]
+            )[user_selection_dim0]
+
             self._source_selection = (data_selection_indices_axis0,) + self.sub_region[1:]
 
     def _get_temporary_buffer(self, convert_after_reading):

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/opencl/src/backproj.cl

@@ -14,7 +14,7 @@ static inline int is_in_circle(float x, float y, float center_x, float center_y,
   This will return arr[y][x] where y is an int (exact access) and x is a float (linear interp horizontally)
 */
 static inline float linear_interpolation(global float* arr, int Nx, float x, int y) {
-    if (x < 0 || x >= Nx) return 0.0f; // texture address mode CLAMP_TO_EDGE
+    if (x < -0.5f || x > Nx - 0.5f) return 0.0f; // texture address mode BORDER (CLAMP_TO_EDGE continues with edge)
     int xm = (int) floor(x);
     int xp = (int) ceil(x);
     if ((xm == xp) || (xp >= Nx)) return arr[y*Nx+xm];
@@ -53,7 +53,7 @@ kernel void backproj(
     uint Gy = get_global_size(1);
 
     #ifdef USE_TEXTURES
-    const sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_LINEAR;
+    const sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_LINEAR;
     #endif
 
     // (xr, yr)    (xrp, yr)
@@ -111,15 +111,15 @@ kernel void backproj(
         #endif
 
         #ifdef USE_TEXTURES
-        if (h1 >= 0 && h1 < num_bins) sum1 += read_imagef(d_sino, sampler, (float2) (h1 +0.5f,proj +0.5f)).x;
-        if (h2 >= 0 && h2 < num_bins) sum2 += read_imagef(d_sino, sampler, (float2) (h2 +0.5f,proj +0.5f)).x;
-        if (h3 >= 0 && h3 < num_bins) sum3 += read_imagef(d_sino, sampler, (float2) (h3 +0.5f,proj +0.5f)).x;
-        if (h4 >= 0 && h4 < num_bins) sum4 += read_imagef(d_sino, sampler, (float2) (h4 +0.5f,proj +0.5f)).x;
+        sum1 += read_imagef(d_sino, sampler, (float2) (h1 +0.5f,proj +0.5f)).x;
+        sum2 += read_imagef(d_sino, sampler, (float2) (h2 +0.5f,proj +0.5f)).x;
+        sum3 += read_imagef(d_sino, sampler, (float2) (h3 +0.5f,proj +0.5f)).x;
+        sum4 += read_imagef(d_sino, sampler, (float2) (h4 +0.5f,proj +0.5f)).x;
         #else
-        if (h1 >= 0 && h1 < num_bins) sum1 += linear_interpolation(d_sino, num_bins, h1, proj);
-        if (h2 >= 0 && h2 < num_bins) sum2 += linear_interpolation(d_sino, num_bins, h2, proj);
-        if (h3 >= 0 && h3 < num_bins) sum3 += linear_interpolation(d_sino, num_bins, h3, proj);
-        if (h4 >= 0 && h4 < num_bins) sum4 += linear_interpolation(d_sino, num_bins, h4, proj);
+        sum1 += linear_interpolation(d_sino, num_bins, h1, proj);
+        sum2 += linear_interpolation(d_sino, num_bins, h2, proj);
+        sum3 += linear_interpolation(d_sino, num_bins, h3, proj);
+        sum4 += linear_interpolation(d_sino, num_bins, h4, proj);
         #endif
     }
 

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/pipeline/estimators.py

@@ -150,7 +150,7 @@ class CORFinderBase:
                 lookup_side = default_lookup_side
             else:
                 lookup_side = initial_cor_pos
-            self._lookup_side = initial_cor_pos
+            self._lookup_side = lookup_side
 
     def _init_cor_finder(self):
         cor_finder_cls = self.search_methods[self.method]["class"]
@@ -542,7 +542,7 @@ class CompositeCORFinder(CORFinderBase):
         else:
             my_flats = None
 
-        if my_flats is not None and len(list(my_flats.keys())):
+        if my_flats is not None and len(list(my_flats.keys())) > 0:
             self.use_flat = True
             self.flatfield = FlatField(
                 (len(self.absolute_indices), self.sy, self.sx),
@@ -750,15 +750,15 @@ class CompositeCORFinder(CORFinderBase):
                 my_blurred_radio1 = np.fliplr(blurred_radio1)
                 my_blurred_radio2 = np.fliplr(blurred_radio2)
 
-            common_left = np.fliplr(my_radio1[:, ovsd_sx - my_z :])[:, : -int(math.ceil(self.ovs * self.high_pass * 2))]
+            common_left = np.fliplr(my_radio1[:, ovsd_sx - my_z :])[:, : -math.ceil(self.ovs * self.high_pass * 2)]
             # adopt a 'safe' margin considering high_pass value (possibly float)
-            common_right = my_radio2[:, ovsd_sx - my_z : -int(math.ceil(self.ovs * self.high_pass * 2))]
+            common_right = my_radio2[:, ovsd_sx - my_z : -math.ceil(self.ovs * self.high_pass * 2)]
 
             common_blurred_left = np.fliplr(my_blurred_radio1[:, ovsd_sx - my_z :])[
-                :, : -int(math.ceil(self.ovs * self.high_pass * 2))
+                :, : -math.ceil(self.ovs * self.high_pass * 2)
             ]
             # adopt a 'safe' margin considering high_pass value (possibly float)
-            common_blurred_right = my_blurred_radio2[:, ovsd_sx - my_z : -int(math.ceil(self.ovs * self.high_pass * 2))]
+            common_blurred_right = my_blurred_radio2[:, ovsd_sx - my_z : -math.ceil(self.ovs * self.high_pass * 2)]
 
             if common_right.size == 0:
                 continue

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/pipeline/fullfield/chunked.py

@@ -4,7 +4,7 @@ from math import ceil
 import numpy as np
 from silx.io.url import DataUrl
 
-from ...utils import get_num_threads, remove_items_from_list
+from ...utils import get_num_threads, remove_items_from_list, get_subregion as get_subregion_xy
 from ...resources.logger import LoggerOrPrint
 from ...resources.utils import extract_parameters
 from ...misc.binning import binning as image_binning
@@ -456,7 +456,8 @@ class ChunkedPipeline:
         self.double_flatfield = self.DoubleFlatFieldClass(
             self.radios_shape,
             result_url=result_url,
-            sub_region=self.sub_region[1:],
+            # DoubleFlatField expects sub_region as (start_x, end_x, start_y, end_y)
+            sub_region=get_subregion_xy(self.sub_region[1:][::-1]),
             input_is_mlog=False,
             output_is_mlog=False,
             average_is_on_log=avg_is_on_log,
@@ -628,6 +629,7 @@ class ChunkedPipeline:
                     "clip_outer_circle": options["clip_outer_circle"],
                     "outer_circle_value": options["outer_circle_value"],
                     "filter_cutoff": options["fbp_filter_cutoff"],
+                    "crop_filtered_data": options["crop_filtered_data"],
                 },
             )
 
@@ -648,6 +650,11 @@ class ChunkedPipeline:
                 },
             )
 
+        if options.get("crop_filtered_data", True) is False:
+            self.logger.warning(
+                "Using [reconstruction] crop_filtered_data = False. This will use a large amount of memory."
+            )
+
         self._allocate_recs(*self.process_config.rec_shape, n_slices=n_slices)
         n_a, _, n_x = self.radios_cropped_shape
         self._tmp_sino = self._allocate_array((n_a, n_x), "f", name="tmp_sino")
@@ -779,7 +786,7 @@ class ChunkedPipeline:
     @pipeline_step("sino_deringer", "Removing rings on sinograms")
     def _destripe_sinos(self):
         sinos = np.rollaxis(self.radios, 1, 0)  # view
-        self.sino_deringer.remove_rings(sinos)  # TODO check it works with non-contiguous view
+        self.sino_deringer.remove_rings(sinos)
 
     @pipeline_step("reconstruction", "Reconstruction")
     def _reconstruct(self):
@@ -804,15 +811,7 @@ class ChunkedPipeline:
         """
         This reconstructs the entire sinograms stack at once
         """
-
-        n_angles, n_z, n_x = self.radios.shape
-
-        # FIXME
-        # can't do a discontiguous single copy...
-        sinos_contig = self._allocate_array((n_z, n_angles, n_x), np.float32, "sinos_cone")
-        for i in range(n_z):
-            sinos_contig[i] = self.radios[:, i, :]
-        # ---
+        sinos_discontig = self.radios.transpose(axes=(1, 0, 2))  # view
 
         # In principle radios are not cropped at this stage,
         # so self.sub_region[2][0] can be used instead of self.get_slice_start_index() instead of self.sub_region[2][0]
@@ -820,7 +819,8 @@ class ChunkedPipeline:
         n_z_tot = self.process_config.radio_shape(binning=True)[0]
 
         self.reconstruction.reconstruct(  # pylint: disable=E1101
-            sinos_contig,
+            # sinos_contig,
+            sinos_discontig,
             output=self.recs,
             relative_z_position=((z_min + z_max) / self.process_config.binning_z / 2) - n_z_tot / 2,
         )

{nabu-2025.1.0.dev5 → nabu-2025.1.0.dev12}/nabu/pipeline/fullfield/computations.py

@@ -128,7 +128,10 @@ def estimate_required_memory(
         if process_config.rec_params["method"] == "cone":
             # In cone-beam reconstruction, need both sinograms and reconstruction inside GPU.
             # That's big!
-            total_memory_needed += 2 * data_volume_size
+            mult_factor = 2
+            if rec_config["crop_filtered_data"] is False:
+                mult_factor = 4
+            total_memory_needed += mult_factor * data_volume_size
 
     if debug:
         print(

nabu-2025.1.0.dev12/nabu/pipeline/fullfield/get_double_flatfield.py

@@ -0,0 +1,147 @@
+"""
+Double-flatfield:
+  - Compute the average of all projections, which gives one resulting image
+  - Apply some filter to this image (DFF)
+  - Subtract or divide this image from all the projections
+"""
+
+from os import path
+from silx.io.url import DataUrl
+from silx.io.dictdump import h5todict
+
+from nabu.io.utils import get_first_hdf5_entry
+
+from ...utils import is_writeable
+from ...app.double_flatfield import DoubleFlatFieldChunks
+from ...resources.nxflatfield import data_url_exists
+
+rel_file_path_template = "{scan_name}_dff.h5"
+data_path_template = "{entry}/double_flatfield"
+
+
+def get_possible_dff_urls(dataset_info, user_dir, output_dir):
+    """
+    See nabu.resources.nxflatfield.get_frame_possible_urls
+    """
+    entry = dataset_info.hdf5_entry or ""
+
+    def make_dataurl(dirname):
+        file_path = path.join(
+            dirname,
+            rel_file_path_template.format(scan_name=dataset_info.scan_basename),
+        )
+        return DataUrl(
+            file_path=file_path,
+            data_path=data_path_template.format(entry=entry),
+            scheme="silx",
+        )
+
+    urls = {"user": None, "dataset": None, "output": None}
+
+    if user_dir is not None:
+        urls["user"] = make_dataurl(user_dir)
+    urls["dataset"] = make_dataurl(dataset_info.scan_dirname)
+    if output_dir is not None:
+        urls["output"] = make_dataurl(output_dir)
+
+    return urls
+
+
+def compute_and_save_dff(dataset_info, possible_urls, dff_options):
+    if possible_urls["user"] is not None:
+        dff_output_file = possible_urls["user"].file_path()
+    elif is_writeable(path.dirname(possible_urls["dataset"].file_path())):
+        dff_output_file = possible_urls["dataset"].file_path()
+    else:
+        dff_output_file = possible_urls["output"].file_path()
+
+    dataset_info.logger.info("Computing double flatfield")
+    dff = DoubleFlatFieldChunks(
+        None,
+        dff_output_file,
+        dataset_info=dataset_info,
+        chunk_size=dff_options.get("chunk_size", 100),
+        sigma=dff_options.get("dff_sigma", None),
+        do_flatfield=dff_options.get("do_flatfield", True),
+        logger=dataset_info.logger,
+    )
+    dff_image = dff.compute_double_flatfield()
+    return dff.write_double_flatfield(dff_image)
+
+
+def check_existing_dff(dff_url, dff_options, logger):
+    # Check that the DFF exists at the given DataUrl, and that its configuration matches the wanted config
+    # Return the DFF file path
+    if not (data_url_exists(dff_url)):
+        raise ValueError("DFF file not found:", dff_url)
+
+    fname = dff_url.file_path()
+    entry = get_first_hdf5_entry(fname)
+    dff_file_options = h5todict(fname, path=entry + "/double_flatfield/configuration", asarray=False)
+
+    ff_file = dff_file_options.get("do_flatfield", True)
+    ff_user = dff_options.get("do_flatfield", True)
+    # Use "==" instead of "is" here, as h5todict() will return something like numpy.True_ instead of True
+    if ff_file != ff_user:
+        msg = "DFF was computed with flatfield=%s, but you asked flatfield=%s" % (ff_file, ff_user)
+        logger.error(msg)
+        return False
+
+    # Use this because h5todict() returns str("None") instead of None
+    def _correct_none(x):
+        if x in [None, "None"]:
+            return None
+        return x
+
+    sigma_file = _correct_none(dff_file_options.get("dff_sigma", None))
+    sigma_user = _correct_none(dff_options.get("dff_sigma", None))
+    if sigma_file != sigma_user:
+        msg = "DFF was computed with dff_sigma=%s, but you asked dff_sigma=%s" % (sigma_file, sigma_user)
+        logger.error(msg)
+        return False
+
+    return fname
+
+
+# pylint: disable=E1136
+def get_double_flatfield(dataset_info, mode, output_dir=None, darks_flats_dir=None, dff_options=None):
+    """
+    See nabu.resources.nxflatfield.update_dataset_info_flats_darks for the logic
+    """
+    if mode is False:
+        return
+    dff_options = dff_options or {}
+
+    possible_urls = get_possible_dff_urls(dataset_info, darks_flats_dir, output_dir)
+
+    if mode == "force-compute":
+        return compute_and_save_dff(dataset_info, possible_urls, dff_options)
+
+    def _can_load_from(folder_type):
+        if possible_urls.get(folder_type, None) is None:
+            return False
+        return data_url_exists(possible_urls[folder_type])
+
+    where_to_load_from = None
+    if possible_urls["user"] is not None and _can_load_from("user"):
+        where_to_load_from = "user"
+    elif _can_load_from("dataset"):
+        where_to_load_from = "dataset"
+    elif _can_load_from("output"):
+        where_to_load_from = "output"
+
+    if where_to_load_from is None:
+        if mode == "force-load":
+            raise ValueError("Could not load double-flatfield file (using 'force-load')")
+        else:
+            return compute_and_save_dff(dataset_info, possible_urls, dff_options)
+
+    fname = check_existing_dff(possible_urls[where_to_load_from], dff_options, dataset_info.logger)
+    if fname is False:
+        if mode == "force-load":
+            raise ValueError("Could not load double-flatfield file (using 'force-load'): wrong configuration")
+        return compute_and_save_dff(dataset_info, possible_urls, dff_options)
+    return fname
+
+    # One possible corner case: if mode == "force-load" and darks_flats_dir is not None (but the actual folder is empty)
+    # then nabu will load a DFF found elsewhere (if any). We might want to raise an error instead.