nabu 2024.1.10__py3-none-any.whl → 2024.2.0__py3-none-any.whl

nabu/pipeline/writer.py

@@ -1,5 +1,6 @@
 from os import path
 from tomoscan.esrf import TIFFVolume, MultiTIFFVolume, EDFVolume, JP2KVolume
+from tomoscan.esrf.volume.singleframebase import VolumeSingleFrameBase
 from ..utils import check_supported, get_num_threads
 from ..resources.logger import LoggerOrPrint
 from ..io.writer import NXProcessWriter, HSTVolVolume, NXVolVolume
@@ -8,7 +9,6 @@ from .params import files_formats
 
 
 class WriterManager:
-
     """
     This class is a wrapper on top of all "writers".
     It will create the right "writer" with all the necessary options, and the histogram writer.
@@ -114,6 +114,7 @@ class WriterManager:
         return vol_writer.data_url.file_path()
 
     def _init_writer(self):
+        self._writer_was_already_initialized = self.extra_options.get("writer_initialized", False)
         if self.file_format in ["tiff", "edf", "jp2", "hdf5"]:
             writer_kwargs = {
                 "folder": self.output_dir,
@@ -144,6 +145,11 @@ class WriterManager:
         self._h5_entry = self.metadata.get("entry", "entry")
         self.writer = self._writer_classes[self.file_format](**writer_kwargs)
         self.fname = self.get_fname(self.writer)
+        # In certain cases, tomoscan needs to remove any previous existing volume filess
+        # and avoid calling 'clean_output_data' when writing downstream (for chunk processing)
+        if isinstance(self.writer, VolumeSingleFrameBase):
+            self.writer.skip_existing_data_files_removal = self._writer_was_already_initialized
+        # ---
         if path.exists(self.fname):
             err = "File already exists: %s" % self.fname
             if self.overwrite:
@@ -188,7 +194,9 @@ class WriterManager:
         self.writer.metadata = self.metadata
         self.writer.save_metadata()
 
-    def write_data(self, data):
+    def write_data(self, data, metadata=None):
         self.writer.data = data
+        if metadata is not None:
+            self.writer.metadata = metadata
         self.writer.save()
         # self._write_metadata()

nabu/preproc/ccd_cuda.py

@@ -118,7 +118,7 @@ class CudaLog(Log):
         self._nthreadsperblock = (16, 16, 4)  # TODO tune ?
         self._nblocks = tuple([updiv(n, p) for n, p in zip([nx, ny, nz], self._nthreadsperblock)])
 
-        self.nlog_kernel = CudaKernel(
+        self.nlog_kernel = CudaKernel(  # pylint: disable=E0606
             "nlog",
             filename=self._nlog_srcfile,
             signature="Piiiff",

nabu/preproc/ctf.py

@@ -18,7 +18,7 @@ class GeoPars:
         self,
         z1_vh=None,
         z2=None,
-        pix_size_det=None,
+        pix_size_det=1e-6,
         wavelength=None,
         magnification=True,
         length_scale=10.0e-6,
@@ -33,8 +33,9 @@ class GeoPars:
            and the horizontaly focused source (vertical line) for KB mirrors.
         z2 : float
            the sample detector distance (meters).
-        pix_size_det: float
-           pixel size (meters)
+        pix_size_det: float or tuple
+           pixel size in meters.
+           If a tuple is passed, it is interpreted as (horizontal_size, vertical_size)
         wavelength: float
            beam wave length (meters).
         magnification: boolean defaults to True
@@ -55,7 +56,11 @@ class GeoPars:
                 self.z1_vh = np.array([z1_vh, z1_vh])
         self.z2 = z2
         self.magnification = magnification
-        self.pix_size_det = pix_size_det
+        if np.isscalar(pix_size_det):
+            self.pix_size_det_xy = (pix_size_det, pix_size_det)
+        else:
+            self.pix_size_det_xy = pix_size_det
+        self.pix_size_det = self.pix_size_det_xy[0]  # COMPAT
 
         if self.magnification and self.z1_vh is not None:
             self.M_vh = (self.z1_vh + self.z2) / self.z1_vh
@@ -69,7 +74,9 @@ class GeoPars:
 
         self.maxM = self.M_vh.max()
 
-        self.pix_size_rec = self.pix_size_det / self.maxM  # we bring everything to highest magnification
+        # we bring everything to highest magnification
+        self.pix_size_rec_xy = [p / self.maxM for p in self.pix_size_det_xy]
+        self.pix_size_rec = self.pix_size_rec_xy[0]  # COMPAT
 
         which_unit = int(np.sum(np.array([self.pix_size_rec > small for small in [1.0e-6, 1.0e-7]]).astype(np.int32)))
         self.pixelsize_string = [
@@ -208,8 +215,8 @@ class CTFPhaseRetrieval:
         padded_img_shape = self.shape_padded
         fsample_vh = np.array(
             [
-                self.geo_pars.length_scale / self.geo_pars.pix_size_rec,
-                self.geo_pars.length_scale / self.geo_pars.pix_size_rec,
+                self.geo_pars.length_scale / self.geo_pars.pix_size_rec_xy[1],
+                self.geo_pars.length_scale / self.geo_pars.pix_size_rec_xy[0],
             ]
         )
 

nabu/preproc/ctf_cuda.py

@@ -15,7 +15,6 @@ if __has_pycuda__:
 #  - better padding scheme (for now 2*shape)
 #  - rework inheritance scheme ? (base class SingleDistancePhaseRetrieval and its cuda counterpart)
 class CudaCTFPhaseRetrieval(CTFPhaseRetrieval):
-
     """
     Cuda back-end of CTFPhaseRetrieval
     """
@@ -37,7 +36,7 @@ class CudaCTFPhaseRetrieval(CTFPhaseRetrieval):
         fft_num_threads=None,
         logger=None,
         cuda_options=None,
-        fft_backend="skcuda",
+        fft_backend="vkfft",
     ):
         """
         Initialize a CudaCTFPhaseRetrieval.
@@ -130,7 +129,7 @@ class CudaCTFPhaseRetrieval(CTFPhaseRetrieval):
         self.set_input(image)
         self.cuda_padding.pad(image, output=self.d_radio_padded)
         if self.normalize_by_mean:
-            m = garray.sum(self.d_radio_padded).get() / np.prod(self.shape_padded)
+            m = garray.sum(self.d_radio_padded).get() / np.prod(self.shape_padded)  # pylint: disable=E0606
             self.d_radio_padded /= m
         self.cufft.fft(self.d_radio_padded, output=self.d_radio_f)
         self.cpxmult_kernel(*self._cpxmult_kernel_args, **self._cpxmult_kernel_kwargs)

nabu/preproc/double_flatfield.py

@@ -2,9 +2,9 @@ from os import path
 import numpy as np
 from scipy.ndimage import gaussian_filter
 from silx.io.url import DataUrl
-from ..utils import check_supported, check_shape, get_2D_3D_shape
-from ..io.reader import Readers
-from ..io.writer import Writers
+from ..utils import check_shape, get_2D_3D_shape
+from ..io.reader import HDF5Reader
+from ..io.writer import NXProcessWriter
 from .ccd import Log
 
 
@@ -64,12 +64,6 @@ class DoubleFlatField:
         self._init_processing(input_is_mlog, output_is_mlog, average_is_on_log, sigma_filter, filter_mode)
         self._computed = False
 
-    def _get_reader_writer_class(self):
-        ext = path.splitext(self.result_url.file_path())[-1].replace(".", "")
-        check_supported(ext, list(Writers.keys()), "file format")
-        self._writer_cls = Writers[ext]
-        self._reader_cls = Readers[ext]
-
     def _load_dff_dump(self):
         res = self.reader.get_data(self.result_url)
         if self.detector_corrector is not None:
@@ -98,15 +92,14 @@ class DoubleFlatField:
         self.reader = None
         if self.result_url is None:
             return
-        self._get_reader_writer_class()
         if path.exists(result_url.file_path()):
             if detector_corrector is None:
                 adapted_subregion = sub_region
             else:
                 adapted_subregion = self.detector_corrector.get_adapted_subregion(sub_region)
-            self.reader = self._reader_cls(sub_region=adapted_subregion)
+            self.reader = HDF5Reader(sub_region=adapted_subregion)
         else:
-            self.writer = self._writer_cls(self.result_url.file_path())
+            self.writer = NXProcessWriter(self.result_url.file_path())
 
     def _init_processing(self, input_is_mlog, output_is_mlog, average_is_on_log, sigma_filter, filter_mode):
         self.input_is_mlog = input_is_mlog

nabu/preproc/double_flatfield_cuda.py

@@ -59,7 +59,7 @@ class CudaDoubleFlatField(DoubleFlatField):
     def _proc_expm(x, o):
         o[:] = x[:]
         o[:] *= -1
-        cumath.exp(o, out=o)
+        cumath.exp(o, out=o)  # pylint: disable=E0606
         return o
 
     def _init_processing(self, input_is_mlog, output_is_mlog, average_is_on_log, sigma_filter, filter_mode):
@@ -108,7 +108,7 @@ class CudaDoubleFlatField(DoubleFlatField):
         recompute: bool, optional
             Whether to recompute the double flatfield if already computed.
         """
-        if not (isinstance(radios, garray.GPUArray)):
+        if not (isinstance(radios, garray.GPUArray)):  # pylint: disable=E0606
             raise ValueError("Expected pycuda.gpuarray.GPUArray for radios")
         if self._computed and not (recompute):
             return self.doubleflatfield

nabu/preproc/flatfield.py

@@ -2,7 +2,7 @@ from multiprocessing.pool import ThreadPool
 from bisect import bisect_left
 import numpy as np
 from ..io.reader import load_images_from_dataurl_dict
-from ..utils import check_supported, get_num_threads
+from ..utils import check_supported, deprecated_class, get_num_threads
 
 
 class FlatFieldArrays:
@@ -228,6 +228,7 @@ class FlatFieldArrays:
                 f_idx, weights = _interp_nearest(idx, prev_next)
             elif self.interpolation == "linear":
                 f_idx, weights = _interp_linear(idx, prev_next)
+            # pylint: disable=E0606
             self.flats_idx[i] = f_idx
             self.flats_weights[i] = weights
 
@@ -376,6 +377,9 @@ class FlatFieldArrays:
 FlatField = FlatFieldArrays
 
 
+@deprecated_class(
+    "FlatFieldDataUrls is deprecated since 2024.2.0 and will be removed in a future version", do_print=True
+)
 class FlatFieldDataUrls(FlatField):
     def __init__(
         self,

nabu/preproc/flatfield_cuda.py

@@ -2,7 +2,7 @@ import numpy as np
 
 from nabu.cuda.processing import CudaProcessing
 from ..preproc.flatfield import FlatFieldArrays
-from ..utils import get_cuda_srcfile
+from ..utils import deprecated_class, get_cuda_srcfile
 from ..io.reader import load_images_from_dataurl_dict
 from ..cuda.utils import __has_pycuda__
 
@@ -114,6 +114,9 @@ class CudaFlatFieldArrays(FlatFieldArrays):
 CudaFlatField = CudaFlatFieldArrays
 
 
+@deprecated_class(
+    "CudaFlatFieldDataUrls is deprecated since version 2024.2.0 and will be removed in a future version", do_print=True
+)
 class CudaFlatFieldDataUrls(CudaFlatField):
     def __init__(
         self,
@@ -138,6 +141,7 @@ class CudaFlatFieldDataUrls(CudaFlatField):
             radios_indices=radios_indices,
             interpolation=interpolation,
             distortion_correction=distortion_correction,
+            nan_value=nan_value,
             radios_srcurrent=radios_srcurrent,
             flats_srcurrent=flats_srcurrent,
             cuda_options=cuda_options,

nabu/preproc/phase.py

@@ -4,7 +4,7 @@ import numpy as np
 from scipy.fft import rfft2, irfft2, fft2, ifft2
 from ..utils import generate_powers, get_decay, check_supported, get_num_threads, deprecation_warning
 
-#
+# COMPAT.
 from .ctf import CTFPhaseRetrieval
 
 #
@@ -50,7 +50,6 @@ class PaganinPhaseRetrieval:
         delta_beta=250.0,
         pixel_size=1e-6,
         padding="edge",
-        margin=None,
         use_rfft=True,
         use_R2C=None,
         fftw_num_threads=None,
@@ -73,42 +72,13 @@ class PaganinPhaseRetrieval:
         delta_beta: float, optional
             delta/beta ratio, where n = (1 - delta) + i*beta is the complex
             refractive index of the sample.
-        pixel_size : float, optional
+        pixel_size : float or tuple, optional
             Detector pixel size in meters. Default is 1e-6 (one micron)
+            If a tuple is passed, the pixel size is set as (horizontal_size, vertical_size).
         padding : str, optional
             Padding method. Available are "zeros", "mean", "edge", "sym",
             "reflect". Default is "edge".
             Please refer to the "Padding" section below for more details.
-        margin: tuple, optional
-            The user may provide integers values U, D, L, R as a tuple under the
-            form ((U, D), (L, R)) (same syntax as numpy.pad()).
-            The resulting filtered radio will have a size equal to
-            (size_vertic - U - D, size_horiz - L - R).
-            These values serve to create a "margin" for the filtering process,
-            where U, D, L R are the margin of the Up, Down, Left and Right part,
-            respectively.
-            The filtering is done on a subset of the input radio. The subset
-            size is (Nrows - U - D, Ncols - R - L).
-            The margins is used to do the padding for the rest of the padded
-            array.
-
-            For example in one dimension, where ``padding="edge"``::
-
-               <------------------------------ padded_size --------------------------->
-               [padding=edge | padding=data | radio data | padding=data | padding=edge]
-               <------ N2 ---><----- L -----><- (N-L-R)--><----- R -----><----- N2 --->
-
-            Some or all the values U, D, L, R can be 0. In this case,
-            the padding of the parts related to the zero values will
-            fall back to the one of "padding" parameter.
-            For example, if padding="edge" and L, R are 0, then
-            the left and right parts will be padded with the edges, while
-            the Up and Down parts will be padded using the the user-provided
-            margins of the radio, and the final data will have shape
-            (Nrows - U - D, Ncols).
-            Some or all the values U, D, L, R can be the string "auto".
-            In this case, the values of U, D, L, R are automatically computed
-            as a function of the Paganin filter width.
         use_rfft: bool, optional
             Whether to use Real-to-Complex (R2C) transform instead of
             standard Complex-to-Complex transform, providing better performances
@@ -171,7 +141,7 @@ class PaganinPhaseRetrieval:
             Journal of Microscopy, Vol 206, Part 1, 2002
         """
         self._init_parameters(distance, energy, pixel_size, delta_beta, padding)
-        self._calc_shape(shape, margin)
+        self._calc_shape(shape)
         # COMPAT.
         if use_R2C is not None:
             deprecation_warning("'use_R2C' is replaced with 'use_rfft'", func_name="pag_r2c")
@@ -186,7 +156,13 @@ class PaganinPhaseRetrieval:
         self.distance_cm = distance * 1e2
         self.distance_micron = distance * 1e6
         self.energy_kev = energy
-        self.pixel_size_micron = pixel_size * 1e6
+        if np.isscalar(pixel_size):
+            self.pixel_size_xy_micron = (pixel_size * 1e6, pixel_size * 1e6)
+        else:
+            self.pixel_size_xy_micron = pixel_size * 1e6
+        # COMPAT.
+        self.pixel_size_micron = self.pixel_size_xy_micron[0]
+        #
         self.delta_beta = delta_beta
         self.wavelength_micron = 1.23984199e-3 / self.energy_kev
         self.padding = padding
@@ -209,34 +185,14 @@ class PaganinPhaseRetrieval:
             self.fft_func = fft2
             self.ifft_func = ifft2
 
-    def _calc_shape(self, shape, margin):
+    def _calc_shape(self, shape):
         if np.isscalar(shape):
             shape = (shape, shape)
         else:
             assert len(shape) == 2
         self.shape = shape
-        self._set_margin_value(margin)
         self._calc_padded_shape()
 
-    def _set_margin_value(self, margin):
-        self.margin = margin
-        if margin is None:
-            self.shape_inner = self.shape
-            self.use_margin = False
-            self.margin = ((0, 0), (0, 0))
-            return
-        self.use_margin = True
-        try:
-            ((U, D), (L, R)) = margin
-        except ValueError:
-            raise ValueError("Expected margin in the format ((U, D), (L, R))")
-        for val in [U, D, L, R]:
-            if isinstance(val, str) and val != "auto":
-                raise ValueError("Expected either an integer, or 'auto'")
-            if int(val) != val or val < 0:
-                raise ValueError("Expected positive integers for margin values")
-        self.shape_inner = (self.shape[0] - U - D, self.shape[1] - L - R)
-
     def _calc_padded_shape(self):
         """
         Compute the padded shape.
@@ -257,19 +213,15 @@ class PaganinPhaseRetrieval:
         nx0 : length of original data
         nx_p : total length of padded data
         """
-        n_y, n_x = self.shape_inner
-        n_y0, n_x0 = self.shape
-        n_y_p = self._get_next_power(max(2 * n_y, n_y0))
-        n_x_p = self._get_next_power(max(2 * n_x, n_x0))
+        n_y, n_x = self.shape
+        n_y_p = self._get_next_power(2 * n_y)
+        n_x_p = self._get_next_power(2 * n_x)
         self.shape_padded = (n_y_p, n_x_p)
         self.data_padded = np.zeros((n_y_p, n_x_p), dtype=np.float64)
-
-        ((U, D), (L, R)) = self.margin
-        n_y0, n_x0 = self.shape
-        self.pad_top_len = (n_y_p - n_y0) // 2
-        self.pad_bottom_len = n_y_p - n_y0 - self.pad_top_len
-        self.pad_left_len = (n_x_p - n_x0) // 2
-        self.pad_right_len = n_x_p - n_x0 - self.pad_left_len
+        self.pad_top_len = (n_y_p - n_y) // 2
+        self.pad_bottom_len = n_y_p - n_y - self.pad_top_len
+        self.pad_left_len = (n_x_p - n_x) // 2
+        self.pad_right_len = n_x_p - n_x - self.pad_left_len
 
     def _get_next_power(self, n):
         """
@@ -284,8 +236,8 @@ class PaganinPhaseRetrieval:
     def compute_filter(self):
         nyp, nxp = self.shape_padded
         fftfreq = np.fft.rfftfreq if self.use_rfft else np.fft.fftfreq
-        fy = np.fft.fftfreq(nyp, d=self.pixel_size_micron)
-        fx = fftfreq(nxp, d=self.pixel_size_micron)
+        fy = np.fft.fftfreq(nyp, d=self.pixel_size_xy_micron[1])
+        fx = fftfreq(nxp, d=self.pixel_size_xy_micron[0])
         self._coords_grid = np.add.outer(fy**2, fx**2)
         #
         k2 = self._coords_grid
@@ -376,12 +328,11 @@ class PaganinPhaseRetrieval:
         radio_f = self.fft_func(self.data_padded, workers=self.fft_num_threads)
         radio_f *= self.paganin_filter
         radio_filtered = self.ifft_func(radio_f, workers=self.fft_num_threads).real
-        s0, s1 = self.shape_inner
-        ((U, _), (L, _)) = self.margin
+        s0, s1 = self.shape
         if output is None:
-            return radio_filtered[U : U + s0, L : L + s1]
+            return radio_filtered[:s0, :s1]
         else:
-            output[:, :] = radio_filtered[U : U + s0, L : L + s1]
+            output[:, :] = radio_filtered[:s0, :s1]
             return output
 
     def lmicron_to_db(self, Lmicron):

nabu/preproc/phase_cuda.py

@@ -18,11 +18,10 @@ class CudaPaganinPhaseRetrieval(PaganinPhaseRetrieval):
         delta_beta=250.0,
         pixel_size=1e-6,
         padding="edge",
-        margin=None,
         cuda_options=None,
         fftw_num_threads=None,  # COMPAT.
         fft_num_threads=None,
-        fft_backend="skcuda",
+        fft_backend="vkfft",
     ):
         """
         Please refer to the documentation of
@@ -37,7 +36,6 @@ class CudaPaganinPhaseRetrieval(PaganinPhaseRetrieval):
             delta_beta=delta_beta,
             pixel_size=pixel_size,
             padding=padding,
-            margin=margin,
             use_rfft=True,
             fft_num_threads=False,
         )
@@ -118,14 +116,13 @@ class CudaPaganinPhaseRetrieval(PaganinPhaseRetrieval):
             raise ValueError("Expected either numpy array, pycuda array or pycuda buffer")
 
     def get_output(self, output):
-        s0, s1 = self.shape_inner
-        ((U, _), (L, _)) = self.margin
+        s0, s1 = self.shape
         if output is None:
             # copy D2H
-            return self.d_radio_padded[U : U + s0, L : L + s1].get()
-        assert output.shape == self.shape_inner
+            return self.d_radio_padded[:s0, :s1].get()
+        assert output.shape == self.shape
         assert output.dtype == np.float32
-        output[:, :] = self.d_radio_padded[U : U + s0, L : L + s1]
+        output[:, :] = self.d_radio_padded[:s0, :s1]
         return output
 
     def apply_filter(self, radio, output=None):

nabu/preproc/tests/test_ctf.py

@@ -1,6 +1,7 @@
 import pytest
 import numpy as np
 import scipy.interpolate
+from nabu.processing.fft_cuda import get_available_fft_implems
 from nabu.testutils import get_data as nabu_get_data
 from nabu.testutils import __do_long_tests__
 from nabu.preproc.flatfield import FlatFieldArrays
@@ -9,11 +10,14 @@ from nabu.preproc import ctf
 from nabu.estimation.distortion import estimate_flat_distortion
 from nabu.misc.filters import correct_spikes
 from nabu.preproc.distortion import DistortionCorrection
-from nabu.cuda.utils import __has_pycuda__, __has_cufft__, get_cuda_context
+from nabu.cuda.utils import __has_pycuda__, get_cuda_context
 
-if __has_pycuda__ and __has_cufft__:
+__has_cufft__ = False
+if __has_pycuda__:
     from nabu.preproc.ctf_cuda import CudaCTFPhaseRetrieval
-    import pycuda.gpuarray as garray
+
+    avail_fft = get_available_fft_implems()
+    __has_cufft__ = len(avail_fft) > 0
 
 
 @pytest.fixture(scope="class")
@@ -39,7 +43,7 @@ def bootstrap_TestCtf(request):
     cls.padded_img_shape_vh = test_data["padded_img_shape_vh"]
     cls.z1_vh = test_data["z1_vh"]
     cls.z2 = test_data["z2"]
-    cls.pix_size_det = test_data["pix_size_det"]
+    cls.pix_size_det = test_data["pix_size_det"][()]
     cls.length_scale = test_data["length_scale"]
     cls.wavelength = test_data["wave_length"]
     cls.remove_spikes_threshold = test_data["remove_spikes_threshold"]
@@ -174,7 +178,7 @@ class TestCtf:
         phase = ctf_filter.retrieve_phase(img)
 
         message = "retrieved phase and reference result differ beyond the accepted tolerance"
-        assert np.abs(phase - self.expected_result).max() < self.abs_tol * (
+        assert np.abs(phase - self.expected_result).max() < 10 * self.abs_tol * (
             np.abs(self.expected_result).mean()
         ), message
 
@@ -219,7 +223,7 @@ class TestCtf:
             phase_fft = ctf_fft.retrieve_phase(img)
             self.check_result(phase_r2c, self.ref_plain, "Something wrong with CtfFilter-FFT")
 
-    @pytest.mark.skipif(not (__has_pycuda__ and __has_cufft__), reason="pycuda and scikit-cuda")
+    @pytest.mark.skipif(not (__has_pycuda__ and __has_cufft__), reason="pycuda and (scikit-cuda or vkfft)")
     def test_cuda_ctf(self):
         data = nabu_get_data("brain_phantom.npz")["data"]
         delta_beta = 50.0
@@ -243,7 +247,7 @@ class TestCtf:
             )
             ref = ctf_filter.retrieve_phase(data)
 
-            d_data = garray.to_gpu(data)
+            d_data = cuda_ctf_filter.cuda_processing.to_device("_d_data", data)
             res = cuda_ctf_filter.retrieve_phase(d_data).get()
             err_max = np.max(np.abs(res - ref))