nabu 2024.2.14__py3-none-any.whl → 2025.1.0__py3-none-any.whl

nabu/app/pcaflats.py

@@ -0,0 +1,122 @@
+import sys
+import os
+import numpy as np
+import h5py
+from .utils import parse_params_values
+from ..utils import is_writeable
+from .cli_configs import PCAFlatsConfig
+from .. import version
+from ..preproc.flatfield import PCAFlatsDecomposer
+from ..io.reader import NXDarksFlats
+
+
+def get_flats_darks_in_nx(filename):
+    dfreader = NXDarksFlats(filename)
+    darks = np.concatenate([d for d in dfreader.get_raw_darks()], axis=0)
+    flats = np.concatenate([f for f in dfreader.get_raw_flats()], axis=0)
+    entry = dfreader.flats_reader.data_path.lstrip("/").split("/")[0]
+    return flats, darks, entry
+
+
+def get_flats_darks_from_h5(filename):
+    flats = []
+    darks = []
+    with h5py.File(filename, "r") as f:
+        for k, v in f.items():
+            if k == "1.1":
+                detector_name = decode_bytes(f["1.1/technique/tomoconfig/detector"][()][0])
+            else:
+                try:
+                    image_key = v["technique/image_key"][()]
+                except:
+                    raise NotImplementedError(
+                        "Legacy h5 file format is not handled. The entry of the h5 file should contain a 'technique/image_key' group."
+                    )
+                if image_key == 2:  # Darks
+                    darks.append(v[f"instrument/{detector_name}/data"][()])
+                elif image_key == 1:  # Flats
+                    flats.append(v[f"instrument/{detector_name}/data"][()])
+
+    flats = np.concatenate([f for f in flats], axis=0)
+    darks = np.concatenate([d for d in darks], axis=0)
+    return flats, darks, "entry0000"  # TODO this will be problematic on the reconstruction side
+
+
+def pcaflats_decomposition(
+    flats, darks, pcaflats_filename="PCAFlats.h5", overwrite=False, entry="entry0000", nsigma=3.0
+):
+    """Compute the PCS decomposition of a series of flats and darks, possibly taken from various scans."""
+    try:
+        decomposer = PCAFlatsDecomposer(flats, darks, nsigma=nsigma)
+        decomposer.save_decomposition(pcaflats_filename, overwrite=overwrite, entry=entry)
+        success = True
+    except:
+        success = False
+        raise ValueError("An error occured in the PCA deccomposition.")
+    return success
+
+
+def decode_bytes(content):
+    if isinstance(content, bytes):
+        return content.decode()
+    else:
+        return content
+
+
+def main(argv=None):
+    """Compute PCA Flats on a series of datasets (h5 or NX)."""
+    if argv is None:
+        argv = sys.argv[1:]
+
+    args = parse_params_values(
+        PCAFlatsConfig,
+        parser_description=f"Compute a PCA Decomposition of flats acquired from various datasets..",
+        program_version="nabu " + version,
+        user_args=argv,
+    )
+
+    # Get "where to write".
+    if args["output_filename"] is None:
+        abspath = os.path.abspath("./PCAFlats.hdf5")
+    else:
+        abspath = os.path.abspath(args["output_filename"])
+
+    pcaflats_dir = os.path.dirname(abspath)
+    pcaflats_filename = os.path.basename(abspath)
+
+    if is_writeable(pcaflats_dir):
+        output_path = os.path.join(pcaflats_dir, pcaflats_filename)
+    else:
+        raise ValueError(f"Output dir {pcaflats_dir} is not writeable.")
+
+    # raise error if file exists and overwrite=False
+    if not args["overwrite"] and os.path.exists(output_path):
+        raise FileExistsError(f"Output file {output_path} already exists. Use --overwrite to overwrite it.")
+
+    # Collect raw darks and flats
+    flats_stack = []
+    darks_stack = []
+
+    for dataset in args["datasets"]:
+        filename = os.path.basename(dataset)
+        kind = filename.split(".")[-1]
+        if kind == "nx":
+            flats, darks, entry = get_flats_darks_in_nx(dataset)
+        elif kind in ("h5", "hdf5"):
+            flats, darks, entry = get_flats_darks_from_h5(dataset)
+
+        flats_stack.append(flats)
+        darks_stack.append(darks)
+
+    flats = np.concatenate(flats_stack, axis=0)
+    darks = np.concatenate(darks_stack, axis=0)
+
+    exit(
+        pcaflats_decomposition(
+            flats, darks, pcaflats_filename=output_path, overwrite=args["overwrite"], entry=entry, nsigma=args["nsigma"]
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

nabu/app/prepare_weights_double.py

@@ -4,7 +4,6 @@ from scipy.special import erf  # pylint: disable=all
 import sys
 import os
 from scipy.ndimage import gaussian_filter
-from nxtomo.nxobject.nxdetector import ImageKey
 from nabu.resources.nxflatfield import update_dataset_info_flats_darks
 from nabu.resources.dataset_analyzer import HDF5DatasetAnalyzer
 from ..io.reader import load_images_from_dataurl_dict
@@ -57,7 +56,7 @@ def main(argv=None):
     beam_profile = 0
     my_flats = load_images_from_dataurl_dict(dataset_info.flats)
 
-    for key, flat in my_flats.items():
+    for flat in my_flats.values():
         beam_profile += flat
     beam_profile = beam_profile / len(list(dataset_info.flats.keys()))
 

nabu/app/reconstruct.py

@@ -1,5 +1,5 @@
-from tomoscan.io import HDF5File
 from .. import version
+from ..io.reader import list_hdf5_entries
 from ..utils import list_match_queries
 from ..pipeline.config import parse_nabu_config_file
 from ..pipeline.config_validators import convert_to_int
@@ -92,12 +92,6 @@ def get_reconstructor(args, overwrite_options=None):
     return reconstructor
 
 
-def list_hdf5_entries(fname):
-    with HDF5File(fname, "r") as f:
-        entries = list(f.keys())
-    return entries
-
-
 def main():
     args = parse_params_values(
         ReconstructConfig,

nabu/app/reconstruct_helical.py

@@ -1,7 +1,6 @@
 from .. import version
 from ..resources.utils import is_hdf5_extension
 from ..pipeline.config import parse_nabu_config_file
-from ..pipeline.config_validators import convert_to_int
 from .cli_configs import ReconstructConfig
 from .utils import parse_params_values
 from .reconstruct import update_reconstruction_start_end, get_log_file
@@ -39,7 +38,7 @@ def main_helical():
         try:
             from silx.math.fft.cufft import CUFFT
         except:  # can't catch narrower - cublasNotInitialized requires cublas !
-            CUFFT = None
+            CUFFT = None  # noqa: F841
     #
 
     logfile = get_log_file(args["logfile"], args["log_file"], forbidden=[args["input_file"]])
@@ -58,9 +57,6 @@ def main_helical():
 
     # Determine which reconstructor to use
     reconstructor_cls = None
-    phase_method = None
-    if "phase" in proc.processing_steps:
-        phase_method = proc.processing_options["phase"]["method"]
 
     # fix the reconstruction roi if not given
     if "reconstruction" in proc.processing_steps:
@@ -71,11 +67,11 @@ def main_helical():
 
         if proc.nabu_config["reconstruction"]["auto_size"]:
             if 2 * rot_center > Nx:
-                w = int(round(2 * rot_center))
+                w = round(2 * rot_center)
             else:
-                w = int(round(2 * Nx - 2 * rot_center))
-            rec_config["start_x"] = int(round(rot_center - w / 2))
-            rec_config["end_x"] = int(round(rot_center + w / 2))
+                w = round(2 * Nx - 2 * rot_center)
+            rec_config["start_x"] = round(rot_center - w / 2)
+            rec_config["end_x"] = round(rot_center + w / 2)
 
             rec_config["start_y"] = rec_config["start_x"]
             rec_config["end_y"] = rec_config["end_x"]

nabu/app/reduce_dark_flat.py

@@ -1,6 +1,4 @@
 import sys
-import logging
-import argparse
 from typing import Optional
 
 from nabu.app.cli_configs import ReduceDarkFlatConfig
@@ -14,6 +12,9 @@ from tomoscan.factory import Factory
 from silx.io.url import DataUrl
 
 
+reduce_methods = tuple(member.value for member in ReduceMethod)
+
+
 def _create_data_urls(output_file: Optional[str], output_data_path: Optional[str], name: str):
     """
     util function to compute reduced Data and metadata url(s)
@@ -67,8 +68,8 @@ def reduce_dark_flat(
     """
     calculation of the darks / flats calling tomoscan utils function
     """
-    dark_method = ReduceMethod.from_value(dark_method) if dark_method is not None else None
-    flat_method = ReduceMethod.from_value(flat_method) if flat_method is not None else None
+    dark_method = ReduceMethod(dark_method) if dark_method is not None else None
+    flat_method = ReduceMethod(flat_method) if flat_method is not None else None
 
     # 1. define url where to save the file
     ## 1.1 for darks

nabu/app/rotate.py

@@ -7,6 +7,8 @@ from multiprocessing.pool import ThreadPool
 import numpy as np
 from tomoscan.io import HDF5File
 from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
+
+from nabu.utils import first_generator_item
 from ..io.utils import get_first_hdf5_entry
 from ..processing.rotation import Rotation
 from ..resources.logger import Logger, LoggerOrPrint
@@ -61,7 +63,7 @@ class HDF5ImagesStackRotation:
         self.output_file = output_file
         copy(self.input_file, output_file)
 
-        first_proj_url = self.dataset_info.projections[list(self.dataset_info.projections.keys())[0]]
+        first_proj_url = self.dataset_info.projections[first_generator_item(self.dataset_info.projections.keys())]
         self.data_path = first_proj_url.data_path()
         dirname, basename = posixpath.split(self.data_path)
         self._data_path_dirname = dirname

nabu/app/stitching.py

@@ -67,12 +67,17 @@ def main():
         futures = {}
         # 2.1 launch jobs
         slurm_job_progress_bars: dict = {}
+
+        # set job name
+        final_output_object_identifier = stitching_config.get_output_object().get_identifier().to_str()
+        stitching_config.slurm_config.job_name = f"stitching-{final_output_object_identifier}"
+
         for i_job, (job, sub_config) in enumerate(
             split_stitching_configuration_to_slurm_job(stitching_config, yield_configuration=True)
         ):
             _logger.info(f"submit job nb {i_job}: handles {sub_config.slices}")
-            output_volume = sub_config.get_output_object().get_identifier().to_str()
-            futures[output_volume] = submit(job, timeout=999999)
+            output_object = sub_config.get_output_object().get_identifier().to_str()
+            futures[output_object] = submit(job, timeout=999999)
             # note on total=100: we only consider percentage in this case (providing advancement from slurm jobs)
             slurm_job_progress_bars[job] = tqdm(
                 total=100,

nabu/app/tests/test_reduce_dark_flat.py

@@ -9,7 +9,7 @@ except ImportError:
     from tomoscan.test.utils import NXtomoMockContext
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture
 def hdf5_scan(tmp_path):
     """simple fixture to create a scan and provide it to another function"""
     test_dir = tmp_path / "my_hdf5_scan"
@@ -26,7 +26,7 @@ def hdf5_scan(tmp_path):
 
 @pytest.mark.parametrize("dark_method", (None, "first", "mean"))
 @pytest.mark.parametrize("flat_method", (None, "last", "median"))
-def test_reduce_dark_flat_hdf5(tmp_path, hdf5_scan, dark_method, flat_method):  # noqa F811
+def test_reduce_dark_flat_hdf5(tmp_path, hdf5_scan, dark_method, flat_method):
     """simply test output - processing is tested at tomoscan side"""
     # test with default url
     default_darks_path = os.path.join(hdf5_scan.path, hdf5_scan.get_dataset_basename() + "_darks.hdf5")

nabu/app/validator.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 import argparse
 import sys
 import os
@@ -19,7 +16,7 @@ def get_scans(path, entries: str):
         if entries == "__all__":
             entries = NXtomoScan.get_valid_entries(path)
         for entry in entries:
-            res.append(NXtomoScan(path, entry))
+            res.append(NXtomoScan(path, entry))  # noqa: PERF401
     else:
         raise TypeError(f"{path} does not looks like a folder containing .EDF or a valid nexus file ")
     return res

nabu/cuda/convolution.py

@@ -1,4 +1,4 @@
-from ..processing.convolution_cuda import *
+from ..processing.convolution_cuda import *  # noqa: F403
 from ..utils import deprecation_warning
 
 deprecation_warning(

nabu/cuda/fft.py

@@ -1,4 +1,4 @@
-from ..processing.fft_cuda import *
+from ..processing.fft_cuda import *  # noqa: F403
 from ..utils import deprecation_warning
 
 deprecation_warning("nabu.cuda.fft has been moved to nabu.processing.fft_cuda", do_print=True, func_name="fft_cuda")

nabu/cuda/medfilt.py

@@ -1,4 +1,4 @@
-from ..processing.medfilt_cuda import *
+from ..processing.medfilt_cuda import *  # noqa: F403
 from ..utils import deprecation_warning
 
 deprecation_warning(

nabu/cuda/padding.py

@@ -1,4 +1,4 @@
-from ..processing.padding_cuda import *
+from ..processing.padding_cuda import *  # noqa: F403
 from ..utils import deprecation_warning
 
 deprecation_warning(

nabu/cuda/src/backproj.cu

@@ -18,8 +18,8 @@ inline __device__ int is_in_circle(int x, int y, float center_x, float center_y,
   This will return arr[y][x] where y is an int (exact access) and x is a float (linear interp horizontally)
 */
 static inline __device__ float linear_interpolation(float* arr, int Nx, float x, int y) {
-    // check commented to gain a bit of speed - the check was done before function call
-    // if (x < 0 || x >= Nx) return 0.0f; // texture address mode CLAMP_TO_EDGE
+    // if (x < 0 || x > Nx-1) return 0.0f; // texture address mode BORDER (CLAMP_TO_EDGE continues with edge)
+    if (x <= -0.5f || x >= Nx - 0.5f) return 0.0f; // texture address mode BORDER (CLAMP_TO_EDGE continues with edge)
     int xm = (int) floorf(x);
     int xp = (int) ceilf(x);
     if ((xm == xp) || (xp >= Nx)) return arr[y*Nx+xm];
@@ -127,10 +127,10 @@ __global__ void backproj(
         #endif
 
         #ifdef USE_TEXTURES
-        if (h1 >= 0 && h1 < num_bins) sum1 += tex2D(tex_projections, h1 + 0.5f, proj + 0.5f);
-        if (h2 >= 0 && h2 < num_bins) sum2 += tex2D(tex_projections, h2 + 0.5f, proj + 0.5f);
-        if (h3 >= 0 && h3 < num_bins) sum3 += tex2D(tex_projections, h3 + 0.5f, proj + 0.5f);
-        if (h4 >= 0 && h4 < num_bins) sum4 += tex2D(tex_projections, h4 + 0.5f, proj + 0.5f);
+        sum1 += tex2D(tex_projections, h1 + 0.5f, proj + 0.5f);
+        sum2 += tex2D(tex_projections, h2 + 0.5f, proj + 0.5f);
+        sum3 += tex2D(tex_projections, h3 + 0.5f, proj + 0.5f);
+        sum4 += tex2D(tex_projections, h4 + 0.5f, proj + 0.5f);
         #else
         if (h1 >= 0 && h1 < num_bins) sum1 += linear_interpolation(d_sino, num_bins, h1, proj);
         if (h2 >= 0 && h2 < num_bins) sum2 += linear_interpolation(d_sino, num_bins, h2, proj);

nabu/cuda/src/cone.cu

@@ -80,7 +80,11 @@ __global__ void devFDK_preweight(void* D_projData, unsigned int projPitch, unsig
 
         const float fWeight = fW / fRayLength;
 
+        #ifndef RADIOS_LAYOUT
         projData[(detectorV*iProjAngles+angle)*projPitch+detectorU] *= fWeight;
+        #else
+        projData[(angle*iProjV+detectorV)*projPitch+detectorU] *= fWeight;
+        #endif
 
         fV += fDetVSize;
     }

nabu/cuda/src/hierarchical_backproj.cu

@@ -6,6 +6,20 @@
     Please cite :
        reference to be added...
 
+
+
+    # Generalized Hierarchical Backprojection (GHBP)
+    # for fast tomographic reconstruction from ultra high resolution images at non-negligible fan angles.
+    #
+    # Authors/Contributions:
+    # - Jonas Graetz, Fraunhofer IIS / Universitat Wurzburg: Algorithm Design and original OpenCL/Python implementation.
+    # - Alessandro Mirone, ESRF: CUDA translation, ESRF / BM18 integration, testing <mirone@esrf.fr>
+    # - Pierre Paleo, ESRF: ESRF / BM18 integration, testing <pierre.paleo@esrf.fr>
+    #
+    # JG was funded by the German Federal Ministry of Education and Research (BMBF), grant 05E2019,
+    # funding the development of BM18 at ESRF in collaboration with the Fraunhofer Gesellschaft,
+    # the Julius-Maximilians-Universitat Wurzburg, and the University of Passau
+
 """
  */
 

nabu/cuda/utils.py

@@ -197,12 +197,12 @@ def cuarray_shape_dtype(cuarray):
 
 
 def get_shape_dtype(arr):
-    if isinstance(arr, garray.GPUArray) or isinstance(arr, np.ndarray):
+    if isinstance(arr, (garray.GPUArray, np.ndarray)):
         return arr.shape, arr.dtype
     elif isinstance(arr, cuda.Array):
         return cuarray_shape_dtype(arr)
     else:
-        raise ValueError("Unknown array type %s" % str(type(arr)))
+        raise TypeError("Unknown array type %s" % str(type(arr)))
 
 
 def copy_array(dst, src, check=False, src_dtype=None, dst_x_in_bytes=0, dst_y=0):

nabu/estimation/alignment.py

@@ -87,20 +87,20 @@ class AlignmentBase:
         if not len(shape_stack) == 3:
             raise ValueError(
                 "A stack of 2-dimensional images is required. Shape of stack: %s"
-                % (" ".join(("%d" % x for x in shape_stack)))
+                % (" ".join("%d" % x for x in shape_stack))
             )
         if not len(shape_pos) == 1:
             raise ValueError(
                 "Positions need to be a 1-dimensional array. Shape of the positions variable: %s"
-                % (" ".join(("%d" % x for x in shape_pos)))
+                % (" ".join("%d" % x for x in shape_pos))
             )
         if not shape_stack[0] == shape_pos[0]:
             raise ValueError(
                 "The same number of images and positions is required."
                 + " Shape of stack: %s, shape of positions variable: %s"
                 % (
-                    " ".join(("%d" % x for x in shape_stack)),
-                    " ".join(("%d" % x for x in shape_pos)),
+                    " ".join("%d" % x for x in shape_stack),
+                    " ".join("%d" % x for x in shape_pos),
                 )
             )
 
@@ -110,18 +110,18 @@ class AlignmentBase:
         shape_2 = np.squeeze(img_2).shape
         if not len(shape_1) == 2:
             raise ValueError(
-                "Images need to be 2-dimensional. Shape of image #1: %s" % (" ".join(("%d" % x for x in shape_1)))
+                "Images need to be 2-dimensional. Shape of image #1: %s" % (" ".join("%d" % x for x in shape_1))
             )
         if not len(shape_2) == 2:
             raise ValueError(
-                "Images need to be 2-dimensional. Shape of image #2: %s" % (" ".join(("%d" % x for x in shape_2)))
+                "Images need to be 2-dimensional. Shape of image #2: %s" % (" ".join("%d" % x for x in shape_2))
             )
         if not np.all(shape_1 == shape_2):
             raise ValueError(
                 "Images need to be of the same shape. Shape of image #1: %s, image #2: %s"
                 % (
-                    " ".join(("%d" % x for x in shape_1)),
-                    " ".join(("%d" % x for x in shape_2)),
+                    " ".join("%d" % x for x in shape_1),
+                    " ".join("%d" % x for x in shape_2),
                 )
             )
 
@@ -153,7 +153,7 @@ class AlignmentBase:
         if not (len(f_vals.shape) == 2):
             raise ValueError(
                 "The fitted values should form a 2-dimensional array. Array of shape: [%s] was given."
-                % (" ".join(("%d" % s for s in f_vals.shape)))
+                % (" ".join("%d" % s for s in f_vals.shape))
             )
         if fy is None:
             fy_half_size = (f_vals.shape[0] - 1) / 2
@@ -161,7 +161,7 @@ class AlignmentBase:
         elif not (len(fy.shape) == 1 and np.all(fy.size == f_vals.shape[0])):
             raise ValueError(
                 "Vertical coordinates should have the same length as values matrix. Sizes of fy: %d, f_vals: [%s]"
-                % (fy.size, " ".join(("%d" % s for s in f_vals.shape)))
+                % (fy.size, " ".join("%d" % s for s in f_vals.shape))
             )
         if fx is None:
             fx_half_size = (f_vals.shape[1] - 1) / 2
@@ -169,7 +169,7 @@ class AlignmentBase:
         elif not (len(fx.shape) == 1 and np.all(fx.size == f_vals.shape[1])):
             raise ValueError(
                 "Horizontal coordinates should have the same length as values matrix. Sizes of fx: %d, f_vals: [%s]"
-                % (fx.size, " ".join(("%d" % s for s in f_vals.shape)))
+                % (fx.size, " ".join("%d" % s for s in f_vals.shape))
             )
 
         fy, fx = np.meshgrid(fy, fx, indexing="ij")
@@ -190,14 +190,7 @@ class AlignmentBase:
         vertex_max_yx = [np.max(fy), np.max(fx)]
         if np.any(vertex_yx < vertex_min_yx) or np.any(vertex_yx > vertex_max_yx):
             raise ValueError(
-                "Fitted (y: {}, x: {}) positions are outside the input margins y: [{}, {}], and x: [{}, {}]".format(
-                    vertex_yx[0],
-                    vertex_yx[1],
-                    vertex_min_yx[0],
-                    vertex_max_yx[0],
-                    vertex_min_yx[1],
-                    vertex_max_yx[1],
-                )
+                f"Fitted (y: {vertex_yx[0]}, x: {vertex_yx[1]}) positions are outside the input margins y: [{vertex_min_yx[0]}, {vertex_max_yx[0]}], and x: [{vertex_min_yx[1]}, {vertex_max_yx[1]}]"
             )
         return vertex_yx
 
@@ -225,10 +218,10 @@ class AlignmentBase:
         float
             Estimated function max, according to the coordinates in fx.
         """
-        if not len(f_vals.shape) in (1, 2):
+        if len(f_vals.shape) not in (1, 2):
             raise ValueError(
                 "The fitted values should be either one or a collection of 1-dimensional arrays. Array of shape: [%s] was given."
-                % (" ".join(("%d" % s for s in f_vals.shape)))
+                % (" ".join("%d" % s for s in f_vals.shape))
             )
         num_vals = f_vals.shape[0]
 
@@ -264,16 +257,9 @@ class AlignmentBase:
         upper_bound_ok = vertex_x < vertex_max_x
         if not np.all(lower_bound_ok * upper_bound_ok):
             if len(f_vals.shape) == 1:
-                message = "Fitted position {} is outide the input margins [{}, {}]".format(
-                    vertex_x, vertex_min_x, vertex_max_x
-                )
+                message = f"Fitted position {vertex_x} is outide the input margins [{vertex_min_x}, {vertex_max_x}]"
             else:
-                message = "Fitted positions outside the input margins [{}, {}]: {} below and {} above".format(
-                    vertex_min_x,
-                    vertex_max_x,
-                    np.sum(1 - lower_bound_ok),
-                    np.sum(1 - upper_bound_ok),
-                )
+                message = f"Fitted positions outside the input margins [{vertex_min_x}, {vertex_max_x}]: {np.sum(1 - lower_bound_ok)} below and {np.sum(1 - upper_bound_ok)} above"
             raise ValueError(message)
         if return_vertex_val:
             vertex_val = coeffs[0, :] + vertex_x * coeffs[1, :] / 2
@@ -354,7 +340,7 @@ class AlignmentBase:
         if not (len(img_shape) == 2):
             raise ValueError(
                 "The input image should be either a 1 or 2-dimensional array. Array of shape: [%s] was given."
-                % (" ".join(("%d" % s for s in cc.shape)))
+                % (" ".join("%d" % s for s in cc.shape))
             )
         other_axis = (axis + 1) % 2
         # get pixel having the maximum value of the correlation array