nabu 2023.2.1__py3-none-any.whl → 2024.1.0rc3__py3-none-any.whl

nabu/reconstruction/tests/test_fbp.py

@@ -4,10 +4,14 @@ from scipy.ndimage import shift
 from nabu.pipeline.params import fbp_filters
 from nabu.utils import clip_circle
 from nabu.testutils import get_data, generate_tests_scenarios, __do_long_tests__
-from nabu.cuda.utils import get_cuda_context, __has_pycuda__, __has_cufft__
+from nabu.cuda.utils import get_cuda_context, __has_pycuda__
 from nabu.opencl.utils import get_opencl_context, __has_pyopencl__
 
-__has_pycuda__ = __has_pycuda__ and __has_cufft__  # need both for using Cuda backprojector
+from nabu.processing.fft_cuda import has_skcuda, has_vkfft as has_vkfft_cu
+from nabu.processing.fft_opencl import has_vkfft as has_vkfft_cl
+
+__has_pycuda__ = __has_pycuda__ and (has_skcuda() or has_vkfft_cu())
+__has_pyopencl__ = __has_pyopencl__ and has_vkfft_cl()
 
 if __has_pycuda__:
     from nabu.reconstruction.fbp import CudaBackprojector
@@ -22,7 +26,7 @@ if __do_long_tests__:
             "backend": ["cuda", "opencl"],
             "input_on_gpu": [False, True],
             "output_on_gpu": [False, True],
-            "opencl_use_textures": [True, False],
+            "use_textures": [True, False],
         }
     )
 
@@ -34,8 +38,8 @@ def bootstrap(request):
     cls.ref_512 = get_data("mri_rec_astra.npz")["data"]
     # always use contiguous arrays
     cls.sino_511 = np.ascontiguousarray(cls.sino_512[:, :-1])
-    cls.tol = 5e-2
-    cls.tol_without_texture = 5.1e-2  # Problem with the border of circle ?!
+    # Could be set to 5.0e-2 when using textures. When not using textures, interpolation slightly differs
+    cls.tol = 5.1e-2
 
     if __has_pycuda__:
         cls.cuda_ctx = get_cuda_context(cleanup_at_exit=False)
@@ -56,19 +60,19 @@ class TestFBP:
     def _get_backprojector(self, config, *bp_args, **bp_kwargs):
         if config["backend"] == "cuda":
             if not (__has_pycuda__):
-                pytest.skip("Need pycuda + scikit-cuda")
+                pytest.skip("Need pycuda + (scikit-cuda or pyvkfft)")
             Backprojector = CudaBackprojector
             ctx = self.cuda_ctx
         else:
             if not (__has_pyopencl__):
-                pytest.skip("Need pyopencl")
+                pytest.skip("Need pyopencl + pyvkfft")
             Backprojector = OpenCLBackprojector
             ctx = self.opencl_ctx
-            if config.get("opencl_use_textures", True) is False:
-                # patch "extra_options"
-                extra_options = bp_kwargs.pop("extra_options", {})
-                extra_options["use_textures"] = False
-                bp_kwargs["extra_options"] = extra_options
+        if config.get("use_textures", True) is False:
+            # patch "extra_options"
+            extra_options = bp_kwargs.pop("extra_options", {})
+            extra_options["use_textures"] = False
+            bp_kwargs["extra_options"] = extra_options
         return Backprojector(*bp_args, **bp_kwargs, backend_options={"ctx": ctx})
 
     @staticmethod
@@ -84,13 +88,6 @@ class TestFBP:
             res = res.get()
         return res
 
-    def get_error_tolerance(self, config):
-        tol = self.tol
-        if config["backend"] == "opencl":
-            tol = self.tol_without_texture
-            tol += 1e-3
-        return tol
-
     @pytest.mark.parametrize("config", scenarios)
     def test_fbp_512(self, config):
         """
@@ -102,7 +99,7 @@ class TestFBP:
         delta_clipped = self.clip_to_inner_circle(res - self.ref_512)
         err_max = np.max(np.abs(delta_clipped))
 
-        assert err_max < self.get_error_tolerance(config), "Something wrong with config=%s" % (str(config))
+        assert err_max < self.tol, "Something wrong with config=%s" % (str(config))
 
     @pytest.mark.parametrize("config", scenarios)
     def test_fbp_511(self, config):
@@ -116,7 +113,7 @@ class TestFBP:
         delta_clipped = self.clip_to_inner_circle(res - ref)
         err_max = np.max(np.abs(delta_clipped))
 
-        assert err_max < self.get_error_tolerance(config), "Something wrong with config=%s" % (str(config))
+        assert err_max < self.tol, "Something wrong with config=%s" % (str(config))
 
     @pytest.mark.parametrize("config", scenarios)
     def test_fbp_roi(self, config):
@@ -146,7 +143,7 @@ class TestFBP:
         }
         for roi, ref in cases.values():
             err_max = backproject_roi(roi, ref)
-            assert err_max < self.get_error_tolerance(config), "Something wrong with ROI = %s for config=%s" % (
+            assert err_max < self.tol, "Something wrong with ROI = %s for config=%s" % (
                 str(roi),
                 str(config),
             )
@@ -183,6 +180,7 @@ class TestFBP:
         """
         sino = self.sino_512
         tol = 1e-5
+
         for rot_center in [None, sino.shape[1] / 2.0 - 10, sino.shape[1] / 2.0 + 15]:
             B = self._get_backprojector(
                 config, sino.shape, rot_center=rot_center, extra_options={"clip_outer_circle": True}
@@ -195,15 +193,8 @@ class TestFBP:
             res_noclip = B0.fbp(sino)
             ref = self.clip_to_inner_circle(res_noclip, radius_factor=1)
 
-            if (
-                rot_center != None
-                and config["backend"] == "opencl"
-                and config.get("opencl_use_textures", True) is False
-            ):
-                # TODO problem when not using textures here - though again outside of circle
-                tol = 10
-
-            err_max = np.max(np.abs(res - ref))
+            abs_diff = np.abs(res - ref)
+            err_max = np.max(abs_diff)
             assert err_max < tol, "Max error is too high for rot_center=%s ; %s" % (str(rot_center), str(config))
 
     @pytest.mark.parametrize("config", scenarios)

nabu/reconstruction/tests/test_filtering.py

@@ -94,7 +94,14 @@ class TestSinoFilter:
             self.sino, sino_filter.dwidth_padded, filter_name=config["filter_name"], padding_mode=config["padding_mode"]
         )
 
-        assert np.allclose(res.get(), ref, atol=6e-5)
+        if not np.allclose(res.get(), ref, atol=6e-5):
+            from spire.utils import ims
+
+            ims([res.get(), ref, res.get() - ref])
+
+        assert np.allclose(res.get(), ref, atol=6e-5), "test_cuda_filter: something wrong with config=%s" % (
+            str(config)
+        )
 
     @pytest.mark.skipif(
         not (__has_pyopencl__ and __has_vkfft__), reason="Need OpenCL + pyopencl + pyvkfft to use OpenCLSinoFilter"
@@ -121,4 +128,6 @@ class TestSinoFilter:
             self.sino, sino_filter.dwidth_padded, filter_name=config["filter_name"], padding_mode=config["padding_mode"]
         )
 
-        assert np.allclose(res.get(), ref, atol=6e-5)
+        assert np.allclose(res.get(), ref, atol=6e-5), "test_opencl_filter: something wrong with config=%s" % (
+            str(config)
+        )

nabu/resources/dataset_analyzer.py

@@ -1,21 +1,16 @@
 import os
+from bisect import bisect_left
 import numpy as np
 from silx.io import get_data
 from silx.io.url import DataUrl
 from tomoscan.esrf.scan.edfscan import EDFTomoScan
-from tomoscan.esrf.scan.hdf5scan import HDF5TomoScan
-
+from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
 from ..utils import check_supported
 from ..io.utils import get_compacted_dataslices
 from .utils import is_hdf5_extension, get_values_from_file
 from .logger import LoggerOrPrint
 
-# Wait for next tomoscan release to ship "nexus_version"
-from packaging.version import parse as parse_version
-from tomoscan.version import version as tomoscan_version
-
-_tomoscan_has_nxversion = parse_version(tomoscan_version) > parse_version("0.6.0")
-#
+from ..pipeline.utils import nabu_env_settings
 
 
 class DatasetAnalyzer:
@@ -56,25 +51,29 @@ class DatasetAnalyzer:
             "output_dir": None,
             "exclude_projections": None,
             "hdf5_entry": None,
+            "nx_version": 1.0,
         }
-        if _tomoscan_has_nxversion:
-            advanced_options["nx_version"] = 1.0
         # --
         advanced_options.update(extra_options)
         self.extra_options = advanced_options
 
+    # pylint: disable=E1136
     def _get_excluded_projections(self):
+        self._ignore_projections_indices = None
+        self._need_rebuild_tomoscan_object_to_exclude_projections = False
         excluded_projs = self.extra_options["exclude_projections"]
         if excluded_projs is None:
             return
-        projs_idx = get_values_from_file(excluded_projs, any_size=True).astype(np.int32).tolist()
-        self.logger.info("Ignoring projections: %s" % (str(projs_idx)))
-        self.extra_options["exclude_projections"] = projs_idx
+        if excluded_projs["type"] == "indices":
+            projs_idx = get_values_from_file(excluded_projs["file"], any_size=True).astype(np.int32).tolist()
+            self._ignore_projections_indices = projs_idx
+        else:
+            self._need_rebuild_tomoscan_object_to_exclude_projections = True
 
     def _init_dataset_scan(self, **kwargs):
         if self._scanner is None:
             raise ValueError("Base class")
-        if self._scanner is HDF5TomoScan:
+        if self._scanner is NXtomoScan:
             if self.extra_options.get("hdf5_entry", None) is not None:
                 kwargs["entry"] = self.extra_options["hdf5_entry"]
             if self.extra_options.get("nx_version", None) is not None:
@@ -82,10 +81,50 @@ class DatasetAnalyzer:
         if self._scanner is EDFTomoScan:
             # Assume 1 frame per file (otherwise too long to open each file)
             kwargs["n_frames"] = 1
+
         self.dataset_scanner = self._scanner(  # pylint: disable=E1102
-            self.location, ignore_projections=self.extra_options["exclude_projections"], **kwargs
+            self.location, ignore_projections=self._ignore_projections_indices, **kwargs
         )
         self.projections = self.dataset_scanner.projections
+
+        # ---
+        if self._need_rebuild_tomoscan_object_to_exclude_projections:
+            # pylint: disable=E1136
+            exclude_projs = self.extra_options["exclude_projections"]
+            rot_angles_deg = np.rad2deg(self.rotation_angles)
+            self._rotation_angles = None  # prevent caching
+            # tomoscan only supports ignore_projections=<list of integers>
+            # However this is cumbersome to use, it's more convenient to use angular range or list of angles
+            # But having angles instead of indices implies to already have information on current scan angular range
+            ignore_projections_indices = []
+            if exclude_projs["type"] == "angular_range":
+                exclude_angle_min, exclude_angle_max = exclude_projs["range"]
+                projections_indices = np.array(sorted(self.dataset_scanner.projections.keys()))
+                for proj_idx, angle in zip(projections_indices, rot_angles_deg):
+                    if exclude_angle_min <= angle and angle <= exclude_angle_max:
+                        ignore_projections_indices.append(proj_idx)
+            elif exclude_projs["type"] == "angles":
+                excluded_angles = get_values_from_file(exclude_projs["file"], any_size=True).astype(np.float32).tolist()
+                for excluded_angle in excluded_angles:
+                    proj_idx = bisect_left(rot_angles_deg, excluded_angle)
+                    if proj_idx < rot_angles_deg.size:
+                        ignore_projections_indices.append(proj_idx)
+            # Rebuild the dataset_scanner instance
+            self._ignore_projections_indices = ignore_projections_indices
+            self.dataset_scanner = self._scanner(  # pylint: disable=E1102
+                self.location, ignore_projections=self._ignore_projections_indices, **kwargs
+            )
+        # ---
+        if self._ignore_projections_indices is not None:
+            self.logger.info("Excluding projections: %s" % str(self._ignore_projections_indices))
+
+        if nabu_env_settings.skip_tomoscan_checks:
+            self.logger.warning(
+                " WARNING: according to nabu_env_settings.skip_tomoscan_checks, skipping virtual layout integrity check of tomoscan which is time consuming"
+            )
+            self.dataset_scanner.set_check_behavior(run_check=False, raise_error=False)
+
+        self.projections = self.dataset_scanner.projections
         self.flats = self.dataset_scanner.flats
         self.darks = self.dataset_scanner.darks
         self.n_angles = len(self.dataset_scanner.projections)
@@ -106,7 +145,8 @@ class DatasetAnalyzer:
         self._energy = None
         self._pixel_size = None
         self._distance = None
-        self.flats_srcurrent = None
+        self._flats_srcurrent = None
+        self._projections_srcurrent = None
 
     @property
     def energy(self):
@@ -183,20 +223,43 @@ class DatasetAnalyzer:
     def detector_tilt(self, tilt):
         self._detector_tilt = tilt
 
-    @property
-    def projections_srcurrent(self):
-        """
-        Return the synchrotron electric current for each projection.
-        """
+    def _get_srcurrent(self, indices):
         srcurrent = self.dataset_scanner.electric_current
         if srcurrent is None or len(srcurrent) == 0:
             return None
         srcurrent_all = np.array(srcurrent)
-        projections_indices = np.array(sorted(self.projections.keys()))
-        if np.any(projections_indices >= len(srcurrent_all)):
+        if np.any(indices >= len(srcurrent_all)):
             self.logger.error("Something wrong with SRCurrent: not enough values!")
             return None
-        return srcurrent_all[projections_indices].astype("f")
+        return srcurrent_all[indices].astype("f")
+
+    @property
+    def projections_srcurrent(self):
+        """
+        Return the synchrotron electric current for each projection.
+        """
+        if self._projections_srcurrent is None:
+            projections_indices = np.array(sorted(self.projections.keys()))
+            self._projections_srcurrent = self._get_srcurrent(projections_indices)
+        return self._projections_srcurrent
+
+    @projections_srcurrent.setter
+    def projections_srcurrent(self, val):
+        self._projections_srcurrent = val
+
+    @property
+    def flats_srcurrent(self):
+        """
+        Return the synchrotron electric current for each flat image.
+        """
+        if self._flats_srcurrent is None:
+            flats_indices = np.array(sorted(self.flats.keys()))
+            self._flats_srcurrent = self._get_srcurrent(flats_indices)
+        return self._flats_srcurrent
+
+    @flats_srcurrent.setter
+    def flats_srcurrent(self, val):
+        self._flats_srcurrent = val
 
     def check_defined_attribute(self, name, error_msg=None):
         """
@@ -252,7 +315,7 @@ class EDFDatasetAnalyzer(DatasetAnalyzer):
             scan_range = self.dataset_scanner.scan_range
             if scan_range is not None:
                 fullturn = abs(scan_range - 360) < abs(scan_range - 180)
-                angles = np.linspace(0, scan_range, num=self.dataset_scanner.tomo_n, endpoint=fullturn, dtype="f")
+                angles = np.linspace(0, scan_range, num=len(self.projections), endpoint=fullturn, dtype="f")
                 self._rotation_angles = np.deg2rad(angles)
         return self._rotation_angles
 
@@ -262,7 +325,7 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
     HDF5 dataset analyzer
     """
 
-    _scanner = HDF5TomoScan
+    _scanner = NXtomoScan
     kind = "hdf5"
 
     @property

nabu/resources/nxflatfield.py

@@ -2,7 +2,7 @@ import os
 import numpy as np
 from silx.io.url import DataUrl
 from tomoscan.io import HDF5File
-from tomoscan.esrf.scan.hdf5scan import HDF5TomoScan
+from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
 from ..utils import check_supported, is_writeable
 
 
@@ -61,7 +61,7 @@ def get_metadata_url(url, frame_type):
     Return the url of the metadata stored alongside flats/darks
     """
     check_supported(frame_type, ["flats", "darks"], "frame type")
-    template_url = getattr(HDF5TomoScan, "REDUCED_%s_METADATAURLS" % frame_type.upper())[0]
+    template_url = getattr(NXtomoScan, "REDUCED_%s_METADATAURLS" % frame_type.upper())[0]
     return DataUrl(
         file_path=url.file_path(),
         data_path=template_url.data_path(),

nabu/resources/tests/test_nxflatfield.py

@@ -4,7 +4,7 @@ from shutil import rmtree
 import pytest
 import numpy as np
 from silx.io import get_data
-from tomoscan.esrf.scan.hdf5scan import ImageKey
+from nxtomo.nxobject.nxdetector import ImageKey
 from nabu.testutils import generate_nx_dataset
 from nabu.resources.nxflatfield import update_dataset_info_flats_darks
 from nabu.resources.dataset_analyzer import HDF5DatasetAnalyzer

nabu/resources/utils.py

@@ -1,7 +1,8 @@
 from ast import literal_eval
 import numpy as np
 from psutil import virtual_memory, cpu_count
-from tomoscan.unitsystem.metricsystem import MetricSystem
+from pyunitsystem.metricsystem import MetricSystem
+from pyunitsystem.energysystem import EnergySI
 
 
 def get_values_from_file(fname, n_values=None, shape=None, sep=None, any_size=False):
@@ -162,6 +163,12 @@ def get_quantities_and_units(string, sep=";"):
         value, unit = value_and_unit.split()
         val = float(value)
         # Convert to SI
-        conversion_factor = MetricSystem.from_str(unit).value
+        try:
+            # handle metrics
+            conversion_factor = MetricSystem.from_str(unit).value
+        except ValueError:
+            # handle energies
+            conversion_factor = EnergySI.from_str(unit).value / EnergySI.KILOELECTRONVOLT.value
+
         result[quantity_name] = val * conversion_factor
     return result

nabu/stitching/alignment.py

@@ -0,0 +1,184 @@
+import h5py
+import numpy
+from typing import Union
+from silx.utils.enum import Enum as _Enum
+from tomoscan.volumebase import VolumeBase
+from tomoscan.esrf.volume.hdf5volume import HDF5Volume
+from nabu.io.utils import DatasetReader
+
+
+class AlignmentAxis2(_Enum):
+    CENTER = "center"
+    LEFT = "left"
+    RIGTH = "right"
+
+
+class AlignmentAxis1(_Enum):
+    FRONT = "front"
+    CENTER = "center"
+    BACK = "back"
+
+
+def align_horizontally(data: numpy.ndarray, alignment: AlignmentAxis2, new_width: int, pad_mode="constant"):
+    """
+    Align data horizontally to make sure new data width will ne `new_width`.
+
+    :param numpy.ndarray data: data to align
+    :param HAlignment alignment: alignment strategy
+    :param int new_width: output data width
+    """
+    current_width = data.shape[-1]
+    alignment = AlignmentAxis2.from_value(alignment)
+
+    if current_width > new_width:
+        raise ValueError(f"data.shape[-1] ({data.shape[-1]}) > new_width ({new_width}). Unable to crop data")
+    elif current_width == new_width:
+        return data
+    else:
+        if alignment is AlignmentAxis2.CENTER:
+            left_width = (new_width - current_width) // 2
+            right_width = (new_width - current_width) - left_width
+        elif alignment is AlignmentAxis2.LEFT:
+            left_width = 0
+            right_width = new_width - current_width
+        elif alignment is AlignmentAxis2.RIGTH:
+            left_width = new_width - current_width
+            right_width = 0
+        else:
+            raise ValueError(f"alignment {alignment.value} is not handled")
+
+        assert left_width >= 0, f"pad width must be positive - left width isn't ({left_width})"
+        assert right_width >= 0, f"pad width must be positive - right width isn't ({right_width})"
+        return numpy.pad(
+            data,
+            pad_width=((0, 0), (left_width, right_width)),
+            mode=pad_mode,
+        )
+
+
+class PaddedRawData:
+    """
+    Util class to extend a data when necessary
+    Must to aplpy to a volume and to an hdf5dataset - array
+    The idea behind is to avoid loading all the data in memory
+    """
+
+    def __init__(self, data: Union[numpy.ndarray, h5py.Dataset], axis_1_pad_width: tuple) -> None:
+        self._axis_1_pad_width = numpy.array(axis_1_pad_width)
+        if not (self._axis_1_pad_width.size == 2 and self._axis_1_pad_width[0] >= 0 and self._axis_1_pad_width[1] >= 0):
+            raise ValueError(f"'axis_1_pad_width' expects to positive elements. Get {axis_1_pad_width}")
+        self._raw_data = data
+        self._raw_data_end = None
+        # note: for now we return only frames with zeros for padded frames.
+        # in the future we could imagine having a method and miror existing volume or extend the closest frame, or get a mean value...
+        self._empty_frame = None
+        self._dtype = None
+        self._shape = None
+        self._raw_data_shape = self.raw_data.shape
+
+    @staticmethod
+    def get_empty_frame(shape, dtype):
+        return numpy.zeros(
+            shape=shape,
+            dtype=dtype,
+        )
+
+    @property
+    def empty_frame(self):
+        if self._empty_frame is None:
+            self._empty_frame = self.get_empty_frame(
+                shape=(self.shape[0], 1, self.shape[2]),
+                dtype=self.dtype,
+            )
+        return self._empty_frame
+
+    @property
+    def shape(self):
+        if self._shape is None:
+            self._shape = tuple(
+                (
+                    self._raw_data_shape[0],
+                    numpy.sum(
+                        numpy.array(self._axis_1_pad_width),
+                    )
+                    + self._raw_data_shape[1],
+                    self._raw_data_shape[2],
+                )
+            )
+        return self._shape
+
+    @property
+    def raw_data(self):
+        return self._raw_data
+
+    @property
+    def raw_data_start(self):
+        return self._axis_1_pad_width[0]
+
+    @property
+    def raw_data_end(self):
+        if self._raw_data_end is None:
+            self._raw_data_end = self._axis_1_pad_width[0] + self._raw_data_shape[1]
+        return self._raw_data_end
+
+    @property
+    def dtype(self):
+        if self._dtype is None:
+            self._dtype = self.raw_data.dtype
+        return self._dtype
+
+    def __getitem__(self, args):
+        if not isinstance(args, tuple) and len(args) == 3:
+            raise ValueError("only handles 3D slicing")
+        elif not (args[0] == slice(None, None, None) and args[2] == slice(None, None, None)):
+            raise ValueError(
+                "slicing only handled along axis 1. First and third tuple item are expected to be empty slice as slice(None, None, None)"
+            )
+        else:
+            if numpy.isscalar(args[1]):
+                args = (
+                    args[0],
+                    slice(args[1], args[1] + 1, 1),
+                    args[2],
+                )
+
+            start = args[1].start
+            if start is None:
+                start = 0
+            stop = args[1].stop
+            if stop is None:
+                stop = self.shape[1]
+            step = args[1].step
+            # some test
+            if start < 0 or stop < 0:
+                raise ValueError("only positive position are handled")
+            if start >= stop:
+                raise ValueError("start >= stop")
+            if stop > self.shape[1]:
+                raise ValueError("stop > self.shape[1]")
+            if step not in (1, None):
+                raise ValueError("for now PaddedVolume only handles steps of 1")
+
+            first_part_array = None
+            if start < self.raw_data_start and (stop - start > 0):
+                stop_first_part = min(stop, self.raw_data_start)
+                first_part_array = numpy.repeat(self.empty_frame, repeats=stop_first_part - start, axis=1)
+                start = stop_first_part
+
+            third_part_array = None
+            if stop > self.raw_data_end and (stop - start > 0):
+                if stop > self.shape[1]:
+                    raise ValueError("requested slice is out of boundaries")
+                start_third_part = max(start, self.raw_data_end)
+                third_part_array = numpy.repeat(self.empty_frame, repeats=stop - start_third_part, axis=1)
+                stop = self.raw_data_end
+
+            if start >= self.raw_data_start and stop >= self.raw_data_start and (stop - start > 0):
+                second_part_array = self.raw_data[:, start - self.raw_data_start : stop - self.raw_data_start, :]
+            else:
+                second_part_array = None
+
+            parts = tuple(filter(lambda a: a is not None, (first_part_array, second_part_array, third_part_array)))
+            return numpy.hstack(
+                parts,
+            )