nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl

nabu/testutils.py

@@ -4,10 +4,7 @@ import os
 import numpy as np
 from scipy.signal.windows import gaussian
 from silx.resources import ExternalResources
-from silx.io.dictdump import dicttoh5, nxtodict, dicttonx
-from silx.io.url import DataUrl
-from tomoscan.io import HDF5File
-from .io.utils import get_compacted_dataslices
+from silx.io.dictdump import nxtodict, dicttonx
 
 utilstest = ExternalResources(
     project="nabu", url_base="http://www.silx.org/pub/nabu/data/", env_key="NABU_DATA", timeout=60
@@ -200,154 +197,6 @@ def compare_shifted_images(img1, img2, fwhm_ratio=0.7, return_upper_bound=False)
         return res
 
 
-class SimpleHDF5TomoScanMock:
-    def __init__(self, image_key):
-        self._image_key = image_key
-
-    @property
-    def image_key(self):
-        return self._image_key
-
-    @image_key.setter
-    def image_key(self, image_key):
-        self._image_key = image_key
-
-    def save_reduced_flats(self, *args, **kwargs):
-        pass
-
-    def save_reduced_darks(self, *args, **kwargs):
-        pass
-
-
-class NXDatasetMock:
-    """
-    An alternative to tomoscan.esrf.mock.MockHDF5, with a different interface.
-    Attributes are not supported !
-    """
-
-    def __init__(self, data_volume, image_keys, rotation_angles=None, incident_energy=19.0, other_params=None):
-        self.data_volume = data_volume
-        self.n_proj = data_volume.shape[0]
-        self.image_key = image_keys
-        if rotation_angles is None:
-            rotation_angles = np.linspace(0, 180, self.n_proj, False)
-        self.rotation_angle = rotation_angles
-        self.incident_energy = incident_energy
-        assert image_keys.size == self.n_proj
-        self._finalize_init(other_params)
-        self.dataset_dict = None
-        self.fname = None
-        # Mocks more attributes
-        self.dataset_scanner = SimpleHDF5TomoScanMock(image_key=self.image_key)
-        self.kind = "hdf5"
-
-    def _finalize_init(self, other_params):
-        if other_params is None:
-            other_params = {}
-        default_params = {
-            "detector": {
-                "count_time": 0.05 * np.ones(self.n_proj, dtype="f"),
-                "distance": 0.5,
-                "field_of_view": "Full",
-                "image_key_control": np.copy(self.image_key),
-                "x_pixel_size": 6.5e-6,
-                "y_pixel_size": 6.5e-6,
-                "x_magnified_pixel_size": 6.5e-5,
-                "y_magnified_pixel_size": 6.5e-5,
-            },
-            "sample": {
-                "name": "dummy sample",
-                "x_translation": 5e-4 * np.ones(self.n_proj, dtype="f"),
-                "y_translation": 5e-4 * np.ones(self.n_proj, dtype="f"),
-                "z_translation": 5e-4 * np.ones(self.n_proj, dtype="f"),
-            },
-        }
-        default_params.update(other_params)
-        self.other_params = default_params
-
-    def generate_dict(self):
-        beam_group = {
-            "incident_energy": self.incident_energy,
-        }
-        detector_other_params = self.other_params["detector"]
-        detector_group = {
-            "count_time": detector_other_params["count_time"],
-            "data": self.data_volume,
-            "distance": detector_other_params["distance"],
-            "field_of_view": detector_other_params["field_of_view"],
-            "image_key": self.image_key,
-            "image_key_control": detector_other_params["image_key_control"],
-            "x_pixel_size": detector_other_params["x_pixel_size"],
-            "y_pixel_size": detector_other_params["y_pixel_size"],
-            "x_magnified_pixel_size": detector_other_params["x_magnified_pixel_size"],
-            "y_magnified_pixel_size": detector_other_params["y_magnified_pixel_size"],
-        }
-        sample_other_params = self.other_params["sample"]
-        sample_group = {
-            "name": sample_other_params["name"],
-            "rotation_angle": self.rotation_angle,
-            "x_translation": sample_other_params["x_translation"],
-            "y_translation": sample_other_params["y_translation"],
-            "z_translation": sample_other_params["z_translation"],
-        }
-        self.dataset_dict = {
-            "beam": beam_group,
-            "instrument": {
-                "detector": detector_group,
-            },
-            "sample": sample_group,
-        }
-
-    def generate_hdf5_file(self, fname, h5path=None):
-        self.fname = fname
-        h5path = h5path or "/entry"
-        if self.dataset_dict is None:
-            self.generate_dict()
-        dicttoh5(self.dataset_dict, fname, h5path=h5path, mode="a")
-        # Patch the "data" field which is exported as string by dicttoh5 (?!)
-        self.dataset_path = os.path.join(h5path, "instrument/detector/data")
-        with HDF5File(fname, "a") as fid:
-            del fid[self.dataset_path]
-            fid[self.dataset_path] = self.dataset_dict["instrument"]["detector"]["data"]
-
-    # Mock some of the HDF5DatasetAnalyzer attributes
-    @property
-    def dataset_hdf5_url(self):
-        if self.fname is None:
-            raise ValueError("generate_hdf5_file() was not called")
-        return DataUrl(file_path=self.fname, data_path=self.dataset_path, scheme="silx")
-
-    def _get_images_with_key(self, key):
-        indices = np.arange(self.image_key.size)[self.image_key == key]
-        urls = [
-            DataUrl(
-                file_path=self.fname,
-                data_path=self.dataset_path,
-                data_slice=slice(img_idx, img_idx + 1),
-                scheme="silx",
-            )
-            for img_idx in indices
-        ]
-        return dict(zip(indices, urls))
-
-    @property
-    def flats(self):
-        return self._get_images_with_key(1)
-
-    @property
-    def darks(self):
-        return self._get_images_with_key(2)
-
-    def get_data_slices(self, what):
-        images = getattr(self, what)
-        # we can't directly use set() on slice() object (unhashable). Use tuples
-        tuples_list = list(
-            set((du.data_slice().start, du.data_slice().stop) for du in get_compacted_dataslices(images).values())
-        )
-        slices_list = [slice(item[0], item[1]) for item in tuples_list]
-        return slices_list
-
-
 # To be improved
 def generate_nx_dataset(out_fname, image_key, data_volume=None, rotation_angle=None):
     nx_template_file = get_file("dummy.nx.tar.gz")

nabu/thirdparty/tomocupy_remove_stripe.py

@@ -4,9 +4,11 @@
 This file is a "GPU" (through cupy) implementation of "remove_all_stripe".
 The original method is implemented by Nghia Vo in the algotom project: https://github.com/algotom/algotom/blob/master/algotom/prep/removal.py
 The implementation using cupy is done by Viktor Nikitin in the tomocupy project: https://github.com/tomography/tomocupy/blame/main/src/tomocupy/remove_stripe.py
-License follows.
+then moved to https://github.com/tomography/tomocupy/blob/main/src/tomocupy/processing/remove_stripe.py
 
 For now we can't rely on off-the-shelf tomocupy as it's not packaged in pypi, and compilation is quite tedious.
+
+License follows.
 """
 
 # *************************************************************************** #
@@ -47,6 +49,7 @@ For now we can't rely on off-the-shelf tomocupy as it's not packaged in pypi, an
 # *************************************************************************** #
 
 try:
+    import pycuda.gpuarray as garray
     import cupy as cp
     import pywt
     from cupyx.scipy.ndimage import median_filter
@@ -559,28 +562,59 @@ def remove_all_stripe(tomo,
     return tomo
 
 
+def remove_all_stripe_sinos(sinos, snr=3, la_size=61, sm_size=21, dim=1):
+    """
+    Same as remove_all_stripe(), but acting on sinograms
+    """
+    n_sinos, n_a, n_x = sinos.shape
+    matindex = _create_matindex(n_x, n_a)
+    for m in range(n_sinos):
+        sino = sinos[m]
+        sino = _rs_dead(sino, snr, la_size, matindex)
+        sino = _rs_sort(sino, sm_size, matindex, dim)
+        sinos[m] = sino
+    return sinos
+
+
 from ..cuda.utils import pycuda_to_cupy
-def remove_all_stripe_pycuda(radios, device_id=0, **kwargs):
+def remove_all_stripe_pycuda(array, layout="radios", device_id=0, **kwargs):
     """
-    Nabu interface to "remove_all_stripe". In-place!
+    Nabu interface to tomocupy "remove_all_stripe".
+    Processing is done in-place to save memory, meaning that the content of "array" will be overwritten.
 
     Parameters
     ----------
-    radios: pycuda.GPUArray
-        Stack of radios in the shape (n_angles, n_y, n_x)
-        so that sinogram number i is radios[:, i, :]
+    array: pycuda.GPUArray
+        Stack of radios in the shape (n_angles, n_y, n_x), if layout == "radios"
+        Stack of sinos in the shape (n_y, n_angles, n_x), if layout == "sinos".
+
 
     Other Parameters
     ----------------
     See parameters of 'remove_all_stripe
     """
 
+    # Init cupy. Nabu currently does not use cupy, with exception of this module,
+    # so the initialization has to be done here.
     if getattr(remove_all_stripe, "_cupy_init", False) is False:
         from cupy import cuda
         cuda.Device(device_id).use()
         setattr(remove_all_stripe, "_cupy_init", True)
 
-    cupy_radios = pycuda_to_cupy(radios) # no memory copy, the internal pointer is passed to pycuda
-    remove_all_stripe(cupy_radios, **kwargs)
-    return radios
+    # remove_all_stripe() in tomocupy expects a 3D array to build the "matindex" data structure.
+    # The usage of this "matindex" array is not clear since this method is supposed to act on individual sinograms.
+    # To avoid memory duplication, we use fake 3D array, i.e, we pass a series of (1, n_a, n_x) sinograms
 
+    if layout == "radios":
+        sinos = array.transpose(axes=(1, 0, 2)) # no copy
+    else:
+        sinos = array
+    # is_contiguous = sinos.flags.c_contigious
+    n_sinos, n_a, n_x = sinos.shape
+    sinos_tmp = garray.zeros((1, n_a, n_x), dtype="f")
+    for i in range(n_sinos):
+        sinos_tmp[0] = sinos[i]
+        cupy_sinos = pycuda_to_cupy(sinos_tmp) # no memory copy, the internal pointer is passed to pycuda
+        remove_all_stripe_sinos(cupy_sinos, **kwargs)
+        sinos[i] = sinos_tmp[0]
+    return array

nabu/utils.py

@@ -1,6 +1,6 @@
+from fnmatch import fnmatch
 from functools import partial
 import os
-import sys
 from functools import partial, lru_cache
 from itertools import product
 import warnings
@@ -50,7 +50,7 @@ def convert_index(idx, idx_max, default_val):
 
 
 def get_folder_path(foldername=""):
-    _file_dir = os.path.dirname(os.path.realpath(__file__))
+    _file_dir = os.path.dirname(os.path.abspath(__file__))
     package_dir = _file_dir
     return os.path.join(package_dir, foldername)
 
@@ -74,6 +74,99 @@ def get_resource_file(filename, subfolder=None):
     return os.path.join(abspath, filename)
 
 
+def indices_to_slices(indices):
+    """
+    From a series of integer indices, return corresponding slice() objects.
+
+    Parameters
+    ----------
+    indices: collection of sorted unique integers
+        Arrays indices
+
+    Examples
+    --------
+    slices_from_indices([0, 1, 2, 3]) returns [slice(0, 4)]
+    slices_from_indices([8, 9, 10, 14, 15, 16]) returns [slice(8, 11), slice(15, 17)]
+    """
+    jumps = np.where(np.diff(indices) > 1)[0]
+    if len(jumps) == 0:
+        return [slice(indices[0], indices[-1] + 1)]
+    jumps = np.hstack([-1, jumps, len(indices) - 1])
+    slices = []
+    for i in range(len(jumps) - 1):
+        slices.append(slice(indices[jumps[i] + 1], indices[jumps[i + 1]] + 1))
+    return slices
+
+
+def merge_slices(slice1, slice2):
+    """
+    Merge two slicing operations in one.
+
+    Examples
+    --------
+    array = numpy.arange(200)
+    array[slice(133, 412, 2)][slice(31, 35, 2)] # gives [195 199]
+    array[merge_slices(slice(133, 412, 2), slice(31, 35, 2))]  # gives [195 199]
+    """
+    step1 = slice1.step or 1
+    step2 = slice2.step or 1
+    step = step1 * step2
+    if step == 1:
+        step = None
+
+    start = slice1.start + step1 * (slice2.start or 0)
+    if slice2.stop is None:
+        stop = slice1.stop
+    else:
+        stop = min(slice1.stop, slice1.start + step1 * slice2.stop)
+    return slice(start, stop, step)
+
+
+def compacted_views(slices_):
+    """
+    From a list of slice objects, returns the slice objects corresponding to a compact view.
+
+    If "array" is obtained with
+        array = np.hstack([big_array[slice1], big_array[slice2]])
+    Then, compacted_views([slice1, slice2]) returns [slice3, slice4] where
+      - slice3 are the indices, in 'array', corresponding to indices of slice1 in 'big_array'
+      - slice4 are the indices, in 'array', corresponding to indices of slice2 in 'big_array'
+
+    Example
+    -------
+    compacted_views([slice(1, 26), slice(526, 551)]) gives [slice(0, 25), slice(25, 50)]
+
+    """
+    prev_start = 0
+    r = []
+    for s in slices_:
+        start = prev_start
+        stop = start + (s.stop - s.start)
+        r.append(slice(start, stop))
+        prev_start = stop
+    return r
+
+
+def get_size_from_sliced_dimension(length, slice_):
+    """
+    From a given array size, returns the size of the array once it is accessed using a slice.
+
+    Examples
+    --------
+    If data.shape = (3500, 2160, 2560)
+    get_size_from_sliced_dimension(data.shape[0], None) returns 3500
+    get_size_from_sliced_dimension(data.shape[0], slice(100, 200)) returns 100
+    """
+    return np.arange(length)[slice_].size
+
+
+def get_shape_from_sliced_dims(shape, slices_):
+    """
+    Same as get_size_from_sliced_dimension() but in 3D
+    """
+    return tuple(get_size_from_sliced_dimension(length, slice_) for length, slice_ in zip(shape, slices_))
+
+
 def get_available_threads():
     try:
         n_threads = len(os.sched_getaffinity(0))
@@ -82,6 +175,18 @@ def get_available_threads():
     return n_threads
 
 
+def list_match_queries(available, queries):
+    """
+    Given a list of strings, return all items matching any of one elements of "queries"
+    """
+    matches = []
+    for a in available:
+        for q in queries:
+            if fnmatch(a, q):
+                matches.append(a)
+    return matches
+
+
 def is_writeable(location):
     """
     Return True if a file/location is writeable.
@@ -93,6 +198,12 @@ def is_int(num, eps=1e-7):
     return abs(num - int(num)) < eps
 
 
+def is_scalar(stuff):
+    if isinstance(stuff, str):
+        return False
+    return np.isscalar(stuff)
+
+
 def _sizeof(Type):
     """
     return the size (in bytes) of a scalar type, like the C behavior
@@ -100,6 +211,22 @@ def _sizeof(Type):
     return np.dtype(Type).itemsize
 
 
+class _Default_format(dict):
+    """
+    https://docs.python.org/3/library/stdtypes.html
+    """
+
+    def __missing__(self, key):
+        return key
+
+
+def safe_format(str_, **kwargs):
+    """
+    Alternative to str.format(), but does not throw a KeyError when fields are missing.
+    """
+    return str_.format_map(_Default_format(**kwargs))
+
+
 def get_ftype(url):
     """
     return supposed filetype of an url
@@ -294,6 +421,11 @@ def partition_dict(dict_, n_partitions):
     return res
 
 
+def first_dict_item(dict_):
+    keys = sorted(list(dict_.keys()))
+    return dict_[keys[0]]
+
+
 def subsample_dict(dic, subsampling_factor):
     """
     Subsample a dict where keys are integers.
@@ -559,64 +691,6 @@ def convert_str_to_tuple(input_str: str, none_if_empty: bool = False):
         return tuple(elmts)
 
 
-class Progress:
-    """Simple interface for defining advancement on a 100 percentage base"""
-
-    def __init__(self, name: str):
-        self._name = name
-        self.set_name(name)
-
-    def set_name(self, name):
-        self._name = name
-        self.reset()
-
-    def reset(self, max_=None):
-        """
-        reset the advancement to n and max advancement to max_
-        :param int max_:
-        """
-        self._n_processed = 0
-        self._max_processed = max_
-
-    def start_process(self) -> None:
-        self.set_advancement(0)
-
-    def set_advancement(self, value: int) -> None:
-        """
-
-        :param int value: set advancement to value
-        """
-        length = 20  # modify this to change the length
-        block = int(round(length * value / 100))
-        blocks_str = "#" * block + "-" * (length - block)
-        msg = "\r{0}: [{1}] {2}%".format(self._name, blocks_str, round(value, 2))
-        if value >= 100:
-            msg += " DONE\r\n"
-        sys.stdout.write(msg)
-        sys.stdout.flush()
-
-    def end_process(self) -> None:
-        """Set advancement to 100 %"""
-        self.set_advancement(100)
-
-    def set_max_advancement(self, n: int) -> None:
-        """
-
-        :param int n: number of steps contained by the advancement. When
-        advancement reach this value, advancement will be 100 %
-        """
-        self._max_processed = n
-
-    def increase_advancement(self, i: int = 1) -> None:
-        """
-
-        :param int i: increase the advancement of n step
-        """
-        self._n_processed += i
-        advancement = int(float(self._n_processed / self._max_processed) * 100)
-        self.set_advancement(advancement)
-
-
 def concatenate_dict(dict_1, dict_2) -> dict:
     """update dict which has dict as values. And we want concatenate those values to"""
     res = dict_1.copy()
@@ -658,12 +732,14 @@ def generate_coords(img_shp, center=None):
     return R, C
 
 
-def clip_circle(img, center=None, radius=None):
+def clip_circle(img, center=None, radius=None, out_value=0):
     R, C = generate_coords(img.shape, center)
     if radius is None:
         radius = R.shape[-1] // 2
     M = R**2 + C**2
     res = np.zeros_like(img)
+    if out_value != 0:
+        res.fill(out_value)
     res[M < radius**2] = img[M < radius**2]
     return res
 
@@ -821,3 +897,24 @@ def docstring(origin):
         If the origin class has not method n case the
     """
     return partial(_docstring, origin=origin)
+
+
+from warnings import catch_warnings
+
+# FIX for python < 3.11
+# catch_warnings() does not have "action=XX" kwarg for python < 3.11
+from sys import version_info
+
+if version_info.major == 3 and version_info.minor < 11:
+
+    def dummy(*args, **kwargs):
+        pass
+
+    catch_warnings_old = catch_warnings
+
+    def catch_warnings(*args, **kwargs):  # pylint: disable=E0102
+        action = kwargs.pop("action", None)
+        return catch_warnings_old(record=(dummy if action == "ignore" else False))
+
+
+# ---

{nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nabu
-Version: 2024.1.9
+Version: 2024.2.0
 Summary: Nabu - Tomography software
 Author-email: Pierre Paleo <pierre.paleo@esrf.fr>, Henri Payno <henri.payno@esrf.fr>, Alessandro Mirone <mirone@esrf.fr>, Jérôme Lesaint <jerome.lesaint@esrf.fr>
 Maintainer-email: Pierre Paleo <pierre.paleo@esrf.fr>
@@ -53,10 +53,11 @@ Requires-Dist: numpy <2,>1.9.0
 Requires-Dist: scipy
 Requires-Dist: h5py >=3.0
 Requires-Dist: silx >=0.15.0
-Requires-Dist: tomoscan >=2.0.4
+Requires-Dist: tomoscan >=2.1.0
 Requires-Dist: psutil
 Requires-Dist: pytest
 Requires-Dist: tifffile
+Requires-Dist: tqdm
 Provides-Extra: doc
 Requires-Dist: sphinx ; extra == 'doc'
 Requires-Dist: cloud-sptheme ; extra == 'doc'
@@ -66,11 +67,17 @@ Provides-Extra: full
 Requires-Dist: scikit-image ; extra == 'full'
 Requires-Dist: PyWavelets ; extra == 'full'
 Requires-Dist: glymur ; extra == 'full'
-Requires-Dist: pycuda <2024.1.1 ; extra == 'full'
+Requires-Dist: pycuda !=2024.1.1 ; extra == 'full'
 Requires-Dist: scikit-cuda ; extra == 'full'
 Requires-Dist: pycudwt ; extra == 'full'
 Requires-Dist: sluurp >=0.3 ; extra == 'full'
 Requires-Dist: pyvkfft ; extra == 'full'
+Provides-Extra: full_nocuda
+Requires-Dist: scikit-image ; extra == 'full_nocuda'
+Requires-Dist: PyWavelets ; extra == 'full_nocuda'
+Requires-Dist: glymur ; extra == 'full_nocuda'
+Requires-Dist: sluurp >=0.3 ; extra == 'full_nocuda'
+Requires-Dist: pyvkfft ; extra == 'full_nocuda'
 
 # Nabu