nabu 2024.1.0rc3__py3-none-any.whl → 2024.1.1__py3-none-any.whl

build/lib/build/lib/build/lib/build/lib/build/lib/nabu/io/reader_helical.py

@@ -0,0 +1,114 @@
+from .reader import *
+
+
+class ChunkReaderHelical(ChunkReader):
+    """implements reading by projections subsets"""
+
+    def _set_subregion(self, sub_region):
+        super()._set_subregion(sub_region)
+        ###########
+        # undo the chun_size setting of the base class
+        # to avoid allocation of Tera bytes in the helical case
+        self.chunk_shape = (1,) + self.shape
+
+    def set_data_buffer(self, data_buffer, pre_allocate=False):
+        if data_buffer is not None:
+            # overwrite out_dtype
+            self.files_data = data_buffer
+            self.out_dtype = data_buffer.dtype
+            if data_buffer.shape[1:] != self.shape:
+                raise ValueError("Expected shape %s but got %s" % (self.shape, data_buffer.shape))
+        if pre_allocate:
+            self.files_data = np.zeros(self.chunk_shape, dtype=self.out_dtype)
+
+        if (self.binning is not None) and (np.dtype(self.out_dtype).kind in ["u", "i"]):
+            raise ValueError(
+                "Output datatype cannot be integer when using binning. Please set the 'convert_float' parameter to True or specify a 'data_buffer'."
+            )
+
+    def get_binning(self):
+        if self.binning is None:
+            return 1, 1
+        else:
+            return self.binning
+
+    def _load_single(self, sub_total_prange_slice=slice(None, None)):
+        if sub_total_prange_slice == slice(None, None):
+            sorted_files_indices = self._sorted_files_indices
+        else:
+            sorted_files_indices = self._sorted_files_indices[sub_total_prange_slice]
+
+        for i, fileidx in enumerate(sorted_files_indices):
+            file_url = self.files[fileidx]
+            self.files_data[i] = self.get_data(file_url)
+            self._fileindex_to_idx[fileidx] = i
+
+    def _apply_subsample_fact(self, t):
+        if t is not None:
+            t = t * self.dataset_subsampling
+        return t
+
+    def _load_multi(self, sub_total_prange_slice=slice(None, None)):
+        if sub_total_prange_slice == slice(None, None):
+            files_to_be_compacted_dict = self.files
+            sorted_files_indices = self._sorted_files_indices
+        else:
+            if self.dataset_subsampling > 1:
+                start, stop, step = list(
+                    map(
+                        self._apply_subsample_fact,
+                        [sub_total_prange_slice.start, sub_total_prange_slice.stop, sub_total_prange_slice.step],
+                    )
+                )
+                sub_total_prange_slice = slice(start, stop, step)
+
+            sorted_files_indices = self._sorted_files_indices[sub_total_prange_slice]
+            files_to_be_compacted_dict = dict(
+                zip(sorted_files_indices, [self.files[idx] for idx in sorted_files_indices])
+            )
+
+        urls_compacted = get_compacted_dataslices(files_to_be_compacted_dict, subsampling=self.dataset_subsampling)
+        loaded = {}
+        start_idx = 0
+        for idx in sorted_files_indices:
+            url = urls_compacted[idx]
+            url_str = str(url)
+            is_loaded = loaded.get(url_str, False)
+            if is_loaded:
+                continue
+            ds = url.data_slice()
+            delta_z = ds.stop - ds.start
+            if ds.step is not None and ds.step > 1:
+                delta_z //= ds.step
+            end_idx = start_idx + delta_z
+            self.files_data[start_idx:end_idx] = self.get_data(url)
+            start_idx += delta_z
+            loaded[url_str] = True
+
+    def load_files(self, overwrite: bool = False, sub_total_prange_slice=slice(None, None)):
+        """
+        Load the files whose links was provided at class instantiation.
+
+        Parameters
+        -----------
+        overwrite: bool, optional
+            Whether to force reloading the files if already loaded.
+        """
+
+        if self._loaded and not (overwrite):
+            raise ValueError("Radios were already loaded. Call load_files(overwrite=True) to force reloading")
+        if self.file_reader.multi_load:
+            self._load_multi(sub_total_prange_slice)
+        else:
+            if self.dataset_subsampling > 1:
+                assert (
+                    False
+                ), " in helica pipeline, load file _load_single has not yet been adapted to angular subsampling "
+            self._load_single(sub_total_prange_slice)
+        self._loaded = True
+
+    load_data = load_files
+
+    @property
+    def data(self):
+        return self.files_data

build/lib/build/lib/build/lib/build/lib/build/lib/nabu/io/tests/test_cast_volume.py

@@ -0,0 +1,339 @@
+import numpy
+from nabu.io.cast_volume import (
+    cast_volume,
+    clamp_and_rescale_data,
+    find_histogram,
+    get_default_output_volume,
+)
+from tomoscan.esrf.volume import (
+    EDFVolume,
+    HDF5Volume,
+    JP2KVolume,
+    MultiTIFFVolume,
+    TIFFVolume,
+)
+from nabu.io.writer import __have_jp2k__
+from tomoscan.esrf.scan.edfscan import EDFTomoScan
+from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
+import pytest
+import h5py
+import os
+from silx.io.url import DataUrl
+
+
+@pytest.mark.skipif(not __have_jp2k__, reason="need jp2k (glymur) for this test")
+def test_get_default_output_volume():
+    """
+    insure nabu.io.cast_volume is working properly
+    """
+    with pytest.raises(TypeError):
+        # test input_volume type
+        get_default_output_volume(input_volume="dsad/dsad/", output_type="jp2")
+
+    with pytest.raises(ValueError):
+        # test output value
+        get_default_output_volume(input_volume=EDFVolume(folder="test"), output_type="toto")
+
+    # test edf to jp2
+    input_volume = EDFVolume(
+        folder="/path/to/my_folder",
+    )
+    output_volume = get_default_output_volume(
+        input_volume=input_volume,
+        output_type="jp2",
+    )
+    assert isinstance(output_volume, JP2KVolume)
+    assert output_volume.data_url.file_path() == "/path/to/vol_cast"
+    assert output_volume.get_volume_basename() == "my_folder"
+
+    # test hdf5 to tiff
+    input_volume = HDF5Volume(
+        file_path="/path/to/my_file.hdf5",
+        data_path="entry0012",
+    )
+
+    output_volume = get_default_output_volume(
+        input_volume=input_volume,
+        output_type="tiff",
+    )
+    assert isinstance(output_volume, TIFFVolume)
+    assert output_volume.data_url.file_path() == "/path/to/vol_cast/my_file"
+    assert output_volume.get_volume_basename() == "my_file"
+
+    # test Multitiff to hdf5
+    input_volume = MultiTIFFVolume(
+        file_path="my_file.tiff",
+    )
+    output_volume = get_default_output_volume(
+        input_volume=input_volume,
+        output_type="hdf5",
+    )
+    assert isinstance(output_volume, HDF5Volume)
+    assert output_volume.data_url.file_path() == "vol_cast/my_file.hdf5"
+    assert output_volume.data_url.data_path() == HDF5Volume.DATA_DATASET_NAME
+    assert output_volume.metadata_url.file_path() == "vol_cast/my_file.hdf5"
+    assert output_volume.metadata_url.data_path() == HDF5Volume.METADATA_GROUP_NAME
+
+    # test jp2 to hdf5
+    input_volume = JP2KVolume(
+        folder="folder",
+        volume_basename="basename",
+    )
+    output_volume = get_default_output_volume(
+        input_volume=input_volume,
+        output_type="hdf5",
+    )
+    assert isinstance(output_volume, HDF5Volume)
+    assert output_volume.data_url.file_path() == "folder/vol_cast/basename.hdf5"
+    assert output_volume.data_url.data_path() == f"/volume/{HDF5Volume.DATA_DATASET_NAME}"
+    assert output_volume.metadata_url.file_path() == "folder/vol_cast/basename.hdf5"
+    assert output_volume.metadata_url.data_path() == f"/volume/{HDF5Volume.METADATA_GROUP_NAME}"
+
+
+def test_find_histogram_hdf5_volume(tmp_path):
+    """
+    test find_histogram function with hdf5 volume
+    """
+    h5_file = os.path.join(tmp_path, "test_file")
+    with h5py.File(h5_file, mode="w") as h5f:
+        h5f.require_group("myentry/histogram/results/data")
+
+    # if volume url provided then can find it
+    assert find_histogram(volume=HDF5Volume(file_path=h5_file, data_path="myentry")) == DataUrl(
+        file_path=h5_file,
+        data_path="myentry/histogram/results/data",
+        scheme="silx",
+    )
+
+    assert find_histogram(volume=HDF5Volume(file_path=h5_file, data_path="entry")) == None
+
+
+def test_find_histogram_single_frame_volume(tmp_path):
+    """
+    test find_histogram function with single frame volume
+
+    TODO: improve: for now histogram file are created manually. If this can be more coupled with the "real" histogram generation it would be way better
+    """
+    # create volume and histogram
+    volume = EDFVolume(
+        folder=tmp_path,
+        volume_basename="volume",
+    )
+    histogram_file = os.path.join(tmp_path, "volumehistogram.hdf5")
+    with h5py.File(histogram_file, mode="w") as h5f:
+        h5f.require_group("entry/histogram/results/data")
+
+    # check behavior
+    assert find_histogram(volume=volume) == None
+
+    assert find_histogram(
+        volume=volume,
+        scan=EDFTomoScan(scan=str(tmp_path)),
+    ) == DataUrl(
+        file_path=histogram_file,
+        data_path="entry",
+        scheme="silx",
+    )
+
+    assert find_histogram(
+        volume=volume,
+        scan=NXtomoScan(scan=str(tmp_path), entry="entry"),
+    ) == DataUrl(
+        file_path=histogram_file,
+        data_path="entry",
+        scheme="silx",
+    )
+
+
+def test_find_histogram_multi_tiff_volume(tmp_path):
+    """
+    test find_histogram function with multi tiff frame volume
+
+    TODO: improve: for now histogram file are created manually. If this can be more coupled with the "real" histogram generation it would be way better
+    """
+    # create volume and histogram
+    tiff_file = os.path.join(tmp_path, "my_tiff.tif")
+    volume = MultiTIFFVolume(
+        file_path=tiff_file,
+    )
+    histogram_file = os.path.join(tmp_path, "my_tiffhistogram.hdf5")
+    with h5py.File(histogram_file, mode="w") as h5f:
+        h5f.require_group("entry/histogram/results/data")
+
+    # check behavior
+    assert find_histogram(volume=volume) == None
+
+    assert find_histogram(
+        volume=volume,
+        scan=EDFTomoScan(scan=str(tmp_path)),
+    ) == DataUrl(
+        file_path=histogram_file,
+        data_path="entry",
+        scheme="silx",
+    )
+
+    assert find_histogram(
+        volume=volume,
+        scan=NXtomoScan(scan=str(tmp_path), entry="entry"),
+    ) == DataUrl(
+        file_path=histogram_file,
+        data_path="entry",
+        scheme="silx",
+    )
+
+
+@pytest.mark.parametrize("input_dtype", (numpy.float32, numpy.float64, numpy.uint8, numpy.uint16))
+def test_clamp_and_rescale_data(input_dtype):
+    """
+    test 'rescale_data' function
+    """
+    array = numpy.linspace(
+        start=1,
+        stop=100,
+        num=100,
+        endpoint=True,
+        dtype=input_dtype,
+    ).reshape(10, 10)
+
+    rescaled_array = clamp_and_rescale_data(
+        data=array,
+        new_min=10,
+        new_max=90,
+        rescale_min_percentile=20,  # provided to insure they will be ignored
+        rescale_max_percentile=80,  # provided to insure they will be ignored
+    )
+    assert rescaled_array.min() == 10
+    assert rescaled_array.max() == 90
+    numpy.testing.assert_equal(rescaled_array.flatten()[0:10], numpy.array([10] * 10))
+    numpy.testing.assert_equal(rescaled_array.flatten()[90:100], numpy.array([90] * 10))
+
+
+def test_cast_volume(tmp_path):
+    """
+    test cast_volume
+    """
+    raw_data = numpy.linspace(
+        start=1,
+        stop=100,
+        num=100,
+        endpoint=True,
+        dtype=numpy.float64,
+    ).reshape(1, 10, 10)
+    volume_hdf5_file_path = os.path.join(tmp_path, "myvolume.hdf5")
+    volume_hdf5 = HDF5Volume(
+        file_path=volume_hdf5_file_path,
+        data_path="myentry",
+        data=raw_data,
+    )
+    volume_edf = EDFVolume(
+        folder=os.path.join(tmp_path, "volume_folder"),
+    )
+
+    # test when no histogram existing
+    cast_volume(
+        input_volume=volume_hdf5,
+        output_volume=volume_edf,
+        output_data_type=numpy.dtype(numpy.uint16),
+        rescale_min_percentile=10,
+        rescale_max_percentile=90,
+        save=True,
+        store=True,
+    )
+    # if percentiles 10 and 90 provided, no data_min and data_max then they will be computed from data min / max
+
+    # append histogram
+    with h5py.File(volume_hdf5_file_path, mode="a") as h5s:
+        hist = numpy.array([20, 20, 20, 20, 20, 20])
+        bins = numpy.array([0, 20, 40, 60, 80, 100])
+        h5s["myentry/histogram/results/data"] = numpy.vstack((hist, bins))
+
+    # and test it again
+    volume_edf.overwrite = True
+    cast_volume(
+        input_volume=volume_hdf5,
+        output_volume=volume_edf,
+        output_data_type=numpy.dtype(numpy.uint16),
+        rescale_min_percentile=20,
+        rescale_max_percentile=60,
+        save=True,
+        store=True,
+    )
+
+    # test to cast the already cast volumes
+    volume_tif = EDFVolume(
+        folder=os.path.join(tmp_path, "second_volume_folder"),
+    )
+    volume_tif.overwrite = True
+    cast_volume(
+        input_volume=volume_edf,
+        output_volume=volume_tif,
+        output_data_type=numpy.dtype(numpy.uint8),
+        save=True,
+        store=True,
+    )
+    assert volume_tif.data.dtype == numpy.uint8
+
+    volume_tif.overwrite = False
+    with pytest.raises(OSError):
+        cast_volume(
+            input_volume=volume_edf,
+            output_volume=volume_tif,
+            output_data_type=numpy.dtype(numpy.uint8),
+            save=True,
+            store=True,
+        )
+
+
+@pytest.mark.skipif(not __have_jp2k__, reason="need jp2k (glymur) for this test")
+def test_jp2k_compression_ratios(tmp_path):
+    """
+    simple test to make sure the compression ratios are handled
+    """
+    import glymur
+
+    raw_data = numpy.random.random(
+        size=(1, 2048, 2048),
+    )
+    raw_data *= 2048.000005
+    volume_hdf5_file_path = os.path.join(tmp_path, "myvolume.hdf5")
+    volume_hdf5 = HDF5Volume(
+        file_path=volume_hdf5_file_path,
+        data_path="myentry",
+        data=raw_data,
+    )
+
+    volume_jp2k_ratios_0 = JP2KVolume(
+        folder=os.path.join(tmp_path, "volume_folder"),
+        cratios=(100, 10),
+    )
+
+    volume_jp2k_ratios_1 = JP2KVolume(
+        folder=os.path.join(tmp_path, "volume_folder_2"),
+        cratios=(1000, 100),
+    )
+
+    # test when no histogram existing
+    cast_volume(
+        input_volume=volume_hdf5,
+        output_volume=volume_jp2k_ratios_0,
+        output_data_type=numpy.dtype(numpy.uint16),
+        save=True,
+        store=True,
+    )
+
+    cast_volume(
+        input_volume=volume_hdf5,
+        output_volume=volume_jp2k_ratios_1,
+        output_data_type=numpy.dtype(numpy.uint16),
+        save=True,
+        store=True,
+    )
+
+    # make sure the ratio have been taking into account
+    frame_0 = glymur.Jp2k(next(volume_jp2k_ratios_0.browse_data_files()))
+    frame_0.layer = 0
+
+    frame_1 = glymur.Jp2k(next(volume_jp2k_ratios_1.browse_data_files()))
+    frame_1.layer = 0
+
+    assert not numpy.array_equal(frame_0, frame_1)

build/lib/build/lib/build/lib/build/lib/build/lib/nabu/io/tests/test_detector_distortion.py

@@ -0,0 +1,178 @@
+import pytest
+import numpy as np
+import scipy.ndimage
+from scipy import sparse
+from nabu.io.detector_distortion import DetectorDistortionBase
+from nabu.processing.rotation import Rotation, __have__skimage__
+
+if __have__skimage__:
+    import skimage
+
+
+@pytest.mark.skipif(not (__have__skimage__), reason="Need scikit-image for rotation")
+def test_detector_distortion():
+    image = scipy.ndimage.gaussian_filter(np.random.random([379, 1357]), 3.0)
+    center_xz = ((image.shape[1] - 1) / 2, (image.shape[0] - 1) / 2)
+
+    part_to_be_retrieved = image[100:279]
+
+    rotated_image = skimage.transform.rotate(image, angle=5.0, center=center_xz[::])
+
+    corrector = DetectorDistortionRotation(detector_full_shape_vh=image.shape, center_xz=center_xz, angle_deg=-5.0)
+    start_x, end_x, start_z, end_z = corrector.set_sub_region_transformation(
+        target_sub_region=(
+            None,
+            None,
+            100,
+            279,
+        )
+    )
+
+    source = rotated_image[start_z:end_z, start_x:end_x]
+
+    retrieved = corrector.transform(source)
+
+    diff = (retrieved - part_to_be_retrieved)[:, 20:-20]
+
+    assert abs(diff).std() < 1e-3
+
+
+class DetectorDistortionRotation(DetectorDistortionBase):
+    """ """
+
+    def __init__(self, detector_full_shape_vh=(0, 0), center_xz=(0, 0), angle_deg=0.0):
+        """This is the basis class.
+        A simple identity transformation which has the only merit to show
+        how it works.Reimplement this function to have more parameters for other
+        transformations
+        """
+        self._build_full_transformation(detector_full_shape_vh, center_xz, angle_deg)
+
+    def _build_full_transformation(self, detector_full_shape_vh, center_xz, angle_deg):
+        """A simple identity.
+        Reimplement this function to have more parameters for other
+        transformations
+        """
+
+        indices = np.indices(detector_full_shape_vh)
+        center_x, center_z = center_xz
+        coordinates = (indices.T - np.array([center_z, center_x])).T
+
+        c = np.cos(np.deg2rad(angle_deg))
+        s = np.sin(np.deg2rad(angle_deg))
+
+        rot_mat = np.array([[c, s], [-s, c]])
+
+        coordinates = np.tensordot(rot_mat, coordinates, axes=[1, 0])
+
+        # padding
+        sz, sx = detector_full_shape_vh
+        total_detector_npixs = sz * sx
+        xs = np.clip(np.array(coordinates[1].flat) + center_x, [[0]], [[sx - 1]])
+        zs = np.clip(np.array(coordinates[0].flat) + center_z, [[0]], [[sz - 1]])
+
+        ix0s = np.floor(xs)
+        ix1s = np.ceil(xs)
+        fx = xs - ix0s
+
+        iz0s = np.floor(zs)
+        iz1s = np.ceil(zs)
+        fz = zs - iz0s
+
+        I_tmp = np.empty([4 * sz * sx], np.int64)
+        J_tmp = np.empty([4 * sz * sx], np.int64)
+        V_tmp = np.ones([4 * sz * sx], "f")
+
+        I_tmp[:] = np.arange(sz * sx * 4) // 4
+
+        J_tmp[0::4] = iz0s * sx + ix0s
+        J_tmp[1::4] = iz0s * sx + ix1s
+        J_tmp[2::4] = iz1s * sx + ix0s
+        J_tmp[3::4] = iz1s * sx + ix1s
+
+        V_tmp[0::4] = (1 - fz) * (1 - fx)
+        V_tmp[1::4] = (1 - fz) * fx
+        V_tmp[2::4] = fz * (1 - fx)
+        V_tmp[3::4] = fz * fx
+
+        self.detector_full_shape_vh = detector_full_shape_vh
+
+        coo_tmp = sparse.coo_matrix((V_tmp.astype("f"), (I_tmp, J_tmp)), shape=(sz * sx, sz * sx))
+
+        csr_tmp = coo_tmp.tocsr()
+
+        self.full_csr_data = csr_tmp.data
+        self.full_csr_indices = csr_tmp.indices
+        self.full_csr_indptr = csr_tmp.indptr
+
+        ## This will be used to save time if the same sub_region argument is requested several time in a row
+        self._status = None
+
+    def _set_sub_region_transformation(
+        self,
+        target_sub_region=(
+            (
+                None,
+                None,
+                0,
+                0,
+            ),
+        ),
+    ):
+        (x_start, x_end, z_start, z_end) = target_sub_region
+
+        if z_start is None:
+            z_start = 0
+        if z_end is None:
+            z_end = self.detector_full_shape_vh[0]
+
+        if (x_start, x_end) not in [(None, None), (0, None), (0, self.detector_full_shape_vh[1])]:
+            message = f""" In the base class DetectorDistortionRotation only vertical slicing is accepted.
+            The sub_region contained (x_start, x_end)={(x_start, x_end)} which would slice the 
+            full horizontal size which is {self.detector_full_shape_vh[1]}
+            """
+            raise ValueError()
+
+        x_start, x_end = 0, self.detector_full_shape_vh[1]
+
+        row_ptr_start = z_start * self.detector_full_shape_vh[1]
+        row_ptr_end = z_end * self.detector_full_shape_vh[1]
+
+        indices_start = self.full_csr_indptr[row_ptr_start]
+        indices_end = self.full_csr_indptr[row_ptr_end]
+
+        data_tmp = self.full_csr_data[indices_start:indices_end]
+
+        target_offset = self.full_csr_indptr[row_ptr_start]
+        indptr_tmp = self.full_csr_indptr[row_ptr_start : row_ptr_end + 1] - target_offset
+
+        indices_tmp = self.full_csr_indices[indices_start:indices_end]
+
+        iz_source = (indices_tmp) // self.detector_full_shape_vh[1]
+
+        z_start_source = iz_source.min()
+        z_end_source = iz_source.max() + 1
+        source_offset = z_start_source * self.detector_full_shape_vh[1]
+        indices_tmp = indices_tmp - source_offset
+
+        target_size = (z_end - z_start) * self.detector_full_shape_vh[1]
+        source_size = (z_end_source - z_start_source) * self.detector_full_shape_vh[1]
+
+        self.transformation_matrix = sparse.csr_matrix(
+            (data_tmp, indices_tmp, indptr_tmp), shape=(target_size, source_size)
+        )
+
+        self.target_shape = ((z_end - z_start), self.detector_full_shape_vh[1])
+
+        ## For the identity matrix the source and the target have the same size.
+        ## The two following lines are trivial.
+        ## For this identity transformation only the slicing of the appropriate part
+        ## of the identity sparse matrix is slightly laborious.
+        ## Practical case will be more complicated and source_sub_region
+        ## will be in general larger than the target_sub_region
+        self._status = {
+            "target_sub_region": ((x_start, x_end, z_start, z_end)),
+            "source_sub_region": ((x_start, x_end, z_start_source, z_end_source)),
+        }
+
+        return self._status["source_sub_region"]