nabu 2024.1.1__py3-none-any.whl → 2024.1.2__py3-none-any.whl

build/lib/nabu/io/reader_helical.py

@@ -1,114 +0,0 @@
-from .reader import *
-
-
-class ChunkReaderHelical(ChunkReader):
-    """implements reading by projections subsets"""
-
-    def _set_subregion(self, sub_region):
-        super()._set_subregion(sub_region)
-        ###########
-        # undo the chun_size setting of the base class
-        # to avoid allocation of Tera bytes in the helical case
-        self.chunk_shape = (1,) + self.shape
-
-    def set_data_buffer(self, data_buffer, pre_allocate=False):
-        if data_buffer is not None:
-            # overwrite out_dtype
-            self.files_data = data_buffer
-            self.out_dtype = data_buffer.dtype
-            if data_buffer.shape[1:] != self.shape:
-                raise ValueError("Expected shape %s but got %s" % (self.shape, data_buffer.shape))
-        if pre_allocate:
-            self.files_data = np.zeros(self.chunk_shape, dtype=self.out_dtype)
-
-        if (self.binning is not None) and (np.dtype(self.out_dtype).kind in ["u", "i"]):
-            raise ValueError(
-                "Output datatype cannot be integer when using binning. Please set the 'convert_float' parameter to True or specify a 'data_buffer'."
-            )
-
-    def get_binning(self):
-        if self.binning is None:
-            return 1, 1
-        else:
-            return self.binning
-
-    def _load_single(self, sub_total_prange_slice=slice(None, None)):
-        if sub_total_prange_slice == slice(None, None):
-            sorted_files_indices = self._sorted_files_indices
-        else:
-            sorted_files_indices = self._sorted_files_indices[sub_total_prange_slice]
-
-        for i, fileidx in enumerate(sorted_files_indices):
-            file_url = self.files[fileidx]
-            self.files_data[i] = self.get_data(file_url)
-            self._fileindex_to_idx[fileidx] = i
-
-    def _apply_subsample_fact(self, t):
-        if t is not None:
-            t = t * self.dataset_subsampling
-        return t
-
-    def _load_multi(self, sub_total_prange_slice=slice(None, None)):
-        if sub_total_prange_slice == slice(None, None):
-            files_to_be_compacted_dict = self.files
-            sorted_files_indices = self._sorted_files_indices
-        else:
-            if self.dataset_subsampling > 1:
-                start, stop, step = list(
-                    map(
-                        self._apply_subsample_fact,
-                        [sub_total_prange_slice.start, sub_total_prange_slice.stop, sub_total_prange_slice.step],
-                    )
-                )
-                sub_total_prange_slice = slice(start, stop, step)
-
-            sorted_files_indices = self._sorted_files_indices[sub_total_prange_slice]
-            files_to_be_compacted_dict = dict(
-                zip(sorted_files_indices, [self.files[idx] for idx in sorted_files_indices])
-            )
-
-        urls_compacted = get_compacted_dataslices(files_to_be_compacted_dict, subsampling=self.dataset_subsampling)
-        loaded = {}
-        start_idx = 0
-        for idx in sorted_files_indices:
-            url = urls_compacted[idx]
-            url_str = str(url)
-            is_loaded = loaded.get(url_str, False)
-            if is_loaded:
-                continue
-            ds = url.data_slice()
-            delta_z = ds.stop - ds.start
-            if ds.step is not None and ds.step > 1:
-                delta_z //= ds.step
-            end_idx = start_idx + delta_z
-            self.files_data[start_idx:end_idx] = self.get_data(url)
-            start_idx += delta_z
-            loaded[url_str] = True
-
-    def load_files(self, overwrite: bool = False, sub_total_prange_slice=slice(None, None)):
-        """
-        Load the files whose links was provided at class instantiation.
-
-        Parameters
-        -----------
-        overwrite: bool, optional
-            Whether to force reloading the files if already loaded.
-        """
-
-        if self._loaded and not (overwrite):
-            raise ValueError("Radios were already loaded. Call load_files(overwrite=True) to force reloading")
-        if self.file_reader.multi_load:
-            self._load_multi(sub_total_prange_slice)
-        else:
-            if self.dataset_subsampling > 1:
-                assert (
-                    False
-                ), " in helica pipeline, load file _load_single has not yet been adapted to angular subsampling "
-            self._load_single(sub_total_prange_slice)
-        self._loaded = True
-
-    load_data = load_files
-
-    @property
-    def data(self):
-        return self.files_data

build/lib/nabu/io/tests/test_cast_volume.py

@@ -1,339 +0,0 @@
-import numpy
-from nabu.io.cast_volume import (
-    cast_volume,
-    clamp_and_rescale_data,
-    find_histogram,
-    get_default_output_volume,
-)
-from tomoscan.esrf.volume import (
-    EDFVolume,
-    HDF5Volume,
-    JP2KVolume,
-    MultiTIFFVolume,
-    TIFFVolume,
-)
-from nabu.io.writer import __have_jp2k__
-from tomoscan.esrf.scan.edfscan import EDFTomoScan
-from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
-import pytest
-import h5py
-import os
-from silx.io.url import DataUrl
-
-
-@pytest.mark.skipif(not __have_jp2k__, reason="need jp2k (glymur) for this test")
-def test_get_default_output_volume():
-    """
-    insure nabu.io.cast_volume is working properly
-    """
-    with pytest.raises(TypeError):
-        # test input_volume type
-        get_default_output_volume(input_volume="dsad/dsad/", output_type="jp2")
-
-    with pytest.raises(ValueError):
-        # test output value
-        get_default_output_volume(input_volume=EDFVolume(folder="test"), output_type="toto")
-
-    # test edf to jp2
-    input_volume = EDFVolume(
-        folder="/path/to/my_folder",
-    )
-    output_volume = get_default_output_volume(
-        input_volume=input_volume,
-        output_type="jp2",
-    )
-    assert isinstance(output_volume, JP2KVolume)
-    assert output_volume.data_url.file_path() == "/path/to/vol_cast"
-    assert output_volume.get_volume_basename() == "my_folder"
-
-    # test hdf5 to tiff
-    input_volume = HDF5Volume(
-        file_path="/path/to/my_file.hdf5",
-        data_path="entry0012",
-    )
-
-    output_volume = get_default_output_volume(
-        input_volume=input_volume,
-        output_type="tiff",
-    )
-    assert isinstance(output_volume, TIFFVolume)
-    assert output_volume.data_url.file_path() == "/path/to/vol_cast/my_file"
-    assert output_volume.get_volume_basename() == "my_file"
-
-    # test Multitiff to hdf5
-    input_volume = MultiTIFFVolume(
-        file_path="my_file.tiff",
-    )
-    output_volume = get_default_output_volume(
-        input_volume=input_volume,
-        output_type="hdf5",
-    )
-    assert isinstance(output_volume, HDF5Volume)
-    assert output_volume.data_url.file_path() == "vol_cast/my_file.hdf5"
-    assert output_volume.data_url.data_path() == HDF5Volume.DATA_DATASET_NAME
-    assert output_volume.metadata_url.file_path() == "vol_cast/my_file.hdf5"
-    assert output_volume.metadata_url.data_path() == HDF5Volume.METADATA_GROUP_NAME
-
-    # test jp2 to hdf5
-    input_volume = JP2KVolume(
-        folder="folder",
-        volume_basename="basename",
-    )
-    output_volume = get_default_output_volume(
-        input_volume=input_volume,
-        output_type="hdf5",
-    )
-    assert isinstance(output_volume, HDF5Volume)
-    assert output_volume.data_url.file_path() == "folder/vol_cast/basename.hdf5"
-    assert output_volume.data_url.data_path() == f"/volume/{HDF5Volume.DATA_DATASET_NAME}"
-    assert output_volume.metadata_url.file_path() == "folder/vol_cast/basename.hdf5"
-    assert output_volume.metadata_url.data_path() == f"/volume/{HDF5Volume.METADATA_GROUP_NAME}"
-
-
-def test_find_histogram_hdf5_volume(tmp_path):
-    """
-    test find_histogram function with hdf5 volume
-    """
-    h5_file = os.path.join(tmp_path, "test_file")
-    with h5py.File(h5_file, mode="w") as h5f:
-        h5f.require_group("myentry/histogram/results/data")
-
-    # if volume url provided then can find it
-    assert find_histogram(volume=HDF5Volume(file_path=h5_file, data_path="myentry")) == DataUrl(
-        file_path=h5_file,
-        data_path="myentry/histogram/results/data",
-        scheme="silx",
-    )
-
-    assert find_histogram(volume=HDF5Volume(file_path=h5_file, data_path="entry")) == None
-
-
-def test_find_histogram_single_frame_volume(tmp_path):
-    """
-    test find_histogram function with single frame volume
-
-    TODO: improve: for now histogram file are created manually. If this can be more coupled with the "real" histogram generation it would be way better
-    """
-    # create volume and histogram
-    volume = EDFVolume(
-        folder=tmp_path,
-        volume_basename="volume",
-    )
-    histogram_file = os.path.join(tmp_path, "volumehistogram.hdf5")
-    with h5py.File(histogram_file, mode="w") as h5f:
-        h5f.require_group("entry/histogram/results/data")
-
-    # check behavior
-    assert find_histogram(volume=volume) == None
-
-    assert find_histogram(
-        volume=volume,
-        scan=EDFTomoScan(scan=str(tmp_path)),
-    ) == DataUrl(
-        file_path=histogram_file,
-        data_path="entry",
-        scheme="silx",
-    )
-
-    assert find_histogram(
-        volume=volume,
-        scan=NXtomoScan(scan=str(tmp_path), entry="entry"),
-    ) == DataUrl(
-        file_path=histogram_file,
-        data_path="entry",
-        scheme="silx",
-    )
-
-
-def test_find_histogram_multi_tiff_volume(tmp_path):
-    """
-    test find_histogram function with multi tiff frame volume
-
-    TODO: improve: for now histogram file are created manually. If this can be more coupled with the "real" histogram generation it would be way better
-    """
-    # create volume and histogram
-    tiff_file = os.path.join(tmp_path, "my_tiff.tif")
-    volume = MultiTIFFVolume(
-        file_path=tiff_file,
-    )
-    histogram_file = os.path.join(tmp_path, "my_tiffhistogram.hdf5")
-    with h5py.File(histogram_file, mode="w") as h5f:
-        h5f.require_group("entry/histogram/results/data")
-
-    # check behavior
-    assert find_histogram(volume=volume) == None
-
-    assert find_histogram(
-        volume=volume,
-        scan=EDFTomoScan(scan=str(tmp_path)),
-    ) == DataUrl(
-        file_path=histogram_file,
-        data_path="entry",
-        scheme="silx",
-    )
-
-    assert find_histogram(
-        volume=volume,
-        scan=NXtomoScan(scan=str(tmp_path), entry="entry"),
-    ) == DataUrl(
-        file_path=histogram_file,
-        data_path="entry",
-        scheme="silx",
-    )
-
-
-@pytest.mark.parametrize("input_dtype", (numpy.float32, numpy.float64, numpy.uint8, numpy.uint16))
-def test_clamp_and_rescale_data(input_dtype):
-    """
-    test 'rescale_data' function
-    """
-    array = numpy.linspace(
-        start=1,
-        stop=100,
-        num=100,
-        endpoint=True,
-        dtype=input_dtype,
-    ).reshape(10, 10)
-
-    rescaled_array = clamp_and_rescale_data(
-        data=array,
-        new_min=10,
-        new_max=90,
-        rescale_min_percentile=20,  # provided to insure they will be ignored
-        rescale_max_percentile=80,  # provided to insure they will be ignored
-    )
-    assert rescaled_array.min() == 10
-    assert rescaled_array.max() == 90
-    numpy.testing.assert_equal(rescaled_array.flatten()[0:10], numpy.array([10] * 10))
-    numpy.testing.assert_equal(rescaled_array.flatten()[90:100], numpy.array([90] * 10))
-
-
-def test_cast_volume(tmp_path):
-    """
-    test cast_volume
-    """
-    raw_data = numpy.linspace(
-        start=1,
-        stop=100,
-        num=100,
-        endpoint=True,
-        dtype=numpy.float64,
-    ).reshape(1, 10, 10)
-    volume_hdf5_file_path = os.path.join(tmp_path, "myvolume.hdf5")
-    volume_hdf5 = HDF5Volume(
-        file_path=volume_hdf5_file_path,
-        data_path="myentry",
-        data=raw_data,
-    )
-    volume_edf = EDFVolume(
-        folder=os.path.join(tmp_path, "volume_folder"),
-    )
-
-    # test when no histogram existing
-    cast_volume(
-        input_volume=volume_hdf5,
-        output_volume=volume_edf,
-        output_data_type=numpy.dtype(numpy.uint16),
-        rescale_min_percentile=10,
-        rescale_max_percentile=90,
-        save=True,
-        store=True,
-    )
-    # if percentiles 10 and 90 provided, no data_min and data_max then they will be computed from data min / max
-
-    # append histogram
-    with h5py.File(volume_hdf5_file_path, mode="a") as h5s:
-        hist = numpy.array([20, 20, 20, 20, 20, 20])
-        bins = numpy.array([0, 20, 40, 60, 80, 100])
-        h5s["myentry/histogram/results/data"] = numpy.vstack((hist, bins))
-
-    # and test it again
-    volume_edf.overwrite = True
-    cast_volume(
-        input_volume=volume_hdf5,
-        output_volume=volume_edf,
-        output_data_type=numpy.dtype(numpy.uint16),
-        rescale_min_percentile=20,
-        rescale_max_percentile=60,
-        save=True,
-        store=True,
-    )
-
-    # test to cast the already cast volumes
-    volume_tif = EDFVolume(
-        folder=os.path.join(tmp_path, "second_volume_folder"),
-    )
-    volume_tif.overwrite = True
-    cast_volume(
-        input_volume=volume_edf,
-        output_volume=volume_tif,
-        output_data_type=numpy.dtype(numpy.uint8),
-        save=True,
-        store=True,
-    )
-    assert volume_tif.data.dtype == numpy.uint8
-
-    volume_tif.overwrite = False
-    with pytest.raises(OSError):
-        cast_volume(
-            input_volume=volume_edf,
-            output_volume=volume_tif,
-            output_data_type=numpy.dtype(numpy.uint8),
-            save=True,
-            store=True,
-        )
-
-
-@pytest.mark.skipif(not __have_jp2k__, reason="need jp2k (glymur) for this test")
-def test_jp2k_compression_ratios(tmp_path):
-    """
-    simple test to make sure the compression ratios are handled
-    """
-    import glymur
-
-    raw_data = numpy.random.random(
-        size=(1, 2048, 2048),
-    )
-    raw_data *= 2048.000005
-    volume_hdf5_file_path = os.path.join(tmp_path, "myvolume.hdf5")
-    volume_hdf5 = HDF5Volume(
-        file_path=volume_hdf5_file_path,
-        data_path="myentry",
-        data=raw_data,
-    )
-
-    volume_jp2k_ratios_0 = JP2KVolume(
-        folder=os.path.join(tmp_path, "volume_folder"),
-        cratios=(100, 10),
-    )
-
-    volume_jp2k_ratios_1 = JP2KVolume(
-        folder=os.path.join(tmp_path, "volume_folder_2"),
-        cratios=(1000, 100),
-    )
-
-    # test when no histogram existing
-    cast_volume(
-        input_volume=volume_hdf5,
-        output_volume=volume_jp2k_ratios_0,
-        output_data_type=numpy.dtype(numpy.uint16),
-        save=True,
-        store=True,
-    )
-
-    cast_volume(
-        input_volume=volume_hdf5,
-        output_volume=volume_jp2k_ratios_1,
-        output_data_type=numpy.dtype(numpy.uint16),
-        save=True,
-        store=True,
-    )
-
-    # make sure the ratio have been taking into account
-    frame_0 = glymur.Jp2k(next(volume_jp2k_ratios_0.browse_data_files()))
-    frame_0.layer = 0
-
-    frame_1 = glymur.Jp2k(next(volume_jp2k_ratios_1.browse_data_files()))
-    frame_1.layer = 0
-
-    assert not numpy.array_equal(frame_0, frame_1)

build/lib/nabu/io/tests/test_detector_distortion.py

@@ -1,178 +0,0 @@
-import pytest
-import numpy as np
-import scipy.ndimage
-from scipy import sparse
-from nabu.io.detector_distortion import DetectorDistortionBase
-from nabu.processing.rotation import Rotation, __have__skimage__
-
-if __have__skimage__:
-    import skimage
-
-
-@pytest.mark.skipif(not (__have__skimage__), reason="Need scikit-image for rotation")
-def test_detector_distortion():
-    image = scipy.ndimage.gaussian_filter(np.random.random([379, 1357]), 3.0)
-    center_xz = ((image.shape[1] - 1) / 2, (image.shape[0] - 1) / 2)
-
-    part_to_be_retrieved = image[100:279]
-
-    rotated_image = skimage.transform.rotate(image, angle=5.0, center=center_xz[::])
-
-    corrector = DetectorDistortionRotation(detector_full_shape_vh=image.shape, center_xz=center_xz, angle_deg=-5.0)
-    start_x, end_x, start_z, end_z = corrector.set_sub_region_transformation(
-        target_sub_region=(
-            None,
-            None,
-            100,
-            279,
-        )
-    )
-
-    source = rotated_image[start_z:end_z, start_x:end_x]
-
-    retrieved = corrector.transform(source)
-
-    diff = (retrieved - part_to_be_retrieved)[:, 20:-20]
-
-    assert abs(diff).std() < 1e-3
-
-
-class DetectorDistortionRotation(DetectorDistortionBase):
-    """ """
-
-    def __init__(self, detector_full_shape_vh=(0, 0), center_xz=(0, 0), angle_deg=0.0):
-        """This is the basis class.
-        A simple identity transformation which has the only merit to show
-        how it works.Reimplement this function to have more parameters for other
-        transformations
-        """
-        self._build_full_transformation(detector_full_shape_vh, center_xz, angle_deg)
-
-    def _build_full_transformation(self, detector_full_shape_vh, center_xz, angle_deg):
-        """A simple identity.
-        Reimplement this function to have more parameters for other
-        transformations
-        """
-
-        indices = np.indices(detector_full_shape_vh)
-        center_x, center_z = center_xz
-        coordinates = (indices.T - np.array([center_z, center_x])).T
-
-        c = np.cos(np.deg2rad(angle_deg))
-        s = np.sin(np.deg2rad(angle_deg))
-
-        rot_mat = np.array([[c, s], [-s, c]])
-
-        coordinates = np.tensordot(rot_mat, coordinates, axes=[1, 0])
-
-        # padding
-        sz, sx = detector_full_shape_vh
-        total_detector_npixs = sz * sx
-        xs = np.clip(np.array(coordinates[1].flat) + center_x, [[0]], [[sx - 1]])
-        zs = np.clip(np.array(coordinates[0].flat) + center_z, [[0]], [[sz - 1]])
-
-        ix0s = np.floor(xs)
-        ix1s = np.ceil(xs)
-        fx = xs - ix0s
-
-        iz0s = np.floor(zs)
-        iz1s = np.ceil(zs)
-        fz = zs - iz0s
-
-        I_tmp = np.empty([4 * sz * sx], np.int64)
-        J_tmp = np.empty([4 * sz * sx], np.int64)
-        V_tmp = np.ones([4 * sz * sx], "f")
-
-        I_tmp[:] = np.arange(sz * sx * 4) // 4
-
-        J_tmp[0::4] = iz0s * sx + ix0s
-        J_tmp[1::4] = iz0s * sx + ix1s
-        J_tmp[2::4] = iz1s * sx + ix0s
-        J_tmp[3::4] = iz1s * sx + ix1s
-
-        V_tmp[0::4] = (1 - fz) * (1 - fx)
-        V_tmp[1::4] = (1 - fz) * fx
-        V_tmp[2::4] = fz * (1 - fx)
-        V_tmp[3::4] = fz * fx
-
-        self.detector_full_shape_vh = detector_full_shape_vh
-
-        coo_tmp = sparse.coo_matrix((V_tmp.astype("f"), (I_tmp, J_tmp)), shape=(sz * sx, sz * sx))
-
-        csr_tmp = coo_tmp.tocsr()
-
-        self.full_csr_data = csr_tmp.data
-        self.full_csr_indices = csr_tmp.indices
-        self.full_csr_indptr = csr_tmp.indptr
-
-        ## This will be used to save time if the same sub_region argument is requested several time in a row
-        self._status = None
-
-    def _set_sub_region_transformation(
-        self,
-        target_sub_region=(
-            (
-                None,
-                None,
-                0,
-                0,
-            ),
-        ),
-    ):
-        (x_start, x_end, z_start, z_end) = target_sub_region
-
-        if z_start is None:
-            z_start = 0
-        if z_end is None:
-            z_end = self.detector_full_shape_vh[0]
-
-        if (x_start, x_end) not in [(None, None), (0, None), (0, self.detector_full_shape_vh[1])]:
-            message = f""" In the base class DetectorDistortionRotation only vertical slicing is accepted.
-            The sub_region contained (x_start, x_end)={(x_start, x_end)} which would slice the 
-            full horizontal size which is {self.detector_full_shape_vh[1]}
-            """
-            raise ValueError()
-
-        x_start, x_end = 0, self.detector_full_shape_vh[1]
-
-        row_ptr_start = z_start * self.detector_full_shape_vh[1]
-        row_ptr_end = z_end * self.detector_full_shape_vh[1]
-
-        indices_start = self.full_csr_indptr[row_ptr_start]
-        indices_end = self.full_csr_indptr[row_ptr_end]
-
-        data_tmp = self.full_csr_data[indices_start:indices_end]
-
-        target_offset = self.full_csr_indptr[row_ptr_start]
-        indptr_tmp = self.full_csr_indptr[row_ptr_start : row_ptr_end + 1] - target_offset
-
-        indices_tmp = self.full_csr_indices[indices_start:indices_end]
-
-        iz_source = (indices_tmp) // self.detector_full_shape_vh[1]
-
-        z_start_source = iz_source.min()
-        z_end_source = iz_source.max() + 1
-        source_offset = z_start_source * self.detector_full_shape_vh[1]
-        indices_tmp = indices_tmp - source_offset
-
-        target_size = (z_end - z_start) * self.detector_full_shape_vh[1]
-        source_size = (z_end_source - z_start_source) * self.detector_full_shape_vh[1]
-
-        self.transformation_matrix = sparse.csr_matrix(
-            (data_tmp, indices_tmp, indptr_tmp), shape=(target_size, source_size)
-        )
-
-        self.target_shape = ((z_end - z_start), self.detector_full_shape_vh[1])
-
-        ## For the identity matrix the source and the target have the same size.
-        ## The two following lines are trivial.
-        ## For this identity transformation only the slicing of the appropriate part
-        ## of the identity sparse matrix is slightly laborious.
-        ## Practical case will be more complicated and source_sub_region
-        ## will be in general larger than the target_sub_region
-        self._status = {
-            "target_sub_region": ((x_start, x_end, z_start, z_end)),
-            "source_sub_region": ((x_start, x_end, z_start_source, z_end_source)),
-        }
-
-        return self._status["source_sub_region"]