tomwer 1.3.7__py3-none-any.whl → 1.3.9__py3-none-any.whl

tomwer/core/process/icat/test/test_gallery.py

@@ -1,170 +0,0 @@
-import os
-
-import h5py
-import numpy
-import pytest
-from silx.io.url import DataUrl
-
-from tomwer.core.process.icat.gallery import (
-    SaveScreenshotsToGalleryTask,
-    IcatScreenshots,
-    deduce_dataset_gallery_location,
-    deduce_proposal_GALLERY_location,
-    select_screenshot_from_volume,
-    PROPOSAL_GALLERY_DIR_NAME,
-)
-from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.nxtomoscan import NXtomoScan
-from tomwer.core.volume.edfvolume import EDFVolume
-from tomwer.core.volume.hdf5volume import HDF5Volume
-
-
-def test_deduce_dataset_gallery_dir():
-    """test the deduce_gallery_dir function"""
-    assert (
-        deduce_dataset_gallery_location(
-            NXtomoScan(scan="/path/to/PROCESSED_DATA/my_scan.nx", entry="entry")
-        )
-        == "/path/to/PROCESSED_DATA/gallery"
-    )
-    assert (
-        deduce_dataset_gallery_location(
-            NXtomoScan(scan="/path/to/RAW_DATA/collection/my_scan.nx", entry="entry")
-        )
-        == "/path/to/PROCESSED_DATA/collection/gallery"
-    )
-    assert (
-        deduce_dataset_gallery_location(
-            NXtomoScan(scan="/any/random/path/my_scan.nx", entry="entry")
-        )
-        == "/any/random/path/gallery"
-    )
-    assert (
-        deduce_dataset_gallery_location(
-            EDFTomoScan(scan="/path/to/PROCESSED_DATA/dataset/toto")
-        )
-        == "/path/to/PROCESSED_DATA/dataset/toto/gallery"
-    )
-    assert (
-        deduce_dataset_gallery_location(EDFTomoScan(scan="/path/to/dataset/toto"))
-        == "/path/to/dataset/toto/gallery"
-    )
-
-
-def test_deduce_proposal_gallery_dir():
-    """test the deduce_gallery_dir function"""
-    assert (
-        deduce_proposal_GALLERY_location(
-            NXtomoScan(scan="/path/to/PROCESSED_DATA/my_scan.nx", entry="entry")
-        )
-        == f"/path/to/{PROPOSAL_GALLERY_DIR_NAME}"
-    )
-    assert (
-        deduce_proposal_GALLERY_location(
-            NXtomoScan(scan="/path/to/PROCESSED_DATA/dataset/my_scan.nx", entry="entry")
-        )
-        == f"/path/to/{PROPOSAL_GALLERY_DIR_NAME}/dataset"
-    )
-    assert (
-        deduce_proposal_GALLERY_location(
-            NXtomoScan(scan="/any/random/path/my_scan.nx", entry="entry")
-        )
-        == f"/any/random/path/{PROPOSAL_GALLERY_DIR_NAME}"
-    )
-    assert (
-        deduce_proposal_GALLERY_location(
-            EDFTomoScan(scan="/path/to/PROCESSED_DATA/dataset/toto")
-        )
-        == "/path/to/GALLERY/dataset/toto"
-    )
-    assert (
-        deduce_proposal_GALLERY_location(EDFTomoScan(scan="/path/to/dataset/toto"))
-        == "/path/to/dataset/toto/GALLERY"
-    )
-
-
-@pytest.mark.parametrize("MetaVolumeClass", (HDF5Volume, EDFVolume))
-def test_select_screenshot_from_volume(MetaVolumeClass, tmp_path):
-    """test the 'select_screenshot_from_volume' function"""
-    output_dir = tmp_path / "volumes"
-
-    if MetaVolumeClass is HDF5Volume:
-        volume = MetaVolumeClass(
-            file_path=os.path.join(output_dir, "test.hdf5"),
-            data_path="data",
-        )
-    elif MetaVolumeClass is EDFVolume:
-        volume = MetaVolumeClass(folder=os.path.join(output_dir, "folder"))
-    else:
-        raise NotImplementedError
-
-    # test with 'standard' volume
-    ## setup the volume
-    volume.data = numpy.arange(1000000).reshape(100, 100, 100)
-    volume.save()  # needed because brosing urls check if url exists or not...
-    ## make sure we get the expected number of screenshot
-    screenshots_as_url = select_screenshot_from_volume(volume=volume)
-    assert len(screenshots_as_url) == 3
-    ## test slices
-    for url in screenshots_as_url.values():
-        if isinstance(volume, HDF5Volume):
-            assert url.data_slice() in (33, 50, 66)
-        elif isinstance(volume, EDFVolume):
-            assert url.file_path().endswith(("33.edf", "50.edf", "66.edf"))
-        else:
-            raise NotImplementedError
-
-    # test with a single frame volume
-    volume.data = numpy.arange(100).reshape(1, 10, 10)
-    volume.overwrite = True
-    volume.save()
-    screenshots_as_url = select_screenshot_from_volume(volume=volume)
-    ## make sure we have a single screenshot in this case
-    screenshot_url = tuple(screenshots_as_url.values())[0]
-    if isinstance(volume, HDF5Volume):
-        assert screenshot_url.data_slice() == 0
-        assert len(screenshots_as_url) == 1
-        # warning: for now this doesn't work for EDF because the volume class will not remove any existing slices...
-        # which is a clear limitation. But not sure we should remove those either...
-    elif isinstance(volume, EDFVolume):
-        # warning: for now this doesn't work for EDF because the volume class will not remove any existing slices...
-        # which is a clear limitation. But not sure we should remove those either...
-        # so here we will still find 3 urls for the screenshot
-        pass
-    else:
-        raise NotImplementedError
-
-
-def test_MoveScreenshotsToGalleryTask(tmp_path):
-    """test the 'MoveScreenshotsToGalleryTask' task"""
-    raw_data_dir = tmp_path / "raw_data"
-    raw_data_dir.mkdir()
-    screenshot_output_data_dir = tmp_path / "screenshot"
-    screenshot_output_data_dir.mkdir()
-
-    raw_data_file = raw_data_dir / "data.hdf5"
-    with h5py.File(raw_data_file, mode="w") as h5f:
-        h5f["data"] = numpy.random.random(100).reshape(10, 10)
-
-    screenshots = IcatScreenshots(
-        data_dir=str(screenshot_output_data_dir),
-        screenshots={
-            "my_screenshot": DataUrl(
-                file_path=raw_data_file,
-                data_path="data",
-                scheme="silx",
-            )
-        },
-        scan=None,
-    )
-
-    task = SaveScreenshotsToGalleryTask(
-        inputs={
-            "screenshots": screenshots,
-            "format": "png",
-        }
-    )
-    task.run()
-
-    expected_output_file = os.path.join(screenshot_output_data_dir, "my_screenshot.png")
-    assert os.path.exists(expected_output_file)

tomwer/core/process/reconstruction/nabu/test/test_castvolume.py

@@ -1,143 +0,0 @@
-# coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "05/07/2022"
-
-
-import os
-
-import numpy
-import pytest
-
-from tomwer.core.process.reconstruction.nabu.castvolume import CastVolumeTask
-from tomwer.core.process.reconstruction.output import NabuOutputFileFormat
-from tomwer.core.scan.nxtomoscan import NXtomoScan
-from tomwer.core.volume.edfvolume import EDFVolume
-from tomwer.core.volume.hdf5volume import HDF5Volume
-from tomwer.core.volume.tiffvolume import TIFFVolume
-
-
-def test_cast_volume_32bitstiff_to_16bits_tiff(tmp_path):
-    """test 32 bits tiffs to 16 bits works"""
-    dir_test = tmp_path / "test_dir"
-    dir_test.mkdir()
-    save_dir = tmp_path / "output_dir"
-    save_dir.mkdir()
-
-    vol_data = numpy.random.random(20 * 100 * 100) * 3600.4
-    vol_data = vol_data.reshape(20, 100, 100)
-    vol_data = vol_data.astype(numpy.float32)
-    volume = TIFFVolume(
-        folder=dir_test,
-        volume_basename="test_dir_vol_",
-        data=vol_data,
-    )
-    volume.save()
-
-    # test with output datadir as None
-    with pytest.raises(ValueError):
-        CastVolumeTask(
-            inputs={
-                "volume": volume,
-                "configuration": {
-                    "output_file_path": None,
-                    "output_file_format": NabuOutputFileFormat.EDF,
-                    "output_data_type": numpy.uint16,
-                },
-            },
-        ).run()
-
-    # test providing save dir and tiff
-    assert len(os.listdir(save_dir)) == 0
-    task = CastVolumeTask(
-        inputs={
-            "volume": volume,
-            "configuration": {
-                "output_dir": str(save_dir),
-                "output_type": NabuOutputFileFormat.EDF,
-                "output_data_type": numpy.uint16,
-            },
-        }
-    )
-    task.run()
-
-    assert len(os.listdir(save_dir)) == 20, "no files have been generated"
-    assert task.outputs.volume.load_data().shape == (20, 100, 100)
-    assert task.outputs.volume.load_data().dtype == numpy.uint16
-
-    # test providing save dir, a scan and hdf5
-    scan = NXtomoScan(scan=os.sep.join([str(tmp_path), "scan"]), entry="entry0000")
-    task = CastVolumeTask(
-        inputs={
-            "volume": volume,
-            "configuration": {
-                "output_dir": str(save_dir),
-                "output_type": NabuOutputFileFormat.HDF5,
-                "output_data_type": numpy.float32,
-            },
-            "scan": scan,
-        },
-    )
-    task.run()
-    assert task.outputs.volume.load_data().shape == (20, 100, 100)
-    assert task.outputs.volume.load_data().dtype == numpy.float32
-
-    # test providing a volume as output volume
-    output_volume = HDF5Volume(
-        file_path=os.path.join(str(tmp_path), "my_volume.hdf5"), data_path="entry0002"
-    )
-    task = CastVolumeTask(
-        inputs={
-            "volume": volume,
-            "configuration": {
-                "output_dir": str(save_dir),
-                "output_type": NabuOutputFileFormat.HDF5,
-                "output_data_type": numpy.uint8,
-            },
-            "output_volume": output_volume,
-        },
-    )
-    task.run()
-    assert task.outputs.volume.load_data().shape == (20, 100, 100)
-    assert task.outputs.volume.load_data().dtype == numpy.uint8
-
-    # test providing a volume us as output volume
-    output_volume = EDFVolume(folder=os.path.join(str(tmp_path), "my_edf_vol"))
-    task = CastVolumeTask(
-        inputs={
-            "volume": volume,
-            "configuration": {
-                "output_dir": str(save_dir),
-                "output_type": NabuOutputFileFormat.HDF5,
-                "output_data_type": numpy.int16,
-            },
-            "output_volume": output_volume.get_identifier().to_str(),
-        },
-    )
-    task.run()
-    assert task.outputs.volume.load_data().shape == (20, 100, 100)
-    assert task.outputs.volume.load_data().dtype == numpy.int16

tomwer/core/process/reconstruction/nabu/test/test_nabu_utils.py

@@ -1,203 +0,0 @@
-import pytest
-
-from tomwer.core.process.reconstruction.nabu.utils import (
-    get_recons_volume_identifier,
-    get_multi_cor_recons_volume_identifiers,
-    nabu_std_err_has_error,
-)
-from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.nxtomoscan import NXtomoScan
-
-_scans = (
-    NXtomoScan(
-        scan="/my_scan_file.nx",
-        entry="entry_test",
-    ),
-    EDFTomoScan("/my_scan_folder"),
-)
-
-
-@pytest.mark.parametrize("scan", _scans)
-@pytest.mark.parametrize("axis", ("YZ", "XZ", "XY"))
-def test_get_recons_volume_identifier(scan, axis):
-    """
-    test get_recons_volume_identifier behavior
-    """
-
-    entry = scan.entry if isinstance(scan, NXtomoScan) else "entry"
-    # check some exceptions
-    with pytest.raises(ValueError):
-        get_recons_volume_identifier(
-            file_prefix="volume_rec",
-            location="/this/is/a/test",
-            file_format="toto",
-            slice_index=1080,
-            scan=scan,
-            axis=axis,
-        )
-
-    # check hdf5 reconstructions
-    id_rec_vols = get_recons_volume_identifier(
-        file_prefix="volume_rec",
-        location="/this/is/a/test",
-        file_format="hdf5",
-        slice_index="1080",
-        scan=scan,
-        axis=axis,
-    )
-    assert len(id_rec_vols) == 1
-    assert (
-        id_rec_vols[0].to_str()
-        == f"hdf5:volume:/this/is/a/test/volume_rec_plane_{axis}_001080.hdf5?path={entry}/reconstruction"
-    )
-
-    # check edf and jp2k slice reconstructions (are expected to have the same behavior)
-    id_rec_vols = get_recons_volume_identifier(
-        file_prefix="volume_rec",
-        location="/this/is/a/test",
-        file_format="edf",
-        slice_index="1080",
-        scan=scan,
-        axis=axis,
-    )
-    assert len(id_rec_vols) == 1
-    assert (
-        id_rec_vols[0].to_str() == "edf:volume:/this/is/a/test?file_prefix=volume_rec"
-    )
-
-    id_rec_vols = get_recons_volume_identifier(
-        file_prefix="volume_recslice",
-        location="/this/is/a/test",
-        file_format="jp2k",
-        slice_index="1080",
-        scan=scan,
-        axis=axis,
-    )
-    assert len(id_rec_vols) == 1
-    assert (
-        id_rec_vols[0].to_str()
-        == "jp2k:volume:/this/is/a/test?file_prefix=volume_recslice"
-    )
-
-    # check tiff slice reconstructions
-    id_rec_vols = get_recons_volume_identifier(
-        file_prefix="volume_rec",
-        location="/this/is/a/test",
-        file_format="tiff",
-        slice_index="1080",
-        scan=scan,
-        axis=axis,
-    )
-    assert len(id_rec_vols) == 1
-    assert (
-        id_rec_vols[0].to_str() == "tiff:volume:/this/is/a/test?file_prefix=volume_rec"
-    )
-
-
-@pytest.mark.parametrize("scan", _scans)
-def test_get_multi_cor_recons_volume_identifier(scan):
-    """
-    test the get_multi_cor_recons_volume_identifier function
-    """
-    entry = scan.entry if isinstance(scan, NXtomoScan) else "entry"
-
-    # dummy test with hdf5
-    id_rec_vols = get_multi_cor_recons_volume_identifiers(
-        scan=scan,
-        slice_index="middle",
-        location="/this/is/a/test",
-        file_prefix="rec",
-        cors=(10, 12),
-        file_format="hdf5",
-    )
-    assert isinstance(id_rec_vols, dict)
-    assert len(id_rec_vols) == 2
-    assert 10 in id_rec_vols.keys()
-    assert 12 in id_rec_vols.keys()
-    assert (
-        id_rec_vols[10].to_str()
-        == f"hdf5:volume:/this/is/a/test/rec_10.000_01024.hdf5?path={entry}/reconstruction"
-    )
-
-    # dummy test with tiff
-    id_rec_vols = get_multi_cor_recons_volume_identifiers(
-        scan=scan,
-        slice_index="middle",
-        location="/this/is/a/test",
-        file_prefix="rec",
-        cors=(10, 12),
-        file_format="tiff",
-    )
-    assert isinstance(id_rec_vols, dict)
-    assert len(id_rec_vols) == 2
-    assert 10 in id_rec_vols.keys()
-    assert 12 in id_rec_vols.keys()
-    assert id_rec_vols[10].to_str() == "tiff:volume:/this/is/a?file_prefix=test"
-
-
-def test_nabu_std_err_has_error():
-    assert nabu_std_err_has_error(None) is False
-    assert nabu_std_err_has_error(b"") is False
-    assert nabu_std_err_has_error(b"this is an error") is True
-    assert (
-        nabu_std_err_has_error(
-            b"warnings.warn('creating CUBLAS context to get version num"
-        )
-        is False
-    )
-    assert (
-        nabu_std_err_has_error(
-            b"warnings.warn('creating CUBLAS context to get version num\n this is an error"
-        )
-        is True
-    )
-
-    assert (
-        nabu_std_err_has_error(
-            b"""/nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/skcuda/cublas.py:284: UserWarning: creating CUBLAS context to get version number
-            warnings.warn('creating CUBLAS context to get version number')
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/pycuda/elementwise.py:47: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            return SourceModule(
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/pycuda/elementwise.py:47: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            return SourceModule(
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/nabu/cuda/kernel.py:49: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            kernel.cu(111): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
-            /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
-            kernel.cu(112): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
-            /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
-            kernel.cu(113): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
-            /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
-            kernel.cu(114): warning #1215-D: function "tex2D(texture<T, 2, cudaReadModeElementType>, float, float) [with T=float]"
-            /cvmfs/hpc.esrf.fr/software/packages/ubuntu20.04/x86_64/cuda/11.8.0//bin/../targets/x86_64-linux/include/texture_fetch_functions.h(198): here was declared deprecated
-            self.module = SourceModule(self.src, **self.sourcemodule_kwargs)
-            /nobackup/lbs0511/tomotools/venvs/2023_10_04/lib/python3.8/site-packages/pycuda/elementwise.py:47: UserWarning: The CUDA compiler succeeded, but said the following:
-            nvcc warning : The 'compute_35', 'compute_37', 'sm_35', and 'sm_37' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-            return SourceModule("""
-        )
-        is False
-    )
-
-    assert (
-        nabu_std_err_has_error(
-            b"""/cvmfs/tomo.esrf.fr/software/packages/linux/x86_64/tomotools/2024_02_26/lib/python3.11/site-packages/cupyx/jit/_interface.py:173: FutureWarning: cupyx.jit.rawkernel is experimental. The interface can change in the future.
-        cupy._util.experimental('cupyx.jit.rawkernel')
-        /cvmfs/tomo.esrf.fr/software/packages/linux/x86_64/tomotools/2024_02_26/lib/python3.11/site-packages/skcuda/cublas.py:284: UserWarning: creating CUBLAS context to get version number
-        warnings.warn('creating CUBLAS context to get version number')"""
-        )
-        is False
-    )