nabu 2023.2.1__py3-none-any.whl → 2024.1.0rc3__py3-none-any.whl

nabu/stitching/slurm_utils.py

@@ -1,22 +1,22 @@
 import os
 import copy
+from typing import Optional, Union
 import numpy
 from silx.io.url import DataUrl
-from typing import Iterable, Optional, Union
 from tomoscan.tomoobject import TomoObject
-from nabu.app.bootstrap_stitching import _SECTIONS_COMMENTS
-from nabu.stitching.config import (
+from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
+from tomoscan.esrf import EDFTomoScan
+from tomoscan.esrf.volume import HDF5Volume, MultiTIFFVolume
+from tomoscan.esrf.volume.singleframebase import VolumeSingleFrameBase
+from ..app.bootstrap_stitching import _SECTIONS_COMMENTS
+from ..pipeline.config import generate_nabu_configfile
+from .config import (
     StitchingConfiguration,
     get_default_stitching_config,
     PreProcessedZStitchingConfiguration,
     PostProcessedZStitchingConfiguration,
     SLURM_SECTION,
 )
-from nabu.pipeline.config import generate_nabu_configfile
-from tomoscan.esrf import HDF5TomoScan
-from tomoscan.esrf import EDFTomoScan
-from tomoscan.esrf.volume import HDF5Volume, MultiTIFFVolume
-from tomoscan.esrf.volume.singleframebase import VolumeSingleFrameBase
 
 try:
     from sluurp.job import SBatchScriptJob
@@ -87,7 +87,7 @@ def split_stitching_configuration_to_slurm_job(
                 original_output_file_path,
                 os.path.basename(original_output_file_path) + f"_part_{i_sub_part}" + file_extension,
             )
-            output_obj = HDF5TomoScan(
+            output_obj = NXtomoScan(
                 scan=sub_configuration.output_file_path,
                 entry=sub_configuration.output_data_path,
             )
@@ -195,7 +195,7 @@ def get_working_directory(obj: TomoObject) -> Optional[str]:
             return None
     elif isinstance(obj, EDFTomoScan):
         return obj.path
-    elif isinstance(obj, HDF5TomoScan):
+    elif isinstance(obj, NXtomoScan):
         if obj.master_file is None:
             return None
         else:

nabu/stitching/tests/test_alignment.py

@@ -0,0 +1,99 @@
+import numpy
+import pytest
+from nabu.stitching.alignment import align_horizontally, PaddedRawData
+from nabu.testutils import get_data
+
+
+def test_alignment_axis_2():
+    """
+    test 'align_horizontally' function
+    """
+    dataset = get_data("chelsea.npz")["data"]  # shape is (300, 451)
+
+    # test if new_width < current_width: should raise an error
+    with pytest.raises(ValueError):
+        align_horizontally(dataset, alignment="center", new_width=10)
+
+    # test some use cases
+    res = align_horizontally(
+        dataset,
+        alignment="center",
+        new_width=600,
+        pad_mode="mean",
+    )
+    assert res.shape == (300, 600)
+    numpy.testing.assert_array_almost_equal(res[:, 74:-75], dataset)
+
+    res = align_horizontally(
+        dataset,
+        alignment="left",
+        new_width=600,
+        pad_mode="median",
+    )
+    assert res.shape == (300, 600)
+    numpy.testing.assert_array_almost_equal(res[:, :451], dataset)
+
+    res = align_horizontally(
+        dataset,
+        alignment="right",
+        new_width=600,
+        pad_mode="reflect",
+    )
+    assert res.shape == (300, 600)
+    numpy.testing.assert_array_almost_equal(res[:, -451:], dataset)
+
+
+def test_PaddedRawData():
+    """
+    test PaddedVolume class
+    """
+    data = numpy.linspace(
+        start=0,
+        stop=20 * 6 * 3,
+        dtype=numpy.int64,
+        num=20 * 6 * 3,
+    )
+    data = data.reshape((3, 6, 20))
+
+    padded_volume = PaddedRawData(data=data, axis_1_pad_width=(4, 1))
+
+    assert padded_volume.shape == (3, 6 + 4 + 1, 20)
+
+    numpy.testing.assert_array_equal(
+        padded_volume[:, 0, :],
+        numpy.zeros(shape=(3, 1, 20), dtype=numpy.int64),
+    )
+    numpy.testing.assert_array_equal(
+        padded_volume[:, 3, :],
+        numpy.zeros(shape=(3, 1, 20), dtype=numpy.int64),
+    )
+    numpy.testing.assert_array_equal(
+        padded_volume[:, 10, :],
+        numpy.zeros(shape=(3, 1, 20), dtype=numpy.int64),
+    )
+    assert padded_volume[:, 3, :].shape == (3, 1, 20)
+    numpy.testing.assert_array_equal(
+        padded_volume[:, 4, :],
+        data[:, 0:1, :],  # TODO: have a look, return a 3D array when a 2D expected...
+    )
+
+    with pytest.raises(ValueError):
+        padded_volume[:, 40, :]
+    with pytest.raises(ValueError):
+        padded_volume[:, 5:1, :]
+
+    arrays = (
+        numpy.zeros(shape=(3, 4, 20), dtype=numpy.int64),
+        data,
+        numpy.zeros(shape=(3, 1, 20), dtype=numpy.int64),
+    )
+    expected_volume = numpy.hstack(
+        arrays,
+    )
+    assert padded_volume[:, :, :].shape == padded_volume.shape
+    assert expected_volume.shape == padded_volume.shape
+
+    numpy.testing.assert_array_equal(
+        padded_volume[:, :, :],
+        expected_volume,
+    )

nabu/stitching/tests/test_config.py

@@ -113,7 +113,14 @@ def test_stitching_config(stitching_type, option_level):
 @pytest.mark.parametrize("overwrite_results", (True, "False", 0, "1"))
 @pytest.mark.parametrize(
     "axis_shifts",
-    ("", None, "None", "", "skimage", "nabu-fft", "shift-grid"),
+    (
+        "",
+        None,
+        "None",
+        "",
+        "skimage",
+        "nabu-fft",
+    ),
 )
 @pytest.mark.parametrize("axis_shifts_params", ("", {}, "window_size=200"))
 @pytest.mark.parametrize(
@@ -135,6 +142,7 @@ def test_stitching_config(stitching_type, option_level):
     "slurm_config",
     (
         {
+            stiching_config.SLURM_MODULES_TO_LOADS: "tomotools",
             stiching_config.SLURM_PREPROCESSING_COMMAND: "",
             stiching_config.SLURM_CLEAN_SCRIPTS: True,
             stiching_config.SLURM_MEM: 56,
@@ -181,6 +189,13 @@ def test_PreProcessedZStitchingConfiguration(
                 stiching_config.NEXUS_VERSION_FIELD: None,
             },
             stiching_config.SLURM_SECTION: slurm_config,
+            stiching_config.NORMALIZATION_BY_SAMPLE_SECTION: {
+                stiching_config.NORMALIZATION_BY_SAMPLE_MARGIN: 1,
+                stiching_config.NORMALIZATION_BY_SAMPLE_SIDE: "right",
+                stiching_config.NORMALIZATION_BY_SAMPLE_ACTIVE_FIELD: True,
+                stiching_config.NORMALIZATION_BY_SAMPLE_METHOD: "mean",
+                stiching_config.NORMALIZATION_BY_SAMPLE_WIDTH: 31,
+            },
         },
     )
 

nabu/stitching/tests/test_overlap.py

@@ -1,6 +1,8 @@
-from nabu.stitching.overlap import compute_image_minimum_divergence, compute_image_higher_signal
-from nabu.testutils import get_data
 import numpy
+import pytest
+
+from nabu.stitching.overlap import compute_image_minimum_divergence, compute_image_higher_signal, check_overlaps
+from nabu.testutils import get_data
 
 
 def test_compute_image_minimum_divergence():
@@ -30,3 +32,67 @@ def test_compute_image_higher_signal():
         stitching,
         raw_data,
     )
+
+
+def test_check_overlaps():
+    """test 'check_overlaps' function"""
+
+    # two frames, ordered and with an overlap
+    check_overlaps(
+        frames=(
+            numpy.ones(10),
+            numpy.ones(20),
+        ),
+        positions=((10,), (0,)),
+        axis=0,
+        raise_error=True,
+    )
+
+    # two frames, ordered and without an overlap
+    with pytest.raises(ValueError):
+        check_overlaps(
+            frames=(
+                numpy.ones(10),
+                numpy.ones(20),
+            ),
+            positions=((0,), (100,)),
+            axis=0,
+            raise_error=True,
+        )
+
+    # two frames, frame 0 fully overlap frame 1
+    with pytest.raises(ValueError):
+        check_overlaps(
+            frames=(
+                numpy.ones(20),
+                numpy.ones(10),
+            ),
+            positions=((8,), (5,)),
+            axis=0,
+            raise_error=True,
+        )
+
+    # three frames 'overlaping' as expected
+    check_overlaps(
+        frames=(
+            numpy.ones(10),
+            numpy.ones(20),
+            numpy.ones(10),
+        ),
+        positions=((20,), (10,), (0,)),
+        axis=0,
+        raise_error=True,
+    )
+
+    # three frames: frame 0 overlap frame 1 but also frame 2
+    with pytest.raises(ValueError):
+        check_overlaps(
+            frames=(
+                numpy.ones(20),
+                numpy.ones(10),
+                numpy.ones(10),
+            ),
+            positions=((20,), (15,), (11,)),
+            axis=0,
+            raise_error=True,
+        )

nabu/stitching/tests/test_sample_normalization.py

@@ -0,0 +1,49 @@
+import numpy
+import pytest
+from nabu.stitching.sample_normalization import normalize_frame, SampleSide, Method
+
+
+def test_normalize_frame():
+    """
+    test normalize_frame function
+    """
+    with pytest.raises(TypeError):
+        normalize_frame("toto", "left", "median")
+    with pytest.raises(TypeError):
+        normalize_frame(numpy.linspace(0, 100), "left", "median")
+
+    frame = numpy.ones((10, 40))
+    frame[:, 15:25] = numpy.arange(1, 101, step=1).reshape((10, 10))
+
+    numpy.testing.assert_array_equal(
+        normalize_frame(
+            frame=frame,
+            side="left",
+            method="mean",
+            sample_width=10,
+            margin_before_sample=2,
+        )[:, 15:25],
+        numpy.arange(0, 100, step=1).reshape((10, 10)),
+    )
+
+    numpy.testing.assert_array_equal(
+        normalize_frame(
+            frame=frame,
+            side="right",
+            method="median",
+            sample_width=10,
+            margin_before_sample=2,
+        )[:, 15:25],
+        numpy.arange(0, 100, step=1).reshape((10, 10)),
+    )
+
+    assert not numpy.array_equal(
+        normalize_frame(
+            frame=frame,
+            side="right",
+            method="mean",
+            sample_width=10,
+            margin_before_sample=20,
+        )[:, 15:25],
+        numpy.arange(0, 100, step=1).reshape((10, 10)),
+    )

nabu/stitching/tests/test_slurm_utils.py

@@ -1,7 +1,7 @@
 import os
 import numpy
 import pytest
-from tomoscan.esrf import HDF5TomoScan
+from tomoscan.esrf import NXtomoScan
 from tomoscan.esrf.volume import HDF5Volume
 from tomoscan.esrf.scan.utils import cwd_context
 from nabu.stitching.config import PreProcessedZStitchingConfiguration, SlurmConfig
@@ -11,7 +11,7 @@ from nabu.stitching.slurm_utils import (
     get_working_directory,
     split_stitching_configuration_to_slurm_job,
 )
-from tomoscan.esrf.mock import MockHDF5
+from tomoscan.esrf.mock import MockNXtomo
 
 try:
     import sluurp
@@ -59,7 +59,7 @@ def test_split_slices():
 
 def test_get_working_directory():
     """test get_working_directory function"""
-    assert get_working_directory(HDF5TomoScan("/this/is/my/hdf5file.hdf5", "entry")) == "/this/is/my"
+    assert get_working_directory(NXtomoScan("/this/is/my/hdf5file.hdf5", "entry")) == "/this/is/my"
     assert get_working_directory(HDF5Volume("/this/is/my/volume.hdf5", "entry")) == "/this/is/my"
 
 
@@ -85,14 +85,14 @@ def test_split_stitching_configuration_to_slurm_job(tmp_path):
     with cwd_context(inputs_dir):
         # the current working directory context help to check file path are moved to absolute.
         # which is important because those jobs will be launched on slurm
-        scan_1 = MockHDF5(
+        scan_1 = MockNXtomo(
             os.path.join("scan_1"),
             n_proj=10,
             n_ini_proj=10,
             dim=100,
         ).scan
 
-        scan_2 = MockHDF5(
+        scan_2 = MockNXtomo(
             os.path.join("scan_2"),
             n_proj=10,
             n_ini_proj=10,

nabu/stitching/tests/test_utils.py

@@ -1,45 +1,15 @@
-from nabu.stitching.utils import shift_grid_search, ScoreMethod, has_itk, find_shift_with_itk
-import scipy.misc
-from scipy.ndimage import shift as scipy_shift
+from nabu.stitching.utils import has_itk, find_shift_with_itk
 from scipy.ndimage import shift as shift_scipy
 import numpy
 import pytest
-
-
-@pytest.mark.parametrize("shift", [(0, 3), (-4, 0), (8, -6)])
-def test_shift_grid_search(shift):
-    """
-    test shift_grid_search algorithm
-    """
-    y = numpy.sin(numpy.linspace(0, numpy.pi, 250))
-
-    weights = numpy.cos(numpy.linspace(-numpy.pi, numpy.pi, 120))
-    image = numpy.outer(y, weights)
-    # add a simple line with different value to ease detection
-    image[150] = 1.0
-    image[:, 50] = 1.2
-
-    image_ref = image
-    image_with_shift = scipy_shift(image_ref.copy(), shift=-numpy.array(shift))
-    score_method = ScoreMethod.TV
-
-    best_shift = shift_grid_search(
-        image_ref,
-        image_with_shift,
-        window_sizes=(40, 20),
-        axis=(0, 1),
-        step_size=1,
-        score_method=score_method,
-    )
-
-    assert tuple(best_shift) == shift
+from nabu.testutils import get_data
 
 
 @pytest.mark.parametrize("data_type", (numpy.float32, numpy.uint16))
 @pytest.mark.skipif(not has_itk, reason="itk not installed")
 def test_find_shift_with_itk(data_type):
     shift = (5, 2)
-    img1 = scipy.misc.ascent().astype(data_type)
+    img1 = get_data("chelsea.npz")["data"].astype(data_type)
     img2 = shift_scipy(
         img1.copy(),
         shift=shift,