tomwer 1.3.26__py3-none-any.whl → 1.4.0__py3-none-any.whl

tomwer/core/process/drac/rawdataset.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import numpy
+import os
+import shutil
+
+from .dracbase import DracDatasetBase
+from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomoscan.esrf.scan.utils import from_bliss_original_file_to_raw
+from tomwer.core.utils.dictutils import concatenate_dict
+from tomwer.io.utils.raw_and_processed_data import to_raw_data_path
+
+__all__ = [
+    "DracRawDataset",
+]
+
+
+class DracRawDataset(DracDatasetBase):
+    """
+    Class to associate a raw dataset (TomwerScanBase) to an icat (raw) dataset
+    """
+
+    def __init__(
+        self,
+        tomo_obj: TomwerScanBase,
+        raw_darks_required: bool = True,
+        raw_flats_required: bool = True,
+        raw_projections_required: bool = True,
+        raw_projections_each: float = 90,
+    ) -> None:
+        if not isinstance(tomo_obj, TomwerScanBase):
+            raise TypeError(
+                f"tomo_obj should be an instance of {TomwerScanBase}. Got {type(tomo_obj)}"
+            )
+        data_dir = DracRawDataset.get_data_dir(scan=tomo_obj)
+        super().__init__(data_dir=data_dir, tomo_obj=tomo_obj)
+        self.__raw_darks_required = raw_darks_required
+        self.__raw_flats_required = raw_flats_required
+        self.__raw_projections_each = raw_projections_each
+        self.__raw_projections_required = raw_projections_required
+
+    @property
+    def metadata(self) -> dict:
+        scan = self.tomo_obj
+        assert isinstance(scan, TomwerScanBase)
+        if scan is not None:
+            return scan.build_drac_metadata()
+        else:
+            return {}
+
+    @property
+    def bliss_raw_datasets(self) -> tuple[str]:
+        return super().from_scan_to_raws(scan=self.tomo_obj)
+
+    @property
+    def raw_darks_required(self) -> bool:
+        return self.__raw_darks_required
+
+    @property
+    def raw_flats_required(self) -> bool:
+        return self.__raw_flats_required
+
+    @property
+    def raw_projections_each(self) -> float:
+        return self.__raw_projections_each
+
+    @property
+    def raw_projections_required(self) -> bool:
+        return self.__raw_projections_required
+
+    def build_gallery(self):
+        if os.path.exists(self.get_gallery_dir()):
+            if self.gallery_overwrite:
+                shutil.rmtree(self.get_gallery_dir())
+            else:
+                raise RuntimeError(
+                    f"Gallery output dir ({self.get_gallery_dir()} already exists and cannot overwrite it.)"
+                )
+        scan = self.tomo_obj
+        # collect all the data to be saved
+        data_to_save: dict[str, numpy.ndarray] = {}
+        if self.raw_darks_required and len(scan.darks) > 0:
+            data_to_save["dark"] = next(iter(scan.darks.values()))
+        if self.raw_flats_required and len(scan.flats) > 0:
+            data_to_save["flat"] = next(iter(scan.flats.values()))
+        if self.raw_projections_required:
+            projections = scan.projections_with_angle()
+            picked_angles = DracRawDataset.select_angles(
+                angles_list=sorted(projections.keys()),
+                each_angle=self.raw_projections_each,
+            )
+            data_to_save = concatenate_dict(
+                data_to_save,
+                {
+                    f"projection_{angle:.1f}": projections[angle]
+                    for angle in picked_angles
+                },
+            )
+
+        # dump to disk
+        for img_name, img_data in data_to_save.items():
+            self.save_to_gallery(
+                output_file_name=os.path.join(self.get_gallery_dir(), img_name),
+                image=img_data,
+            )
+
+    @staticmethod
+    def select_angles(angles_list: tuple, each_angle: int) -> tuple:
+        angles_list = sorted(angles_list)
+        if len(angles_list) > 0:
+            start_angle = angles_list[0]
+            stop_angle = angles_list[-1]
+            picked_angle = numpy.arange(start_angle, stop_angle + 1, step=each_angle)
+            return tuple(
+                [
+                    DracRawDataset.get_closest_projection(angle, angles_list)
+                    for angle in picked_angle
+                ]
+            )
+        else:
+            return tuple()
+
+    @staticmethod
+    def get_closest_projection(angle, angles_list):
+        idx_closest = numpy.argmin(numpy.abs(angles_list - angle))
+        return angles_list[idx_closest]
+
+    @staticmethod
+    def get_data_dir(scan: HDF5TomoScan) -> str | None:
+        """Deduce (bliss) data directory from a scan.
+        If the scan can get the information from 'get_bliss_original_files' then we use it as this is most reliable information.
+        Else we use the nxtomo location as a fallback (in case the data doesn't comes from bliss for example).
+        """
+        # the most reliable way is to get it from 'get_bliss_original_files' as this is set during conversion
+        # else
+        raw_data_file = scan.get_bliss_original_files()
+        if scan.master_file is None:
+            return None
+        else:
+            raw_data_file = to_raw_data_path(scan.master_file)
+        return from_bliss_original_file_to_raw(os.path.dirname(raw_data_file))

tomwer/core/process/drac/tests/test_gallery.py

@@ -0,0 +1,71 @@
+from tomwer.core.process.drac.gallery import (
+    deduce_dataset_gallery_location,
+    deduce_proposal_GALLERY_location,
+    PROPOSAL_GALLERY_DIR_NAME,
+)
+from tomwer.core.scan.edfscan import EDFTomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+
+
+def test_deduce_dataset_gallery_dir():
+    """test the deduce_gallery_dir function"""
+    assert (
+        deduce_dataset_gallery_location(
+            NXtomoScan(scan="/path/to/PROCESSED_DATA/my_scan.nx", entry="entry")
+        )
+        == "/path/to/PROCESSED_DATA/gallery"
+    )
+    assert (
+        deduce_dataset_gallery_location(
+            NXtomoScan(scan="/path/to/RAW_DATA/collection/my_scan.nx", entry="entry")
+        )
+        == "/path/to/PROCESSED_DATA/collection/gallery"
+    )
+    assert (
+        deduce_dataset_gallery_location(
+            NXtomoScan(scan="/any/random/path/my_scan.nx", entry="entry")
+        )
+        == "/any/random/path/gallery"
+    )
+    assert (
+        deduce_dataset_gallery_location(
+            EDFTomoScan(scan="/path/to/PROCESSED_DATA/dataset/toto")
+        )
+        == "/path/to/PROCESSED_DATA/dataset/toto/gallery"
+    )
+    assert (
+        deduce_dataset_gallery_location(EDFTomoScan(scan="/path/to/dataset/toto"))
+        == "/path/to/dataset/toto/gallery"
+    )
+
+
+def test_deduce_proposal_gallery_dir():
+    """test the deduce_gallery_dir function"""
+    assert (
+        deduce_proposal_GALLERY_location(
+            NXtomoScan(scan="/path/to/PROCESSED_DATA/my_scan.nx", entry="entry")
+        )
+        == f"/path/to/{PROPOSAL_GALLERY_DIR_NAME}"
+    )
+    assert (
+        deduce_proposal_GALLERY_location(
+            NXtomoScan(scan="/path/to/PROCESSED_DATA/dataset/my_scan.nx", entry="entry")
+        )
+        == f"/path/to/{PROPOSAL_GALLERY_DIR_NAME}/dataset"
+    )
+    assert (
+        deduce_proposal_GALLERY_location(
+            NXtomoScan(scan="/any/random/path/my_scan.nx", entry="entry")
+        )
+        == f"/any/random/path/{PROPOSAL_GALLERY_DIR_NAME}"
+    )
+    assert (
+        deduce_proposal_GALLERY_location(
+            EDFTomoScan(scan="/path/to/PROCESSED_DATA/dataset/toto")
+        )
+        == "/path/to/GALLERY/dataset/toto"
+    )
+    assert (
+        deduce_proposal_GALLERY_location(EDFTomoScan(scan="/path/to/dataset/toto"))
+        == "/path/to/dataset/toto/GALLERY"
+    )

tomwer/core/process/drac/tests/test_icat_processed_dataset.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import os
+import pytest
+import numpy
+
+from tomwer.core.process.drac.processeddataset import DracReconstructedVolumeDataset
+from tomwer.tests.conftest import nxtomo_scan_180  # noqa F811
+from tomwer.core.volume.hdf5volume import HDF5Volume
+from tomwer.core.volume.edfvolume import EDFVolume
+
+
+@pytest.mark.parametrize(
+    "output_dir_use_case", ("default", "change_name", "change_dir")
+)
+@pytest.mark.parametrize("MetaVolumeClass", (HDF5Volume, EDFVolume))
+def test_IcatReconstructedVolumeDataset(
+    output_dir_use_case: str, nxtomo_scan_180, MetaVolumeClass, tmp_path  # noqa F811
+):
+
+    if output_dir_use_case == "default":
+        output_dir = os.path.join(
+            os.path.dirname(nxtomo_scan_180.master_file), "reconstructed_volumes"
+        )
+    elif output_dir_use_case == "change_name":
+        output_dir = os.path.join(
+            os.path.dirname(nxtomo_scan_180.master_file), "my_vols"
+        )
+    elif output_dir_use_case == "change_dir":
+        output_dir = tmp_path / "test" / "reconstructed_volumes"
+        output_dir.mkdir(parents=True)
+
+    if MetaVolumeClass is HDF5Volume:
+        volume = MetaVolumeClass(
+            file_path=os.path.join(output_dir, "test.hdf5"),
+            data_path="data",
+        )
+    elif MetaVolumeClass is EDFVolume:
+        volume = MetaVolumeClass(folder=os.path.join(output_dir, "folder"))
+    else:
+        raise NotImplementedError
+    volume.data = numpy.random.random((10, 10, 10))
+    volume.save()
+
+    icat_dataset = DracReconstructedVolumeDataset(
+        tomo_obj=volume, source_scan=nxtomo_scan_180
+    )
+    gallery_dir = icat_dataset.get_gallery_dir()
+    assert gallery_dir == os.path.join(output_dir, "gallery")
+    assert not os.path.exists(gallery_dir)
+    assert icat_dataset.get_slices_to_extract() == (
+        (0, 2),
+        (0, 5),
+        (0, 8),
+        (1, 2),
+        (1, 5),
+        (1, 8),
+        (2, 2),
+        (2, 5),
+        (2, 8),
+    )
+    icat_dataset.build_gallery()
+    assert os.path.exists(gallery_dir)
+    # make sure we have three slice over each direction
+    assert len(os.listdir(gallery_dir)) == 3
+    assert "XY" in os.listdir(gallery_dir)
+    if MetaVolumeClass is HDF5Volume:
+        assert "test_capture_XY_000008.png" in os.listdir(
+            os.path.join(gallery_dir, "XY")
+        )
+        assert "test_capture_YZ_000002.png" in os.listdir(
+            os.path.join(gallery_dir, "YZ")
+        )
+    elif MetaVolumeClass is EDFVolume:
+        assert "folder_capture_XY_000008.png" in os.listdir(
+            os.path.join(gallery_dir, "XY")
+        )
+        assert "folder_capture_YZ_000002.png" in os.listdir(
+            os.path.join(gallery_dir, "YZ")
+        )

tomwer/core/process/drac/tests/test_icat_raw_dataset.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import os
+import pytest
+import numpy
+from tomwer.tests.conftest import nxtomo_scan_180  # noqa F811
+from tomwer.core.process.drac.rawdataset import DracRawDataset
+
+
+def test_IcatRawDataset(nxtomo_scan_180):  # noqa F811
+    """
+    test CreateRawDataScreenshotsTask task
+    """
+    # test nothing requested
+    dataset = DracRawDataset(
+        tomo_obj=nxtomo_scan_180,
+        raw_projections_required=False,
+        raw_darks_required=False,
+        raw_flats_required=False,
+    )
+
+    assert not os.path.exists(dataset.get_gallery_dir())
+    dataset.build_gallery()
+    assert not os.path.exists(dataset.get_gallery_dir())
+
+    # test only flat requested
+    dataset = DracRawDataset(
+        tomo_obj=nxtomo_scan_180,
+        raw_projections_required=False,
+        raw_darks_required=False,
+        raw_flats_required=True,
+    )
+    dataset.build_gallery()
+    assert len(os.listdir(dataset.get_gallery_dir())) == 1
+
+    # test only dark requested
+    dataset = DracRawDataset(
+        tomo_obj=nxtomo_scan_180,
+        raw_projections_required=False,
+        raw_darks_required=True,
+        raw_flats_required=False,
+    )
+    dataset.gallery_overwrite = False
+    with pytest.raises(RuntimeError):
+        dataset.build_gallery()
+    dataset.gallery_overwrite = True
+    dataset.build_gallery()
+    assert len(os.listdir(dataset.get_gallery_dir())) == 1
+
+    # test only projection requested
+    dataset = DracRawDataset(
+        tomo_obj=nxtomo_scan_180,
+        raw_projections_required=True,
+        raw_darks_required=False,
+        raw_flats_required=False,
+    )
+    dataset.gallery_overwrite = True
+    dataset.build_gallery()
+    assert len(os.listdir(dataset.get_gallery_dir())) == 2
+
+    # test all requested
+    dataset = DracRawDataset(
+        tomo_obj=nxtomo_scan_180,
+        raw_projections_required=True,
+        raw_darks_required=True,
+        raw_flats_required=True,
+        raw_projections_each=10,
+    )
+    dataset.gallery_overwrite = True
+    dataset.build_gallery()
+    assert len(os.listdir(dataset.get_gallery_dir())) == 12
+
+
+def test_select_angles():
+    """test the select_angles function"""
+    numpy.testing.assert_allclose(
+        DracRawDataset.select_angles(
+            numpy.linspace(start=-20, stop=40, num=201, endpoint=True), each_angle=20
+        ),
+        numpy.array([-20, 0.1, 19.9, 40]),
+    )
+
+    assert DracRawDataset.select_angles((), each_angle=20) == ()
+    assert DracRawDataset.select_angles((10,), each_angle=10) == (10,)
+    angles = DracRawDataset.select_angles(
+        numpy.linspace(start=0, stop=10, num=400, endpoint=False), each_angle=1
+    )
+    assert len(angles) == 11
+    assert numpy.isclose(angles[0], 0, atol=0.1)
+    assert numpy.isclose(angles[-1], 10, atol=0.1)

tomwer/core/process/edit/darkflatpatch.py

@@ -1,31 +1,5 @@
 # coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "04/11/2020"
+from __future__ import annotations
 
 
 import nxtomomill.version
@@ -129,7 +103,6 @@ class DarkFlatPatchTask(
         """
 
         :return: configuration of the process
-        :rtype: dict
         """
         return self.inputs.configuration
 

tomwer/core/process/edit/imagekeyeditor.py

@@ -1,32 +1,5 @@
 # coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "26/10/2020"
-
+from __future__ import annotations
 
 import copy
 import logging
@@ -103,7 +76,7 @@ class ImageKeyEditorTask(
     optional_input_names=("serialize_output_data",),
 ):
     """
-    task to edit `image_key` field of an NXtomo ('data' input)
+    task to edit `image_key` field of a NXtomo ('data' input)
     """
 
     def __init__(
@@ -165,7 +138,7 @@ class ImageKeyEditorTask(
 
     @staticmethod
     def definition():
-        return "Modify image keys on an NXTomo"
+        return "Modify image keys on a NXtomo"
 
 
 class ImageKeyUpgraderTask(
@@ -245,7 +218,7 @@ class ImageKeyUpgraderTask(
 
     @staticmethod
     def definition():
-        return "Modify image keys on an NXTomo"
+        return "Modify image keys on a NXtomo"
 
     @deprecated(
         since_version="1.2",
@@ -256,7 +229,6 @@ class ImageKeyUpgraderTask(
         """
 
         :return: configuration of the process
-        :rtype: dict
         """
         return self.inputs.configuration