tomwer 1.3.5__py3-none-any.whl → 1.3.7__py3-none-any.whl

tomwer/core/log/test/test_processlog.py

@@ -0,0 +1,41 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "18/03/2022"
+
+
+from tomwer.core.log.logger import TomwerLogger
+
+
+def test_process_log():
+    logger = TomwerLogger("tomtom")
+    logger.processStarted("process started")
+    logger.processEnded("process ended")
+    logger.processSkipped("process skipped")
+    logger.inform("information")
+    logger.processSucceed("process succeed")
+    logger.processFailed("process failed")

tomwer/core/process/control/datalistener/datalistener.py

@@ -37,7 +37,7 @@ import typing
 from ewokscore.task import Task as EwoksTask
 from nxtomomill import converter as nxtomomill_converter
 from silx.io.utils import h5py_read_dataset
-from tomoscan.io import HDF5File
+from silx.io.utils import open as open_hdf5
 
 import tomwer.version
 from tomwer.core import settings
@@ -229,7 +229,7 @@ class DataListener(BaseProcessInfo):
         """Return the list of scans dir for this bliss_file / entry"""
 
         def get_scan_indexes():
-            with HDF5File(bliss_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+            with open_hdf5(bliss_file) as h5f:
                 entry_node = h5f[entry]
                 if _SCAN_NUMBER_PATH in entry_node:
                     return h5py_read_dataset(entry_node[_SCAN_NUMBER_PATH])
@@ -254,7 +254,7 @@ class DataListener(BaseProcessInfo):
         """Return the proposal file of the experimentation if registred by the
         data listener"""
         if entry is None:
-            with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -269,7 +269,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -311,7 +311,7 @@ class DataListener(BaseProcessInfo):
         """Return the proposal file of the experimentation if registred by the
         data listener"""
         if entry is None:
-            with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -326,7 +326,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -360,7 +360,7 @@ class DataListener(BaseProcessInfo):
         """Return the proposal file of the experimentation if registred by the
         data listener"""
         if entry is None:
-            with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -375,7 +375,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -410,7 +410,7 @@ class DataListener(BaseProcessInfo):
         specific sequence data
         """
         if entry is None:
-            with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+            with open_hdf5(process_file) as h5f:
                 entries = BaseProcessInfo._get_process_nodes(
                     root_node=h5f, process=DataListener
                 )
@@ -425,7 +425,7 @@ class DataListener(BaseProcessInfo):
                     entry = list(entries.keys())[0]
                     _logger.info("take %s as default entry" % entry)
 
-        with HDF5File(process_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+        with open_hdf5(process_file) as h5f:
             dl_nodes = BaseProcessInfo._get_process_nodes(
                 root_node=h5f[entry], process=DataListener
             )
@@ -503,7 +503,7 @@ class DataListener(BaseProcessInfo):
         # write
         def sequence_is_finished():
             try:
-                with HDF5File(bliss_file, "r", swmr=DataListener.SWMR_MODE) as h5f:
+                with open_hdf5(bliss_file) as h5f:
                     end_scan_path = "/".join((entry, "end_time"))
                     return end_scan_path in h5f
             except Exception:

tomwer/core/process/edit/test/test_darkflatpatch.py

@@ -0,0 +1,269 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2017-2021 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "18/06/2021"
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import h5py
+import numpy
+from silx.io.url import DataUrl
+from silx.io.utils import h5py_read_dataset
+from tomoscan.esrf.scan.mock import MockNXtomo
+
+from tomwer.core.process.edit.darkflatpatch import DarkFlatPatchTask
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+
+
+class BaseTestAddDarkAndFlats(unittest.TestCase):
+    """
+    Unit test on nxtomomill.utils.add_dark_flat_nx_file function
+    """
+
+    def setUp(self) -> None:
+        self.tmpdir = tempfile.mkdtemp()
+        simple_nx_path = os.path.join(self.tmpdir, "simple_case")
+        self.dim = 55
+        self.nproj = 20
+        self._simple_nx = MockNXtomo(
+            scan_path=simple_nx_path,
+            n_proj=self.nproj,
+            n_ini_proj=self.nproj,
+            create_ini_dark=False,
+            create_ini_flat=False,
+            create_final_flat=False,
+            dim=self.dim,
+        ).scan
+        with h5py.File(self._simple_nx.master_file, mode="r") as h5s:
+            data_path = "/".join(
+                (self._simple_nx.entry, "instrument", "detector", "data")
+            )
+            self._raw_data = h5py_read_dataset(h5s[data_path])
+        nx_with_vds_path = os.path.join(self.tmpdir, "case_with_vds")
+        self._nx_with_virtual_dataset = MockNXtomo(
+            scan_path=nx_with_vds_path,
+            n_proj=0,
+            n_ini_proj=0,
+            create_ini_dark=False,
+            create_ini_flat=False,
+            create_final_flat=False,
+            dim=self.dim,
+        ).scan
+        self._create_vds(
+            source_file=self._simple_nx.master_file,
+            source_data_path=self._simple_nx.entry,
+            target_file=self._nx_with_virtual_dataset.master_file,
+            target_data_path=self._nx_with_virtual_dataset.entry,
+            copy_other_data=True,
+        )
+        self._patch_nxtomo_flags(self._nx_with_virtual_dataset)
+
+        nx_with_vds_path_and_links = os.path.join(
+            self.tmpdir, "case_with_vds_and_links"
+        )
+        self._nx_with_virtual_dataset_with_link = MockNXtomo(
+            scan_path=nx_with_vds_path_and_links,
+            n_proj=0,
+            n_ini_proj=0,
+            create_ini_dark=False,
+            create_ini_flat=False,
+            create_final_flat=False,
+            dim=self.dim,
+        ).scan
+        self._create_vds(
+            source_file=self._simple_nx.master_file,
+            source_data_path=self._simple_nx.entry,
+            target_file=self._nx_with_virtual_dataset_with_link.master_file,
+            target_data_path=self._nx_with_virtual_dataset_with_link.entry,
+            copy_other_data=True,
+        )
+        self._patch_nxtomo_flags(self._nx_with_virtual_dataset_with_link)
+
+        # create dark
+        self.start_dark = (
+            numpy.random.random((self.dim * self.dim))
+            .reshape(1, self.dim, self.dim)
+            .astype("f")
+        )
+        self.start_dark_file = os.path.join(self.tmpdir, "dark.hdf5")
+        self.start_dark_entry = "data"
+        self.start_dark_url = self._save_raw(
+            data=self.start_dark,
+            entry=self.start_dark_entry,
+            file_path=self.start_dark_file,
+        )
+
+        self.end_dark = (
+            numpy.random.random((self.dim * self.dim * 2))
+            .reshape(2, self.dim, self.dim)
+            .astype("f")
+        )
+        self.end_dark_file = os.path.join(self.tmpdir, "dark.hdf5")
+        self.end_dark_entry = "data2"
+        self.end_dark_url = self._save_raw(
+            data=self.end_dark, entry=self.end_dark_entry, file_path=self.end_dark_file
+        )
+
+        # create flats
+        self.start_flat = (
+            numpy.random.random((self.dim * self.dim * 3))
+            .reshape(3, self.dim, self.dim)
+            .astype("f")
+        )
+        self.start_flat_file = os.path.join(self.tmpdir, "start_flat.hdf5")
+        self.start_flat_entry = "/root/flat"
+        self.start_flat_url = self._save_raw(
+            data=self.start_flat,
+            entry=self.start_flat_entry,
+            file_path=self.start_flat_file,
+        )
+
+        self.end_flat = (
+            numpy.random.random((self.dim * self.dim))
+            .reshape(1, self.dim, self.dim)
+            .astype("f")
+        )
+        # save the end flat in the simple case file to insure all cases are
+        # consider
+        self.end_flat_file = self._simple_nx.master_file
+        self.end_flat_entry = "flat"
+        self.end_flat_url = self._save_raw(
+            data=self.end_flat, entry=self.end_flat_entry, file_path=self.end_flat_file
+        )
+
+    def _save_raw(self, data, entry, file_path) -> DataUrl:
+        with h5py.File(file_path, mode="a") as h5s:
+            h5s[entry] = data
+        return DataUrl(file_path=file_path, data_path=entry, scheme="silx")
+
+    def _create_vds(
+        self,
+        source_file: str,
+        source_data_path: str,
+        target_file: str,
+        target_data_path: str,
+        copy_other_data: bool,
+    ):
+        """Create virtual dataset and links from source to target
+
+        :param str source_file:
+        :param str source_data_path:
+        :param str target_file:
+        :param str target_data_path:
+        :param bool copy_other_data: we want to create two cases: one copying
+                                     datasets 'image_key'... and the other
+                                     one linking them. Might have a difference
+                                     of behavior when overwriting for example
+        """
+        assert source_file != target_file, "file should be different"
+        # link data
+        n_frames = 0
+        # for now we only consider the original data
+        with h5py.File(source_file, mode="r") as o_h5s:
+            old_path = os.path.join(source_data_path, "instrument", "detector", "data")
+            n_frames += o_h5s[old_path].shape[0]
+            shape = o_h5s[old_path].shape
+            data_type = o_h5s[old_path].dtype
+
+            layout = h5py.VirtualLayout(shape=shape, dtype=data_type)
+            assert os.path.exists(source_file)
+            with h5py.File(source_file, mode="r") as ppp:
+                assert source_data_path in ppp
+            layout[:] = h5py.VirtualSource(path_or_dataset=o_h5s[old_path])
+
+            det_path = os.path.join(target_data_path, "instrument", "detector")
+            with h5py.File(target_file, mode="a") as h5s:
+                detector_node = h5s.require_group(det_path)
+                detector_node.create_virtual_dataset("data", layout, fillvalue=-5)
+
+        for path in (
+            os.path.join("instrument", "detector", "image_key"),
+            os.path.join("instrument", "detector", "image_key_control"),
+            os.path.join("instrument", "detector", "count_time"),
+            os.path.join("sample", "rotation_angle"),
+        ):
+            old_path = os.path.join(source_data_path, path)
+            new_path = os.path.join(target_data_path, path)
+            with h5py.File(target_file, mode="a") as h5s:
+                if copy_other_data:
+                    with h5py.File(source_file, mode="r") as o_h5s:
+                        if new_path in h5s:
+                            del h5s[new_path]
+                        h5s[new_path] = h5py_read_dataset(o_h5s[old_path])
+                elif source_file == target_file:
+                    h5s[new_path] = h5py.SoftLink(old_path)
+                else:
+                    relpath = os.path.relpath(source_file, os.path.dirname(target_file))
+                    h5s[new_path] = h5py.ExternalLink(relpath, old_path)
+
+    def _patch_nxtomo_flags(self, scan):
+        """Insure necessary flags are here"""
+        with h5py.File(scan.master_file, mode="a") as h5s:
+            instrument_path = os.path.join(scan.entry, "instrument")
+            instrument_node = h5s.require_group(instrument_path)
+            if "NX_class" not in instrument_node.attrs:
+                instrument_node.attrs["NX_class"] = "NXinstrument"
+            detector_node = instrument_node.require_group("detector")
+            if "NX_class" not in detector_node.attrs:
+                detector_node.attrs["NX_class"] = "NXdetector"
+            if "data" in instrument_node:
+                if "interpretation" not in instrument_node.attrs:
+                    instrument_node["data"].attrs["interpretation"] = "image"
+
+            sample_path = os.path.join(scan.entry, "sample")
+            sample_node = h5s.require_group(sample_path)
+            if "NX_class" not in sample_node:
+                sample_node.attrs["NX_class"] = "NXsample"
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.tmpdir)
+
+
+class TestDaarkFlatPatch(BaseTestAddDarkAndFlats):
+    def test(self):
+        scan = NXtomoScan(
+            self._nx_with_virtual_dataset_with_link.master_file,
+            self._nx_with_virtual_dataset_with_link.entry,
+        )
+        process = DarkFlatPatchTask(
+            inputs={
+                "data": scan,
+                "serialize_output_data": False,
+                "configuration": {
+                    "darks_start": self.start_dark_url,
+                },
+            },
+        )
+        process.definition()
+        process.program_version()
+        process.program_name()
+        # TODO: properties should be part of inputs
+        process.run()

tomwer/core/process/edit/test/test_imagekey_editor.py

@@ -0,0 +1,125 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2017-2021 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "18/06/2021"
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import pytest
+from nxtomo.nxobject.nxdetector import ImageKey
+
+from tomwer.core.process.edit.imagekeyeditor import (
+    ImageKeyEditorTask,
+    ImageKeyUpgraderTask,
+)
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+from tomwer.core.utils.scanutils import MockNXtomo
+
+
+class TestImageKeyEditor(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        dim = 10
+        mock = MockNXtomo(
+            scan_path=os.path.join(self.tempdir, "scan1"),
+            n_proj=10,
+            n_ini_proj=10,
+            scan_range=180,
+            dim=dim,
+        )
+        self.scan = mock.scan
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test(self):
+        process = ImageKeyEditorTask(
+            inputs={
+                "data": self.scan,
+                "serialize_output_data": False,
+                "configuration": {
+                    "modifications": {
+                        0: ImageKey.INVALID.value,
+                        3: ImageKey.ALIGNMENT.value,
+                    },
+                },
+            }
+        )
+        process.definition()
+        process.program_version()
+        process.program_name()
+        process.run()
+
+
+def test_ImageKeyUpgraderTask(tmp_path):
+    """
+    test ImageKeyUpgraderTask task
+    """
+    test_dir = tmp_path / "test_image_key_upgrader"
+    os.makedirs(test_dir)
+    scan = MockNXtomo(
+        scan_path=test_dir,
+        n_proj=20,
+        n_ini_proj=20,
+        create_ini_dark=False,
+    ).scan
+
+    operations = {
+        ImageKey.PROJECTION: ImageKey.DARK_FIELD,
+    }
+
+    with pytest.raises(TypeError):
+        ImageKeyUpgraderTask(
+            inputs={
+                "data": None,
+                "operations": operations,
+            },
+        ).run()
+
+    with pytest.raises(TypeError):
+        ImageKeyUpgraderTask(
+            inputs={
+                "data": scan,
+                "operations": None,
+            },
+        ).run()
+
+    task = ImageKeyUpgraderTask(
+        inputs={
+            "data": scan,
+            "operations": operations,
+        },
+    )
+    task.run()
+
+    scan = NXtomoScan(scan.master_file, scan.entry)
+    assert len(scan.projections) == 0
+    assert len(scan.darks) == 20

tomwer/core/process/icat/test/test_create_screenshots.py

@@ -0,0 +1,98 @@
+import numpy
+from tomwer.tests.conftest import nxtomo_scan_180  # noqa F811
+from tomwer.core.process.icat.createscreenshots import (
+    CreateRawDataScreenshotsTask,
+    select_angles,
+)
+
+
+def test_CreateRawDataScreenshotsTask(nxtomo_scan_180):  # noqa F811
+    """
+    test CreateRawDataScreenshotsTask task
+    """
+    # test nothing requested
+    task = CreateRawDataScreenshotsTask(
+        inputs={
+            "data": nxtomo_scan_180,
+            "raw_projections_required": False,
+            "raw_darks_required": False,
+            "raw_flats_required": False,
+        }
+    )
+    task.run()
+    assert len(task.outputs.screenshots.screenshots) == 0
+
+    # test only dark requested
+    task = CreateRawDataScreenshotsTask(
+        inputs={
+            "data": nxtomo_scan_180,
+            "raw_projections_required": False,
+            "raw_darks_required": False,
+            "raw_flats_required": True,
+        }
+    )
+    task.run()
+    assert len(task.outputs.screenshots.screenshots) == 1
+
+    # test only flat requested
+    task = CreateRawDataScreenshotsTask(
+        inputs={
+            "data": nxtomo_scan_180,
+            "raw_projections_required": False,
+            "raw_darks_required": True,
+            "raw_flats_required": False,
+        }
+    )
+    task.run()
+    assert len(task.outputs.screenshots.screenshots) == 1
+
+    # test only projection requested
+    task = CreateRawDataScreenshotsTask(
+        inputs={
+            "data": nxtomo_scan_180,
+            "raw_projections_required": True,
+            "raw_darks_required": False,
+            "raw_flats_required": False,
+        }
+    )
+    task.run()
+    assert len(task.outputs.screenshots.screenshots) == 2
+
+    # test all requested
+    task = CreateRawDataScreenshotsTask(
+        inputs={
+            "data": nxtomo_scan_180,
+            "raw_projections_required": True,
+            "raw_darks_required": True,
+            "raw_flats_required": True,
+            "raw_projections_each": 10,
+        }
+    )
+    task.run()
+    assert len(task.outputs.screenshots.screenshots) == 12
+
+
+def test_select_angles():
+    """test the select_angles function"""
+    numpy.testing.assert_allclose(
+        select_angles(
+            numpy.linspace(start=-20, stop=40, num=201, endpoint=True), each_angle=20
+        ),
+        numpy.array([-20, 0.1, 19.9, 40]),
+    )
+
+    assert select_angles((), each_angle=20) == ()
+    assert select_angles((10,), each_angle=10) == (10,)
+    angles = select_angles(
+        numpy.linspace(start=0, stop=10, num=400, endpoint=False), each_angle=1
+    )
+    assert len(angles) == 11
+    assert numpy.isclose(angles[0], 0, atol=0.1)
+    assert numpy.isclose(angles[-1], 10, atol=0.1)
+
+
+# def test_CreateRawDataScreenshotsWidgetWorkflow():
+#     """
+#     test CreateRawDataScreenshotsTask with a MoveScreenshotsToGalleryTask task to make sure the two work well together
+#     """
+#     raise NotImplementedError