tomwer 1.3.6__py3-none-any.whl → 1.3.7__py3-none-any.whl

tomwer/io/utils/test/test_utils.py

@@ -0,0 +1,92 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+"""
+contains utils for inputs and outputs
+"""
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "15/12/2021"
+
+import os
+
+import h5py
+import numpy
+from silx.io.url import DataUrl
+from tomoscan.io import HDF5File
+
+from tomwer.core.utils import scanutils
+from tomwer.io.utils import (
+    get_default_directory,
+    get_linked_files_with_entry,
+    get_slice_data,
+)
+
+
+def test_get_linked_files_with_entry(tmp_path):
+    """test get_linked_files_with_entry function"""
+    dir_test = tmp_path / "sub"
+    dir_test.mkdir()
+
+    layout = h5py.VirtualLayout(shape=(4, 10, 10), dtype="i4")
+    for i_file in range(4):
+        file_name = os.path.join(dir_test, f"file_{i_file}.hdf5")
+        with HDF5File(file_name, "w") as h5s:
+            h5s.create_dataset("data", (10, 10), "i4", numpy.ones((10, 10)))
+            vsource = h5py.VirtualSource(file_name, "data", shape=(10, 10))
+            layout[i_file] = vsource
+
+    test_with_vds = os.path.join(dir_test, "file_with_vds")
+    with HDF5File(test_with_vds, "w") as h5s:
+        h5s.create_virtual_dataset("vdata", layout, fillvalue=-1)
+
+    assert len(get_linked_files_with_entry(test_with_vds, "vdata")) == 4
+
+
+def test_get_default_directory():
+    get_default_directory()
+
+
+def test_get_slice_data_vol(tmp_path):
+    """test load of a .vol file"""
+    dir_test = tmp_path / "test_vol"
+    dir_test.mkdir()
+    vol_file_path = os.path.join(dir_test, "volume.vol")
+    vol_info_file_path = os.path.join(dir_test, "volume.vol.info")
+
+    shape = (4, 50, 20)
+    data = numpy.ones(shape)
+    data.astype(numpy.float32).tofile(vol_file_path)
+    scanutils.MockEDF._createVolInfoFile(
+        filePath=vol_info_file_path,
+        shape=shape,
+    )
+
+    vol = get_slice_data(DataUrl(file_path=vol_file_path))
+    assert vol is not None
+    assert vol.shape == shape
+    vol = get_slice_data(DataUrl(file_path=vol_info_file_path))
+    assert vol is not None
+    assert vol.shape == shape

tomwer/synctools/stacks/reconstruction/castvolume.py

@@ -94,11 +94,26 @@ class CastVolumeProcessStack(FIFO, qt.QObject):
             self.scan_ready(scan=data)
         else:
             self._data_currently_computed = data
-            self._computationThread.init(data=data, configuration=configuration)
-            # need to manage connect before starting it because
-            fct_callback = functools.partial(self._end_threaded_computation, callback)
-            self._computationThread.finished.connect(fct_callback)
-            self._computationThread.start()
+            try:
+                self._computationThread.init(data=data, configuration=configuration)
+            except ValueError as e:
+                # initialization can fail (for example for cast volume is there is no volume or be case this will raise an error)
+                # then we want to keep the thread active
+                self._data_currently_computed = None
+                ProcessManager().notify_dataset_state(
+                    dataset=data, process=self._process_id, state=DatasetState.SKIPPED
+                )
+                _logger.processSkipped(f"thread initialization failed. Error is {e}")
+                if callback is not None:
+                    callback()
+                self.scan_ready(scan=data)
+            else:
+                # need to manage connect before starting it because
+                fct_callback = functools.partial(
+                    self._end_threaded_computation, callback
+                )
+                self._computationThread.finished.connect(fct_callback)
+                self._computationThread.start()
 
     def _end_computation(self, data, future_tomo_obj, callback):
         """

tomwer/tests/test_ewoks/test_conversion.py

@@ -0,0 +1,104 @@
+import pytest
+from ewoksorange.bindings import owsconvert
+from tomwer.tests.datasets import TomwerCIDatasets
+
+
+ows_files_to_convert = ("from_nxtomo_to_volume.ows",)
+
+
+@pytest.mark.parametrize("orange_file_name", ows_files_to_convert)
+def test_conversion(orange_file_name):
+    """test orange file can be converted to ewoks Graph"""
+    scheme_file = TomwerCIDatasets.get_dataset(f"workflows/{orange_file_name}")
+    graph = owsconvert.ows_to_ewoks(scheme_file)
+    # make sure settings where correctly saved
+    assert len(graph.graph.nodes) == 7
+    assert len(graph.graph.edges) == 6
+    # make sure the settings are properly loaded
+    for node in graph.graph.nodes.values():
+        assert node["task_type"] == "class"
+        assert node["task_identifier"].startswith("tomwer")
+    # check settings of some specific nodes
+    # dark and flat
+    dark_flat_node = graph.graph.nodes["1"]
+    assert (
+        dark_flat_node["task_identifier"]
+        == "tomwer.core.process.reconstruction.darkref.darkrefscopy.DarkRefsCopy"
+    )
+    assert "default_inputs" in dark_flat_node
+
+    def fileter_dark_ref_params(my_dict):
+        return my_dict["name"] == "dark_ref_params"
+
+    dark_ref_params_settings = tuple(
+        filter(fileter_dark_ref_params, dark_flat_node["default_inputs"])
+    )
+    assert len(dark_ref_params_settings) == 1
+    dark_ref_params_as_dict = dark_ref_params_settings[0]["value"]
+
+    assert "DOWHEN" in dark_ref_params_as_dict
+    assert "DARKCAL" in dark_ref_params_as_dict
+    assert "DKFILE" in dark_ref_params_as_dict
+
+    # axis
+    axis_node = graph.graph.nodes["2"]
+    assert (
+        axis_node["task_identifier"]
+        == "tomwer.core.process.reconstruction.axis.axis.AxisTask"
+    )
+    assert "default_inputs" in axis_node
+    assert len(axis_node["default_inputs"]) == 2  # axis_params + gui metadata
+
+    def filter_axis_params(my_dict):
+        return my_dict["name"] == "axis_params"
+
+    axis_params_settings = tuple(
+        filter(filter_axis_params, axis_node["default_inputs"])
+    )
+    assert len(axis_params_settings) == 1
+    axis_params_as_dict = axis_params_settings[0]["value"]
+
+    assert "MODE" in axis_params_as_dict
+    assert "POSITION_VALUE" in axis_params_as_dict
+    assert "SINOGRAM_LINE" in axis_params_as_dict
+
+    # nabu slice
+    nabu_slice_node = graph.graph.nodes["3"]
+    assert (
+        nabu_slice_node["task_identifier"]
+        == "tomwer.core.process.reconstruction.nabu.nabuslices.NabuSlicesTask"
+    )
+    assert "default_inputs" in axis_node
+
+    def filter_nabu_params(my_dict):
+        return my_dict["name"] == "nabu_params"
+
+    nabu_params_settings = tuple(
+        filter(filter_nabu_params, nabu_slice_node["default_inputs"])
+    )
+    assert len(nabu_params_settings) == 1
+    nabu_params_as_dict = nabu_params_settings[0]["value"]
+
+    assert "preproc" in nabu_params_as_dict
+    assert "phase" in nabu_params_as_dict
+    assert "tomwer_slices" in nabu_params_as_dict
+
+    # nabu volume
+    nabu_volume_node = graph.graph.nodes["4"]
+    assert (
+        nabu_volume_node["task_identifier"]
+        == "tomwer.core.process.reconstruction.nabu.nabuvolume.NabuVolumeTask"
+    )
+    assert "default_inputs" in axis_node
+
+    def filter_nabu_volume_params(my_dict):
+        return my_dict["name"] == "nabu_volume_params"
+
+    nabu_volume_params_settings = tuple(
+        filter(filter_nabu_volume_params, nabu_volume_node["default_inputs"])
+    )
+    assert len(nabu_volume_params_settings) == 1
+    nabu_volume_params_as_dict = nabu_volume_params_settings[0]["value"]
+
+    assert "start_z" in nabu_volume_params_as_dict
+    assert "end_z" in nabu_volume_params_as_dict

tomwer/tests/test_ewoks/test_single_node_execution.py

@@ -0,0 +1,112 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["C.Nemoz", "H.Payno"]
+__license__ = "MIT"
+__date__ = "18/08/2021"
+
+import os
+import tempfile
+from glob import glob
+
+import pytest
+from ewokscore.graph import analysis, load_graph
+from ewokscore.graph.validate import validate_graph
+
+from tomwer.core.utils.scanutils import HDF5MockContext
+from nabu.pipeline.config import get_default_nabu_config
+from nabu.pipeline.fullfield.nabu_config import (
+    nabu_config as nabu_fullfield_default_config,
+)
+
+try:
+    from nabu.pipeline.fullfield.reconstruction import (  # noqa F401
+        FullFieldReconstructor,
+    )
+except ImportError:
+    try:
+        from nabu.pipeline.fullfield.local_reconstruction import (  # noqa F401
+            ChunkedReconstructor,
+        )
+    except ImportError:
+        has_nabu = False
+    else:
+        has_nabu = True
+else:
+    has_nabu = True
+
+pytest.mark.skipif(condition=not has_nabu, reason="nabu not installed")
+
+
+classes_to_test = {
+    "darkref": "tomwer.core.process.reconstruction.darkref.darkrefs.DarkRefs",
+    "axis": "tomwer.core.process.reconstruction.axis.axis.AxisProcess",
+    "nabu slices": "tomwer.core.process.reconstruction.nabu.nabuslices.NabuSlices",
+}
+
+
+@pytest.mark.parametrize("node_name, node_qual_name", classes_to_test.items())
+def test_single_class_instanciation(node_name, node_qual_name):
+    with HDF5MockContext(
+        scan_path=os.path.join(tempfile.mkdtemp(), "scan_test"), n_proj=100
+    ) as scan:
+        # insure no cfg yet
+        assert len(glob(os.path.join(scan.path, "*.cfg"))) == 0
+
+        graph = load_graph(
+            {
+                "nodes": [
+                    {
+                        "id": node_name,
+                        "task_type": "class",
+                        "task_identifier": node_qual_name,
+                        "default_inputs": [
+                            {
+                                "name": "data",
+                                "value": scan,
+                            },
+                            {
+                                "name": "nabu_params",
+                                "value": get_default_nabu_config(
+                                    nabu_fullfield_default_config
+                                ),
+                            },
+                        ],
+                    },
+                ]
+            }
+        )
+
+        assert graph.is_cyclic is False, "graph is expected to be acyclic"
+        assert analysis.start_nodes(graph.graph) == {
+            node_name
+        }, "graph is expected to have only on start nodes"
+        validate_graph(graph.graph)
+        result = graph.execute(varinfo=None)
+
+        assert analysis.end_nodes(graph.graph) == {
+            node_name
+        }, "graph is expected to have only one end node"
+        assert "data" in result, "data is expected to be part of the output_values"

tomwer/tests/test_ewoks/test_workflows.py

@@ -0,0 +1,160 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["C.Nemoz", "H.Payno"]
+__license__ = "MIT"
+__date__ = "18/08/2021"
+
+
+import os
+import tempfile
+from glob import glob
+
+from ewoks import execute_graph
+from ewokscore.graph import analysis, load_graph
+from ewokscore.graph.validate import validate_graph
+
+from tomwer.core.utils.scanutils import HDF5MockContext
+from tomwer.core.process.reconstruction.output import PROCESS_FOLDER_NAME
+
+try:
+    from nabu.pipeline.fullfield.reconstruction import (  # noqa F401
+        FullFieldReconstructor,
+    )
+except ImportError:
+    try:
+        from nabu.pipeline.fullfield.local_reconstruction import (  # noqa F401
+            ChunkedReconstructor,
+        )
+    except ImportError:
+        has_nabu = False
+    else:
+        has_nabu = True
+else:
+    has_nabu = True
+
+
+def test_simple_workflow_nabu():
+    """Test the workflow: darkref -> axis -> nabu slices -> nabu volume"""
+
+    with HDF5MockContext(
+        scan_path=os.path.join(tempfile.mkdtemp(), "scan_test"), n_proj=100
+    ) as scan:
+        # insure no cfg yet
+        assert len(glob(os.path.join(scan.path, "*.cfg"))) == 0
+        assert not os.path.exists(
+            scan.process_file
+        ), "check tomwer should not have been processed yet"
+
+        graph = load_graph(
+            source={
+                "nodes": [
+                    {
+                        "id": "darkref",
+                        "task_type": "class",
+                        "task_identifier": "tomwer.core.process.reconstruction.darkref.darkrefs.DarkRefs",
+                    },
+                    {
+                        "id": "axis",
+                        "task_type": "class",
+                        "task_identifier": "tomwer.core.process.reconstruction.axis.axis.AxisProcess",
+                        "default_inputs": [
+                            {
+                                "name": "axis_params",
+                                "value": {
+                                    "MODE": "manual",
+                                    "POSITION_VALUE": 0.2,
+                                },
+                            },
+                        ],
+                    },
+                    {
+                        "id": "nabu slices",
+                        "task_type": "class",
+                        "task_identifier": "tomwer.core.process.reconstruction.nabu.nabuslices.NabuSlices",
+                        "dry_run": True,  # as this is a mock dataset avoid to reconstruct it and only check for the .cfg file created
+                        "default_inputs": [
+                            {
+                                "name": "nabu_params",
+                                "value": {"tomwer_slices": 2},
+                            }
+                        ],
+                    },
+                    {
+                        "id": "nabu volume",
+                        "task_type": "class",
+                        "task_identifier": "tomwer.core.process.reconstruction.nabu.nabuvolume.NabuVolume",
+                        "dry_run": True,
+                        "default_inputs": [
+                            {
+                                "name": "dry_run",
+                                "value": True,
+                            }
+                        ],
+                    },
+                ],
+                "links": [
+                    {
+                        "source": "darkref",
+                        "target": "axis",
+                        "map_all_data": True,
+                    },
+                    {
+                        "source": "axis",
+                        "target": "nabu slices",
+                        "data_mapping": [  # same as all arguments but just here to test both
+                            {
+                                "source_output": "data",
+                                "target_input": "data",
+                            },
+                        ],
+                    },
+                    {
+                        "source": "nabu slices",
+                        "target": "nabu volume",
+                        "map_all_data": True,
+                    },
+                ],
+            }
+        )
+        assert graph.is_cyclic is False, "graph is expected to be acyclic"
+        assert analysis.start_nodes(graph=graph.graph) == {
+            "darkref"
+        }, "start node should be a single task `darkref`"
+        validate_graph(graph.graph)
+        result = execute_graph(
+            graph,
+            inputs=[
+                {"id": "darkref", "name": "data", "value": scan},
+            ],
+        )
+        assert analysis.end_nodes(graph=graph.graph) == {
+            "nabu volume"
+        }, "should only have one result nodes"
+        assert "data" in result, f"cannot find `nabu volume` in {result}"
+        assert os.path.exists(scan.process_file), "check tomwer has been processed"
+        assert os.path.exists(
+            os.path.join(scan.path, PROCESS_FOLDER_NAME, "nabu_cfg_files")
+        ), "nabu has not been executed (even in dry mode)"

tomwer/version.py

@@ -77,7 +77,7 @@ RELEASE_LEVEL_VALUE = {
 
 MAJOR = 1
 MINOR = 3
-MICRO = 6
+MICRO = 7
 RELEV = "final"  # <16
 SERIAL = 0  # <16
 

{tomwer-1.3.6.dist-info → tomwer-1.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tomwer
-Version: 1.3.6
+Version: 1.3.7
 Summary: "tomography workflow tools"
 Home-page: https://gitlab.esrf.fr/tomotools/tomwer
 Author: data analysis unit