tomwer 1.3.26__py3-none-any.whl → 1.4.0__py3-none-any.whl

tomwer/core/process/reconstruction/nabu/test/test_nabunormalization.py

@@ -0,0 +1,199 @@
+# coding: utf-8
+from __future__ import annotations
+
+import configparser
+from collections import namedtuple
+
+import numpy
+import pytest
+from silx.io.url import DataUrl
+from tomoscan.io import HDF5File
+from nabu.pipeline.config import get_default_nabu_config
+from nabu.pipeline.fullfield.nabu_config import (
+    nabu_config as nabu_fullfield_default_config,
+)
+from tomwer.core.process.reconstruction.nabu import settings as nabu_settings
+from tomwer.core.process.reconstruction.nabu.nabuslices import NabuSlicesTask
+from tomwer.core.process.reconstruction.normalization.normalization import (
+    SinoNormalizationTask,
+)
+from tomwer.core.process.reconstruction.normalization.params import (
+    SinoNormalizationParams,
+)
+from tomwer.core.utils.scanutils import MockNXtomo
+from tomwer.core.process.reconstruction.output import (
+    PROCESS_FOLDER_RECONSTRUCTED_SLICES,
+)
+
+try:
+    import nabu
+except ImportError:
+    has_nabu = False
+else:
+    has_nabu = True
+import os
+
+_norm_setting = namedtuple("_norm_setting", ["method", "source", "extra_infos"])
+
+_nabu_norm_field = namedtuple(
+    "_nabu_norm_field",
+    ["sino_normalization", "has_sino_normalization_file"],
+)
+
+
+normalization_config_test = (
+    # no normalization
+    (
+        _norm_setting(method="none", source=None, extra_infos=None),
+        _nabu_norm_field(
+            sino_normalization="",
+            has_sino_normalization_file=False,
+        ),
+    ),
+    # chebyshev normalization
+    (
+        _norm_setting(method="chebyshev", source=None, extra_infos=None),
+        _nabu_norm_field(
+            sino_normalization="chebyshev",
+            has_sino_normalization_file=False,
+        ),
+    ),
+    # divide by a scalar
+    (
+        _norm_setting(method="division", source="scalar", extra_infos={"value": 12.0}),
+        _nabu_norm_field(
+            sino_normalization="division",
+            has_sino_normalization_file=True,
+        ),
+    ),
+    # substract a roi
+    (
+        _norm_setting(
+            method="subtraction",
+            source="manual ROI",
+            extra_infos={
+                "start_x": 0.0,
+                "end_x": 1.0,
+                "start_y": 0.0,
+                "end_y": 1.0,
+                "calc_fct": "mean",
+                "calc_area": "volume",
+                "calc_method": "scalar",
+            },
+        ),
+        _nabu_norm_field(
+            sino_normalization="subtraction",
+            has_sino_normalization_file=True,
+        ),
+    ),
+    # substract from a dataset
+    (
+        _norm_setting(
+            method="subtraction",
+            source="from dataset",
+            extra_infos={
+                "dataset_url": DataUrl(
+                    file_path="random_dataset.hdf5",
+                    data_path="data",
+                    scheme="silx",
+                ),
+                "calc_fct": "median",
+                "calc_area": "volume",
+                "calc_method": "scalar",
+            },
+        ),
+        _nabu_norm_field(
+            sino_normalization="subtraction",
+            has_sino_normalization_file=True,
+        ),
+    ),
+)
+
+
+@pytest.mark.skipif(
+    (not has_nabu or nabu.version < "2022.2"),
+    reason="nabu not available or the current version doesn't handle sino normlaization yet",
+)
+@pytest.mark.parametrize(
+    "norm_setting, expected_nabu_conf", [item for item in normalization_config_test]
+)
+def test_normalization(norm_setting, expected_nabu_conf, tmp_path):
+    """
+    Insure normalization is correctly provided to nabu configuration file
+    For this run a normalization process followed by a nabu process.
+    """
+    scan_dir = tmp_path / "scan"
+    scan_dir.mkdir()
+
+    nabu_cfg_folders = os.path.join(scan_dir, nabu_settings.NABU_CFG_FILE_FOLDER)
+    os.makedirs(nabu_cfg_folders, exist_ok=True)
+
+    random_dataset_file = os.path.join(nabu_cfg_folders, "random_dataset.hdf5")
+
+    # create a random dataset if necessary
+    with HDF5File(random_dataset_file, mode="w") as h5f:
+        h5f["data"] = numpy.ones((20, 20))
+
+    mock = MockNXtomo(
+        scan_path=scan_dir,
+        n_proj=10,
+        n_ini_proj=10,
+        scan_range=180,
+        dim=20,
+    )
+    scan = mock.scan
+
+    cfg_folder = os.path.join(
+        str(scan_dir), PROCESS_FOLDER_RECONSTRUCTED_SLICES, "nabu_cfg_files"
+    )
+    cfg_file = os.path.join(cfg_folder, "entry_scan.cfg")
+    assert not os.path.exists(cfg_file)
+    norm_params = SinoNormalizationParams(
+        method=norm_setting.method,
+        source=norm_setting.source,
+        extra_infos=norm_setting.extra_infos,
+    )
+
+    normalization = SinoNormalizationTask(
+        inputs={
+            "data": scan,
+            "configuration": norm_params.to_dict(),
+            "serialize_output_data": False,
+        },
+        varinfo=None,
+    )
+    normalization.run()
+
+    # insure the method is style valid
+    assert scan.intensity_normalization.method.value == norm_setting.method
+    assert (
+        "tomwer_processing_res_code" in scan.intensity_normalization.get_extra_infos()
+    )
+
+    process = NabuSlicesTask(
+        inputs={
+            "data": scan,
+            "dry_run": True,
+            "nabu_params": get_default_nabu_config(nabu_fullfield_default_config),
+            "serialize_output_data": False,
+        },
+        varinfo=None,
+    )
+    process.run()
+    assert os.path.exists(cfg_file)
+
+    configuration = configparser.ConfigParser(allow_no_value=True)
+    configuration.read(cfg_file)
+
+    preproc_section = configuration["preproc"]
+    sino_normalization = preproc_section.get("sino_normalization", "")
+    sino_normalization_file = preproc_section.get("sino_normalization_file", "")
+    assert sino_normalization == expected_nabu_conf.sino_normalization
+    if expected_nabu_conf.has_sino_normalization_file:
+        url = DataUrl(path=sino_normalization_file)
+        assert url.is_valid()
+    else:
+        assert sino_normalization_file == ""
+    assert (
+        "tomwer_processing_res_code" in scan.intensity_normalization.get_extra_infos()
+    )

tomwer/core/process/reconstruction/nabu/utils.py

@@ -1,37 +1,9 @@
 # coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2020 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "06/08/2020"
-
+from __future__ import annotations
 
 import datetime
 import logging
 import os
-import typing
 from contextlib import AbstractContextManager
 
 from nabu.pipeline.config import generate_nabu_configfile, parse_nabu_config_file
@@ -78,7 +50,7 @@ def retrieve_lst_of_value_from_str(my_string: str, type_) -> tuple:
     Return a list of value from a string like '12,23' or '(12, 23)',
     '[12;23]', '12;23' or with the pattern from:to:step like '0:10:1'
 
-    :param str mystring:
+    :param mystring:
     :return: list of single value
     """
     if not isinstance(my_string, str):
@@ -113,31 +85,10 @@ def retrieve_lst_of_value_from_str(my_string: str, type_) -> tuple:
         return tuple(res)
 
 
-def get_nabu_resources_desc(scan: TomwerScanBase, method, workers=1) -> dict:
-    """
-    Create the descriptor of nabu's resources
-
-    :param TomwerScanBase scan:
-    :param str method:
-    :return: nabu's description of resources to be used
-    """
-    assert isinstance(scan, TomwerScanBase)
-    res = {
-        "method": method,
-        "cpu_workers": workers,
-        "partition": "gpu",
-        "memory_per_node": "90%",
-        "threads_per_node": "100%",
-    }
-    return res
-
-
-def get_nabu_about_desc(overwrite) -> dict:
+def get_nabu_about_desc(overwrite: bool) -> dict:
     """
-    Create the description for nabu's about
+    Create the description for nabu's 'about'
 
-    :param self:
-    :return:
     """
     return {"overwrite_results": str(bool(overwrite))}
 
@@ -147,7 +98,7 @@ def get_recons_volume_identifier(
     location: str,
     file_format: str,
     scan: TomwerScanBase,
-    slice_index: typing.Union[int, None],
+    slice_index: int | None,
     axis: NabuPlane,
 ) -> tuple:
     """
@@ -240,11 +191,11 @@ def get_multi_cor_recons_volume_identifiers(
     """
     util to retrieve Volumes (identifier) reconstructed by nabu-multicor
 
-    :param TomwerScanBase scan: scam processed by the nabu-multicor
-    :param str location: location of the files
-    :param tuple cors: cors for which we want the reconstructed slices. As this extension is created by nabu
+    :param scan: scam processed by the nabu-multicor
+    :param location: location of the files
+    :param cors: cors for which we want the reconstructed slices. As this extension is created by nabu
                        the cor reference is in absolute.
-    :param str file_format: file format of the reconstruction
+    :param file_format: file format of the reconstruction
 
     :return: a dict with absolute cor value as key and the Volume identifier as value
     """
@@ -442,7 +393,7 @@ class RingCorrectionMethod(_Enum):
     MEAN_DIVISION = "mean-division"
 
 
-def nabu_std_err_has_error(errs: typing.Optional[bytes]):
+def nabu_std_err_has_error(errs: bytes | None):
     """
     small util to parse stderr where some warning can exists.
     But I don't think we want to catch all warnings from nabu so this is a (bad) concession
@@ -513,7 +464,7 @@ def update_cfg_file_after_transfer(config_file_path, old_path, new_path):
 
 
 def slice_index_to_int(
-    slice_index: typing.Union[int, str],
+    slice_index: int | str,
     scan: TomwerScanBase,
     axis=NabuPlane.XY,
 ) -> int:

tomwer/core/process/reconstruction/normalization/normalization.py

@@ -1,38 +1,9 @@
 # coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
+
 """
 material for radio and sinogram normalization
 """
-
-
-__authors__ = [
-    "H. Payno",
-]
-__license__ = "MIT"
-__date__ = "25/06/2021"
-
+from __future__ import annotations
 
 import functools
 import logging
@@ -208,7 +179,6 @@ class SinoNormalizationTask(
         :param end_y:
         :param calc_area:
         :return:
-        :rtype: dict
         """
         if start_x is None or end_x is None or start_y is None or end_y is None:
             raise ValueError("min_x, max_x, min_y and max_y should all be provided.")

tomwer/core/process/reconstruction/normalization/params.py

@@ -1,40 +1,8 @@
 # coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
 """
 material for radio and sinogram normalization
 """
-
-
-__authors__ = [
-    "H. Payno",
-]
-__license__ = "MIT"
-__date__ = "25/06/2021"
-
-
-import typing
+from __future__ import annotations
 
 from silx.utils.enum import Enum as _Enum
 from tomoscan.normalization import Method
@@ -70,7 +38,7 @@ class _DatasetInfos:
         return self._scope
 
     @scope.setter
-    def scope(self, scope: typing.Union[str, _DatasetScope]):
+    def scope(self, scope: str | _DatasetScope):
         self._scope = _DatasetScope.from_value(scope)
 
     @property
@@ -115,7 +83,7 @@ class SinoNormalizationParams:
         return self._method
 
     @method.setter
-    def method(self, method: typing.Union[str, Method, None]):
+    def method(self, method: str | Method | None):
         if method is None:
             method = Method.NONE
         self._method = Method.from_value(method)

tomwer/core/process/reconstruction/output.py

@@ -4,22 +4,17 @@ import logging
 from silx.utils.enum import Enum as _Enum
 from tomwer.io.utils.raw_and_processed_data import (
     to_processed_data_path,
-    to_raw_data_path,
 )
 from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.process.output import ProcessDataOutputDirMode
 from tomwer.core.utils.scanutils import format_output_location
 
 _logger = logging.getLogger(__name__)
 
 
-PROCESS_FOLDER_NAME = "reconstructed_volumes"
+PROCESS_FOLDER_RECONSTRUCTED_VOLUMES = "reconstructed_volumes"
 
-
-class ProcessDataOutputDirMode(_Enum):
-    IN_SCAN_FOLDER = "same folder as scan"
-    PROCESSED_DATA_FOLDER = "PROCESSED_DATA folder"
-    RAW_DATA_FOLDER = "RAW_DATA folder"
-    OTHER = "other"
+PROCESS_FOLDER_RECONSTRUCTED_SLICES = "reconstructed_slices"
 
 
 class NabuOutputFileFormat(_Enum):
@@ -59,8 +54,16 @@ def get_output_folder_from_scan(
     nabu_location: str | None,
     file_basename: str,
     file_format: NabuOutputFileFormat,
+    processed_data_folder_name: str | None,
 ) -> tuple[str, str]:
     """
+
+    :param mode: output mode, should save this to raw data, processed data...
+    :param scan: scan for which we want to get the output folder
+    :param nabu_location: output location provided by user (in case mode is 'other')
+    :param file_basename: file basename to take in order to create output files
+    :param file_format: output volume format (edf...)
+    :param processed_data_folder_name: name of the processed data folder. Like 'reconstructed_volumes' or 'reconstructed_slices'...
     :return: (location, location_cfg_files). Location is the nabu configuration field 'output/location' 'location_cfg_files' is the information on where to save the nabu configuration file
 
     """
@@ -68,27 +71,22 @@ def get_output_folder_from_scan(
     file_format = NabuOutputFileFormat.from_value(file_format)
 
     if output_mode is ProcessDataOutputDirMode.OTHER and nabu_location in ("", None):
-        _logger.info(
-            "output dir requested is other bit no path provided. Fall back on the output dir to the scan folder"
+        _logger.error(
+            "'other' output dir requested but no path provided. Fall back on the output dir to the scan folder"
         )
         # note: this is only an info because we expect to pass by this one for all .ows configuration (before 1.3 version)
         # as there was no different option by the time
         output_mode = ProcessDataOutputDirMode.IN_SCAN_FOLDER
 
     if output_mode is ProcessDataOutputDirMode.OTHER:
-        assert nabu_location not in (
-            "",
-            None,
-        ), "nabu_location not provided when expected"
         location = format_output_location(nabu_location, scan=scan)
         location_cfg_files = location
     elif output_mode in (
         ProcessDataOutputDirMode.IN_SCAN_FOLDER,
         ProcessDataOutputDirMode.PROCESSED_DATA_FOLDER,
-        ProcessDataOutputDirMode.RAW_DATA_FOLDER,
     ):
         # otherwise default location will be the data root level
-        location = os.path.join(scan.path, PROCESS_FOLDER_NAME)
+        location = os.path.join(scan.path, processed_data_folder_name)
         location_cfg_files = location
         if file_format in (
             NabuOutputFileFormat.EDF.value,
@@ -99,9 +97,6 @@ def get_output_folder_from_scan(
         if output_mode is ProcessDataOutputDirMode.PROCESSED_DATA_FOLDER:
             location = to_processed_data_path(location)
             location_cfg_files = to_processed_data_path(location_cfg_files)
-        if output_mode is ProcessDataOutputDirMode.RAW_DATA_FOLDER:
-            location = to_raw_data_path(location)
-            location_cfg_files = to_raw_data_path(location_cfg_files)
     else:
         raise NotImplementedError(f"mode {output_mode.value} is not implemented yet")
 

tomwer/core/process/reconstruction/paramsbase.py

@@ -1,32 +1,5 @@
 # coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "05/03/2019"
-
+from __future__ import annotations
 
 import copy
 import itertools
@@ -109,16 +82,14 @@ class _ReconsParam(object):
             self.__unmanaged_params[parameter] = value
             self.changed()
 
-    def to_unique_recons_set(self, as_to_dict=False) -> tuple:
+    def to_unique_recons_set(self, as_to_dict: bool = False) -> tuple:
         """
 
         :return: a tuple of unique reconstruction parameters in order to have
                  each parameter as a single value (and no list)
-        :param bool as_to_dict: True if we want to get the same value as if we
+        :param as_to_dict: True if we want to get the same value as if we
                                 where exporting it with to_dict function. There
                                 is some cast in the to_dict function.
-        :type: Union[tuple, None]
-        :rtype: tuple
         """
         params_list = OrderedDict()
         dict_values = self.to_dict()
@@ -174,7 +145,7 @@ class _ReconsParam(object):
         """
         copy parameters value from other_rp
 
-        :param _ReconsParam: reconsparam to copy
+        :param other_rp: reconsparam to copy
         """
         if other_rp is None:
             return

tomwer/core/process/reconstruction/saaxis/params.py

@@ -1,35 +1,7 @@
-# coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "18/02/2021"
-
-
-from collections.abc import Iterable
-from typing import Union
+from __future__ import annotations
+
+
+from typing import Iterable
 
 import numpy
 from silx.utils.enum import Enum as _Enum
@@ -88,7 +60,7 @@ class SAAxisParams(SABaseParams):
         self._mode = mode
 
     @property
-    def image_width(self) -> Union[None, float]:
+    def image_width(self) -> float | None:
         return self._image_width
 
     @image_width.setter