tomwer 1.3.27__py3-none-any.whl → 1.4.0rc0__py3-none-any.whl

tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -1,42 +1,13 @@
-# coding: utf-8
-###########################################################################
-# Copyright (C) 2016 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-#############################################################################
-
 """contain utils for score process
 """
 
-__authors__ = [
-    "H.Payno",
-]
-__license__ = "MIT"
-__date__ = "27/10/2021"
-
+from __future__ import annotations
 
 import logging
 import os
 import subprocess
 import uuid
-from typing import Iterable, Optional, Union
+from typing import Iterable
 import signal
 import psutil
 import sys
@@ -189,7 +160,7 @@ class ResultSlurmRun(ResultsWithStd):
     def __init__(
         self,
         future_slurm_jobs: tuple,
-        job_id: Optional[int],
+        job_id: int | None,
         *args,
         **kwargs,
     ) -> None:
@@ -202,7 +173,7 @@ class ResultSlurmRun(ResultsWithStd):
         return self.__future_slurm_jobs
 
     @property
-    def job_id(self) -> Optional[int]:
+    def job_id(self) -> int | None:
         return self.__job_id
 
     def __str__(self) -> str:
@@ -226,7 +197,7 @@ class _NabuBaseReconstructor:
         scan: TomwerScanBase,
         dry_run: bool,
         target: Target,
-        cluster_config: Optional[Union[dict, SlurmClusterConfiguration]],
+        cluster_config: dict | SlurmClusterConfiguration | None,
         process_name: str,
         axis: NabuPlane = NabuPlane.XY,
     ) -> None:
@@ -273,6 +244,11 @@ class _NabuBaseReconstructor:
     def axis(self) -> NabuPlane:
         return self._axis
 
+    @property
+    def processed_data_folder_name(self) -> str:
+        """return the specific processed folder name associated to this type of reconstruction."""
+        raise NotImplementedError("Base class")
+
     def only_create_config_file(self):
         """Should we run the reconstruction or only create the configuration file"""
         return False
@@ -290,13 +266,13 @@ class _NabuBaseReconstructor:
         config_to_dump: dict,
         config_file: str,
         file_format: str,
-        info: Optional[str],
+        info: str | None,
         process_name: str,
     ):
         """
         process provided configuration
 
-        :param str info:
+        :param info:
         """
         if self.dry_run is True or self.only_create_config_file():
             return ResultsRun(
@@ -347,11 +323,10 @@ class _NabuBaseReconstructor:
         """
         run locally nabu for a single configuration file.
 
-        :param str conf_file: path to the nabu .cfg file
-        :param str file_format: format of the generated file
-        :param dict config_to_dump: configuration saved in the .cfg as a dictionary
+        :param conf_file: path to the nabu .cfg file
+        :param file_format: format of the generated file
+        :param config_to_dump: configuration saved in the .cfg as a dictionary
         :return: results of the local run
-        :rtype: ResultsLocalRun
         """
         if not has_nabu:
             raise ImportError("Fail to import nabu")
@@ -365,7 +340,7 @@ class _NabuBaseReconstructor:
                 sys.executable,
                 "-m",
                 settings.NABU_FULL_FIELD_APP_PATH,
-                conf_file,
+                f"'{conf_file}'",  # adding ' around file name allows to handle config file with spaces
                 "--gpu_mem_fraction",
                 str(gpu_mem_fraction),
                 "--cpu_mem_fraction",
@@ -415,7 +390,6 @@ class _NabuBaseReconstructor:
         Run a nabu reconstruction on slurm of a single configuration
 
         :return: results of the slurm run
-        :rtype: ResultSlurmRun
         """
         if not isinstance(conf_file, str):
             raise TypeError(f"conf_file is expected to be a strg not {type(conf_file)}")
@@ -535,6 +509,7 @@ class _NabuBaseReconstructor:
                 file_format=config["output"].get(
                     "file_format", NabuOutputFileFormat.HDF5
                 ),
+                processed_data_folder_name=self.processed_data_folder_name,
             )
             # add reconstruction path to the list. scan `reconstruction_paths` register all the existing path where
             # reconstruction are saved in order to be able to browse them all
@@ -647,7 +622,7 @@ class _NabuBaseReconstructor:
 
 
 def dump_normalization_array_for_nabu(
-    scan: TomwerScanBase, output_file: str, array: Union[numpy.ndarray, float, int]
+    scan: TomwerScanBase, output_file: str, array: numpy.ndarray | float | int
 ) -> DataUrl:
     if not isinstance(array, (numpy.ndarray, float, int)):
         raise TypeError(

tomwer/core/process/reconstruction/nabu/nabuscores.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 try:
     from nabu.pipeline.fullfield.reconstruction import (  # noqa F401
         FullFieldReconstructor,
@@ -19,8 +21,9 @@ import os
 import sys
 from copy import deepcopy
 import subprocess
-from typing import Iterable, Optional, Union
+from typing import Iterable
 import numpy
+from tqdm import tqdm
 
 from nabu.pipeline.fullfield.nabu_config import (
     nabu_config as nabu_fullfield_default_config,
@@ -40,7 +43,6 @@ from tomwer.core.process.reconstruction.nabu.utils import (
     get_nabu_multicor_file_prefix,
 )
 from tomwer.core.process.reconstruction.nabu.target import Target
-from tomwer.core.progress import Progress
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.slurm import get_slurm_script_name, is_slurm_available
 from tomwer.utils import docstring
@@ -56,30 +58,29 @@ _logger = logging.getLogger(__name__)
 
 def run_nabu_one_slice_several_config(
     scan: TomwerScanBase,
-    nabu_configs: Union[list, tuple],
-    cluster_config: Optional[dict],
+    nabu_configs: list | tuple,
+    cluster_config: dict | None,
     dry_run: bool,
-    slice_index: Union[int, str],
+    slice_index: int | str,
     file_format: str,
     axis: NabuPlane,
-    advancement: Optional[Progress] = None,
-    process_id: Optional[int] = None,
-    instanciate_classes_only: bool = False,
+    advancement: tqdm | None = None,
+    process_id: int | None = None,
+    instantiate_classes_only: bool = False,
     output_file_prefix_pattern=None,
 ) -> tuple:
     """
     Run several reconstruction of a specific slice.
 
     :param scan: dataset
-    :param Iterable nabu_configs: set of nabu configurations to be run
-    :param bool dry_run:
-    :param int slice_index: slice index to reconstruct or "middle"
-    :param Progress advancement: optional class to display advancement
-    :param Optional[int] process_id: id of the process requesting this computation
-    :param Optional[dict] cluster_config: cluster configuration if
+    :param nabu_configs: set of nabu configurations to be run
+    :param dry_run:
+    :param slice_index: slice index to reconstruct or "middle"
+    :param advancement: optional class to display advancement
+    :param process_id: id of the process requesting this computation
+    :param cluster_config: cluster configuration if
     :return: success, recons_urls (list of output urls), tuple of outs, tuples of errs, dict future_scans (key is cor, value is future_scan)
-             if `instanciate_classes_only` set to True then return a list of :class:`_Reconstructor`
-    :rtype: tuple
+             if `instantiate_classes_only` set to True then return a list of :class:`_Reconstructor`
     """
     if cluster_config in (None, {}):
         target = Target.LOCAL
@@ -113,7 +114,7 @@ def run_nabu_one_slice_several_config(
         output_file_prefix_pattern=output_file_prefix_pattern,
         axis=axis,
     )
-    if instanciate_classes_only:
+    if instantiate_classes_only:
         return (reconstructor,)
 
     try:
@@ -151,12 +152,12 @@ def run_nabu_multicor(
     scan: TomwerScanBase,
     nabu_config: dict,
     cors: tuple,
-    cluster_config: Optional[dict],
+    cluster_config: dict | None,
     dry_run: bool,
-    slice_index: Union[int, str],
+    slice_index: int | str,
     file_format: str,
-    process_id: Optional[int] = None,
-    instanciate_classes_only: bool = False,
+    process_id: int | None = None,
+    instantiate_classes_only: bool = False,
     output_file_prefix_pattern=None,
 ):
     if cluster_config in (None, {}):
@@ -199,7 +200,7 @@ def run_nabu_multicor(
         axis=axis,
         output_file_prefix_pattern=output_file_prefix_pattern,
     )
-    if instanciate_classes_only:
+    if instantiate_classes_only:
         return reconstructor
 
     try:
@@ -233,13 +234,13 @@ class _Reconstructor(_NabuBaseReconstructor):
         self,
         scan: TomwerScanBase,
         nabu_configs: Iterable,
-        advancement: Optional[Progress],
-        slice_index: Union[int, str],
-        axis: Union[str, NabuPlane],
+        advancement: tqdm | None,
+        slice_index: int | str,
+        axis: str | NabuPlane,
         target: Target,
         dry_run: bool,
         file_format: str,
-        cluster_config: Optional[dict],
+        cluster_config: dict | None,
         process_name: str,
         output_file_prefix_pattern=None,
     ) -> None:
@@ -252,7 +253,7 @@ class _Reconstructor(_NabuBaseReconstructor):
         :param target: is the reconstruction is to made locally or remotly
         :param file_format: reconstructed volume file format
         :param cluster_config: cluster configuration in the case of a remote execution
-        :param str extra_output_file_pattern: possible extra file name pattern like for cor we want to add 'cor_' as prefix and cor value as suffix.
+        :param extra_output_file_pattern: possible extra file name pattern like for cor we want to add 'cor_' as prefix and cor value as suffix.
                                               To make the file name unique. For delta/beta it is already forseen to be unique. For now keywords are:
                                               * file_name: default file name according to db values and dataset name
                                               * value: value of the nabu_configs keys
@@ -283,7 +284,7 @@ class _Reconstructor(_NabuBaseReconstructor):
 
         results = {}
         if self.advancement:
-            self.advancement.setMaxAdvancement(len(self.nabu_configs))
+            self.advancement.total = len(self.nabu_configs)
         for var_value, config in self.nabu_configs.items():
             if self._cancelled:
                 break
@@ -308,7 +309,7 @@ class _Reconstructor(_NabuBaseReconstructor):
             )
             # specific treatment for cor: rename output files
             if self.advancement:
-                self.advancement.increaseAdvancement(1)
+                self.advancement.update()
         return results
 
     def _format_file_prefix(self, file_prefix, value):
@@ -395,9 +396,9 @@ class _Reconstructor(_NabuBaseReconstructor):
             dataset_params.update(config["dataset"])
         config["dataset"] = dataset_params
 
-        config["resources"] = utils.get_nabu_resources_desc(
-            scan=self.scan, workers=1, method="local"
-        )
+        config["resources"] = config.get("resources", {})
+        config["resources"]["method"] = "local"
+
         # force overwrite results
         if "output" not in config:
             config["output"] = {}
@@ -423,10 +424,10 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
     def __init__(
         self,
         nabu_config: dict,
-        axis: Union[str, NabuPlane],
+        axis: str | NabuPlane,
         cors: tuple,
         file_format,
-        slice_index: Union[int, str] = "middle",
+        slice_index: int | str = "middle",
         output_file_prefix_pattern=None,
         *args,
         **kwargs,
@@ -448,7 +449,7 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
         return self.__cors
 
     @property
-    def slice_index(self) -> Union[int, str]:
+    def slice_index(self) -> int | str:
         return self.__slice_index
 
     @property
@@ -460,13 +461,13 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
         config_to_dump: dict,
         config_file: str,
         file_format: str,
-        info: Optional[str],
+        info: str | None,
         process_name: str,
     ):
         """
         process provided configuration
 
-        :param str info:
+        :param info:
         """
         if self.dry_run is True or self.only_create_config_file():
             return ResultsRun(
@@ -497,7 +498,7 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
     @staticmethod
     def convert_cor_from_rel_to_abs(scan, cor):
         if scan.dim_1 is not None:
-            return cor + scan.dim_1 / 2.0
+            return cor + (scan.dim_1 - 1) / 2.0
         else:
             _logger.warning("enable to get image half width. Set it to 1024")
             return cor + 1024
@@ -505,7 +506,7 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
     @staticmethod
     def convert_cor_from_abs_to_rel(scan, cor):
         if scan.dim_1 is not None:
-            return cor - scan.dim_1 / 2.0
+            return cor - (scan.dim_1 - 1) / 2.0
         else:
             _logger.warning("enable to get image half width. Set it to 1024")
             return cor - 1024
@@ -519,11 +520,10 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
         """
         run locally nabu for a single configuration file.
 
-        :param str conf_file: path to the nabu .cfg file
-        :param str file_format: format of the generated file
-        :param dict config_to_dump: configuration saved in the .cfg as a dictionary
+        :param conf_file: path to the nabu .cfg file
+        :param file_format: format of the generated file
+        :param config_to_dump: configuration saved in the .cfg as a dictionary
         :return: results of the local run
-        :rtype: ResultsLocalRun
         """
         if not has_nabu:
             raise ImportError("Fail to import nabu")
@@ -538,7 +538,7 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
                 sys.executable,
                 "-m",
                 settings.NABU_MULTICOR_PATH,
-                conf_file,  # input file
+                f"'{conf_file}'",  # input file
                 f"{slice_index}",  # slice
                 f"{cor_in_nabu_ref}",  # cor
             )
@@ -595,7 +595,6 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
         Run a nabu reconstruction on slurm of a single configuration
 
         :return: results of the slurm run
-        :rtype: ResultSlurmRun
         """
         if not isinstance(conf_file, str):
             raise TypeError(f"conf_file is expected to be a strg not {type(conf_file)}")
@@ -664,9 +663,9 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
             dataset_params.update(config["dataset"])
         config["dataset"] = dataset_params
 
-        config["resources"] = utils.get_nabu_resources_desc(
-            scan=self.scan, workers=1, method="local"
-        )
+        config["resources"] = config.get("resources", {})
+        config["resources"]["method"] = "local"
+
         # force overwrite results
         if "output" not in config:
             config["output"] = {}