tomwer 1.3.0.dev2__py3-none-any.whl → 1.3.0rc10__py3-none-any.whl

tomwer/core/process/reconstruction/axis/params.py

@@ -45,7 +45,7 @@ from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.spec import getParametersFromParOrInfo
 
 from .anglemode import CorAngleMode
-from .mode import AxisMode
+from .mode import AxisMode, AXIS_MODE_METADATAS
 from .projectiontype import ProjectionType
 
 from nabu.preproc.phase import PaganinPhaseRetrieval
@@ -160,9 +160,9 @@ class AxisResource(object):
         else:
             self.__norme_data = scan.data_flat_field_correction(data=self.data)
             if log_ is True:
-                self.__norme_data[
-                    numpy.isnan(self.__norme_data)
-                ] = self.__norme_data.max()
+                self.__norme_data[numpy.isnan(self.__norme_data)] = (
+                    self.__norme_data.max()
+                )
                 if self.__norme_data.max() < 0:
                     _logger.error("max data value < 0 unable to compute log")
                 else:
@@ -173,9 +173,9 @@ class AxisResource(object):
                     except Exception as e:
                         _logger.error("Fail to apply log on radio" + e)
             else:
-                self.__norme_data[
-                    numpy.isnan(self.__norme_data)
-                ] = self.__norme_data.min()
+                self.__norme_data[numpy.isnan(self.__norme_data)] = (
+                    self.__norme_data.min()
+                )
 
         return self.__norme_data
 
@@ -885,6 +885,9 @@ class AxisRP:
         pass
 
     def get_simple_str(self):
+        """
+        special information as a str for mode able to handle both sinogram and radios
+        """
         results = f"{self.mode.value}"
         if self.mode in (
             AxisMode.growing_window_radios,
@@ -898,20 +901,20 @@ class AxisRP:
 
     def get_nabu_cor_options_as_dict(self) -> str:
         options = {}
-        if self.mode in (
-            AxisMode.sliding_window_sinogram,
-            AxisMode.sliding_window_radios,
-            AxisMode.growing_window_sinogram,
-            AxisMode.growing_window_radios,
-        ):
-            options["side"] = self.side
-        elif self.mode in (AxisMode.composite_coarse_to_fine, AxisMode.near):
-            near_pos = self.composite_options.get("near_pos", 0.0)
+        if self.mode is AxisMode.near:
+            self.side = "near"
+        request_side = len(AXIS_MODE_METADATAS[self.mode].valid_sides) > 0
+        if request_side:
+
+            if self.side == "near":
+                options["side"] = self.composite_options.get(
+                    "near_pos", self.estimated_cor
+                )
+            else:
+                options["side"] = self.side
+
+        if self.side == "near":
             near_width = self.composite_options.get("near_width", 20.0)
-            if self.mode is AxisMode.near:
-                self.side = "near"
-            options["side"] = self.side
-            options["near_pos"] = near_pos
             options["near_width"] = near_width
 
         # append "extra_cor_options" to already handled cor options

tomwer/core/process/reconstruction/darkref/darkrefs.py

@@ -396,19 +396,18 @@ class DarkRefsTask(
             if scan.process_file is not None and not (
                 isinstance(scan, NXtomoScan) and not self.__new_hdf5_entry_created
             ):
-                with scan.acquire_process_file_lock():
-                    entry = "entry"
-                    if hasattr(scan, "entry"):
-                        entry = scan.entry
-                    self.register_process(
-                        process_file=scan.process_file,
-                        entry=entry,
-                        configuration=self.recons_params.to_dict(),
-                        results=results,
-                        interpretations=interpretations,
-                        process_index=scan.pop_process_index(),
-                        overwrite=True,
-                    )
+                entry = "entry"
+                if hasattr(scan, "entry"):
+                    entry = scan.entry
+                self.register_process(
+                    process_file=scan.process_file,
+                    entry=entry,
+                    configuration=self.recons_params.to_dict(),
+                    results=results,
+                    interpretations=interpretations,
+                    process_index=scan.pop_process_index(),
+                    overwrite=True,
+                )
         logger.processSucceed(f"Dark and flat reduction succeeded for {scan}")
         self.notify_to_state_to_managed(
             dataset=scan, state=DatasetState.SUCCEED, details=None

tomwer/core/process/reconstruction/nabu/castvolume.py

@@ -231,9 +231,9 @@ class CastVolumeTask(
                     working_directory, "slurm_scripts", script_name
                 )
             # for now force job name
-            self._cluster_config[
-                "job_name"
-            ] = f"tomwer-cast-volume {input_volume.get_identifier().to_str()} to {output_volume.get_identifier().to_str()}"
+            self._cluster_config["job_name"] = (
+                f"tomwer-cast-volume {input_volume.get_identifier().to_str()} to {output_volume.get_identifier().to_str()}"
+            )
             job = SBatchScriptJob(
                 slurm_config=self._cluster_config,
                 script=(get_command(),),

tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -39,15 +39,17 @@ import uuid
 from typing import Iterable, Optional, Union
 import signal
 import psutil
+import sys
 
 import numpy
 from silx.io.url import DataUrl
 from sluurp.executor import submit as submit_to_slurm_cluster
 from sluurp.job import SBatchScriptJob
-from tomoscan.io import HDF5File
+from tomoscan.io import HDF5File, get_swmr_mode
 from tomoscan.normalization import Method as INormMethod
 from tomoscan.identifier import VolumeIdentifier
 
+from tomwer.core.process.reconstruction.nabu.plane import NabuPlane
 from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.process.reconstruction.nabu.target import Target
 from tomwer.core.process.reconstruction.nabu.utils import (
@@ -66,6 +68,7 @@ from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.slurm import get_slurm_script_name, is_slurm_available
+from tomwer.core.volume.volumefactory import VolumeFactory
 
 from . import settings, utils
 
@@ -147,10 +150,27 @@ class ResultsLocalRun(ResultsWithStd):
             raise TypeError(
                 f"results_urls is expected to be an Iterable not {type(results_identifiers)}"
             )
-        for identifier in results_identifiers:
-            if not isinstance(identifier, VolumeIdentifier):
-                raise TypeError("identifiers are expected to be VolumeIdentifier")
-        self.__results_identifiers = results_identifiers
+
+        # check all identifiers
+        def check_identifier(identifier):
+            if isinstance(identifier, str):
+                vol = VolumeFactory.create_tomo_object_from_identifier(
+                    identifier=identifier
+                )
+                return vol.get_identifier()
+            elif not isinstance(identifier, VolumeIdentifier):
+                raise TypeError(
+                    f"identifiers are expected to be VolumeIdentifier. Get {type(identifier)} instead."
+                )
+            else:
+                return identifier
+
+        self.__results_identifiers = tuple(
+            [
+                check_identifier(identifier=identifier)
+                for identifier in results_identifiers
+            ]
+        )
 
     @property
     def results_identifiers(self) -> tuple:
@@ -207,6 +227,7 @@ class _NabuBaseReconstructor:
         target: Target,
         cluster_config: Optional[Union[dict, SlurmClusterConfiguration]],
         process_name: str,
+        axis: NabuPlane = NabuPlane.XY,
     ) -> None:
         self._scan = scan
         self._target = Target.from_value(target)
@@ -214,6 +235,7 @@ class _NabuBaseReconstructor:
         self._process_name = process_name
         self._process = None
         self._cancelled = False
+        self._axis = NabuPlane.from_value(axis)
         # nabu subprocess if run locally
         if isinstance(cluster_config, SlurmClusterConfiguration):
             self._cluster_config = cluster_config
@@ -246,6 +268,10 @@ class _NabuBaseReconstructor:
     def process_name(self):
         return self._process_name
 
+    @property
+    def axis(self) -> NabuPlane:
+        return self._axis
+
     def only_create_config_file(self):
         """Should we run the reconstruction or only create the configuration file"""
         return False
@@ -263,8 +289,6 @@ class _NabuBaseReconstructor:
         config_to_dump: dict,
         config_file: str,
         file_format: str,
-        start_z: Optional[int],
-        end_z: Optional[int],
         info: Optional[str],
         process_name: str,
     ):
@@ -284,8 +308,7 @@ class _NabuBaseReconstructor:
                 conf_file=config_file,
                 file_format=file_format,
                 config_to_dump=config_to_dump,
-                start_z=start_z,
-                end_z=end_z,
+                axis=config_to_dump["reconstruction"].get("slice_plane", "XY"),
             )
         elif self.target is Target.SLURM:
             _logger.info(
@@ -295,8 +318,6 @@ class _NabuBaseReconstructor:
                 conf_file=config_file,
                 config_to_dump=config_to_dump,
                 cluster_config=self.cluster_config.to_dict(),
-                start_z=start_z,
-                end_z=end_z,
                 process_name=process_name,
                 info=info,
             )
@@ -317,11 +338,10 @@ class _NabuBaseReconstructor:
 
     def _run_nabu_locally(
         self,
+        axis: NabuPlane,
         conf_file: str,
         file_format: str,
         config_to_dump: dict,
-        start_z: int,
-        end_z: int,
     ) -> ResultsLocalRun:
         """
         run locally nabu for a single configuration file.
@@ -341,7 +361,7 @@ class _NabuBaseReconstructor:
 
         command = " ".join(
             (
-                "python3",
+                sys.executable,
                 "-m",
                 settings.NABU_FULL_FIELD_APP_PATH,
                 conf_file,
@@ -372,6 +392,7 @@ class _NabuBaseReconstructor:
             slice_index=None,
             scan=self.scan,
             file_format=file_format,
+            axis=axis,
         )
         return ResultsLocalRun(
             success=not nabu_std_err_has_error(errs),
@@ -386,8 +407,6 @@ class _NabuBaseReconstructor:
         conf_file: str,
         config_to_dump: dict,
         cluster_config: dict,
-        start_z: int,
-        end_z: int,
         process_name: str,
         info: str,
     ) -> ResultSlurmRun:
@@ -494,9 +513,14 @@ class _NabuBaseReconstructor:
                                 "delta_beta"
                             ][0]
                     db = round(float(config["phase"]["delta_beta"]))
+        # retrieve axis used
+        axis = config.get("reconstruction", {}).get("slice_plane", "XY")
+
         # handle output
         if "output" in config:
-            _file_name = self._get_file_basename_reconstruction(pag=pag, db=db, ctf=ctf)
+            _file_name = self._get_file_basename_reconstruction(
+                pag=pag, db=db, ctf=ctf, axis=axis
+            )
             config["output"]["file_prefix"] = _file_name
             location, location_cfg_files = get_output_folder_from_scan(
                 mode=ProcessDataOutputDirMode.from_value(
@@ -558,7 +582,7 @@ class _NabuBaseReconstructor:
                     dataset_url = DataUrl(path=dataset_url)
                     if os.path.exists(dataset_url.file_path()):
                         with HDF5File(
-                            dataset_url.file_path(), mode="r", swmr="True"
+                            dataset_url.file_path(), mode="r", swmr=get_swmr_mode()
                         ) as h5f:
                             if dataset_url.data_path() not in h5f:
                                 dataset_url = None
@@ -594,7 +618,7 @@ class _NabuBaseReconstructor:
             raise NotImplementedError(f"source type {source.value} is not handled")
         return config, nabu_cfg_folder
 
-    def _get_file_basename_reconstruction(self, pag, db, ctf):
+    def _get_file_basename_reconstruction(self, pag, db, ctf, axis):
         """return created file base name"""
         raise NotImplementedError("Base class")
 

tomwer/core/process/reconstruction/nabu/nabuscores.py

@@ -16,6 +16,7 @@ else:
     has_nabu = True
 import logging
 import os
+import sys
 from copy import deepcopy
 import subprocess
 from typing import Iterable, Optional, Union
@@ -44,6 +45,7 @@ from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.slurm import get_slurm_script_name, is_slurm_available
 from tomwer.utils import docstring
 from tomwer.core.process.reconstruction.nabu.utils import nabu_std_err_has_error
+from tomwer.core.process.reconstruction.nabu.plane import NabuPlane
 
 from ..nabu import settings as nabu_settings
 from . import utils, settings
@@ -59,6 +61,7 @@ def run_nabu_one_slice_several_config(
     dry_run: bool,
     slice_index: Union[int, str],
     file_format: str,
+    axis: NabuPlane,
     advancement: Optional[Progress] = None,
     process_id: Optional[int] = None,
     instanciate_classes_only: bool = False,
@@ -108,6 +111,7 @@ def run_nabu_one_slice_several_config(
         cluster_config=cluster_config,
         process_name=process_name,
         output_file_prefix_pattern=output_file_prefix_pattern,
+        axis=axis,
     )
     if instanciate_classes_only:
         return (reconstructor,)
@@ -180,6 +184,8 @@ def run_nabu_multicor(
     # half acquisition it will be refused
     nabu_config["reconstruction"]["rotation_axis_position"] = numpy.mean(cors)
 
+    axis = nabu_config["reconstruction"].get("slice_plane", "XY")
+
     reconstructor = _ReconstructorMultiCor(
         scan=scan,
         nabu_config=nabu_config,
@@ -190,6 +196,7 @@ def run_nabu_multicor(
         file_format=file_format,
         cluster_config=cluster_config,
         process_name=process_name,
+        axis=axis,
         output_file_prefix_pattern=output_file_prefix_pattern,
     )
     if instanciate_classes_only:
@@ -227,6 +234,7 @@ class _Reconstructor(_NabuBaseReconstructor):
         nabu_configs: Iterable,
         advancement: Optional[Progress],
         slice_index: Union[int, str],
+        axis: Union[str, NabuPlane],
         target: Target,
         dry_run: bool,
         file_format: str,
@@ -235,6 +243,14 @@ class _Reconstructor(_NabuBaseReconstructor):
         output_file_prefix_pattern=None,
     ) -> None:
         """
+        :param scan: scan to reconstruct
+        :param nabu_configs: all the configuration to run
+        :param advancement: Progress object to notify advancement
+        :param slice_index: index of the slice to reconstruct.
+        :param axis: axis over which we want to do the reconstruction
+        :param target: is the reconstruction is to made locally or remotly
+        :param file_format: reconstructed volume file format
+        :param cluster_config: cluster configuration in the case of a remote execution
         :param str extra_output_file_pattern: possible extra file name pattern like for cor we want to add 'cor_' as prefix and cor value as suffix.
                                               To make the file name unique. For delta/beta it is already forseen to be unique. For now keywords are:
                                               * file_name: default file name according to db values and dataset name
@@ -246,12 +262,12 @@ class _Reconstructor(_NabuBaseReconstructor):
             target=target,
             cluster_config=cluster_config,
             process_name=process_name,
+            axis=axis,
         )
         if not isinstance(slice_index, (int, str)):
             raise TypeError(
                 f"slice_index should be an int or a string not {type(slice_index)}"
             )
-
         self.advancement = advancement
         self.slice_index = slice_index
         self.nabu_configs = nabu_configs
@@ -286,8 +302,6 @@ class _Reconstructor(_NabuBaseReconstructor):
                 config_to_dump=config_to_dump,
                 config_file=conf_file,
                 file_format=self.file_format,
-                start_z=None,
-                end_z=None,
                 info="nabu slice reconstruction",
                 process_name=self.process_name,
             )
@@ -349,6 +363,7 @@ class _Reconstructor(_NabuBaseReconstructor):
                 pag=pag,
                 db=db,
                 ctf=ctf,
+                axis=self.axis,
             )
             file_prefix = self._format_file_prefix(file_prefix=file_prefix, value=value)
             _config["output"]["file_prefix"] = file_prefix
@@ -358,8 +373,19 @@ class _Reconstructor(_NabuBaseReconstructor):
 
         if "reconstruction" not in _config:
             _config["reconstruction"] = {}
-        _config["reconstruction"]["start_z"] = self.slice_index
-        _config["reconstruction"]["end_z"] = self.slice_index
+        if self.axis is NabuPlane.YZ:
+            _config["reconstruction"]["start_x"] = self.slice_index
+            _config["reconstruction"]["end_x"] = self.slice_index
+        elif self.axis is NabuPlane.XZ:
+            _config["reconstruction"]["start_y"] = self.slice_index
+            _config["reconstruction"]["end_y"] = self.slice_index
+        elif self.axis is NabuPlane.XY:
+            _config["reconstruction"]["start_z"] = self.slice_index
+            _config["reconstruction"]["end_z"] = self.slice_index
+        else:
+            raise ValueError(
+                f"self.axis has an invalid value: {self.axis} when expected to be in {NabuPlane.values()}"
+            )
         return _config, file_prefix
 
     def preprocess_config(self, config, value) -> tuple:
@@ -396,6 +422,7 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
     def __init__(
         self,
         nabu_config: dict,
+        axis: Union[str, NabuPlane],
         cors: tuple,
         file_format,
         slice_index: Union[int, str] = "middle",
@@ -507,7 +534,7 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
         cor_in_nabu_ref = ",".join([str(cor) for cor in cor_in_nabu_ref])
         command = " ".join(
             (
-                "python3",
+                sys.executable,
                 "-m",
                 settings.NABU_MULTICOR_PATH,
                 conf_file,  # input file
@@ -549,7 +576,7 @@ class _ReconstructorMultiCor(_NabuBaseReconstructor):
         }
         return ResultsLocalRun(
             success=not nabu_std_err_has_error(errs),
-            results_identifiers=recons_vol_identifiers,
+            results_identifiers=recons_vol_identifiers.values(),
             std_out=outs,
             std_err=errs,
             config=config_to_dump,  # config_slices,