tomwer 1.2.9__py3-none-any.whl → 1.3.0a0__py3-none-any.whl

tomwer/core/process/output.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+import os
+from silx.utils.enum import Enum as _Enum
+
+
+class ProcessDataOutputDirMode(_Enum):
+    IN_SCAN_FOLDER = "same folder as scan"
+    PROCESSED_DATA_FOLDER = "PROCESSED_DATA folder"
+    RAW_DATA_FOLDER = "RAW_DATA folder"
+    OTHER = "other"
+
+    @classmethod
+    def from_value(cls, value):
+        # ensure backward compatibility
+        if value == "near input":
+            return cls.IN_SCAN_FOLDER
+        elif value == "processed data dir":
+            return cls.PROCESSED_DATA_FOLDER
+        elif value == "raw data dir":
+            return cls.RAW_DATA_FOLDER
+        return super().from_value(value)
+
+
+class NabuOutputFileFormat(_Enum):
+    TIFF = "tiff"
+    HDF5 = "hdf5"
+    JP2K = "jp2"
+    EDF = "edf"
+    RAW = "vol"
+
+    @classmethod
+    def from_value(cls, value):
+        if isinstance(value, str):
+            value = value.lstrip(".")
+        return super().from_value(value)
+
+
+def get_file_format(file_str):
+    extension = os.path.splitext(file_str.lower())[-1]
+    extension = extension.lstrip(".")
+    if extension in ("tiff", "tif"):
+        return NabuOutputFileFormat.TIFF
+    elif extension in ("hdf5", "hdf", "h5"):
+        return NabuOutputFileFormat.HDF5
+    elif extension in ("jp2", "jp2k", "jpg2k"):
+        return NabuOutputFileFormat.JP2K
+    elif extension in ("edf",):
+        return NabuOutputFileFormat.EDF
+    elif extension in ("vol", "raw"):
+        return NabuOutputFileFormat.RAW
+    else:
+        raise ValueError(f"Unrecognized file extension {extension} from {file_str}")

tomwer/core/process/reconstruction/axis/axis.py

@@ -43,12 +43,12 @@ from nabu.pipeline.estimators import SinoCORFinder
 from nabu.resources.nxflatfield import update_dataset_info_flats_darks
 from processview.core.manager import DatasetState, ProcessManager
 from processview.core.superviseprocess import SuperviseProcess
-from silx.utils.deprecation import deprecated_warning
+from tomwer.core.utils.deprecation import deprecated_warning
 
 import tomwer.version
 from tomwer.core.process.reconstruction.utils.cor import absolute_pos_to_relative
 from tomwer.core.process.task import Task
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils import image, logconfig
@@ -108,7 +108,7 @@ def adapt_tomwer_scan_to_nabu(scan: TomwerScanBase):
     updating infos regarding flat and dark if needed
     """
     dataset_infos = scan.to_nabu_dataset_analyser()
-    if isinstance(scan, HDF5TomoScan):
+    if isinstance(scan, NXtomoScan):
         try:
             update_dataset_info_flats_darks(
                 dataset_infos,
@@ -183,12 +183,15 @@ def compute_cor_nabu_growing_window_sinogram(
 
     :return:
     """
+    has_darks = scan.reduced_darks is not None and len(scan.reduced_darks) > 0
+    has_flats = scan.reduced_flats is not None and len(scan.reduced_flats) > 0
+
     corfinder = SinoCORFinder(
         dataset_info=adapt_tomwer_scan_to_nabu(scan),
         method="growing-window",
         slice_idx=scan.axis_params.sinogram_line or "middle",
         subsampling=scan.axis_params.sinogram_subsampling,
-        do_flatfield=True,
+        do_flatfield=has_darks and has_flats,
         cor_options=scan.axis_params.get_nabu_cor_options_as_dict(),
         logger=_logger,
     )
@@ -213,12 +216,15 @@ def compute_scan_sino_coarse_to_fine(scan):
     :param scan:
     :return:
     """
+    has_darks = scan.reduced_darks is not None and len(scan.reduced_darks) > 0
+    has_flats = scan.reduced_flats is not None and len(scan.reduced_flats) > 0
+
     corfinder = SinoCORFinder(
         dataset_info=adapt_tomwer_scan_to_nabu(scan),
         method=AxisMode.sino_coarse_to_fine.value,
         slice_idx=scan.axis_params.sinogram_line or "middle",
         subsampling=scan.axis_params.sinogram_subsampling,
-        do_flatfield=True,
+        do_flatfield=has_darks and has_flats,
         cor_options=scan.axis_params.get_nabu_cor_options_as_dict(),
         logger=_logger,
     )
@@ -252,8 +258,6 @@ def compute_scan_composite_coarse_to_fine(scan: TomwerScanBase):
     # allow some tolerance for the "side" argument that is there only
     # in the new one
 
-    extra_args = {}
-
     cor_options = scan.axis_params.get_nabu_cor_options_as_dict()
     for key in "low_pass", "high_pass":
         if key in cor_options:
@@ -266,7 +270,6 @@ def compute_scan_composite_coarse_to_fine(scan: TomwerScanBase):
         cor_options=cor_options,
         logger=_logger,
         take_log=take_log,
-        **extra_args,
     )
     res = corfinder.find_cor()
     return _absolute_pos_to_relative_with_warning(
@@ -381,12 +384,15 @@ def compute_cor_nabu_sliding_window_sinogram(
 
     :return:
     """
+    has_darks = scan.reduced_darks is not None and len(scan.reduced_darks) > 0
+    has_flats = scan.reduced_flats is not None and len(scan.reduced_flats) > 0
+
     corfinder = SinoCORFinder(
         dataset_info=adapt_tomwer_scan_to_nabu(scan),
         method="sliding-window",
         slice_idx=scan.axis_params.sinogram_line or "middle",
         subsampling=scan.axis_params.sinogram_subsampling,
-        do_flatfield=True,
+        do_flatfield=has_darks and has_flats,
         cor_options=scan.axis_params.get_nabu_cor_options_as_dict(),
         logger=_logger,
     )
@@ -597,6 +603,7 @@ class AxisTask(
         AxisMode.growing_window_radios: compute_scan_cor_nabu_growing_window,
         AxisMode.sino_coarse_to_fine: compute_scan_sino_coarse_to_fine,
         AxisMode.composite_coarse_to_fine: compute_scan_composite_coarse_to_fine,
+        AxisMode.near: compute_scan_composite_coarse_to_fine,
     }
 
     def __init__(
@@ -722,7 +729,7 @@ class AxisTask(
         assert isinstance(scan, TomwerScanBase)
         if scan.process_file is not None:
             entry = "entry"
-            if isinstance(scan, HDF5TomoScan):
+            if isinstance(scan, NXtomoScan):
                 entry = scan.entry
             with scan.acquire_process_file_lock():
                 self.register_process(

tomwer/core/process/reconstruction/axis/mode.py

@@ -56,6 +56,8 @@ class AxisMode(_Enum):
     sino_coarse_to_fine = "sino-coarse-to-fine"
     composite_coarse_to_fine = "composite-coarse-to-fine"
     read = "read"
+    # alias to composite_coarse_to_fine with near mode
+    near = "near"
 
     @classmethod
     def from_value(cls, value):
@@ -87,9 +89,11 @@ _VALID_INPUTS = {
         _InputType.COMPOSITE,
     ),  # in fact it is more an n radio constrain
     AxisMode.read: None,
+    AxisMode.near: (_InputType.COMPOSITE,),  # in fact it is more an n radio constrain
 }
 
 _AXIS_MODE_CONSTRAIN = {
     AxisMode.sino_coarse_to_fine: (_Constrain.FULL_TURN,),
     AxisMode.composite_coarse_to_fine: (_Constrain.FULL_TURN,),
+    AxisMode.near: (_Constrain.FULL_TURN,),
 }

tomwer/core/process/reconstruction/axis/params.py

@@ -42,7 +42,7 @@ from tomoscan.esrf.scan.utils import get_data
 
 from tomwer.core.process.task import Task
 from tomwer.core.scan.scanbase import TomwerScanBase
-from tomwer.core.utils import getParametersFromParOrInfo
+from tomwer.core.utils.spec import getParametersFromParOrInfo
 
 from .anglemode import CorAngleMode
 from .mode import AxisMode
@@ -905,9 +905,11 @@ class AxisRP:
             AxisMode.growing_window_radios,
         ):
             options["side"] = self.side
-        elif self.mode in (AxisMode.composite_coarse_to_fine,):
+        elif self.mode in (AxisMode.composite_coarse_to_fine, AxisMode.near):
             near_pos = self.composite_options.get("near_pos", 0.0)
             near_width = self.composite_options.get("near_width", 20.0)
+            if self.mode is AxisMode.near:
+                self.side = "near"
             options["side"] = self.side
             options["near_pos"] = near_pos
             options["near_width"] = near_width
@@ -917,8 +919,11 @@ class AxisRP:
 
         if extra_cor_options != "":
             for opt in self.extra_cor_options.replace(" ", "").split(";"):
-                key, value = opt.split("=")
-                options[key] = value
+                if len(opt.split("=")) == 2:
+                    key, value = opt.split("=")
+                    options[key] = value
+                else:
+                    _logger.info(f"ignore option {opt}. Invalid synthax")
         return options
 
     @deprecated(replacement="get_nabu_cor_options_as_str", since_version="1.1")

tomwer/core/process/reconstruction/darkref/darkrefs.py

@@ -37,14 +37,16 @@ from queue import Queue
 
 from processview.core.manager import DatasetState, ProcessManager
 from processview.core.superviseprocess import SuperviseProcess
-from silx.utils.deprecation import deprecated_warning
+from tomwer.core.utils.deprecation import deprecated_warning
 
-from tomoscan.framereducerbase import REDUCER_TARGET, ReduceMethod
+from tomoscan.framereducer.target import REDUCER_TARGET
+from tomoscan.framereducer.method import ReduceMethod
 
 import tomwer.version
-from tomwer.core import settings, utils
+from tomwer.core import settings
+from tomwer.core.utils.lbsram import is_low_on_memory
 from tomwer.core.process.task import Task
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import data_identifier_to_scan
@@ -302,7 +304,7 @@ class DarkRefsTask(
         logger.processStarted(f"start dark and ref for {scan}")
         if (
             settings.isOnLbsram(scan)
-            and utils.isLowOnMemory(settings.get_lbsram_path()) is True
+            and is_low_on_memory(settings.get_lbsram_path()) is True
         ):
             mess = (
                 "low memory, do compute dark and flat field mean/median "
@@ -392,7 +394,7 @@ class DarkRefsTask(
         if len(results) > 0:
             # if some processing to be registered
             if scan.process_file is not None and not (
-                isinstance(scan, HDF5TomoScan) and not self.__new_hdf5_entry_created
+                isinstance(scan, NXtomoScan) and not self.__new_hdf5_entry_created
             ):
                 with scan.acquire_process_file_lock():
                     entry = "entry"

tomwer/core/process/reconstruction/darkref/darkrefscopy.py

@@ -44,7 +44,7 @@ from tomoscan.esrf.scan.utils import (
     copy_h5_dict_flats_to,
     cwd_context,
 )
-from tomoscan.framereducerbase import REDUCER_TARGET
+from tomoscan.framereducer.target import REDUCER_TARGET
 from tomoscan.io import HDF5File
 
 from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefsTask

tomwer/core/process/reconstruction/darkref/params.py

@@ -31,7 +31,7 @@ __date__ = "07/03/2019"
 import enum
 
 from silx.utils.enum import Enum as _Enum
-from tomoscan.framereducerbase import ReduceMethod
+from tomoscan.framereducer.method import ReduceMethod
 
 from tomwer.core.process.reconstruction.darkref.settings import (
     DARKHST_PREFIX,

tomwer/core/process/reconstruction/lamino/tofu.py

@@ -44,12 +44,12 @@ from tomwer.core.process.reconstruction.darkref.settings import (
     REFHST_PREFIX,
 )
 from tomwer.core.process.task import Task
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
-from tomwer.core.utils import getDim1Dim2
+from tomwer.core.utils.spec import getDim1Dim2
 from tomwer.core.utils.char import PSI_CHAR
-from silx.utils.deprecation import deprecated_warning
+from tomwer.core.utils.deprecation import deprecated_warning
 
 logger = logging.getLogger(__name__)
 
@@ -871,7 +871,7 @@ class LaminoReconstructionTask(
         )
         if not self.dry_run:
             entry = "entry"
-            if isinstance(scan, HDF5TomoScan):
+            if isinstance(scan, NXtomoScan):
                 entry = scan.entry
             with scan.acquire_process_file_lock():
                 self.register_process(

tomwer/core/process/reconstruction/nabu/castvolume.py

@@ -47,7 +47,7 @@ from tomoscan.volumebase import VolumeBase
 from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.futureobject import FutureTomwerObject
 from tomwer.core.process.reconstruction.nabu import settings
-from tomwer.core.process.reconstruction.nabu.nabucommon import NabuOutputFileFormat
+from tomwer.core.process.reconstruction.output import NabuOutputFileFormat
 from tomwer.core.process.task import Task
 from tomwer.core.utils.slurm import get_slurm_script_name
 from tomwer.core.utils.volumeutils import volume_identifier_to_volume

tomwer/core/process/reconstruction/nabu/helical.py

@@ -6,7 +6,7 @@ from nabu.io.reader import load_images_from_dataurl_dict
 from nabu.app.prepare_weights_double import create_heli_maps
 
 from tomwer.core.process.task import TaskWithProgress
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.utils.scanutils import format_output_location
 
 
@@ -17,8 +17,10 @@ class NabuHelicalPrepareWeightsDouble(
     TaskWithProgress,
     input_names=(
         "data",
-        "transition_width",
+        "transition_width_vertical",
+        "transition_width_horizontal",
         "processes_file",
+        "rotation_axis_position",
     ),
     optional_input_names=("progress",),
     output_names=("data",),
@@ -26,8 +28,8 @@ class NabuHelicalPrepareWeightsDouble(
     def run(self):
         # TODO: handle future /cluster config ???
         scan = self.inputs.data
-        if not isinstance(scan, HDF5TomoScan):
-            raise TypeError(f"data is expected to be an instance of {HDF5TomoScan}")
+        if not isinstance(scan, NXtomoScan):
+            raise TypeError(f"data is expected to be an instance of {NXtomoScan}")
         dataset_info = HDF5DatasetAnalyzer(
             scan.master_file,
             extra_options={"h5_entry": scan.entry},
@@ -48,7 +50,9 @@ class NabuHelicalPrepareWeightsDouble(
             profile=mappe,
             process_file_name=scan.helical.processes_files,
             entry_name=scan.entry,
-            transition_width=self.inputs.transition_width,
+            transition_width_vertical=self.inputs.transition_width_vertical,
+            transition_width_horizontal=self.inputs.transition_width_horizontal,
+            rotation_axis_position=self.inputs.rotation_axis_position,
         )
 
         self.outputs.data = scan

tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -42,11 +42,11 @@ import psutil
 
 import numpy
 from silx.io.url import DataUrl
-from silx.utils.enum import Enum as _Enum
 from sluurp.executor import submit as submit_to_slurm_cluster
 from sluurp.job import SBatchScriptJob
 from tomoscan.io import HDF5File
 from tomoscan.normalization import Method as INormMethod
+from tomoscan.identifier import VolumeIdentifier
 
 from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.process.reconstruction.nabu.target import Target
@@ -54,13 +54,17 @@ from tomwer.core.process.reconstruction.nabu.utils import (
     _NabuPhaseMethod,
     nabu_std_err_has_error,
 )
+from tomwer.core.process.reconstruction.output import (
+    ProcessDataOutputDirMode,
+    get_output_folder_from_scan,
+    NabuOutputFileFormat,
+)
 from tomwer.core.process.reconstruction.normalization.params import (
     _ValueSource as INormSource,
 )
 from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
-from tomwer.core.utils.scanutils import format_output_location
 from tomwer.core.utils.slurm import get_slurm_script_name, is_slurm_available
 
 from . import settings, utils
@@ -85,37 +89,6 @@ else:
     has_nabu = True
 
 
-class NabuOutputFileFormat(_Enum):
-    TIFF = "tiff"
-    HDF5 = "hdf5"
-    JP2K = "jp2"
-    EDF = "edf"
-    RAW = "vol"
-
-    @classmethod
-    def from_value(cls, value):
-        if isinstance(value, str):
-            value = value.lstrip(".")
-        return super().from_value(value)
-
-
-def get_file_format(file_str):
-    extension = os.path.splitext(file_str.lower())[-1]
-    extension = extension.lstrip(".")
-    if extension in ("tiff", "tif"):
-        return NabuOutputFileFormat.TIFF
-    elif extension in ("hdf5", "hdf", "h5"):
-        return NabuOutputFileFormat.HDF5
-    elif extension in ("jp2", "jp2k", "jpg2k"):
-        return NabuOutputFileFormat.JP2K
-    elif extension in ("edf",):
-        return NabuOutputFileFormat.EDF
-    elif extension in ("vol", "raw"):
-        return NabuOutputFileFormat.RAW
-    else:
-        raise ValueError(f"Unrecognized file extension {extension} from {file_str}")
-
-
 class ResultsRun:
     """
     Base class of results for nabu
@@ -165,24 +138,27 @@ class ResultsLocalRun(ResultsWithStd):
 
     def __init__(
         self,
-        results_urls: tuple,
+        results_identifiers: tuple,
         *args,
         **kwargs,
     ) -> None:
         super().__init__(*args, **kwargs)
-        if not isinstance(results_urls, Iterable):
+        if not isinstance(results_identifiers, Iterable):
             raise TypeError(
-                f"results_urls is expected to be an Iterable not {type(results_urls)}"
+                f"results_urls is expected to be an Iterable not {type(results_identifiers)}"
             )
-        self.__results_urls = results_urls
+        for identifier in results_identifiers:
+            if not isinstance(identifier, VolumeIdentifier):
+                raise TypeError("identifiers are expected to be VolumeIdentifier")
+        self.__results_identifiers = results_identifiers
 
     @property
-    def results_urls(self) -> tuple:
-        return self.__results_urls
+    def results_identifiers(self) -> tuple:
+        return self.__results_identifiers
 
     def __str__(self) -> str:
         res = super().__str__()
-        res += f"\n - result urls: {self.results_urls}"
+        res += f"\n - result urls: {self.results_identifiers}"
         return res
 
 
@@ -396,13 +372,10 @@ class _NabuBaseReconstructor:
             slice_index=None,
             scan=self.scan,
             file_format=file_format,
-            start_z=start_z,
-            end_z=end_z,
-            expects_single_slice=self.EXPECTS_SINGLE_SLICE,
         )
         return ResultsLocalRun(
             success=not nabu_std_err_has_error(errs),
-            results_urls=recons_urls,
+            results_identifiers=recons_urls,
             std_out=outs,
             std_err=errs,
             config=config_to_dump,  # config_slices,
@@ -525,22 +498,19 @@ class _NabuBaseReconstructor:
         if "output" in config:
             _file_name = self._get_file_basename_reconstruction(pag=pag, db=db, ctf=ctf)
             config["output"]["file_prefix"] = _file_name
-            location = config["output"].get("location", None)
-            if location not in ("", None):
-                location = format_output_location(location, scan=self.scan)
-                location_cfg_files = location
-            else:
-                # otherwise default location will be the data root level
-                location = self.scan.path
-                location_cfg_files = location
-                # TODO: if is a single file - append prefix
-                if config["output"].get("file_format") in (
-                    NabuOutputFileFormat.EDF.value,
-                    NabuOutputFileFormat.TIFF.value,
-                    NabuOutputFileFormat.JP2K.value,
-                ):  # if user specify the location
-                    location = "/".join([location, _file_name])
-
+            location, location_cfg_files = get_output_folder_from_scan(
+                mode=ProcessDataOutputDirMode.from_value(
+                    config["output"].get(
+                        "output_dir_mode", ProcessDataOutputDirMode.OTHER
+                    )
+                ),
+                nabu_location=config["output"].get("location", None),
+                scan=self.scan,
+                file_basename=_file_name,
+                file_format=config["output"].get(
+                    "file_format", NabuOutputFileFormat.HDF5
+                ),
+            )
             # add reconstruction path to the list. scan `reconstruction_paths` register all the existing path where
             # reconstruction are saved in order to be able to browse them all
             self.scan.add_reconstruction_path(location)
@@ -661,7 +631,7 @@ def dump_normalization_array_for_nabu(
             f"array is expected to be a numpy array or a scalar and not {type(array)}"
         )
     # save the value to a dedicated path in "nabu_tomwer_serving_hatch"
-    if isinstance(scan, HDF5TomoScan):
+    if isinstance(scan, NXtomoScan):
         entry_path = scan.entry
     elif isinstance(scan, EDFTomoScan):
         entry_path = "entry"