tomwer 1.2.9__py3-none-any.whl → 1.3.0a0__py3-none-any.whl

tomwer/core/process/reconstruction/nabu/nabuvolume.py

@@ -43,21 +43,27 @@ from processview.core.manager.manager import ProcessManager, DatasetState
 from processview.core.superviseprocess import SuperviseProcess
 
 from silx.io.utils import h5py_read_dataset
-from silx.utils.deprecation import deprecated_warning
+from tomwer.core.utils.deprecation import deprecated_warning
 
 from tomoscan.io import HDF5File
 
 from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.futureobject import FutureTomwerObject
 from tomwer.core.process.task import Task
+from tomwer.core.process.icat.gallery import (
+    IcatScreenshots,
+    deduce_dataset_gallery_location,
+    select_screenshot_from_volume,
+)
 from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
+from tomwer.core.utils.dictutils import concatenate_dict
 from tomwer.core.utils.scanutils import data_identifier_to_scan
+from tomwer.core.volume.volumefactory import VolumeFactory
 from tomwer.io.utils.h5pyutils import EntryReader
 from tomwer.utils import docstring
-from tomwer.core.utils import concatenate_dict
 from tomwer.io.utils import format_stderr_stdout
 
 from . import settings, utils
@@ -143,8 +149,8 @@ def run_volume_reconstruction(
         assert len(results) == 1, "only one volume should be reconstructed"
         res = results[0]
         # tag latest reconstructions
-        if isinstance(res, ResultsLocalRun) and res.results_urls is not None:
-            scan.set_latest_vol_reconstructions(res.results_urls)
+        if isinstance(res, ResultsLocalRun) and res.results_identifiers is not None:
+            scan.set_latest_vol_reconstructions(res.results_identifiers)
         # create future if needed
         if isinstance(res, ResultSlurmRun):
             future_tomo_obj = FutureTomwerObject(
@@ -292,8 +298,8 @@ class VolumeRunner(_NabuBaseReconstructor):
             def process(self, fn):
                 if fn.done() and not (fn.cancelled() or fn.exception()):
                     # update reconstruction urls only if processing succeed.
-                    recons_urls = self.f_partial()
-                    self.scan.add_latest_vol_reconstructions(recons_urls)
+                    recons_identifiers = self.f_partial()
+                    self.scan.add_latest_vol_reconstructions(recons_identifiers)
 
         file_format = config_to_dump["output"]["file_format"]
         callback = functools.partial(
@@ -303,9 +309,6 @@ class VolumeRunner(_NabuBaseReconstructor):
             file_format=file_format,
             scan=self.scan,
             slice_index=None,
-            start_z=config_to_dump["reconstruction"]["start_z"],
-            end_z=config_to_dump["reconstruction"]["end_z"],
-            expects_single_slice=False,
         )
 
         return (CallBack(callback, self.scan),)
@@ -346,7 +349,7 @@ class VolumeRunner(_NabuBaseReconstructor):
         """
         assert type(db) in (int, type(None))
         assert not pag == ctf == True, "cannot ask for both pag and ctf active"
-        if isinstance(self.scan, HDF5TomoScan):
+        if isinstance(self.scan, NXtomoScan):
             basename, _ = os.path.splitext(self.scan.master_file)
             basename = os.path.basename(basename)
             try:
@@ -371,7 +374,7 @@ class NabuVolumeTask(
     Task,
     SuperviseProcess,
     input_names=("data", "nabu_params"),
-    output_names=("data", "volumes", "future_tomo_obj"),
+    output_names=("data", "volumes", "future_tomo_obj", "screenshots"),
     optional_input_names=(
         "dry_run",
         "nabu_extra_params",  # some parameter that must update 'nabu_params' before launching the reconstruction. Such as z range...
@@ -462,8 +465,8 @@ class NabuVolumeTask(
             future_tomo_obj = None
         else:
             # tag latest reconstructions
-            if isinstance(res, ResultsLocalRun) and res.results_urls is not None:
-                scan.set_latest_vol_reconstructions(res.results_urls)
+            if isinstance(res, ResultsLocalRun) and res.results_identifiers is not None:
+                scan.set_latest_vol_reconstructions(res.results_identifiers)
             # create future if needed
             if isinstance(res, ResultSlurmRun):
                 future_tomo_obj = FutureTomwerObject(
@@ -527,6 +530,26 @@ class NabuVolumeTask(
         self.outputs.volumes = scan.latest_vol_reconstructions
         self.outputs.future_tomo_obj = future_tomo_obj
 
+        # build screenshots
+        try:
+            screenshots = {}
+            [
+                screenshots.update(
+                    select_screenshot_from_volume(
+                        VolumeFactory.create_tomo_object_from_identifier(rec_identifier)
+                    )
+                )
+                for rec_identifier in scan.latest_vol_reconstructions
+            ]
+        except Exception as e:
+            _logger.error(f"screenshot creation failed. Error is {e}")
+        else:
+            self.outputs.screenshots = IcatScreenshots(
+                data_dir=deduce_dataset_gallery_location(scan),
+                screenshots=screenshots,
+                scan=scan,
+            )
+
     def set_configuration(self, configuration: dict) -> None:
         Task.set_configuration(self, configuration=configuration)
         if "dry_run" in configuration:

tomwer/core/process/reconstruction/nabu/settings.py

@@ -33,8 +33,6 @@ import logging
 _logger = logging.getLogger(__name__)
 
 
-NABU_FILE_PER_GROUP = 100
-
 NABU_CONFIG_FILE_EXTENSION = ".cfg"
 
 NABU_CFG_FILE_FOLDER = "nabu_cfg_files"
@@ -60,3 +58,5 @@ else:
 
 
 NABU_CAST_APP_PATH = "nabu.app.cast_volume"
+
+NABU_MULTICOR_PATH = "nabu.app.multicor"

tomwer/core/process/reconstruction/nabu/utils.py

@@ -42,7 +42,7 @@ from silx.utils.enum import Enum as _Enum
 
 import tomwer.version
 from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.volume.edfvolume import EDFVolume
 from tomwer.core.volume.hdf5volume import HDF5Volume
@@ -148,9 +148,6 @@ def get_recons_volume_identifier(
     file_format: str,
     scan: TomwerScanBase,
     slice_index: typing.Union[int, None],
-    start_z: typing.Union[int, None],
-    end_z: typing.Union[int, None],
-    expects_single_slice: bool,
 ) -> tuple:
     """
     return tuple of DataUrl for existings slices
@@ -165,7 +162,7 @@ def get_recons_volume_identifier(
         file_name = ".".join((file_name, file_format))
         file_path = os.path.join(location, file_name)
 
-        if isinstance(scan, HDF5TomoScan):
+        if isinstance(scan, NXtomoScan):
             entry = scan.entry
         elif isinstance(scan, EDFTomoScan):
             entry = "entry"
@@ -223,6 +220,75 @@ def get_recons_volume_identifier(
     return tuple([volume.get_identifier() for volume in volumes])
 
 
+def get_multi_cor_recons_volume_identifiers(
+    scan: TomwerScanBase,
+    slice_index: int,
+    location: str,
+    file_prefix: str,
+    cors: tuple,
+    file_format: str,
+) -> dict:
+    """
+    util to retrieve Volumes (identifier) reconstructed by nabu-multicor
+
+    :param TomwerScanBase scan: scam processed by the nabu-multicor
+    :param str location: location of the files
+    :param tuple cors: cors for which we want the reconstructed slices. As this extension is created by nabu
+                       the cor reference is in absolute.
+    :param str file_format: file format of the reconstruction
+
+    :return: a dict with absolute cor value as key and the Volume identifier as value
+    """
+    _logger.info("Deduce volume identifiers for nabu-multicor")
+    if isinstance(slice_index, str):
+        slice_index = slice_index_to_int(slice_index=slice_index, scan=scan)
+    assert isinstance(
+        slice_index, int
+    ), "slice_index is expected to be an int or to be converted to it"
+    res = {}
+    if isinstance(scan, EDFTomoScan):
+        entry = "entry"
+    else:
+        entry = scan.entry
+
+    for cor in cors:
+        file_path = os.path.join(
+            location,
+            f"{file_prefix}_{cor:.3f}_{str(slice_index).zfill(5)}.{file_format}",
+        )
+
+        if file_format in ("hdf5", "h5", "hdf"):
+            file_path = os.path.join(
+                location,
+                f"{file_prefix}_{cor:.3f}",  # this level is expected to be removed on the future nabu versions
+                f"{file_prefix}_{cor:.3f}_{str(slice_index).zfill(5)}.{file_format}",
+            )
+            volume = HDF5Volume(
+                file_path=file_path,
+                data_path="/".join([entry, "reconstruction"]),
+            )
+        elif file_format in ("vol", "raw"):
+            volume = (RawVolume(file_path=file_path),)
+        elif file_format in ("jp2", "jp2k", "edf", "tiff"):
+            if file_format in ("jp2k", "jp2"):
+                constructor = JP2KVolume
+            elif file_format == "edf":
+                constructor = EDFVolume
+            elif file_format == "tiff":
+                constructor = TIFFVolume
+            else:
+                raise NotImplementedError
+            file_path = location
+            volume = constructor(
+                folder=os.path.dirname(file_path),
+                volume_basename=os.path.basename(file_path),
+            )
+        else:
+            raise ValueError(f"file_format {file_format} is not handled for now")
+        res[cor] = volume.get_identifier()
+    return res
+
+
 class _NabuMode(_Enum):
     FULL_FIELD = "standard acquisition"
     HALF_ACQ = "half acquisition"
@@ -302,24 +368,6 @@ class _NabuProcessing(_Enum):
         return (_NabuProcessing.RECONSTRUCTION,)
 
 
-class ConfigurationLevel(_Enum):
-    REQUIRED = "required"
-    OPTIONAL = "optional"
-    ADVANCED = "advanced"
-
-    def _get_num_value(self) -> int:
-        if self is self.REQUIRED:
-            return 0
-        elif self is self.OPTIONAL:
-            return 1
-        elif self is self.ADVANCED:
-            return 2
-
-    def __le__(self, other):
-        assert isinstance(other, ConfigurationLevel)
-        return self._get_num_value() <= other._get_num_value()
-
-
 class _NabuPostProcessing(_Enum):
     """Define all the post processing action available"""
 
@@ -374,9 +422,12 @@ class _NabuPaddingType(_Enum):
     EDGES = "edges"
 
 
-class _RingCorrectionMethod(_Enum):
+class RingCorrectionMethod(_Enum):
     NONE = "None"
     MUNCH = "munch"
+    VO = "vo"
+    MEAN_SUBTRACTION = "mean-subtraction"
+    MEAN_DIVISION = "mean-division"
 
 
 def nabu_std_err_has_error(errs: typing.Optional[bytes]):
@@ -397,6 +448,7 @@ def nabu_std_err_has_error(errs: typing.Optional[bytes]):
             or "self.module = SourceModule(self.src, **self.sourcemodule_kwargs)"
             in line
             or "return SourceModule(" in line
+            or "CUBLAS" in line
         )
 
     if errs is None:
@@ -438,3 +490,56 @@ def update_cfg_file_after_transfer(config_file_path, old_path, new_path):
         config=config_as_dict,
         options_level="advanced",
     )
+
+
+def slice_index_to_int(
+    slice_index: typing.Union[int, str], scan: TomwerScanBase
+) -> int:
+    """
+    cast a slice to an index. The slice can be a string in ['first', 'last', 'middle']
+    """
+    if slice_index == "fisrt":
+        return 0
+    elif slice_index == "last":
+        if scan is None:
+            # backward compatibility in the case the scan is not provided. Should not happen anymore
+            _logger.warning("Scan not provided. Consider the 2048 width detector")
+            return 2047
+        elif scan.dim_2 is not None:
+            return scan.dim_2 - 1
+        else:
+            # this could happen on some EDF scans. Not expected to happen for HDF5
+            _logger.warning("unable to get dim size, guess this is 2048 width")
+            # in this
+            return 2047
+    elif slice_index == "middle":
+        if scan is None:
+            # backward compatibility in the case the scan is not provided. Should not happen anymore
+            _logger.warning("Scan not provided. Consider the 1024 width detector")
+            # default middle.
+            return 1024
+        elif scan.dim_2 is None:
+            _logger.warning("unable to get dim size, guess this is 2048 width")
+            return 1024
+        else:
+            return scan.dim_2 // 2
+    else:
+        return int(slice_index)
+
+
+def get_nabu_multicor_file_prefix(scan):
+    if isinstance(scan, EDFTomoScan):
+        dataset_path = scan.path
+    elif isinstance(scan, NXtomoScan):
+        dataset_path = scan.master_file
+    else:
+        raise TypeError(f"{type(scan)} is not handled")
+
+    if os.path.isfile(dataset_path):  # hdf5
+        file_prefix = os.path.basename(dataset_path).split(".")[0]
+    elif os.path.isdir(dataset_path):
+        file_prefix = os.path.basename(dataset_path)
+    else:
+        raise ValueError(f"dataset location {scan.path} is neither a file or directory")
+    file_prefix += "_rec"  # avoid overwriting dataset
+    return file_prefix

tomwer/core/process/reconstruction/output.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+import os
+import logging
+from silx.utils.enum import Enum as _Enum
+from tomwer.io.utils.raw_and_processed_data import (
+    to_processed_data_path,
+    to_raw_data_path,
+)
+from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.utils.scanutils import format_output_location
+
+_logger = logging.getLogger(__name__)
+
+
+PROCESS_FOLDER_NAME = "reconstructed_volumes"
+
+
+class ProcessDataOutputDirMode(_Enum):
+    IN_SCAN_FOLDER = "same folder as scan"
+    PROCESSED_DATA_FOLDER = "PROCESSED_DATA folder"
+    RAW_DATA_FOLDER = "RAW_DATA folder"
+    OTHER = "other"
+
+
+class NabuOutputFileFormat(_Enum):
+    TIFF = "tiff"
+    HDF5 = "hdf5"
+    JP2K = "jp2"
+    EDF = "edf"
+    RAW = "vol"
+
+    @classmethod
+    def from_value(cls, value):
+        if isinstance(value, str):
+            value = value.lstrip(".")
+        return super().from_value(value)
+
+
+def get_file_format(file_str):
+    extension = os.path.splitext(file_str.lower())[-1]
+    extension = extension.lstrip(".")
+    if extension in ("tiff", "tif"):
+        return NabuOutputFileFormat.TIFF
+    elif extension in ("hdf5", "hdf", "h5"):
+        return NabuOutputFileFormat.HDF5
+    elif extension in ("jp2", "jp2k", "jpg2k"):
+        return NabuOutputFileFormat.JP2K
+    elif extension in ("edf",):
+        return NabuOutputFileFormat.EDF
+    elif extension in ("vol", "raw"):
+        return NabuOutputFileFormat.RAW
+    else:
+        raise ValueError(f"Unrecognized file extension {extension} from {file_str}")
+
+
+def get_output_folder_from_scan(
+    mode: ProcessDataOutputDirMode,
+    scan: TomwerScanBase,
+    nabu_location: str | None,
+    file_basename: str,
+    file_format: NabuOutputFileFormat,
+) -> tuple[str, str]:
+    """
+    :return: (location, location_cfg_files). Location is the nabu configuration field 'output/location' 'location_cfg_files' is the information on where to save the nabu configuration file
+
+    """
+    output_mode = ProcessDataOutputDirMode.from_value(mode)
+    file_format = NabuOutputFileFormat.from_value(file_format)
+
+    if output_mode is ProcessDataOutputDirMode.OTHER and nabu_location in ("", None):
+        _logger.info(
+            "output dir requested is other bit no path provided. Fall back on the output dir to the scan folder"
+        )
+        # note: this is only an info because we expect to pass by this one for all .ows configuration (before 1.3 version)
+        # as there was no different option by the time
+        output_mode = ProcessDataOutputDirMode.IN_SCAN_FOLDER
+
+    if output_mode is ProcessDataOutputDirMode.OTHER:
+        assert nabu_location not in (
+            "",
+            None,
+        ), "nabu_location not provided when expected"
+        location = format_output_location(nabu_location, scan=scan)
+        location_cfg_files = location
+    elif output_mode in (
+        ProcessDataOutputDirMode.IN_SCAN_FOLDER,
+        ProcessDataOutputDirMode.PROCESSED_DATA_FOLDER,
+        ProcessDataOutputDirMode.RAW_DATA_FOLDER,
+    ):
+        # otherwise default location will be the data root level
+        location = os.path.join(scan.path, PROCESS_FOLDER_NAME)
+        location_cfg_files = location
+        if file_format in (
+            NabuOutputFileFormat.EDF.value,
+            NabuOutputFileFormat.TIFF.value,
+            NabuOutputFileFormat.JP2K.value,
+        ):  # if user specify the location
+            location = "/".join([location, file_basename])
+        if output_mode is ProcessDataOutputDirMode.PROCESSED_DATA_FOLDER:
+            location = to_processed_data_path(location)
+            location_cfg_files = to_processed_data_path(location_cfg_files)
+        if output_mode is ProcessDataOutputDirMode.RAW_DATA_FOLDER:
+            location = to_raw_data_path(location)
+            location_cfg_files = to_raw_data_path(location_cfg_files)
+    else:
+        raise NotImplementedError(f"mode {output_mode.value} is not implemented yet")
+
+    return location, location_cfg_files