tomwer 1.2.1__py3-none-any.whl → 1.3.12__py3-none-any.whl

tomwer/core/process/reconstruction/nabu/utils.py

@@ -41,8 +41,9 @@ from nabu.pipeline.fullfield.nabu_config import (
 from silx.utils.enum import Enum as _Enum
 
 import tomwer.version
+from tomwer.core.process.reconstruction.nabu.plane import NabuPlane
 from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.volume.edfvolume import EDFVolume
 from tomwer.core.volume.hdf5volume import HDF5Volume
@@ -148,27 +149,30 @@ def get_recons_volume_identifier(
     file_format: str,
     scan: TomwerScanBase,
     slice_index: typing.Union[int, None],
-    start_z: typing.Union[int, None],
-    end_z: typing.Union[int, None],
-    expects_single_slice: bool,
+    axis: NabuPlane,
 ) -> tuple:
     """
     return tuple of DataUrl for existings slices
     """
+    axis = NabuPlane.from_value(axis)
     file_format = file_format.lower()
     if file_format in ("hdf5", "h5", "hdf"):
         if slice_index is not None:
             # case of a single hdf5 file
-            file_name = "_".join((file_prefix, str(slice_index).zfill(6)))
+            file_name = "_".join(
+                (file_prefix, "plane", axis.value, str(slice_index).zfill(6))
+            )
         else:
             file_name = file_prefix
         file_name = ".".join((file_name, file_format))
         file_path = os.path.join(location, file_name)
 
-        if isinstance(scan, HDF5TomoScan):
+        if isinstance(scan, NXtomoScan):
             entry = scan.entry
         elif isinstance(scan, EDFTomoScan):
             entry = "entry"
+        else:
+            raise NotImplementedError(f"unrecognized scan type ({type(scan)})")
 
         volumes = (
             HDF5Volume(
@@ -179,7 +183,9 @@ def get_recons_volume_identifier(
     elif file_format in ("vol", "raw"):
         if slice_index is not None:
             # case of a single hdf5 file
-            file_name = "_".join((file_prefix, str(slice_index).zfill(6)))
+            file_name = "_".join(
+                (file_prefix, "plane", axis.value, str(slice_index).zfill(6))
+            )
         else:
             file_name = file_prefix
         file_name = ".".join((file_name, file_format))
@@ -223,6 +229,79 @@ def get_recons_volume_identifier(
     return tuple([volume.get_identifier() for volume in volumes])
 
 
+def get_multi_cor_recons_volume_identifiers(
+    scan: TomwerScanBase,
+    slice_index: int,
+    location: str,
+    file_prefix: str,
+    cors: tuple,
+    file_format: str,
+    axis=NabuPlane.XY,
+) -> dict:
+    """
+    util to retrieve Volumes (identifier) reconstructed by nabu-multicor
+
+    :param TomwerScanBase scan: scam processed by the nabu-multicor
+    :param str location: location of the files
+    :param tuple cors: cors for which we want the reconstructed slices. As this extension is created by nabu
+                       the cor reference is in absolute.
+    :param str file_format: file format of the reconstruction
+
+    :return: a dict with absolute cor value as key and the Volume identifier as value
+    """
+    _logger.info("Deduce volume identifiers for nabu-multicor")
+    if isinstance(slice_index, str):
+        slice_index = slice_index_to_int(
+            slice_index=slice_index,
+            scan=scan,
+            axis=axis,  # for now we always expect the multicor to be done along Z
+        )
+    assert isinstance(
+        slice_index, int
+    ), "slice_index is expected to be an int or to be converted to it"
+    res = {}
+    if isinstance(scan, EDFTomoScan):
+        entry = "entry"
+    else:
+        entry = scan.entry
+
+    for cor in cors:
+        file_path = os.path.join(
+            location,
+            f"{file_prefix}_{cor:.3f}_{str(slice_index).zfill(5)}.{file_format}",
+        )
+
+        if file_format in ("hdf5", "h5", "hdf"):
+            file_path = os.path.join(
+                location,
+                f"{file_prefix}_{cor:.3f}_{str(slice_index).zfill(5)}.{file_format}",
+            )
+            volume = HDF5Volume(
+                file_path=file_path,
+                data_path="/".join([entry, "reconstruction"]),
+            )
+        elif file_format in ("vol", "raw"):
+            volume = (RawVolume(file_path=file_path),)
+        elif file_format in ("jp2", "jp2k", "edf", "tiff"):
+            if file_format in ("jp2k", "jp2"):
+                constructor = JP2KVolume
+            elif file_format == "edf":
+                constructor = EDFVolume
+            elif file_format == "tiff":
+                constructor = TIFFVolume
+            else:
+                raise NotImplementedError
+            file_path = location
+            volume = constructor(
+                folder=os.path.dirname(file_path),
+                volume_basename=os.path.basename(file_path),
+            )
+        else:
+            raise ValueError(f"file_format {file_format} is not handled for now")
+        res[cor] = volume.get_identifier()
+    return res
+
+
 class _NabuMode(_Enum):
     FULL_FIELD = "standard acquisition"
     HALF_ACQ = "half acquisition"
@@ -302,24 +381,6 @@ class _NabuProcessing(_Enum):
         return (_NabuProcessing.RECONSTRUCTION,)
 
 
-class ConfigurationLevel(_Enum):
-    REQUIRED = "required"
-    OPTIONAL = "optional"
-    ADVANCED = "advanced"
-
-    def _get_num_value(self) -> int:
-        if self is self.REQUIRED:
-            return 0
-        elif self is self.OPTIONAL:
-            return 1
-        elif self is self.ADVANCED:
-            return 2
-
-    def __le__(self, other):
-        assert isinstance(other, ConfigurationLevel)
-        return self._get_num_value() <= other._get_num_value()
-
-
 class _NabuPostProcessing(_Enum):
     """Define all the post processing action available"""
 
@@ -374,9 +435,12 @@ class _NabuPaddingType(_Enum):
     EDGES = "edges"
 
 
-class _RingCorrectionMethod(_Enum):
+class RingCorrectionMethod(_Enum):
     NONE = "None"
     MUNCH = "munch"
+    VO = "vo"
+    MEAN_SUBTRACTION = "mean-subtraction"
+    MEAN_DIVISION = "mean-division"
 
 
 def nabu_std_err_has_error(errs: typing.Optional[bytes]):
@@ -390,10 +454,19 @@ def nabu_std_err_has_error(errs: typing.Optional[bytes]):
         return (
             "warning" in line
             or "Warning" in line
+            or "cupyx.jit.rawkernel" in line
             or "deprecated" in line
             or line.replace(" ", "") == ""
             or "unable to load" in line
             or "deprecated" in line
+            or "self.module = SourceModule(self.src, **self.sourcemodule_kwargs)"
+            in line
+            or "return SourceModule(" in line
+            or "CUBLAS" in line
+            or "Not supported for EDF"
+            in line  # debatable but very disturbing from the gui side... anyway EDF days are coming to an end
+            or "PerformanceWarning" in line
+            or "jitify._init_module()" in line
         )
 
     if errs is None:
@@ -435,3 +508,68 @@ def update_cfg_file_after_transfer(config_file_path, old_path, new_path):
         config=config_as_dict,
         options_level="advanced",
     )
+
+
+def slice_index_to_int(
+    slice_index: typing.Union[int, str],
+    scan: TomwerScanBase,
+    axis=NabuPlane.XY,
+) -> int:
+    """
+    cast a slice to an index. The slice can be a string in ['first', 'last', 'middle']
+    """
+    axis = NabuPlane.from_value(axis)
+    if slice_index == "fisrt":
+        return 0
+    elif slice_index == "last":
+        if scan is None:
+            # backward compatibility in the case the scan is not provided. Should not happen anymore
+            _logger.warning("Scan not provided. Consider the 2048 width detector")
+            return 2047
+        elif scan.dim_2 is not None:
+            return scan.dim_2 - 1
+        else:
+            # this could happen on some EDF scans. Not expected to happen for HDF5
+            _logger.warning("unable to get dim size, guess this is 2048 width")
+            # in this
+            return 2047
+    elif slice_index == "middle":
+        if scan is None:
+            # backward compatibility in the case the scan is not provided. Should not happen anymore
+            _logger.warning("Scan not provided. Consider the 1024 width detector")
+            # default middle.
+            return 1024
+        elif axis is NabuPlane.XY:
+            if scan.dim_2 is None:
+                _logger.warning("unable to get dim size, guess this is 2048 height")
+                return 1024
+            else:
+                return scan.dim_2 // 2
+        elif axis in (NabuPlane.YZ, NabuPlane.XZ):
+            if scan.dim_1 is None:
+                _logger.warning("unable to get dim size, guess this is 2048 width")
+                return 1024
+            else:
+                return scan.dim_1 // 2
+        else:
+            raise ValueError(f"axis {axis} is not handled")
+    else:
+        return int(slice_index)
+
+
+def get_nabu_multicor_file_prefix(scan):
+    if isinstance(scan, EDFTomoScan):
+        dataset_path = scan.path
+    elif isinstance(scan, NXtomoScan):
+        dataset_path = scan.master_file
+    else:
+        raise TypeError(f"{type(scan)} is not handled")
+
+    if os.path.isfile(dataset_path):  # hdf5
+        file_prefix = os.path.basename(dataset_path).split(".")[0]
+    elif os.path.isdir(dataset_path):
+        file_prefix = os.path.basename(dataset_path)
+    else:
+        raise ValueError(f"dataset location {scan.path} is neither a file or directory")
+    file_prefix += "_rec"  # avoid overwriting dataset
+    return file_prefix

tomwer/core/process/reconstruction/output.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+import os
+import logging
+from silx.utils.enum import Enum as _Enum
+from tomwer.io.utils.raw_and_processed_data import (
+    to_processed_data_path,
+    to_raw_data_path,
+)
+from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.utils.scanutils import format_output_location
+
+_logger = logging.getLogger(__name__)
+
+
+PROCESS_FOLDER_NAME = "reconstructed_volumes"
+
+
+class ProcessDataOutputDirMode(_Enum):
+    IN_SCAN_FOLDER = "same folder as scan"
+    PROCESSED_DATA_FOLDER = "PROCESSED_DATA folder"
+    RAW_DATA_FOLDER = "RAW_DATA folder"
+    OTHER = "other"
+
+
+class NabuOutputFileFormat(_Enum):
+    TIFF = "tiff"
+    HDF5 = "hdf5"
+    JP2K = "jp2"
+    EDF = "edf"
+    RAW = "vol"
+
+    @classmethod
+    def from_value(cls, value):
+        if isinstance(value, str):
+            value = value.lstrip(".")
+        return super().from_value(value)
+
+
+def get_file_format(file_str):
+    extension = os.path.splitext(file_str.lower())[-1]
+    extension = extension.lstrip(".")
+    if extension in ("tiff", "tif"):
+        return NabuOutputFileFormat.TIFF
+    elif extension in ("hdf5", "hdf", "h5"):
+        return NabuOutputFileFormat.HDF5
+    elif extension in ("jp2", "jp2k", "jpg2k"):
+        return NabuOutputFileFormat.JP2K
+    elif extension in ("edf",):
+        return NabuOutputFileFormat.EDF
+    elif extension in ("vol", "raw"):
+        return NabuOutputFileFormat.RAW
+    else:
+        raise ValueError(f"Unrecognized file extension {extension} from {file_str}")
+
+
+def get_output_folder_from_scan(
+    mode: ProcessDataOutputDirMode,
+    scan: TomwerScanBase,
+    nabu_location: str | None,
+    file_basename: str,
+    file_format: NabuOutputFileFormat,
+) -> tuple[str, str]:
+    """
+    :return: (location, location_cfg_files). Location is the nabu configuration field 'output/location' 'location_cfg_files' is the information on where to save the nabu configuration file
+
+    """
+    output_mode = ProcessDataOutputDirMode.from_value(mode)
+    file_format = NabuOutputFileFormat.from_value(file_format)
+
+    if output_mode is ProcessDataOutputDirMode.OTHER and nabu_location in ("", None):
+        _logger.info(
+            "output dir requested is other bit no path provided. Fall back on the output dir to the scan folder"
+        )
+        # note: this is only an info because we expect to pass by this one for all .ows configuration (before 1.3 version)
+        # as there was no different option by the time
+        output_mode = ProcessDataOutputDirMode.IN_SCAN_FOLDER
+
+    if output_mode is ProcessDataOutputDirMode.OTHER:
+        assert nabu_location not in (
+            "",
+            None,
+        ), "nabu_location not provided when expected"
+        location = format_output_location(nabu_location, scan=scan)
+        location_cfg_files = location
+    elif output_mode in (
+        ProcessDataOutputDirMode.IN_SCAN_FOLDER,
+        ProcessDataOutputDirMode.PROCESSED_DATA_FOLDER,
+        ProcessDataOutputDirMode.RAW_DATA_FOLDER,
+    ):
+        # otherwise default location will be the data root level
+        location = os.path.join(scan.path, PROCESS_FOLDER_NAME)
+        location_cfg_files = location
+        if file_format in (
+            NabuOutputFileFormat.EDF.value,
+            NabuOutputFileFormat.TIFF.value,
+            NabuOutputFileFormat.JP2K.value,
+        ):  # if user specify the location
+            location = "/".join([location, file_basename])
+        if output_mode is ProcessDataOutputDirMode.PROCESSED_DATA_FOLDER:
+            location = to_processed_data_path(location)
+            location_cfg_files = to_processed_data_path(location_cfg_files)
+        if output_mode is ProcessDataOutputDirMode.RAW_DATA_FOLDER:
+            location = to_raw_data_path(location)
+            location_cfg_files = to_raw_data_path(location_cfg_files)
+    else:
+        raise NotImplementedError(f"mode {output_mode.value} is not implemented yet")
+
+    return location, location_cfg_files