tomwer 1.3.27__py3-none-any.whl → 1.4.0__py3-none-any.whl

tomwer/core/process/control/timer.py

@@ -1,31 +1,5 @@
 # coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "12/12/2018"
+from __future__ import annotations
 
 
 import logging

tomwer/core/process/control/tomoobjseries.py

@@ -0,0 +1,12 @@
+from ewokscore.task import Task as EwoksTask
+
+
+class _TomoobjseriesPlaceHolder(
+    EwoksTask, input_names=["series"], output_names=["series"]
+):
+    """
+    task to define a tomography 'serie'
+    """
+
+    def run(self):
+        self.outputs.s = self.inputs.series

tomwer/core/process/drac/binning.py

@@ -0,0 +1,37 @@
+import numpy
+from silx.utils.enum import Enum as _Enum
+
+
+class Binning(_Enum):
+    ONE_BY_ONE = "1x1"
+    TWO_BY_TWO = "2x2"
+    FOUR_BY_FOUR = "4x4"
+    HEIGHT_BY_HEIGHT = "8x8"
+    SIXTEEN_BY_SIXTEEN = "16x16"
+    THIRTY_TWO_BY_THIRTY_TWO = "32x32"
+    SIXTY_FOUR_BY_SIXTY_FOUR = "64x64"
+    ONE_HUNDRED_TWENTY_HEIGHT_BY_ONE_HUNDRED_TWENTY_HEIGHT = "128x128"
+
+    @staticmethod
+    def _bin_data(data, binning):
+        if not isinstance(data, numpy.ndarray):
+            raise TypeError("data should be a numpy array")
+        if not data.ndim == 2:
+            raise ValueError("data is expected to be 2d")
+        binning = Binning.from_value(binning)
+        if binning is Binning.ONE_BY_ONE:
+            return data
+        elif binning is Binning.TWO_BY_TWO:
+            return data[::2, ::2]
+        elif binning is Binning.FOUR_BY_FOUR:
+            return data[::4, ::4]
+        elif binning is Binning.HEIGHT_BY_HEIGHT:
+            return data[::8, ::8]
+        elif binning is Binning.SIXTEEN_BY_SIXTEEN:
+            return data[::16, ::16]
+        elif binning is Binning.THIRTY_TWO_BY_THIRTY_TWO:
+            return data[::32, ::32]
+        elif binning is Binning.SIXTY_FOUR_BY_SIXTY_FOUR:
+            return data[::64, ::64]
+        else:
+            raise NotImplementedError

tomwer/core/process/drac/dracbase.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import os
+import numpy
+import logging
+from PIL import Image
+from silx.io.url import DataUrl
+
+from .binning import Binning
+from .output import OutputFormat
+
+from tomoscan.utils.io import filter_esrf_mounting_points
+from tomoscan.esrf.scan.utils import get_data, from_bliss_original_file_to_raw
+
+from tomwer.core.process.drac.output import DATASET_GALLERY_DIR_NAME
+from tomwer.core.tomwer_object import TomwerObject
+from tomwer.core.scan.scanbase import TomwerScanBase
+
+
+_logger = logging.getLogger(__name__)
+
+__all__ = [
+    "DracDatasetBase",
+]
+
+
+class DracDatasetBase:
+    """
+    Abstract class for a drac dataset (that should be publish to the data portal).
+    A drac dataset is defined by the following elements:
+    * a directory
+    * metadata
+    """
+
+    def __init__(self, data_dir: str, tomo_obj: TomwerObject) -> None:
+        """
+        :param data_dir: directory containing processed data
+        :param tomo_obj: tomwer object referee for the processing
+        """
+        self._data_dir = filter_esrf_mounting_points(data_dir)
+        self._tomo_obj = tomo_obj
+        self._gallery_file_binning = Binning.TWO_BY_TWO
+        self._gallery_output_format = OutputFormat.PNG
+        self._gallery_overwrite = True
+
+    @property
+    def gallery_output_binning(self) -> Binning:
+        "Binning factor to be used when saving gallery images"
+        return self._gallery_file_binning
+
+    @gallery_output_binning.setter
+    def gallery_output_binning(self, binning: Binning):
+        binning = Binning(binning)
+        self._gallery_file_binning = binning
+
+    @property
+    def gallery_output_format(self) -> OutputFormat:
+        "Output format of the images stored in the gallery"
+        return self._gallery_output_format
+
+    @gallery_output_format.setter
+    def gallery_output_format(self, format: OutputFormat):
+        format = OutputFormat(format)
+        self._gallery_output_format = format
+
+    @property
+    def gallery_overwrite(self) -> bool:
+        return self._gallery_overwrite
+
+    @gallery_overwrite.setter
+    def gallery_overwrite(self, overwrite: bool) -> None:
+        self._gallery_overwrite = overwrite
+
+    @property
+    def dataset_name(self) -> str:
+        """
+        Return the name of the dataset to be given to the drac dataset.
+        A drac dataset is store according to a (dataset_path, dataset_name). By default if several are pushed with the same key,
+        only the most recent one will be stored
+        """
+        raise NotImplementedError("Base class")
+
+    def get_gallery_dir(self) -> str:
+        return os.path.join(self.data_dir, DATASET_GALLERY_DIR_NAME)
+
+    def build_gallery(self):
+        """callback to build the drac-dataset gallery"""
+        pass
+
+    @property
+    def data_dir(self):
+        return self._data_dir
+
+    @property
+    def tomo_obj(self):
+        return self._tomo_obj
+
+    @property
+    def metadata(self) -> dict:
+        raise NotImplementedError("Base class")
+
+    @property
+    def bliss_raw_datasets(self) -> tuple[str]:
+        raise NotImplementedError("Base class")
+
+    @staticmethod
+    def from_scan_to_raws(scan: TomwerScanBase) -> tuple[str]:
+        assert isinstance(
+            scan, TomwerScanBase
+        ), f"scan should be an instance of {TomwerScanBase}. Got {type(scan)}"
+        bliss_original_files = scan.get_bliss_original_files()
+        if bliss_original_files is None:
+            bliss_original_files = ()
+        return tuple(
+            [
+                from_bliss_original_file_to_raw(bliss_original_file)
+                for bliss_original_file in bliss_original_files
+            ]
+        )
+
+    def to_dict(self):
+        return {
+            "output_dir": self.data_dir,
+            "drac_metadata": self.metadata,
+        }
+
+    def save_to_gallery(
+        self,
+        output_file_name: str,
+        image: numpy.ndarray | DataUrl,
+        lower_bound: float | None = None,
+        upper_bound: float | None = None,
+    ) -> None:
+        format = self.gallery_output_format
+        overwrite = self.gallery_overwrite
+        binning = self.gallery_output_binning
+
+        gallery_dir = self.get_gallery_dir()
+        os.makedirs(gallery_dir, exist_ok=True)
+
+        if isinstance(image, DataUrl):
+            image = get_data(image)
+
+        if image.ndim == 3 and image.shape[0] == 1:
+            image = image.reshape(image.shape[1:])
+        elif image.ndim != 2:
+            raise ValueError(f"Only 2D grayscale images are handled. Got {image.shape}")
+
+        # If both bounds are provided, apply clamping and normalization
+        # If not it will use the min/max of each image
+        if lower_bound is not None and upper_bound is not None:
+            image = numpy.clip(image, lower_bound, upper_bound)
+            image = image - lower_bound
+            image *= 255.0 / (upper_bound - lower_bound)
+        else:
+            min_val, max_val = image.min(), image.max()
+            image = image - min_val
+            if max_val > min_val:  # Avoid division by zero if image is constant
+                image *= 255.0 / (max_val - min_val)
+
+        # Binning the data if required
+        image = Binning._bin_data(data=image, binning=binning)
+
+        # Convert to a PIL Image and save
+        img = Image.fromarray(image.astype(numpy.uint8), mode="L")
+        if not output_file_name.endswith(f".{format.value}"):
+            output_file_name = output_file_name + f".{format.value}"
+        if not overwrite and os.path.exists(output_file_name):
+            raise OSError(f"File already exists ({output_file_name})")
+        img.save(output_file_name)

tomwer/core/process/drac/gallery.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import os
+
+from nxtomomill.converter.hdf5.utils import PROCESSED_DATA_DIR_NAME, RAW_DATA_DIR_NAME
+
+from tomwer.io.utils.raw_and_processed_data import to_processed_data_path
+from tomwer.core.scan.scanbase import TomwerScanBase
+from .output import DATASET_GALLERY_DIR_NAME, PROPOSAL_GALLERY_DIR_NAME
+
+__all__ = ["deduce_dataset_gallery_location", "deduce_proposal_GALLERY_location"]
+
+
+def deduce_dataset_gallery_location(scan_obj: TomwerScanBase) -> str:
+    """
+    From scan path deduce the 'dataset' path to the gallery.
+    Warning: dataset gallery is different then the 'proposal' gallery
+    """
+    if not isinstance(scan_obj, TomwerScanBase):
+        raise TypeError(f"'scan_obj' is expected to be an instance of {TomwerScanBase}")
+
+    file_path = os.path.abspath(scan_obj.path)
+
+    split_path = file_path.split(os.sep)
+    # reverse it to find the lower level value of 'PROCESSED_DATA_DIR_NAME' or 'RAW_DATA_DIR_NAME' if by any 'chance' has several in the path
+    # then we will replace the 'lower one' in the string. This is where the GALLERY will be added
+    split_path = split_path[::-1]
+    # check if already contain in a "PROCESSED_DATA" directory
+    try:
+        index_processed_data = split_path.index(PROCESSED_DATA_DIR_NAME)
+    except ValueError:
+        pass
+        index_processed_data = None
+    try:
+        index_raw_data = split_path.index(RAW_DATA_DIR_NAME)
+    except ValueError:
+        # if the value is not in the list
+        index_raw_data = None
+
+    if index_processed_data is None and index_raw_data is None:
+        # if not in any "PROCESSED_DATA" or "RAW_DATA" directory
+        return scan_obj.get_relative_file(
+            file_name=DATASET_GALLERY_DIR_NAME, with_dataset_prefix=False
+        )
+    elif index_processed_data is not None and index_raw_data is not None:
+        if index_raw_data > index_processed_data:
+            # if PROCESSED_DATA lower in the path than RAW_DATA
+            split_path[index_processed_data] = RAW_DATA_DIR_NAME
+    # reorder path to original
+    split_path = list(split_path[::-1])
+    split_path.append(DATASET_GALLERY_DIR_NAME)
+    # move it to PROCESSED_DATA when possible
+    path = os.sep.join(split_path)
+    path = to_processed_data_path(path)
+    return path
+
+
+def deduce_proposal_GALLERY_location(scan_obj: TomwerScanBase) -> str:
+    """
+    Policy: look if the scan_obj.path is in 'PROCESSED_DATA_DIR_NAME' or 'RAW_DATA_DIR_NAME' directories.
+    If find any (before any 'GALLERY_DIR_NAME' directory) replace it "GALLERY_DIR_NAME".
+    If none of those are found then create it at the same level as the scan
+
+    :param scan_obj: scan_obj for which we want the GALLERY directory
+    :return: gallery path (to save screenshots for example)
+    """
+    if not isinstance(scan_obj, TomwerScanBase):
+        raise TypeError(f"'scan_obj' is expected to be an instance of {TomwerScanBase}")
+
+    file_path = os.path.abspath(scan_obj.path)
+
+    split_path = file_path.split(os.sep)
+    # reverse it to find the lower level value of 'PROCESSED_DATA_DIR_NAME' or 'RAW_DATA_DIR_NAME' if by any 'chance' has several in the path
+    # then we will replace the 'lower one' in the string. This is where the GALLERY will be added
+    split_path = split_path[::-1]
+    # check if already contain in a "PROCESSED_DATA" directory
+    try:
+        index_processed_data = split_path.index(PROCESSED_DATA_DIR_NAME)
+    except ValueError:
+        pass
+        index_processed_data = None
+    try:
+        index_raw_data = split_path.index(RAW_DATA_DIR_NAME)
+    except ValueError:
+        # if the value is not in the list
+        index_raw_data = None
+
+    if index_processed_data is None and index_raw_data is None:
+        # if not in any "PROCESSED_DATA" or "RAW_DATA" directory
+        return scan_obj.get_relative_file(
+            file_name=PROPOSAL_GALLERY_DIR_NAME, with_dataset_prefix=False
+        )
+    elif index_processed_data is not None and index_raw_data is not None:
+        if index_raw_data > index_processed_data:
+            # if PROCESSED_DATA lower in the path than RAW_DATA
+            split_path[index_processed_data] = PROPOSAL_GALLERY_DIR_NAME
+        else:
+            # if RAW_DATA lower in the path than PROCESSED_DATA
+            split_path[index_raw_data] = PROPOSAL_GALLERY_DIR_NAME
+    elif index_raw_data is not None:
+        # if the path contains only PROCESSED_DATA or RAW_DATA (expected behavior for online acquistion)
+        split_path[index_raw_data] = PROPOSAL_GALLERY_DIR_NAME
+    else:
+        assert index_processed_data is not None, "index_processed_data is None"
+        split_path[index_processed_data] = PROPOSAL_GALLERY_DIR_NAME
+
+    # reorder path to original
+    split_path = split_path[::-1]
+    return os.sep.join(split_path)

tomwer/core/process/drac/output.py

@@ -0,0 +1,12 @@
+from silx.utils.enum import Enum as _Enum
+
+
+PROPOSAL_GALLERY_DIR_NAME = "GALLERY"
+DATASET_GALLERY_DIR_NAME = "gallery"
+
+
+class OutputFormat(_Enum):
+    """possible output format to save screenshots"""
+
+    PNG = "png"
+    JPEG = "jpg"

tomwer/core/process/drac/processeddataset.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import os
+import numpy
+from .dracbase import DracDatasetBase
+from tomoscan.esrf.volume.singleframebase import VolumeSingleFrameBase
+from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.volume.volumebase import TomwerVolumeBase
+from tomwer.core.process.reconstruction.nabu.plane import NabuPlane
+from tomwer.core.process.reconstruction.output import (
+    PROCESS_FOLDER_RECONSTRUCTED_VOLUMES,
+)
+
+
+__all__ = ["DracReconstructedVolumeDataset"]
+
+
+class DracReconstructedVolumeDataset(DracDatasetBase):
+    """
+    Class to associate reconstructed volume(s) to an drac (processed) dataset
+    """
+
+    def __init__(self, tomo_obj: TomwerVolumeBase, source_scan: TomwerScanBase) -> None:
+        if not isinstance(tomo_obj, TomwerVolumeBase):
+            raise TypeError(
+                f"tomo_obj should be an instance of {TomwerVolumeBase}. Got {type(tomo_obj)}"
+            )
+
+        super().__init__(tomo_obj=tomo_obj, data_dir=tomo_obj.icat_data_dir)
+        self._n_slices_per_axis = 3
+        self.__source_scan = source_scan
+        self.__bliss_raw_dataset = self.from_scan_to_raws(scan=self.__source_scan)
+
+    @staticmethod
+    def make_serializable(obj):
+        """
+        Recursively convert numpy arrays to lists and replace string "None" with None in a given object.
+        """
+        if isinstance(obj, dict):
+            return {
+                key: DracReconstructedVolumeDataset.make_serializable(value)
+                for key, value in obj.items()
+            }
+        elif isinstance(obj, (list, tuple)):
+            return [
+                DracReconstructedVolumeDataset.make_serializable(item) for item in obj
+            ]
+        elif isinstance(obj, numpy.ndarray):
+            return obj.tolist()
+        else:
+            return obj
+
+    @property
+    def metadata(self) -> dict:
+        assert isinstance(self.tomo_obj, TomwerVolumeBase)
+        metadata = self.tomo_obj.build_drac_metadata()
+        metadata["Sample_name"] = self.source_scan.sample_name
+        return metadata
+
+    @property
+    def dataset_name(self) -> str:
+        """name to give to the drac (processed) dataset."""
+        return PROCESS_FOLDER_RECONSTRUCTED_VOLUMES
+
+    @property
+    def source_scan(self) -> TomwerScanBase:
+        return self.__source_scan
+
+    @property
+    def bliss_raw_datasets(self) -> tuple[str]:
+        return self.__bliss_raw_dataset
+
+    @property
+    def n_slices_per_axis(self) -> int:
+        "number of slices to sample per axis"
+        return self._n_slices_per_axis
+
+    @n_slices_per_axis.setter
+    def n_slices_per_axis(self, n: int) -> None:
+        self._n_slices_per_axis = n
+
+    def get_slices_to_extract(self) -> tuple[tuple[int, tuple[int]]]:
+        """
+        Compute the slices to be retrieve along each dimension according to 'n_slices_per_axis'
+
+        return tuple (A) is a two elements tuple. First element if the axis (B).
+        Second is the tuple of indices to extract along the axis (B)
+        indices are equally spaced in each dimensions
+        """
+        result: list[tuple[int, tuple[int]]] = []
+        volume = self.tomo_obj
+        if not isinstance(volume, TomwerVolumeBase):
+            raise TypeError(
+                f"Volume is expected to be an instance of {TomwerVolumeBase}. Got {type(volume)}"
+            )
+        volume_shape = volume.get_volume_shape()
+        for axis, axis_len in enumerate(volume_shape):
+            for slice_index in numpy.linspace(
+                0, axis_len, endpoint=False, num=self.n_slices_per_axis + 1
+            )[1:]:
+                result.append((axis, numpy.round(slice_index).astype(numpy.uint16)))
+        return tuple(result)
+
+    def build_gallery(self):
+        gallery_dir = self.get_gallery_dir()
+        volume = self.tomo_obj
+        slices = volume.get_slices(slices=self.get_slices_to_extract())
+
+        # Stack all slices into a single array to compute the global percentiles
+        all_slices = numpy.concatenate([slice_ for slice_ in slices.values()])
+        lower_bound = numpy.percentile(all_slices, 1)
+        upper_bound = numpy.percentile(all_slices, 99)
+
+        for (axis, slice_index), slice_ in slices.items():
+            self.save_to_gallery(
+                output_file_name=self.get_output_file_name(
+                    output_dir=gallery_dir,
+                    axis=axis,
+                    slice_index=slice_index,
+                    volume=volume,
+                ),
+                image=slice_,
+                lower_bound=lower_bound,
+                upper_bound=upper_bound,
+            )
+
+    @staticmethod
+    def get_output_file_name(
+        output_dir: str, axis: int, slice_index: int, volume: TomwerVolumeBase
+    ) -> str:
+        axis = NabuPlane.from_value(axis)
+        if isinstance(volume, VolumeSingleFrameBase):
+            basename = volume.get_volume_basename()
+        else:
+            basename = os.path.splitext(os.path.basename(volume.data_url.file_path()))[
+                0
+            ]
+        axis_folder = os.path.join(
+            os.path.abspath(output_dir),
+            str(axis.value),
+        )
+        os.makedirs(axis_folder, exist_ok=True)
+        # warning: for ordering drac needs to have the axis in a dedicated folder
+        return os.path.join(
+            axis_folder,
+            f"{basename}_capture_{axis.value}_{str(slice_index).zfill(6)}",
+        )

tomwer/core/process/drac/publish.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+import logging
+
+from tomwer.core.process.task import Task
+from tomwer.core.process.drac.dracbase import DracDatasetBase
+
+from processview.core.manager import DatasetState, ProcessManager
+
+try:
+    from pyicat_plus.client.main import IcatClient  # noqa F401
+except ImportError:
+    has_pyicat_plus = False
+else:
+    has_pyicat_plus = True
+
+_logger = logging.getLogger(__name__)
+
+__all__ = [
+    "PublishICatDatasetTask",
+]
+
+
+class PublishICatDatasetTask(
+    Task,
+    input_names=(
+        "data_portal_processed_datasets",
+        "beamline",
+        "proposal",
+        "dataset",
+    ),
+    optional_input_names=(
+        "dry_run",
+        "__process__",
+    ),
+):
+    """
+    publish a list of 'IcatDataBase' instances.
+
+    `IcatDataBase` provide API to retrieve data and metadata to be publish
+
+    input field:
+    * data_portal_processed_datasets: list of 'DracDatasetBase' instances.
+    * beamline: name of the beamline (bm05, id19...)
+    * proposal: proposal name
+    """
+
+    def run(self):
+        for icat_data in self.inputs.data_portal_processed_datasets:
+            if not isinstance(icat_data, DracDatasetBase):
+                raise TypeError(f"icat_data should be an instance of {DracDatasetBase}")
+
+            # build gallery (if needed by the icat data)
+
+            process = self.get_input_value("__process__", None)
+
+            if process is not None and icat_data.tomo_obj is not None:
+                ProcessManager().notify_dataset_state(
+                    dataset=icat_data.tomo_obj,
+                    process=process(),
+                    state=DatasetState.ON_GOING,
+                )
+
+            icat_data.build_gallery()
+
+            try:
+                self.publish_to_data_portal(
+                    path=icat_data.data_dir,
+                    metadata=icat_data.metadata,
+                    raw=icat_data.bliss_raw_datasets,
+                    dataset=icat_data.dataset_name,
+                )
+
+            except Exception as e:
+                if process is not None and icat_data.tomo_obj is not None:
+                    ProcessManager().notify_dataset_state(
+                        dataset=icat_data.tomo_obj,
+                        process=process(),
+                        state=DatasetState.FAILED,
+                    )
+                    raise e
+            else:
+                if process is not None and icat_data.tomo_obj is not None:
+                    ProcessManager().notify_dataset_state(
+                        dataset=icat_data.tomo_obj,
+                        process=process(),
+                        state=DatasetState.SUCCEED,
+                    )
+
+    def publish_to_data_portal(self, path: str, metadata: dict, raw: str, dataset: str):
+        """publish path to data_portal (drac) with given metadata"""
+        if not self.inputs.dry_run:
+            if not has_pyicat_plus:
+                raise RuntimeError(
+                    "pyicat_plus not installed it. Please install it to be able to publish dataset to icat"
+                )
+            icat_client = IcatClient(
+                metadata_urls=("bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613")
+            )
+
+            _logger.info(
+                "publish to icat: %s",
+                {
+                    "path": path,
+                    "beamline": self.inputs.beamline,
+                    "proposal": self.inputs.proposal,
+                    "raw": raw,
+                    "dataset": dataset,
+                    "metadata": metadata,
+                },
+            )
+            icat_client.store_processed_data(
+                beamline=self.inputs.beamline,
+                proposal=self.inputs.proposal,
+                dataset=dataset,
+                path=path,
+                metadata=metadata,
+                raw=raw,
+            )