tomwer 1.3.26__py3-none-any.whl → 1.4.0__py3-none-any.whl

tomwer/core/process/edit/nxtomoeditor.py

@@ -0,0 +1,307 @@
+from __future__ import annotations
+
+import gc
+import logging
+import h5py
+import numpy
+
+from collections import namedtuple
+from typing import Any
+
+from silx.io.utils import h5py_read_dataset
+
+from tomoscan.io import HDF5File
+
+from nxtomo.paths.nxtomo import get_paths as get_nexus_paths
+from nxtomo.nxobject.nxtransformations import NXtransformations
+from nxtomo.utils.transformation import (
+    DetYFlipTransformation,
+    DetZFlipTransformation,
+)
+
+from tomwer.core.process.task import Task
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+
+_EditorFieldInfo = namedtuple(
+    "_EditorFieldInfo", ["nexus_path", "name", "expected_type", "units", "n_value"]
+)
+
+_logger = logging.getLogger(__name__)
+
+
+class NXtomoEditorKeys:
+    """namespace to store all keys used by the nxtomo editor"""
+
+    ENERGY = "instrument.beam.energy"
+    X_PIXEL_SIZE = "instrument.detector.x_pixel_size"
+    Y_PIXEL_SIZE = "instrument.detector.y_pixel_size"
+    SAMPLE_DETECTOR_DISTANCE = "instrument.detector.distance"
+    FIELD_OF_VIEW = "instrument.detector.field_of_view"
+    X_FLIPPED = "instrument.detector.x_flipped"
+    Y_FLIPPED = "instrument.detector.y_flipped"
+    X_TRANSLATION = "sample.x_translation"
+    Y_TRANSLATION = "sample.y_translation"
+    Z_TRANSLATION = "sample.z_translation"
+
+
+class NXtomoEditorTask(
+    Task,
+    input_names=("data", "configuration"),
+    output_names=("data",),
+):
+    """
+    task to edit a couple of field of a NXtomo
+    """
+
+    Y_FLIP_MATRIX = DetYFlipTransformation(flip=True).as_matrix()
+    # up-down flip matrix
+    Z_FLIP_MATRIX = DetZFlipTransformation(flip=True).as_matrix()
+    # left-right flip matrix
+    Y_AND_Z_flip_MATRIX = numpy.matmul(
+        Y_FLIP_MATRIX,
+        Z_FLIP_MATRIX,
+    )
+
+    def run(self):
+        scan = self.inputs.data
+        if scan is None:
+            _logger.warning("no scan found to be saved")
+            return
+        if not isinstance(scan, NXtomoScan):
+            raise TypeError(
+                f"data is expected to be an instance of {NXtomoScan}. Got {type(scan)}"
+            )
+
+        configuration = self.inputs.configuration
+        if not isinstance(configuration, dict):
+            raise TypeError(
+                f"configuration is expected to be an instance of {dict}. Got {type(configuration)}"
+            )
+
+        n_frames = len(scan.image_key_control)
+        mapping = self.build_mapping(
+            nexus_version=scan.nexus_version, n_frames=n_frames
+        )
+
+        x_flip = configuration.pop(NXtomoEditorKeys.X_FLIPPED, None)
+        y_flip = configuration.pop(NXtomoEditorKeys.Y_FLIPPED, None)
+
+        with HDF5File(scan.master_file, mode="a") as h5f:
+            entry = h5f[scan.entry]
+
+            for field, field_value in configuration.items():
+                if field not in mapping:
+                    raise ValueError(f"field unknown ({field})")
+                self.__write_to_file(
+                    entry=entry,
+                    path=mapping[field].nexus_path,
+                    value=field_value,
+                    name=mapping[field].name,
+                    expected_type=mapping[field].expected_type,
+                    units=mapping[field].units,
+                    n_value=mapping[field].n_value,
+                )
+
+            # solve NXtransformations
+            if x_flip is not None or y_flip is not None:
+                nx_transformations, detector_transformation_path = (
+                    self.get_detector_transformations(
+                        scan_entry=scan.entry,
+                        x_flip=x_flip if x_flip is not None else scan.x_flipped,
+                        y_flip=y_flip if y_flip is not None else scan.y_flipped,
+                        nexus_version=scan.nexus_version,
+                    )
+                )
+            else:
+                nx_transformations = detector_transformation_path = None
+
+        # make sure the file has been removed (fix https://gitlab.esrf.fr/tomotools/tomwer/-/merge_requests/874#note_366688).
+        gc.collect()
+        if nx_transformations is not None:
+            nx_transformations.save(
+                file_path=scan.master_file,
+                data_path=detector_transformation_path,
+                nexus_path_version=scan.nexus_version,
+                overwrite=True,
+            )
+        # clear caches to make sure all modifications will be considered
+        scan.clear_caches()
+        scan.clear_frames_caches()
+        self.outputs.data = scan
+
+    @staticmethod
+    def get_detector_transformations(
+        scan_entry: str, x_flip: bool, y_flip: bool, nexus_version: float | None
+    ) -> dict:
+        nexus_paths = get_nexus_paths(nexus_version)
+        if nexus_paths.nx_detector_paths.NX_TRANSFORMATIONS is None:
+            # old NXtomo are not handling NX_TRANSFORMATIONS
+            _logger.debug(
+                "Old version of NXtomo found. No information about transformation will be saved"
+            )
+            return
+
+        nx_transformations = NXtransformations()
+        nx_transformations.add_transformation(DetYFlipTransformation(flip=y_flip))
+        nx_transformations.add_transformation(DetZFlipTransformation(flip=x_flip))
+
+        detector_transformation_path = "/".join(
+            (
+                scan_entry,
+                nexus_paths.INSTRUMENT_PATH,
+                nexus_paths.nx_instrument_paths.DETECTOR_PATH,
+                nexus_paths.nx_detector_paths.NX_TRANSFORMATIONS,
+            ),
+        )
+        return nx_transformations, detector_transformation_path
+
+    @staticmethod
+    def __write_to_file(
+        entry: h5py.Group,
+        path: str,
+        value: Any,
+        name: str,
+        expected_type: type,
+        n_value: int = 1,
+        units=None,
+    ) -> None:
+        if path is None:
+            # if the path does not exists (no handled by this version of nexus for example)
+            return
+
+        # try to cast the value
+        if isinstance(value, str):
+            value = value.replace(" ", "")
+            if value.lower() == "none" or "..." in value:
+                # if value is not defined or is an array not overwrite by the user (case of the ... )
+                return
+        elif value is None:
+            pass
+        else:
+            try:
+                value = expected_type(value)
+            except (ValueError, TypeError) as e:
+                _logger.error(f"Fail to overwrite {name} of {entry.name}. Error is {e}")
+                return
+
+        if path in entry:
+            if not NXtomoEditorTask.isFieldValueDifferent(
+                dataset=entry[path], new_value=value, units=units
+            ):
+                # if no need to overwrite
+                return
+            else:
+                del entry[path]
+        if value is None:
+            return
+        elif n_value == 1:
+            entry[path] = value
+        else:
+            entry[path] = numpy.array([value] * n_value)
+        if units is not None:
+            entry[path].attrs["units"] = units
+
+    @staticmethod
+    def isFieldValueDifferent(dataset: h5py.Dataset, new_value, units) -> bool:
+        """
+        return False if the given value is the same as the one stored.
+
+        This is a small improvement to avoid rewrite field if we can avoid it.
+        The reason behind is that the orange widget will update the field when it receive a scan (of the unlock field).
+        But then users can edit any field. As we don't want to bother with complex stuff like for each field keep track if
+        It has been modified or not the orange widget will always require to modify all possible field even is not modified.
+        """
+        current_value = h5py_read_dataset(dataset)
+        attrs = dataset.attrs
+        current_unit = attrs.get("units", attrs.get("unit", None))
+        if units != current_unit:
+            # if the unit is not the same, even if the value is the same we will overwrite it
+            return True
+        else:
+            if isinstance(new_value, numpy.ndarray) and isinstance(
+                current_value, numpy.ndarray
+            ):
+                return not numpy.array_equal(new_value, current_value)
+            elif numpy.isscalar(current_value) and numpy.isscalar(new_value):
+                return current_value != new_value
+            else:
+                return True
+
+    @staticmethod
+    def build_mapping(
+        nexus_version: float | None, n_frames: int
+    ) -> dict[NXtomoEditorKeys, _EditorFieldInfo]:
+        """
+        for the different field that can be edited we provide an instance of _EditorFieldInfo
+        with all metadata needed to save this particular field.
+
+        Note: X_flip and y_flip are saved differently as the two parameters are coupled we want either to save both or none of it.
+        """
+        nexus_paths = get_nexus_paths(nexus_version)
+
+        return {
+            # energy
+            NXtomoEditorKeys.ENERGY: _EditorFieldInfo(
+                nexus_path=nexus_paths.ENERGY_PATH,
+                expected_type=float,
+                units="kev",
+                name="energy",
+                n_value=1,
+            ),
+            # x pixel size
+            NXtomoEditorKeys.X_PIXEL_SIZE: _EditorFieldInfo(
+                nexus_path=nexus_paths.X_PIXEL_SIZE_PATH,
+                expected_type=float,
+                units="m",
+                name="x pixel size",
+                n_value=1,
+            ),
+            # y pixel size
+            NXtomoEditorKeys.Y_PIXEL_SIZE: _EditorFieldInfo(
+                nexus_path=nexus_paths.Y_PIXEL_SIZE_PATH,
+                expected_type=float,
+                units="m",
+                name="y pixel size",
+                n_value=1,
+            ),
+            # sample / detector distance
+            NXtomoEditorKeys.SAMPLE_DETECTOR_DISTANCE: _EditorFieldInfo(
+                nexus_path=nexus_paths.DISTANCE_PATH,
+                expected_type=float,
+                units="m",
+                name="sample detector distance",
+                n_value=1,
+            ),
+            # overwrite FOV
+            NXtomoEditorKeys.FIELD_OF_VIEW: _EditorFieldInfo(
+                nexus_path=nexus_paths.FOV_PATH,
+                expected_type=str,
+                units=None,
+                name="field of view",
+                n_value=1,
+            ),
+            # x translation
+            NXtomoEditorKeys.X_TRANSLATION: _EditorFieldInfo(
+                nexus_path=nexus_paths.X_TRANS_PATH,
+                expected_type=float,
+                units="m",
+                name="x translation",
+                n_value=n_frames,
+            ),
+            # y translation
+            NXtomoEditorKeys.Y_TRANSLATION: _EditorFieldInfo(
+                nexus_path=nexus_paths.Y_TRANS_PATH,
+                expected_type=float,
+                units="m",
+                name="y translation",
+                n_value=n_frames,
+            ),
+            # z translation
+            NXtomoEditorKeys.Z_TRANSLATION: _EditorFieldInfo(
+                nexus_path=nexus_paths.Z_TRANS_PATH,
+                expected_type=float,
+                units="m",
+                name="z translation",
+                n_value=n_frames,
+            ),
+        }

tomwer/core/process/edit/tests/test_darkflatpatch.py

@@ -0,0 +1,243 @@
+# coding: utf-8
+from __future__ import annotations
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import h5py
+import numpy
+from silx.io.url import DataUrl
+from silx.io.utils import h5py_read_dataset
+from tomoscan.esrf.scan.mock import MockNXtomo
+
+from tomwer.core.process.edit.darkflatpatch import DarkFlatPatchTask
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+
+
+class BaseTestAddDarkAndFlats(unittest.TestCase):
+    """
+    Unit test on nxtomomill.utils.add_dark_flat_nx_file function
+    """
+
+    def setUp(self) -> None:
+        self.tmpdir = tempfile.mkdtemp()
+        simple_nx_path = os.path.join(self.tmpdir, "simple_case")
+        self.dim = 55
+        self.nproj = 20
+        self._simple_nx = MockNXtomo(
+            scan_path=simple_nx_path,
+            n_proj=self.nproj,
+            n_ini_proj=self.nproj,
+            create_ini_dark=False,
+            create_ini_flat=False,
+            create_final_flat=False,
+            dim=self.dim,
+        ).scan
+        with h5py.File(self._simple_nx.master_file, mode="r") as h5s:
+            data_path = "/".join(
+                (self._simple_nx.entry, "instrument", "detector", "data")
+            )
+            self._raw_data = h5py_read_dataset(h5s[data_path])
+        nx_with_vds_path = os.path.join(self.tmpdir, "case_with_vds")
+        self._nx_with_virtual_dataset = MockNXtomo(
+            scan_path=nx_with_vds_path,
+            n_proj=0,
+            n_ini_proj=0,
+            create_ini_dark=False,
+            create_ini_flat=False,
+            create_final_flat=False,
+            dim=self.dim,
+        ).scan
+        self._create_vds(
+            source_file=self._simple_nx.master_file,
+            source_data_path=self._simple_nx.entry,
+            target_file=self._nx_with_virtual_dataset.master_file,
+            target_data_path=self._nx_with_virtual_dataset.entry,
+            copy_other_data=True,
+        )
+        self._patch_nxtomo_flags(self._nx_with_virtual_dataset)
+
+        nx_with_vds_path_and_links = os.path.join(
+            self.tmpdir, "case_with_vds_and_links"
+        )
+        self._nx_with_virtual_dataset_with_link = MockNXtomo(
+            scan_path=nx_with_vds_path_and_links,
+            n_proj=0,
+            n_ini_proj=0,
+            create_ini_dark=False,
+            create_ini_flat=False,
+            create_final_flat=False,
+            dim=self.dim,
+        ).scan
+        self._create_vds(
+            source_file=self._simple_nx.master_file,
+            source_data_path=self._simple_nx.entry,
+            target_file=self._nx_with_virtual_dataset_with_link.master_file,
+            target_data_path=self._nx_with_virtual_dataset_with_link.entry,
+            copy_other_data=True,
+        )
+        self._patch_nxtomo_flags(self._nx_with_virtual_dataset_with_link)
+
+        # create dark
+        self.start_dark = (
+            numpy.random.random((self.dim * self.dim))
+            .reshape(1, self.dim, self.dim)
+            .astype("f")
+        )
+        self.start_dark_file = os.path.join(self.tmpdir, "dark.hdf5")
+        self.start_dark_entry = "data"
+        self.start_dark_url = self._save_raw(
+            data=self.start_dark,
+            entry=self.start_dark_entry,
+            file_path=self.start_dark_file,
+        )
+
+        self.end_dark = (
+            numpy.random.random((self.dim * self.dim * 2))
+            .reshape(2, self.dim, self.dim)
+            .astype("f")
+        )
+        self.end_dark_file = os.path.join(self.tmpdir, "dark.hdf5")
+        self.end_dark_entry = "data2"
+        self.end_dark_url = self._save_raw(
+            data=self.end_dark, entry=self.end_dark_entry, file_path=self.end_dark_file
+        )
+
+        # create flats
+        self.start_flat = (
+            numpy.random.random((self.dim * self.dim * 3))
+            .reshape(3, self.dim, self.dim)
+            .astype("f")
+        )
+        self.start_flat_file = os.path.join(self.tmpdir, "start_flat.hdf5")
+        self.start_flat_entry = "/root/flat"
+        self.start_flat_url = self._save_raw(
+            data=self.start_flat,
+            entry=self.start_flat_entry,
+            file_path=self.start_flat_file,
+        )
+
+        self.end_flat = (
+            numpy.random.random((self.dim * self.dim))
+            .reshape(1, self.dim, self.dim)
+            .astype("f")
+        )
+        # save the end flat in the simple case file to insure all cases are
+        # consider
+        self.end_flat_file = self._simple_nx.master_file
+        self.end_flat_entry = "flat"
+        self.end_flat_url = self._save_raw(
+            data=self.end_flat, entry=self.end_flat_entry, file_path=self.end_flat_file
+        )
+
+    def _save_raw(self, data, entry, file_path) -> DataUrl:
+        with h5py.File(file_path, mode="a") as h5s:
+            h5s[entry] = data
+        return DataUrl(file_path=file_path, data_path=entry, scheme="silx")
+
+    def _create_vds(
+        self,
+        source_file: str,
+        source_data_path: str,
+        target_file: str,
+        target_data_path: str,
+        copy_other_data: bool,
+    ):
+        """Create virtual dataset and links from source to target
+
+        :param source_file:
+        :param source_data_path:
+        :param target_file:
+        :param target_data_path:
+        :param copy_other_data: we want to create two cases: one copying
+                                     datasets 'image_key'... and the other
+                                     one linking them. Might have a difference
+                                     of behavior when overwriting for example
+        """
+        assert source_file != target_file, "file should be different"
+        # link data
+        n_frames = 0
+        # for now we only consider the original data
+        with h5py.File(source_file, mode="r") as o_h5s:
+            old_path = os.path.join(source_data_path, "instrument", "detector", "data")
+            n_frames += o_h5s[old_path].shape[0]
+            shape = o_h5s[old_path].shape
+            data_type = o_h5s[old_path].dtype
+
+            layout = h5py.VirtualLayout(shape=shape, dtype=data_type)
+            assert os.path.exists(source_file)
+            with h5py.File(source_file, mode="r") as ppp:
+                assert source_data_path in ppp
+            layout[:] = h5py.VirtualSource(path_or_dataset=o_h5s[old_path])
+
+            det_path = os.path.join(target_data_path, "instrument", "detector")
+            with h5py.File(target_file, mode="a") as h5s:
+                detector_node = h5s.require_group(det_path)
+                detector_node.create_virtual_dataset("data", layout, fillvalue=-5)
+
+        for path in (
+            os.path.join("instrument", "detector", "image_key"),
+            os.path.join("instrument", "detector", "image_key_control"),
+            os.path.join("instrument", "detector", "count_time"),
+            os.path.join("sample", "rotation_angle"),
+        ):
+            old_path = os.path.join(source_data_path, path)
+            new_path = os.path.join(target_data_path, path)
+            with h5py.File(target_file, mode="a") as h5s:
+                if copy_other_data:
+                    with h5py.File(source_file, mode="r") as o_h5s:
+                        if new_path in h5s:
+                            del h5s[new_path]
+                        h5s[new_path] = h5py_read_dataset(o_h5s[old_path])
+                elif source_file == target_file:
+                    h5s[new_path] = h5py.SoftLink(old_path)
+                else:
+                    relpath = os.path.relpath(source_file, os.path.dirname(target_file))
+                    h5s[new_path] = h5py.ExternalLink(relpath, old_path)
+
+    def _patch_nxtomo_flags(self, scan):
+        """Insure necessary flags are here"""
+        with h5py.File(scan.master_file, mode="a") as h5s:
+            instrument_path = os.path.join(scan.entry, "instrument")
+            instrument_node = h5s.require_group(instrument_path)
+            if "NX_class" not in instrument_node.attrs:
+                instrument_node.attrs["NX_class"] = "NXinstrument"
+            detector_node = instrument_node.require_group("detector")
+            if "NX_class" not in detector_node.attrs:
+                detector_node.attrs["NX_class"] = "NXdetector"
+            if "data" in instrument_node:
+                if "interpretation" not in instrument_node.attrs:
+                    instrument_node["data"].attrs["interpretation"] = "image"
+
+            sample_path = os.path.join(scan.entry, "sample")
+            sample_node = h5s.require_group(sample_path)
+            if "NX_class" not in sample_node:
+                sample_node.attrs["NX_class"] = "NXsample"
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.tmpdir)
+
+
+class TestDaarkFlatPatch(BaseTestAddDarkAndFlats):
+    def test(self):
+        scan = NXtomoScan(
+            self._nx_with_virtual_dataset_with_link.master_file,
+            self._nx_with_virtual_dataset_with_link.entry,
+        )
+        process = DarkFlatPatchTask(
+            inputs={
+                "data": scan,
+                "serialize_output_data": False,
+                "configuration": {
+                    "darks_start": self.start_dark_url,
+                },
+            },
+        )
+        process.definition()
+        process.program_version()
+        process.program_name()
+        # TODO: properties should be part of inputs
+        process.run()

tomwer/core/process/edit/tests/test_imagekey_editor.py

@@ -0,0 +1,99 @@
+# coding: utf-8
+from __future__ import annotations
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import pytest
+from nxtomo.nxobject.nxdetector import ImageKey
+
+from tomwer.core.process.edit.imagekeyeditor import (
+    ImageKeyEditorTask,
+    ImageKeyUpgraderTask,
+)
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+from tomwer.core.utils.scanutils import MockNXtomo
+
+
+class TestImageKeyEditor(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        dim = 10
+        mock = MockNXtomo(
+            scan_path=os.path.join(self.tempdir, "scan1"),
+            n_proj=10,
+            n_ini_proj=10,
+            scan_range=180,
+            dim=dim,
+        )
+        self.scan = mock.scan
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test(self):
+        process = ImageKeyEditorTask(
+            inputs={
+                "data": self.scan,
+                "serialize_output_data": False,
+                "configuration": {
+                    "modifications": {
+                        0: ImageKey.INVALID.value,
+                        3: ImageKey.ALIGNMENT.value,
+                    },
+                },
+            }
+        )
+        process.definition()
+        process.program_version()
+        process.program_name()
+        process.run()
+
+
+def test_ImageKeyUpgraderTask(tmp_path):
+    """
+    test ImageKeyUpgraderTask task
+    """
+    test_dir = tmp_path / "test_image_key_upgrader"
+    os.makedirs(test_dir)
+    scan = MockNXtomo(
+        scan_path=test_dir,
+        n_proj=20,
+        n_ini_proj=20,
+        create_ini_dark=False,
+    ).scan
+
+    operations = {
+        ImageKey.PROJECTION: ImageKey.DARK_FIELD,
+    }
+
+    with pytest.raises(TypeError):
+        ImageKeyUpgraderTask(
+            inputs={
+                "data": None,
+                "operations": operations,
+            },
+        ).run()
+
+    with pytest.raises(TypeError):
+        ImageKeyUpgraderTask(
+            inputs={
+                "data": scan,
+                "operations": None,
+            },
+        ).run()
+
+    task = ImageKeyUpgraderTask(
+        inputs={
+            "data": scan,
+            "operations": operations,
+        },
+    )
+    task.run()
+
+    scan = NXtomoScan(scan.master_file, scan.entry)
+    assert len(scan.projections) == 0
+    assert len(scan.darks) == 20

tomwer/core/process/output.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 import os
+import warnings
 from silx.utils.enum import Enum as _Enum
 
 
 class ProcessDataOutputDirMode(_Enum):
     IN_SCAN_FOLDER = "same folder as scan"
     PROCESSED_DATA_FOLDER = "PROCESSED_DATA folder"
-    RAW_DATA_FOLDER = "RAW_DATA folder"
     OTHER = "other"
 
     @classmethod
@@ -17,7 +17,14 @@ class ProcessDataOutputDirMode(_Enum):
         elif value == "processed data dir":
             return cls.PROCESSED_DATA_FOLDER
         elif value == "raw data dir":
-            return cls.RAW_DATA_FOLDER
+            # Deprecation logic
+            warnings.warn(
+                "'RAW_DATA output' is deprecated since writing to 'RAW_DATA' folder is forbidden."
+                "Redirecting to 'PROCESSED_DATA_FOLDER'.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            return cls.PROCESSED_DATA_FOLDER
         return super().from_value(value)