darfix 4.2.0__py3-none-any.whl → 4.3.0__py3-none-any.whl

darfix/core/data_selection.py

@@ -4,6 +4,7 @@ import urllib.request
 from typing import Optional
 from typing import Union
 
+from esrf_pathlib import ESRFPath
 from silx.io.url import DataUrl
 
 from darfix.core.dataset import ImageDataset
@@ -26,7 +27,6 @@ def load_process_data(
     :param detector_url: detector_url to be loaded.
     :param metadata_url: path to the scan metadata for HDF5 containing positioner information in order to load metadata for non-edf files
     """
-    indices = li_indices = None
     root_dir_specified = bool(root_dir)
 
     if isinstance(detector_url, DataUrl):
@@ -87,9 +87,18 @@ def load_process_data(
 
     assert dataset.data is not None and dataset.data.size > 0, "No data was loaded!"
 
-    return dataset, indices, li_indices, bg_dataset
+    return dataset, bg_dataset
 
 
 def _get_root_dir(filename: str) -> str:
     url = urllib.parse.urlparse(filename, scheme="file")
     return os.path.dirname(urllib.request.url2pathname(url.path))
+
+
+def get_default_output_directory(raw_data_file: str) -> str:
+    esrf_raw_data_file = ESRFPath(raw_data_file)
+    try:
+        return str(esrf_raw_data_file.processed_dataset_path)
+    except AttributeError:
+        # Not an ESRF path : Default directory is cwd.
+        return os.getcwd()

darfix/core/dataset.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import os
+import threading
 import warnings
 from pathlib import Path
 from typing import Literal
@@ -22,9 +23,9 @@ from sklearn.exceptions import ConvergenceWarning
 from darfix import __version__
 from darfix.core.dimension import AcquisitionDims
 from darfix.core.dimension import find_dimensions_from_metadata
-from darfix.core.imageRegistration import apply_opencv_shift
-from darfix.core.imageRegistration import shift_detection
-from darfix.core.imageStack import ImageStack
+from darfix.core.image_registration import apply_opencv_shift
+from darfix.core.image_registration import shift_detection
+from darfix.core.image_stack import ImageStack
 from darfix.core.mapping import calculate_RSM_histogram
 from darfix.core.mapping import compute_magnification
 from darfix.core.mapping import compute_moments
@@ -33,15 +34,14 @@ from darfix.core.positioners import Positioners
 from darfix.core.rocking_curves import fit_rocking_curve_parallel
 from darfix.core.roi import apply_2D_ROI
 from darfix.core.roi import apply_3D_ROI
-from darfix.core.state_of_operation import Operation
-from darfix.core.state_of_operation import StateOfOperations
 from darfix.core.utils import NoDimensionsError
 from darfix.core.utils import TooManyDimensionsForRockingCurvesError
 from darfix.decomposition.nica import NICA
 from darfix.io import utils as io_utils_legacy
-from darfix.processing.imageOperations import threshold_removal
+from darfix.processing.image_operations import threshold_removal
 
 from ..math import Vector3D
+from .moment_types import MomentsPerDimension
 from .moment_types import MomentType
 from .transformation import Transformation
 
@@ -99,7 +99,7 @@ class ImageDataset(ImageStack):
         title: str | None = None,
         metadata_url: DataUrl | str | None = None,
         positioners: Positioners | None = None,
-        moments: dict[MomentType, numpy.ndarray] = {},
+        moments: MomentsPerDimension = {},
     ):
 
         if raw_data is None:
@@ -108,7 +108,6 @@ class ImageDataset(ImageStack):
 
         self._frames_intensity = []
         self.moments_dims = moments
-        self.state_of_operations = StateOfOperations()
         self._dir = _dir
         self._transformation = transformation
         self._title = title or ""
@@ -135,18 +134,6 @@ class ImageDataset(ImageStack):
             # No metadata in Dark dataset for instance
             self._positioners = None
 
-    def stop_operation(self, operation):
-        """
-        Method used for cases where threads are created to apply functions to the dataset.
-        If method is called, the flag concerning the stop is set to 0 so that if the concerned
-        operation is running in another thread it knows to stop.
-
-        :param int operation: operation to stop
-        :type int: Union[int, `Operation`]
-        """
-        if self.state_of_operations.is_running(operation):
-            self.state_of_operations.stop(operation)
-
     @property
     def transformation(self) -> Transformation:
         return self._transformation
@@ -175,6 +162,10 @@ class ImageDataset(ImageStack):
             return {}
         return self._positioners.data
 
+    @property
+    def positioners(self) -> Positioners:
+        return self._positioners
+
     def compute_frames_intensity(self, kernel=(3, 3), sigma=20):
         """
         Returns for every image a number representing its intensity. This number
@@ -183,20 +174,17 @@ class ImageDataset(ImageStack):
         _logger.info("Computing intensity per frame")
         io_utils_legacy.advancement_display(0, self.nframes, "Computing intensity")
         frames_intensity = []
-        with self.state_of_operations.run_context(Operation.PARTITION):
-            for i in range(self.nframes):
-                import cv2
-
-                if not self.state_of_operations.is_running(Operation.PARTITION):
-                    return
-                frames_intensity += [
-                    cv2.GaussianBlur(self.as_array3d(i), kernel, sigma).var()
-                ]
-                io_utils_legacy.advancement_display(
-                    i + 1, self.nframes, "Computing intensity"
-                )
-            self._frames_intensity = frames_intensity
-            return frames_intensity
+        for i in range(self.nframes):
+            import cv2
+
+            frames_intensity += [
+                cv2.GaussianBlur(self.as_array3d()[i], kernel, sigma).var()
+            ]
+            io_utils_legacy.advancement_display(
+                i + 1, self.nframes, "Computing intensity"
+            )
+        self._frames_intensity = frames_intensity
+        return frames_intensity
 
     def partition_by_intensity(
         self,
@@ -277,8 +265,8 @@ class ImageDataset(ImageStack):
             )
         self.__dims = _dims
 
-    def zsum(self, indices=None, dimension=None) -> numpy.ndarray:
-        data = self.get_filtered_data(indices, dimension)
+    def zsum(self, dimension=None) -> numpy.ndarray:
+        data = self.get_filtered_data(dimension)
         return data.sum(axis=0)
 
     def reshape_data(self):
@@ -307,31 +295,27 @@ class ImageDataset(ImageStack):
         """
         self.__dims = find_dimensions_from_metadata(self.metadata_dict, tolerance)
 
-    def get_metadata_values(self, key: str, indices=None) -> numpy.ndarray:
+    def get_metadata_values(self, key: str) -> numpy.ndarray:
         if key not in self.metadata_dict:
             # The key does not exist -> return an array of the desired size and with NaN value
             return numpy.full(self.nframes, numpy.nan)
-        if indices is None:
-            return self.metadata_dict[key]
-        return self.metadata_dict[key][indices]
+        return self.metadata_dict[key]
 
-    def get_dimensions_values(self, indices=None):
+    def get_dimensions_values(self):
         """
         Returns all the metadata values of the dimensions.
         The values are assumed to be numbers.
 
         :returns: array_like
         """
-        if not self._dimensions_values or indices is not None:
+        if not self._dimensions_values:
             for dimension in self.__dims.values():
                 self._dimensions_values[dimension.name] = self.get_metadata_values(
-                    key=dimension.name, indices=indices
+                    key=dimension.name
                 )
         return self._dimensions_values
 
-    def apply_roi(
-        self, origin=None, size=None, center=None, indices=None, roi_dir=None
-    ):
+    def apply_roi(self, origin=None, size=None, center=None, roi_dir=None):
         """
         Applies a region of interest to the data.
 
@@ -341,14 +325,9 @@ class ImageDataset(ImageStack):
         :type origin: Union[2d vector, None]
         :type center: Union[2d vector, None]
         :type size: Union[2d vector, None]
-        :param indices: Indices of the images to apply background subtraction.
-            If None, the roi is applied to all the data.
-        :type indices: Union[None, array_like]
         :param roi_dir: Directory path for the new dataset
         :type roi_dir: str
         :returns: dataset with data with roi applied.
-            Note: To preserve consistence of shape between images, if `indices`
-            is not None, only the data modified is returned.
         :rtype: Dataset
         """
 
@@ -383,16 +362,14 @@ class ImageDataset(ImageStack):
         return self.data.sum(axis_array)
 
     def find_shift(
-        self, selected_axis: int | None = None, steps=50, indices=None
+        self, selected_axis: int | None = None, steps=50
     ) -> tuple[float, float]:
         """
         Find shift of the data or part of it.
 
         :param int selected_axis: specify one motor (axis). The method try to find shift along this axis.
-        :param array_like indices: Boolean index list with True in the images to apply the shift to.
         :returns: tuple with shift x and y
         """
-        # TODO : indices are not used. Maybe it is possible to use it for performance or maybe it is better to simply remove it.
         data = (
             self.z_sum_along_axis(selected_axis)
             if selected_axis is not None
@@ -406,7 +383,6 @@ class ImageDataset(ImageStack):
         shift: tuple[float, float],
         axis: int | None = None,
         shift_approach: str | None = None,
-        indices=None,
     ):
         """
         Apply shift of the data or part of it and save new data into disk.
@@ -414,7 +390,6 @@ class ImageDataset(ImageStack):
         :param array_like shift: Shift per frame.
         :param int axis: Select one axis (motor) where to apply the shift. If None, the shift id not applied to one axis but to the whole stack.
         :param Literal['fft', 'linear'] shift_approach: Method to use to apply the shift.
-        :param array_like indices: Boolean index list with True in the images to apply the shift to.
             If None, the hot pixel removal is applied to all the data.
         """
         assert len(shift) > 0, "Shift list can't be empty"
@@ -445,9 +420,6 @@ class ImageDataset(ImageStack):
             if numpy.isclose(shift[0], 0) and numpy.isclose(shift[1], 0):
                 # No need to apply a (0,0) shift
                 continue
-            if indices is not None and img_index not in indices:
-                # Not really optimized but the simplest way i found to take indices into account.
-                continue
             data[img_index] = apply_opencv_shift(
                 data[img_index],
                 shift,
@@ -458,7 +430,6 @@ class ImageDataset(ImageStack):
         self,
         selected_axis: int | None = None,
         steps=100,
-        indices=None,
         shift_approach: str = "linear",
     ):
         """
@@ -467,16 +438,17 @@ class ImageDataset(ImageStack):
         :param int selected_axis: specify one motor (axis). The method try to find shift along this axis.
         :param float h_step: See `core.imageRegistration.shift_detection`
         :param Union['fft', 'linear'] shift_approach: Method to use to apply the shift.
-        :param array_like indices: Indices of the images to find and apply the shift to.
         :param Literal['fft', 'linear'] shift_approach: Method to use to apply the shift.
         """
-        shift = self.find_shift(selected_axis, steps, indices=indices)
-        self.apply_shift(
-            shift, selected_axis, indices=indices, shift_approach=shift_approach
-        )
+        shift = self.find_shift(selected_axis, steps)
+        self.apply_shift(shift, selected_axis, shift_approach=shift_approach)
 
     def _waterfall_nmf(
-        self, num_components, iterations, vstep=None, hstep=None, indices=None
+        self,
+        num_components,
+        iterations,
+        vstep=None,
+        hstep=None,
     ):
         """
         This method is used as a way to improve the speed of convergence of
@@ -492,7 +464,6 @@ class ImageDataset(ImageStack):
         :param int num_components: Number of components to find.
         :param array_like iterations: Array with number of iterations per step of the waterfall.
             The size of the array sets the size of the waterfall.
-        :param Union[None, array_like] indices: If not None, apply method only to indices of data.
         """
 
         from skimage.transform import resize
@@ -523,7 +494,7 @@ class ImageDataset(ImageStack):
 
         return H, W
 
-    def pca(self, num_components=None, indices=None, return_vals=False):
+    def pca(self, num_components=None, return_vals=False):
         """
         Compute Principal Component Analysis on the data.
         The method, first converts, if not already, the data into an hdf5 file object
@@ -533,8 +504,6 @@ class ImageDataset(ImageStack):
             If None, it uses the minimum between the number of images and the
             number of pixels.
         :type num_components: Union[None, int]
-        :param indices: If not None, apply method only to indices of data, defaults to None
-        :type indices: Union[None, array_like], optional
         :param return_vals: If True, returns only the singular values of PCA, else returns
             the components and the mixing matrix, defaults to False
         :type return_vals: bool, optional
@@ -546,7 +515,7 @@ class ImageDataset(ImageStack):
 
         model = decomposition.PCA(n_components=num_components)
 
-        W = model.fit_transform(self.as_array2d((indices)))
+        W = model.fit_transform(self.as_array2d())
 
         H, vals, W = model.components_, model.singular_values_, W
 
@@ -558,7 +527,6 @@ class ImageDataset(ImageStack):
         chunksize=None,
         num_iter=500,
         error_step=None,
-        indices=None,
     ):
         """
         Compute Non-negative Independent Component Analysis on the data.
@@ -572,8 +540,6 @@ class ImageDataset(ImageStack):
         :type num_iter: int, optional
         :param error_step: If not None, find the error every error_step and compares it
             to check for convergence. TODO: not able for huge datasets.
-        :param indices: If not None, apply method only to indices of data, defaults to None
-        :type indices: Union[None, array_like], optional
 
         :return: (H, W): The components matrix and the mixing matrix.
         """
@@ -582,7 +548,6 @@ class ImageDataset(ImageStack):
             self.as_array2d(),
             num_components,
             chunksize,
-            indices=indices,
         )
         model.fit_transform(max_iter=num_iter, error_step=error_step)
         return numpy.abs(model.H), numpy.abs(model.W)
@@ -597,7 +562,6 @@ class ImageDataset(ImageStack):
         W=None,
         vstep=100,
         hstep=1000,
-        indices=None,
         init=None,
     ):
         """
@@ -621,8 +585,6 @@ class ImageDataset(ImageStack):
         :type H: Union[None, array_like], optional
         :param W: Init matrix for W of shape (n_features, n_components), defaults to None
         :type W: Union[None, array_like], optional
-        :param indices: If not None, apply method only to indices of data, defaults to None
-        :type indices: Union[None, array_like], optional
 
         :return: (H, W): The components matrix and the mixing matrix.
         """
@@ -632,7 +594,7 @@ class ImageDataset(ImageStack):
         model = decomposition.NMF(
             n_components=num_components, init=init, max_iter=num_iter
         )
-        X = self.as_array2d(indices)
+        X = self.as_array2d()
 
         if numpy.any(X[:, :] < 0):
             _logger.warning("Setting negative values to 0 to compute NMF")
@@ -655,13 +617,12 @@ class ImageDataset(ImageStack):
         error_step=None,
         vstep=100,
         hstep=1000,
-        indices=None,
     ):
         """
         Applies both NICA and NMF to the data. The init H and W for NMF are the
         result of NICA.
         """
-        H, W = self.nica(num_components, chunksize, num_iter, indices=indices)
+        H, W = self.nica(num_components, chunksize, num_iter)
 
         # Initial NMF factorization: X = F0 * G0
         W = numpy.abs(W)
@@ -676,27 +637,23 @@ class ImageDataset(ImageStack):
             W,
             vstep,
             hstep,
-            indices=indices,
             init="custom",
         )
 
     def apply_moments(
         self,
-        indices=None,
     ):
         """
         Compute the COM, FWHM, skewness and kurtosis of the data for very dimension.
 
-        :param indices: If not None, apply method only to indices of data, defaults to None
-        :type indices: Union[None, array_like], optional
         """
 
         if not self.dims.ndim:
             raise NoDimensionsError("apply_moments")
         for axis, dim in self.dims.items():
             # Get motor values per image of the stack
-            values = self.get_dimensions_values(indices)[dim.name]
-            mean, fwhm, skew, kurt = compute_moments(values, self.as_array3d(indices))
+            values = self.get_dimensions_values()[dim.name]
+            mean, fwhm, skew, kurt = compute_moments(values, self.as_array3d())
             self.moments_dims[axis] = {
                 MomentType.COM: mean,
                 MomentType.FWHM: fwhm,
@@ -708,17 +665,12 @@ class ImageDataset(ImageStack):
 
     def apply_fit(
         self,
-        indices=None,
         int_thresh: float | None = 15.0,
         method: str | None = None,
-        _dir: str | None = None,
+        abort_event: threading.Event = threading.Event(),
     ) -> Tuple[ImageDataset, numpy.ndarray]:
         """
         Fits the data around axis 0 and saves the new data into disk.
-
-        :param indices: Indices of the images to fit.
-            If None, the fit is done to all the data.
-        :type indices: Union[None, array_like]
         :param int_thresh: see `mapping.fit_pixel`
         :type int_thresh: Union[None, float]
         :returns: dataset with data of same size as `self.data` but with the
@@ -727,46 +679,32 @@ class ImageDataset(ImageStack):
         """
         if not self.dims.ndim:
             raise NoDimensionsError("apply_fit")
-        if indices is None:
-            indices = Ellipsis
 
-        new_dataset = self.copy(new_data=self.data.copy())
+        if self.dims.ndim == 1:
+            dim0 = self.dims.get(0)
+            motor_values = self.metadata_dict[dim0.name].ravel()
+        elif self.dims.ndim <= 3:
 
-        with self.state_of_operations.run_context(Operation.FIT):
-            data = new_dataset.as_array3d()
-            # Fit can only be done if rocking curves are at least of size 3
-            if len(self.data) < 3:
-                raise ValueError(
-                    f"Fit can only be done if dataset has not at least 3 dimensions. Got {len(data)}"
-                )
+            ndim = self.dims.ndim
+            dimension_names = [self.dims.get(dim_idx).name for dim_idx in range(ndim)]
 
-            if self.dims.ndim == 1:
-                dim0 = self.dims.get(0)
-                motor_values = self.metadata_dict[dim0.name].ravel()[indices]
-            elif self.dims.ndim <= 3:
-
-                ndim = self.dims.ndim
-                dimension_names = [
-                    self.dims.get(dim_idx).name for dim_idx in range(ndim)
-                ]
-
-                motor_values = [
-                    self.metadata_dict[dim_name].ravel()[indices]
-                    for dim_name in dimension_names
-                ]
-
-            else:
-                raise TooManyDimensionsForRockingCurvesError()
-
-            data[indices], maps = fit_rocking_curve_parallel(
-                data=self.as_array3d(indices),
-                motor_values=motor_values,
-                thresh=int_thresh,
-                method=method,
-            )
+            motor_values = [
+                self.metadata_dict[dim_name].ravel() for dim_name in dimension_names
+            ]
+
+        else:
+            raise TooManyDimensionsForRockingCurvesError()
+
+        data, maps = fit_rocking_curve_parallel(
+            data=self.as_array3d(),
+            motor_values=motor_values,
+            thresh=int_thresh,
+            method=method,
+            abort_event=abort_event,
+        )
 
         return (
-            new_dataset,
+            self.copy(new_data=data),
             maps,
         )
 
@@ -823,7 +761,7 @@ class ImageDataset(ImageStack):
             )
         self.transformation = Transformation(kind, x, y, rotate)
 
-    def project_data(self, dimension: Sequence[int], indices=None, _dir=None):
+    def project_data(self, dimension: Sequence[int]):
         """
         Applies a projection to the data.
         The new Dataset will have the same size as the chosen dimension, where
@@ -831,9 +769,6 @@ class ImageDataset(ImageStack):
 
         :param dimension: Dimensions to project the data onto
         :type dimension: array_like
-        :param indices: Indices of the images to use for the projection.
-            If None, the projection is done using all data.
-        :type indices: Union[None, array_like]
         :param str _dir: Directory filename to save the new data
         """
 
@@ -846,7 +781,7 @@ class ImageDataset(ImageStack):
             dim = self.dims.get(dimension[0])
             data = []
             for i in range(dim.size):
-                _sum = self.zsum(indices=indices, dimension=[dimension[0], i])
+                _sum = self.zsum(dimension=[dimension[0], i])
                 if len(_sum):
                     data += [_sum]
             data = numpy.array(data)
@@ -860,7 +795,7 @@ class ImageDataset(ImageStack):
             data = []
             for i in range(dim1.size):
                 for j in range(dim2.size):
-                    _sum = self.zsum(indices=indices, dimension=[dimension, [i, j]])
+                    _sum = self.zsum(dimension=[dimension, [i, j]])
                     if len(_sum):
                         data += [_sum]
             data = numpy.array(data)
@@ -906,35 +841,15 @@ class ImageDataset(ImageStack):
             E=energy,
         )
 
-    def apply_binning(self, scale, _dir=None):
-        from darfix.tasks.binning import Binning  # avoid cyclic import
-
-        _dir = self.dir if _dir is None else _dir
-        task = Binning(
-            inputs={
-                "dataset": self,
-                "output_dir": _dir,
-                "scale": scale,
-            }
-        )
-        task.run()
-        return task.outputs.dataset
-
-    def recover_weak_beam(self, n, indices=None):
+    def recover_weak_beam(self, n):
         """
         Set to zero all pixels higher than n times the standard deviation across the stack dimension
 
         :param n: Increase or decrease the top threshold by this fixed value.
         :type n: float
-        :param indices: Indices of the images to use for the filtering.
-            If None, the filtering is done using all data.
-        :type indices: Union[None, array_like]
         """
-        std = numpy.std(self.as_array3d(indices).view(numpy.ndarray), axis=0)
-
-        dataset = self.copy(new_data=self.data.copy())
-        threshold_removal(dataset.as_array3d(indices), top=n * std)
-        return dataset
+        std = numpy.std(self.as_array3d(), axis=0)
+        threshold_removal(self.as_array3d(), top=n * std)
 
     @staticmethod
     def load(file: str):