pymodaq_data 5.2.0a1__tar.gz → 5.2.0a3__tar.gz

{pymodaq_data-5.2.0a1 → pymodaq_data-5.2.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pymodaq_data
-Version: 5.2.0a1
+Version: 5.2.0a3
 Summary: Modular Data Acquisition with Python
 Project-URL: Homepage, http://pymodaq.cnrs.fr
 Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
@@ -43,16 +43,19 @@ Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Software Development :: User Interfaces
 Requires-Python: >=3.8
+Requires-Dist: h5py
 Requires-Dist: pymodaq-utils>=0.0.8
 Requires-Dist: scipy
-Requires-Dist: tables>=3.10
 Provides-Extra: dev
 Requires-Dist: flake8; extra == 'dev'
-Requires-Dist: h5py; extra == 'dev'
+Requires-Dist: h5py>=2.5.0; extra == 'dev'
 Requires-Dist: hatch; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest-xdist; extra == 'dev'
+Requires-Dist: tables>=3.10; extra == 'dev'
+Provides-Extra: xarray
+Requires-Dist: xarray>=2024.10; extra == 'xarray'
 Description-Content-Type: text/x-rst
 
 .. ############################################################

{pymodaq_data-5.2.0a1 → pymodaq_data-5.2.0a3}/pyproject.toml

@@ -33,14 +33,16 @@ classifiers = [
 dependencies = [
     "pymodaq_utils>=0.0.8",
     "scipy",
-    "tables>=3.10",
+    "h5py",
 ]
 
 [project.optional-dependencies]
+xarray = ["xarray>=2024.10"]
 dev = [
     "hatch", 
     "flake8",
-    "h5py",
+    "h5py>=2.5.0",
+    "tables>=3.10",
     "pytest",
     "pytest-cov",
     "pytest-xdist",

{pymodaq_data-5.2.0a1 → pymodaq_data-5.2.0a3}/src/pymodaq_data/__init__.py

@@ -13,7 +13,7 @@ try:
     try:
         logger = set_logger('pymodaq_data', add_handler=True, base_logger=True)
     except Exception:
-        print("Couldn't create the local folder to store logs , presets...")
+        print("Couldn't create the local folder to store logs , experiments...")
 
     import pymodaq_data.config
     logger.info('************************')

{pymodaq_data-5.2.0a1 → pymodaq_data-5.2.0a3}/src/pymodaq_data/data.py

@@ -7,8 +7,8 @@ Created the 28/10/2022
 from __future__ import annotations
 
 from abc import ABCMeta, abstractmethod
-import numbers
 from copy import deepcopy
+import numbers
 
 import numpy as np
 from numpy.lib.mixins import NDArrayOperatorsMixin
@@ -184,19 +184,18 @@ class DataDistribution(BaseEnum):
 
 def _compute_slices_from_axis(axis: Axis, _slice, *ignored, is_index=True, **ignored_also):
     if not is_index:
-        if isinstance(_slice, numbers.Number) or isinstance(_slice, Q_):
+        if isinstance(_slice, (numbers.Number, Q_)):
             if not is_index:
                 _slice = axis.find_index(_slice)
         elif _slice is Ellipsis:
             return _slice
-        elif isinstance(_slice, slice):
-            if not (_slice.start is None and
-                    _slice.stop is None and _slice.step is None):
-                start = axis.find_index(
-                    _slice.start if _slice.start is not None else axis.get_data()[0])
-                stop = axis.find_index(
-                    _slice.stop if _slice.stop is not None else axis.get_data()[-1])
-                _slice = slice(start, stop)
+        elif isinstance(_slice, slice) and not (_slice.start is None and
+                _slice.stop is None and _slice.step is None):
+            start = axis.find_index(
+                _slice.start if _slice.start is not None else axis.get_data()[0])
+            stop = axis.find_index(
+                _slice.stop if _slice.stop is not None else axis.get_data()[-1])
+            _slice = slice(start, stop)
     return _slice
 
 
@@ -437,8 +436,7 @@ class Axis(SerializableBase):
         ----------
         indexes:
         """
-        if not (isinstance(indexes, np.ndarray) or isinstance(indexes, slice) or
-                isinstance(indexes, int)):
+        if not (isinstance(indexes, (np.ndarray, slice, int))):
             indexes = np.array(indexes)
         return self.get_data()[indexes]
 
@@ -910,6 +908,16 @@ class DataBase(DataLowLevel, NDArrayOperatorsMixin):
             else:
                 return [float(np.mean(data_array)) for data_array in self.data]
 
+    def equal_to(self, other: 'DataBase', epsilon: Union[float, Q_])-> bool:
+        """ Check if two data object are equal within epsilon """
+        if isinstance(epsilon, numbers.Number):
+            epsilon = Q_(epsilon, self.units)
+        try:
+            return bool(np.all([np.abs(self.quantities[ind] - other.quantities[ind])
+                                <= epsilon for ind in range(len(self))]))
+        except pint.errors.DimensionalityError as e:
+            return False
+
     def as_dte(self, name: str = 'mydte') -> DataToExport:
         """Convenience method to wrap the DataWithAxes object into a DataToExport"""
         return DataToExport(name, data=[self])
@@ -918,11 +926,11 @@ class DataBase(DataLowLevel, NDArrayOperatorsMixin):
         """ Convenience method to split each ndarray into a DataWithAxes object """
         return DataToExport(name, data=[type(self)(self.labels[ind],
                                                    source=self.source,
-                                                   dim = self.dim,
+                                                   dim=self.dim,
                                                    data=[array],
-                                                   labels = [self.labels[ind]],
-                                                   axes = deepcopy(self.axes),
-                                                   units = self.units,
+                                                   labels=[self.labels[ind]],
+                                                   axes=deepcopy(self.axes),
+                                                   units=self.units,
                                                    ) for ind, array in enumerate(self)])
 
     def add_extra_attribute(self, **kwargs):
@@ -1009,7 +1017,7 @@ class DataBase(DataLowLevel, NDArrayOperatorsMixin):
 
     def _comparison_common(self, other, operator='__eq__'):
         if isinstance(other, DataBase):
-            if not (# no more checking for name equality but take care ot the pop/remove methods
+            if not (  # no more checking for name equality but take care ot the pop/remove methods
                     len(self) == len(other) and
                     Unit(self.units).is_compatible_with(other.units)):
                 return False
@@ -1021,7 +1029,7 @@ class DataBase(DataLowLevel, NDArrayOperatorsMixin):
                     eq = False
                     break
                 if operator == '__eq__':
-                    eq = eq and np.allclose(self.quantities[ind], other.quantities[ind])
+                    eq = eq and np.allclose(self.quantities[ind], other.quantities[ind], equal_nan=True)
                 else:
                     eq = eq and np.all(getattr(self.quantities[ind], operator)(other.quantities[ind]))
             # extra attributes are not relevant as they may contain module specific data...
@@ -1211,38 +1219,37 @@ class DataBase(DataLowLevel, NDArrayOperatorsMixin):
         return self.data[index]
 
     def _check_data_type(self, data: List[Union[np.ndarray, Q_]]) -> List[np.ndarray]:
-        """make sure data is a list of nd-arrays"""
-        is_valid = True
+        """Make sure data is a list of non-empty nd-arrays."""
         if data is None:
-            is_valid = False
+            raise TypeError('Data should be a non-empty list of non-empty numpy arrays')
+
+        # Convert single Q_, ndarray, or Number to a list
         if not isinstance(data, list):
-            # try to transform the data to regular type
             if isinstance(data, Q_):
                 self.force_units(str(data.units))
                 data = [data.magnitude]
             elif isinstance(data, np.ndarray):
-                warnings.warn(DataTypeWarning(f'Your data should be a list of numpy arrays not just a single numpy'
-                                              f' array, wrapping them with a list'))
+                warnings.warn(DataTypeWarning('Your data should be a list of numpy arrays, not just a single numpy array. Wrapping it in a list.'))
                 data = [data]
             elif isinstance(data, numbers.Number):
-                warnings.warn(DataTypeWarning(f'Your data should be a list of numpy arrays not just a single numpy'
-                                              f' array, wrapping them with a list'))
+                warnings.warn(DataTypeWarning('Your data should be a list of numpy arrays, not just a single number. Wrapping it in a list.'))
                 data = [np.array([data])]
             else:
-                is_valid = False
-        if isinstance(data, list):
-            if len(data) == 0:
-                is_valid = False
-            elif not (isinstance(data[0], np.ndarray) or
-                             isinstance(data[0], Q_)):
-                is_valid = False
-            elif len(data[0].shape) == 0:
-                is_valid = False
-        if not is_valid:
-            raise TypeError(f'Data should be an non-empty list of non-empty numpy arrays')
+                raise TypeError('Data should be a non-empty list of non-empty numpy arrays')
+
+        # Validate the list
+        if not data or not all(isinstance(item, (np.ndarray, Q_)) for item in data):
+            raise TypeError('Data should be a non-empty list of non-empty numpy arrays')
+
+        # Check for non-empty arrays
+        if any(len(item.shape) == 0 for item in data):
+            raise TypeError('Data should be a non-empty list of non-empty numpy arrays')
+
+        # Convert Q_ to magnitude
         if isinstance(data[0], Q_):
             self.force_units(str(data[0].units))
             data = [array.magnitude for array in data]
+
         return data
 
     def check_shape_from_data(self, data: List[np.ndarray]):
@@ -2265,7 +2272,7 @@ class DataWithAxes(DataBase, SerializableBase):
             dat_sum.append(np.atleast_1d(np.sum(dat, axis=axis)))
         return self.deepcopy_with_new_data(dat_sum, remove_axes_index=axis)
 
-    def interp(self,  new_axis_data: Union[Axis, np.ndarray], **kwargs) -> DataWithAxes:
+    def interp(self, new_axis_data: Union[Axis, np.ndarray], **kwargs) -> DataWithAxes:
         """Performs linear interpolation for 1D data only.
         
         For more complex ones, see :py:meth:`scipy.interpolate`
@@ -2477,11 +2484,11 @@ class DataWithAxes(DataBase, SerializableBase):
 
             dte.append(DataCalculated(f'{self.labels[ind]}',
                                       data=[self[ind][peaks_indices[-1]],
-                                            peaks_indices[-1]
+                                            peaks_indices[-1],
                                             ],
                                       labels=['peak value', 'peak indexes'],
                                       axes=[Axis('peak position', self.axes[0].units,
-                                                 data=self.axes[0].get_data_at(peaks_indices[-1]))])
+                                                 data=self.axes[0].get_data_at(peaks_indices[-1]))]),
                        )
         return dte
 
@@ -2632,7 +2639,7 @@ class DataWithAxes(DataBase, SerializableBase):
         list(slice): a version as index of the input argument
         """
         _slices_as_index = []
-        if isinstance(slices, numbers.Number) or isinstance(slices, Q_) or isinstance(slices, slice):
+        if isinstance(slices, (numbers.Number, Q_, slice)):
             slices = [slices]
         if is_navigation:
             indexes = self._am.nav_indexes
@@ -2694,7 +2701,7 @@ class DataWithAxes(DataBase, SerializableBase):
             Object of the same type as the initial data, derived from DataWithAxes. But with lower
             data size due to the slicing and with eventually less axes.
         """
-        if isinstance(slices, numbers.Number) or isinstance(slices, slice):
+        if isinstance(slices, (numbers.Number, slice)):
             slices = [slices]
 
         total_slices, slices = self._compute_slices(slices, is_navigation, is_index=is_index)
@@ -2851,6 +2858,218 @@ class DataWithAxes(DataBase, SerializableBase):
         """ Get the underlying data selected from the list at index, returned as a DataWithAxes"""
         return self.deepcopy_with_new_data([self[index]])
 
+    def to_xarray(self):
+        """Convert this DataWithAxes to an xarray.Dataset.
+
+        Each array in self.data becomes a data variable (keyed by its label).
+        Each Axis becomes a coordinate on the corresponding dimension.
+        Error arrays (if present) are stored as ``<label>_error`` data variables.
+
+        Returns
+        -------
+        xr.Dataset
+
+        Raises
+        ------
+        ImportError
+            If xarray is not installed.
+        """
+        try:
+            import xarray as xr
+        except ImportError:
+            raise ImportError(
+                "xarray is required for to_xarray(). "
+                "Install it with: pip install 'pymodaq_data[xarray]'",
+            )
+
+        ndim = len(self.shape)
+
+        # --- build dim names (one per shape dimension) ---
+        dim_names = []
+        seen_dim_names = {}
+        for i in range(ndim):
+            axes_at_i = self.get_axis_from_index(i)
+            if axes_at_i and axes_at_i[0] is not None and axes_at_i[0].label:
+                base = axes_at_i[0].label
+            else:
+                base = f'dim_{i}'
+            # deduplicate
+            if base in seen_dim_names:
+                seen_dim_names[base] += 1
+                name = f'{base}_{seen_dim_names[base]}'
+            else:
+                seen_dim_names[base] = 0
+                name = base
+            dim_names.append(name)
+
+        # --- build coordinates ---
+        coords = {}
+        spread_dim_names = []
+        for axis in self.axes:
+            dim_name = dim_names[axis.index]
+            axis_data = axis.get_data()
+            if axis_data is None:
+                continue
+            if axis.spread_order > 0:
+                coord_name = f'{axis.label}_{axis.spread_order}' if axis.label else f'{dim_name}_{axis.spread_order}'
+                coord_attrs = {
+                    'units': axis.units,
+                    'pymodaq_label': axis.label,
+                    'spread_order': axis.spread_order,
+                }
+                if dim_name not in spread_dim_names:
+                    spread_dim_names.append(dim_name)
+            else:
+                coord_name = axis.label if axis.label else dim_name
+                coord_attrs = {
+                    'units': axis.units,
+                    'pymodaq_label': axis.label,
+                }
+            coords[coord_name] = xr.Variable(dim_name, axis_data, attrs=coord_attrs)
+
+        # --- build data variables ---
+        data_vars = {}
+        label_list = list(self.labels)
+        error_var_names = []
+        for i, (array, label) in enumerate(zip(self.data, label_list)):
+            var_name = label if label else f'data_{i}'
+            data_vars[var_name] = xr.Variable(dim_names, array)
+            if self.errors is not None:
+                err_name = f'{var_name}_error'
+                data_vars[err_name] = xr.Variable(dim_names, self.errors[i])
+                error_var_names.append(err_name)
+
+        # --- dataset attrs ---
+        attrs = {
+            'pymodaq_name': self.name,
+            'pymodaq_origin': self.origin,
+            'pymodaq_source': self.source.name,
+            'pymodaq_distribution': self.distribution.name,
+            'pymodaq_units': self.units,
+            'pymodaq_nav_indexes': list(self.nav_indexes),
+            'pymodaq_labels': label_list,
+        }
+        if error_var_names:
+            attrs['pymodaq_error_vars'] = error_var_names
+        if spread_dim_names:
+            attrs['pymodaq_spread_dim_names'] = spread_dim_names
+
+        return xr.Dataset(data_vars, coords=coords, attrs=attrs)
+
+    @classmethod
+    def from_xarray(cls, ds) -> 'DataWithAxes':
+        """Construct a DataWithAxes from an xarray Dataset (or DataArray).
+
+        Parameters
+        ----------
+        ds : xr.Dataset or xr.DataArray
+
+        Returns
+        -------
+        DataWithAxes
+
+        Raises
+        ------
+        ImportError
+            If xarray is not installed.
+        """
+        try:
+            import xarray as xr
+        except ImportError:
+            raise ImportError(
+                "xarray is required for from_xarray(). "
+                "Install it with: pip install 'pymodaq_data[xarray]'",
+            )
+
+        if isinstance(ds, xr.DataArray):
+            var_name = ds.name if ds.name else 'data'
+            ds = ds.to_dataset(name=var_name)
+
+        attrs = ds.attrs
+        name = attrs.get('pymodaq_name', 'from_xarray')
+        origin = attrs.get('pymodaq_origin', '')
+        source_str = attrs.get('pymodaq_source', 'raw')
+        distribution_str = attrs.get('pymodaq_distribution', 'uniform')
+        units = attrs.get('pymodaq_units', '')
+        nav_indexes = tuple(attrs.get('pymodaq_nav_indexes', []))
+        stored_labels = list(attrs.get('pymodaq_labels', []))
+        error_var_names = list(attrs.get('pymodaq_error_vars', []))
+
+        # separate error vars from regular data vars
+        regular_vars = {k: v for k, v in ds.data_vars.items() if k not in error_var_names}
+        error_vars = {k: v for k, v in ds.data_vars.items() if k in error_var_names}
+
+        # reconstruct data arrays and labels
+        data_arrays = []
+        labels = []
+        for i, (var_name, var) in enumerate(regular_vars.items()):
+            data_arrays.append(var.values)
+            label = stored_labels[i] if i < len(stored_labels) else var_name
+            labels.append(label)
+
+        # reconstruct error arrays (matched by position in error_var_names)
+        errors = None
+        if error_vars:
+            errors = [error_vars[err_name].values for err_name in error_var_names]
+
+        # reconstruct Axis objects from dims and coords
+        dim_names = list(ds.dims)
+        axes = []
+        for i, dim_name in enumerate(dim_names):
+            # find coords that live on this dimension
+            dim_coords = [
+                (cname, cvar) for cname, cvar in ds.coords.items()
+                if list(cvar.dims) == [dim_name]
+            ]
+            if dim_coords:
+                # primary coord: spread_order == 0 (or missing) comes first
+                primary = None
+                secondary = []
+                for cname, cvar in dim_coords:
+                    spread_order = int(cvar.attrs.get('spread_order', 0))
+                    axis_label = cvar.attrs.get('pymodaq_label', cname)
+                    axis_units = cvar.attrs.get('units', '')
+                    axis = Axis(
+                        label=axis_label,
+                        units=axis_units,
+                        data=cvar.values,
+                        index=i,
+                        spread_order=spread_order,
+                    )
+                    if spread_order == 0:
+                        primary = axis
+                    else:
+                        secondary.append(axis)
+                if primary is not None:
+                    axes.append(primary)
+                axes.extend(secondary)
+            else:
+                # no coord: create size-only axis
+                dim_size = ds.sizes[dim_name]
+                axes.append(Axis(label=dim_name, index=i, size=dim_size))
+
+        try:
+            source = DataSource[source_str]
+        except KeyError:
+            source = DataSource.raw
+        try:
+            distribution = DataDistribution[distribution_str]
+        except KeyError:
+            distribution = DataDistribution.uniform
+
+        return cls(
+            name=name,
+            source=source,
+            distribution=distribution,
+            data=data_arrays,
+            labels=labels,
+            units=units,
+            axes=axes,
+            nav_indexes=nav_indexes,
+            origin=origin,
+            errors=errors,
+        )
+
 
 @ser_factory.register_decorator()
 class DataRaw(DataWithAxes):
@@ -2878,7 +3097,7 @@ class DataRaw(DataWithAxes):
                          axes=axes,
                          nav_indexes=nav_indexes,
                          errors=errors,
-                         **kwargs
+                         **kwargs,
                          )
 
 
@@ -3252,9 +3471,9 @@ class DataToExport(DataLowLevel, SerializableBase):
     def get_data_from_full_name(self, full_name: str, deepcopy=False) -> DataWithAxes:
         """Get the DataWithAxes with matching full name"""
         if deepcopy:
-            data = self.get_data_from_name_origin(full_name.split('/')[1], full_name.split('/')[0]).deepcopy()
+            data = self.get_data_from_name_origin('/'.join(full_name.split('/')[1:]), full_name.split('/')[0]).deepcopy()
         else:
-            data = self.get_data_from_name_origin(full_name.split('/')[1], full_name.split('/')[0])
+            data = self.get_data_from_name_origin('/'.join(full_name.split('/')[1:]), full_name.split('/')[0])
         return data
 
     def get_data_from_full_names(self, full_names: List[str], deepcopy=False) -> DataToExport:
@@ -3530,6 +3749,75 @@ class DataToExport(DataLowLevel, SerializableBase):
         if isinstance(dte, DataToExport):
             self.append(dte.data)
 
+    def to_xarray(self):
+        """Convert this DataToExport to an xarray.DataTree.
+
+        The root node carries ``pymodaq_name`` in its attrs. Each DataWithAxes
+        becomes a child node whose dataset is produced by
+        ``DataWithAxes.to_xarray()``.
+
+        Returns
+        -------
+        xr.DataTree
+
+        Raises
+        ------
+        ImportError
+            If xarray is not installed.
+        """
+        try:
+            import xarray as xr
+        except ImportError:
+            raise ImportError(
+                "xarray is required for to_xarray(). "
+                "Install it with: pip install 'pymodaq_data[xarray]'",
+            )
+
+        children = {dwa.name: xr.DataTree(dataset=dwa.to_xarray()) for dwa in self}
+        root_ds = xr.Dataset(attrs={'pymodaq_name': self.name})
+        return xr.DataTree(dataset=root_ds, children=children)
+
+    @classmethod
+    def from_xarray(cls, dt, name: str = None) -> 'DataToExport':
+        """Construct a DataToExport from an xarray.DataTree or a dict of Datasets.
+
+        Parameters
+        ----------
+        dt : xr.DataTree or dict[str, xr.Dataset]
+        name : str, optional
+            Override the name; if None, read from ``dt.attrs['pymodaq_name']``.
+
+        Returns
+        -------
+        DataToExport
+
+        Raises
+        ------
+        ImportError
+            If xarray is not installed.
+        """
+        try:
+            import xarray as xr
+        except ImportError:
+            raise ImportError(
+                "xarray is required for from_xarray(). "
+                "Install it with: pip install 'pymodaq_data[xarray]'",
+            )
+
+        if isinstance(dt, xr.DataTree):
+            root_attrs = dt.dataset.attrs if dt.dataset is not None else {}
+            dte_name = name or root_attrs.get('pymodaq_name', 'from_xarray')
+            data_list = [
+                DataWithAxes.from_xarray(child.dataset)
+                for child in dt.children.values()
+            ]
+        else:
+            # dict[str, xr.Dataset] fallback
+            dte_name = name or 'from_xarray'
+            data_list = [DataWithAxes.from_xarray(ds) for ds in dt.values()]
+
+        return cls(name=dte_name, data=data_list)
+
 
 if __name__ == '__main__':
     d = DataRaw('hjk', units='m', data=[np.array([0, 1, 2])])
@@ -3550,7 +3838,7 @@ if __name__ == '__main__':
 
     dat = np.zeros((Nnav, Nsig))
     for ind in range(Nnav):
-        dat[ind] = mutils.gauss1D(x,  50 * (ind -Nnav / 2), 25 / np.sqrt(2))
+        dat[ind] = mutils.gauss1D(x, 50 * (ind -Nnav / 2), 25 / np.sqrt(2))
 
     data = DataRaw('mydata', data=[dat], nav_indexes=(0,),
                    axes=[Axis('nav', data=np.linspace(0, Nnav-1, Nnav), index=0),