pymodaq 3.6.12__py3-none-any.whl → 4.0.1__py3-none-any.whl

pymodaq/utils/h5modules/h5logging.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+"""
+Created the 15/11/2022
+
+@author: Sebastien Weber
+"""
+
+import logging
+import numpy as np
+
+from pymodaq.utils.logger import set_logger, get_module_name, get_module_name
+from pymodaq.utils.config import Config
+
+from pymodaq.utils.abstract.logger import AbstractLogger
+from pymodaq.utils import daq_utils as utils
+from .saving import H5Saver
+
+
+config = Config()
+
+logger = set_logger(get_module_name(__file__))
+
+
+class H5LogHandler(logging.StreamHandler):
+    def __init__(self, h5saver):
+        super().__init__()
+        self.h5saver = h5saver
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        self.setFormatter(formatter)
+
+    def emit(self, record):
+        msg = self.format(record)
+        self.h5saver.add_log(msg)
+
+
+class H5Logger(AbstractLogger):
+    def __init__(self, *args, **kwargs):
+        self.h5saver = H5Saver(*args, save_type='logger', **kwargs)
+
+    def close(self):
+        self.h5saver.close_file()
+
+    @property
+    def settings_tree(self):
+        return self.h5saver.settings_tree
+
+    @property
+    def settings(self):
+        return self.h5saver.settings
+
+    def init_logger(self, settings):
+        self.h5saver.init_file(update_h5=True, metadata=dict(settings=settings))
+        self.h5saver.flush()
+        return True
+
+    def get_handler(self):
+        return H5LogHandler(self.h5saver)
+
+    def add_detector(self, name, settings):
+        if name not in self.h5saver.raw_group.children_name():
+            group = self.h5saver.add_det_group(self.h5saver.raw_group, name, settings)
+            self.h5saver.add_navigation_axis(np.array([0.0, ]),
+                                             group, 'time_axis', enlargeable=True,
+                                             title='Time axis',
+                                             metadata=dict(label='Time axis', units='s', nav_index=0))
+
+    def add_actuator(self, name, settings):
+        if name not in self.h5saver.raw_group.children_name():
+            group = self.h5saver.add_move_group(self.h5saver.raw_group, name, settings)
+            self.h5saver.add_navigation_axis(np.array([0.0, ]),
+                                     group, 'time_axis', enlargeable=True,
+                                     title='Time axis',
+                                     metadata=dict(label='Time axis', units='s', nav_index=0))
+
+    def add_data(self, data):
+        name = data['name']
+        group = self.h5saver.get_group_by_title(self.h5saver.raw_group, name)
+        time_array = self.h5saver.get_node(group, 'Logger_time_axis')
+        time_array.append(np.array([data['acq_time_s']]))
+
+        data_types = ['data0D', 'data1D']
+        if self.settings['save_2D']:
+            data_types.extend(['data2D', 'dataND'])
+
+        for data_type in data_types:
+            if data_type in data.keys() and len(data[data_type]) != 0:
+                if not self.h5saver.is_node_in_group(group, data_type):
+                    data_group = self.h5saver.add_data_group(group, data_type, metadata=dict(type='scan'))
+                else:
+                    data_group = self.h5saver.get_node(group, utils.capitalize(data_type))
+                for ind_channel, channel in enumerate(data[data_type]):
+                    channel_group = self.h5saver.get_group_by_title(data_group, channel)
+                    if channel_group is None:
+                        channel_group = self.h5saver.add_CH_group(data_group, title=channel)
+                        data_array = self.h5saver.add_data(channel_group, data[data_type][channel],
+                                                   scan_type='scan1D', enlargeable=True)
+                    else:
+                        data_array = self.h5saver.get_node(channel_group, 'Data')
+                    if data_type == 'data0D' and not isinstance(data[data_type][channel]['data'], np.ndarray):
+                        #this is a security as accessing an element in an array can be converted
+                        # to a scalar... Made some other attempts but found this is the most reliable here.
+                        logger.debug('Some data seems to not be properly formated as ndarrays')
+                        data_array.append(np.array([data[data_type][channel]['data']]))
+                    else:
+                        data_array.append(data[data_type][channel]['data'])
+        self.h5saver.flush()
+        self.settings.child('N_saved').setValue(self.settings.child('N_saved').value() + 1)
+
+    def stop_logger(self):
+        self.h5saver.flush()

pymodaq/utils/h5modules/module_saving.py

@@ -0,0 +1,350 @@
+# -*- coding: utf-8 -*-
+"""
+Created the 23/11/2022
+
+@author: Sebastien Weber
+"""
+from __future__ import annotations
+
+from typing import Union, List, Dict, Tuple, TYPE_CHECKING
+import xml.etree.ElementTree as ET
+
+
+import numpy as np
+
+from pymodaq.utils.abstract import ABCMeta, abstract_attribute, abstractmethod
+from pymodaq.utils.daq_utils import capitalize
+from pymodaq.utils.data import Axis, DataDim, DataWithAxes, DataToExport, DataDistribution
+from .saving import H5SaverLowLevel
+from .backends import GROUP, CARRAY, Node, GroupType
+from .data_saving import DataToExportSaver, AxisSaverLoader, DataToExportTimedSaver, DataToExportExtendedSaver
+from pymodaq.utils.parameter import ioxml
+
+if TYPE_CHECKING:
+    from pymodaq.extensions.daq_scan import DAQScan
+    from pymodaq.control_modules.daq_viewer import DAQ_Viewer
+    from pymodaq.control_modules.daq_move import DAQ_Move
+
+
+class ModuleSaver(metaclass=ABCMeta):
+    """Abstract base class to save info and data from main modules (DAQScan, DAQViewer, DAQMove, ...)"""
+    group_type: GroupType = abstract_attribute()
+    _module = abstract_attribute()
+    _h5saver: H5SaverLowLevel = abstract_attribute()
+    _module_group: GROUP = abstract_attribute()
+    main_module = True
+
+    def get_set_node(self, where: Union[Node, str] = None, name: str = None) -> GROUP:
+        """Get the node corresponding to this particular Module instance
+
+        Parameters
+        ----------
+        where: Union[Node, str]
+            the path of a given node or the node itself
+        new: bool
+            if True force the creation of a new indexed node of this class type
+            if False return the last node (or create one if None)
+
+        Returns
+        -------
+        GROUP: the Node associated with this module which should be a GROUP node
+        """
+        if where is None:
+            where = self._h5saver.raw_group
+        if name is None:
+            name = self._module.title
+        group = self._h5saver.get_node_from_title(where, name)
+        if group is not None:
+            self._module_group = group
+            return group  # if I got one I return it else I create one
+
+        self._module_group = self._add_module(where)
+        return self._module_group
+
+    def get_last_node(self, where: Union[Node, str] = None):
+        """Get the last node corresponding to this particular Module instance
+
+        Parameters
+        ----------
+        where: Union[Node, str]
+           the path of a given node or the node itself
+        new: bool
+           if True force the creation of a new indexed node of this class type
+           if False return the last node (or create one if None)
+
+        Returns
+        -------
+        GROUP: the Node associated with this module which should be a GROUP node
+        """
+        if where is None:
+            where = self._h5saver.raw_group
+
+        group = self._h5saver.get_last_group(where, self.group_type)
+        self._module_group = group
+        return self._module_group
+
+    @abstractmethod
+    def _add_module(self, where: Union[Node, str] = None, metadata={}):
+        ...
+
+    @property
+    def module(self):
+        return self._module
+
+    @property
+    def module_group(self):
+        return self._module_group
+
+    @property
+    def h5saver(self):
+        return self._h5saver
+
+    @h5saver.setter
+    def h5saver(self, _h5saver: H5SaverLowLevel):
+        self._h5saver = _h5saver
+        self.update_after_h5changed()
+
+    @abstractmethod
+    def update_after_h5changed(self):
+        ...
+
+    def get_last_node_index(self, where: Union[Node, str] = None):
+        node = self.get_last_node(where)
+        return int(node.name.split(capitalize(self.group_type.name))[1])
+
+    def get_next_node_name(self, where: Union[Node, str] = None):
+        index = self.get_last_node_index(where)
+        return f'{capitalize(self.group_type.name)}{index+1:03d}'
+
+
+class DetectorSaver(ModuleSaver):
+    """Implementation of the ModuleSaver class dedicated to DAQ_Viewer modules
+
+    Parameters
+    ----------
+    module
+    """
+    group_type = GroupType['detector']
+
+    def __init__(self, module: DAQ_Viewer):
+        self._datatoexport_saver: DataToExportSaver = None
+
+        self._module: 'DAQ_Viewer' = module
+        self._module_group: GROUP = None
+        self._h5saver = None
+
+    def update_after_h5changed(self, ):
+        self._datatoexport_saver = DataToExportSaver(self.h5saver)
+
+    def _add_module(self, where: Union[Node, str] = None, metadata={}) -> Node:
+        """
+
+        Parameters
+        ----------
+        where: Union[Node, str]
+            the path of a given node or the node itself
+        metadata: dict
+
+        Returns
+        -------
+
+        """
+        if where is None:
+            where = self._h5saver.raw_group
+
+        settings_xml = ET.Element('All_settings', type='group')
+        settings_xml.append(ioxml.walk_parameters_to_xml(param=self._module.settings))
+        if self.main_module:
+            saver_xml = ET.SubElement(settings_xml, 'H5Saver', type='group')
+            saver_xml.append(ioxml.walk_parameters_to_xml(param=self._h5saver.settings))
+
+        if self._module.ui is not None:
+            for ind, viewer in enumerate(self._module.viewers):
+                if hasattr(viewer, 'roi_manager'):
+                    roi_xml = ET.SubElement(settings_xml, f'ROI_Viewer_{ind:02d}', type='group')
+                    roi_xml.append(ioxml.walk_parameters_to_xml(param=viewer.roi_manager.settings))
+
+        return self._h5saver.add_det_group(where, title=self._module.title, settings_as_xml=ET.tostring(settings_xml),
+                                           metadata=metadata)
+
+    def add_data(self, where: Union[Node, str], data: DataToExport):
+        self._datatoexport_saver.add_data(where, data)
+
+    def add_bkg(self, where: Union[Node, str], data_bkg: DataToExport):
+        self._datatoexport_saver.add_bkg(where, data_bkg)
+
+    def add_external_h5(self, other_h5data: H5SaverLowLevel):
+        if other_h5data is not None:
+            external_group = self._h5saver.add_group('external_data', 'external_h5', self.module_group)
+            try:
+                if not other_h5data.isopen:
+                    h5saver = H5SaverLowLevel()
+                    h5saver.init_file(addhoc_file_path=other_h5data.filename)
+                    h5_file = h5saver.h5_file
+                else:
+                    h5_file = other_h5data
+                h5_file.copy_children(h5_file.get_node('/'), external_group, recursive=True)
+                h5_file.flush()
+                h5_file.close()
+
+            except Exception as e:
+                self.logger.exception(str(e))
+
+
+class DetectorEnlargeableSaver(DetectorSaver):
+    """Implementation of the ModuleSaver class dedicated to DAQ_Viewer modules in order to save enlargeable data
+
+    Parameters
+    ----------
+    module
+    """
+    group_type = GroupType['detector']
+
+    def __init__(self, module: DAQ_Viewer):
+        super().__init__(module)
+        self._datatoexport_saver: DataToExportTimedSaver = None
+
+    def update_after_h5changed(self, ):
+        self._datatoexport_saver = DataToExportTimedSaver(self.h5saver)
+
+
+class DetectorExtendedSaver(DetectorSaver):
+    """Implementation of the ModuleSaver class dedicated to DAQ_Viewer modules in order to save enlargeable data
+
+    Parameters
+    ----------
+    module
+    """
+    group_type = GroupType['detector']
+
+    def __init__(self, module: DAQ_Viewer, extended_shape: Tuple[int]):
+        super().__init__(module)
+        self._extended_shape = extended_shape
+        self._datatoexport_saver: DataToExportExtendedSaver = None
+
+    def update_after_h5changed(self, ):
+        self._datatoexport_saver = DataToExportExtendedSaver(self.h5saver, self._extended_shape)
+
+    def add_data(self, where: Union[Node, str], data: DataToExport, indexes: Tuple[int],
+                 distribution=DataDistribution['uniform']):
+        self._datatoexport_saver.add_data(where, data, indexes=indexes, distribution=distribution)
+
+    def add_nav_axes(self, where: Union[Node, str], axes: List[Axis]):
+        self._datatoexport_saver.add_nav_axes(where, axes)
+
+
+class ActuatorSaver(ModuleSaver):
+    """Implementation of the ModuleSaver class dedicated to DAQ_Move modules
+
+    Parameters
+    ----------
+    h5saver
+    module
+    """
+    group_type = GroupType['actuator']
+
+    def __init__(self, module: DAQ_Move):
+        self._axis_saver = None
+        self._module_group: GROUP = None
+        self._module: DAQ_Move = module
+        self._h5saver = None
+
+    def update_after_h5changed(self):
+        self._axis_saver = AxisSaverLoader(self.h5saver)
+
+    def _add_module(self, where: Union[Node, str] = None, metadata={}):
+        if where is None:
+            where = self._h5saver.raw_group
+
+        settings_xml = ET.Element('All_settings')
+        settings_xml.append(ioxml.walk_parameters_to_xml(param=self._module.settings))
+
+        return self._h5saver.add_act_group(where, title=self._module.title, settings_as_xml=ET.tostring(settings_xml),
+                                           metadata=metadata)
+
+
+class ScanSaver(ModuleSaver):
+    """Implementation of the ModuleSaver class dedicated to DAQScan module
+
+    Parameters
+    ----------
+    h5saver
+    module
+    """
+    group_type = GroupType['scan']
+
+    def __init__(self, module):
+        self._module_group: GROUP = None
+        self._module: 'DAQScan' = module
+        self._h5saver = None
+
+    def update_after_h5changed(self):
+        for module in self._module.modules_manager.modules_all:
+            if hasattr(module, 'module_and_data_saver'):
+                module.module_and_data_saver.h5saver = self.h5saver
+
+    def get_set_node(self, where: Union[Node, str] = None, new=False) -> GROUP:
+        """Get the last group scan node
+
+        Get the last Scan Group or create one
+        get the last Scan Group if:
+        * there is one already created
+        * new is False
+
+        Parameters
+        ----------
+        where: Union[Node, str]
+            the path of a given node or the node itself
+        new: bool
+
+        Returns
+        -------
+        GROUP: the GROUP associated with this module
+        """
+        self._module_group = self.get_last_node(where)
+        new = new or (self._module_group is None)
+        if new:
+            self._module_group = self._add_module(where)
+        for module in self._module.modules_manager.modules:
+            module.module_and_data_saver.main_module = False
+            module.module_and_data_saver.get_set_node(self._module_group)
+        return self._module_group
+
+    def _add_module(self, where: Union[Node, str] = None, metadata={}) -> Node:
+        """
+
+        Parameters
+        ----------
+        where: Union[Node, str]
+            the path of a given node or the node itself
+        metadata: dict
+
+        Returns
+        -------
+
+        """
+        if where is None:
+            where = self._h5saver.raw_group
+
+        settings_xml = ET.Element('All_settings', type='group')
+        settings_xml.append(ioxml.walk_parameters_to_xml(param=self._module.settings))
+        if self.main_module:
+            saver_xml = ET.SubElement(settings_xml, 'H5Saver', type='group')
+            saver_xml.append(ioxml.walk_parameters_to_xml(param=self._h5saver.settings))
+
+        return self._h5saver.add_scan_group(where, title=self._module.title,
+                                            settings_as_xml=ET.tostring(settings_xml),
+                                            metadata=metadata)
+
+    def add_nav_axes(self, axes: List[Axis]):
+        for detector in self._module.modules_manager.detectors:
+            detector.module_and_data_saver.add_nav_axes(self._module_group, axes)
+
+    def add_data(self, indexes: Tuple[int] = None, distribution=DataDistribution['uniform']):
+        for detector in self._module.modules_manager.detectors:
+            try:
+                detector.insert_data(indexes, where=self._module_group, distribution=distribution)
+            except Exception as e:
+                pass
+
+