pymodaq 3.6.12__py3-none-any.whl → 4.0.1__py3-none-any.whl

pymodaq/{daq_utils/h5backend.py → utils/h5modules/backends.py}

@@ -1,18 +1,24 @@
-#Standard imports
+# -*- coding: utf-8 -*-
+"""
+Created the 15/11/2022
+
+@author: Sebastien Weber
+"""
+import numpy as np
 import importlib
 import pickle
+from typing import Dict
 
-#3rd party imports
-import numpy as np
+from pymodaq.utils.logger import set_logger, get_module_name
+from pymodaq.utils.config import Config
+from pymodaq.utils.daq_utils import capitalize, JsonConverter
+from pymodaq.utils import daq_utils as utils
+from pymodaq.utils.enums import BaseEnum, enum_checker
 
-#Project imports
-from pymodaq.daq_utils import daq_utils as utils
-from pymodaq.daq_utils.daq_utils import JsonConverter
-from pymodaq.daq_utils.exceptions import InvalidGroupType
 
-logger = utils.set_logger(utils.get_module_name(__file__))
+config = Config()
+logger = set_logger(get_module_name(__file__))
 
-#Initialized at import
 backends_available = []
 
 # default backend
@@ -31,7 +37,7 @@ try:
     backends_available.append('h5py')
 except Exception as e:                              # pragma: no cover
     logger.warning(str(e))
-    is_h5y = False
+    is_h5py = False
 
 is_h5pyd = True
 # this one is to be used for remote reading/writing towards a HSDS server (or h5serv), see HDFGroup
@@ -40,12 +46,112 @@ try:
     backends_available.append('h5pyd')
 except Exception as e:                              # pragma: no cover
     logger.warning(str(e))
-    is_h5yd = False
+    is_h5pyd = False
 
 if not (is_tables or is_h5py or is_h5pyd):
     logger.exception('No valid hdf5 backend has been installed, please install either pytables or h5py')
 
-#As I understand, the Node object should never be instanciated
+
+class SaveType(BaseEnum):
+    scan = 0
+    detector = 1
+    logger = 2
+    custom = 3
+
+
+class GroupType(BaseEnum):
+    detector = 0
+    actuator = 1
+    data = 2
+    ch = 3
+    scan = 4
+    external_h5 = 5
+    data_dim = 6
+
+
+class InvalidExport(Exception):
+    pass
+
+
+def check_mandatory_attrs(attr_name, attr):
+    """for cross compatibility between different backends. If these attributes have binary value, then decode them
+
+    Parameters
+    ----------
+    attr_name
+    attr
+
+    Returns
+    -------
+
+    """
+    if attr_name == 'TITLE' or attr_name == 'CLASS' or attr_name == 'EXTDIM':
+        if isinstance(attr, bytes):
+            return attr.decode()
+        else:
+            return attr
+    else:
+        return attr
+
+
+def get_attr(node, attr_name, backend='tables'):
+    if backend == 'tables':
+        if attr_name is not None:
+            attr = node._v_attrs[attr_name]
+            attr = check_mandatory_attrs(attr_name, attr)
+            return JsonConverter.json2object(attr)
+        else:
+            attrs = dict([])
+            for attr_name in node._v_attrs._v_attrnames:
+                attrval = node._v_attrs[attr_name]
+                attrval = check_mandatory_attrs(attr_name, attrval)
+                attrs[attr_name] = JsonConverter.json2object(attrval)
+            return attrs
+    else:
+        if attr_name is not None:
+            attr = node.attrs[attr_name]
+            attr = check_mandatory_attrs(attr_name, attr)
+            return JsonConverter.json2object(attr)
+        else:
+            attrs = dict([])
+            for attr_name in node.attrs.keys():
+                attrval = node.attrs[attr_name]
+                attrval = check_mandatory_attrs(attr_name, attrval)
+                attrs[attr_name] = JsonConverter.json2object(attrval)
+            return attrs
+
+
+def set_attr(node, attr_name, attr_value, backend='tables'):
+    if backend == 'tables':
+        node._v_attrs[attr_name] = JsonConverter.object2json(attr_value)
+    else:
+        node.attrs[attr_name] = JsonConverter.object2json(attr_value)
+
+
+class InvalidGroupType(Exception):
+    pass
+
+
+class InvalidSave(Exception):
+    pass
+
+
+class InvalidGroupDataType(Exception):
+    pass
+
+
+class InvalidDataType(Exception):
+    pass
+
+
+class InvalidDataDimension(Exception):
+    pass
+
+
+class InvalidScanType(Exception):
+    pass
+
+
 class Node(object):
     def __init__(self, node, backend):
         if isinstance(node, Node):  # to ovoid recursion if one call Node(Node()) or even more
@@ -71,10 +177,10 @@ class Node(object):
         return self.node == other.node
 
     @property
-    def parent_node(self):
+    def parent_node(self) -> 'GROUP':
         if self.path == '/':
             return None
-        mod = importlib.import_module('.h5backend', 'pymodaq.daq_utils')
+        mod = importlib.import_module('.backends', 'pymodaq.utils.h5modules')
 
         if self.backend == 'tables':
             p = self.node._v_parent
@@ -107,6 +213,10 @@ class Node(object):
             else:
                 return path.split('/')[-1]
 
+    @property
+    def title(self):
+        return self.attrs['TITLE']
+
     @property
     def path(self):
         """return node path
@@ -123,15 +233,6 @@ class Node(object):
         else:
             return self._node.name
 
-    def get_file(self):
-        """ Return node file. Depending on the backend, will return either a h5py or a pytables object.
-
-        """
-        if self.backend == 'tables':
-            return self.node._v_file
-        else:
-            return self.node.file
-
 
 class GROUP(Node):
     def __init__(self, node, backend):
@@ -158,8 +259,10 @@ class GROUP(Node):
 
         return "%s\n  children := %s" % (str(self), childlist)
 
-    def children(self):
-        """Get a dict containing all children node hanging from self with their name as keys
+
+
+    def children(self) -> Dict[str, Node]:
+        """Get a dict containing all children node hanging from self whith their name as keys
 
         Returns
         -------
@@ -169,8 +272,7 @@ class GROUP(Node):
         --------
         children_name
         """
-        # mod = importlib.import_module('.h5modules', 'pymodaq.daq_utils')
-        mod = importlib.import_module('.h5backend', 'pymodaq.daq_utils')
+        mod = importlib.import_module('.backends', 'pymodaq.utils.h5modules')
         children = dict([])
         if self.backend == 'tables':
             for child_name, child in self.node._v_children.items():
@@ -191,19 +293,30 @@ class GROUP(Node):
                 children[child_name] = _cls(child, self.backend)
         return children
 
+    def get_child(self, name: str) -> Node:
+        return self.children()[name]
+
     def children_name(self):
-        """Gets the list of children name hanging from self
+        """Gets the sorted list of children name hanging from self
 
         Returns
         -------
         list: list of name of the children
         """
         if self.backend == 'tables':
-            return list(self.node._v_children.keys())
+            return sorted(list(self.node._v_children.keys()))
         else:
-            return list(self.node.keys())
+            return sorted(list(self.node.keys()))
         pass
 
+    def remove_children(self):
+        children_dict = self.children()
+        for child_name in children_dict:
+            if self.backend == 'tables':
+                children_dict[child_name].node._f_remove(recursive=True)
+            else:
+                self.node.__delitem__(child_name)
+
 
 class CARRAY(Node):
     def __init__(self, node, backend):
@@ -305,14 +418,39 @@ class Attributes(object):
         self.backend = backend
 
     def __getitem__(self, item):
+        if item == 'title':
+            item = item.upper()
         attr = get_attr(self._node.node, item, backend=self.backend)
         # if isinstance(attr, bytes):
         #    attr = attr.decode()
         return attr
 
     def __setitem__(self, key, value):
+        if key == 'title':
+            key = key.upper()
         set_attr(self._node.node, key, value, backend=self.backend)
 
+    def __iter__(self):
+        self._iter_index = 0
+        return self
+
+    def __next__(self):
+        if self._iter_index < len(self):
+            self._iter_index += 1
+            return self.attrs_name[self._iter_index-1]
+        else:
+            raise StopIteration
+
+    def __len__(self):
+        return len(self.attrs_name)
+
+    def to_dict(self) -> dict:
+        """Returns attributes name/value as a dict"""
+        attrs_dict = dict()
+        for name in self.attrs_name:
+            attrs_dict[name] = self[name]
+        return attrs_dict
+
     @property
     def node(self):
         return self._node
@@ -351,80 +489,26 @@ class Attributes(object):
             return str(self)
 
 
-def get_attr(node, attr_name, backend='tables'):
-    if backend == 'tables':
-        if attr_name is not None:
-            attr = node._v_attrs[attr_name]
-            attr = check_mandatory_attrs(attr_name, attr)
-            return JsonConverter.json2object(attr)
-        else:
-            attrs = dict([])
-            for attr_name in node._v_attrs._v_attrnames:
-                attrval = node._v_attrs[attr_name]
-                attrval = check_mandatory_attrs(attr_name, attrval)
-                attrs[attr_name] = JsonConverter.json2object(attrval)
-            return attrs
-    else:
-        if attr_name is not None:
-            attr = node.attrs[attr_name]
-            attr = check_mandatory_attrs(attr_name, attr)
-            return JsonConverter.json2object(attr)
-        else:
-            attrs = dict([])
-            for attr_name in node.attrs.keys():
-                attrval = node.attrs[attr_name]
-                attrval = check_mandatory_attrs(attr_name, attrval)
-                attrs[attr_name] = JsonConverter.json2object(attrval)
-            return attrs
-
-def set_attr(node, attr_name, attr_value, backend='tables'):
-    if backend == 'tables':
-        node._v_attrs[attr_name] = JsonConverter.object2json(attr_value)
-    else:
-        node.attrs[attr_name] = JsonConverter.object2json(attr_value)
-
-def check_mandatory_attrs(attr_name, attr):
-    """for cross compatibility between different backends. If these attributes have binary value, then decode them
-
-    Parameters
-    ----------
-    attr_name
-    attr
-
-    Returns
-    -------
-
-    """
-    if attr_name == 'TITLE' or attr_name == 'CLASS' or attr_name == 'EXTDIM':
-        if isinstance(attr, bytes):
-            return attr.decode()
-        else:
-            return attr
-    else:
-        return attr
-
-group_types = ['raw_datas', 'scan', 'detector', 'move', 'data', 'ch', '', 'external_h5']
-
 class H5Backend:
     def __init__(self, backend='tables'):
 
-        self._h5file = None # _h5file is the object created by one of the backends. Typically a  tables.File
+        self._h5file = None
         self.backend = backend
         self.file_path = None
         self.compression = None
         if backend == 'tables':
             if is_tables:
-                self.h5module = tables
+                self.h5_library = tables
             else:
                 raise ImportError('the pytables module is not present')
         elif backend == 'h5py':
             if is_h5py:
-                self.h5module = h5py
+                self.h5_library = h5py
             else:
                 raise ImportError('the h5py module is not present')
         elif backend == 'h5pyd':
             if is_h5pyd:
-                self.h5module = h5pyd
+                self.h5_library = h5pyd
             else:
                 raise ImportError('the h5pyd module is not present')
 
@@ -437,6 +521,10 @@ class H5Backend:
         self.file_path = file.filename
         self._h5file = file
 
+    @property
+    def filename(self):
+        return self._h5file.filename
+
     def isopen(self):
         if self._h5file is None:
             return False
@@ -461,26 +549,23 @@ class H5Backend:
     def open_file(self, fullpathname, mode='r', title='PyMoDAQ file', **kwargs):
         self.file_path = fullpathname
         if self.backend == 'tables':
-            self._h5file = self.h5module.open_file(str(fullpathname), mode=mode, title=title, **kwargs)
+            self._h5file = self.h5_library.open_file(str(fullpathname), mode=mode, title=title, **kwargs)
             if mode == 'w':
                 self.root().attrs['pymodaq_version'] = utils.get_version()
             return self._h5file
         else:
-            self._h5file = self.h5module.File(str(fullpathname), mode=mode, **kwargs)
+            self._h5file = self.h5_library.File(str(fullpathname), mode=mode, **kwargs)
 
             if mode == 'w':
                 self.root().attrs['TITLE'] = title
                 self.root().attrs['pymodaq_version'] = utils.get_version()
             return self._h5file
 
-    def save_file_as(self, filenamepath='h5copy.h5'):
-        """"""
+    def save_file_as(self, filenamepath='h5copy.txt'):
         if self.backend == 'tables':
             self.h5file.copy_file(str(filenamepath))
         else:
-            with h5py.File(filenamepath, 'w') as f_dest:
-                self.h5file.copy(self.h5file, f_dest)
-            # raise Warning(f'Not possible to copy the file with the "{self.backend}" backend')
+            raise Warning(f'Not possible to copy the file with the "{self.backend}" backend')
 
     def root(self):
         if self.backend == 'tables':
@@ -498,6 +583,9 @@ class H5Backend:
             node = node.node
         return set_attr(node, attr_name, attr_value, self.backend)
 
+    def has_attr(self, node, attr_name):
+        return attr_name in self.get_node(node).attrs.attrs_name
+
     def flush(self):
         if self._h5file is not None:
             self._h5file.flush()
@@ -514,7 +602,7 @@ class H5Backend:
         if self.backend == 'tables':
             if compression == 'gzip':
                 compression = 'zlib'
-            self.compression = self.h5module.Filters(complevel=compression_opts, complib=compression)
+            self.compression = self.h5_library.Filters(complevel=compression_opts, complib=compression)
         else:
             if compression == 'zlib':
                 compression = 'gzip'
@@ -585,12 +673,9 @@ class H5Backend:
         return name.lower() in [name.lower() for name in self.get_children(where)]
 
     def get_node(self, where, name=None) -> Node:
-        """This method returns a node object (for sure?) that"""
-        #This casts where into a string object for the pytables backend but otherwise not.
         if isinstance(where, Node):
             where = where.node
 
-        #Then we get back to a node object but backend-dependent
         if self.backend == 'tables':
             node = self._h5file.get_node(where, name)
         else:
@@ -666,9 +751,11 @@ class H5Backend:
     def get_children(self, where):
         """Get a dict containing all children node hanging from where with their name as keys and types among Node,
         CARRAY, EARRAY, VLARRAY or StringARRAY
+
         Parameters
         ----------
-        where (str or node instance), see h5py and pytables documentation on nodes, and Node objects of this module
+        where (str or node instance)
+            see h5py and pytables documentation on nodes, and Node objects of this module
 
         Returns
         -------
@@ -676,14 +763,14 @@ class H5Backend:
 
         See Also
         --------
-        children_name, Node, CARRAY, EARRAY, VLARRAY or StringARRAY
+        :meth:`.GROUP.children_name`
+
         """
         where = self.get_node(where)  # return a node object in case where is a string
         if isinstance(where, Node):
             where = where.node
 
-        mod = importlib.import_module('.h5backend', 'pymodaq.daq_utils')
-        # mod = importlib.import_module('.h5modules', 'pymodaq.daq_utils')
+        mod = importlib.import_module('.backends', 'pymodaq.utils.h5modules')
         children = dict([])
         if self.backend == 'tables':
             for child_name, child in where._v_children.items():
@@ -773,7 +860,7 @@ class H5Backend:
         shape = tuple(shape)
 
         if self.backend == 'tables':
-            atom = self.h5module.Atom.from_dtype(dtype)
+            atom = self.h5_library.Atom.from_dtype(dtype)
             array = EARRAY(self._h5file.create_earray(where, name, atom, shape=shape, title=title,
                                                       filters=self.compression), self.backend)
         else:
@@ -823,7 +910,7 @@ class H5Backend:
             dtype = np.dtype(dtype)
             subdtype = ''
         if self.backend == 'tables':
-            atom = self.h5module.Atom.from_dtype(dtype)
+            atom = self.h5_library.Atom.from_dtype(dtype)
             if subdtype == 'string':
                 array = StringARRAY(self._h5file.create_vlarray(where, name, atom, title=title,
                                                                 filters=self.compression), self.backend)
@@ -833,7 +920,7 @@ class H5Backend:
         else:
             maxshape = (None,)
             if self.backend == 'h5py':
-                dt = self.h5module.vlen_dtype(dtype)
+                dt = self.h5_library.vlen_dtype(dtype)
             else:
                 dt = h5pyd.special_dtype(dtype)
             if self.compression is not None:
@@ -861,13 +948,14 @@ class H5Backend:
         array.attrs['backend'] = self.backend
         return array
 
-    def add_group(self, group_name, group_type, where, title='', metadata=dict([])):
+    def add_group(self, group_name, group_type, where, title='', metadata=dict([])) -> GROUP:
         """
         Add a node in the h5 file tree of the group type
         Parameters
         ----------
         group_name: (str) a custom name for this group
-        group_type: (str) one of the possible values of **group_types**
+        group_type: str or GroupType enum
+            one of the possible values of GroupType
         where: (str or node) parent node where to create the new group
         metadata: (dict) extra metadata to be saved with this new group node
 
@@ -877,16 +965,16 @@ class H5Backend:
         """
         if isinstance(where, Node):
             where = where.node
-        if group_type not in group_types:
-            raise InvalidGroupType('Invalid group type')
+
+        group_type = enum_checker(GroupType, group_type)
 
         if group_name in self.get_children(self.get_node(where)):
             node = self.get_node(where, group_name)
 
         else:
             node = self.get_set_group(where, utils.capitalize(group_name), title)
-            node.attrs['type'] = group_type.lower()
+            node.attrs['type'] = group_type.name.lower()
             for metadat in metadata:
                 node.attrs[metadat] = metadata[metadat]
         node.attrs['backend'] = self.backend
-        return node
+        return node