pymodaq 5.1.6__py3-none-any.whl

pymodaq/extensions/daq_logger/db/db_logger.py

@@ -0,0 +1,300 @@
+from contextlib import contextmanager
+import logging
+import datetime
+from typing import List
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy_utils import database_exists, create_database
+
+from pymodaq_utils.logger import set_logger, get_module_name
+from pymodaq_utils.config import Config
+
+from pymodaq_gui.h5modules.saving import dashboard_submodules_params
+from pymodaq_gui.messenger import messagebox
+from pymodaq_gui.managers.parameter_manager import ParameterManager
+
+from pymodaq_data.data import DataToExport
+
+
+from .db_logger_models import (Base, Data0D, Data1D, Data2D, LogInfo,
+                               Configuration, ControlModule)
+from ..abstract import AbstractLogger
+
+
+logger = set_logger(get_module_name(__file__))
+config = Config()
+
+
+class DBLogHandler(logging.StreamHandler):
+    def __init__(self, dblogger):
+        super().__init__()
+        self.dblogger = dblogger
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        self.setFormatter(formatter)
+
+    def emit(self, record):
+        msg = self.format(record)
+        self.dblogger.add_log(msg)
+
+
+class DbLogger:
+    user = config('network', 'logging', 'user', 'username')
+    user_pwd = config('network', 'logging', 'user', 'pwd')
+
+    def __init__(self, database_name, ip_address=config('network', 'logging', 'sql', 'ip'),
+                 port=config('network', 'logging', 'sql', 'port'), save2D=False):
+        """
+
+        Parameters
+        ----------
+        models_base
+        ip_address
+        port
+        database_name
+        """
+
+        self.ip_address = ip_address
+        self.port = port
+        self.database_name = database_name
+
+        self.engine = None
+        self.Session = None
+        self._save2D = save2D
+
+    @property
+    def save2D(self):
+        return self._save2D
+
+    @save2D.setter
+    def save2D(self, value):
+        self._save2D = value
+
+    @contextmanager
+    def session_scope(self):
+        """Provide a transactional scope around a series of operations."""
+        session = self.Session()
+        try:
+            yield session
+            session.commit()
+        except Exception as e:
+            logger.error(str(e))
+            session.rollback()
+        finally:
+            session.close()
+
+    def connect_db(self):
+        url = f"postgresql://{self.user}:{self.user_pwd}@{self.ip_address}:"\
+              f"{self.port}/{self.database_name}"
+        logger.debug(f'Connecting database using: {url}')
+        try:
+            self.engine = create_engine(f"postgresql://{self.user}:{self.user_pwd}@{self.ip_address}:"
+                                        f"{self.port}/{self.database_name}")
+        except ModuleNotFoundError as e:
+            messagebox('warning', 'ModuleError',
+                       f'The postgresql backend *psycopg2* has not been installed.\n'
+                       f'Could not connect to your database')
+            logger.warning(str(e))
+            return False
+
+        try:
+            if not database_exists(self.engine.url):
+                create_database(self.engine.url)
+            assert database_exists(self.engine.url)
+        except Exception as e:
+            logger.critical(f'Impossible to connect to the DB: {str(e)}')
+            return False
+
+        self.create_table()
+        self.Session = sessionmaker(bind=self.engine)
+        logger.debug(f'Database Connected')
+        return True
+
+    def close(self):
+        if self.engine is not None:
+            self.engine.dispose()
+
+    def create_table(self):
+        # create tables if not existing
+        if self.engine is not None:
+            Base.metadata.create_all(self.engine)
+
+    def get_detectors(self, session) -> List[str]:
+        """Returns the list of detectors name
+
+        Parameters
+        ----------
+        session: (Session) SQLAlchemy session instance for db transactions
+
+        Returns
+        -------
+        the list of all created detectors
+        """
+        return [res[0] for res in session.query(ControlModule.name)]
+
+    def get_actuators(self, session) -> List[str]:
+        """Returns the list of actuators name
+
+        Parameters
+        ----------
+        session: (Session) SQLAlchemy session instance for db transactions
+
+        Returns
+        -------
+        the list of all created actuators
+        """
+        return [res[0] for res in session.query(ControlModule.name)]
+
+    def add_detectors(self, detectors):
+        """
+        add detectors in the detectors table
+        Parameters
+        ----------
+        detectors: (list) list of dict with keys: name and settings_xml
+        """
+        self.add_control_modules(detectors, 'DAQ_Viewer')
+
+    def add_actuators(self, actuators):
+        """
+        add actuators in the actuators table
+        Parameters
+        ----------
+        actuators: list
+            list of dict with keys: name and settings_xml
+        """
+        self.add_control_modules(actuators, 'DAQ_Move')
+
+    def add_control_modules(self, modules, module_type='DAQ_Viewer'):
+        if not isinstance(modules, list):
+            modules = [modules]
+        with self.session_scope() as session:
+            existing_modules = [d.name for d in session.query(ControlModule)]
+            for mod in modules:
+                if mod['name'] not in existing_modules:
+                    session.add(ControlModule(name=mod['name'], module_type=module_type,
+                                              settings_xml=mod['xml_settings']))
+
+    def add_config(self, config_settings):
+        with self.session_scope() as session:
+            session.add(Configuration(timestamp=datetime.datetime.now().timestamp(), settings_xml=config_settings))
+
+    def add_log(self, log):
+        with self.session_scope() as session:
+            session.add(LogInfo(log))
+
+    def add_data(self, data: DataToExport):
+        with self.session_scope() as session:
+            time_stamp = data.timestamp
+            module_name = data.name
+
+            if session.query(ControlModule).filter_by(name=module_name).count() == 0:
+                self.add_control_modules(session, dict(name=module_name), data.control_module)
+
+            module_id = session.query(ControlModule).filter_by(name=module_name).one().id  # detector/actuator names should/are unique
+
+            for dwa in data.get_data_from_dim('Data0D'):
+                for ind, data_array in enumerate(dwa):
+                    session.add(Data0D(timestamp=dwa.timestamp, control_module_id=module_id, channel=dwa.labels[ind],
+                                       value=float(data_array[0])))
+
+            for dwa in data.get_data_from_dim('Data1D'):
+                for ind, data_array in enumerate(dwa):
+                    session.add(Data1D(timestamp=dwa.timestamp, control_module_id=module_id,
+                                       channel=dwa.labels[ind],
+                                       value=data_array.tolist()))
+            if self.save2D:
+                for dwa in data.get_data_from_dim('Data2D'):
+                    for ind, data_array in enumerate(dwa):
+                        session.add(Data2D(timestamp=dwa.timestamp, control_module_id=module_id,
+                                           channel=dwa.labels[ind],
+                                           value=data_array.tolist()))
+
+            # not yet dataND as db should not know where to save these datas
+
+
+class DbLoggerGUI(DbLogger, ParameterManager):
+    params = [
+        {'title': 'Database:', 'name': 'database_type', 'type': 'list', 'value': 'PostgreSQL',
+            'limits': ['PostgreSQL', ]},
+        {'title': 'Server IP:', 'name': 'server_ip', 'type': 'str',
+            'value': config('network', 'logging', 'sql', 'ip'),
+         'tip':'Either localhost if the database server is on the same computer or the IP address of the server'},
+        {'title': 'Server port:', 'name': 'server_port', 'type': 'int',
+            'value': config('network', 'logging', 'sql', 'port')},
+        {'title': 'Connect:', 'name': 'connect_db', 'type': 'bool_push', 'value': False},
+        {'title': 'Connected:', 'name': 'connected_db', 'type': 'led', 'value': False},
+    ] + dashboard_submodules_params
+
+    def __init__(self, database_name):
+        DbLogger.__init__(self, database_name, ip_address=config('network', 'logging', 'sql', 'ip'),
+                          port=config('network', 'logging', 'sql', 'port'), save2D=False)
+        ParameterManager.__init__(self)
+
+        self.settings.child('do_save').hide()
+
+    def value_changed(self, param):
+        if param.name() == 'server_ip':
+            self.ip_address = param.value()
+
+        elif param.name() == 'server_port':
+            self.port = param.value()
+
+        elif param.name() == 'connect_db':
+            if param.value():
+                status = self.connect_db()
+                self.settings.child('connected_db').setValue(status)
+            else:
+                self.close()
+                self.settings.child('connected_db').setValue(False)
+
+        elif param.name() == 'save_2D':
+            self.save2D = param.value()
+
+
+class DataBaseLogger(AbstractLogger):
+    def __init__(self, database_name):
+        self.dblogger = DbLoggerGUI(database_name)
+
+    @property
+    def settings_tree(self):
+        return self.dblogger.settings_tree
+
+    @property
+    def settings(self):
+        return self.dblogger.settings
+
+    def init_logger(self, settings):
+        if not self.settings.child('connected_db').value():
+            status = self.dblogger.connect_db()
+            if not status:
+                logger.critical('the Database is not and cannot be connnect')
+                return False
+
+        self.dblogger.add_config(settings)
+        return True
+
+    def get_handler(self):
+        return DBLogHandler(self.dblogger)
+
+    def add_detector(self, det_name, settings):
+        self.dblogger.add_detectors([dict(name=det_name, xml_settings=settings)])
+
+    def add_actuator(self, act_name, settings):
+        self.dblogger.add_actuators([dict(name=act_name, xml_settings=settings)])
+
+    def add_data(self, data):
+        self.dblogger.add_data(data)
+        self.settings.child('N_saved').setValue(
+            self.settings.child('N_saved').value() + 1)
+
+    def stop_logger(self):
+        pass
+
+    def close(self):
+        pass
+
+
+if __name__ == '__main__':
+    db = DbLogger('preset_default')
+    db.connect_db()
+    pass

pymodaq/extensions/daq_logger/db/db_logger_models.py

@@ -0,0 +1,100 @@
+import datetime
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker, relationship
+from sqlalchemy import Column, Integer, String, Float, ForeignKey
+from sqlalchemy.dialects.postgresql import ARRAY as Array
+
+
+Base = declarative_base()
+
+
+class Configuration(Base):
+    __tablename__ = 'configurations'
+    id = Column(Integer, primary_key=True)
+    timestamp = Column(Integer, nullable=False, index=True)
+    settings_xml = Column(String)
+
+    def __repr__(self):
+        return f"<Config(date='{datetime.datetime.fromtimestamp(self.timestamp).isoformat()}', settings_xml='{self.settings_xml[0:20]}')>"
+
+# class Actuator(Base):
+#     __tablename__ = 'actuators'
+#     id = Column(Integer, primary_key=True)
+#     name = Column(String(128))
+#     settings_xml = Column(String)
+#     datas0D = relationship("Data0D", backref='actuators')
+#
+#
+# class Detector(Base):
+#     __tablename__ = 'detectors'
+#     id = Column(Integer, primary_key=True)
+#     name = Column(String(128))
+#     settings_xml = Column(String)
+#     datas0D = relationship("Data0D", backref='detectors')
+#     datas1D = relationship("Data1D", backref='detectors')
+#     datas2D = relationship("Data2D", backref='detectors')
+#
+#     def __repr__(self):
+#         return f"<Detector(name='{self.name}', settings_xml='{self.settings_xml[0:20]}')>"
+
+
+class ControlModule(Base):
+    __tablename__ = 'control_modules'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(128))
+    module_type = Column(String(128))
+    settings_xml = Column(String)
+    datas0D = relationship("Data0D", backref='control_modules')
+    datas1D = relationship("Data1D", backref='control_modules')
+    datas2D = relationship("Data2D", backref='control_modules')
+
+    def __repr__(self):
+        return f"<Control Module {self.type} (name='{self.name}', settings_xml='{self.settings_xml[0:20]}')>"
+
+
+class LogInfo(Base):
+    __tablename__ = 'loginfo'
+    id = Column(Integer, primary_key=True)
+    value = Column(String)
+
+    def __repr__(self):
+        return f"<Loginfo(timestamp='{self.timestamp}', value='{self.value}')>"
+
+
+class Data0D(Base):
+    __tablename__ = 'datas0D'
+
+    id = Column(Integer, primary_key=True)
+    timestamp = Column(Integer, nullable=False, index=True)
+    control_module_id = Column(Integer, ForeignKey('control_modules.id'), index=True)
+    channel = Column(String(128))
+    value = Column(Float)
+
+    def __repr__(self):
+        return f"<Data0D(channel='{self.channel}', timestamp='{self.timestamp}', value='{self.value}')>"
+
+
+class Data1D(Base):
+    __tablename__ = 'datas1D'
+
+    id = Column(Integer, primary_key=True)
+    timestamp = Column(Integer, nullable=False, index=True)
+    control_module_id = Column(Integer, ForeignKey('control_modules.id'), index=True)
+    channel = Column(String(128))
+    value = Column(Array(Float, dimensions=1))
+
+    def __repr__(self):
+        return f"<Data1D(channel='{self.channel}', timestamp='{self.timestamp}', value='{self.value}')>"
+
+
+class Data2D(Base):
+    __tablename__ = 'datas2D'
+
+    id = Column(Integer, primary_key=True)
+    timestamp = Column(Integer, nullable=False, index=True)
+    control_module_id = Column(Integer, ForeignKey('control_modules.id'), index=True)
+    channel = Column(String(128))
+    value = Column(Array(Float, dimensions=2))
+
+    def __repr__(self):
+        return f"<Data2D(channel='{self.channel}', timestamp='{self.timestamp}', value='{self.value}')>"

pymodaq/extensions/daq_logger/h5logging.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+Created the 15/11/2022
+
+@author: Sebastien Weber
+"""
+
+import logging
+import numpy as np
+
+from pymodaq_utils.logger import set_logger, get_module_name
+from pymodaq_utils.config import Config
+from pymodaq_gui.h5modules.saving import H5Saver
+from pymodaq_data.data import DataToExport
+
+from pymodaq.utils.managers.modules_manager import ModulesManager
+from pymodaq.utils.h5modules import module_saving
+from .abstract import AbstractLogger
+
+
+config = Config()
+
+logger = set_logger(get_module_name(__file__))
+
+
+class H5LogHandler(logging.StreamHandler):
+    def __init__(self, h5saver):
+        super().__init__()
+        self.h5saver = h5saver
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        self.setFormatter(formatter)
+
+    def emit(self, record):
+        msg = self.format(record)
+        self.h5saver.add_log(msg)
+
+
+class H5Logger(AbstractLogger):
+    def __init__(self, modules_manager, *args, **kwargs):
+
+        self.title = self.__class__.__name__
+
+        self.modules_manager: ModulesManager = modules_manager
+        self.h5saver = H5Saver(*args, save_type='logger', **kwargs)
+
+        self.module_and_data_saver = module_saving.LoggerSaver(self)
+        for det in self.modules_manager.detectors_all:
+            det.module_and_data_saver = module_saving.DetectorTimeSaver(det)
+        self.module_and_data_saver.h5saver = self.h5saver  # will update its h5saver and all submodules's h5saver
+
+    def close(self):
+        self.h5saver.close_file()
+
+    @property
+    def settings_tree(self):
+        return self.h5saver.settings_tree
+
+    @property
+    def settings(self):
+        return self.h5saver.settings
+
+    def init_logger(self, settings):
+        self.h5saver.init_file(update_h5=True, metadata=dict(settings=settings))
+        self.h5saver.flush()
+        self.module_and_data_saver.h5saver = self.h5saver
+        logger_node = self.module_and_data_saver.get_set_node(new=True)
+        return True
+
+    def get_handler(self):
+        return H5LogHandler(self.h5saver)
+
+    def add_detector(self, name, settings):
+        pass
+
+    def add_actuator(self, name, settings):
+        pass
+
+    def add_data(self, dte: DataToExport):
+        self.module_and_data_saver.add_data(dte)
+
+        self.settings.child('N_saved').setValue(self.settings.child('N_saved').value() + 1)
+
+    def stop_logger(self):
+        self.h5saver.flush()