holado 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

holado_logging/common/logging/log_config.py

@@ -0,0 +1,128 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+
+
+# logger = logging.getLogger(__name__)
+
+
+class LogConfig(object):
+    TLogger = None
+    TManager = None
+    default_level = logging.INFO
+    
+    @classmethod
+    def configure(cls, initialize_logging=True, log_level=None):
+        # HolAdo needs at least to add logging level TRACE and PRINT
+        cls.add_logging_level_trace()
+        cls.add_logging_level_print()
+        
+        if log_level:
+            if isinstance(log_level, str):
+                cls.default_level = logging._nameToLevel[log_level]
+            else:
+                cls.default_level = log_level
+        
+        if initialize_logging:
+            from holado_logging.common.logging.holado_logger import HALogger
+            
+            HALogger.default_message_size_limit = 10000
+            cls.TLogger = HALogger
+            cls.TManager = logging.Manager
+            
+            # Configure logging
+            cls.configure_logging()
+    
+    @classmethod
+    def configure_logging(cls):
+        #TODO EKL: make loggers configuration optional
+        # Configure loggers
+        from holado_logging.common.logging.holado_logger import TestRootLogger
+        logging.root = TestRootLogger(cls.default_level)
+        logging.Logger.root = logging.root
+        logging.Logger.manager = cls.TManager(cls.TLogger.root)
+        
+        logging.setLoggerClass(cls.TLogger)
+        
+    
+    @classmethod
+    def add_logging_level_print(cls):
+        if not cls.has_logging_level("PRINT"):
+            cls.add_logging_level("PRINT", 45, None)
+    
+    @classmethod
+    def add_logging_level_trace(cls):
+        if not cls.has_logging_level("TRACE"):
+            cls.add_logging_level("TRACE", 5, None)
+        
+    
+    @classmethod
+    def has_logging_level(cls, levelName):
+        return hasattr(logging, levelName)
+    
+    @classmethod
+    def add_logging_level(cls, levelName, levelNum, methodName=None):
+        """
+        This method was implemented and shared by the author of library haggis (https://haggis.readthedocs.io).
+        
+        Comprehensively adds a new logging level to the `logging` module and the
+        currently configured logging class.
+    
+        `levelName` becomes an attribute of the `logging` module with the value
+        `levelNum`. `methodName` becomes a convenience method for both `logging`
+        itself and the class returned by `logging.getLoggerClass()` (usually just
+        `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is
+        used.
+    
+        To avoid accidental clobberings of existing attributes, this method will
+        raise an `AttributeError` if the level name is already an attribute of the
+        `logging` module or if the method name is already present 
+    
+        Example
+        -------
+        >>> addLoggingLevel('TRACE', logging.DEBUG - 5)
+        >>> logging.getLogger(__name__).setLevel("TRACE")
+        >>> logging.getLogger(__name__).trace('that worked')
+        >>> logging.trace('so did this')
+        >>> logging.TRACE
+        5
+    
+        """
+        if not methodName:
+            methodName = levelName.lower()
+    
+        if hasattr(logging, levelName):
+            raise AttributeError('{} already defined in logging module'.format(levelName))
+        if hasattr(logging, methodName):
+            raise AttributeError('{} already defined in logging module'.format(methodName))
+        if hasattr(logging.getLoggerClass(), methodName):
+            raise AttributeError('{} already defined in logger class'.format(methodName))
+    
+        # This method was inspired by the answers to Stack Overflow post
+        # http://stackoverflow.com/q/2183233/2988730, especially
+        # http://stackoverflow.com/a/13638084/2988730
+        def logForLevel(self, message, *args, **kwargs):
+            if self.isEnabledFor(levelNum):
+                self._log(levelNum, message, args, **kwargs)
+        def logToRoot(message, *args, **kwargs):
+            logging.log(levelNum, message, *args, **kwargs)
+    
+        logging.addLevelName(levelNum, levelName)
+        setattr(logging, levelName, levelNum)
+        setattr(logging.getLoggerClass(), methodName, logForLevel)
+        setattr(logging, methodName, logToRoot)
+    
+    
+    
+    

holado_logging/common/logging/log_manager.py

@@ -0,0 +1,292 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import threading
+import logging
+import sys
+import os.path
+from pathlib import Path
+import configparser
+from holado_logging.common.logging.log_config import LogConfig
+from holado_core.common.tools.tools import Tools
+
+
+logger = logging.getLogger(__name__)
+
+def filter_thread_native_id(record):
+    """Inject thread_native_id to log records"""
+    from holado_multitask.multitasking.multitask_manager import MultitaskManager
+    record.thread_native_id = MultitaskManager.get_thread_id(native=True)
+    return record
+
+class LogManager(object):
+
+    def __init__(self):
+        self.__loggers_levels = []
+        
+        self.__config_file_path = None
+        
+        self.on_console = False
+        self.__console_handler = None
+        
+        self.in_file = True
+        self.__files_lock = threading.Lock()
+        self.__file_names = []
+        self.__file_handlers = {}
+        self.__root_file_name = None
+        self.__root_file_handler = None
+        self.__root_file_handler_active = False
+        
+        # Initialize format according python version
+        from holado_multitask.multitasking.multitask_manager import MultitaskManager
+        if MultitaskManager.has_thread_native_id():
+            # Exists since python 3.8
+            # self.format = '{asctime:s} | {thread_native_id:-5d} | {levelname:5s} | {name:50s} | {message:s}'
+            self.format = '{asctime:s} | {process:-5d}-{thread_native_id:-5d} | {levelname:5s} | {name:50s} | {message:s}'
+            self.style = '{'
+        else:
+            if sys.version_info > (3, 2):
+                # self.format = '{asctime:s} | {thread:-5d} | {levelname:5s} | {module:35s} | {message:s}'
+                self.format = '{asctime:s} | {process:-5d}-{thread:-5d} | {levelname:5s} | {name:50s} | {message:s}'
+                self.style = '{'
+            else: 
+                self.format = '%(asctime)s | %(process)-5d-%(thread)-5d | %(levelname)5s | %(module)35s | %(message)s'
+                self.style = '%'
+        
+    def initialize(self, log_on_console=True):
+        """
+        Initialize log manager.
+        If log_on_console is True, logs are published on console until a new configuration by calling method set_config
+        """
+        if log_on_console:
+            self.on_console = True
+            self.__console_handler = self.__new_console_handler()
+            logging.basicConfig(format=self.format, style=self.style, level=LogConfig.default_level, handlers=[self.__console_handler])
+        else:
+            logging.basicConfig(format=self.format, style=self.style, level=LogConfig.default_level, handlers=[])
+        
+    def set_config_file_path(self, file_path, update_default_level=True):
+        self.__config_file_path = file_path
+        
+        config = configparser.ConfigParser()
+        config.read(self.__config_file_path)
+        if update_default_level:
+            LogConfig.default_level = config.get("logger_root", "level")
+        
+        if config.has_section("loggers_levels"):
+            self.__loggers_levels = config.items(section="loggers_levels")
+    
+    def has_log_file(self, file_name):
+        with self.__files_lock:
+            return file_name in self.__file_names or file_name == self.__root_file_name
+    
+    def set_root_log_file(self, file_name):
+        with self.__files_lock:
+            if file_name is not None and len(file_name) > 0:
+                self.__root_file_name = file_name
+                
+    def reset_log_files(self):
+        with self.__files_lock:
+            self.__file_names.clear()
+                
+    def add_log_file(self, file_name):
+        from holado_core.common.exceptions.technical_exception import TechnicalException
+        
+        if file_name is not None and len(file_name) > 0:
+            if file_name == self.__root_file_name:
+                raise TechnicalException(f"Log file '{file_name}' is already set as root log file")
+            with self.__files_lock:
+                if file_name in self.__file_names:
+                    raise TechnicalException(f"Log file '{file_name}' is already set")
+                self.__file_names.append(file_name)
+    
+    def remove_log_file(self, file_name):
+        with self.__files_lock:
+            if file_name is not None and len(file_name) > 0 and file_name in self.__file_names:
+                self.__file_names.remove(file_name)
+        
+    def __new_console_handler(self):
+        from holado_multitask.multitasking.multitask_manager import MultitaskManager
+        
+        res = logging.StreamHandler()
+        res.setFormatter(logging.Formatter(fmt=self.format, style=self.style))
+        if MultitaskManager.has_thread_native_id():
+            res.addFilter(filter_thread_native_id)
+        return res
+    
+    def set_config(self):
+        # print(f"Set logging config: {LogConfig.default_level=} ; {self.on_console=} ; {self.in_file=} ; {self.__root_file_name=}")
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Set logging config: {LogConfig.default_level=} ; {self.on_console=} ; {self.in_file=} ; {self.__root_file_name=}")
+
+        # if self.__config_file_path is not None:
+        #     logging.config.fileConfig(self.__config_file_path, defaults=None, disable_existing_loggers=False)
+        
+        logger_ = logging.getLogger()
+
+        # Update log destination to console
+        if self.on_console:
+            if self.__console_handler is not None:
+                if Tools.do_log(logger, logging.DEBUG):
+                    logger.debug("Log destination already set to console.")
+            else:
+                if Tools.do_log(logger, logging.DEBUG):
+                    logger.debug("Adding log destination to console.")
+                self.__console_handler = self.__new_console_handler()
+                logger_.addHandler(self.__console_handler)
+        else:
+            if self.__console_handler is not None:
+                if Tools.do_log(logger, logging.DEBUG):
+                    logger.debug("Removing log destination to console.")
+                logger_.removeHandler(self.__console_handler)
+                self.__console_handler = None
+                
+        # Update log destination to files
+        with self.__files_lock:
+            # Remove old log files
+            # if self.__root_file_name:
+            #     self.remove_root_file_handler(do_reset=True)
+            for file_name in list(self.__file_handlers.keys()):
+                if file_name not in self.__file_names:
+                    self.remove_file_handler(file_name, do_remove_log_file=False)
+            
+            # Add new log files
+            if self.__root_file_name and not self.__root_file_handler:
+                self.add_root_file_handler()
+            for file_name in self.__file_names:
+                if file_name not in list(self.__file_handlers.keys()):
+                    self.add_file_handler(file_name)
+                
+        
+        # level
+        if logger_.getEffectiveLevel() != LogConfig.default_level:
+            logger_.setLevel(LogConfig.default_level)
+    
+        # Loggers levels
+        for name, level in self.__loggers_levels:
+            if not name.startswith("#"):
+                logging.getLogger(name).setLevel(level)
+                
+    def set_level(self, log_level, do_set_config=True):
+        from holado_core.common.exceptions.technical_exception import TechnicalException
+        
+        if isinstance(log_level, str):
+            if hasattr(logging, log_level):
+                log_level = getattr(logging, log_level)
+            else:
+                raise TechnicalException(f"Unexpected log level string '{log_level}'")
+            
+        LogConfig.default_level = log_level
+        if do_set_config:
+            self.set_config()
+        
+    def add_root_file_handler(self):
+        from holado_core.common.exceptions.technical_exception import TechnicalException
+        from holado_multitask.multitasking.multitask_manager import MultitaskManager
+        
+        logger_ = logging.getLogger()
+            
+        if not self.__root_file_name:
+            raise TechnicalException("Root log file is not defined")
+            
+        if self.__root_file_handler is None:
+            logger_.info("Creating file handler to root file '{}'.".format(self.__root_file_name))
+            Path(os.path.dirname(self.__root_file_name)).mkdir(parents=True, exist_ok=True)
+            
+            self.__root_file_handler = logging.FileHandler(self.__root_file_name, mode='w', encoding='utf8')
+            self.__root_file_handler.setFormatter(logging.Formatter(fmt=self.format, style=self.style))
+            if MultitaskManager.has_thread_native_id():
+                self.__root_file_handler.addFilter(filter_thread_native_id)
+            
+        logger_.addHandler(self.__root_file_handler)
+        self.__root_file_handler_active = True
+        
+    def remove_root_file_handler(self, do_reset=False):
+        logger_ = logging.getLogger()
+        if self.__root_file_handler:
+            logger_.info(f"Removing log destination to root file '{self.__root_file_name}'.")
+            logger_.removeHandler(self.__root_file_handler)
+            self.__root_file_handler_active = False
+            if do_reset:
+                self.__root_file_handler = None
+        
+    def add_file_handler(self, file_name, logger_=None):
+        from holado_multitask.multitasking.multitask_manager import MultitaskManager
+        
+        if logger_ is None:
+            logger_ = logging.getLogger()
+            
+        # In case this method is called outside "set_config" method, add file_name as a configured log_file
+        if not self.has_log_file(file_name):
+            self.add_log_file(file_name)
+            
+        if file_name in self.__file_handlers:
+            logger_.debug("Log destination already set to file '{}'.".format(file_name))
+        else:
+            logger_.info("Adding log destination to file '{}'.".format(file_name))
+            Path(os.path.dirname(file_name)).mkdir(parents=True, exist_ok=True)
+            
+            file_handler = logging.FileHandler(file_name, mode='w', encoding='utf8')
+            file_handler.setFormatter(logging.Formatter(fmt=self.format, style=self.style))
+            if MultitaskManager.has_thread_native_id():
+                file_handler.addFilter(filter_thread_native_id)
+            
+            self.__file_handlers[file_name] = file_handler
+            logger_.addHandler(file_handler)
+                
+    def remove_file_handler(self, file_name, logger_=None, do_remove_log_file=True):
+        from holado_core.common.exceptions.technical_exception import TechnicalException
+        
+        if logger_ is None:
+            logger_ = logging.getLogger()
+            
+        if file_name not in self.__file_handlers:
+            raise TechnicalException(f"Not set log destination to file '{file_name}'")
+        
+        logger_.info("Removing log destination to file '{}'.".format(file_name))
+        logger_.removeHandler(self.__file_handlers[file_name])
+        del self.__file_handlers[file_name]
+        
+        if do_remove_log_file:
+            self.remove_log_file(file_name)
+    
+    def __remove_existing_file_handlers(self, log_level_if_exists=None):
+        with self.__files_lock:
+            if len(self.__file_names) > 0:
+                if log_level_if_exists is not None:
+                    logger_ = logging.getLogger()
+                    logger_.log(log_level_if_exists, f"Removing existing file handlers: {self.__file_names}")
+                file_names = list(self.__file_names)
+            else:
+                file_names = []
+        
+        # Note: removes are done outside lock to avoid a deadlock
+        for fn in file_names:
+            self.remove_file_handler(fn)
+    
+    def enter_log_file(self, file_name, do_remove_root_file_handler=True, do_remove_other_file_handlers=True):
+        if do_remove_other_file_handlers:
+            self.__remove_existing_file_handlers(log_level_if_exists=logging.ERROR)
+        
+        self.add_file_handler(file_name)
+        
+        if do_remove_root_file_handler:
+            self.remove_root_file_handler(do_reset=False)
+            
+    def leave_log_file(self, file_name, do_remove_log_file=True):
+        if not self.__root_file_handler_active:
+            self.add_root_file_handler()
+        self.remove_file_handler(file_name, do_remove_log_file=do_remove_log_file)
+    
+    
+    

holado_multitask/__init__.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+
+def dependencies():
+    return None
+
+def register():
+    from holado.common.context.session_context import SessionContext
+    
+    from holado_multitask.multitasking.multitask_manager import MultitaskManager
+    SessionContext.instance().services.register_service_type("multitask_manager", MultitaskManager)
+    
+    from holado_multitask.multiprocessing.processesmanager import ProcessesManager
+    SessionContext.instance().services.register_service_type("processes_manager", 
+                                                        lambda: ProcessesManager(SessionContext.instance().multitask_manager))
+    
+    from holado_multitask.multithreading.threadsmanager import ThreadsManager
+    SessionContext.instance().services.register_service_type("threads_manager", 
+                                                        lambda: ThreadsManager(SessionContext.instance().multitask_manager))
+
+

holado_multitask/multiprocessing/context/process_context.py

@@ -0,0 +1,35 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+from builtins import super
+from holado.common.context.context import Context
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessContext(Context):
+    def __init__(self, name, unique_name):
+        super().__init__(f"Process[{name};{unique_name}]")
+        
+        self.__process_id = None
+        self.__process_name = name
+        self.__process_unique_name = unique_name
+
+    @property
+    def process_name(self):
+        return self.__process_name
+    
+    
+    
+    

holado_multitask/multiprocessing/function_process.py

@@ -0,0 +1,102 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado_core.common.block.function import Function
+from holado_core.common.exceptions.functional_exception import FunctionalException
+from holado_core.common.tools.tools import Tools
+from holado_multitask.multiprocessing.process import InterruptableProcess
+
+logger = logging.getLogger(__name__)
+
+
+class FunctionProcess(InterruptableProcess):
+    '''
+    Execute in a process given function with given arguments
+    '''
+    
+    def __init__(self, target, args=None, kwargs=None, name = None, default_wait_timeout = None, register_process = True, delay_before_run_sec=None):
+        super().__init__(name if name is not None else f"FunctionProcess({repr(target)})", default_wait_timeout = default_wait_timeout, register_process = register_process, delay_before_run_sec=delay_before_run_sec)
+        
+        if args is None:
+            args = []
+        if kwargs is None:
+            kwargs = {}
+        
+        self._function = Function(target, *args, **kwargs)
+        self.__error = None
+        self.__result = None
+        self.__interrupt_function = None
+        self.__callback_function = None
+
+    @property
+    def error(self):
+        return self.__error
+
+    @property
+    def result(self):
+        return self.__result
+
+    @property
+    def interrupt_function(self):
+        return self.__interrupt_function
+
+    @interrupt_function.setter
+    def interrupt_function(self, func):
+        """Set the Function to call on interrupt"""
+        self.__interrupt_function = func
+
+    @property
+    def callback_function(self):
+        return self.__callback_function
+
+    @callback_function.setter
+    def callback_function(self, func):
+        """Set the callback function"""
+        self.__callback_function = func
+ 
+    def run(self):
+        # Start running by superclass run
+        super().run()
+        
+        # self._raise_if_interrupted()
+        self.wait_before_run()
+            
+        # self._raise_if_interrupted()
+        try:
+#             logging.debug("+++++++++ Launching function [{}({})]".format(repr(self._target), repr(self._args)))
+            self.__result = self._function.run()
+        except Exception as exc:
+            logger.exception(f"[{self.name}] Exception catched during processed function [{self._function.represent()}]")
+            self.__error = exc
+        
+        # self._raise_if_interrupted()
+        try:
+            if self.__callback_function:
+                self.__callback_function.run(self.result, self.error)
+        except Exception as exc:
+            msg = f"[{self.name}] Exception catched while calling callback function:\n    callback_function: {repr(self.__callback_function)}\n    result: [{Tools.truncate_text(repr(self.result))}]\n    error: [{repr(self.error)}]"
+            logger.exception(msg)
+            raise FunctionalException(msg) from exc
+        
+        logger.info(f"[{self.name}] Process has finished")
+
+    def throw_if_error(self):
+        if self.error:
+            raise FunctionalException(f"[{self.name}] Error during call of function [{repr(self._target)}({repr(self._args)})]") from self.error
+    
+    def interrupt(self):
+        if not self.is_interrupted and self.is_alive():
+            super().interrupt()
+            if self.interrupt_function is not None:
+                self.interrupt_function.run()