holado 0.13.22__py3-none-any.whl

holado_grpc/ipc/rpc/grpc_services.py

@@ -0,0 +1,165 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+import os
+from holado_core.common.exceptions.functional_exception import FunctionalException
+from holado_core.common.exceptions.technical_exception import TechnicalException
+import sys
+import importlib
+from holado_core.common.tools.tools import Tools
+
+logger = logging.getLogger(__name__)
+
+try:
+    import grpc_requests  # @UnusedImport
+    with_grpc_requests = True
+except Exception as exc:
+    if Tools.do_log(logger, logging.DEBUG):
+        logger.debug(f"GRpcServices is not available. Initialization failed on error: {exc}")
+    with_grpc_requests = False
+
+
+class GRpcServices(object):
+    @classmethod
+    def is_available(cls):
+        return with_grpc_requests
+    
+    def __init__(self): 
+        self.__service_descriptors_by_fullname = {}
+        self.__services_by_fullname = {}
+    
+    def import_all_compiled_proto(self, compiled_proto_path, raise_if_not_exist=True):
+        """Register a folder path containing compiled proto files. Usually it corresponds to the parameter '--python_out' passed to proto compiler."""
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"[GRpcServices] Importing all compiled proto in '{compiled_proto_path}'...")
+        if os.path.exists(compiled_proto_path):
+            if os.path.isfile(compiled_proto_path):
+                proto_path = os.path.dirname(compiled_proto_path)
+                sys.path.append(proto_path)
+                self.__import_compiled_proto(proto_path, compiled_proto_path)
+            elif os.path.isdir(compiled_proto_path):
+                sys.path.append(compiled_proto_path)
+                self.__import_all_compiled_proto(compiled_proto_path, "")
+            else:
+                raise TechnicalException(f"Unmanaged path '{proto_path}'")
+        else:
+            msg = f"Path '{compiled_proto_path}' doesn't exist"
+            if raise_if_not_exist:
+                raise TechnicalException(msg)
+            else:
+                logger.warning(msg)
+    
+    def __import_all_compiled_proto(self, compiled_proto_path, package_name):
+        if Tools.do_log(logger, logging.TRACE):  # @UndefinedVariable
+            logger.trace(f"[GRpcServices] Importing all compiled proto in '{compiled_proto_path}' (package: '{package_name}')...")
+        if os.path.isdir(compiled_proto_path):
+            lp = os.listdir(compiled_proto_path)
+            for cp in lp:
+                if not cp.startswith((".", "_")):
+                    cur_proto_path = os.path.join(compiled_proto_path, cp)
+                    
+                    if os.path.isfile(cur_proto_path):
+                        self.__import_compiled_proto(cur_proto_path, package_name)
+                    elif os.path.isdir(cur_proto_path):
+                        cur_package_name = f"{package_name}.{cp}" if package_name is not None and len(package_name) > 0 else cp
+                        self.__import_all_compiled_proto(cur_proto_path, cur_package_name)
+                    else:
+                        raise TechnicalException(f"Unmanaged path '{cur_proto_path}'")
+        else:
+            raise TechnicalException(f"Unmanaged path '{compiled_proto_path}'")
+    
+    def __import_compiled_proto(self, compiled_proto_file_path, package_name):
+        if not os.path.isfile(compiled_proto_file_path):
+            raise TechnicalException(f"Compiled proto path '{compiled_proto_file_path}' is not a file")
+        if not compiled_proto_file_path.endswith("_pb2.py"):
+            return
+
+        if Tools.do_log(logger, logging.TRACE):  # @UndefinedVariable
+            logger.trace(f"[GRpcServices] Importing compiled proto file '{compiled_proto_file_path}' (package: '{package_name}')...")
+        
+        filename = os.path.splitext(os.path.basename(compiled_proto_file_path))[0]
+        module_name = f"{package_name}.{filename}" if package_name is not None and len(package_name) > 0 else filename
+        if Tools.do_log(logger, logging.TRACE):  # @UndefinedVariable
+            logger.trace(f"[GRpcServices] Importing module '{module_name}'")
+        module = importlib.import_module(module_name)
+        
+        if hasattr(module.DESCRIPTOR, 'services_by_name'):
+            service_package = module.DESCRIPTOR.package if hasattr(module.DESCRIPTOR, 'package') else package_name
+            
+            module_grpc_name = module_name + '_grpc'
+            module_spec = importlib.util.find_spec(module_grpc_name)
+            if module_spec is not None:
+                module_grpc = importlib.import_module(module_grpc_name)
+            
+            for s_name in module.DESCRIPTOR.services_by_name:
+                s_fullname = f"{service_package}.{s_name}" if service_package is not None and len(service_package) > 0 else s_name
+                self.__service_descriptors_by_fullname[s_fullname] = module.DESCRIPTOR.services_by_name[s_name]
+                if Tools.do_log(logger, logging.DEBUG):
+                    logger.debug(f"[GRpcServices] New managed service descriptor '{s_fullname}'")
+                if module_grpc is not None:
+                    self.__services_by_fullname[s_fullname] = getattr(module_grpc, s_name)
+                    if Tools.do_log(logger, logging.DEBUG):
+                        logger.debug(f"[GRpcServices] New managed service '{s_fullname}' (type: '{self.__services_by_fullname[s_fullname].__qualname__}')")
+
+    def has_service(self, service_fullname):
+        """Return if service fullname is known."""
+        return service_fullname in self.__services_by_fullname
+
+    def get_service(self, service_fullname):
+        """Return service for given service fullname."""
+        if self.has_service(service_fullname):
+            return self.__services_by_fullname[service_fullname]
+        else:
+            raise FunctionalException(f"Unknown service '{service_fullname}' (existing services: {list(self.__services_by_fullname.keys())})")
+
+    def has_service_descriptor(self, service_fullname):
+        """Return if service descriptor fullname is known."""
+        return service_fullname in self.__service_descriptors_by_fullname
+
+    def get_service_descriptor(self, service_fullname):
+        """Return service descriptor for given service fullname."""
+        if self.has_service_descriptor(service_fullname):
+            return self.__service_descriptors_by_fullname[service_fullname]
+        else:
+            raise FunctionalException(f"Unknown service '{service_fullname}' (existing services: {list(self.__service_descriptors_by_fullname.keys())})")
+
+    def has_method_descriptor(self, service_fullname, method_name):
+        """Return if method exists in service descriptor fullname."""
+        service_descriptor = self.get_service_descriptor(service_fullname)
+        return method_name in service_descriptor.methods_by_name
+
+    def get_method_descriptor(self, service_fullname, method_name):
+        """Return service method descriptor for given service fullname and method name."""
+        # Get service descriptor to verify in same type that it exists
+        service_descriptor = self.get_service_descriptor(service_fullname)
+        
+        # Get method descriptor
+        if method_name in service_descriptor.methods_by_name:
+            return service_descriptor.methods_by_name[method_name]
+        else:
+            raise FunctionalException(f"Unknown method '{method_name}' in service '{service_fullname}' (existing methods in service: {list(service_descriptor.methods_by_name.keys())})")
+    
+    def get_all_service_fullnames(self):
+        return tuple(self.__service_descriptors_by_fullname.keys())
+    
+    def get_all_method_fullnames(self):
+        res = []
+        for service_fullname in self.get_all_service_fullnames():
+            service_descriptor = self.get_service_descriptor(service_fullname)
+            for method_name in service_descriptor.methods_by_name:
+                res.append(f"{service_fullname}.{method_name}")
+        return res
+    
+    
+        

holado_grpc/tests/behave/steps/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+
+from .api.grpc_client_steps import *

holado_grpc/tests/behave/steps/api/grpc_client_steps.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+
+from holado_test.scenario.step_tools import StepTools
+from holado.common.context.session_context import SessionContext
+from holado_test.behave.behave import *  # @UnusedWildImport
+from holado_grpc.api.rpc.grpc_client import GRpcClient
+import logging
+from holado_test.behave.scenario.behave_step_tools import BehaveStepTools
+from holado_value.common.tables.converters.value_table_converter import ValueTableConverter
+from holado_core.common.tools.tools import Tools
+from holado_system.system.global_system import GlobalSystem
+import queue
+from holado_python.standard_library.typing import Typing
+
+logger = logging.getLogger(__name__)
+
+
+if GRpcClient.is_available():
+
+    def __get_session_context():
+        return SessionContext.instance()
+    
+    def __get_scenario_context():
+        return __get_session_context().get_scenario_context()
+    
+    def __get_text_interpreter():
+        return __get_scenario_context().get_text_interpreter()
+    
+    def __get_variable_manager():
+        return __get_scenario_context().get_variable_manager()
+    
+    def __get_grpc_services():
+        return __get_session_context().grpc_services
+
+        
+    @Given(r"(?P<var_name>{Variable}) = new gRPC client")
+    def step_impl(context, var_name):
+        var_name = StepTools.evaluate_variable_name(var_name)
+        table = BehaveStepTools.convert_step_table_2_value_table_with_header(context.table)
+        kwargs = ValueTableConverter.convert_name_value_table_2_dict(table)
+        
+        client = __get_session_context().grpc_manager.new_client(None, **kwargs)
+        
+        __get_variable_manager().register_variable(var_name, client)
+        
+    @Given(r"(?P<var_name>{Variable}) = service names \(gRPC client: (?P<var_client>{Variable})\)")
+    def step_impl(context, var_name, var_client):
+        var_name = StepTools.evaluate_variable_name(var_name)
+        client = StepTools.evaluate_variable_value(var_client)
+        
+        service_names = client.internal_client.service_names
+        
+        __get_variable_manager().register_variable(var_name, service_names)
+        
+    @Step(r"(?P<var_name>{Variable}) = input type fullname of request (?P<service_method>{Str}) \(gRPC client: (?P<var_client>{Variable})\)")
+    def step_impl(context, var_name, service_method, var_client):
+        var_name = StepTools.evaluate_variable_name(var_name)
+        service_method = StepTools.evaluate_scenario_parameter(service_method)
+        client = StepTools.evaluate_variable_value(var_client)
+        
+        service_name, method_name = service_method.rsplit('.', 1)
+        res = client.get_request_data_type_fullname(service_name, method_name)
+        
+        __get_variable_manager().register_variable(var_name, res)
+        
+    @When(r"(?P<var_name>{Variable}) = request (?P<service_method>{Str}) \(gRPC client: (?P<var_client>{Variable})(?: ; with Protobuf (?P<with_proto_request>request)?(?: & )?(?P<with_proto_response>response)?)?\)")
+    def step_impl(context, var_name, service_method, var_client, with_proto_request, with_proto_response):
+        var_name = StepTools.evaluate_variable_name(var_name)
+        service_method = StepTools.evaluate_scenario_parameter(service_method)
+        client = StepTools.evaluate_variable_value(var_client)
+        table = BehaveStepTools.get_step_table(context)
+        
+        service_name, method_name = service_method.rsplit('.', 1)
+        request_data = client.build_request_data(service_name, method_name, params_table=table, as_proto=with_proto_request is not None)
+        res = client.request(service_name, method_name, request_data, raw_output=with_proto_response is not None)
+        
+        __get_variable_manager().register_variable(var_name, res)
+        
+    @When(r"(?P<var_name>{Variable}) = request (?P<service_method>{Str}) with data (?P<request_data>{Str}) \(gRPC client: (?P<var_client>{Variable})(?: ; with Protobuf (?P<with_proto_response>response)?)?\)")
+    def step_impl(context, var_name, service_method, request_data, var_client, with_proto_response):
+        var_name = StepTools.evaluate_variable_name(var_name)
+        service_method = StepTools.evaluate_scenario_parameter(service_method)
+        request_data = StepTools.evaluate_scenario_parameter(request_data)
+        client = StepTools.evaluate_variable_value(var_client)
+        
+        service_name, method_name = service_method.rsplit('.', 1)
+        res = client.request(service_name, method_name, request_data, raw_output=with_proto_response is not None)
+        
+        __get_variable_manager().register_variable(var_name, res)
+        
+    @Step(r"(?P<var_name>{Variable}) = data for request (?P<service_method>{Str}) \(gRPC client: (?P<var_client>{Variable})(?P<as_proto> ; as Protobuf)?\)")
+    def step_impl(context, var_name, service_method, var_client, as_proto):
+        var_name = StepTools.evaluate_variable_name(var_name)
+        service_method = StepTools.evaluate_scenario_parameter(service_method)
+        client = StepTools.evaluate_variable_value(var_client)
+        table = BehaveStepTools.get_step_table(context)
+        
+        service_name, method_name = service_method.rsplit('.', 1)
+        res = client.build_request_data(service_name, method_name, params_table=table, as_proto=as_proto is not None)
+        
+        __get_variable_manager().register_variable(var_name, res)
+        
+    @When(r"request (?P<service_method>{Str}) for all of queue (?P<var_queue>{Variable}) \(gRPC client: (?P<var_client>{Variable})(?: ; rate log period: (?P<rate_period_s>{Int}) s)?\)")
+    def step_impl(context, service_method, var_queue, var_client, rate_period_s):
+        service_method = StepTools.evaluate_scenario_parameter(service_method)
+        q = StepTools.evaluate_variable_value(var_queue)
+        q_name = StepTools.evaluate_variable_name(var_queue)
+        client = StepTools.evaluate_variable_value(var_client)
+        client_name = StepTools.evaluate_variable_name(var_client)
+        rate_period_s = StepTools.evaluate_scenario_parameter(rate_period_s)
+        
+        counter = 0
+        beg = Tools.timer_s()
+        c_beg, t_beg = counter, beg
+        log_counter = 0
+        service_name, method_name = service_method.rsplit('.', 1)
+        
+        try:
+            while True:
+                # Get next message and publish
+                request_data = q.get()
+                if q.is_sentinel(request_data):
+                    break
+                counter += 1
+                
+                try:
+                    # Note: according internal library implementation, less actions are done with raw_output=True
+                    client.request(service_name, method_name, request_data, raw_output=True)
+                except Exception as exc:
+                    logger.warning(f"[Queue {q_name} -> Client {client_name}] Failed to request data {counter}: [{Typing.get_object_class_fullname(exc)}] {str(exc)}")
+                    # logger.warning(f"[Queue {q_name} -> Publisher {pub_name}] Failed to publish message {counter}: [{Typing.get_object_class_fullname(exc)}] {Tools.represent_exception(exc)}")
+                finally:
+                    q.task_done()
+                
+                # Log rate if needed
+                if rate_period_s and counter % 10 == 0:
+                    t_end = Tools.timer_s()
+                    if t_end > t_beg + rate_period_s:
+                        c_end, s_end = counter, q.qsize()
+                        rate = (c_end - c_beg) / (t_end - t_beg)
+                        log_counter += 1
+                        logger.print(f"[Queue {q_name} -> Client {client_name}] Rate: {int(rate)} msg/s ; Nb messages: {counter} ; Queue size: {s_end}")
+                        c_beg, t_beg = c_end, t_end
+                        
+                # Yield processor in case of multithreading
+                if counter % 10 == 0:
+                    GlobalSystem.yield_processor()
+        except queue.Empty:
+            # Without block, or with timeout, this exception occurs when queue is empty
+            pass
+        
+        if rate_period_s:
+            end = Tools.timer_s()
+            rate = counter / (end - beg)
+            logger.print(f"[Queue {q_name} -> Client {client_name}] Mean rate: {int(rate)} msg/s ; Nb messages: {counter}")
+
+
+
+
+    

holado_grpc/tests/behave/steps/private/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+
+from .api.grpc_steps import *

holado_grpc/tests/behave/steps/private/api/grpc_steps.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+
+from holado.common.context.session_context import SessionContext
+from holado_test.behave.behave import *  # @UnusedWildImport
+import os.path
+from holado_core.tools.abstracts.blocking_command_service import BlockingCommandService
+import logging
+from holado_test.scenario.step_tools import StepTools
+
+logger = logging.getLogger(__name__)
+
+
+def __get_session_context():
+    return SessionContext.instance()
+
+def __get_scenario_context():
+    return __get_session_context().get_scenario_context()
+
+def __get_text_interpreter():
+    return __get_scenario_context().get_text_interpreter()
+
+def __get_variable_manager():
+    return __get_scenario_context().get_variable_manager()
+
+
+@Given(r"(?P<var_name>{Variable}) = start internal gRPC server")
+def step_impl(context, var_name):
+    """
+    This step runs the api_grpc/api1 gRPC server available in testing tools.
+    
+    It was written with djangogrpcframework that is not maintained anymore.
+    Version 0.2.1 of djangogrpcframework is not compatible as is with Django>=4.1.4.
+    A workaround working currently is to comment line 20 "requires_system_checks = False" in file "django_grpc_framework/management/commands/grpcrunserver.py".
+    TODO: update tools by using library https://github.com/socotecio/django-socio-grpc instead of djangogrpcframework
+    """
+    var_name = StepTools.evaluate_variable_name(var_name)
+    
+    here = os.path.abspath(os.path.dirname(__file__))
+    django_project_path = os.path.normpath(os.path.join(here, "..", "..", "..", "..", "..", "..", "..", "tests", "behave", "test_holado", "tools", "django", "api_grpc"))
+    
+    execute_steps(u"""
+        Given {var_name} = new gRPC Django server
+            | Name                   | Value                          |
+            | 'name'                 | 'gRPC server for holado tests' |
+            | 'django_project_path'  | '{django_project_path}'        |
+        When start (Django server: {var_name})
+        """.format(var_name=var_name, django_project_path=django_project_path))
+    
+    # Update imported grpc messages and services
+    proto_path = os.path.join(django_project_path, "api_grpc", "api1", "proto")
+    __get_session_context().protobuf_messages.import_all_compiled_proto(proto_path)
+    __get_session_context().grpc_services.import_all_compiled_proto(proto_path)
+    
+    
+@Given(r"(?P<var_name>{Variable}) = new internal gRPC client on service (?P<service_name>{Str})")
+def step_impl(context, var_name, service_name):
+    var_name = StepTools.evaluate_variable_name(var_name)
+    
+    execute_steps(u"""
+        Given {var_name} = new gRPC client
+            | Name                      | Value           |
+            | 'endpoint'                | '0.0.0.0:50051' |
+            | 'service'                 | {service_name}  |
+        """.format(var_name=var_name, service_name=service_name))

holado_helper/__init__.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+# This module is a helper for use cases using HolAdo solution.
+# Usage: copy needed files and adapt them for specific usage.
+
+
+
+def dependencies():
+    """
+    Return None or the list of names of modules it depend on
+    """
+    return None
+
+def register():
+    """
+    Register the services of this module
+    """
+    # from holado.common.context.session_context import SessionContext
+    
+    # from holado_xxx.xxx import XXXManager
+    # SessionContext.instance().services.register_service_type("xxx_manager", XXXManager)
+    pass
+

holado_helper/debug/README.txt

@@ -0,0 +1,32 @@
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+This module gives some tools for debugging purposes.
+Each tool uses internally a third party library.
+
+
+Existing external tools can be found in following links:
+    https://wiki.python.org/moin/PythonDebuggingTools
+    
+    For memory profiling:
+        https://www.datacamp.com/tutorial/memory-profiling-python
+        https://stackify.com/top-5-python-memory-profilers/
+
+    For deadlock analyse:
+        pystack, py-spy
+        
+        Commands with py-spy:
+            sudo env "PATH=$PATH" py-spy dump --pid XXX --full-filenames > py-spy_dump_summary.txt
+            sudo env "PATH=$PATH" py-spy dump --pid XXX --full-filenames --locals --subprocesses > py-spy_dump_details.txt
+            sudo env "PATH=$PATH" py-spy dump --pid XXX --full-filenames --locals --subprocesses --json > py-spy_dump_details.json
+        
+

holado_helper/debug/memory/memory_profiler.py

@@ -0,0 +1,106 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+from holado_core.common.tools.tools import Tools
+import logging
+from holado_core.common.exceptions.technical_exception import TechnicalException
+from holado_python.standard_library.typing import Typing
+
+logger = logging.getLogger(__name__)
+
+
+try:
+    import pympler
+    from pympler.tracker import SummaryTracker
+    from pympler.tracker import ObjectTracker
+    with_pympler = True
+except Exception as exc:
+    if Tools.do_log(logger, logging.DEBUG):
+        logger.debug(f"Pympler is not available for memory profiling. Initialization failed on error: {exc}")
+    with_pympler = False
+
+
+class MemoryProfiler():
+    """
+    Manage many methods to profile memory.
+    It uses internally different libraries, depending on profiling method.
+    """
+    __stored_trackers = {}
+    
+    @classmethod
+    def is_tracker_available(cls):
+        return with_pympler
+    
+    @classmethod
+    def create_or_reset_tracker_of_objects_summary_changes(cls, name):
+        if name in cls.__stored_trackers:
+            cls.reset_tracker(cls.__stored_trackers[name])
+        else:
+            cls.__stored_trackers[name] = cls.new_tracker_of_objects_summary_changes()
+        return cls.__stored_trackers[name]
+    
+    @classmethod
+    def create_or_reset_tracker_of_objects_changes(cls, name):
+        if name in cls.__stored_trackers:
+            cls.reset_tracker(cls.__stored_trackers[name])
+        else:
+            cls.__stored_trackers[name] = cls.new_tracker_of_objects_changes()
+        return cls.__stored_trackers[name]
+    
+    @classmethod
+    def new_tracker_of_objects_summary_changes(cls):
+        if cls.is_tracker_available():
+            return SummaryTracker()
+        else:
+            raise TechnicalException("Third library 'Pympler' is needed for this profiling method")
+    
+    @classmethod
+    def new_tracker_of_objects_changes(cls):
+        if cls.is_tracker_available():
+            return ObjectTracker()
+        else:
+            raise TechnicalException("Third library 'Pympler' is needed for this profiling method")
+    
+    @classmethod
+    def reset_tracker(cls, tracker):
+        if isinstance(tracker, SummaryTracker):
+            _ = tracker.diff()
+        elif isinstance(tracker, ObjectTracker):
+            _ = tracker.get_diff()
+        else:
+            raise TechnicalException(f"Unexpected tracker type '{Typing.get_object_class_fullname(tracker)}'")
+    
+    @classmethod
+    def log_tracker_diff(cls, name=None, tracker=None, prefix=None, level=logging.DEBUG, logger_=None):
+        if logger_ is None:
+            logger_ = logger
+        if not logger_.isEnabledFor(level):
+            return
+        
+        if name is not None:
+            tracker = cls.__stored_trackers.get(name)
+        if tracker is None:
+            raise TechnicalException(f"Parameter 'tracker' or 'name' must be defined")
+        
+        if isinstance(tracker, SummaryTracker):
+            tracker_descr = "objects summary"
+        elif isinstance(tracker, ObjectTracker):
+            tracker_descr = "objects summary"
+        else:
+            raise TechnicalException(f"Unexpected tracker type '{Typing.get_object_class_fullname(tracker)}'")
+        
+        logger_.log(level, f"{prefix if prefix else ''}Preparing changes in {tracker_descr}...")
+        formatted_diff = "\n".join(tracker.format_diff())
+        logger_.log(level, f"{prefix if prefix else ''}Changes in {tracker_descr}:\n{formatted_diff}")
+        
+

holado_helper/docker/init_user.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Script to init a new docker user as existing user appuser.
+#
+# Default HolAdo Dockerfile creates a user appuser, configure it so that it can run tests, and it runs as this user.
+# When defining a specific new user when running docker image, this user is not configured to be able to run tests.
+# The purpose of this script is to give a simple solution to initialize this new user as user appuser.
+#
+# USAGE:
+#     In the command run in docker, call this script before the command to execute.
+#
+#     Example: to run all non-reg tests as a specific user defined in run command, set in yaml file
+#
+#           user: ${TEST_UID}:${TEST_GID}
+#           command: "bash -c \"/code/holado/python/src/holado_helper/docker/init_user.sh && ./run_test_nonreg.sh\""
+
+
+# Activate python venv
+source /code/env/bin/activate
+
+# Set environment variables needed by HolAdo
+export HOLADO_PATH=/code/holado/python/
+
+