naeural-core 7.7.243__py3-none-any.whl → 7.7.245__py3-none-any.whl

naeural_core/business/base/base_plugin_biz.py

@@ -5,7 +5,7 @@ import numpy as np
 import traceback
 import json
 
-from time import time, sleep
+from time import time, sleep, perf_counter
 from copy import deepcopy
 from collections import deque, OrderedDict
 from functools import partial
@@ -1933,6 +1933,16 @@ class BasePluginExecutor(
       return updates
 
     def _update_instance_config(self):
+      debug_load_timings = self.log.config_data.get('PLUGINS_DEBUG_LOAD_TIMINGS', False)
+      if debug_load_timings:
+        cfg_start = perf_counter()
+        env_merge_s = 0.0
+        default_merge_s = 0.0
+        validate_s = 0.0
+        warnings_s = 0.0
+        alerters_s = 0.0
+        timebins_s = 0.0
+        shmem_s = 0.0
       if self._instance_config is not None:
         reconfig = True
         self.P("* * * * Reconfiguring plugin {} * *".format(self), color='b')
@@ -1946,30 +1956,46 @@ class BasePluginExecutor(
         # initially this call did not have `default_config = self._instance_config` that led
         # to wrongly using at each update the _default_config
         self.__set_loop_stage('_update_instance_config._environment_variables')
+        if debug_load_timings:
+          env_start = perf_counter()
         self._instance_config = self._merge_prepare_config(
           default_config=self._instance_config,
           delta_config=self._environment_variables,
           debug=self.__debug_config_changes,
         )
+        if debug_load_timings:
+          env_merge_s = perf_counter() - env_start
       # now normal delta config from upstream
       self.__set_loop_stage('_update_instance_config._default_config')
 
       updates = self._check_delta_config(self._upstream_config)  # check if there are any updates
       is_instance_command_only = len(updates) == 1 and 'INSTANCE_COMMAND' in updates
 
+      if debug_load_timings:
+        default_merge_start = perf_counter()
       self._instance_config = self._merge_prepare_config(
         default_config=self._instance_config,
         debug=self.__debug_config_changes,
       )
+      if debug_load_timings:
+        default_merge_s = perf_counter() - default_merge_start
 
       self.__set_loop_stage('_update_instance_config.setup_config_and_validate')
 
       # if we do not run the next line then the config will not be updated for each of the cfg_* handlers
       # as they will point to the old config_data dict
+      if debug_load_timings:
+        validate_start = perf_counter()
       self.setup_config_and_validate(self._instance_config)  # here _instance_config will be copyed to config_data
+      if debug_load_timings:
+        validate_s = perf_counter() - validate_start
 
       self.__set_loop_stage('_update_instance_config._print_warnings')
+      if debug_load_timings:
+        warnings_start = perf_counter()
       self._print_warnings()
+      if debug_load_timings:
+        warnings_s = perf_counter() - warnings_start
 
       # now for the second part of the config: reset/set stuff
       if is_instance_command_only:
@@ -1981,18 +2007,30 @@ class BasePluginExecutor(
           # TODO: make sure you set RESTART_ALERTERS_ON_CONFIG to False for plugins
           #       that should NOT restart alerters
           self.__set_loop_stage('_update_instance_config._create_alert_state_machine')
+          if debug_load_timings:
+            alerters_start = perf_counter()
           self._create_alert_state_machine()
+          if debug_load_timings:
+            alerters_s = perf_counter() - alerters_start
 
         # Create time bins mixin default # TODO: refactor similar with alerters
         self.__set_loop_stage('_update_instance_config.timebins_create_bin')
+        if debug_load_timings:
+          timebins_start = perf_counter()
         self.timebins_create_bin()
+        if debug_load_timings:
+          timebins_s = perf_counter() - timebins_start
 
         # now maybe re-init plugin shmem if uptate or new initialization
         if (self.cfg_restart_shmem_on_config and reconfig) or not reconfig:
           # `global_shmem` here is WRONG: we should NOT allow  plugins to access global shmem!
           #  so instead we use `plugins_shmem` initiated by the BizMgr
           self.__set_loop_stage('_update_instance_config.init_plugins_shared_memory')
+          if debug_load_timings:
+            shmem_start = perf_counter()
           self.init_plugins_shared_memory(self.plugins_shmem)
+          if debug_load_timings:
+            shmem_s = perf_counter() - shmem_start
 
       self.last_config_timestamp = self.log.now_str(nice_print=True, short=False)
       if self.is_plugin_stopped:
@@ -2000,6 +2038,24 @@ class BasePluginExecutor(
           msg='Plugin configured while being stopped',
           notif_code=ct.NOTIFICATION_CODES.PLUGIN_CONFIG_IN_PAUSE_OK,
         )
+      if debug_load_timings:
+        total_s = perf_counter() - cfg_start
+        other_s = total_s - (env_merge_s + default_merge_s + validate_s + warnings_s + alerters_s + timebins_s + shmem_s)
+        self.P(
+          "CONFIG_TIMING {} total={:.3f}s other={:.3f}s env_merge={:.3f}s default_merge={:.3f}s validate={:.3f}s warnings={:.3f}s alerters={:.3f}s timebins={:.3f}s shmem={:.3f}s".format(
+            self,
+            total_s,
+            other_s,
+            env_merge_s,
+            default_merge_s,
+            validate_s,
+            warnings_s,
+            alerters_s,
+            timebins_s,
+            shmem_s,
+          ),
+          color='b',
+        )
       self.__set_loop_stage('_update_instance_config.EXIT_update_instance_config')
       return
 

naeural_core/business/base/base_plugin_biz_loop.py

@@ -1,7 +1,7 @@
 import traceback
 import numpy as np
 
-from time import time, sleep
+from time import time, sleep, perf_counter
 from collections import deque
 
 from naeural_core import constants as ct
@@ -387,6 +387,9 @@ class _BasePluginLoopMixin(object):
   
   def __on_init(self):
     self.P("Running build-in & custom on_init events...")
+    debug_load_timings = self.log.config_data.get('PLUGINS_DEBUG_LOAD_TIMINGS', False)
+    if debug_load_timings:
+      init_start = perf_counter()
     if self.cfg_disabled:
       self.P(f"WARNING: This plugin instance of `{self.__class__.__name__}` is DISABLED", boxed=True, color='r')
       return
@@ -400,7 +403,13 @@ class _BasePluginLoopMixin(object):
         msg=f"Custom on_init event failed: {e}",
         displayed=True,
       )
+      if debug_load_timings:
+        init_s = perf_counter() - init_start
+        self.P("PLUGIN_INIT_TIMING {} status=error on_init={:.3f}s".format(self, init_s), color='r')
       raise e
+    if debug_load_timings:
+      init_s = perf_counter() - init_start
+      self.P("PLUGIN_INIT_TIMING {} status=ok on_init={:.3f}s".format(self, init_s), color='b')
     self._init_process_finalized = True
     return
   
@@ -537,4 +546,3 @@ class _BasePluginLoopMixin(object):
     #### del self.__upstream_inputs_deque ### this should NOT be called here - please do not ADD/DECOMMENT
     self._on_close()
     return
-

naeural_core/business/business_manager.py

@@ -1,6 +1,7 @@
 import json
 import gc
 import os
+import sys
 
 import traceback
 
@@ -67,6 +68,15 @@ class BusinessManager(Manager):
   def startup(self):
     super().startup()
     self._dct_current_instances = self._dct_subalterns # this allows usage of `self.get_subaltern(instance_hash)`
+    if self.config_data.get('PLUGINS_DEBUG_LOAD_TIMINGS', False):
+      self.P(
+        "Plugin timing env: python={} dont_write_bytecode={} env_PYTHONDONTWRITEBYTECODE={}".format(
+          sys.version.split()[0],
+          getattr(sys, 'dont_write_bytecode', None),
+          os.environ.get('PYTHONDONTWRITEBYTECODE'),
+        ),
+        color='b',
+      )
     return
 
   @property
@@ -246,14 +256,17 @@ class BusinessManager(Manager):
     self.set_loop_stage('2.bm.refresh._check_instances.get_current_jobs')
     all_jobs = self.get_current_jobs()
     n_all_jobs = len(all_jobs)
-    self.P("Checking {} business plugin instances...".format(n_all_jobs))
-    total_start = perf_counter()
+    debug_load_timings = self.config_data.get('PLUGINS_DEBUG_LOAD_TIMINGS', False)
+    if debug_load_timings:
+      self.P("Checking {} business plugin instances...".format(n_all_jobs))
+      total_start = perf_counter()
     for idx_job, (initiator_addr, initiator_id, modified_by_addr, modified_by_id, session_id, stream_name, signature, instance_id, upstream_config) in enumerate(all_jobs):
-      iter_start = perf_counter()
-      get_class_s = 0.0
-      instantiate_s = 0.0
-      start_thread_s = 0.0
-      update_config_s = 0.0
+      if debug_load_timings:
+        iter_start = perf_counter()
+        get_class_s = 0.0
+        instantiate_s = 0.0
+        start_thread_s = 0.0
+        update_config_s = 0.0
       is_new_instance = False
       try:
         obj_identification = (stream_name, signature, instance_id)
@@ -279,7 +292,8 @@ class BusinessManager(Manager):
           self.set_loop_stage('2.bm.refresh.get_class.{}:{}'.format(signature,instance_id))
           if 'update_monitor' in signature.lower():
             print('debug')
-          get_class_start = perf_counter()
+          if debug_load_timings:
+            get_class_start = perf_counter()
           _module_name, _class_name, _cls_def, _config_dict = self._get_module_name_and_class(
             locations=ct.PLUGIN_SEARCH.LOC_BIZ_PLUGINS,
             name=signature,
@@ -289,7 +303,8 @@ class BusinessManager(Manager):
             safe_locations=ct.PLUGIN_SEARCH.SAFE_BIZ_PLUGINS,
             safe_imports=ct.PLUGIN_SEARCH.SAFE_BIZ_IMPORTS
           )
-          get_class_s = perf_counter() - get_class_start
+          if debug_load_timings:
+            get_class_s = perf_counter() - get_class_start
 
           self.set_loop_stage('2.bm.refresh.check_class.{}:{}'.format(signature,instance_id))
           
@@ -320,7 +335,8 @@ class BusinessManager(Manager):
             
             _module_version = _config_dict.get('MODULE_VERSION', '0.0.0')
             
-            instantiate_start = perf_counter()
+            if debug_load_timings:
+              instantiate_start = perf_counter()
             plugin = _cls_def(
               log=self.log,
               global_shmem=self.shmem, # this SHOULD NOT be used for inter-plugin mem access
@@ -344,7 +360,8 @@ class BusinessManager(Manager):
               pipelines_view_function=self.owner.get_pipelines_view,
               pipeline_use_local_comms_only=self._dct_config_streams[stream_name].get(ct.CONFIG_STREAM.K_USE_LOCAL_COMMS_ONLY, False),
             )
-            instantiate_s = perf_counter() - instantiate_start
+            if debug_load_timings:
+              instantiate_s = perf_counter() - instantiate_start
             if plugin.cfg_runs_only_on_supervisor_node:
               if not self.is_supervisor_node:
                 self.P(
@@ -384,40 +401,45 @@ class BusinessManager(Manager):
 
           self.P("New plugin instance {} added for exec.".format(plugin), color='g')
           if self._run_on_threads:
-            start_thread_start = perf_counter()
+            if debug_load_timings:
+              start_thread_start = perf_counter()
             plugin.start_thread()
-            start_thread_s = perf_counter() - start_thread_start
-        #endif new instance
+            if debug_load_timings:
+              start_thread_s = perf_counter() - start_thread_start
+          #endif new instance
         else:
           # I do have the instance, I just need to modify the config
           plugin = self._dct_current_instances[instance_hash]
           if plugin is not None:
             # next we need to check if the config has changed and handle also the particular
             # case when the plugin just received a INSTANCE_COMMAND
-            update_config_start = perf_counter()
+            if debug_load_timings:
+              update_config_start = perf_counter()
             plugin.maybe_update_instance_config(
               upstream_config=upstream_config,
               session_id=session_id,
               modified_by_addr=modified_by_addr,
               modified_by_id=modified_by_id,
             )
-            update_config_s = perf_counter() - update_config_start
+            if debug_load_timings:
+              update_config_s = perf_counter() - update_config_start
             self.set_loop_stage('2.bm.refresh.maybe_update_instance_config.DONE: {}:{}:{}'.format(stream_name, signature, instance_id))
           #endif
         #endif
       finally:
-        # Done to avoid spam logs - only log new instances
-        if is_new_instance:
+        if is_new_instance and debug_load_timings:
           iter_total_s = perf_counter() - iter_start
           total_elapsed_s = perf_counter() - total_start
+          other_s = iter_total_s - (get_class_s + instantiate_s + start_thread_s + update_config_s)
           self.P(
-            " START Plugin {}/{} {}:{} new={} total={:.2f}s get_class={:.2f}s init={:.2f}s start_thread={:.2f}s update_cfg={:.2f}s (ALL={:.2f}s)".format(
+            " START Plugin {}/{} {}:{} new={} total={:.2f}s other={:.2f}s get_class={:.2f}s init={:.2f}s start_thread={:.2f}s update_cfg={:.2f}s (ALL={:.2f}s)".format(
               idx_job + 1,
               n_all_jobs,
               signature,
               instance_id,
               is_new_instance,
               iter_total_s,
+              other_s,
               get_class_s,
               instantiate_s,
               start_thread_s,
@@ -426,7 +448,7 @@ class BusinessManager(Manager):
             ),
             boxed=True
           )
-        # endif is new instance
+        # endif debug_load_timings
       # end try-finally
 
     return current_instances

naeural_core/main/orchestrator.py

@@ -160,8 +160,14 @@ class Orchestrator(DecentrAIObject,
 
 
   def startup(self):
-    super().startup()
-    self.P("Starting Execution Engine '{}' core v.{}".format(self.cfg_eeid, self.__version__))
+    super().startup()    
+    self.P(
+      "Starting Execution Engine '{}' v.{}, core {}, (WORK_OFFLINE={})".format(
+        self.cfg_eeid, self.__version__, self.core_version,
+        self.cfg_work_offline,
+      ),
+      boxed=True
+    )
     if self.runs_in_docker:
       self.P("Running in Docker container detected.")
     self._maybe_env_and_docker_setup()

naeural_core/main/ver.py

@@ -1,4 +1,4 @@
-__VER__ = '7.7.243'
+__VER__ = '7.7.245'
 
 
 

naeural_core/manager.py

@@ -1,5 +1,6 @@
 from naeural_core import constants as ct
 
+from time import perf_counter
 from naeural_core import Logger
 from ratio1 import _PluginsManagerMixin
 from naeural_core import DecentrAIObject
@@ -75,13 +76,21 @@ class Manager(DecentrAIObject, _PluginsManagerMixin):
   def _get_module_name_and_class(self, locations, name, suffix=None, verbose=1, safety_check=True, safe_locations=None, safe_imports=None):
     if safe_locations is None:
       self.P("  Warning: no safe location provided", color='r')
+    debug_load_timings = self.config_data.get('PLUGINS_DEBUG_LOAD_TIMINGS', False)
+    cache_key = name.lower()
+    cache_hit = cache_key in self.plugin_locations_cache
+    if debug_load_timings:
+      load_start = perf_counter()
+      lookup_s = 0.0
     self.P("Attempting to load {} plugin '{}'".format(
       self.__class__.__name__, name,
     ))
-    if name.lower() in self.plugin_locations_cache:
-      _module_name, _class_name, _cls_def, _config_dict = self.plugin_locations_cache[name.lower()]
+    if cache_hit:
+      _module_name, _class_name, _cls_def, _config_dict = self.plugin_locations_cache[cache_key]
       self.P(f'Attempting to load plugin {name} from cache.')
     else:
+      if debug_load_timings:
+        lookup_start = perf_counter()
       _module_name, _class_name, _cls_def, _config_dict = super()._get_module_name_and_class(
         locations=locations,
         name=name,
@@ -92,9 +101,25 @@ class Manager(DecentrAIObject, _PluginsManagerMixin):
         search_in_packages=ct.PLUGIN_SEARCH.SEARCH_IN_PACKAGES,
         safe_imports=safe_imports,
       )
-      self.plugin_locations_cache[name.lower()] = _module_name, _class_name, _cls_def, _config_dict
+      if debug_load_timings:
+        lookup_s = perf_counter() - lookup_start
+      self.plugin_locations_cache[cache_key] = _module_name, _class_name, _cls_def, _config_dict
     # endif name in cache
 
+    if debug_load_timings:
+      total_s = perf_counter() - load_start
+      self.P(
+        "Plugin load timing name='{}' cache_hit={} lookup={:.3f}s total={:.3f}s module={} class={}".format(
+          name,
+          cache_hit,
+          lookup_s,
+          total_s,
+          _module_name,
+          _class_name,
+        ),
+        color='b',
+      )
+
     if _cls_def is None and verbose >= 1:
       self._create_notification(
         notif=ct.STATUS_TYPE.STATUS_EXCEPTION,

{naeural_core-7.7.243.dist-info → naeural_core-7.7.245.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: naeural_core
-Version: 7.7.243
+Version: 7.7.245
 Summary: Ratio1 Core is the backbone of the Ratio1 Edge Protocol.
 Project-URL: Homepage, https://github.com/Ratio1/naeural_core
 Project-URL: Bug Tracker, https://github.com/Ratio1/naeural_core/issues

{naeural_core-7.7.243.dist-info → naeural_core-7.7.245.dist-info}/RECORD

@@ -2,15 +2,15 @@ naeural_core/__init__.py,sha256=DqUL1kr0r9wXGYpH4OQAdHgSe00nfdukr2fekqkgiiY,88
 naeural_core/constants.py,sha256=jOOkIy93obAkD_x73rzBbQgcYNSK6JGyJ7Pk2wZEG9g,38027
 naeural_core/data_structures.py,sha256=Qp7VL8N8nKOkzUfRPbJDWbcWyo1Rjc6QarRK7pol-g0,13848
 naeural_core/decentrai_object.py,sha256=pw-NRmlQRAzvUVFfh1xTJdj1Ltd2Y-UVrpompriUQzA,4408
-naeural_core/manager.py,sha256=fMyVjM2UXQ0LxHE33z1t9AQxmxn68uUsu9EseK5zqDo,3406
+naeural_core/manager.py,sha256=BJrcIBIwykW_46xhbFHJXM9tbnLiF0LDJJc3Dxwi9Wg,4147
 naeural_core/bc/__init__.py,sha256=Su7YuJjTPI1P3a6L8WGRqW2YvDADyLMYSipcZognAEM,176
 naeural_core/business/__init__.py,sha256=9boWpUoISx1OkCftejQXaw-sqv6M88LLt7CLNZjdnfU,46
-naeural_core/business/business_manager.py,sha256=ZPnw19TH9ycsVeVL_mtWs7rC9OSQn7nMCXZMAisgqTo,28815
+naeural_core/business/business_manager.py,sha256=gbupztHqlm_u-8LW-gLfLcYuSJGEjTG3ppMvcLuHWjk,29736
 naeural_core/business/utils.py,sha256=ykl7d1snVldTPR7LyDjSSPL4LOGag0pgkRfOnGZXJKg,15765
 naeural_core/business/base/__init__.py,sha256=OEhyheZKcCk2GafJuO6bDVi1ph1pZyW0A8wan_SuesA,270
-naeural_core/business/base/base_plugin_biz.py,sha256=KlMD1c50meTWYHoS-GaXeICWhvNhsXiXgaYpMfQCFn4,74532
+naeural_core/business/base/base_plugin_biz.py,sha256=FPI5sDVnj3PkWu9jmzfSrCAvs3trAy9BW4HBxhrPW9Q,76653
 naeural_core/business/base/base_plugin_biz_api.py,sha256=r1xwa1UrPvtD7ArFHlH54zwwLSxZVXOo4Jmc8eTyxU8,16956
-naeural_core/business/base/base_plugin_biz_loop.py,sha256=WkQqylLCY18yfQHlu1-8EgXtyia2Z0P-ggx9jCpSZUg,18462
+naeural_core/business/base/base_plugin_biz_loop.py,sha256=IIVPKXcxW5odZNokI3whmJi01MW5o4ZM2xu8la_LUTI,18964
 naeural_core/business/base/cv_plugin_executor.py,sha256=1k8m26-1POBOts13AMMJfxZb0GzaAoAO6qXbCYL1Ymw,36661
 naeural_core/business/base/network_processor.py,sha256=9HXRufgHpwM6PChHmQgE_pmfKpByk2hc-rBDHtW6D_8,953
 naeural_core/business/base/sd_plugin_executor.py,sha256=sdAQZBmFkgvvKCMirDdwgeklBq3j-rZcgkvBEojcHw4,2952
@@ -344,8 +344,8 @@ naeural_core/main/epochs_manager.py,sha256=lH01Pv9E_uz5fdvh_W2dZ29hZLM0CL2NZfuYJ
 naeural_core/main/geoloc.py,sha256=TEqyuNzpVqZSBCo0OOrpHYncIsHSClvRt28hgvxJ35o,24909
 naeural_core/main/main_loop_data_handler.py,sha256=hABB65OUBhtur3rd2mYsEhdAc54jVILzybrvxml5h0s,13815
 naeural_core/main/net_mon.py,sha256=qlyo1fqTeQy_M9VfJOxon_PBbQat0QO9Zbu_93FMbLc,88144
-naeural_core/main/orchestrator.py,sha256=0FhxLgpJN9Wjw1L7Ojx4Mi0P82u8dNDRTfUdqGh0Tqs,70626
-naeural_core/main/ver.py,sha256=Ej_BKrKSBFiPw5XvW16axNWnd5gnXYHCBK7jdTY70Sg,335
+naeural_core/main/orchestrator.py,sha256=EodPSzrJd-Yg_RnBIA6-aIsMBa94I5uV1ATZUR5j3Kg,70750
+naeural_core/main/ver.py,sha256=ULuSaZSXo9u6tjfLVaOUf9FIoBZf8ONzlyZwK0NdiOs,335
 naeural_core/main/orchestrator_mixins/__init__.py,sha256=MNleg48vdlqsyAR8Vamjl4ahG2jwCH5kLbQN5CfU57E,149
 naeural_core/main/orchestrator_mixins/managers_init.py,sha256=sQVqpr99a5WP9HCloYCyaWDW5J3IypEImlf703bqTF4,6692
 naeural_core/main/orchestrator_mixins/utils.py,sha256=jMa0uStVNLQmp0VhNMRvfBDjo387ORLlUVLthRNBKqc,1866
@@ -555,7 +555,7 @@ naeural_core/utils/tracing/onnx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 naeural_core/utils/tracing/onnx/base_trt_scripter.py,sha256=1FelEBo7JGsc8hbJ3sevzxnM-J61nvBHz6L1VLpZrVc,2043
 naeural_core/utils/tracing/onnx/utils.py,sha256=IKmqUWakrMWn34uJvbRjNLacdszD8jkkQBFPUhgJtOQ,5618
 naeural_core/utils/web_app/favicon.ico,sha256=zU6-Jxx4ol1A9FJvcQELYV9DiqwqyvjPS89xQybZE74,15406
-naeural_core-7.7.243.dist-info/METADATA,sha256=QOw-MVuhPkWufArzSVPVC9SlMIstJ7CkD72AAPG0xl0,6522
-naeural_core-7.7.243.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-naeural_core-7.7.243.dist-info/licenses/LICENSE,sha256=SPHPWjOdAUUUUI020nI5VNCtFjmTOlJpi1cZxyB3gKo,11339
-naeural_core-7.7.243.dist-info/RECORD,,
+naeural_core-7.7.245.dist-info/METADATA,sha256=YBBsJaDZLe6Pwtp_OZa5CmVJOxHELr4eAhMVF-0u8w4,6522
+naeural_core-7.7.245.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+naeural_core-7.7.245.dist-info/licenses/LICENSE,sha256=SPHPWjOdAUUUUI020nI5VNCtFjmTOlJpi1cZxyB3gKo,11339
+naeural_core-7.7.245.dist-info/RECORD,,