naeural-core 7.7.239__py3-none-any.whl → 7.7.242__py3-none-any.whl

naeural_core/comm/communication_manager.py

@@ -74,13 +74,18 @@ class CommunicationManager(Manager, _ConfigHandlerMixin):
     _id = self.log.config_data.get(ct.CONFIG_STARTUP_v2.K_EE_ID, '')[:ct.EE_ALIAS_MAX_SIZE]
     return _id  
 
+  def _has_failed_comms(self):
+    for comm in self._dct_comm_plugins.values():
+      if comm.comm_failed_after_retries:
+        return comm
+    return None
 
   @property
   def has_failed_comms(self):
-    for comm in self._dct_comm_plugins.values():
-      if comm.comm_failed_after_retries:
-        self.P("Detected total communication failure on comm {}. This may generate shutdown/restart.".format(comm.__class__.__name__), color='error')
-        return True
+    comm = self._has_failed_comms()
+    if comm is not None:
+      self.P("Detected total communication failure on comm {}. This may generate shutdown/restart.".format(comm.__class__.__name__), color='error')
+      return True
     return False  
 
 

naeural_core/comm/mixins/default_comm_mixin.py

@@ -80,4 +80,3 @@ class _DefaultCommMixin(object):
     self.P('`run_thread` finished')
     self._thread_stopped = True
     return
-

naeural_core/core_logging/logger_mixins/gpu_mixin.py

@@ -136,6 +136,28 @@ class _GPUMixin(object):
 
     return processes_by_uuid
 
+  def _get_uuid_by_index(self, timeout=1.5):
+    import subprocess, shutil
+    smi = shutil.which("nvidia-smi")
+    if not smi:
+      return {}
+
+    out = subprocess.run(
+      [smi, "--query-gpu=index,uuid", "--format=csv,noheader,nounits"],
+      capture_output=True, text=True, timeout=timeout
+    )
+    if out.returncode != 0:
+      return {}
+
+    d = {}
+    for line in out.stdout.splitlines():
+      line = line.strip()
+      if not line:
+        continue
+      idx_s, uuid = [p.strip() for p in line.split(",", 1)]
+      d[int(idx_s)] = uuid
+    return d
+
   def gpu_info(self, show=False, mb=False, current_pid=False):
     """
     Collects GPU info. Must have torch installed & non-mandatory nvidia-smi
@@ -215,20 +237,12 @@ class _GPUMixin(object):
           fan_speed, fan_speed_unit = -1, "N/A"
           if pynvml_avail:
             # --- get an NVML handle that matches torch's CUDA device ordering when possible ---
-            handle = None
-            try:
-              # This helps when CUDA_VISIBLE_DEVICES remaps indices:
-              # torch device 0 may not be NVML index 0.
-              pci_bus_id = getattr(device_props, "pci_bus_id", None)
-              if pci_bus_id:
-                if hasattr(pynvml, "nvmlDeviceGetHandleByPciBusId_v2"):
-                  handle = pynvml.nvmlDeviceGetHandleByPciBusId_v2(pci_bus_id)
-                elif hasattr(pynvml, "nvmlDeviceGetHandleByPciBusId"):
-                  handle = pynvml.nvmlDeviceGetHandleByPciBusId(pci_bus_id)
-            except Exception:
-              handle = None
-
-            if handle is None:
+            uuid_by_index = self._get_uuid_by_index()
+            # inside your for device_id in range(n_gpus):
+            uuid = uuid_by_index.get(device_id)
+            if uuid:
+              handle = pynvml.nvmlDeviceGetHandleByUUID(uuid)
+            else:
               handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
 
             # --- memory (NVML returns bytes) ---

naeural_core/main/orchestrator.py

@@ -121,6 +121,7 @@ class Orchestrator(DecentrAIObject,
     self._current_dct_config_streams = {}
     self._should_send_initial_log = False
     self._initial_log_sent = False
+    self._last_offline_log = 0
     self.loop_timings = deque(maxlen=3600)
     self._reset_timers = False
     self.__is_mlstop_dangerous = False
@@ -709,6 +710,10 @@ class Orchestrator(DecentrAIObject,
   def cfg_main_loop_resolution(self):
     return self.config_data.get('MAIN_LOOP_RESOLUTION', 20)
 
+  @property
+  def cfg_work_offline(self):
+    return self.config_data.get('WORK_OFFLINE', False)
+
   @property
   def cfg_sequential_streams(self):
     """
@@ -1613,6 +1618,27 @@ class Orchestrator(DecentrAIObject,
     self._comm_manager.maybe_show_info()    
     return
 
+  def _maybe_log_offline_status(self):
+    if not self.cfg_work_offline or self._comm_manager is None or not self._comm_manager._has_failed_comms():
+      return
+    now = time()
+    if (now - self._last_offline_log) < ct.COMMS.COMM_SECS_SHOW_INFO:
+      return
+    self._last_offline_log = now
+
+    comm_attempts = [
+      "{}:try={} fails={}".format(
+        name,
+        getattr(comm, "_nr_conn_retry_iters", None),
+        getattr(comm, "_total_conn_fails", None),
+      )
+      for name, comm in self._comm_manager._dct_comm_plugins.items()
+      if comm is not None
+    ]
+    attempts_str = "; ".join(comm_attempts) if len(comm_attempts) > 0 else "no comm plugins"
+    self.P(f"WORK_OFFLINE enabled; reconnect attempts: {attempts_str}", color='r')
+    return
+
   def _save_exception_main_loop_state(self, txt, **save_kwargs):
     fn = '{}_main_loop_exception'.format(self.log.now_str())
     self.log.save_pickle_to_output(data=save_kwargs, fn=fn + '.pickle', subfolder_path='main_loop_exceptions')
@@ -1829,6 +1855,7 @@ class Orchestrator(DecentrAIObject,
         #9. Comm info, timers, ... - later we gonna check for total comm failures
         self.__loop_stage = '9.logs'
         self.comm_manager_show_info()
+        self._maybe_log_offline_status()
 
 
         self.log.stop_timer(self._main_loop_timer_name)
@@ -1844,7 +1871,7 @@ class Orchestrator(DecentrAIObject,
         return_code = self._return_code
 
         self.__loop_stage = '10.checks'
-        if self.comm_manager.has_failed_comms:
+        if (not self.cfg_work_offline) and self.comm_manager.has_failed_comms:
           self.P("Shutdown initiated due to multiple failure in communication!", color='r')
           return_code = ct.CODE_EXCEPTION
 

naeural_core/main/ver.py

@@ -1,4 +1,4 @@
-__VER__ = '7.7.239'
+__VER__ = '7.7.242'
 
 
 

{naeural_core-7.7.239.dist-info → naeural_core-7.7.242.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: naeural_core
-Version: 7.7.239
+Version: 7.7.242
 Summary: Ratio1 Core is the backbone of the Ratio1 Edge Protocol.
 Project-URL: Homepage, https://github.com/Ratio1/naeural_core
 Project-URL: Bug Tracker, https://github.com/Ratio1/naeural_core/issues

{naeural_core-7.7.239.dist-info → naeural_core-7.7.242.dist-info}/RECORD

@@ -102,7 +102,7 @@ naeural_core/business/training/minio_download_dataset.py,sha256=7uoo5CFLynWbLkBh
 naeural_core/business/training/minio_upload_dataset.py,sha256=5stm_E_L3SLwcjd2znUVMIC2PWO07f6QfBdcrKRiHCo,2162
 naeural_core/business/training/second_stage_training_process.py,sha256=z8LG9xx2G6s5AqeSD-t5rBegIctkFMEf4ZqEJVXZcz0,3152
 naeural_core/comm/__init__.py,sha256=SpAWJIyYdOouZSImzVrEF_M4-nrCrY9p3cVYwvmbt20,105
-naeural_core/comm/communication_manager.py,sha256=n70Khv464kQbseo6ch33cTel_JZkC1eItGyKah27sHM,30347
+naeural_core/comm/communication_manager.py,sha256=swMxO3DPTnT0SeSsv6PWz0gP0WsWOzA_BlYawoWHSB4,30471
 naeural_core/comm/base/__init__.py,sha256=rDzAtPwcMOsW3aCp0t07GpJz5qweLiJgafTngHwEMOo,44
 naeural_core/comm/base/base_comm_thread.py,sha256=DVPzLN9UfKxjiVf0FtjOr5001-YzkjPMmC4RJFhoGG0,27034
 naeural_core/comm/default/amqp.py,sha256=-6_qGlOIjOUz42nkwnfMQZEVfdeUKGTNgjcTuo49v4E,4145
@@ -110,7 +110,7 @@ naeural_core/comm/default/mqtt.py,sha256=F2VOOmjXkz7diFC-PVdxZgBI7O9aCwul5KfA6r4
 naeural_core/comm/default/readme.md,sha256=hNY9V5HU8yW0JjyseiPWMkV8l7YU0ZEBw_iq_lpW-Uk,162
 naeural_core/comm/mixins/__init__.py,sha256=d8o2tKAkQ-P9voRB6REnEmObVyi4AiQgNVZuAKKObKo,290
 naeural_core/comm/mixins/commandcontrol_comm_mixin.py,sha256=VhAGzR23-x8INn8VGBd7z2YUtT4GIkeTNwNK0CmAjio,4781
-naeural_core/comm/mixins/default_comm_mixin.py,sha256=aAqFPCkFcDIY6CvtmW1OkfNJtsJSobCD412euG3_j_M,2950
+naeural_core/comm/mixins/default_comm_mixin.py,sha256=CMnaYjBk5jazIygEyGXb4j4DSCPij7LgmDoCsGz6W3E,2949
 naeural_core/comm/mixins/heartbeats_comm_mixin.py,sha256=_PqCqn4R3KmzE-2aRnMRHSMelc0v52QRms2HoZgTvFE,3307
 naeural_core/comm/mixins/notifications_comm_mixin.py,sha256=Otcb7vUkNOu2xAiEMm0ukndBqocsr2dYdvyRv-5YLWA,1929
 naeural_core/comm/mixins/telemetry_mixin.py,sha256=mIox-rJe1XTy3_vgi3qT6gb0SiupUksyn7CPWACZCsY,4025
@@ -137,7 +137,7 @@ naeural_core/core_logging/logger_mixins/confusion_matrix_mixin.py,sha256=fLJOeyp
 naeural_core/core_logging/logger_mixins/dataframe_mixin.py,sha256=hkOtoTzoBDacpagdFYp2kawsw7rzbgLw2-_pzXLBU6Q,11491
 naeural_core/core_logging/logger_mixins/deploy_models_in_production_mixin.py,sha256=J2j1tnt0Cd2qD31rL8Sov0sz9_T5-h6Ukd-4sl1ITcQ,5986
 naeural_core/core_logging/logger_mixins/fit_debug_tfkeras_mixin.py,sha256=6efE5W59a3VWWR1UbPd9iNfQl0nuse7SW3IV0RFpHOc,9344
-naeural_core/core_logging/logger_mixins/gpu_mixin.py,sha256=sgtEfuROphSL03Id8n4Wd8SbxGoSuBMVx_v-N_cKvXc,14023
+naeural_core/core_logging/logger_mixins/gpu_mixin.py,sha256=1gh83RB8ziZJG-UdF1BnOPJWWFAq4xbbhGuQvUPUn1E,14193
 naeural_core/core_logging/logger_mixins/grid_search_mixin.py,sha256=lo3bTyEmcsk03ttTLs0lC_N0beAC1eTiGI9kZX2ib-A,5961
 naeural_core/core_logging/logger_mixins/histogram_mixin.py,sha256=ro5q99VXdcLSBi6XMZK4fJIXNX8n4wBzay-0SXuj7qc,6413
 naeural_core/core_logging/logger_mixins/keras_callbacks_mixin.py,sha256=ELlTb8TycdFnuO2dMDAherlzRd1rfHWogIDq-svnZ7w,3940
@@ -344,8 +344,8 @@ naeural_core/main/epochs_manager.py,sha256=lH01Pv9E_uz5fdvh_W2dZ29hZLM0CL2NZfuYJ
 naeural_core/main/geoloc.py,sha256=TEqyuNzpVqZSBCo0OOrpHYncIsHSClvRt28hgvxJ35o,24909
 naeural_core/main/main_loop_data_handler.py,sha256=hABB65OUBhtur3rd2mYsEhdAc54jVILzybrvxml5h0s,13815
 naeural_core/main/net_mon.py,sha256=qlyo1fqTeQy_M9VfJOxon_PBbQat0QO9Zbu_93FMbLc,88144
-naeural_core/main/orchestrator.py,sha256=SKnW5jWksBm2-fZcfeT5dddDmFSTSyA-CcODKX664KI,69657
-naeural_core/main/ver.py,sha256=NCXuXm5fEgpFjFfI3Rlr_co4S3oap5g9XZOHDtGhUPM,335
+naeural_core/main/orchestrator.py,sha256=0FhxLgpJN9Wjw1L7Ojx4Mi0P82u8dNDRTfUdqGh0Tqs,70626
+naeural_core/main/ver.py,sha256=Ga9K1mXR6U7fDBbh2_lozzdOHGPe7E40abF2oYh_SWI,335
 naeural_core/main/orchestrator_mixins/__init__.py,sha256=MNleg48vdlqsyAR8Vamjl4ahG2jwCH5kLbQN5CfU57E,149
 naeural_core/main/orchestrator_mixins/managers_init.py,sha256=sQVqpr99a5WP9HCloYCyaWDW5J3IypEImlf703bqTF4,6692
 naeural_core/main/orchestrator_mixins/utils.py,sha256=jMa0uStVNLQmp0VhNMRvfBDjo387ORLlUVLthRNBKqc,1866
@@ -555,7 +555,7 @@ naeural_core/utils/tracing/onnx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 naeural_core/utils/tracing/onnx/base_trt_scripter.py,sha256=1FelEBo7JGsc8hbJ3sevzxnM-J61nvBHz6L1VLpZrVc,2043
 naeural_core/utils/tracing/onnx/utils.py,sha256=IKmqUWakrMWn34uJvbRjNLacdszD8jkkQBFPUhgJtOQ,5618
 naeural_core/utils/web_app/favicon.ico,sha256=zU6-Jxx4ol1A9FJvcQELYV9DiqwqyvjPS89xQybZE74,15406
-naeural_core-7.7.239.dist-info/METADATA,sha256=jWc9x_Znx1X-FNdp69ujFdvVHKIH5uiEqsS55f673E0,6522
-naeural_core-7.7.239.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-naeural_core-7.7.239.dist-info/licenses/LICENSE,sha256=SPHPWjOdAUUUUI020nI5VNCtFjmTOlJpi1cZxyB3gKo,11339
-naeural_core-7.7.239.dist-info/RECORD,,
+naeural_core-7.7.242.dist-info/METADATA,sha256=f7kZ1SrW7zJuC9edJQDUX9cBCV_bOTTxYVcpC4UnBkU,6522
+naeural_core-7.7.242.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+naeural_core-7.7.242.dist-info/licenses/LICENSE,sha256=SPHPWjOdAUUUUI020nI5VNCtFjmTOlJpi1cZxyB3gKo,11339
+naeural_core-7.7.242.dist-info/RECORD,,