learning-loop-node 0.13.4__py3-none-any.whl → 0.13.6__py3-none-any.whl

learning_loop_node/detector/detector_logic.py

@@ -6,41 +6,38 @@ import numpy as np
 
 from ..data_classes import ImageMetadata, ModelInformation
 from ..globals import GLOBALS
+from .exceptions import NodeNeedsRestartError
 
 
 class DetectorLogic():
 
     def __init__(self, model_format: str) -> None:
         self.model_format: str = model_format
-        self._model_info: Optional[ModelInformation] = None
+        self.model_info: Optional[ModelInformation] = None
 
-    async def soft_reload(self):
-        self._model_info = None
-
-    @property
-    def model_info(self) -> ModelInformation:
-        if self._model_info is None:
-            raise Exception('Model not loaded')
-        return self._model_info
+        self._remaining_init_attempts: int = 2
 
-    @property
-    def is_initialized(self) -> bool:
-        return self._model_info is not None
+    async def soft_reload(self):
+        self.model_info = None
 
-    def load_model(self):
+    def load_model_info_and_init_model(self):
         logging.info('Loading model from %s', GLOBALS.data_folder)
-        model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
-        if model_info is None:
+        self.model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
+        if self.model_info is None:
             logging.error('No model found')
-            self._model_info = None
-            raise Exception('No model found')
+            self.model_info = None
+            return
+
         try:
-            self._model_info = model_info
             self.init()
-            logging.info('Successfully loaded model %s', self._model_info)
+            logging.info('Successfully loaded model %s', self.model_info)
+            self._remaining_init_attempts = 2
         except Exception:
-            self._model_info = None
-            logging.error('Could not init model %s', model_info)
+            self._remaining_init_attempts -= 1
+            self.model_info = None
+            logging.error('Could not init model %s. Retries left: %s', self.model_info, self._remaining_init_attempts)
+            if self._remaining_init_attempts == 0:
+                raise NodeNeedsRestartError('Could not init model') from None
             raise
 
     @abstractmethod

learning_loop_node/detector/detector_node.py

@@ -1,8 +1,10 @@
 import asyncio
 import contextlib
+import math
 import os
 import shutil
 import subprocess
+import sys
 from dataclasses import asdict
 from datetime import datetime
 from threading import Thread
@@ -14,8 +16,16 @@ from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
 from socketio import AsyncClient
 
-from ..data_classes import (AboutResponse, Category, Context, DetectionStatus, ImageMetadata, ModelInformation,
-                            ModelVersionResponse, Shape)
+from ..data_classes import (
+    AboutResponse,
+    Category,
+    Context,
+    DetectionStatus,
+    ImageMetadata,
+    ModelInformation,
+    ModelVersionResponse,
+    Shape,
+)
 from ..data_classes.socket_response import SocketResponse
 from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
@@ -23,6 +33,7 @@ from ..globals import GLOBALS
 from ..helpers import environment_reader
 from ..node import Node
 from .detector_logic import DetectorLogic
+from .exceptions import NodeNeedsRestartError
 from .inbox_filter.relevance_filter import RelevanceFilter
 from .outbox import Outbox
 from .rest import about as rest_about
@@ -80,13 +91,13 @@ class DetectorNode(Node):
         return AboutResponse(
             operation_mode=self.operation_mode.value,
             state=self.status.state,
-            model_info=self.detector_logic._model_info,  # pylint: disable=protected-access
+            model_info=self.detector_logic.model_info,  # pylint: disable=protected-access
             target_model=self.target_model.version if self.target_model else None,
             version_control=self.version_control.value
         )
 
     def get_model_version_response(self) -> ModelVersionResponse:
-        current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else 'None'  # pylint: disable=protected-access
+        current_version = self.detector_logic.model_info.version if self.detector_logic.model_info is not None else 'None'  # pylint: disable=protected-access
         target_version = self.target_model.version if self.target_model is not None else 'None'
         loop_version = self.loop_deployment_target.version if self.loop_deployment_target is not None else 'None'
 
@@ -170,13 +181,13 @@ class DetectorNode(Node):
 
         # simulate startup
         await self.detector_logic.soft_reload()
-        self.detector_logic.load_model()
+        self.detector_logic.load_model_info_and_init_model()
         self.operation_mode = OperationMode.Idle
 
     async def on_startup(self) -> None:
         try:
             self.outbox.ensure_continuous_upload()
-            self.detector_logic.load_model()
+            self.detector_logic.load_model_info_and_init_model()
         except Exception:
             self.log.exception("error during 'startup'")
         self.operation_mode = OperationMode.Idle
@@ -235,7 +246,7 @@ class DetectorNode(Node):
 
         @self.sio.event
         async def info(sid) -> Dict:
-            if self.detector_logic.is_initialized:
+            if self.detector_logic.model_info is not None:
                 return asdict(self.detector_logic.model_info)
             return {"status": "No model loaded"}
 
@@ -272,7 +283,7 @@ class DetectorNode(Node):
             '''upload an image with detections'''
 
             detection_data = data.get('detections', {})
-            if detection_data and self.detector_logic.is_initialized:
+            if detection_data and self.detector_logic.model_info is not None:
                 try:
                     image_metadata = from_dict(data_class=ImageMetadata, data=detection_data)
                 except Exception as e:
@@ -321,9 +332,12 @@ class DetectorNode(Node):
                 self.log.debug('not checking for updates; no target model selected')
                 return
 
-            current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else None  # pylint: disable=protected-access
+            if self.detector_logic.model_info is not None:
+                current_version = self.detector_logic.model_info.version
+            else:
+                current_version = None
 
-            if not self.detector_logic.is_initialized or self.target_model.version != current_version:
+            if current_version != self.target_model.version:
                 self.log.info('Current model "%s" needs to be updated to %s',
                               current_version or "-", self.target_model.version)
 
@@ -349,7 +363,10 @@ class DetectorNode(Node):
                     self.log.info('Updated symlink for model to %s', os.readlink(model_symlink))
 
                     try:
-                        self.detector_logic.load_model()
+                        self.detector_logic.load_model_info_and_init_model()
+                    except NodeNeedsRestartError:
+                        self.log.error('Node needs restart')
+                        sys.exit(0)
                     except Exception:
                         self.log.exception('Could not load model, will retry download on next check')
                         shutil.rmtree(target_model_folder, ignore_errors=True)
@@ -383,9 +400,9 @@ class DetectorNode(Node):
             self.log.info('Status sync failed: not connected')
             raise Exception('Status sync failed: not connected')
 
-        try:
+        if self.detector_logic.model_info is not None:
             current_model = self.detector_logic.model_info.version
-        except Exception:
+        else:
             current_model = None
 
         target_model_version = self.target_model.version if self.target_model else None

learning_loop_node/detector/exceptions.py

@@ -0,0 +1,7 @@
+
+
+class NodeNeedsRestartError(Exception):
+    '''
+    NodeNeedsRestartError is raised when the node needs to be restarted.
+    This is e.g. the case when the GPU is not available anymore.
+    '''

learning_loop_node/node.py

@@ -74,6 +74,7 @@ class Node(FastAPI):
         self.repeat_loop_lock = asyncio.Lock()
 
         self.previous_state: Optional[str] = None
+        self.repeat_loop_cycle_sec = 5
 
     def log_status_on_change(self, current_state_str: str, full_status: Any):
         if self.previous_state != current_state_str:
@@ -146,7 +147,7 @@ class Node(FastAPI):
             except Exception:
                 self.log.exception('error in repeat loop')
 
-            await asyncio.sleep(5)
+            await asyncio.sleep(self.repeat_loop_cycle_sec)
 
     async def _ensure_sio_connection(self):
         if self.socket_connection_broken or self._sio_client is None or not self.sio_client.connected:

learning_loop_node/tests/detector/conftest.py

@@ -129,10 +129,6 @@ class MockDetectorLogic(DetectorLogic):  # pylint: disable=abstract-method
                                          model_name="mock",
                                          )])
 
-    @property
-    def is_initialized(self):
-        return True
-
     def evaluate_with_all_info(self, image: np.ndarray, tags: List[str], source: Optional[str] = None, creation_date: Optional[str] = None):
         return self.image_metadata
 

learning_loop_node/tests/detector/test_client_communication.py

@@ -51,9 +51,6 @@ def test_rest_detect(test_detector_node: DetectorNode, grouping_key: str):
     headers = {grouping_key: '0:0:0:0', 'tags':  'some_tag'}
 
     assert isinstance(test_detector_node.detector_logic, TestingDetectorLogic)
-    # test_detector_node.detector_logic.mock_is_initialized = True
-    # print(test_detector_node.detector_logic.mock_is_initialized)
-    # print(test_detector_node.detector_logic.is_initialized)
     response = requests.post(f'http://localhost:{GLOBALS.detector_port}/detect',
                              files=image, headers=headers, timeout=30)
     assert response.status_code == 200

{learning_loop_node-0.13.4.dist-info → learning_loop_node-0.13.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.13.4
+Version: 0.13.6
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.13.4.dist-info → learning_loop_node-0.13.6.dist-info}/RECORD

@@ -11,8 +11,9 @@ learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM
 learning_loop_node/data_classes/training.py,sha256=FFPsr2AA7ynYz39MLZaFJ0sF_9Axll5HHbAA8nnirp0,5726
 learning_loop_node/data_exchanger.py,sha256=IG5ki3f3IsVuXbyw6q_gUIakgv-GMT6e9nhOhzjKgW4,9055
 learning_loop_node/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/detector/detector_logic.py,sha256=FhGbu0mdF0tW0Gg8cr8xM6ZmZzEigGf0IcAWBjoxFrs,2191
-learning_loop_node/detector/detector_node.py,sha256=2DpolliA36HIOvFqNeZaBOX2vwKL58i1rEJidPcFymM,25151
+learning_loop_node/detector/detector_logic.py,sha256=s1EFLrk_SFvLJOsIj9b0lp-Oq0DgfVWxT6Q34Vmi_JE,2243
+learning_loop_node/detector/detector_node.py,sha256=6MUdR_WzzcubwUAiaUo7TDF2xs-VkEvB_y1sNsrONYI,25461
+learning_loop_node/detector/exceptions.py,sha256=C6KbNPlSbtfgDrZx2Hbhm7Suk9jVoR3fMRCO0CkrMsQ,196
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=1PHgXRrhSQ34HSFw7mdX8ndRxHf_i1aP5nXXnrZxhAY,3312
 learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=NPEmrAtuGjIWCtHS0B3zDmnYWkhVFCLbd_7RUp08_AM,1372
@@ -38,7 +39,7 @@ learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa
 learning_loop_node/helpers/log_conf.py,sha256=hqVAa_9NnYEU6N0dcOKmph82p7MpgKqeF_eomTLYzWY,961
 learning_loop_node/helpers/misc.py,sha256=J29iBmsEUAraKKDN1m1NKiHQ3QrP5ub5HBU6cllSP2g,7384
 learning_loop_node/loop_communication.py,sha256=kc7GrkUS14Ka5OICaaOd_LZ61D-6O19GcyDEwckTxvM,7286
-learning_loop_node/node.py,sha256=rBdZiBzp7VqQkIt9GJ8b4nK2Y54lUJFzK-gc7rb14cA,10966
+learning_loop_node/node.py,sha256=IRV81q1G3-A6_BLNqB3NBT7T_dN5OXegBoM9JHMJuLM,11030
 learning_loop_node/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/rest.py,sha256=omwlRHLnyG-kgCBVnZDk5_SAPobL9g7slWeX21wsPGw,1551
 learning_loop_node/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -47,14 +48,14 @@ learning_loop_node/tests/annotator/conftest.py,sha256=G4ZvdZUdvPp9bYCzg3eEVkGCeX
 learning_loop_node/tests/annotator/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/annotator/test_annotator_node.py,sha256=UWRXRSBc1e795ftkp7xrEXbyR4LYvFDDHRpZGqC3vr8,1974
 learning_loop_node/tests/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/tests/detector/conftest.py,sha256=4zNW8dnwj3CDKCkFNVCPbHgFTYtDvdaqnUM4s_I-cq4,5328
+learning_loop_node/tests/detector/conftest.py,sha256=gut-RaacarhWJNCvGEz7O7kj3cS7vJ4SvAxCmR87PIw,5263
 learning_loop_node/tests/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/detector/inbox_filter/test_observation.py,sha256=k4WYdvnuV7d_r7zI4M2aA8WuBjm0aycQ0vj1rGE2q4w,1370
 learning_loop_node/tests/detector/inbox_filter/test_relevance_group.py,sha256=r-wABFQVsTNTjv7vYGr8wbHfOWy43F_B14ZDWHfiZ-A,7613
 learning_loop_node/tests/detector/inbox_filter/test_unexpected_observations_count.py,sha256=3KKwf-J9oJRMIuuVju2vT9IM9vWhKvswPiXJI8KxmcU,1661
 learning_loop_node/tests/detector/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/detector/test.jpg,sha256=msA-vHPmvPiro_D102Qmn1fn4vNfooqYYEXPxZUmYpk,161390
-learning_loop_node/tests/detector/test_client_communication.py,sha256=RUsdmZRQE1YO587JPHB2c2-bTE_tmThQodWuhjzBPEk,9180
+learning_loop_node/tests/detector/test_client_communication.py,sha256=PUjnWnY-9RCZe-gqrtWf3o0ylCNH3WuzHoL7v3eAjAQ,8984
 learning_loop_node/tests/detector/test_detector_node.py,sha256=0ZMV6coAvdq-nH8CwY9_LR2tUcH9VLcAB1CWuwHQMpo,3023
 learning_loop_node/tests/detector/test_outbox.py,sha256=IfCz4iBmYA4bm3TK4q2NmWyzQCwZWhUbBrKQNHGxZM4,3007
 learning_loop_node/tests/detector/test_relevance_filter.py,sha256=ZKcCstFWCDxJzKdVlAe8E6sZzv5NiH8mADhaZjokHoU,2052
@@ -96,6 +97,6 @@ learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvg
 learning_loop_node/trainer/trainer_logic.py,sha256=eK-01qZzi10UjLMCQX8vy5eW2FoghPj3rzzDC-s3Si4,8792
 learning_loop_node/trainer/trainer_logic_generic.py,sha256=RQqon8JIVzxaNh0KdEe6tMxebsY0DgZllEohHR-AgqU,26846
 learning_loop_node/trainer/trainer_node.py,sha256=Dl4ZQAjjXQggibeBjvhXAoFClw1ZX2Kkt3v_fjrJnCI,4508
-learning_loop_node-0.13.4.dist-info/METADATA,sha256=PCQ2UaAA26NGRvbsCnjf2uWla3Qf-WBjfCs1o5gsgJs,12761
-learning_loop_node-0.13.4.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.13.4.dist-info/RECORD,,
+learning_loop_node-0.13.6.dist-info/METADATA,sha256=z7dmbFXZUgUXgs-y0N-w-KwBDo8H2oVngoc1ja2Giok,12761
+learning_loop_node-0.13.6.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.13.6.dist-info/RECORD,,