learning-loop-node 0.13.4__tar.gz → 0.13.6__tar.gz

{learning_loop_node-0.13.4 → learning_loop_node-0.13.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.13.4
+Version: 0.13.6
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.13.4 → learning_loop_node-0.13.6}/learning_loop_node/detector/detector_logic.py

@@ -6,41 +6,38 @@ import numpy as np
 
 from ..data_classes import ImageMetadata, ModelInformation
 from ..globals import GLOBALS
+from .exceptions import NodeNeedsRestartError
 
 
 class DetectorLogic():
 
     def __init__(self, model_format: str) -> None:
         self.model_format: str = model_format
-        self._model_info: Optional[ModelInformation] = None
+        self.model_info: Optional[ModelInformation] = None
 
-    async def soft_reload(self):
-        self._model_info = None
-
-    @property
-    def model_info(self) -> ModelInformation:
-        if self._model_info is None:
-            raise Exception('Model not loaded')
-        return self._model_info
+        self._remaining_init_attempts: int = 2
 
-    @property
-    def is_initialized(self) -> bool:
-        return self._model_info is not None
+    async def soft_reload(self):
+        self.model_info = None
 
-    def load_model(self):
+    def load_model_info_and_init_model(self):
         logging.info('Loading model from %s', GLOBALS.data_folder)
-        model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
-        if model_info is None:
+        self.model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
+        if self.model_info is None:
             logging.error('No model found')
-            self._model_info = None
-            raise Exception('No model found')
+            self.model_info = None
+            return
+
         try:
-            self._model_info = model_info
             self.init()
-            logging.info('Successfully loaded model %s', self._model_info)
+            logging.info('Successfully loaded model %s', self.model_info)
+            self._remaining_init_attempts = 2
         except Exception:
-            self._model_info = None
-            logging.error('Could not init model %s', model_info)
+            self._remaining_init_attempts -= 1
+            self.model_info = None
+            logging.error('Could not init model %s. Retries left: %s', self.model_info, self._remaining_init_attempts)
+            if self._remaining_init_attempts == 0:
+                raise NodeNeedsRestartError('Could not init model') from None
             raise
 
     @abstractmethod

{learning_loop_node-0.13.4 → learning_loop_node-0.13.6}/learning_loop_node/detector/detector_node.py

@@ -1,8 +1,10 @@
 import asyncio
 import contextlib
+import math
 import os
 import shutil
 import subprocess
+import sys
 from dataclasses import asdict
 from datetime import datetime
 from threading import Thread
@@ -14,8 +16,16 @@ from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
 from socketio import AsyncClient
 
-from ..data_classes import (AboutResponse, Category, Context, DetectionStatus, ImageMetadata, ModelInformation,
-                            ModelVersionResponse, Shape)
+from ..data_classes import (
+    AboutResponse,
+    Category,
+    Context,
+    DetectionStatus,
+    ImageMetadata,
+    ModelInformation,
+    ModelVersionResponse,
+    Shape,
+)
 from ..data_classes.socket_response import SocketResponse
 from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
@@ -23,6 +33,7 @@ from ..globals import GLOBALS
 from ..helpers import environment_reader
 from ..node import Node
 from .detector_logic import DetectorLogic
+from .exceptions import NodeNeedsRestartError
 from .inbox_filter.relevance_filter import RelevanceFilter
 from .outbox import Outbox
 from .rest import about as rest_about
@@ -80,13 +91,13 @@ class DetectorNode(Node):
         return AboutResponse(
             operation_mode=self.operation_mode.value,
             state=self.status.state,
-            model_info=self.detector_logic._model_info,  # pylint: disable=protected-access
+            model_info=self.detector_logic.model_info,  # pylint: disable=protected-access
             target_model=self.target_model.version if self.target_model else None,
             version_control=self.version_control.value
         )
 
     def get_model_version_response(self) -> ModelVersionResponse:
-        current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else 'None'  # pylint: disable=protected-access
+        current_version = self.detector_logic.model_info.version if self.detector_logic.model_info is not None else 'None'  # pylint: disable=protected-access
         target_version = self.target_model.version if self.target_model is not None else 'None'
         loop_version = self.loop_deployment_target.version if self.loop_deployment_target is not None else 'None'
 
@@ -170,13 +181,13 @@ class DetectorNode(Node):
 
         # simulate startup
         await self.detector_logic.soft_reload()
-        self.detector_logic.load_model()
+        self.detector_logic.load_model_info_and_init_model()
         self.operation_mode = OperationMode.Idle
 
     async def on_startup(self) -> None:
         try:
             self.outbox.ensure_continuous_upload()
-            self.detector_logic.load_model()
+            self.detector_logic.load_model_info_and_init_model()
         except Exception:
             self.log.exception("error during 'startup'")
         self.operation_mode = OperationMode.Idle
@@ -235,7 +246,7 @@ class DetectorNode(Node):
 
         @self.sio.event
         async def info(sid) -> Dict:
-            if self.detector_logic.is_initialized:
+            if self.detector_logic.model_info is not None:
                 return asdict(self.detector_logic.model_info)
             return {"status": "No model loaded"}
 
@@ -272,7 +283,7 @@ class DetectorNode(Node):
             '''upload an image with detections'''
 
             detection_data = data.get('detections', {})
-            if detection_data and self.detector_logic.is_initialized:
+            if detection_data and self.detector_logic.model_info is not None:
                 try:
                     image_metadata = from_dict(data_class=ImageMetadata, data=detection_data)
                 except Exception as e:
@@ -321,9 +332,12 @@ class DetectorNode(Node):
                 self.log.debug('not checking for updates; no target model selected')
                 return
 
-            current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else None  # pylint: disable=protected-access
+            if self.detector_logic.model_info is not None:
+                current_version = self.detector_logic.model_info.version
+            else:
+                current_version = None
 
-            if not self.detector_logic.is_initialized or self.target_model.version != current_version:
+            if current_version != self.target_model.version:
                 self.log.info('Current model "%s" needs to be updated to %s',
                               current_version or "-", self.target_model.version)
 
@@ -349,7 +363,10 @@ class DetectorNode(Node):
                     self.log.info('Updated symlink for model to %s', os.readlink(model_symlink))
 
                     try:
-                        self.detector_logic.load_model()
+                        self.detector_logic.load_model_info_and_init_model()
+                    except NodeNeedsRestartError:
+                        self.log.error('Node needs restart')
+                        sys.exit(0)
                     except Exception:
                         self.log.exception('Could not load model, will retry download on next check')
                         shutil.rmtree(target_model_folder, ignore_errors=True)
@@ -383,9 +400,9 @@ class DetectorNode(Node):
             self.log.info('Status sync failed: not connected')
             raise Exception('Status sync failed: not connected')
 
-        try:
+        if self.detector_logic.model_info is not None:
             current_model = self.detector_logic.model_info.version
-        except Exception:
+        else:
             current_model = None
 
         target_model_version = self.target_model.version if self.target_model else None

learning_loop_node-0.13.6/learning_loop_node/detector/exceptions.py

@@ -0,0 +1,7 @@
+
+
+class NodeNeedsRestartError(Exception):
+    '''
+    NodeNeedsRestartError is raised when the node needs to be restarted.
+    This is e.g. the case when the GPU is not available anymore.
+    '''

{learning_loop_node-0.13.4 → learning_loop_node-0.13.6}/learning_loop_node/node.py

@@ -74,6 +74,7 @@ class Node(FastAPI):
         self.repeat_loop_lock = asyncio.Lock()
 
         self.previous_state: Optional[str] = None
+        self.repeat_loop_cycle_sec = 5
 
     def log_status_on_change(self, current_state_str: str, full_status: Any):
         if self.previous_state != current_state_str:
@@ -146,7 +147,7 @@ class Node(FastAPI):
             except Exception:
                 self.log.exception('error in repeat loop')
 
-            await asyncio.sleep(5)
+            await asyncio.sleep(self.repeat_loop_cycle_sec)
 
     async def _ensure_sio_connection(self):
         if self.socket_connection_broken or self._sio_client is None or not self.sio_client.connected:

{learning_loop_node-0.13.4 → learning_loop_node-0.13.6}/learning_loop_node/tests/detector/conftest.py

@@ -129,10 +129,6 @@ class MockDetectorLogic(DetectorLogic):  # pylint: disable=abstract-method
                                          model_name="mock",
                                          )])
 
-    @property
-    def is_initialized(self):
-        return True
-
     def evaluate_with_all_info(self, image: np.ndarray, tags: List[str], source: Optional[str] = None, creation_date: Optional[str] = None):
         return self.image_metadata
 

{learning_loop_node-0.13.4 → learning_loop_node-0.13.6}/learning_loop_node/tests/detector/test_client_communication.py

@@ -51,9 +51,6 @@ def test_rest_detect(test_detector_node: DetectorNode, grouping_key: str):
     headers = {grouping_key: '0:0:0:0', 'tags':  'some_tag'}
 
     assert isinstance(test_detector_node.detector_logic, TestingDetectorLogic)
-    # test_detector_node.detector_logic.mock_is_initialized = True
-    # print(test_detector_node.detector_logic.mock_is_initialized)
-    # print(test_detector_node.detector_logic.is_initialized)
     response = requests.post(f'http://localhost:{GLOBALS.detector_port}/detect',
                              files=image, headers=headers, timeout=30)
     assert response.status_code == 200

{learning_loop_node-0.13.4 → learning_loop_node-0.13.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "learning_loop_node"
-version = "v0.13.4"
+version = "v0.13.6"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = ["Zauberzeug GmbH <info@zauberzeug.com>"]
 license = "MIT"