learning-loop-node 0.13.5__tar.gz → 0.13.7__tar.gz

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.13.5
+Version: 0.13.7
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT
@@ -63,8 +63,8 @@ You can configure connection to our Learning Loop by specifying the following en
 | LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
 | LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
 | LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector                  |
-| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector (opt.)           |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  |
+| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector (opt.)           |
 | MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
 | MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
 | INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
@@ -73,6 +73,8 @@ You can configure connection to our Learning Loop by specifying the following en
 | TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
 | USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
 
+Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
+
 #### Testing
 
 We use github actions for CI. Tests can also be executed locally by running

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/README.md

@@ -23,8 +23,8 @@ You can configure connection to our Learning Loop by specifying the following en
 | LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
 | LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
 | LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector                  |
-| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector (opt.)           |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  |
+| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector (opt.)           |
 | MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
 | MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
 | INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
@@ -33,6 +33,8 @@ You can configure connection to our Learning Loop by specifying the following en
 | TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
 | USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
 
+Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
+
 #### Testing
 
 We use github actions for CI. Tests can also be executed locally by running

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/learning_loop_node/data_exchanger.py

@@ -143,7 +143,7 @@ class DataExchanger():
         logging.info('Downloading model data for uuid %s from the loop to %s..', model_uuid, target_folder)
 
         path = f'/{context.organization}/projects/{context.project}/models/{model_uuid}/{model_format}/file'
-        response = await self.loop_communicator.get(path, requires_login=False)
+        response = await self.loop_communicator.get(path, requires_login=False, timeout=60*10)
         if response.status_code != 200:
             decoded_content = response.content.decode('utf-8')
             logging.error('could not download loop/%s: %s, content: %s', path,

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/learning_loop_node/detector/detector_logic.py

@@ -13,37 +13,29 @@ class DetectorLogic():
 
     def __init__(self, model_format: str) -> None:
         self.model_format: str = model_format
-        self._model_info: Optional[ModelInformation] = None
+        self.model_info: Optional[ModelInformation] = None
 
         self._remaining_init_attempts: int = 2
 
     async def soft_reload(self):
-        self._model_info = None
-
-    @property
-    def model_info(self) -> ModelInformation:
-        if self._model_info is None:
-            raise Exception('Model not loaded')
-        return self._model_info
-
-    @property
-    def is_initialized(self) -> bool:
-        return self._model_info is not None
+        self.model_info = None
 
     def load_model_info_and_init_model(self):
         logging.info('Loading model from %s', GLOBALS.data_folder)
-        self._model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
-        if self._model_info is None:
+        self.model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
+        if self.model_info is None:
             logging.error('No model found')
-            self._model_info = None
+            self.model_info = None
+            return
+
         try:
             self.init()
-            logging.info('Successfully loaded model %s', self._model_info)
+            logging.info('Successfully loaded model %s', self.model_info)
             self._remaining_init_attempts = 2
         except Exception:
             self._remaining_init_attempts -= 1
-            self._model_info = None
-            logging.error('Could not init model %s. Retries left: %s', self._model_info, self._remaining_init_attempts)
+            self.model_info = None
+            logging.error('Could not init model %s. Retries left: %s', self.model_info, self._remaining_init_attempts)
             if self._remaining_init_attempts == 0:
                 raise NodeNeedsRestartError('Could not init model') from None
             raise

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/learning_loop_node/detector/detector_node.py

@@ -1,5 +1,6 @@
 import asyncio
 import contextlib
+import math
 import os
 import shutil
 import subprocess
@@ -15,8 +16,16 @@ from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
 from socketio import AsyncClient
 
-from ..data_classes import (AboutResponse, Category, Context, DetectionStatus, ImageMetadata, ModelInformation,
-                            ModelVersionResponse, Shape)
+from ..data_classes import (
+    AboutResponse,
+    Category,
+    Context,
+    DetectionStatus,
+    ImageMetadata,
+    ModelInformation,
+    ModelVersionResponse,
+    Shape,
+)
 from ..data_classes.socket_response import SocketResponse
 from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
@@ -82,13 +91,13 @@ class DetectorNode(Node):
         return AboutResponse(
             operation_mode=self.operation_mode.value,
             state=self.status.state,
-            model_info=self.detector_logic._model_info,  # pylint: disable=protected-access
+            model_info=self.detector_logic.model_info,  # pylint: disable=protected-access
             target_model=self.target_model.version if self.target_model else None,
             version_control=self.version_control.value
         )
 
     def get_model_version_response(self) -> ModelVersionResponse:
-        current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else 'None'  # pylint: disable=protected-access
+        current_version = self.detector_logic.model_info.version if self.detector_logic.model_info is not None else 'None'  # pylint: disable=protected-access
         target_version = self.target_model.version if self.target_model is not None else 'None'
         loop_version = self.loop_deployment_target.version if self.loop_deployment_target is not None else 'None'
 
@@ -237,7 +246,7 @@ class DetectorNode(Node):
 
         @self.sio.event
         async def info(sid) -> Dict:
-            if self.detector_logic.is_initialized:
+            if self.detector_logic.model_info is not None:
                 return asdict(self.detector_logic.model_info)
             return {"status": "No model loaded"}
 
@@ -274,7 +283,7 @@ class DetectorNode(Node):
             '''upload an image with detections'''
 
             detection_data = data.get('detections', {})
-            if detection_data and self.detector_logic.is_initialized:
+            if detection_data and self.detector_logic.model_info is not None:
                 try:
                     image_metadata = from_dict(data_class=ImageMetadata, data=detection_data)
                 except Exception as e:
@@ -323,7 +332,7 @@ class DetectorNode(Node):
                 self.log.debug('not checking for updates; no target model selected')
                 return
 
-            if self.detector_logic.is_initialized:
+            if self.detector_logic.model_info is not None:
                 current_version = self.detector_logic.model_info.version
             else:
                 current_version = None
@@ -335,16 +344,21 @@ class DetectorNode(Node):
                 with step_into(GLOBALS.data_folder):
                     model_symlink = 'model'
                     target_model_folder = f'models/{self.target_model.version}'
-                    if not os.path.exists(target_model_folder):
-                        os.makedirs(target_model_folder)
-                        await self.data_exchanger.download_model(target_model_folder,
-                                                                 Context(organization=self.organization,
-                                                                         project=self.project),
-                                                                 self.target_model.id,
-                                                                 self.detector_logic.model_format)
-                        self.log.info('Downloaded model %s', self.target_model.version)
-                    else:
+                    if os.path.exists(target_model_folder) and len(os.listdir(target_model_folder)) > 0:
                         self.log.info('No need to download model %s (already exists)', self.target_model.version)
+                    else:
+                        os.makedirs(target_model_folder, exist_ok=True)
+                        try:
+                            await self.data_exchanger.download_model(target_model_folder,
+                                                                     Context(organization=self.organization,
+                                                                             project=self.project),
+                                                                     self.target_model.id,
+                                                                     self.detector_logic.model_format)
+                            self.log.info('Downloaded model %s', self.target_model.version)
+                        except Exception:
+                            self.log.exception('Could not download model %s', self.target_model.version)
+                            shutil.rmtree(target_model_folder, ignore_errors=True)
+                            return
                     try:
                         os.unlink(model_symlink)
                         os.remove(model_symlink)
@@ -391,9 +405,9 @@ class DetectorNode(Node):
             self.log.info('Status sync failed: not connected')
             raise Exception('Status sync failed: not connected')
 
-        try:
+        if self.detector_logic.model_info is not None:
             current_model = self.detector_logic.model_info.version
-        except Exception:
+        else:
             current_model = None
 
         target_model_version = self.target_model.version if self.target_model else None
@@ -413,16 +427,23 @@ class DetectorNode(Node):
         self.log_status_on_change(status.state or 'None', status)
 
         # NOTE: sending organization and project is no longer required!
-        response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
+        try:
+            response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
+        except TimeoutError:
+            self.socket_connection_broken = True
+            self.log.exception('TimeoutError for sending status update (will try to reconnect):')
+            raise Exception('Status update failed due to timeout') from None
+
         if not response:
             self.socket_connection_broken = True
-            return
+            self.log.error('Status update failed (will try to reconnect): %s', response)
+            raise Exception('Status update failed: Did not receive a response from the learning loop')
 
         socket_response = from_dict(data_class=SocketResponse, data=response)
         if not socket_response.success:
             self.socket_connection_broken = True
-            self.log.error('Statusupdate failed: %s', response)
-            raise Exception(f'Statusupdate failed: {response}')
+            self.log.error('Status update failed (will try to reconnect): %s', response)
+            raise Exception(f'Status update failed. Response from learning loop: {response}')
 
         assert socket_response.payload is not None
 

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/learning_loop_node/loop_communication.py

@@ -100,31 +100,31 @@ class LoopCommunicator():
                 raise TimeoutError('Backend not ready within timeout')
             await asyncio.sleep(10)
 
-    async def retry_on_401(self, func: Callable[..., Awaitable[httpx.Response]], *args, **kwargs) -> httpx.Response:
+    async def _retry_on_401(self, func: Callable[..., Awaitable[httpx.Response]], *args, **kwargs) -> httpx.Response:
         response = await func(*args, **kwargs)
         if response.status_code == 401:
             await self.ensure_login(relogin=True)
             response = await func(*args, **kwargs)
         return response
 
-    async def get(self, path: str, requires_login: bool = True, api_prefix: str = '/api') -> httpx.Response:
+    async def get(self, path: str, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._get, path, api_prefix)
+            return await self._retry_on_401(self._get, path, api_prefix, timeout)
         return await self._get(path, api_prefix)
 
     @retry_on_429
-    async def _get(self, path: str, api_prefix: str) -> httpx.Response:
-        return await self.async_client.get(api_prefix+path)
+    async def _get(self, path: str, api_prefix: str, timeout: int = 60) -> httpx.Response:
+        return await self.async_client.get(api_prefix+path, timeout=timeout)
 
-    async def put(self, path: str, files: Optional[List[str]] = None, requires_login: bool = True, api_prefix: str = '/api', **kwargs) -> httpx.Response:
+    async def put(self, path: str, files: Optional[List[str]] = None, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60, **kwargs) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._put, path, files, api_prefix, **kwargs)
-        return await self._put(path, files, api_prefix, **kwargs)
+            return await self._retry_on_401(self._put, path, files, api_prefix, timeout, **kwargs)
+        return await self._put(path, files, api_prefix, timeout, **kwargs)
 
     @retry_on_429
-    async def _put(self, path: str, files: Optional[List[str]], api_prefix: str, **kwargs) -> httpx.Response:
+    async def _put(self, path: str, files: Optional[List[str]], api_prefix: str, timeout: int = 60, **kwargs) -> httpx.Response:
         if files is None:
             return await self.async_client.put(api_prefix+path, **kwargs)
 
@@ -139,29 +139,29 @@ class LoopCommunicator():
 
         try:
             file_list = [('files', fh) for fh in file_handles]  # Use file handles
-            response = await self.async_client.put(api_prefix+path, files=file_list)
+            response = await self.async_client.put(api_prefix+path, files=file_list, timeout=timeout)
         finally:
             for fh in file_handles:
                 fh.close()  # Ensure all files are closed
 
         return response
 
-    async def post(self, path: str, requires_login: bool = True, api_prefix: str = '/api', **kwargs) -> httpx.Response:
+    async def post(self, path: str, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60, **kwargs) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._post, path, api_prefix, **kwargs)
+            return await self._retry_on_401(self._post, path, api_prefix, timeout, **kwargs)
         return await self._post(path, api_prefix, **kwargs)
 
     @retry_on_429
-    async def _post(self, path, api_prefix='/api', **kwargs) -> httpx.Response:
-        return await self.async_client.post(api_prefix+path, **kwargs)
+    async def _post(self, path, api_prefix='/api', timeout: int = 60, **kwargs) -> httpx.Response:
+        return await self.async_client.post(api_prefix+path, timeout=timeout, **kwargs)
 
-    async def delete(self, path: str, requires_login: bool = True, api_prefix: str = '/api', **kwargs) -> httpx.Response:
+    async def delete(self, path: str, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60, **kwargs) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._delete, path, api_prefix, **kwargs)
+            return await self._retry_on_401(self._delete, path, api_prefix, timeout, **kwargs)
         return await self._delete(path, api_prefix, **kwargs)
 
     @retry_on_429
-    async def _delete(self, path, api_prefix, **kwargs) -> httpx.Response:
-        return await self.async_client.delete(api_prefix+path, **kwargs)
+    async def _delete(self, path, api_prefix, timeout: int = 60, **kwargs) -> httpx.Response:
+        return await self.async_client.delete(api_prefix+path, timeout=timeout, **kwargs)

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/learning_loop_node/node.py

@@ -74,6 +74,7 @@ class Node(FastAPI):
         self.repeat_loop_lock = asyncio.Lock()
 
         self.previous_state: Optional[str] = None
+        self.repeat_loop_cycle_sec = 5
 
     def log_status_on_change(self, current_state_str: str, full_status: Any):
         if self.previous_state != current_state_str:
@@ -146,7 +147,7 @@ class Node(FastAPI):
             except Exception:
                 self.log.exception('error in repeat loop')
 
-            await asyncio.sleep(5)
+            await asyncio.sleep(self.repeat_loop_cycle_sec)
 
     async def _ensure_sio_connection(self):
         if self.socket_connection_broken or self._sio_client is None or not self.sio_client.connected:

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/learning_loop_node/tests/detector/conftest.py

@@ -129,10 +129,6 @@ class MockDetectorLogic(DetectorLogic):  # pylint: disable=abstract-method
                                          model_name="mock",
                                          )])
 
-    @property
-    def is_initialized(self):
-        return True
-
     def evaluate_with_all_info(self, image: np.ndarray, tags: List[str], source: Optional[str] = None, creation_date: Optional[str] = None):
         return self.image_metadata
 

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/learning_loop_node/tests/detector/test_client_communication.py

@@ -51,9 +51,6 @@ def test_rest_detect(test_detector_node: DetectorNode, grouping_key: str):
     headers = {grouping_key: '0:0:0:0', 'tags':  'some_tag'}
 
     assert isinstance(test_detector_node.detector_logic, TestingDetectorLogic)
-    # test_detector_node.detector_logic.mock_is_initialized = True
-    # print(test_detector_node.detector_logic.mock_is_initialized)
-    # print(test_detector_node.detector_logic.is_initialized)
     response = requests.post(f'http://localhost:{GLOBALS.detector_port}/detect',
                              files=image, headers=headers, timeout=30)
     assert response.status_code == 200

{learning_loop_node-0.13.5 → learning_loop_node-0.13.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "learning_loop_node"
-version = "v0.13.5"
+version = "v0.13.7"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = ["Zauberzeug GmbH <info@zauberzeug.com>"]
 license = "MIT"