learning-loop-node 0.10.10__tar.gz → 0.10.11__tar.gz

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.10
+Version: 0.10.11
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT
@@ -57,19 +57,20 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                    | Alias        | Purpose                                                      | Required by          |
-| ----------------------- | ------------ | ------------------------------------------------------------ | -------------------- |
-| LOOP_HOST               | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
-| LOOP_USERNAME           | USERNAME     | Learning Loop user name                                      | all besides Detector |
-| LOOP_PASSWORD           | PASSWORD     | Learning Loop password                                       | all besides Detector |
-| LOOP_SSL_CERT_PATH      | -            | Path to the SSL certificate                                  | all (opt.)           |
-| LOOP_ORGANIZATION       | ORGANIZATION | Organization name                                            | Detector             |
-| LOOP_PROJECT            | PROJECT      | Project name                                                 | Detector             |
-| MIN_UNCERTAIN_THRESHOLD | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
-| MAX_UNCERTAIN_THRESHOLD | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
-| INFERENCE_BATCH_SIZE    | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING  | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS      | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
+| Name                     | Alias        | Purpose                                                      | Required by          |
+| ------------------------ | ------------ | ------------------------------------------------------------ | -------------------- |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)           |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector             |
+| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector             |
+| MIN_UNCERTAIN_THRESHOLD  | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
+| MAX_UNCERTAIN_THRESHOLD  | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)       |
 
 #### Testing
 
@@ -104,6 +105,24 @@ The detector also has a sio **upload endpoint** that can be used to upload image
 
 The endpoint returns None if the upload was successful and an error message otherwise.
 
+### Changing the model version
+
+The detector can be configured to one of the following behaviors:
+
+- download use a specific model version
+- automatically update the model version according to the learning loop deployment target
+- pause the model updates and use the version that was last loaded
+
+The model versioning configuration can be accessed/changed via a REST endpoint. Example Usage:
+
+- Fetch the current model versioning configuration: `curl http://localhost/model_version`
+- Configure the detector to use a specific model version: `curl -X PUT -d "1.0" http://localhost/model_version`
+- Configure the detector to automatically update the model version: `curl -X PUT -d "follow_loop" http://localhost/model_version`
+- Pause the model updates: `curl -X PUT -d "pause" http://localhost/model_version`
+
+Note that the configuration is not persistent, however, the default behavior on startup can be configured via the environment variable `VERSION_CONTROL_DEFAULT`.
+If the environment variable is set to `VERSION_CONTROL_DEFAULT=PAUSE`, the detector will pause the model updates on startup. Otherwise, the detector will automatically follow the loop deployment target.
+
 ### Changing the outbox mode
 
 If the autoupload is set to `all` or `filtered` (selected) images and the corresponding detections are saved on HDD (the outbox). A background thread will upload the images and detections to the Learning Loop. The outbox is located in the `outbox` folder in the root directory of the node. The outbox can be cleared by deleting the files in the folder.

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/README.md

@@ -17,19 +17,20 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                    | Alias        | Purpose                                                      | Required by          |
-| ----------------------- | ------------ | ------------------------------------------------------------ | -------------------- |
-| LOOP_HOST               | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
-| LOOP_USERNAME           | USERNAME     | Learning Loop user name                                      | all besides Detector |
-| LOOP_PASSWORD           | PASSWORD     | Learning Loop password                                       | all besides Detector |
-| LOOP_SSL_CERT_PATH      | -            | Path to the SSL certificate                                  | all (opt.)           |
-| LOOP_ORGANIZATION       | ORGANIZATION | Organization name                                            | Detector             |
-| LOOP_PROJECT            | PROJECT      | Project name                                                 | Detector             |
-| MIN_UNCERTAIN_THRESHOLD | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
-| MAX_UNCERTAIN_THRESHOLD | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
-| INFERENCE_BATCH_SIZE    | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING  | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS      | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
+| Name                     | Alias        | Purpose                                                      | Required by          |
+| ------------------------ | ------------ | ------------------------------------------------------------ | -------------------- |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)           |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector             |
+| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector             |
+| MIN_UNCERTAIN_THRESHOLD  | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
+| MAX_UNCERTAIN_THRESHOLD  | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)       |
 
 #### Testing
 
@@ -64,6 +65,24 @@ The detector also has a sio **upload endpoint** that can be used to upload image
 
 The endpoint returns None if the upload was successful and an error message otherwise.
 
+### Changing the model version
+
+The detector can be configured to one of the following behaviors:
+
+- download use a specific model version
+- automatically update the model version according to the learning loop deployment target
+- pause the model updates and use the version that was last loaded
+
+The model versioning configuration can be accessed/changed via a REST endpoint. Example Usage:
+
+- Fetch the current model versioning configuration: `curl http://localhost/model_version`
+- Configure the detector to use a specific model version: `curl -X PUT -d "1.0" http://localhost/model_version`
+- Configure the detector to automatically update the model version: `curl -X PUT -d "follow_loop" http://localhost/model_version`
+- Pause the model updates: `curl -X PUT -d "pause" http://localhost/model_version`
+
+Note that the configuration is not persistent, however, the default behavior on startup can be configured via the environment variable `VERSION_CONTROL_DEFAULT`.
+If the environment variable is set to `VERSION_CONTROL_DEFAULT=PAUSE`, the detector will pause the model updates on startup. Otherwise, the detector will automatically follow the loop deployment target.
+
 ### Changing the outbox mode
 
 If the autoupload is set to `all` or `filtered` (selected) images and the corresponding detections are saved on HDD (the outbox). A background thread will upload the images and detections to the Learning Loop. The outbox is located in the `outbox` folder in the root directory of the node. The outbox can be cleared by deleting the files in the folder.

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/data_classes/general.py

@@ -50,7 +50,7 @@ class ModelInformation():
     organization: str
     project: str
     version: str
-    categories: List[Category]
+    categories: List[Category] = field(default_factory=list)
     resolution: Optional[int] = None
     model_root_path: Optional[str] = None
     model_size: Optional[str] = None

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/data_exchanger.py

@@ -77,7 +77,7 @@ class DataExchanger():
             logging.info('got empty list. No images were downloaded')
             return []
 
-        progress_factor = 0.5 / num_image_ids  # 50% of progress is for downloading data
+        progress_factor = 0.5 / num_image_ids  # first 50% of progress is for downloading data
         images_data: List[Dict] = []
         for i in range(0, num_image_ids, chunk_size):
             self.progress = i * progress_factor
@@ -100,20 +100,21 @@ class DataExchanger():
         new_image_uuids = [id for id in image_uuids if id not in existing_uuids]
 
         paths, _ = create_resource_paths(self.context.organization, self.context.project, new_image_uuids)
-        num_image_ids = len(image_uuids)
+        num_new_image_ids = len(new_image_uuids)
         os.makedirs(image_folder, exist_ok=True)
 
-        progress_factor = 0.5 / num_image_ids  # second 50% of progress is for downloading images
-        for i in range(0, num_image_ids, chunk_size):
+        progress_factor = 0.5 / num_new_image_ids  # second 50% of progress is for downloading images
+        for i in range(0, num_new_image_ids, chunk_size):
             self.progress = 0.5 + i * progress_factor
             chunk_paths = paths[i:i+chunk_size]
-            chunk_ids = image_uuids[i:i+chunk_size]
+            chunk_ids = new_image_uuids[i:i+chunk_size]
             tasks = []
             for j, chunk_j in enumerate(chunk_paths):
                 start = time()
                 tasks.append(create_task(self._download_one_image(chunk_j, chunk_ids[j], image_folder)))
                 await asyncio.sleep(max(0, 0.02 - (time() - start)))  # prevent too many requests at once
             await asyncio.gather(*tasks)
+        self.progress = 1.0
 
     async def _download_one_image(self, path: str, image_id: str, image_folder: str) -> None:
         response = await self.loop_communicator.get(path)
@@ -124,7 +125,10 @@ class DataExchanger():
         async with aiofiles.open(filename, 'wb') as f:
             await f.write(response.content)
         if not await is_valid_image(filename, self.check_jpeg):
+            logging.error('Invalid image "%s". Removing it..', filename)
             os.remove(filename)
+        else:
+            logging.debug('Downloaded image "%s"', filename)
 
     async def download_model(self, target_folder: str, context: Context, model_uuid: str, model_format: str) -> List[str]:
         """Downloads a model (and additional meta data like model.json) and returns the paths of the downloaded files.

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/detector/detector_node.py

@@ -6,7 +6,7 @@ import subprocess
 from dataclasses import asdict
 from datetime import datetime
 from threading import Thread
-from typing import Dict, List, Literal, Optional, Union
+from typing import Dict, List, Optional, Union
 
 import numpy as np
 from dacite import from_dict
@@ -26,6 +26,7 @@ from .outbox import Outbox
 from .rest import about as rest_about
 from .rest import backdoor_controls
 from .rest import detect as rest_detect
+from .rest import model_version_control as rest_version_control
 from .rest import operation_mode as rest_mode
 from .rest import outbox_mode as rest_outbox_mode
 from .rest import upload as rest_upload
@@ -52,13 +53,22 @@ class DetectorNode(Node):
             self.loop_communicator)
 
         self.relevance_filter: RelevanceFilter = RelevanceFilter(self.outbox)
-        self.target_model: Optional[str] = None
+
+        # NOTE: version_control controls the behavior of the detector node.
+        # FollowLoop: the detector node will follow the loop and update the model if necessary
+        # SpecificVersion: the detector node will update to a specific version, set via the /model_version endpoint
+        # Pause: the detector node will not update the model
+        self.version_control: rest_version_control.VersionMode = rest_version_control.VersionMode.Pause if os.environ.get(
+            'VERSION_CONTROL_DEFAULT', 'follow_loop').lower() == 'pause' else rest_version_control.VersionMode.FollowLoop
+        self.target_model: Optional[ModelInformation] = None
+        self.loop_deployment_target: Optional[ModelInformation] = None
 
         self.include_router(rest_detect.router, tags=["detect"])
         self.include_router(rest_upload.router, prefix="")
         self.include_router(rest_mode.router, tags=["operation_mode"])
         self.include_router(rest_about.router, tags=["about"])
         self.include_router(rest_outbox_mode.router, tags=["outbox_mode"])
+        self.include_router(rest_version_control.router, tags=["model_version"])
 
         if use_backdoor_controls:
             self.include_router(backdoor_controls.router)
@@ -75,6 +85,8 @@ class DetectorNode(Node):
             Context(organization=self.organization, project=self.project),
             self.loop_communicator)
         self.relevance_filter = RelevanceFilter(self.outbox)
+        self.version_control = rest_version_control.VersionMode.Pause if os.environ.get(
+            'VERSION_CONTROL_DEFAULT', 'follow_loop').lower() == 'pause' else rest_version_control.VersionMode.FollowLoop
         self.target_model = None
         # self.setup_sio_server()
 
@@ -183,20 +195,12 @@ class DetectorNode(Node):
             return
         try:
             self.log.info(f'Current operation mode is {self.operation_mode}')
-            update_to_model_id = await self.send_status()
-            if not update_to_model_id:
-                self.log.info('could not check for updates')
+            try:
+                await self.sync_status_with_learning_loop()
+            except Exception as e:
+                self.log.error(f'Could not check for updates: {e}')
                 return
 
-            # TODO: solve race condition (it should not be required to recheck if model_info is not None, but it is!)
-            if self.detector_logic.is_initialized:
-                model_info = self.detector_logic._model_info  # pylint: disable=protected-access
-                if model_info is not None:
-                    self.log.info(f'Current model: {model_info.version} with id {model_info.id}')
-                else:
-                    self.log.info('no model loaded')
-            else:
-                self.log.info('no model loaded')
             if self.operation_mode != OperationMode.Idle:
                 self.log.info(f'not checking for updates; operation mode is {self.operation_mode}')
                 return
@@ -206,25 +210,22 @@ class DetectorNode(Node):
                 self.log.info('not checking for updates; no target model selected')
                 return
 
-            self.log.info('going to check for new updates')  # TODO: solve race condition !!!
-            model_info = self.detector_logic._model_info  # pylint: disable=protected-access
-            if model_info is not None:
-                version = model_info.version
-            else:
-                version = None
-            if not self.detector_logic.is_initialized or self.target_model != version:
-                cur_model = version or "-"
-                self.log.info(f'Current model "{cur_model}" needs to be updated to {self.target_model}')
+            current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else None
+
+            if not self.detector_logic.is_initialized or self.target_model.version != current_version:
+                self.log.info(
+                    f'Current model "{current_version or "-"}" needs to be updated to {self.target_model.version}')
+
                 with step_into(GLOBALS.data_folder):
                     model_symlink = 'model'
-                    target_model_folder = f'models/{self.target_model}'
+                    target_model_folder = f'models/{self.target_model.version}'
                     shutil.rmtree(target_model_folder, ignore_errors=True)
                     os.makedirs(target_model_folder)
 
                     await self.data_exchanger.download_model(target_model_folder,
                                                              Context(organization=self.organization,
                                                                      project=self.project),
-                                                             update_to_model_id, self.detector_logic.model_format)
+                                                             self.target_model.id, self.detector_logic.model_format)
                     try:
                         os.unlink(model_symlink)
                         os.remove(model_symlink)
@@ -234,26 +235,42 @@ class DetectorNode(Node):
                     self.log.info(f'Updated symlink for model to {os.readlink(model_symlink)}')
 
                     self.detector_logic.load_model()
-                    await self.send_status()
+                    try:
+                        await self.sync_status_with_learning_loop()
+                    except Exception:
+                        pass
                     # self.reload(reason='new model installed')
-            else:
-                self.log.info('Versions are identic. Nothing to do.')
+
         except Exception as e:
             self.log.exception('check_for_update failed')
             msg = e.cause if isinstance(e, DownloadError) else str(e)
             self.status.set_error('update_model', f'Could not update model: {msg}')
-            await self.send_status()
+            try:
+                await self.sync_status_with_learning_loop()
+            except Exception:
+                pass
+
+    async def sync_status_with_learning_loop(self) -> None:
+        """Sync status of the detector with the Learning Loop.
+        The Learning Loop will respond with the model info of the deployment target.
+        If version_control is set to FollowLoop, the detector will update the target_model.
+        Return if the communication was successful.
+
+        Raises:
+            Exception: If the communication with the Learning Loop failed.
+        """
 
-    async def send_status(self) -> Union[str, Literal[False]]:
         if not self.sio_client.connected:
-            self.log.info('could not send status -- we are not connected to the Learning Loop')
-            return False
+            self.log.info('Status sync failed: not connected')
+            raise Exception('Status sync failed: not connected')
 
         try:
             current_model = self.detector_logic.model_info.version
         except Exception:
             current_model = None
 
+        target_model_version = self.target_model.version if self.target_model else None
+
         status = DetectionStatus(
             id=self.uuid,
             name=self.name,
@@ -262,27 +279,38 @@ class DetectorNode(Node):
             uptime=int((datetime.now() - self.startup_datetime).total_seconds()),
             operation_mode=self.operation_mode,
             current_model=current_model,
-            target_model=self.target_model,
+            target_model=target_model_version,
             model_format=self.detector_logic.model_format,
         )
 
         self.log.info(f'sending status {status}')
         response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
+
         assert response is not None
         socket_response = from_dict(data_class=SocketResponse, data=response)
         if not socket_response.success:
             self.log.error(f'Statusupdate failed: {response}')
-            return False
+            raise Exception(f'Statusupdate failed: {response}')
 
         assert socket_response.payload is not None
-        # TODO This is weird because target_model_version is stored in self and target_model_id is returned
-        self.target_model = socket_response.payload['target_model_version']
-        self.log.info(f'After sending status. Target_model is {self.target_model}')
-        return socket_response.payload['target_model_id']
+
+        deployment_target_model_id = socket_response.payload['target_model_id']
+        deployment_target_model_version = socket_response.payload['target_model_version']
+        self.loop_deployment_target = ModelInformation(organization=self.organization, project=self.project,
+                                                       host="", categories=[],
+                                                       id=deployment_target_model_id,
+                                                       version=deployment_target_model_version)
+
+        if self.version_control == rest_version_control.VersionMode.FollowLoop:
+            self.target_model = self.loop_deployment_target
+            self.log.info(f'After sending status. Target_model is {self.target_model.version}')
 
     async def set_operation_mode(self, mode: OperationMode):
         self.operation_mode = mode
-        await self.send_status()
+        try:
+            await self.sync_status_with_learning_loop()
+        except Exception as e:
+            self.log.warning(f'Operation mode set to {mode}, but sync failed: {e}')
 
     def reload(self, reason: str):
         '''provide a cause for the reload'''

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/detector/rest/about.py

@@ -21,5 +21,5 @@ async def get_about(request: Request):
         'operation_mode': app.operation_mode.value,
         'state': app.status.state,
         'model_info':  app.detector_logic._model_info,  # pylint: disable=protected-access
-        'target_model': app.target_model,  # pylint: disable=protected-access
+        'target_model': app.target_model.version if app.target_model is not None else 'None',
     }

learning_loop_node-0.10.11/learning_loop_node/detector/rest/model_version_control.py

@@ -0,0 +1,101 @@
+
+import os
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter, HTTPException, Request
+
+from ...data_classes import ModelInformation
+from ...globals import GLOBALS
+
+if TYPE_CHECKING:
+    from ..detector_node import DetectorNode
+
+router = APIRouter()
+
+
+class VersionMode(str, Enum):
+    FollowLoop = 'follow_loop'  # will follow the loop
+    SpecificVersion = 'specific_version'  # will follow the specific version
+    Pause = 'pause'  # will pause the updates
+
+
+@router.get("/model_version")
+async def get_version(request: Request):
+    '''
+    Example Usage
+        curl http://localhost/model_version
+    '''
+    # pylint: disable=protected-access
+
+    app: 'DetectorNode' = request.app
+
+    current_version = app.detector_logic._model_info.version if app.detector_logic._model_info is not None else 'None'
+    target_version = app.target_model.version if app.target_model is not None else 'None'
+    loop_version = app.loop_deployment_target.version if app.loop_deployment_target is not None else 'None'
+
+    local_versions: list[str] = []
+
+    local_models = os.listdir(os.path.join(GLOBALS.data_folder, 'models'))
+    for model in local_models:
+        if model.replace('.', '').isdigit():
+            local_versions.append(model)
+
+    return {
+        'current_version': current_version,
+        'target_version': target_version,
+        'loop_version': loop_version,
+        'local_versions': local_versions,
+        'version_control': app.version_control.value,
+    }
+
+
+@router.put("/model_version")
+async def put_version(request: Request):
+    '''
+    Example Usage
+        curl -X PUT -d "follow_loop" http://localhost/model_version
+        curl -X PUT -d "pause" http://localhost/model_version
+        curl -X PUT -d "13.6" http://localhost/model_version
+    '''
+    app: 'DetectorNode' = request.app
+    content = str(await request.body(), 'utf-8')
+
+    if content == 'follow_loop':
+        app.version_control = VersionMode.FollowLoop
+    elif content == 'pause':
+        app.version_control = VersionMode.Pause
+    else:
+        app.version_control = VersionMode.SpecificVersion
+        if not content or not content.replace('.', '').isdigit():
+            raise HTTPException(400, 'Invalid version number')
+        target_version = content
+
+        if app.target_model is not None and app.target_model.version == target_version:
+            return "OK"
+
+        # Fetch the model uuid by version from the loop
+        uri = f'/{app.organization}/projects/{app.project}/models'
+        response = await app.loop_communicator.get(uri)
+        if response.status_code != 200:
+            app.version_control = VersionMode.Pause
+            raise HTTPException(500, 'Failed to load models from learning loop')
+
+        models = response.json()['models']
+        models_with_target_version = [m for m in models if m['version'] == target_version]
+        if len(models_with_target_version) == 0:
+            app.version_control = VersionMode.Pause
+            raise HTTPException(400, f'No Model with version {target_version}')
+        if len(models_with_target_version) > 1:
+            app.version_control = VersionMode.Pause
+            raise HTTPException(500, f'Multiple models with version {target_version}')
+
+        model_id = models_with_target_version[0]['id']
+        model_host = models_with_target_version[0].get('host', 'unknown')
+
+        app.target_model = ModelInformation(organization=app.organization, project=app.project,
+                                            host=model_host, categories=[],
+                                            id=model_id,
+                                            version=target_version)
+
+    return "OK"

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/node.py

@@ -16,7 +16,7 @@ from socketio import AsyncClient
 from .data_classes import NodeStatus
 from .data_exchanger import DataExchanger
 from .helpers import log_conf
-from .helpers.misc import activate_asyncio_warnings, ensure_socket_response, read_or_create_uuid
+from .helpers.misc import ensure_socket_response, read_or_create_uuid
 from .loop_communication import LoopCommunicator
 
 

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/tests/detector/test_client_communication.py

@@ -107,6 +107,62 @@ async def test_about_endpoint(test_detector_node: DetectorNode):
     assert any(c.name == 'purple point' for c in model_information.categories)
 
 
+async def test_model_version_api(test_detector_node: DetectorNode):
+    await asyncio.sleep(3)
+
+    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/model_version', timeout=30)
+    assert response.status_code == 200
+    response_dict = json.loads(response.content)
+    assert response_dict['current_version'] == '1.1'
+    assert response_dict['target_version'] == '1.1'
+    assert response_dict['loop_version'] == '1.1'
+    assert response_dict['local_versions'] == ['1.1']
+    assert response_dict['version_control'] == 'follow_loop'
+
+    response = requests.put(f'http://localhost:{GLOBALS.detector_port}/model_version', data='1.0', timeout=30)
+    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/model_version', timeout=30)
+    assert response.status_code == 200
+    response_dict = json.loads(response.content)
+    assert response_dict['current_version'] == '1.1'
+    assert response_dict['target_version'] == '1.0'
+    assert response_dict['loop_version'] == '1.1'
+    assert response_dict['local_versions'] == ['1.1']
+    assert response_dict['version_control'] == 'specific_version'
+
+    await asyncio.sleep(11)
+
+    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/model_version', timeout=30)
+    assert response.status_code == 200
+    response_dict = json.loads(response.content)
+    assert response_dict['current_version'] == '1.0'
+    assert response_dict['target_version'] == '1.0'
+    assert response_dict['loop_version'] == '1.1'
+    assert set(response_dict['local_versions']) == set(['1.1', '1.0'])
+    assert response_dict['version_control'] == 'specific_version'
+
+    response = requests.put(f'http://localhost:{GLOBALS.detector_port}/model_version', data='pause', timeout=30)
+    await asyncio.sleep(11)
+    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/model_version', timeout=30)
+    assert response.status_code == 200
+    response_dict = json.loads(response.content)
+    assert response_dict['current_version'] == '1.0'
+    assert response_dict['target_version'] == '1.0'
+    assert response_dict['loop_version'] == '1.1'
+    assert set(response_dict['local_versions']) == set(['1.1', '1.0'])
+    assert response_dict['version_control'] == 'pause'
+
+    response = requests.put(f'http://localhost:{GLOBALS.detector_port}/model_version', data='follow_loop', timeout=30)
+    await asyncio.sleep(11)
+    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/model_version', timeout=30)
+    assert response.status_code == 200
+    response_dict = json.loads(response.content)
+    assert response_dict['current_version'] == '1.1'
+    assert response_dict['target_version'] == '1.1'
+    assert response_dict['loop_version'] == '1.1'
+    assert set(response_dict['local_versions']) == set(['1.1', '1.0'])
+    assert response_dict['version_control'] == 'follow_loop'
+
+
 async def test_rest_outbox_mode(test_detector_node: DetectorNode):
     await asyncio.sleep(3)
 

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/trainer/trainer_node.py

@@ -1,3 +1,6 @@
+import os
+import sys
+import time
 from dataclasses import asdict
 from typing import Dict, Optional
 
@@ -7,7 +10,7 @@ from socketio import AsyncClient, exceptions
 from ..data_classes import TrainingStatus
 from ..node import Node
 from .io_helpers import LastTrainingIO
-from .rest import backdoor_controls, controls
+from .rest import backdoor_controls
 from .trainer_logic_generic import TrainerLogicGeneric
 
 
@@ -20,7 +23,15 @@ class TrainerNode(Node):
         self.last_training_io = LastTrainingIO(self.uuid)
         self.trainer_logic._last_training_io = self.last_training_io
 
-        self.include_router(controls.router, tags=["controls"])
+        self.first_idle_time: float | None = None
+        if os.environ.get('TRAINER_IDLE_TIMEOUT_SEC', 0.0):
+            self.idle_timeout = float(os.environ.get('TRAINER_IDLE_TIMEOUT_SEC', 0.0))
+        else:
+            self.idle_timeout = 0.0
+        if self.idle_timeout:
+            self.log.info(
+                f'Trainer started with an idle_timeout of {self.idle_timeout} seconds. Note that shutdown does not work if docker container has the restart policy set to always')
+
         if use_backdoor_controls:
             self.include_router(backdoor_controls.router, tags=["controls"])
 
@@ -38,6 +49,7 @@ class TrainerNode(Node):
             if await self.trainer_logic.try_continue_run_if_incomplete():
                 return  # NOTE: we prevent sending idle status after starting a continuation
             await self.send_status()
+            self.check_idle_timeout()
         except exceptions.TimeoutError:
             self.log.warning('timeout when sending status to learning loop, reconnecting sio_client')
             await self.sio_client.disconnect()  # NOTE: reconnect happens in node._on_repeat
@@ -90,3 +102,19 @@ class TrainerNode(Node):
         result = await self.sio_client.call('update_trainer', jsonable_encoder(asdict(status)), timeout=30)
         if isinstance(result, Dict) and not result['success']:
             self.log.error(f'Error when sending status update: Response from loop was:\n {result}')
+
+    def check_idle_timeout(self):
+        if not self.idle_timeout:
+            return
+
+        if self.trainer_logic.state == 'idle':
+            if self.first_idle_time is None:
+                self.first_idle_time = time.time()
+            idle_time = time.time() - self.first_idle_time
+            if idle_time > self.idle_timeout:
+                self.log.info('Trainer has been idle for %.2f s (with timeout %.2f s). Shutting down.',
+                              idle_time, self.idle_timeout)
+                sys.exit(0)
+            self.log.debug('idle time: %.2f s / %.2f s', idle_time, self.idle_timeout)
+        else:
+            self.first_idle_time = None

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "learning_loop_node"
-version = "v0.10.10"
+version = "v0.10.11"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = ["Zauberzeug GmbH <info@zauberzeug.com>"]
 license = "MIT"

learning_loop_node-0.10.10/learning_loop_node/trainer/rest/controls.py

@@ -1,28 +0,0 @@
-
-import logging
-
-from fastapi import APIRouter, HTTPException, Request
-
-from learning_loop_node.trainer.trainer_logic import TrainerLogic
-
-router = APIRouter()
-
-# pylint: disable=protected-access
-
-
-@router.post("/controls/detect/{organization}/{project}/{version}")
-async def operation_mode(organization: str, project: str, version: str, request: Request):
-    '''
-    Example Usage
-        curl -X POST localhost/controls/detect/<organization>/<project>/<model_version>
-    '''
-    path = f'/{organization}/projects/{project}/models'
-    response = await request.app.loop_communication.get(path)
-    if response.status_code != 200:
-        raise HTTPException(404, 'could not load latest model')
-    models = response.json()['models']
-    model_id = next(m for m in models if m['version'] == version)['id']
-    logging.info(model_id)
-    trainer: TrainerLogic = request.app.trainer
-    await trainer._do_detections()
-    return "OK"

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/tests/detector/conftest.py

@@ -1,8 +1,8 @@
-import shutil
 import asyncio
 import logging
 import multiprocessing
 import os
+import shutil
 import socket
 from glob import glob
 from multiprocessing import Process, log_to_stderr

{learning_loop_node-0.10.10 → learning_loop_node-0.10.11}/learning_loop_node/trainer/downloader.py

@@ -17,8 +17,8 @@ class TrainingsDownloader():
         return (image_data, skipped_image_count)
 
     async def download_images_and_annotations(self, image_ids: List[str], image_folder: str) -> Tuple[List[Dict], int]:
-        await self.data_exchanger.download_images(image_ids, image_folder)
         image_data = await self.data_exchanger.download_images_data(image_ids)
+        await self.data_exchanger.download_images(image_ids, image_folder)
         logging.info('filtering corrupt images')  # download only safes valid images
         valid_image_data: List[Dict] = []
         skipped_image_count = 0