learning-loop-node 0.10.11__tar.gz → 0.10.13__tar.gz

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.11
+Version: 0.10.13
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT
@@ -57,20 +57,21 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                     | Alias        | Purpose                                                      | Required by          |
-| ------------------------ | ------------ | ------------------------------------------------------------ | -------------------- |
-| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
-| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector |
-| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector |
-| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)           |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector             |
-| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector             |
-| MIN_UNCERTAIN_THRESHOLD  | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
-| MAX_UNCERTAIN_THRESHOLD  | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
-| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
-| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)       |
+| Name                     | Alias        | Purpose                                                      | Required by               |
+| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector                  |
+| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector (opt.)           |
+| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
+| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
+| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
 
 #### Testing
 

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/README.md

@@ -17,20 +17,21 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                     | Alias        | Purpose                                                      | Required by          |
-| ------------------------ | ------------ | ------------------------------------------------------------ | -------------------- |
-| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
-| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector |
-| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector |
-| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)           |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector             |
-| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector             |
-| MIN_UNCERTAIN_THRESHOLD  | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
-| MAX_UNCERTAIN_THRESHOLD  | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
-| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
-| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)       |
+| Name                     | Alias        | Purpose                                                      | Required by               |
+| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector                  |
+| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector (opt.)           |
+| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
+| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
+| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
 
 #### Testing
 

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/data_classes/detections.py

@@ -118,7 +118,8 @@ class Detections():
     classification_detections: List[ClassificationDetection] = field(default_factory=list)
     tags: List[str] = field(default_factory=list)
     date: Optional[str] = field(default_factory=current_datetime)
-    image_id: Optional[str] = None  # used for detection of trainers
+    image_id: Optional[str] = None  # (actually UUID) used for detection of trainers
+    source: Optional[str] = None
 
     def __len__(self):
         return len(self.box_detections) + len(self.point_detections) + len(self.segmentation_detections) + len(self.classification_detections)

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/data_exchanger.py

@@ -101,6 +101,12 @@ class DataExchanger():
 
         paths, _ = create_resource_paths(self.context.organization, self.context.project, new_image_uuids)
         num_new_image_ids = len(new_image_uuids)
+        if num_new_image_ids == 0:
+            logging.info('All images are already downloaded. Nothing to do.')
+            self.progress = 1.0
+            return
+
+        logging.info(f'Downloading {num_new_image_ids} new images to {image_folder}..')
         os.makedirs(image_folder, exist_ok=True)
 
         progress_factor = 0.5 / num_new_image_ids  # second 50% of progress is for downloading images

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/detector/detector_logic.py

@@ -1,6 +1,6 @@
 import logging
 from abc import abstractmethod
-from typing import Optional
+from typing import List, Optional
 
 import numpy as np
 
@@ -46,6 +46,12 @@ class DetectorLogic():
     def init(self):
         """Called when a (new) model was loaded. Initialize the model. Model information available via `self.model_info`"""
 
+    def evaluate_with_all_info(self, image: np.ndarray, tags: List[str], source: Optional[str] = None) -> Detections:  # pylint: disable=unused-argument
+        """Called by the detector node when an image should be evaluated (REST or SocketIO).
+        Tags, source come from the caller and may be used in this function. 
+        By default, this function simply calls `evaluate`"""
+        return self.evaluate(image)
+
     @abstractmethod
     def evaluate(self, image: np.ndarray) -> Detections:
         """Evaluate the image and return the detections.

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/detector/detector_node.py

@@ -9,9 +9,9 @@ from threading import Thread
 from typing import Dict, List, Optional, Union
 
 import numpy as np
+import socketio
 from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
-from fastapi_socketio import SocketManager
 from socketio import AsyncClient
 
 from ..data_classes import Category, Context, Detections, DetectionStatus, ModelInformation, Shape
@@ -41,7 +41,7 @@ class DetectorNode(Node):
         self.organization = environment_reader.organization()
         self.project = environment_reader.project()
         assert self.organization and self.project, 'Detector node needs an organization and an project'
-        self.log.info(f'Using {self.organization}/{self.project}')
+        self.log.info('Using %s/%s', self.organization, self.project)
         self.operation_mode: OperationMode = OperationMode.Startup
         self.connected_clients: List[str] = []
 
@@ -70,7 +70,7 @@ class DetectorNode(Node):
         self.include_router(rest_outbox_mode.router, tags=["outbox_mode"])
         self.include_router(rest_version_control.router, tags=["model_version"])
 
-        if use_backdoor_controls:
+        if use_backdoor_controls or os.environ.get('USE_BACKDOOR_CONTROLS', '0').lower() in ('1', 'true'):
             self.include_router(backdoor_controls.router)
 
         self.setup_sio_server()
@@ -126,22 +126,32 @@ class DetectorNode(Node):
 
     def setup_sio_server(self) -> None:
         """The DetectorNode acts as a SocketIO server. This method sets up the server and defines the event handlers."""
-
         # pylint: disable=unused-argument
 
-        async def _detect(sid, data: Dict) -> Dict:
-            self.log.info('running detect via socketio')
+        # Initialize the Socket.IO server
+        self.sio = socketio.AsyncServer(async_mode='asgi')
+        # Initialize and mount the ASGI app
+        self.sio_app = socketio.ASGIApp(self.sio, socketio_path='/socket.io')
+        self.mount('/ws', self.sio_app)
+        # Register event handlers
+
+        self.log.info('>>>>>>>>>>>>>>>>>>>>>>> Setting up the SIO server')
+
+        @self.sio.event
+        async def detect(sid, data: Dict) -> Dict:
+            self.log.debug('running detect via socketio')
             try:
                 np_image = np.frombuffer(data['image'], np.uint8)
                 det = await self.get_detections(
                     raw_image=np_image,
                     camera_id=data.get('camera-id', None) or data.get('mac', None),
                     tags=data.get('tags', []),
-                    autoupload=data.get('autoupload', None),
+                    source=data.get('source', None),
+                    autoupload=data.get('autoupload', None)
                 )
                 if det is None:
                     return {'error': 'no model loaded'}
-                self.log.info('detect via socketio finished')
+                self.log.debug('detect via socketio finished')
                 return det
             except Exception as e:
                 self.log.exception('could not detect via socketio')
@@ -149,12 +159,14 @@ class DetectorNode(Node):
                     f.write(data['image'])
                 return {'error': str(e)}
 
-        async def _info(sid) -> Union[str, Dict]:
+        @self.sio.event
+        async def info(sid) -> Union[str, Dict]:
             if self.detector_logic.is_initialized:
                 return asdict(self.detector_logic.model_info)
             return 'No model loaded'
 
-        async def _upload(sid, data: Dict) -> Optional[Dict]:
+        @self.sio.event
+        async def upload(sid, data: Dict) -> Optional[Dict]:
             '''upload an image with detections'''
 
             detection_data = data.get('detections', {})
@@ -171,38 +183,33 @@ class DetectorNode(Node):
             tags = data.get('tags', [])
             tags.append('picked_by_system')
 
+            source = data.get('source', None)
+
             loop = asyncio.get_event_loop()
             try:
-                await loop.run_in_executor(None, self.outbox.save, data['image'], detections, tags)
+                await loop.run_in_executor(None, self.outbox.save, data['image'], detections, tags, source)
             except Exception as e:
                 self.log.exception('could not upload via socketio')
                 return {'error': str(e)}
             return None
 
-        def _connect(sid, environ, auth) -> None:
+        @self.sio.event
+        def connect(sid, environ, auth) -> None:
             self.connected_clients.append(sid)
 
-        print('>>>>>>>>>>>>>>>>>>>>>>> setting up sio server', flush=True)
-
-        self.sio_server = SocketManager(app=self)
-        self.sio_server.on('detect', _detect)
-        self.sio_server.on('info', _info)
-        self.sio_server.on('upload', _upload)
-        self.sio_server.on('connect', _connect)
-
     async def _check_for_update(self) -> None:
         if self.operation_mode == OperationMode.Startup:
             return
         try:
-            self.log.info(f'Current operation mode is {self.operation_mode}')
+            self.log.info('Current operation mode is %s', self.operation_mode)
             try:
                 await self.sync_status_with_learning_loop()
             except Exception as e:
-                self.log.error(f'Could not check for updates: {e}')
+                self.log.error('Could not check for updates: %s', e)
                 return
 
             if self.operation_mode != OperationMode.Idle:
-                self.log.info(f'not checking for updates; operation mode is {self.operation_mode}')
+                self.log.info('not checking for updates; operation mode is %s', self.operation_mode)
                 return
 
             self.status.reset_error('update_model')
@@ -210,11 +217,11 @@ class DetectorNode(Node):
                 self.log.info('not checking for updates; no target model selected')
                 return
 
-            current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else None
+            current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else None  # pylint: disable=protected-access
 
             if not self.detector_logic.is_initialized or self.target_model.version != current_version:
-                self.log.info(
-                    f'Current model "{current_version or "-"}" needs to be updated to {self.target_model.version}')
+                self.log.info('Current model "%s" needs to be updated to %s',
+                              current_version or "-", self.target_model.version)
 
                 with step_into(GLOBALS.data_folder):
                     model_symlink = 'model'
@@ -232,7 +239,7 @@ class DetectorNode(Node):
                     except Exception:
                         pass
                     os.symlink(target_model_folder, model_symlink)
-                    self.log.info(f'Updated symlink for model to {os.readlink(model_symlink)}')
+                    self.log.info('Updated symlink for model to %s', os.readlink(model_symlink))
 
                     self.detector_logic.load_model()
                     try:
@@ -283,13 +290,13 @@ class DetectorNode(Node):
             model_format=self.detector_logic.model_format,
         )
 
-        self.log.info(f'sending status {status}')
+        self.log.info('sending status %s', status)
         response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
 
         assert response is not None
         socket_response = from_dict(data_class=SocketResponse, data=response)
         if not socket_response.success:
-            self.log.error(f'Statusupdate failed: {response}')
+            self.log.error('Statusupdate failed: %s', response)
             raise Exception(f'Statusupdate failed: {response}')
 
         assert socket_response.payload is not None
@@ -303,19 +310,19 @@ class DetectorNode(Node):
 
         if self.version_control == rest_version_control.VersionMode.FollowLoop:
             self.target_model = self.loop_deployment_target
-            self.log.info(f'After sending status. Target_model is {self.target_model.version}')
+            self.log.info('After sending status. Target_model is %s', self.target_model.version)
 
     async def set_operation_mode(self, mode: OperationMode):
         self.operation_mode = mode
         try:
             await self.sync_status_with_learning_loop()
         except Exception as e:
-            self.log.warning(f'Operation mode set to {mode}, but sync failed: {e}')
+            self.log.warning('Operation mode set to %s, but sync failed: %s', mode, e)
 
     def reload(self, reason: str):
         '''provide a cause for the reload'''
 
-        self.log.info(f'########## reloading app because {reason}')
+        self.log.info('########## reloading app because %s', reason)
         if os.path.isfile('/app/app_code/restart/restart.py'):
             subprocess.call(['touch', '/app/app_code/restart/restart.py'])
         elif os.path.isfile('/app/main.py'):
@@ -325,32 +332,36 @@ class DetectorNode(Node):
         else:
             self.log.error('could not reload app')
 
-    async def get_detections(self, raw_image: np.ndarray, camera_id: Optional[str], tags: List[str], autoupload: Optional[str] = None) -> Optional[Dict]:
-        """Note: raw_image is a numpy array of type uint8, but not in the correrct shape!
+    async def get_detections(self,
+                             raw_image: np.ndarray,
+                             camera_id: Optional[str],
+                             tags: List[str],
+                             source: Optional[str] = None,
+                             autoupload: Optional[str] = None) -> Optional[Dict]:
+        """ Main processing function for the detector node when an image is received via REST or SocketIO.
+        This function infers the detections from the image, cares about uploading to the loop and returns the detections as a dictionary.
+        Note: raw_image is a numpy array of type uint8, but not in the correct shape!
         It can be converted e.g. using cv2.imdecode(raw_image, cv2.IMREAD_COLOR)"""
-        loop = asyncio.get_event_loop()
+
         await self.detection_lock.acquire()
-        detections: Detections = await loop.run_in_executor(None, self.detector_logic.evaluate, raw_image)
+        loop = asyncio.get_event_loop()
+        detections = await loop.run_in_executor(None, self.detector_logic.evaluate_with_all_info, raw_image, tags, source)
         self.detection_lock.release()
-        for seg_detection in detections.segmentation_detections:
-            if isinstance(seg_detection.shape, Shape):
-                shapes = ','.join([str(value) for p in seg_detection.shape.points for _,
-                                   value in asdict(p).items()])
-                seg_detection.shape = shapes  # TODO This seems to be a quick fix.. check how loop upload detections deals with this
 
+        fix_shape_detections(detections)
         n_bo, n_cl = len(detections.box_detections), len(detections.classification_detections)
         n_po, n_se = len(detections.point_detections), len(detections.segmentation_detections)
-        self.log.info(f'detected:{n_bo} boxes, {n_po} points, {n_se} segs, {n_cl} classes')
+        self.log.debug('Detected: %d boxes, %d points, %d segs, %d classes', n_bo, n_po, n_se, n_cl)
 
         if autoupload is None or autoupload == 'filtered':  # NOTE default is filtered
             Thread(target=self.relevance_filter.may_upload_detections,
-                   args=(detections, camera_id, raw_image, tags)).start()
+                   args=(detections, camera_id, raw_image, tags, source)).start()
         elif autoupload == 'all':
-            Thread(target=self.outbox.save, args=(raw_image, detections, tags)).start()
+            Thread(target=self.outbox.save, args=(raw_image, detections, tags, source)).start()
         elif autoupload == 'disabled':
             pass
         else:
-            self.log.error(f'unknown autoupload value {autoupload}')
+            self.log.error('unknown autoupload value %s', autoupload)
         return jsonable_encoder(asdict(detections))
 
     async def upload_images(self, images: List[bytes]):
@@ -393,3 +404,12 @@ def step_into(new_dir):
         yield
     finally:
         os.chdir(previous_dir)
+
+
+def fix_shape_detections(detections: Detections):
+    # TODO This is a quick fix.. check how loop upload detections deals with this
+    for seg_detection in detections.segmentation_detections:
+        if isinstance(seg_detection.shape, Shape):
+            points = ','.join([str(value) for p in seg_detection.shape.points for _,
+                               value in asdict(p).items()])
+            seg_detection.shape = points

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/detector/inbox_filter/relevance_filter.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from ...data_classes.detections import Detections
 from ..outbox import Outbox
@@ -11,7 +11,13 @@ class RelevanceFilter():
         self.cam_histories: Dict[str, CamObservationHistory] = {}
         self.outbox: Outbox = outbox
 
-    def may_upload_detections(self, dets: Detections, cam_id: str, raw_image: bytes, tags: List[str]) -> List[str]:
+    def may_upload_detections(self,
+                              dets: Detections,
+                              cam_id: str,
+                              raw_image: bytes,
+                              tags: List[str],
+                              source: Optional[str] = None
+                              ) -> List[str]:
         for group in self.cam_histories.values():
             group.forget_old_detections()
 
@@ -23,5 +29,5 @@ class RelevanceFilter():
         if len(causes) > 0:
             tags = tags if tags is not None else []
             tags.extend(causes)
-            self.outbox.save(raw_image, dets, tags)
+            self.outbox.save(raw_image, dets, tags, source)
         return causes

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/detector/outbox.py

@@ -55,7 +55,13 @@ class Outbox():
 
         self.upload_counter = 0
 
-    def save(self, image: bytes, detections: Optional[Detections] = None, tags: Optional[List[str]] = None) -> None:
+    def save(self,
+             image: bytes,
+             detections: Optional[Detections] = None,
+             tags: Optional[List[str]] = None,
+             source: Optional[str] = None
+             ) -> None:
+
         if not self._is_valid_jpg(image):
             self.log.error('Invalid jpg image')
             return
@@ -71,6 +77,7 @@ class Outbox():
         tmp = f'{GLOBALS.data_folder}/tmp/{identifier}'
         detections.tags = tags
         detections.date = identifier
+        detections.source = source or 'unknown'
         os.makedirs(tmp, exist_ok=True)
 
         with open(tmp + '/image.json', 'w') as f:

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/detector/rest/about.py

@@ -22,4 +22,5 @@ async def get_about(request: Request):
         'state': app.status.state,
         'model_info':  app.detector_logic._model_info,  # pylint: disable=protected-access
         'target_model': app.target_model.version if app.target_model is not None else 'None',
+        'version_control': app.version_control.value,
     }

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/detector/rest/backdoor_controls.py

@@ -22,10 +22,8 @@ async def _socketio(request: Request):
         curl -X PUT -d "on" http://localhost:8007/socketio
     '''
     state = str(await request.body(), 'utf-8')
-    await _switch_socketio(state, request.app)
+    detector_node: 'DetectorNode' = request.app
 
-
-async def _switch_socketio(state: str, detector_node: 'DetectorNode'):
     if state == 'off':
         logging.info('BC: turning socketio off')
         await detector_node.sio_client.disconnect()

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/detector/rest/detect.py

@@ -1,10 +1,14 @@
 import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
 from fastapi import APIRouter, File, Header, Request, UploadFile
 from fastapi.responses import JSONResponse
 
+if TYPE_CHECKING:
+    from ..detector_node import DetectorNode
+
+
 router = APIRouter()
 
 
@@ -15,6 +19,7 @@ async def http_detect(
     camera_id: Optional[str] = Header(None),
     mac: Optional[str] = Header(None),
     tags: Optional[str] = Header(None),
+    source: Optional[str] = Header(None),
     autoupload: Optional[str] = Header(None),
 ):
     """
@@ -35,10 +40,12 @@ async def http_detect(
         raise Exception(f'Uploaded file {file.filename} is no image file.') from exc
 
     try:
-        detections = await request.app.get_detections(raw_image=np_image,
-                                                      camera_id=camera_id or mac or None,
-                                                      tags=tags.split(',') if tags else [],
-                                                      autoupload=autoupload,)
+        app: 'DetectorNode' = request.app
+        detections = await app.get_detections(raw_image=np_image,
+                                              camera_id=camera_id or mac or None,
+                                              tags=tags.split(',') if tags else [],
+                                              source=source,
+                                              autoupload=autoupload)
     except Exception as exc:
         logging.exception(f'Error during detection of image {file.filename}.')
         raise Exception(f'Error during detection of image {file.filename}.') from exc

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/helpers/log_conf.py

@@ -23,6 +23,11 @@ LOGGING_CONF = {
             'level': 'INFO',
             'propagate': False,
         },
+        'Node': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False,
+        },
     },
 }
 

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/node.py

@@ -18,6 +18,7 @@ from .data_exchanger import DataExchanger
 from .helpers import log_conf
 from .helpers.misc import ensure_socket_response, read_or_create_uuid
 from .loop_communication import LoopCommunicator
+from .rest import router
 
 
 class Node(FastAPI):
@@ -41,7 +42,7 @@ class Node(FastAPI):
         self.uuid = uuid or read_or_create_uuid(self.name)
         self.needs_login = needs_login
 
-        self.log = logging.getLogger()
+        self.log = logging.getLogger('Node')
         self.loop_communicator = LoopCommunicator()
         self.websocket_url = self.loop_communicator.websocket_url()
         self.data_exchanger = DataExchanger(None, self.loop_communicator)
@@ -56,6 +57,8 @@ class Node(FastAPI):
 
         self.repeat_task: Any = None
 
+        self.include_router(router)
+
     @property
     def sio_client(self) -> AsyncClient:
         if self._sio_client is None:

learning_loop_node-0.10.13/learning_loop_node/rest.py

@@ -0,0 +1,32 @@
+import logging
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter, Request, HTTPException
+
+if TYPE_CHECKING:
+    from .node import Node
+
+
+router = APIRouter()
+logger = logging.getLogger('Node.rest')
+
+
+@router.put("/debug_logging")
+async def _debug_logging(request: Request):
+    '''
+    Example Usage
+
+        curl -X PUT -d "on" http://localhost:8007/debug_logging
+    '''
+    state = str(await request.body(), 'utf-8')
+    node: 'Node' = request.app
+
+    if state == 'off':
+        logger.info('turning debug logging off')
+        node.log.setLevel('INFO')
+        return 'off'
+    if state == 'on':
+        logger.info('turning debug logging on')
+        node.log.setLevel('DEBUG')
+        return 'on'
+    raise HTTPException(status_code=400, detail='Invalid state')

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/tests/detector/conftest.py

@@ -12,6 +12,9 @@ import pytest
 import socketio
 import uvicorn
 
+from learning_loop_node.data_classes import BoxDetection, Detections
+from learning_loop_node.detector.detector_logic import DetectorLogic
+
 from ...detector.detector_node import DetectorNode
 from ...detector.outbox import Outbox
 from ...globals import GLOBALS
@@ -113,6 +116,37 @@ def get_outbox_files(outbox: Outbox):
     files = glob(f'{outbox.path}/**/*', recursive=True)
     return [file for file in files if os.path.isfile(file)]
 
+
+@pytest.fixture
+def mock_detector_logic():
+    class MockDetectorLogic(DetectorLogic):  # pylint: disable=abstract-method
+        def __init__(self):
+            super().__init__('mock')
+            self.detections = Detections(
+                box_detections=[BoxDetection(category_name="test",
+                                             category_id="1",
+                                             confidence=0.9,
+                                             x=0, y=0, width=10, height=10,
+                                             model_name="mock",
+                                             )]
+            )
+
+        @property
+        def is_initialized(self):
+            return True
+
+        def evaluate_with_all_info(self, image, tags, source):  # pylint: disable=signature-differs
+            return self.detections
+
+    return MockDetectorLogic()
+
+
+@pytest.fixture
+def detector_node(mock_detector_logic):
+    os.environ['ORGANIZATION'] = 'test_organization'
+    os.environ['PROJECT'] = 'test_project'
+    return DetectorNode(name="test_node", detector=mock_detector_logic)
+
 # ====================================== REDUNDANT FIXTURES IN ALL CONFTESTS ! ======================================
 
 

learning_loop_node-0.10.13/learning_loop_node/tests/detector/test_detector_node.py

@@ -0,0 +1,86 @@
+import numpy as np
+import pytest
+
+from learning_loop_node.detector.detector_node import DetectorNode
+
+
+@pytest.mark.asyncio
+async def test_get_detections(detector_node: DetectorNode, monkeypatch):
+    # Mock raw image data
+    raw_image = np.zeros((100, 100, 3), dtype=np.uint8)
+
+    # Mock relevance_filter and outbox
+    filtered_upload_called = False
+    save_called = False
+
+    save_args = []
+
+    def mock_filtered_upload(*args, **kwargs):  # pylint: disable=unused-argument
+        nonlocal filtered_upload_called
+        filtered_upload_called = True
+
+    def mock_save(*args, **kwargs):
+        nonlocal save_called
+        nonlocal save_args
+        save_called = True
+        save_args = (args, kwargs)
+
+    monkeypatch.setattr(detector_node.relevance_filter, "may_upload_detections", mock_filtered_upload)
+    monkeypatch.setattr(detector_node.outbox, "save", mock_save)
+
+    # Test cases
+    test_cases = [
+        (None, True, False),
+        ("filtered", True, False),
+        ("all", False, True),
+        ("disabled", False, False),
+    ]
+
+    expected_save_args = {
+        'image': raw_image,
+        'detections': detector_node.detector_logic.detections,  # type: ignore
+        'tags': ['test_tag'],
+        'source': 'test_source',
+    }
+
+    for autoupload, expect_filtered, expect_all in test_cases:
+        filtered_upload_called = False
+        save_called = False
+
+        result = await detector_node.get_detections(
+            raw_image=raw_image,
+            camera_id="test_camera",
+            tags=["test_tag"],
+            source="test_source",
+            autoupload=autoupload
+        )
+
+        # Check if detections were processed
+        assert result is not None
+        assert "box_detections" in result
+        assert len(result["box_detections"]) == 1
+        assert result["box_detections"][0]["category_name"] == "test"
+
+        # Check if the correct upload method was called
+        assert filtered_upload_called == expect_filtered
+        assert save_called == expect_all
+
+        if save_called:
+            save_pos_args, save_kwargs = save_args  # pylint: disable=unbalanced-tuple-unpacking
+            expected_values = list(expected_save_args.values())
+            assert len(save_pos_args) + len(save_kwargs) == len(expected_values)
+
+            # Check positional arguments
+            for arg, expected in zip(save_pos_args, expected_values[:len(save_pos_args)]):
+                if isinstance(arg, (list, np.ndarray)):
+                    assert np.array_equal(arg, expected)
+                else:
+                    assert arg == expected
+
+            # Check keyword arguments
+            for key, value in save_kwargs.items():
+                expected = expected_save_args[key]
+                if isinstance(value, (list, np.ndarray)):
+                    assert np.array_equal(value, expected)
+                else:
+                    assert value == expected

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/trainer/io_helpers.py

@@ -16,12 +16,9 @@ from ..loop_communication import LoopCommunicator
 
 class EnvironmentVars:
     def __init__(self) -> None:
-        self.restart_after_training = os.environ.get(
-            'RESTART_AFTER_TRAINING', 'FALSE').lower() in ['true', '1']
-        self.keep_old_trainings = os.environ.get(
-            'KEEP_OLD_TRAININGS', 'FALSE').lower() in ['true', '1']
-        self.inference_batch_size = int(
-            os.environ.get('INFERENCE_BATCH_SIZE', '10'))
+        self.restart_after_training = os.environ.get('RESTART_AFTER_TRAINING', 'FALSE').lower() in ['true', '1']
+        self.keep_old_trainings = os.environ.get('KEEP_OLD_TRAININGS', 'FALSE').lower() in ['true', '1']
+        self.inference_batch_size = int(os.environ.get('INFERENCE_BATCH_SIZE', '10'))
 
 
 class LastTrainingIO:

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/trainer/trainer_logic.py

@@ -25,7 +25,7 @@ class TrainerLogic(TrainerLogicGeneric):
         self._detection_progress: Optional[float] = None
         self._executor: Optional[Executor] = None
         self.start_training_task: Optional[Coroutine] = None
-        self.inference_batch_size = 10
+        self.inference_batch_size = self._environment_vars.inference_batch_size
 
     # ---------------------------------------- IMPLEMENTED ABSTRACT PROPERTIES ----------------------------------------
 
@@ -92,7 +92,7 @@ class TrainerLogic(TrainerLogicGeneric):
 
         shutil.rmtree(tmp_folder, ignore_errors=True)
         os.makedirs(tmp_folder)
-        logging.info(f'downloading detection model to {tmp_folder}')
+        logging.info('downloading detection model to %s', tmp_folder)
 
         await self.node.data_exchanger.download_model(tmp_folder, context, model_id, self.model_format)
         with open(f'{tmp_folder}/model.json', 'r') as f:
@@ -104,10 +104,10 @@ class TrainerLogic(TrainerLogicGeneric):
         image_ids = []
         for state, p in zip(['inbox', 'annotate', 'review', 'complete'], [0.1, 0.2, 0.3, 0.4]):
             self._detection_progress = p
-            logging.info(f'fetching image ids of {state}')
+            logging.info('fetching image ids of state %s', state)
             new_ids = await self.node.data_exchanger.fetch_image_uuids(query_params=f'state={state}')
             image_ids += new_ids
-            logging.info(f'downloading {len(new_ids)} images')
+            logging.info('downloading %d images', len(new_ids))
             await self.node.data_exchanger.download_images(new_ids, image_folder)
         self._detection_progress = 0.42
         # await delete_corrupt_images(image_folder)

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/trainer/trainer_logic_generic.py

@@ -210,7 +210,7 @@ class TrainerLogicGeneric(ABC):
 
         self._active_training_io = ActiveTrainingIO(
             self._training.training_folder, self.node.loop_communicator, context)
-        logger.info(f'new training initialized: {self._training}')
+        logger.info('new training initialized: %s', self._training)
 
     async def _run(self) -> None:
         """Called on `begin_training` event from the Learning Loop. 
@@ -229,8 +229,8 @@ class TrainerLogicGeneric(ABC):
                 self._may_restart()
             else:
                 logger.info('CancelledError in _run - shutting down')
-        except Exception as e:
-            logger.exception(f'Error in train: {e}')
+        except Exception:
+            logger.exception('(Ignored) exception in trainer_logic._run:')
 
     # ---------------------------------------- TRAINING STATES ----------------------------------------
 
@@ -271,7 +271,7 @@ class TrainerLogicGeneric(ABC):
         '''
 
         await asyncio.sleep(0.1)
-        logger.info(f'Performing state: {state_during}')
+        logger.info('Performing state: %s', state_during)
         previous_state = self.training.training_state
         self.training.training_state = state_during
         await asyncio.sleep(0.1)
@@ -283,12 +283,12 @@ class TrainerLogicGeneric(ABC):
 
         except asyncio.CancelledError:
             if self.shutdown_event.is_set():
-                logger.info(f'CancelledError in {state_during} - shutdown event set')
+                logger.info('CancelledError in %s - shutdown event set', state_during)
                 raise
-            logger.info(f'CancelledError in {state_during} - cleaning up')
+            logger.info('CancelledError in %s - cleaning up', state_during)
             self.training.training_state = TrainerState.ReadyForCleanup
         except CriticalError as e:
-            logger.error(f'CriticalError in {state_during} - Exception: {e}')
+            logger.error('CriticalError in %s - Exception: %s', state_during, e)
             self.errors.set(error_key, str(e))
             self.training.training_state = TrainerState.ReadyForCleanup
         except Exception as e:
@@ -297,7 +297,7 @@ class TrainerLogicGeneric(ABC):
             self.training.training_state = previous_state
             return
         else:
-            logger.info(f'Successfully finished state: {state_during}')
+            logger.info('Successfully finished state: %s', state_during)
             if not reset_early:
                 self.errors.reset(error_key)
             self.training.training_state = state_after

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/learning_loop_node/trainer/trainer_node.py

@@ -32,7 +32,7 @@ class TrainerNode(Node):
             self.log.info(
                 f'Trainer started with an idle_timeout of {self.idle_timeout} seconds. Note that shutdown does not work if docker container has the restart policy set to always')
 
-        if use_backdoor_controls:
+        if use_backdoor_controls or os.environ.get('USE_BACKDOOR_CONTROLS', '0').lower() in ('1', 'true'):
             self.include_router(backdoor_controls.router, tags=["controls"])
 
     # ----------------------------------- NODE LIVECYCLE METHODS --------------------------

{learning_loop_node-0.10.11 → learning_loop_node-0.10.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "learning_loop_node"
-version = "v0.10.11"
+version = "v0.10.13"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = ["Zauberzeug GmbH <info@zauberzeug.com>"]
 license = "MIT"