learning-loop-node 0.13.7__py3-none-any.whl → 0.15.0__py3-none-any.whl

learning_loop_node/data_classes/__init__.py

@@ -3,7 +3,7 @@ from .detections import (BoxDetection, ClassificationDetection, Detections, Obse
                          SegmentationDetection, Shape)
 from .general import (AboutResponse, AnnotationNodeStatus, Category, Context, DetectionStatus, ErrorConfiguration,
                       ModelInformation, ModelVersionResponse, NodeState, NodeStatus)
-from .image_metadata import ImageMetadata
+from .image_metadata import ImageMetadata, ImagesMetadata
 from .socket_response import SocketResponse
 from .training import Errors, PretrainedModel, Training, TrainingError, TrainingOut, TrainingStateData, TrainingStatus
 
@@ -12,7 +12,7 @@ __all__ = [
     'BoxDetection', 'ClassificationDetection', 'ImageMetadata', 'Observation', 'Point', 'PointDetection',
     'SegmentationDetection', 'Shape', 'Detections',
     'AnnotationNodeStatus', 'Category', 'Context', 'DetectionStatus', 'ErrorConfiguration',
-    'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse',
+    'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse', 'ImagesMetadata',
     'SocketResponse',
     'Errors', 'PretrainedModel', 'Training',
     'TrainingError', 'TrainingOut', 'TrainingStateData', 'TrainingStatus',

learning_loop_node/data_classes/image_metadata.py

@@ -35,3 +35,8 @@ class ImageMetadata():
 
     def __len__(self):
         return len(self.box_detections) + len(self.point_detections) + len(self.segmentation_detections) + len(self.classification_detections)
+
+
+@dataclass(**KWONLY_SLOTS)
+class ImagesMetadata():
+    items: List[ImageMetadata] = field(default_factory=list, metadata={'description': 'List of image metadata'})

learning_loop_node/data_classes/training.py

@@ -8,6 +8,7 @@ from uuid import uuid4
 
 from ..enums import TrainerState
 from ..helpers.misc import create_image_folder, create_training_folder
+
 # pylint: disable=no-name-in-module
 from .general import Category, Context
 
@@ -52,7 +53,7 @@ class Training():
     training_folder: str  # f'{project_folder}/trainings/{trainings_id}'
 
     categories: List[Category]
-    hyperparameters: dict
+    hyperparameters: Dict[str, Any]
 
     training_number: int
     training_state: str
@@ -63,7 +64,7 @@ class Training():
     base_model_uuid: Optional[str] = None  # model uuid to continue training (is loaded from loop)
 
     # NOTE: these are set later after the model has been uploaded
-    image_data: Optional[List[dict]] = None
+    image_data: Optional[List[Dict]] = None
     skipped_image_count: Optional[int] = None
     model_uuid_for_detecting: Optional[str] = None  # Model uuid to load from the loop after training and upload
 

learning_loop_node/data_exchanger.py

@@ -7,7 +7,7 @@ from glob import glob
 from http import HTTPStatus
 from io import BytesIO
 from time import time
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import aiofiles  # type: ignore
 
@@ -68,7 +68,7 @@ class DataExchanger():
         assert response.status_code == 200, response
         return (response.json())['image_ids']
 
-    async def download_images_data(self, image_uuids: List[str], chunk_size: int = 100) -> List[Dict]:
+    async def download_images_data(self, image_uuids: List[str], chunk_size: int = 100) -> List[Dict[str, Any]]:
         """Download image annotations, tags, set and other information for the given image uuids."""
         logging.info('Fetching annotations, tags, sets, etc. for %s images..', len(image_uuids))
 
@@ -78,7 +78,7 @@ class DataExchanger():
             return []
 
         progress_factor = 0.5 / num_image_ids  # first 50% of progress is for downloading data
-        images_data: List[Dict] = []
+        images_data: List[Dict[str, Any]] = []
         for i in range(0, num_image_ids, chunk_size):
             self.progress = i * progress_factor
             chunk_ids = image_uuids[i:i+chunk_size]

learning_loop_node/detector/detector_logic.py

@@ -2,9 +2,7 @@ import logging
 from abc import abstractmethod
 from typing import List, Optional
 
-import numpy as np
-
-from ..data_classes import ImageMetadata, ModelInformation
+from ..data_classes import ImageMetadata, ImagesMetadata, ModelInformation
 from ..globals import GLOBALS
 from .exceptions import NodeNeedsRestartError
 
@@ -44,13 +42,18 @@ class DetectorLogic():
     def init(self):
         """Called when a (new) model was loaded. Initialize the model. Model information available via `self.model_info`"""
 
-    def evaluate_with_all_info(self, image: np.ndarray, tags: List[str], source: Optional[str] = None, creation_date: Optional[str] = None) -> ImageMetadata:  # pylint: disable=unused-argument
+    def evaluate_with_all_info(self, image: bytes, tags: List[str], source: Optional[str] = None, creation_date: Optional[str] = None) -> ImageMetadata:  # pylint: disable=unused-argument
         """Called by the detector node when an image should be evaluated (REST or SocketIO).
         Tags, source come from the caller and may be used in this function. 
         By default, this function simply calls `evaluate`"""
         return self.evaluate(image)
 
     @abstractmethod
-    def evaluate(self, image: np.ndarray) -> ImageMetadata:
+    def evaluate(self, image: bytes) -> ImageMetadata:
         """Evaluate the image and return the detections.
         The object should return empty detections if it is not initialized"""
+
+    @abstractmethod
+    def batch_evaluate(self, images: List[bytes]) -> ImagesMetadata:
+        """Evaluate a batch of images and return the detections.
+        The object should return empty detections if it is not initialized"""

learning_loop_node/detector/detector_node.py

@@ -1,36 +1,25 @@
 import asyncio
 import contextlib
-import math
 import os
 import shutil
 import subprocess
 import sys
 from dataclasses import asdict
 from datetime import datetime
-from threading import Thread
 from typing import Dict, List, Optional
 
-import numpy as np
 import socketio
 from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
 from socketio import AsyncClient
 
-from ..data_classes import (
-    AboutResponse,
-    Category,
-    Context,
-    DetectionStatus,
-    ImageMetadata,
-    ModelInformation,
-    ModelVersionResponse,
-    Shape,
-)
+from ..data_classes import (AboutResponse, Category, Context, DetectionStatus, ImageMetadata, ImagesMetadata,
+                            ModelInformation, ModelVersionResponse, Shape)
 from ..data_classes.socket_response import SocketResponse
 from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
 from ..globals import GLOBALS
-from ..helpers import environment_reader
+from ..helpers import background_tasks, environment_reader, run
 from ..node import Node
 from .detector_logic import DetectorLogic
 from .exceptions import NodeNeedsRestartError
@@ -227,7 +216,7 @@ class DetectorNode(Node):
         async def detect(sid, data: Dict) -> Dict:
             try:
                 det = await self.get_detections(
-                    raw_image=np.frombuffer(data['image'], np.uint8),
+                    raw_image=data['image'],
                     camera_id=data.get('camera-id', None) or data.get('mac', None),
                     tags=data.get('tags', []),
                     source=data.get('source', None),
@@ -240,8 +229,29 @@ class DetectorNode(Node):
                 return detection_dict
             except Exception as e:
                 self.log.exception('could not detect via socketio')
-                with open('/tmp/bad_img_from_socket_io.jpg', 'wb') as f:
-                    f.write(data['image'])
+                # with open('/tmp/bad_img_from_socket_io.jpg', 'wb') as f:
+                #     f.write(data['image'])
+                return {'error': str(e)}
+
+        @self.sio.event
+        async def batch_detect(sid, data: Dict) -> Dict:
+            try:
+                det = await self.get_batch_detections(
+                    raw_images=data['images'],
+                    tags=data.get('tags', []),
+                    camera_id=data.get('camera-id', None) or data.get('mac', None),
+                    source=data.get('source', None),
+                    autoupload=data.get('autoupload', None),
+                    creation_date=data.get('creation_date', None)
+                )
+                if det is None:
+                    return {'error': 'no model loaded'}
+                detection_dict = jsonable_encoder(asdict(det))
+                return detection_dict
+            except Exception as e:
+                self.log.exception('could not detect via socketio')
+                # with open('/tmp/bad_img_from_socket_io.jpg', 'wb') as f:
+                #     f.write(data['image'])
                 return {'error': str(e)}
 
         @self.sio.event
@@ -279,9 +289,10 @@ class DetectorNode(Node):
                 return {'error': str(e)}
 
         @self.sio.event
-        async def upload(sid, data: Dict) -> Optional[Dict]:
-            '''upload an image with detections'''
+        async def upload(sid, data: Dict) -> Dict:
+            """Upload an image with detections"""
 
+            self.log.debug('Processing upload via socketio.')
             detection_data = data.get('detections', {})
             if detection_data and self.detector_logic.model_info is not None:
                 try:
@@ -293,22 +304,19 @@ class DetectorNode(Node):
             else:
                 image_metadata = ImageMetadata()
 
-            tags = data.get('tags', [])
-            tags.append('picked_by_system')
-
-            source = data.get('source', None)
-            creation_date = data.get('creation_date', None)
-
-            self.log.debug('running upload via socketio. tags: %s, source: %s, creation_date: %s',
-                           tags, source, creation_date)
-
-            loop = asyncio.get_event_loop()
             try:
-                await loop.run_in_executor(None, self.outbox.save, data['image'], image_metadata, tags, source, creation_date)
+                await self.upload_images(
+                    images=[data['image']],
+                    image_metadata=image_metadata,
+                    tags=data.get('tags', []),
+                    source=data.get('source', None),
+                    creation_date=data.get('creation_date', None),
+                    upload_priority=data.get('upload_priority', False)
+                )
             except Exception as e:
                 self.log.exception('could not upload via socketio')
                 return {'error': str(e)}
-            return None
+            return {'status': 'OK'}
 
         @self.sio.event
         def connect(sid, environ, auth) -> None:
@@ -469,7 +477,7 @@ class DetectorNode(Node):
             self.log.warning('Operation mode set to %s, but sync failed: %s', mode, e)
 
     def reload(self, reason: str):
-        '''provide a cause for the reload'''
+        """provide a cause for the reload"""
 
         self.log.info('########## reloading app because %s', reason)
         if os.path.isfile('/app/app_code/restart/restart.py'):
@@ -482,20 +490,20 @@ class DetectorNode(Node):
             self.log.error('could not reload app')
 
     async def get_detections(self,
-                             raw_image: np.ndarray,
-                             camera_id: Optional[str],
+                             raw_image: bytes,
                              tags: List[str],
+                             *,
+                             camera_id: Optional[str] = None,
                              source: Optional[str] = None,
                              autoupload: Optional[str] = None,
                              creation_date: Optional[str] = None) -> ImageMetadata:
         """ Main processing function for the detector node when an image is received via REST or SocketIO.
-        This function infers the detections from the image, cares about uploading to the loop and returns the detections as a dictionary.
+        This function infers the detections from the image, cares about uploading to the loop and returns the detections as ImageMetadata object.
         Note: raw_image is a numpy array of type uint8, but not in the correct shape!
         It can be converted e.g. using cv2.imdecode(raw_image, cv2.IMREAD_COLOR)"""
 
         await self.detection_lock.acquire()
-        loop = asyncio.get_event_loop()
-        detections = await loop.run_in_executor(None, self.detector_logic.evaluate_with_all_info, raw_image, tags, source, creation_date)
+        detections = await run.io_bound(self.detector_logic.evaluate_with_all_info, raw_image, tags, source, creation_date)
         self.detection_lock.release()
 
         fix_shape_detections(detections)
@@ -503,21 +511,74 @@ class DetectorNode(Node):
         n_po, n_se = len(detections.point_detections), len(detections.segmentation_detections)
         self.log.debug('Detected: %d boxes, %d points, %d segs, %d classes', n_bo, n_po, n_se, n_cl)
 
-        if autoupload is None or autoupload == 'filtered':  # NOTE default is filtered
-            Thread(target=self.relevance_filter.may_upload_detections,
-                   args=(detections, camera_id, raw_image, tags, source, creation_date)).start()
+        autoupload = autoupload or 'filtered'
+        if autoupload == 'filtered' and camera_id is not None:
+            background_tasks.create(self.relevance_filter.may_upload_detections(
+                detections, camera_id, raw_image, tags, source, creation_date
+            ))
         elif autoupload == 'all':
-            Thread(target=self.outbox.save, args=(raw_image, detections, tags, source, creation_date)).start()
+            background_tasks.create(self.outbox.save(raw_image, detections, tags, source, creation_date))
         elif autoupload == 'disabled':
             pass
         else:
             self.log.error('unknown autoupload value %s', autoupload)
         return detections
 
-    async def upload_images(self, images: List[bytes], source: Optional[str], creation_date: Optional[str]):
-        loop = asyncio.get_event_loop()
+    async def get_batch_detections(self,
+                                   raw_images: List[bytes],
+                                   tags: List[str],
+                                   *,
+                                   camera_id: Optional[str] = None,
+                                   source: Optional[str] = None,
+                                   autoupload: Optional[str] = None,
+                                   creation_date: Optional[str] = None) -> ImagesMetadata:
+        """ Processing function for the detector node when a a batch inference is requested via SocketIO.
+        This function infers the detections from all images, cares about uploading to the loop and returns the detections as a list of ImageMetadata."""
+
+        await self.detection_lock.acquire()
+        all_detections = await run.io_bound(self.detector_logic.batch_evaluate, raw_images)
+        self.detection_lock.release()
+
+        for detections, raw_image in zip(all_detections.items, raw_images):
+            fix_shape_detections(detections)
+            n_bo, n_cl = len(detections.box_detections), len(detections.classification_detections)
+            n_po, n_se = len(detections.point_detections), len(detections.segmentation_detections)
+            self.log.debug('Detected: %d boxes, %d points, %d segs, %d classes', n_bo, n_po, n_se, n_cl)
+
+            autoupload = autoupload or 'filtered'
+            if autoupload == 'filtered' and camera_id is not None:
+                background_tasks.create(self.relevance_filter.may_upload_detections(
+                    detections, camera_id, raw_image, tags, source, creation_date
+                ))
+            elif autoupload == 'all':
+                background_tasks.create(self.outbox.save(raw_image, detections, tags, source, creation_date))
+            elif autoupload == 'disabled':
+                pass
+            else:
+                self.log.error('unknown autoupload value %s', autoupload)
+        return all_detections
+
+    async def upload_images(
+            self, *,
+            images: List[bytes],
+            image_metadata: Optional[ImageMetadata] = None,
+            tags: Optional[List[str]] = None,
+            source: Optional[str],
+            creation_date: Optional[str],
+            upload_priority: bool = False
+    ) -> None:
+        """Save images to the outbox using an asyncio executor.
+        Used by SIO and REST upload endpoints."""
+
+        if image_metadata is None:
+            image_metadata = ImageMetadata()
+        if tags is None:
+            tags = []
+
+        tags.append('picked_by_system')
+
         for image in images:
-            await loop.run_in_executor(None, self.outbox.save, image, ImageMetadata(), ['picked_by_system'], source, creation_date)
+            await self.outbox.save(image, image_metadata, tags, source, creation_date, upload_priority)
 
     def add_category_id_to_detections(self, model_info: ModelInformation, image_metadata: ImageMetadata):
         def find_category_id_by_name(categories: List[Category], category_name: str):

learning_loop_node/detector/inbox_filter/relevance_filter.py

@@ -11,14 +11,16 @@ class RelevanceFilter():
         self.cam_histories: Dict[str, CamObservationHistory] = {}
         self.outbox: Outbox = outbox
 
-    def may_upload_detections(self,
-                              image_metadata: ImageMetadata,
-                              cam_id: str,
-                              raw_image: bytes,
-                              tags: List[str],
-                              source: Optional[str] = None,
-                              creation_date: Optional[str] = None
-                              ) -> List[str]:
+    async def may_upload_detections(self,
+                                    image_metadata: ImageMetadata,
+                                    cam_id: str,
+                                    raw_image: bytes,
+                                    tags: List[str],
+                                    source: Optional[str] = None,
+                                    creation_date: Optional[str] = None) -> List[str]:
+        """Check if the detection should be uploaded to the outbox.
+        If so, upload it and return the list of causes for the upload.
+        """
         for group in self.cam_histories.values():
             group.forget_old_detections()
 
@@ -30,5 +32,5 @@ class RelevanceFilter():
         if len(causes) > 0:
             tags = tags if tags is not None else []
             tags.extend(causes)
-            self.outbox.save(raw_image, image_metadata, tags, source, creation_date)
+            await self.outbox.save(raw_image, image_metadata, tags, source, creation_date)
         return causes