learning-loop-node 0.13.6__py3-none-any.whl → 0.14.0__py3-none-any.whl

learning_loop_node/data_exchanger.py

@@ -143,7 +143,7 @@ class DataExchanger():
         logging.info('Downloading model data for uuid %s from the loop to %s..', model_uuid, target_folder)
 
         path = f'/{context.organization}/projects/{context.project}/models/{model_uuid}/{model_format}/file'
-        response = await self.loop_communicator.get(path, requires_login=False)
+        response = await self.loop_communicator.get(path, requires_login=False, timeout=60*10)
         if response.status_code != 200:
             decoded_content = response.content.decode('utf-8')
             logging.error('could not download loop/%s: %s, content: %s', path,

learning_loop_node/detector/detector_logic.py

@@ -2,8 +2,6 @@ import logging
 from abc import abstractmethod
 from typing import List, Optional
 
-import numpy as np
-
 from ..data_classes import ImageMetadata, ModelInformation
 from ..globals import GLOBALS
 from .exceptions import NodeNeedsRestartError
@@ -44,13 +42,13 @@ class DetectorLogic():
     def init(self):
         """Called when a (new) model was loaded. Initialize the model. Model information available via `self.model_info`"""
 
-    def evaluate_with_all_info(self, image: np.ndarray, tags: List[str], source: Optional[str] = None, creation_date: Optional[str] = None) -> ImageMetadata:  # pylint: disable=unused-argument
+    def evaluate_with_all_info(self, image: bytes, tags: List[str], source: Optional[str] = None, creation_date: Optional[str] = None) -> ImageMetadata:  # pylint: disable=unused-argument
         """Called by the detector node when an image should be evaluated (REST or SocketIO).
         Tags, source come from the caller and may be used in this function. 
         By default, this function simply calls `evaluate`"""
         return self.evaluate(image)
 
     @abstractmethod
-    def evaluate(self, image: np.ndarray) -> ImageMetadata:
+    def evaluate(self, image: bytes) -> ImageMetadata:
         """Evaluate the image and return the detections.
         The object should return empty detections if it is not initialized"""

learning_loop_node/detector/detector_node.py

@@ -1,14 +1,12 @@
 import asyncio
 import contextlib
-import math
 import os
 import shutil
 import subprocess
 import sys
 from dataclasses import asdict
 from datetime import datetime
-from threading import Thread
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, cast
 
 import numpy as np
 import socketio
@@ -30,7 +28,7 @@ from ..data_classes.socket_response import SocketResponse
 from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
 from ..globals import GLOBALS
-from ..helpers import environment_reader
+from ..helpers import background_tasks, environment_reader, run
 from ..node import Node
 from .detector_logic import DetectorLogic
 from .exceptions import NodeNeedsRestartError
@@ -227,7 +225,7 @@ class DetectorNode(Node):
         async def detect(sid, data: Dict) -> Dict:
             try:
                 det = await self.get_detections(
-                    raw_image=np.frombuffer(data['image'], np.uint8),
+                    raw_image=data['image'],
                     camera_id=data.get('camera-id', None) or data.get('mac', None),
                     tags=data.get('tags', []),
                     source=data.get('source', None),
@@ -279,9 +277,10 @@ class DetectorNode(Node):
                 return {'error': str(e)}
 
         @self.sio.event
-        async def upload(sid, data: Dict) -> Optional[Dict]:
-            '''upload an image with detections'''
+        async def upload(sid, data: Dict) -> Dict:
+            """Upload an image with detections"""
 
+            self.log.debug('Processing upload via socketio.')
             detection_data = data.get('detections', {})
             if detection_data and self.detector_logic.model_info is not None:
                 try:
@@ -293,22 +292,19 @@ class DetectorNode(Node):
             else:
                 image_metadata = ImageMetadata()
 
-            tags = data.get('tags', [])
-            tags.append('picked_by_system')
-
-            source = data.get('source', None)
-            creation_date = data.get('creation_date', None)
-
-            self.log.debug('running upload via socketio. tags: %s, source: %s, creation_date: %s',
-                           tags, source, creation_date)
-
-            loop = asyncio.get_event_loop()
             try:
-                await loop.run_in_executor(None, self.outbox.save, data['image'], image_metadata, tags, source, creation_date)
+                await self.upload_images(
+                    images=[data['image']],
+                    image_metadata=image_metadata,
+                    tags=data.get('tags', []),
+                    source=data.get('source', None),
+                    creation_date=data.get('creation_date', None),
+                    upload_priority=data.get('upload_priority', False)
+                )
             except Exception as e:
                 self.log.exception('could not upload via socketio')
                 return {'error': str(e)}
-            return None
+            return {'status': 'OK'}
 
         @self.sio.event
         def connect(sid, environ, auth) -> None:
@@ -344,16 +340,21 @@ class DetectorNode(Node):
                 with step_into(GLOBALS.data_folder):
                     model_symlink = 'model'
                     target_model_folder = f'models/{self.target_model.version}'
-                    if not os.path.exists(target_model_folder):
-                        os.makedirs(target_model_folder)
-                        await self.data_exchanger.download_model(target_model_folder,
-                                                                 Context(organization=self.organization,
-                                                                         project=self.project),
-                                                                 self.target_model.id,
-                                                                 self.detector_logic.model_format)
-                        self.log.info('Downloaded model %s', self.target_model.version)
-                    else:
+                    if os.path.exists(target_model_folder) and len(os.listdir(target_model_folder)) > 0:
                         self.log.info('No need to download model %s (already exists)', self.target_model.version)
+                    else:
+                        os.makedirs(target_model_folder, exist_ok=True)
+                        try:
+                            await self.data_exchanger.download_model(target_model_folder,
+                                                                     Context(organization=self.organization,
+                                                                             project=self.project),
+                                                                     self.target_model.id,
+                                                                     self.detector_logic.model_format)
+                            self.log.info('Downloaded model %s', self.target_model.version)
+                        except Exception:
+                            self.log.exception('Could not download model %s', self.target_model.version)
+                            shutil.rmtree(target_model_folder, ignore_errors=True)
+                            return
                     try:
                         os.unlink(model_symlink)
                         os.remove(model_symlink)
@@ -422,16 +423,23 @@ class DetectorNode(Node):
         self.log_status_on_change(status.state or 'None', status)
 
         # NOTE: sending organization and project is no longer required!
-        response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
+        try:
+            response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
+        except TimeoutError:
+            self.socket_connection_broken = True
+            self.log.exception('TimeoutError for sending status update (will try to reconnect):')
+            raise Exception('Status update failed due to timeout') from None
+
         if not response:
             self.socket_connection_broken = True
-            return
+            self.log.error('Status update failed (will try to reconnect): %s', response)
+            raise Exception('Status update failed: Did not receive a response from the learning loop')
 
         socket_response = from_dict(data_class=SocketResponse, data=response)
         if not socket_response.success:
             self.socket_connection_broken = True
-            self.log.error('Statusupdate failed: %s', response)
-            raise Exception(f'Statusupdate failed: {response}')
+            self.log.error('Status update failed (will try to reconnect): %s', response)
+            raise Exception(f'Status update failed. Response from learning loop: {response}')
 
         assert socket_response.payload is not None
 
@@ -457,7 +465,7 @@ class DetectorNode(Node):
             self.log.warning('Operation mode set to %s, but sync failed: %s', mode, e)
 
     def reload(self, reason: str):
-        '''provide a cause for the reload'''
+        """provide a cause for the reload"""
 
         self.log.info('########## reloading app because %s', reason)
         if os.path.isfile('/app/app_code/restart/restart.py'):
@@ -470,7 +478,7 @@ class DetectorNode(Node):
             self.log.error('could not reload app')
 
     async def get_detections(self,
-                             raw_image: np.ndarray,
+                             raw_image: bytes,
                              camera_id: Optional[str],
                              tags: List[str],
                              source: Optional[str] = None,
@@ -482,8 +490,7 @@ class DetectorNode(Node):
         It can be converted e.g. using cv2.imdecode(raw_image, cv2.IMREAD_COLOR)"""
 
         await self.detection_lock.acquire()
-        loop = asyncio.get_event_loop()
-        detections = await loop.run_in_executor(None, self.detector_logic.evaluate_with_all_info, raw_image, tags, source, creation_date)
+        detections = await run.io_bound(self.detector_logic.evaluate_with_all_info, raw_image, tags, source, creation_date)
         self.detection_lock.release()
 
         fix_shape_detections(detections)
@@ -491,21 +498,40 @@ class DetectorNode(Node):
         n_po, n_se = len(detections.point_detections), len(detections.segmentation_detections)
         self.log.debug('Detected: %d boxes, %d points, %d segs, %d classes', n_bo, n_po, n_se, n_cl)
 
-        if autoupload is None or autoupload == 'filtered':  # NOTE default is filtered
-            Thread(target=self.relevance_filter.may_upload_detections,
-                   args=(detections, camera_id, raw_image, tags, source, creation_date)).start()
+        autoupload = autoupload or 'filtered'
+        if autoupload == 'filtered' and camera_id is not None:
+            background_tasks.create(self.relevance_filter.may_upload_detections(
+                detections, camera_id, raw_image, tags, source, creation_date
+            ))
         elif autoupload == 'all':
-            Thread(target=self.outbox.save, args=(raw_image, detections, tags, source, creation_date)).start()
+            background_tasks.create(self.outbox.save(raw_image, detections, tags, source, creation_date))
         elif autoupload == 'disabled':
             pass
         else:
             self.log.error('unknown autoupload value %s', autoupload)
         return detections
 
-    async def upload_images(self, images: List[bytes], source: Optional[str], creation_date: Optional[str]):
-        loop = asyncio.get_event_loop()
+    async def upload_images(
+            self, *,
+            images: List[bytes],
+            image_metadata: Optional[ImageMetadata] = None,
+            tags: Optional[List[str]] = None,
+            source: Optional[str],
+            creation_date: Optional[str],
+            upload_priority: bool = False
+    ) -> None:
+        """Save images to the outbox using an asyncio executor.
+        Used by SIO and REST upload endpoints."""
+
+        if image_metadata is None:
+            image_metadata = ImageMetadata()
+        if tags is None:
+            tags = []
+
+        tags.append('picked_by_system')
+
         for image in images:
-            await loop.run_in_executor(None, self.outbox.save, image, ImageMetadata(), ['picked_by_system'], source, creation_date)
+            await self.outbox.save(image, image_metadata, tags, source, creation_date, upload_priority)
 
     def add_category_id_to_detections(self, model_info: ModelInformation, image_metadata: ImageMetadata):
         def find_category_id_by_name(categories: List[Category], category_name: str):

learning_loop_node/detector/inbox_filter/relevance_filter.py

@@ -11,14 +11,16 @@ class RelevanceFilter():
         self.cam_histories: Dict[str, CamObservationHistory] = {}
         self.outbox: Outbox = outbox
 
-    def may_upload_detections(self,
-                              image_metadata: ImageMetadata,
-                              cam_id: str,
-                              raw_image: bytes,
-                              tags: List[str],
-                              source: Optional[str] = None,
-                              creation_date: Optional[str] = None
-                              ) -> List[str]:
+    async def may_upload_detections(self,
+                                    image_metadata: ImageMetadata,
+                                    cam_id: str,
+                                    raw_image: bytes,
+                                    tags: List[str],
+                                    source: Optional[str] = None,
+                                    creation_date: Optional[str] = None) -> List[str]:
+        """Check if the detection should be uploaded to the outbox.
+        If so, upload it and return the list of causes for the upload.
+        """
         for group in self.cam_histories.values():
             group.forget_old_detections()
 
@@ -30,5 +32,5 @@ class RelevanceFilter():
         if len(causes) > 0:
             tags = tags if tags is not None else []
             tags.extend(causes)
-            self.outbox.save(raw_image, image_metadata, tags, source, creation_date)
+            await self.outbox.save(raw_image, image_metadata, tags, source, creation_date)
         return causes

learning_loop_node/detector/outbox.py

@@ -5,13 +5,15 @@ import logging
 import os
 import shutil
 from asyncio import Task
+from collections import deque
 from dataclasses import asdict
 from datetime import datetime
 from glob import glob
 from io import BufferedReader, TextIOWrapper
 from multiprocessing import Event
 from multiprocessing.synchronize import Event as SyncEvent
-from typing import List, Optional, Tuple, Union
+from threading import Lock
+from typing import List, Optional, Tuple, TypeVar, Union
 
 import aiohttp
 import PIL
@@ -21,14 +23,27 @@ from fastapi.encoders import jsonable_encoder
 from ..data_classes import ImageMetadata
 from ..enums import OutboxMode
 from ..globals import GLOBALS
-from ..helpers import environment_reader
+from ..helpers import environment_reader, run
+
+T = TypeVar('T')
 
 
 class Outbox():
+    """
+    Outbox is a class that handles the uploading of images to the learning loop.
+    It uploads images from an internal queue (lifo) in batches of 20 every 5 seconds.
+    It handles upload failures by splitting the upload into two smaller batches until the problematic image is identified - and removed.
+    Any image can be saved to the normal or the priority queue.
+    Images in the priority queue are uploaded first.
+    The total queue length is limited to 1000 images.
+    """
+
     def __init__(self) -> None:
         self.log = logging.getLogger()
         self.path = f'{GLOBALS.data_folder}/outbox'
         os.makedirs(self.path, exist_ok=True)
+        os.makedirs(f'{self.path}/priority', exist_ok=True)
+        os.makedirs(f'{self.path}/normal', exist_ok=True)
 
         self.log = logging.getLogger()
         host = environment_reader.host()
@@ -42,6 +57,8 @@ class Outbox():
         self.log.info('Outbox initialized with target_uri: %s', self.target_uri)
 
         self.BATCH_SIZE = 20
+        self.MAX_UPLOAD_LENGTH = 1000  # only affects the `upload_folders` list
+        self.UPLOAD_INTERVAL_S = 5
         self.UPLOAD_TIMEOUT_S = 30
 
         self.shutdown_event: SyncEvent = Event()
@@ -49,15 +66,24 @@ class Outbox():
 
         self.upload_counter = 0
 
-    def save(self,
-             image: bytes,
-             image_metadata: Optional[ImageMetadata] = None,
-             tags: Optional[List[str]] = None,
-             source: Optional[str] = None,
-             creation_date: Optional[str] = None
-             ) -> None:
+        self.priority_upload_folders: List[str] = []
+        self.upload_folders: deque[str] = deque()
+        self.folders_lock = Lock()
+
+        for file in glob(f'{self.path}/priority/*'):
+            self.priority_upload_folders.append(file)
+        for file in glob(f'{self.path}/normal/*'):
+            self.upload_folders.append(file)
 
-        if not self._is_valid_jpg(image):
+    async def save(self,
+                   image: bytes,
+                   image_metadata: Optional[ImageMetadata] = None,
+                   tags: Optional[List[str]] = None,
+                   source: Optional[str] = None,
+                   creation_date: Optional[str] = None,
+                   upload_priority: bool = False) -> None:
+
+        if not await run.io_bound(self._is_valid_jpg, image):
             self.log.error('Invalid jpg image')
             return
 
@@ -66,9 +92,33 @@ class Outbox():
         if not tags:
             tags = []
         identifier = datetime.now().isoformat(sep='_', timespec='microseconds')
-        if os.path.exists(self.path + '/' + identifier):
-            self.log.error('Directory with identifier %s already exists', identifier)
+
+        try:
+            await run.io_bound(self._save_files_to_disk, identifier, image, image_metadata, tags, source, creation_date, upload_priority)
+        except Exception as e:
+            self.log.error('Failed to save files for image %s: %s', identifier, e)
             return
+
+        if upload_priority:
+            self.priority_upload_folders.append(f'{self.path}/priority/{identifier}')
+        else:
+            self.upload_folders.appendleft(f'{self.path}/normal/{identifier}')
+
+        await self._trim_upload_queue()
+
+    def _save_files_to_disk(self,
+                            identifier: str,
+                            image: bytes,
+                            image_metadata: ImageMetadata,
+                            tags: List[str],
+                            source: Optional[str],
+                            creation_date: Optional[str],
+                            upload_priority: bool) -> None:
+        subpath = 'priority' if upload_priority else 'normal'
+        full_path = f'{self.path}/{subpath}/{identifier}'
+        if os.path.exists(full_path):
+            raise FileExistsError(f'Directory with identifier {identifier} already exists')
+
         tmp = f'{GLOBALS.data_folder}/tmp/{identifier}'
         image_metadata.tags = tags
         if self._is_valid_isoformat(creation_date):
@@ -77,6 +127,7 @@ class Outbox():
             image_metadata.created = identifier
 
         image_metadata.source = source or 'unknown'
+
         os.makedirs(tmp, exist_ok=True)
 
         with open(tmp + f'/image_{identifier}.json', 'w') as f:
@@ -85,10 +136,34 @@ class Outbox():
         with open(tmp + f'/image_{identifier}.jpg', 'wb') as f:
             f.write(image)
 
-        if os.path.exists(tmp):
-            os.rename(tmp, self.path + '/' + identifier)  # NOTE rename is atomic so upload can run in parallel
-        else:
-            self.log.error('Could not rename %s to %s', tmp, self.path + '/' + identifier)
+        if not os.path.exists(tmp):
+            self.log.error('Could not rename %s to %s', tmp, full_path)
+            raise FileNotFoundError(f'Could not rename {tmp} to {full_path}')
+        os.rename(tmp, full_path)
+
+    async def _trim_upload_queue(self) -> None:
+        if len(self.upload_folders) > self.MAX_UPLOAD_LENGTH:
+            excess = len(self.upload_folders) - self.MAX_UPLOAD_LENGTH
+            self.log.info('Dropping %s images from upload list', excess)
+
+            folders_to_delete = []
+            for _ in range(excess):
+                if self.upload_folders:
+                    try:
+                        folder = self.upload_folders.pop()
+                        folders_to_delete.append(folder)
+                    except Exception:
+                        self.log.exception('Failed to get item from upload_folders')
+
+            await run.io_bound(self._delete_folders, folders_to_delete)
+
+    def _delete_folders(self, folders_to_delete: List[str]) -> None:
+        for folder in folders_to_delete:
+            try:
+                shutil.rmtree(folder)
+                self.log.debug('Deleted %s', folder)
+            except Exception:
+                self.log.exception('Failed to delete %s', folder)
 
     def _is_valid_isoformat(self, date: Optional[str]) -> bool:
         if date is None:
@@ -99,10 +174,11 @@ class Outbox():
         except Exception:
             return False
 
-    def get_data_files(self):
-        return glob(f'{self.path}/*')
+    def get_upload_folders(self) -> List[str]:
+        with self.folders_lock:
+            return self.priority_upload_folders + list(self.upload_folders)
 
-    def ensure_continuous_upload(self):
+    def ensure_continuous_upload(self) -> None:
         self.log.debug('start_continuous_upload')
         if self._upload_process_alive():
             self.log.debug('Upload thread already running')
@@ -111,44 +187,58 @@ class Outbox():
         self.shutdown_event.clear()
         self.upload_task = asyncio.create_task(self._continuous_upload())
 
-    async def _continuous_upload(self):
+    async def _continuous_upload(self) -> None:
         self.log.info('continuous upload started')
         assert self.shutdown_event is not None
         while not self.shutdown_event.is_set():
             await self.upload()
-            await asyncio.sleep(5)
+            await asyncio.sleep(self.UPLOAD_INTERVAL_S)
         self.log.info('continuous upload ended')
 
-    async def upload(self):
-        items = self.get_data_files()
+    async def upload(self) -> None:
+        items = self.get_upload_folders()
         if not items:
             self.log.debug('No images found to upload')
             return
 
         self.log.info('Found %s images to upload', len(items))
-        for i in range(0, len(items), self.BATCH_SIZE):
-            batch_items = items[i:i+self.BATCH_SIZE]
-            if self.shutdown_event.is_set():
-                break
-            try:
-                await self._upload_batch(batch_items)
-            except Exception:
-                self.log.exception('Could not upload files')
 
-    async def _upload_batch(self, items: List[str]):
+        batch_items = items[:self.BATCH_SIZE]
+        try:
+            await self._upload_batch(batch_items)
+        except Exception:
+            self.log.exception('Could not upload files')
 
-        # NOTE: keys are not relevant for the server, but using a fixed key like 'files'
-        # results in a post failure on the first run of the test in a docker environment (WTF)
+    async def _clear_item(self, item: str) -> None:
+        try:
+            if item in self.upload_folders:
+                self.upload_folders.remove(item)
+            if item in self.priority_upload_folders:
+                self.priority_upload_folders.remove(item)
+            await run.io_bound(shutil.rmtree, item, ignore_errors=True)
+            self.log.debug('Deleted %s', item)
+        except Exception:
+            self.log.exception('Failed to delete %s', item)
+
+    async def _upload_batch(self, items: List[str]) -> None:
+        """
+        Uploads a batch of images to the server.
+        :param items: List of folders to upload (each folder contains an image and a metadata file)
+        """
 
         data: List[Tuple[str, Union[TextIOWrapper, BufferedReader]]] = []
         for item in items:
+            if not os.path.exists(item):
+                await self._clear_item(item)
+                continue
             identifier = os.path.basename(item)
             data.append(('files', open(f'{item}/image_{identifier}.json', 'r')))
             data.append(('files', open(f'{item}/image_{identifier}.jpg', 'rb')))
 
         try:
             async with aiohttp.ClientSession() as session:
-                response = await session.post(self.target_uri, data=data, timeout=self.UPLOAD_TIMEOUT_S)
+                response = await session.post(self.target_uri, data=data, timeout=aiohttp.ClientTimeout(total=self.UPLOAD_TIMEOUT_S))
+                await response.read()
         except Exception:
             self.log.exception('Could not upload images')
             return
@@ -159,23 +249,23 @@ class Outbox():
 
         if response.status == 200:
             self.upload_counter += len(items)
+            self.log.debug('Uploaded %s images', len(items))
             for item in items:
-                try:
-                    shutil.rmtree(item)
-                    self.log.debug('Deleted %s', item)
-                except Exception:
-                    self.log.exception('Failed to delete %s', item)
-            self.log.info('Uploaded %s images successfully', len(items))
-
-        elif response.status == 422:
+                await self._clear_item(item)
+            self.log.debug('Cleared %s images', len(items))
+            return
+
+        if response.status == 422:
             if len(items) == 1:
                 self.log.error('Broken content in image: %s\n Skipping.', items[0])
-                shutil.rmtree(items[0], ignore_errors=True)
+                await self._clear_item(items[0])
                 return
 
             self.log.exception('Broken content in batch. Splitting and retrying')
             await self._upload_batch(items[:len(items)//2])
             await self._upload_batch(items[len(items)//2:])
+        elif response.status == 429:
+            self.log.warning('Too many requests: %s', response.content)
         else:
             self.log.error('Could not upload images: %s', response.content)
 

learning_loop_node/detector/rest/detect.py

@@ -1,7 +1,6 @@
 import logging
 from typing import TYPE_CHECKING, Optional
 
-import numpy as np
 from fastapi import APIRouter, File, Header, Request, UploadFile
 
 from ...data_classes.image_metadata import ImageMetadata
@@ -35,14 +34,15 @@ async def http_detect(
 
     """
     try:
-        np_image = np.fromfile(file.file, np.uint8)
+        # Read file directly to bytes instead of using numpy
+        file_bytes = file.file.read()
     except Exception as exc:
         logging.exception('Error during reading of image %s.', file.filename)
         raise Exception(f'Uploaded file {file.filename} is no image file.') from exc
 
     try:
         app: 'DetectorNode' = request.app
-        detections = await app.get_detections(raw_image=np_image,
+        detections = await app.get_detections(raw_image=file_bytes,
                                               camera_id=camera_id or mac or None,
                                               tags=tags.split(',') if tags else [],
                                               source=source,

learning_loop_node/detector/rest/upload.py

@@ -12,7 +12,8 @@ router = APIRouter()
 async def upload_image(request: Request,
                        files: List[UploadFile] = File(...),
                        source: Optional[str] = Query(None, description='Source of the image'),
-                       creation_date: Optional[str] = Query(None, description='Creation date of the image')):
+                       creation_date: Optional[str] = Query(None, description='Creation date of the image'),
+                       upload_priority: bool = Query(False, description='Upload the image with priority')):
     """
     Upload an image or multiple images to the learning loop.
 
@@ -21,9 +22,9 @@ async def upload_image(request: Request,
 
     Example Usage
 
-        curl -X POST -F 'files=@test.jpg' "http://localhost:/upload?source=test&creation_date=2024-01-01T00:00:00"
+        curl -X POST -F 'files=@test.jpg' "http://localhost:/upload?source=test&creation_date=2024-01-01T00:00:00&upload_priority=true"
     """
     raw_files = [await file.read() for file in files]
     node: DetectorNode = request.app
-    await node.upload_images(raw_files, source, creation_date)
+    await node.upload_images(images=raw_files, source=source, creation_date=creation_date, upload_priority=upload_priority)
     return 200, "OK"

learning_loop_node/helpers/background_tasks.py

@@ -0,0 +1,78 @@
+# Copy of Nicegui background_tasks.py
+# MIT License
+
+# Copyright (c) 2021 Zauberzeug GmbH
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""inspired from https://quantlane.com/blog/ensure-asyncio-task-exceptions-get-logged/"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Awaitable, Dict, Set
+
+running_tasks: Set[asyncio.Task] = set()
+lazy_tasks_running: Dict[str, asyncio.Task] = {}
+lazy_tasks_waiting: Dict[str, Awaitable] = {}
+
+
+def create(coroutine: Awaitable, *, name: str = 'unnamed task') -> asyncio.Task:
+    """Wraps a loop.create_task call and ensures there is an exception handler added to the task.
+
+    If the task raises an exception, it is logged and handled by the global exception handlers.
+    Also a reference to the task is kept until it is done, so that the task is not garbage collected mid-execution.
+    See https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task.
+    """
+    loop = asyncio.get_event_loop()
+    coroutine = coroutine if asyncio.iscoroutine(coroutine) else asyncio.wait_for(coroutine, None)
+    task: asyncio.Task = loop.create_task(coroutine, name=name)
+    task.add_done_callback(_handle_task_result)
+    running_tasks.add(task)
+    task.add_done_callback(running_tasks.discard)
+    return task
+
+
+def create_lazy(coroutine: Awaitable, *, name: str) -> None:
+    """Wraps a create call and ensures a second task with the same name is delayed until the first one is done.
+
+    If a third task with the same name is created while the first one is still running, the second one is discarded.
+    """
+    if name in lazy_tasks_running:
+        if name in lazy_tasks_waiting:
+            asyncio.Task(lazy_tasks_waiting[name]).cancel()
+        lazy_tasks_waiting[name] = coroutine
+        return
+
+    def finalize(name: str) -> None:
+        lazy_tasks_running.pop(name)
+        if name in lazy_tasks_waiting:
+            create_lazy(lazy_tasks_waiting.pop(name), name=name)
+    task = create(coroutine, name=name)
+    lazy_tasks_running[name] = task
+    task.add_done_callback(lambda _: finalize(name))
+
+
+def _handle_task_result(task: asyncio.Task) -> None:
+    try:
+        task.result()
+    except asyncio.CancelledError:
+        pass
+    except Exception:
+        logging.exception('Background task %s raised an exception', task.get_name())

learning_loop_node/helpers/run.py

@@ -0,0 +1,21 @@
+import asyncio
+import sys
+from typing import Any, Callable, TypeVar
+
+T = TypeVar('T')
+
+if sys.version_info >= (3, 10):
+    from typing import ParamSpec
+    P = ParamSpec('P')
+
+    async def io_bound(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
+        """Run a blocking function in a thread pool executor.
+        This is useful for disk I/O operations that would block the event loop."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, lambda: func(*args, **kwargs))
+else:
+    async def io_bound(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
+        """Run a blocking function in a thread pool executor.
+        This is useful for disk I/O operations that would block the event loop."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, lambda: func(*args, **kwargs))

learning_loop_node/loop_communication.py

@@ -100,31 +100,31 @@ class LoopCommunicator():
                 raise TimeoutError('Backend not ready within timeout')
             await asyncio.sleep(10)
 
-    async def retry_on_401(self, func: Callable[..., Awaitable[httpx.Response]], *args, **kwargs) -> httpx.Response:
+    async def _retry_on_401(self, func: Callable[..., Awaitable[httpx.Response]], *args, **kwargs) -> httpx.Response:
         response = await func(*args, **kwargs)
         if response.status_code == 401:
             await self.ensure_login(relogin=True)
             response = await func(*args, **kwargs)
         return response
 
-    async def get(self, path: str, requires_login: bool = True, api_prefix: str = '/api') -> httpx.Response:
+    async def get(self, path: str, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._get, path, api_prefix)
+            return await self._retry_on_401(self._get, path, api_prefix, timeout)
         return await self._get(path, api_prefix)
 
     @retry_on_429
-    async def _get(self, path: str, api_prefix: str) -> httpx.Response:
-        return await self.async_client.get(api_prefix+path)
+    async def _get(self, path: str, api_prefix: str, timeout: int = 60) -> httpx.Response:
+        return await self.async_client.get(api_prefix+path, timeout=timeout)
 
-    async def put(self, path: str, files: Optional[List[str]] = None, requires_login: bool = True, api_prefix: str = '/api', **kwargs) -> httpx.Response:
+    async def put(self, path: str, files: Optional[List[str]] = None, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60, **kwargs) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._put, path, files, api_prefix, **kwargs)
-        return await self._put(path, files, api_prefix, **kwargs)
+            return await self._retry_on_401(self._put, path, files, api_prefix, timeout, **kwargs)
+        return await self._put(path, files, api_prefix, timeout, **kwargs)
 
     @retry_on_429
-    async def _put(self, path: str, files: Optional[List[str]], api_prefix: str, **kwargs) -> httpx.Response:
+    async def _put(self, path: str, files: Optional[List[str]], api_prefix: str, timeout: int = 60, **kwargs) -> httpx.Response:
         if files is None:
             return await self.async_client.put(api_prefix+path, **kwargs)
 
@@ -139,29 +139,29 @@ class LoopCommunicator():
 
         try:
             file_list = [('files', fh) for fh in file_handles]  # Use file handles
-            response = await self.async_client.put(api_prefix+path, files=file_list)
+            response = await self.async_client.put(api_prefix+path, files=file_list, timeout=timeout)
         finally:
             for fh in file_handles:
                 fh.close()  # Ensure all files are closed
 
         return response
 
-    async def post(self, path: str, requires_login: bool = True, api_prefix: str = '/api', **kwargs) -> httpx.Response:
+    async def post(self, path: str, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60, **kwargs) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._post, path, api_prefix, **kwargs)
+            return await self._retry_on_401(self._post, path, api_prefix, timeout, **kwargs)
         return await self._post(path, api_prefix, **kwargs)
 
     @retry_on_429
-    async def _post(self, path, api_prefix='/api', **kwargs) -> httpx.Response:
-        return await self.async_client.post(api_prefix+path, **kwargs)
+    async def _post(self, path, api_prefix='/api', timeout: int = 60, **kwargs) -> httpx.Response:
+        return await self.async_client.post(api_prefix+path, timeout=timeout, **kwargs)
 
-    async def delete(self, path: str, requires_login: bool = True, api_prefix: str = '/api', **kwargs) -> httpx.Response:
+    async def delete(self, path: str, requires_login: bool = True, api_prefix: str = '/api', timeout: int = 60, **kwargs) -> httpx.Response:
         if requires_login:
             await self.ensure_login()
-            return await self.retry_on_401(self._delete, path, api_prefix, **kwargs)
+            return await self._retry_on_401(self._delete, path, api_prefix, timeout, **kwargs)
         return await self._delete(path, api_prefix, **kwargs)
 
     @retry_on_429
-    async def _delete(self, path, api_prefix, **kwargs) -> httpx.Response:
-        return await self.async_client.delete(api_prefix+path, **kwargs)
+    async def _delete(self, path, api_prefix, timeout: int = 60, **kwargs) -> httpx.Response:
+        return await self.async_client.delete(api_prefix+path, timeout=timeout, **kwargs)

learning_loop_node/node.py

@@ -76,6 +76,8 @@ class Node(FastAPI):
         self.previous_state: Optional[str] = None
         self.repeat_loop_cycle_sec = 5
 
+        self._client_session: Optional[aiohttp.ClientSession] = None
+
     def log_status_on_change(self, current_state_str: str, full_status: Any):
         if self.previous_state != current_state_str:
             self.previous_state = current_state_str
@@ -127,6 +129,8 @@ class Node(FastAPI):
         await self.loop_communicator.shutdown()
         if self._sio_client is not None:
             await self._sio_client.disconnect()
+        if self._client_session is not None:
+            await self._client_session.close()
         self.log.info('successfully disconnected from loop.')
         await self.on_shutdown()
 
@@ -205,12 +209,15 @@ class Node(FastAPI):
             ssl_context.verify_mode = ssl.CERT_REQUIRED
             connector = TCPConnector(ssl=ssl_context)
 
+        if self._client_session is not None:
+            await self._client_session.close()
+
         if self.needs_login:
-            self._sio_client = AsyncClient(request_timeout=20, http_session=aiohttp.ClientSession(
-                cookies=cookies, connector=connector))
+            self._client_session = aiohttp.ClientSession(cookies=cookies, connector=connector)
         else:
-            self._sio_client = AsyncClient(request_timeout=20, http_session=aiohttp.ClientSession(
-                connector=connector))
+            self._client_session = aiohttp.ClientSession(connector=connector)
+
+        self._sio_client = AsyncClient(request_timeout=20, http_session=self._client_session)
 
         # pylint: disable=protected-access
         self._sio_client._trigger_event = ensure_socket_response(self._sio_client._trigger_event)

learning_loop_node/tests/annotator/conftest.py

@@ -3,18 +3,23 @@ import logging
 import os
 import shutil
 
+# ====================================== REDUNDANT FIXTURES IN ALL CONFTESTS ! ======================================
+import sys
+
 import pytest
 
 from ...globals import GLOBALS
 from ...loop_communication import LoopCommunicator
 
-# ====================================== REDUNDANT FIXTURES IN ALL CONFTESTS ! ======================================
-
 
 @pytest.fixture()
 async def setup_test_project():  # pylint: disable=redefined-outer-name
     loop_communicator = LoopCommunicator()
-    await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_annotator?keep_images=true")
+    try:
+        await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_annotator?keep_images=true", timeout=10)
+    except Exception:
+        logging.warning("Failed to delete project pytest_nodelib_annotator")
+        sys.exit(1)
     await asyncio.sleep(1)
     project_conf = {
         'project_name': 'pytest_nodelib_annotator', 'inbox': 0, 'annotate': 0, 'review': 0, 'complete': 3, 'image_style': 'beautiful',
@@ -22,7 +27,7 @@ async def setup_test_project():  # pylint: disable=redefined-outer-name
         'trainings': 1, 'box_detections': 3, 'box_annotations': 0}
     assert (await loop_communicator.post("/zauberzeug/projects/generator", json=project_conf)).status_code == 200
     yield
-    await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_annotator?keep_images=true")
+    await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_annotator?keep_images=true", timeout=10)
     await loop_communicator.shutdown()
 
 

learning_loop_node/tests/annotator/test_annotator_node.py

@@ -7,7 +7,14 @@ from fastapi.encoders import jsonable_encoder
 
 from ...annotation.annotator_logic import AnnotatorLogic
 from ...annotation.annotator_node import AnnotatorNode
-from ...data_classes import AnnotationData, Category, Context, Point, ToolOutput, UserInput
+from ...data_classes import (
+    AnnotationData,
+    Category,
+    Context,
+    Point,
+    ToolOutput,
+    UserInput,
+)
 from ...enums import AnnotationEventType, CategoryType
 
 
@@ -37,7 +44,8 @@ def default_user_input() -> UserInput:
 
 
 @pytest.mark.asyncio
-async def test_image_download(setup_test_project):  # pylint: disable=unused-argument
+@pytest.mark.usefixtures('setup_test_project')
+async def test_image_download():
     image_folder = '/tmp/learning_loop_lib_data/zauberzeug/pytest_nodelib_annotator/images'
 
     assert os.path.exists(image_folder) is False or len(os.listdir(image_folder)) == 0

learning_loop_node/tests/detector/inbox_filter/test_unexpected_observations_count.py

@@ -24,10 +24,11 @@ l_conf_point_det = PointDetection(category_name='point', x=100, y=100,
       ['uncertain', 'unexpected_observations_count']),
      (ImageMetadata(box_detections=[h_conf_box_det], point_detections=[l_conf_point_det]),
       ['uncertain'])])
-def test_unexpected_observations_count(detections: ImageMetadata, reason: List[str]):
+@pytest.mark.asyncio
+async def test_unexpected_observations_count(detections: ImageMetadata, reason: List[str]):
     os.environ['LOOP_ORGANIZATION'] = 'zauberzeug'
     os.environ['LOOP_PROJECT'] = 'demo'
     outbox = Outbox()
 
-    r_filter = RelevanceFilter(outbox)
-    assert r_filter.may_upload_detections(detections, raw_image=b'', cam_id='0:0:0:0', tags=[]) == reason
+    relevance_filter = RelevanceFilter(outbox)
+    assert await relevance_filter.may_upload_detections(detections, raw_image=b'', cam_id='0:0:0:0', tags=[]) == reason

learning_loop_node/tests/detector/test_client_communication.py

@@ -84,7 +84,7 @@ async def test_sio_upload(test_detector_node: DetectorNode, sio_client):
     with open(test_image_path, 'rb') as f:
         image_bytes = f.read()
     result = await sio_client.call('upload', {'image': image_bytes})
-    assert result is None
+    assert result.get('status') == 'OK'
     assert len(get_outbox_files(test_detector_node.outbox)) == 2, 'There should be one image and one .json file.'
 
 
@@ -175,25 +175,3 @@ async def test_rest_outbox_mode(test_detector_node: DetectorNode):
     check_switch_to_mode('stopped')
     check_switch_to_mode('continuous_upload')
     check_switch_to_mode('stopped')
-
-
-async def test_api_responsive_during_large_upload(test_detector_node: DetectorNode):
-    assert len(get_outbox_files(test_detector_node.outbox)) == 0
-
-    with open(test_image_path, 'rb') as f:
-        image_bytes = f.read()
-
-    for _ in range(200):
-        test_detector_node.outbox.save(image_bytes)
-
-    outbox_size_early = len(get_outbox_files(test_detector_node.outbox))
-    await asyncio.sleep(5)  # NOTE: we wait 5 seconds because the continuous upload is running every 5 seconds
-
-    # check if api is still responsive
-    response = requests.get(f'http://localhost:{GLOBALS.detector_port}/outbox_mode', timeout=2)
-    assert response.status_code == 200, response.content
-
-    await asyncio.sleep(5)
-    outbox_size_late = len(get_outbox_files(test_detector_node.outbox))
-    assert outbox_size_late > 0, 'The outbox should not be fully cleared, maybe the node was too fast.'
-    assert outbox_size_early > outbox_size_late, 'The outbox should have been partially emptied.'

learning_loop_node/tests/detector/test_outbox.py

@@ -6,8 +6,6 @@ import shutil
 import pytest
 from PIL import Image
 
-from ...data_classes import ImageMetadata
-from ...detector.detector_node import DetectorNode
 from ...detector.outbox import Outbox
 from ...globals import GLOBALS
 
@@ -26,31 +24,24 @@ async def test_outbox():
     shutil.rmtree(test_outbox.path, ignore_errors=True)
 
 
-@pytest.mark.asyncio
-async def test_files_are_automatically_uploaded_by_node(test_detector_node: DetectorNode):
-    test_detector_node.outbox.save(get_test_image_binary(), ImageMetadata())
-    assert await wait_for_outbox_count(test_detector_node.outbox, 1)
-    assert await wait_for_outbox_count(test_detector_node.outbox, 0)
-
-
 @pytest.mark.asyncio
 async def test_set_outbox_mode(test_outbox: Outbox):
     await test_outbox.set_mode('stopped')
-    test_outbox.save(get_test_image_binary())
+    await test_outbox.save(get_test_image_binary())
     assert await wait_for_outbox_count(test_outbox, 1)
     await asyncio.sleep(6)
     assert await wait_for_outbox_count(test_outbox, 1), 'File was cleared even though outbox should be stopped'
 
     await test_outbox.set_mode('continuous_upload')
-    assert await wait_for_outbox_count(test_outbox, 0), 'File was not cleared even though outbox should be in continuous_upload'
+    assert await wait_for_outbox_count(test_outbox, 0, timeout=15), 'File was not cleared even though outbox should be in continuous_upload'
     assert test_outbox.upload_counter == 1
 
 
 @pytest.mark.asyncio
 async def test_outbox_upload_is_successful(test_outbox: Outbox):
-    test_outbox.save(get_test_image_binary())
+    await test_outbox.save(get_test_image_binary())
     await asyncio.sleep(1)
-    test_outbox.save(get_test_image_binary())
+    await test_outbox.save(get_test_image_binary())
     assert await wait_for_outbox_count(test_outbox, 2)
     await test_outbox.upload()
     assert await wait_for_outbox_count(test_outbox, 0)
@@ -60,8 +51,8 @@ async def test_outbox_upload_is_successful(test_outbox: Outbox):
 @pytest.mark.asyncio
 async def test_invalid_jpg_is_not_saved(test_outbox: Outbox):
     invalid_bytes = b'invalid jpg'
-    test_outbox.save(invalid_bytes)
-    assert len(test_outbox.get_data_files()) == 0
+    await test_outbox.save(invalid_bytes)
+    assert len(test_outbox.get_upload_folders()) == 0
 
 
 # ------------------------------ Helper functions --------------------------------------
@@ -90,7 +81,7 @@ def get_test_image_binary():
 
 async def wait_for_outbox_count(outbox: Outbox, count: int, timeout: int = 10) -> bool:
     for _ in range(timeout):
-        if len(outbox.get_data_files()) == count:
+        if len(outbox.get_upload_folders()) == count:
             return True
         await asyncio.sleep(1)
     return False

learning_loop_node/tests/general/conftest.py

@@ -2,6 +2,7 @@ import asyncio
 import logging
 import os
 import shutil
+import sys
 
 import pytest
 
@@ -15,7 +16,12 @@ from ...loop_communication import LoopCommunicator
 async def create_project_for_module():
 
     loop_communicator = LoopCommunicator()
-    await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_general?keep_images=true")
+    try:
+        await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_general", timeout=10)
+    except Exception:
+        logging.warning("Failed to delete project pytest_nodelib_general")
+        sys.exit(1)
+
     await asyncio.sleep(1)
     project_configuration = {
         'project_name': 'pytest_nodelib_general', 'inbox': 0, 'annotate': 0, 'review': 0, 'complete': 3, 'image_style': 'beautiful',
@@ -23,7 +29,7 @@ async def create_project_for_module():
         'trainings': 1, 'box_detections': 3, 'box_annotations': 0}
     assert (await loop_communicator.post("/zauberzeug/projects/generator", json=project_configuration)).status_code == 200
     yield
-    await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_general?keep_images=true")
+    await loop_communicator.delete("/zauberzeug/projects/pytest_nodelib_general", timeout=10)
     await loop_communicator.shutdown()
 
 

{learning_loop_node-0.13.6.dist-info → learning_loop_node-0.14.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.13.6
+Version: 0.14.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT
@@ -63,8 +63,8 @@ You can configure connection to our Learning Loop by specifying the following en
 | LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
 | LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
 | LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector                  |
-| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector (opt.)           |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  |
+| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector (opt.)           |
 | MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
 | MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
 | INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
@@ -73,6 +73,8 @@ You can configure connection to our Learning Loop by specifying the following en
 | TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
 | USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
 
+Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
+
 #### Testing
 
 We use github actions for CI. Tests can also be executed locally by running
@@ -103,6 +105,9 @@ The detector also has a sio **upload endpoint** that can be used to upload image
 - `image`: the image data in jpg format
 - `tags`: a list of strings. If not provided the tag is `picked_by_system`
 - `detections`: a dictionary representing the detections. UUIDs for the classes are automatically determined based on the category names. This field is optional. If not provided, no detections are uploaded.
+- `source`: optional source identifier for the image
+- `creation_date`: optional creation date for the image
+- `upload_priority`: boolean flag to prioritize the upload (defaults to False)
 
 The endpoint returns None if the upload was successful and an error message otherwise.
 
@@ -185,58 +190,52 @@ Upload a model with
 The model should now be available for the format 'format_a'
 `curl "https://learning-loop.ai/api/zauberzeug/projects/demo/models?format=format_a"`
 
-````
-
+```json
 {
-"models": [
-{
-"id": "3c20d807-f71c-40dc-a996-8a8968aa5431",
-"version": "4.0",
-"formats": [
-"format_a"
-],
-"created": "2021-06-01T06:28:21.289092",
-"comment": "uploaded at 2021-06-01 06:28:21.288442",
-...
+  "models": [
+    {
+    "id": "3c20d807-f71c-40dc-a996-8a8968aa5431",
+    "version": "4.0",
+    "formats": [
+      "format_a"
+    ],
+    "created": "2021-06-01T06:28:21.289092",
+    "comment": "uploaded at 2021-06-01 06:28:21.288442",
+    ...
+    }
+  ]
 }
-]
-}
-
 ```
 
 but not in the format_b
 `curl "https://learning-loop.ai/api/zauberzeug/projects/demo/models?format=format_b"`
 
-```
-
+```json
 {
-"models": []
+  "models": []
 }
-
 ```
 
 Connect the Node to the Learning Loop by simply starting the container.
 After a short time the converted model should be available as well.
 `curl https://learning-loop.ai/api/zauberzeug/projects/demo/models?format=format_b`
 
-```
-
-{
-"models": [
+```json
 {
-"id": "3c20d807-f71c-40dc-a996-8a8968aa5431",
-"version": "4.0",
-"formats": [
-"format_a",
-"format_b",
-],
-"created": "2021-06-01T06:28:21.289092",
-"comment": "uploaded at 2021-06-01 06:28:21.288442",
-...
-}
-]
+  "models": [
+  {
+  "id": "3c20d807-f71c-40dc-a996-8a8968aa5431",
+    "version": "4.0",
+    "formats": [
+      "format_a",
+      "format_b",
+    ],
+    "created": "2021-06-01T06:28:21.289092",
+    "comment": "uploaded at 2021-06-01 06:28:21.288442",
+    ...
+  }
+  ]
 }
-
 ```
 
 ## About Models (the currency between Nodes)
@@ -255,6 +254,4 @@ After a short time the converted model should be available as well.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
   - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, ` resolution_x`` resolution_y ` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings
-```
-````
 

{learning_loop_node-0.13.6.dist-info → learning_loop_node-0.14.0.dist-info}/RECORD

@@ -9,23 +9,23 @@ learning_loop_node/data_classes/general.py,sha256=r7fVfuQvbo8qOTT7zylgfM45TbIvYu
 learning_loop_node/data_classes/image_metadata.py,sha256=56nNSf_7aMlvKsJOG8vKCzJHcqKGHVRoULp85pJ2imA,1598
 learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
 learning_loop_node/data_classes/training.py,sha256=FFPsr2AA7ynYz39MLZaFJ0sF_9Axll5HHbAA8nnirp0,5726
-learning_loop_node/data_exchanger.py,sha256=IG5ki3f3IsVuXbyw6q_gUIakgv-GMT6e9nhOhzjKgW4,9055
+learning_loop_node/data_exchanger.py,sha256=2gV2epi24NQm8MgZKhi-sUNAP8CmcFLwihLagHxzKgA,9070
 learning_loop_node/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/detector/detector_logic.py,sha256=s1EFLrk_SFvLJOsIj9b0lp-Oq0DgfVWxT6Q34Vmi_JE,2243
-learning_loop_node/detector/detector_node.py,sha256=6MUdR_WzzcubwUAiaUo7TDF2xs-VkEvB_y1sNsrONYI,25461
+learning_loop_node/detector/detector_logic.py,sha256=AnXnAWzZfPMxRwKImNy2uiffnTacE3ArE4IxwxspgBU,2213
+learning_loop_node/detector/detector_node.py,sha256=TrBJlx9QEcyMYy4szVfw-g0xp9Yu5fdKgGWPJVGb4YQ,26629
 learning_loop_node/detector/exceptions.py,sha256=C6KbNPlSbtfgDrZx2Hbhm7Suk9jVoR3fMRCO0CkrMsQ,196
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=1PHgXRrhSQ34HSFw7mdX8ndRxHf_i1aP5nXXnrZxhAY,3312
-learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=NPEmrAtuGjIWCtHS0B3zDmnYWkhVFCLbd_7RUp08_AM,1372
-learning_loop_node/detector/outbox.py,sha256=i12X28FJka8HMm4iqE7SCODT1uCEtM53tIBul3uxKFw,8828
+learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=rI46jL9ZuI0hiDVxWCfXllB8DlQyyewNs6oZ6MnglMc,1540
+learning_loop_node/detector/outbox.py,sha256=KjQ2C8OokFtXtSOUKiYihADGI4QgkBX8QVRV109Bdr0,12716
 learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/rest/about.py,sha256=evHJ2svUZY_DFz0FSef5u9c5KW4Uc3GL7EbPinG9-dg,583
 learning_loop_node/detector/rest/backdoor_controls.py,sha256=ZNaFOvC0OLWNtcLiG-NIqS_y1kkLP4csgk3CHhp8Gis,885
-learning_loop_node/detector/rest/detect.py,sha256=ofJ3ysTarbCpiH1YAD6gSJbrDOzAcsLRuGxhr57dtk0,2503
+learning_loop_node/detector/rest/detect.py,sha256=wYf9cCgtImMgnHbrcE6GMXE2aBopdZciKvGmc92ZCGw,2533
 learning_loop_node/detector/rest/model_version_control.py,sha256=P4FOG0U9HT6QtCoNt-1s1pT6drtgdVjGZWEuCAyuNmA,1370
 learning_loop_node/detector/rest/operation_mode.py,sha256=1_xfutA_6nzdb4Q_jZiHQ5m_wA83bcG5jSIy-sfNIvk,1575
 learning_loop_node/detector/rest/outbox_mode.py,sha256=H8coDNbgLGEfXmKQrhtXWeUHBAHpnrdZktuHXQz0xis,1148
-learning_loop_node/detector/rest/upload.py,sha256=5YWY0Ku4duZqKd6tjyJzq-Ga83o2UYb1VmzuxBIgo0w,1061
+learning_loop_node/detector/rest/upload.py,sha256=GMDKyN3UNfzsKq5GtBBlv828lht0bztgqRqT_PQHkZM,1250
 learning_loop_node/enums/__init__.py,sha256=tjSrhztIQ8W656_QuXfTbbVNtH_wDXP5hpYZgzfgRhc,285
 learning_loop_node/enums/annotator.py,sha256=mtTAw-8LJIrHcYkBjYHCZuhYEEHS6QzSK8k6BhLusvQ,285
 learning_loop_node/enums/detector.py,sha256=Qvm5LWWR9BfsDxHEQ8YzaPaUuSmp4BescYuV4X4ikwE,512
@@ -34,34 +34,36 @@ learning_loop_node/enums/trainer.py,sha256=VaD63guLO4aKgVfXT0EryPlXKQGegSET3Cp4R
 learning_loop_node/examples/novelty_score_updater.py,sha256=1DRgM9lxjFV-q2JvGDDsNLz_ic_rhEZ9wc6ZdjcxwPE,2038
 learning_loop_node/globals.py,sha256=tgw_8RYOipPV9aYlyUhYtXfUxvJKRvfUk6u-qVAtZmY,174
 learning_loop_node/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/helpers/background_tasks.py,sha256=sNKyHyk9J5vNn-0GG1OzNJbB-F7GXGcbCWKE3MbRrno,3346
 learning_loop_node/helpers/environment_reader.py,sha256=6DxDJecLHxiGczByhyVa_JssAwwft7vuNCGaEzoSY2I,1662
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
 learning_loop_node/helpers/log_conf.py,sha256=hqVAa_9NnYEU6N0dcOKmph82p7MpgKqeF_eomTLYzWY,961
 learning_loop_node/helpers/misc.py,sha256=J29iBmsEUAraKKDN1m1NKiHQ3QrP5ub5HBU6cllSP2g,7384
-learning_loop_node/loop_communication.py,sha256=kc7GrkUS14Ka5OICaaOd_LZ61D-6O19GcyDEwckTxvM,7286
-learning_loop_node/node.py,sha256=IRV81q1G3-A6_BLNqB3NBT7T_dN5OXegBoM9JHMJuLM,11030
+learning_loop_node/helpers/run.py,sha256=_uox-j3_K_bL3yCAwy3JYSOiIxrnhzVxyxWpCe8_J9U,876
+learning_loop_node/loop_communication.py,sha256=opulqBKRLXlUQgjA3t0pg8CNA-JXJRCPPUspRxRuuGw,7556
+learning_loop_node/node.py,sha256=-Tw8kbvDKm8bPMm51MsFEOQKxPJx3n6DZ65cWGVQ5Zw,11262
 learning_loop_node/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/rest.py,sha256=omwlRHLnyG-kgCBVnZDk5_SAPobL9g7slWeX21wsPGw,1551
 learning_loop_node/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/annotator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/tests/annotator/conftest.py,sha256=G4ZvdZUdvPp9bYCzg3eEVkGCeXn9INZ3AcN7d5CyLkU,1931
+learning_loop_node/tests/annotator/conftest.py,sha256=e83I8WNAUgCFmum1GCx_nSjP9uwAoPIwPk72elypNQY,2098
 learning_loop_node/tests/annotator/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
-learning_loop_node/tests/annotator/test_annotator_node.py,sha256=UWRXRSBc1e795ftkp7xrEXbyR4LYvFDDHRpZGqC3vr8,1974
+learning_loop_node/tests/annotator/test_annotator_node.py,sha256=AuTqFvFyQYuxEdkNmjBZqBB7RYRgpoSuDsi7SjBVHfo,1997
 learning_loop_node/tests/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/detector/conftest.py,sha256=gut-RaacarhWJNCvGEz7O7kj3cS7vJ4SvAxCmR87PIw,5263
 learning_loop_node/tests/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/detector/inbox_filter/test_observation.py,sha256=k4WYdvnuV7d_r7zI4M2aA8WuBjm0aycQ0vj1rGE2q4w,1370
 learning_loop_node/tests/detector/inbox_filter/test_relevance_group.py,sha256=r-wABFQVsTNTjv7vYGr8wbHfOWy43F_B14ZDWHfiZ-A,7613
-learning_loop_node/tests/detector/inbox_filter/test_unexpected_observations_count.py,sha256=3KKwf-J9oJRMIuuVju2vT9IM9vWhKvswPiXJI8KxmcU,1661
+learning_loop_node/tests/detector/inbox_filter/test_unexpected_observations_count.py,sha256=JbUnPZVjzdtAlp6cTZVAdXUluQYNueGU9eITNJKY-tU,1710
 learning_loop_node/tests/detector/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/detector/test.jpg,sha256=msA-vHPmvPiro_D102Qmn1fn4vNfooqYYEXPxZUmYpk,161390
-learning_loop_node/tests/detector/test_client_communication.py,sha256=PUjnWnY-9RCZe-gqrtWf3o0ylCNH3WuzHoL7v3eAjAQ,8984
+learning_loop_node/tests/detector/test_client_communication.py,sha256=cVviUmAwbLY3LsJcY-D3ve-Jwxk9WVOrVupeh-PdKtA,8013
 learning_loop_node/tests/detector/test_detector_node.py,sha256=0ZMV6coAvdq-nH8CwY9_LR2tUcH9VLcAB1CWuwHQMpo,3023
-learning_loop_node/tests/detector/test_outbox.py,sha256=IfCz4iBmYA4bm3TK4q2NmWyzQCwZWhUbBrKQNHGxZM4,3007
+learning_loop_node/tests/detector/test_outbox.py,sha256=8L2k792oBhS82fnw2D7sw-Kh1vok_-4PzGjrK7r1WpM,2629
 learning_loop_node/tests/detector/test_relevance_filter.py,sha256=ZKcCstFWCDxJzKdVlAe8E6sZzv5NiH8mADhaZjokHoU,2052
 learning_loop_node/tests/detector/testing_detector.py,sha256=MZajybyzISz2G1OENfLHgZhBcLCYzTR4iN9JkWpq5-s,551
 learning_loop_node/tests/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/tests/general/conftest.py,sha256=oVuE-XZfUPjOXE8KLJgDbIMKIF9Mmgfna2rlupC44TE,2298
+learning_loop_node/tests/general/conftest.py,sha256=kEtkuVA2wgny-YBkLDn7Ff5j6ShOPghQUU0cH9IIl_8,2430
 learning_loop_node/tests/general/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/general/test_data/file_1.txt,sha256=Lis06nfvbFPVCBZyEgQlfI_Nle2YDq1GQBlYvEfFtxw,19
 learning_loop_node/tests/general/test_data/file_2.txt,sha256=Xp8EETGhZBdVAgb4URowSSpOytwwwJdV0Renkdur7R8,19
@@ -97,6 +99,6 @@ learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvg
 learning_loop_node/trainer/trainer_logic.py,sha256=eK-01qZzi10UjLMCQX8vy5eW2FoghPj3rzzDC-s3Si4,8792
 learning_loop_node/trainer/trainer_logic_generic.py,sha256=RQqon8JIVzxaNh0KdEe6tMxebsY0DgZllEohHR-AgqU,26846
 learning_loop_node/trainer/trainer_node.py,sha256=Dl4ZQAjjXQggibeBjvhXAoFClw1ZX2Kkt3v_fjrJnCI,4508
-learning_loop_node-0.13.6.dist-info/METADATA,sha256=z7dmbFXZUgUXgs-y0N-w-KwBDo8H2oVngoc1ja2Giok,12761
-learning_loop_node-0.13.6.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.13.6.dist-info/RECORD,,
+learning_loop_node-0.14.0.dist-info/METADATA,sha256=8gcoFu72XaljmTvPUIRXGk86GV0dhR9WGrGHM6zhRsQ,13186
+learning_loop_node-0.14.0.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.14.0.dist-info/RECORD,,