learning-loop-node 0.10.12__py3-none-any.whl → 0.10.14__py3-none-any.whl

learning_loop_node/detector/rest/about.py

@@ -1,25 +1,50 @@
 
-from typing import TYPE_CHECKING
+import sys
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Optional
 
 from fastapi import APIRouter, Request
 
+from ...data_classes import ModelInformation
+
 if TYPE_CHECKING:
     from ..detector_node import DetectorNode
+KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
 
 router = APIRouter()
 
 
-@router.get("/about")
+@dataclass(**KWONLY_SLOTS)
+class AboutResponse:
+    operation_mode: str = field(metadata={"description": "The operation mode of the detector node"})
+    state: Optional[str] = field(metadata={
+        "description": "The state of the detector node",
+        "example": "idle, online, detecting"})
+    model_info: Optional[ModelInformation] = field(metadata={
+        "description": "Information about the model of the detector node"})
+    target_model: Optional[str] = field(metadata={"description": "The target model of the detector node"})
+    version_control: str = field(metadata={
+        "description": "The version control mode of the detector node",
+        "example": "follow_loop, specific_version, pause"})
+
+
+@router.get("/about", response_model=AboutResponse)
 async def get_about(request: Request):
     '''
+    Get information about the detector node.
+
     Example Usage
-        curl http://localhost/about
+
+        curl http://hosturl/about
     '''
     app: 'DetectorNode' = request.app
 
-    return {
-        'operation_mode': app.operation_mode.value,
-        'state': app.status.state,
-        'model_info':  app.detector_logic._model_info,  # pylint: disable=protected-access
-        'target_model': app.target_model.version if app.target_model is not None else 'None',
-    }
+    response = AboutResponse(
+        operation_mode=app.operation_mode.value,
+        state=app.status.state,
+        model_info=app.detector_logic._model_info,  # pylint: disable=protected-access
+        target_model=app.target_model.version if app.target_model is not None else None,
+        version_control=app.version_control.value
+    )
+
+    return response

learning_loop_node/detector/rest/backdoor_controls.py

@@ -14,43 +14,24 @@ if TYPE_CHECKING:
 router = APIRouter()
 
 
-@router.put("/socketio")
-async def _socketio(request: Request):
+@router.post("/reset")
+async def _reset(request: Request):
     '''
+    Soft-Reset the detector node.
+
     Example Usage
 
-        curl -X PUT -d "on" http://localhost:8007/socketio
+        curl -X POST http://hosturl/reset
     '''
-    state = str(await request.body(), 'utf-8')
-    await _switch_socketio(state, request.app)
-
-
-async def _switch_socketio(state: str, detector_node: 'DetectorNode'):
-    if state == 'off':
-        logging.info('BC: turning socketio off')
-        await detector_node.sio_client.disconnect()
-    if state == 'on':
-        logging.info('BC: turning socketio on')
-        await detector_node.connect_sio()
-
-
-@router.post("/reset")
-async def _reset(request: Request):
     logging.info('BC: reset')
+    detector_node: 'DetectorNode' = request.app
+
     try:
         shutil.rmtree(GLOBALS.data_folder, ignore_errors=True)
         os.makedirs(GLOBALS.data_folder, exist_ok=True)
 
-        # get file dir
-        # restart_path = Path(os.path.realpath(__file__)) / 'restart' / 'restart.py'
-        # restart_path = Path(os.getcwd()).absolute() / 'app_code' / 'restart' / 'restart.py'
-        # restart_path.touch()
-        # assert isinstance(request.app, 'DetectorNode')
-        await request.app.soft_reload()
-
-        # assert isinstance(request.app, DetectorNode)
-        # request.app.reload(reason='------- reset was called from backdoor controls',)
-    except Exception as e:
-        logging.error(f'BC: could not reset: {e}')
+        await detector_node.soft_reload()
+    except Exception:
+        logging.exception('BC: could not reset:')
         return False
     return True

learning_loop_node/detector/rest/detect.py

@@ -1,45 +1,53 @@
 import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
 from fastapi import APIRouter, File, Header, Request, UploadFile
 from fastapi.responses import JSONResponse
 
+from ...data_classes.detections import Detections
+
+if TYPE_CHECKING:
+    from ..detector_node import DetectorNode
+
 router = APIRouter()
 
 
-@router.post("/detect")
+@router.post("/detect", response_model=Detections)
 async def http_detect(
     request: Request,
-    file: UploadFile = File(...),
-    camera_id: Optional[str] = Header(None),
-    mac: Optional[str] = Header(None),
-    tags: Optional[str] = Header(None),
-    autoupload: Optional[str] = Header(None),
+    file: UploadFile = File(..., description='The image file to run detection on'),
+    camera_id: Optional[str] = Header(None, description='The camera id (used by learning loop)'),
+    mac: Optional[str] = Header(None, description='The camera mac address (used by learning loop)'),
+    tags: Optional[str] = Header(None, description='Tags to add to the image (used by learning loop)'),
+    source: Optional[str] = Header(None, description='The source of the image (used by learning loop)'),
+    autoupload: Optional[str] = Header(None, description='Mode to decide whether to upload the image to the learning loop',
+                                       examples=['filtered', 'all', 'disabled']),
 ):
     """
-    Example Usage
+    Single image example:
+
+        curl --request POST -F 'file=@test.jpg' localhost:8004/detect -H 'autoupload: all' -H 'camera-id: front_cam' -H 'source: test' -H 'tags: test,test2'
 
-        curl --request POST -F 'file=@test.jpg' localhost:8004/detect
+    Multiple images example:
 
         for i in `seq 1 10`; do time curl --request POST -F 'file=@test.jpg' localhost:8004/detect; done
 
-        You can additionally provide the following camera parameters:
-          - `autoupload`: configures auto-submission to the learning loop; `filtered` (default), `all`, `disabled` (example curl parameter `-H 'autoupload: all'`)
-          - `camera-id`: a string which groups images for submission together (example curl parameter `-H 'camera-id: front_cam'`)
     """
     try:
         np_image = np.fromfile(file.file, np.uint8)
     except Exception as exc:
-        logging.exception(f'Error during reading of image {file.filename}.')
+        logging.exception('Error during reading of image %s.', file.filename)
         raise Exception(f'Uploaded file {file.filename} is no image file.') from exc
 
     try:
-        detections = await request.app.get_detections(raw_image=np_image,
-                                                      camera_id=camera_id or mac or None,
-                                                      tags=tags.split(',') if tags else [],
-                                                      autoupload=autoupload,)
+        app: 'DetectorNode' = request.app
+        detections = await app.get_detections(raw_image=np_image,
+                                              camera_id=camera_id or mac or None,
+                                              tags=tags.split(',') if tags else [],
+                                              source=source,
+                                              autoupload=autoupload)
     except Exception as exc:
-        logging.exception(f'Error during detection of image {file.filename}.')
+        logging.exception('Error during detection of image %s.', file.filename)
         raise Exception(f'Error during detection of image {file.filename}.') from exc
-    return JSONResponse(detections)
+    return detections

learning_loop_node/detector/rest/model_version_control.py

@@ -1,7 +1,9 @@
 
 import os
+import sys
+from dataclasses import dataclass, field
 from enum import Enum
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List
 
 from fastapi import APIRouter, HTTPException, Request
 
@@ -10,6 +12,7 @@ from ...globals import GLOBALS
 
 if TYPE_CHECKING:
     from ..detector_node import DetectorNode
+KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
 
 router = APIRouter()
 
@@ -20,9 +23,20 @@ class VersionMode(str, Enum):
     Pause = 'pause'  # will pause the updates
 
 
+@dataclass(**KWONLY_SLOTS)
+class ModelVersionResponse:
+    current_version: str = field(metadata={"description": "The version of the model currently used by the detector."})
+    target_version: str = field(metadata={"description": "The target model version set in the detector."})
+    loop_version: str = field(metadata={"description": "The target model version specified by the loop."})
+    local_versions: List[str] = field(metadata={"description": "The locally available versions of the model."})
+    version_control: str = field(metadata={"description": "The version control mode."})
+
+
 @router.get("/model_version")
 async def get_version(request: Request):
     '''
+    Get information about the model version control and the current model version.
+
     Example Usage
         curl http://localhost/model_version
     '''
@@ -35,28 +49,31 @@ async def get_version(request: Request):
     loop_version = app.loop_deployment_target.version if app.loop_deployment_target is not None else 'None'
 
     local_versions: list[str] = []
-
-    local_models = os.listdir(os.path.join(GLOBALS.data_folder, 'models'))
+    models_path = os.path.join(GLOBALS.data_folder, 'models')
+    local_models = os.listdir(models_path) if os.path.exists(models_path) else []
     for model in local_models:
         if model.replace('.', '').isdigit():
             local_versions.append(model)
 
-    return {
-        'current_version': current_version,
-        'target_version': target_version,
-        'loop_version': loop_version,
-        'local_versions': local_versions,
-        'version_control': app.version_control.value,
-    }
+    response = ModelVersionResponse(
+        current_version=current_version,
+        target_version=target_version,
+        loop_version=loop_version,
+        local_versions=local_versions,
+        version_control=app.version_control.value,
+    )
+    return response
 
 
 @router.put("/model_version")
 async def put_version(request: Request):
     '''
+    Set the model version control mode.
+
     Example Usage
-        curl -X PUT -d "follow_loop" http://localhost/model_version
-        curl -X PUT -d "pause" http://localhost/model_version
-        curl -X PUT -d "13.6" http://localhost/model_version
+        curl -X PUT -d "follow_loop" http://hosturl/model_version
+        curl -X PUT -d "pause" http://hosturl/model_version
+        curl -X PUT -d "13.6" http://hosturl/model_version
     '''
     app: 'DetectorNode' = request.app
     content = str(await request.body(), 'utf-8')

learning_loop_node/detector/rest/operation_mode.py

@@ -22,7 +22,10 @@ class OperationMode(str, Enum):
 @router.put("/operation_mode")
 async def put_operation_mode(request: Request):
     '''
+    Set the operation mode of the detector node.
+
     Example Usage
+
         curl -X PUT -d "check_for_updates" http://localhost/operation_mode
         curl -X PUT -d "detecting" http://localhost/operation_mode
     '''
@@ -34,22 +37,25 @@ async def put_operation_mode(request: Request):
         raise HTTPException(422, str(exc)) from exc
     node: DetectorNode = request.app
 
-    logging.info(f'current node state : {node.status.state}')
-    logging.info(f'current operation mode : {node.operation_mode.value}')
-    logging.info(f'target operation mode : {target_mode}')
+    logging.info('current node state : %s', node.status.state)
+    logging.info('current operation mode : %s', node.operation_mode.value)
+    logging.info('target operation mode : %s', target_mode)
     if target_mode == node.operation_mode:
         logging.info('operation mode already set')
         return "OK"
 
     await node.set_operation_mode(target_mode)
-    logging.info(f'operation mode set to : {target_mode}')
+    logging.info('operation mode set to : %s', target_mode)
     return "OK"
 
 
-@router.get("/operation_mode")
+@router.get("/operation_mode", response_class=PlainTextResponse)
 async def get_operation_mode(request: Request):
     '''
+    Get the operation mode of the detector node.
+
     Example Usage
+
         curl http://localhost/operation_mode
     '''
     return PlainTextResponse(request.app.operation_mode.value)

learning_loop_node/detector/rest/outbox_mode.py

@@ -6,10 +6,13 @@ from ..outbox import Outbox
 router = APIRouter()
 
 
-@router.get("/outbox_mode")
+@router.get("/outbox_mode", response_class=PlainTextResponse)
 async def get_outbox_mode(request: Request):
     '''
+    Get the outbox mode of the detector node.
+
     Example Usage
+
         curl http://localhost/outbox_mode
     '''
     outbox: Outbox = request.app.outbox
@@ -19,7 +22,10 @@ async def get_outbox_mode(request: Request):
 @router.put("/outbox_mode")
 async def put_outbox_mode(request: Request):
     '''
+    Set the outbox mode of the detector node.
+
     Example Usage
+
         curl -X PUT -d "continuous_upload" http://localhost/outbox_mode
         curl -X PUT -d "stopped" http://localhost/outbox_mode
     '''

learning_loop_node/helpers/log_conf.py

@@ -23,6 +23,11 @@ LOGGING_CONF = {
             'level': 'INFO',
             'propagate': False,
         },
+        'Node': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False,
+        },
     },
 }
 

learning_loop_node/node.py

@@ -8,7 +8,6 @@ from datetime import datetime
 from typing import Any, Optional
 
 import aiohttp
-import socketio
 from aiohttp import TCPConnector
 from fastapi import FastAPI
 from socketio import AsyncClient
@@ -18,6 +17,11 @@ from .data_exchanger import DataExchanger
 from .helpers import log_conf
 from .helpers.misc import ensure_socket_response, read_or_create_uuid
 from .loop_communication import LoopCommunicator
+from .rest import router
+
+
+class NodeConnectionError(Exception):
+    pass
 
 
 class Node(FastAPI):
@@ -41,9 +45,8 @@ class Node(FastAPI):
         self.uuid = uuid or read_or_create_uuid(self.name)
         self.needs_login = needs_login
 
-        self.log = logging.getLogger()
-        self.loop_communicator = LoopCommunicator()
-        self.websocket_url = self.loop_communicator.websocket_url()
+        self.log = logging.getLogger('Node')
+        self.init_loop_communicator()
         self.data_exchanger = DataExchanger(None, self.loop_communicator)
 
         self.startup_datetime = datetime.now()
@@ -55,6 +58,19 @@ class Node(FastAPI):
                             'nodeType': node_type}
 
         self.repeat_task: Any = None
+        self.socket_connection_broken = False
+        self._skip_repeat_loop = False
+
+        self.include_router(router)
+
+        self.CONNECTED_TO_LOOP = asyncio.Event()
+        self.DISCONNECTED_FROM_LOOP = asyncio.Event()
+
+        self.repeat_loop_lock = asyncio.Lock()
+
+    def init_loop_communicator(self):
+        self.loop_communicator = LoopCommunicator()
+        self.websocket_url = self.loop_communicator.websocket_url()
 
     @property
     def sio_client(self) -> AsyncClient:
@@ -66,7 +82,10 @@ class Node(FastAPI):
     @asynccontextmanager
     async def lifespan(self, app: FastAPI):  # pylint: disable=unused-argument
         try:
-            await self._on_startup()
+            try:
+                await self._on_startup()
+            except Exception:
+                self.log.exception('Fatal error during startup: %s')
             self.repeat_task = asyncio.create_task(self.repeat_loop())
             yield
         finally:
@@ -80,13 +99,10 @@ class Node(FastAPI):
 
     async def _on_startup(self):
         self.log.info('received "startup" lifecycle-event')
-        # activate_asyncio_warnings()
-        if self.needs_login:
-            await self.loop_communicator.backend_ready()
-            self.log.info('ensuring login')
-            await self.loop_communicator.ensure_login()
-        self.log.info('create sio client')
-        await self.create_sio_client()
+        try:
+            await self.reconnect_to_loop()
+        except Exception:
+            self.log.warning('Could not establish sio connection to loop during startup')
         self.log.info('done')
         await self.on_startup()
 
@@ -99,55 +115,93 @@ class Node(FastAPI):
         await self.on_shutdown()
 
     async def repeat_loop(self) -> None:
-        """NOTE: with the lifespan approach, we cannot use @repeat_every anymore :("""
         while True:
+            if self._skip_repeat_loop:
+                self.log.debug('node is muted, skipping repeat loop')
+                await asyncio.sleep(1)
+                continue
             try:
-                await self._on_repeat()
+                async with self.repeat_loop_lock:
+                    await self._ensure_sio_connection()
+                    await self.on_repeat()
             except asyncio.CancelledError:
                 return
-            except Exception as e:
-                self.log.exception(f'error in repeat loop: {e}')
+            except Exception:
+                self.log.exception('error in repeat loop')
+
             await asyncio.sleep(5)
 
-    async def _on_repeat(self):
-        if not self.sio_client.connected:
-            self.log.info('Reconnecting to loop via sio')
-            await self.connect_sio()
-            if not self.sio_client.connected:
-                self.log.warning('Could not connect to loop via sio')
-                return
-        await self.on_repeat()
+    async def _ensure_sio_connection(self):
+        if self.socket_connection_broken or self._sio_client is None or not self.sio_client.connected:
+            self.log.info('Reconnecting to loop via sio due to %s',
+                          'broken connection' if self.socket_connection_broken else 'no connection')
+            await self.reconnect_to_loop()
+
+    async def reconnect_to_loop(self):
+        """Initialize the loop communicator, log in if needed and reconnect to the loop via socket.io."""
+        self.init_loop_communicator()
+        if self.needs_login:
+            await self.loop_communicator.ensure_login(relogin=True)
+        try:
+            await self._reconnect_socketio()
+        except NodeConnectionError:
+            self.log.exception('Could not reset sio connection to loop')
+            self.socket_connection_broken = True
+            raise
+
+        self.socket_connection_broken = False
+
+    def set_skip_repeat_loop(self, value: bool):
+        self._skip_repeat_loop = value
+        self.log.info('node is muted: %s', value)
 
     # --------------------------------------------------- SOCKET.IO ---------------------------------------------------
 
-    async def create_sio_client(self):
-        """Create a socket.io client that communicates with the learning loop and register the events.
-        Note: The method is called in startup and soft restart of detector, so the _sio_client should always be available."""
+    async def _reconnect_socketio(self):
+        """Create a socket.io client, connect it to the learning loop and register its events.
+        The current client is disconnected and deleted if it already exists."""
+
+        self.log.debug('-------------- Connecting to loop via socket.io -------------------')
+        self.log.debug('HTTP Cookies: %s\n', self.loop_communicator.get_cookies())
+
+        if self._sio_client is not None:
+            try:
+                await self.sio_client.disconnect()
+                self.log.info('disconnected from loop via sio')
+                # NOTE: without waiting for the disconnect event, we might disconnect the next connection too early
+                await asyncio.wait_for(self.DISCONNECTED_FROM_LOOP.wait(), timeout=5)
+            except asyncio.TimeoutError:
+                self.log.warning(
+                    'Did not receive disconnect event from loop within 5 seconds.\nContinuing with new connection...')
+            except Exception as e:
+                self.log.warning('Could not disconnect from loop via sio: %s.\nIgnoring...', e)
+            self._sio_client = None
 
+        connector = None
         if self.loop_communicator.ssl_cert_path:
-            logging.info(f'SIO using SSL certificate path: {self.loop_communicator.ssl_cert_path}')
+            logging.info('SIO using SSL certificate path: %s', self.loop_communicator.ssl_cert_path)
             ssl_context = ssl.create_default_context(cafile=self.loop_communicator.ssl_cert_path)
             ssl_context.check_hostname = False
             ssl_context.verify_mode = ssl.CERT_REQUIRED
             connector = TCPConnector(ssl=ssl_context)
-            self._sio_client = AsyncClient(request_timeout=20,
-                                           http_session=aiohttp.ClientSession(cookies=self.loop_communicator.get_cookies(),
-                                                                              connector=connector))
 
-        else:
-            self._sio_client = AsyncClient(request_timeout=20,
-                                           http_session=aiohttp.ClientSession(cookies=self.loop_communicator.get_cookies()))
+        self._sio_client = AsyncClient(request_timeout=20, http_session=aiohttp.ClientSession(
+            cookies=self.loop_communicator.get_cookies(), connector=connector))
 
         # pylint: disable=protected-access
-        self.sio_client._trigger_event = ensure_socket_response(self.sio_client._trigger_event)
+        self._sio_client._trigger_event = ensure_socket_response(self._sio_client._trigger_event)
 
         @self._sio_client.event
         async def connect():
             self.log.info('received "connect" via sio from loop.')
+            self.CONNECTED_TO_LOOP.set()
+            self.DISCONNECTED_FROM_LOOP.clear()
 
         @self._sio_client.event
         async def disconnect():
             self.log.info('received "disconnect" via sio from loop.')
+            self.DISCONNECTED_FROM_LOOP.set()
+            self.CONNECTED_TO_LOOP.clear()
 
         @self._sio_client.event
         async def restart():
@@ -155,21 +209,15 @@ class Node(FastAPI):
             sys.exit(0)
 
         self.register_sio_events(self._sio_client)
-
-    async def connect_sio(self):
         try:
-            await self.sio_client.disconnect()
-        except Exception:
-            pass
-
-        self.log.info(f'(re)connecting to Learning Loop at {self.websocket_url}')
-        try:
-            await self.sio_client.connect(f"{self.websocket_url}", headers=self.sio_headers, socketio_path="/ws/socket.io")
-            self.log.info('connected to Learning Loop')
-        except socketio.exceptions.ConnectionError:  # type: ignore
-            self.log.warning('connection error')
-        except Exception:
-            self.log.exception(f'error while connecting to "{self.websocket_url}". Exception:')
+            await self._sio_client.connect(f"{self.websocket_url}", headers=self.sio_headers, socketio_path="/ws/socket.io")
+        except Exception as e:
+            self.log.exception('Could not connect socketio client to loop')
+            raise NodeConnectionError('Could not connect socketio client to loop') from e
+
+        if not self._sio_client.connected:
+            self.log.exception('Could not connect socketio client to loop')
+            raise NodeConnectionError('Could not connect socketio client to loop')
 
     # --------------------------------------------------- ABSTRACT METHODS ---------------------------------------------------
 

learning_loop_node/rest.py

@@ -0,0 +1,55 @@
+import logging
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter, HTTPException, Request
+
+if TYPE_CHECKING:
+    from .node import Node
+
+
+router = APIRouter()
+logger = logging.getLogger('Node.rest')
+
+
+@router.put("/debug_logging")
+async def _debug_logging(request: Request) -> str:
+    '''
+    Example Usage
+
+        curl -X PUT -d "on" http://localhost:8007/debug_logging
+    '''
+    state = str(await request.body(), 'utf-8')
+    node: 'Node' = request.app
+
+    if state == 'off':
+        logger.info('turning debug logging off')
+        node.log.setLevel('INFO')
+        return 'off'
+    if state == 'on':
+        logger.info('turning debug logging on')
+        node.log.setLevel('DEBUG')
+        return 'on'
+    raise HTTPException(status_code=400, detail='Invalid state')
+
+
+@router.put("/socketio")
+async def _socketio(request: Request) -> str:
+    '''
+    Enable or disable the socketio connection to the learning loop.
+    Not intended to be used outside of testing.
+
+    Example Usage
+
+        curl -X PUT -d "on" http://hosturl/socketio
+    '''
+    state = str(await request.body(), 'utf-8')
+    node: 'Node' = request.app
+
+    if state == 'off':
+        await node.sio_client.disconnect()
+        node.set_skip_repeat_loop(True)  # Prevent auto-reconnection
+        return 'off'
+    if state == 'on':
+        node.set_skip_repeat_loop(False)  # Allow auto-reconnection (1 sec delay)
+        return 'on'
+    raise HTTPException(status_code=400, detail='Invalid state')