learning-loop-node 0.15.0__tar.gz → 0.16.1__tar.gz

{learning_loop_node-0.15.0 → learning_loop_node-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.15.0
+Version: 0.16.1
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.15.0 → learning_loop_node-0.16.1}/learning_loop_node/annotation/annotator_node.py

@@ -18,7 +18,7 @@ from .annotator_logic import AnnotatorLogic
 class AnnotatorNode(Node):
 
     def __init__(self, name: str, annotator_logic: AnnotatorLogic, uuid: Optional[str] = None):
-        super().__init__(name, uuid, 'annotation_node')
+        super().__init__(name, uuid=uuid, node_type='annotation_node')
         self.tool = annotator_logic
         self.histories: Dict = {}
         annotator_logic.init(self)
@@ -35,6 +35,9 @@ class AnnotatorNode(Node):
             return self.tool.logout_user(sid)
 
     async def _handle_user_input(self, user_input_dict: Dict) -> str:
+        if not self.sio_client or not self.sio_client.connected:
+            raise ConnectionError('SocketIO client is not connected')
+
         user_input = from_dict(data_class=UserInput, data=user_input_dict)
 
         if user_input.data.key_up == 'Escape':
@@ -66,6 +69,9 @@ class AnnotatorNode(Node):
 
     async def send_status(self):
 
+        if not self.sio_client or not self.sio_client.connected:
+            raise ConnectionError('SocketIO client is not connected')
+
         status = AnnotationNodeStatus(
             id=self.uuid,
             name=self.name,

{learning_loop_node-0.15.0 → learning_loop_node-0.16.1}/learning_loop_node/data_classes/__init__.py

@@ -1,17 +1,43 @@
 from .annotations import AnnotationData, SegmentationAnnotation, ToolOutput, UserInput
-from .detections import (BoxDetection, ClassificationDetection, Detections, Observation, Point, PointDetection,
-                         SegmentationDetection, Shape)
-from .general import (AboutResponse, AnnotationNodeStatus, Category, Context, DetectionStatus, ErrorConfiguration,
-                      ModelInformation, ModelVersionResponse, NodeState, NodeStatus)
+from .detections import (
+    BoxDetection,
+    ClassificationDetection,
+    Detections,
+    Observation,
+    Point,
+    PointDetection,
+    SegmentationDetection,
+    Shape,
+)
+from .general import (
+    AboutResponse,
+    AnnotationNodeStatus,
+    Category,
+    Context,
+    DetectorStatus,
+    ErrorConfiguration,
+    ModelInformation,
+    ModelVersionResponse,
+    NodeState,
+    NodeStatus,
+)
 from .image_metadata import ImageMetadata, ImagesMetadata
 from .socket_response import SocketResponse
-from .training import Errors, PretrainedModel, Training, TrainingError, TrainingOut, TrainingStateData, TrainingStatus
+from .training import (
+    Errors,
+    PretrainedModel,
+    Training,
+    TrainingError,
+    TrainingOut,
+    TrainingStateData,
+    TrainingStatus,
+)
 
 __all__ = [
     'AboutResponse', 'AnnotationData', 'SegmentationAnnotation', 'ToolOutput', 'UserInput',
     'BoxDetection', 'ClassificationDetection', 'ImageMetadata', 'Observation', 'Point', 'PointDetection',
     'SegmentationDetection', 'Shape', 'Detections',
-    'AnnotationNodeStatus', 'Category', 'Context', 'DetectionStatus', 'ErrorConfiguration',
+    'AnnotationNodeStatus', 'Category', 'Context', 'DetectorStatus', 'ErrorConfiguration',
     'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse', 'ImagesMetadata',
     'SocketResponse',
     'Errors', 'PretrainedModel', 'Training',

{learning_loop_node-0.15.0 → learning_loop_node-0.16.1}/learning_loop_node/data_classes/general.py

@@ -148,8 +148,8 @@ class NodeState(str, Enum):
 class NodeStatus():
     id: str
     name: str
-    state: Optional[NodeState] = NodeState.Online
-    uptime: Optional[int] = 0
+    state: NodeState = NodeState.Online
+    uptime: int = 0
     errors: Dict = field(default_factory=dict)
     capabilities: List[str] = field(default_factory=list)
 
@@ -175,14 +175,13 @@ class AnnotationNodeStatus(NodeStatus):
 
 
 @dataclass(**KWONLY_SLOTS)
-class DetectionStatus():
-    id: str
+class DetectorStatus():
+    uuid: str
     name: str
+    state: NodeState
+    uptime: int
     model_format: str
-
-    state: Optional[NodeState] = None
-    errors: Optional[Dict] = None
-    uptime: Optional[int] = None
-    current_model: Optional[str] = None
-    target_model: Optional[str] = None
-    operation_mode: Optional[str] = None
+    current_model: Optional[str]
+    target_model: Optional[str]
+    errors: Dict
+    operation_mode: str

{learning_loop_node-0.15.0 → learning_loop_node-0.16.1}/learning_loop_node/detector/detector_node.py

@@ -13,9 +13,17 @@ from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
 from socketio import AsyncClient
 
-from ..data_classes import (AboutResponse, Category, Context, DetectionStatus, ImageMetadata, ImagesMetadata,
-                            ModelInformation, ModelVersionResponse, Shape)
-from ..data_classes.socket_response import SocketResponse
+from ..data_classes import (
+    AboutResponse,
+    Category,
+    Context,
+    DetectorStatus,
+    ImageMetadata,
+    ImagesMetadata,
+    ModelInformation,
+    ModelVersionResponse,
+    Shape,
+)
 from ..data_exchanger import DataExchanger, DownloadError
 from ..enums import OperationMode, VersionMode
 from ..globals import GLOBALS
@@ -37,7 +45,7 @@ from .rest import upload as rest_upload
 class DetectorNode(Node):
 
     def __init__(self, name: str, detector: DetectorLogic, uuid: Optional[str] = None, use_backdoor_controls: bool = False) -> None:
-        super().__init__(name, uuid, 'detector', False)
+        super().__init__(name, uuid=uuid, node_type='detector', needs_login=False, needs_sio=False)
         self.detector_logic = detector
         self.organization = environment_reader.organization()
         self.project = environment_reader.project()
@@ -64,6 +72,10 @@ class DetectorNode(Node):
         self.target_model: Optional[ModelInformation] = None
         self.loop_deployment_target: Optional[ModelInformation] = None
 
+        self._regular_status_sync_cycles: int = int(os.environ.get('SYNC_CYCLES', '6'))
+        """sync status every 6 cycles (6*10s = 1min)"""
+        self._repeat_cycles_to_next_sync: int = 0
+
         self.include_router(rest_detect.router, tags=["detect"])
         self.include_router(rest_upload.router, prefix="")
         self.include_router(rest_mode.router, tags=["operation_mode"])
@@ -74,7 +86,7 @@ class DetectorNode(Node):
         if use_backdoor_controls or os.environ.get('USE_BACKDOOR_CONTROLS', '0').lower() in ('1', 'true'):
             self.include_router(backdoor_controls.router)
 
-        self.setup_sio_server()
+        self._setup_sio_server()
 
     def get_about_response(self) -> AboutResponse:
         return AboutResponse(
@@ -190,13 +202,7 @@ class DetectorNode(Node):
         except Exception:
             self.log.exception("error during 'shutdown'")
 
-    async def on_repeat(self) -> None:
-        try:
-            await self._check_for_update()
-        except Exception:
-            self.log.exception("error during '_check_for_update'")
-
-    def setup_sio_server(self) -> None:
+    def _setup_sio_server(self) -> None:
         """The DetectorNode acts as a SocketIO server. This method sets up the server and defines the event handlers."""
         # pylint: disable=unused-argument
 
@@ -322,96 +328,22 @@ class DetectorNode(Node):
         def connect(sid, environ, auth) -> None:
             self.connected_clients.append(sid)
 
-    async def _check_for_update(self) -> None:
-        try:
-            self.log.debug('Current operation mode is %s', self.operation_mode)
-            try:
-                await self.sync_status_with_learning_loop()
-            except Exception:
-                self.log.exception('Sync with learning loop failed (could not check for updates):')
-                return
-
-            if self.operation_mode != OperationMode.Idle:
-                self.log.debug('not checking for updates; operation mode is %s', self.operation_mode)
-                return
-
-            self.status.reset_error('update_model')
-            if self.target_model is None:
-                self.log.debug('not checking for updates; no target model selected')
-                return
-
-            if self.detector_logic.model_info is not None:
-                current_version = self.detector_logic.model_info.version
-            else:
-                current_version = None
+# ================================== Repeat Cycle, sync and model updates ==================================
 
-            if current_version != self.target_model.version:
-                self.log.info('Current model "%s" needs to be updated to %s',
-                              current_version or "-", self.target_model.version)
-
-                with step_into(GLOBALS.data_folder):
-                    model_symlink = 'model'
-                    target_model_folder = f'models/{self.target_model.version}'
-                    if os.path.exists(target_model_folder) and len(os.listdir(target_model_folder)) > 0:
-                        self.log.info('No need to download model %s (already exists)', self.target_model.version)
-                    else:
-                        os.makedirs(target_model_folder, exist_ok=True)
-                        try:
-                            await self.data_exchanger.download_model(target_model_folder,
-                                                                     Context(organization=self.organization,
-                                                                             project=self.project),
-                                                                     self.target_model.id,
-                                                                     self.detector_logic.model_format)
-                            self.log.info('Downloaded model %s', self.target_model.version)
-                        except Exception:
-                            self.log.exception('Could not download model %s', self.target_model.version)
-                            shutil.rmtree(target_model_folder, ignore_errors=True)
-                            return
-                    try:
-                        os.unlink(model_symlink)
-                        os.remove(model_symlink)
-                    except Exception:
-                        pass
-                    os.symlink(target_model_folder, model_symlink)
-                    self.log.info('Updated symlink for model to %s', os.readlink(model_symlink))
-
-                    try:
-                        self.detector_logic.load_model_info_and_init_model()
-                    except NodeNeedsRestartError:
-                        self.log.error('Node needs restart')
-                        sys.exit(0)
-                    except Exception:
-                        self.log.exception('Could not load model, will retry download on next check')
-                        shutil.rmtree(target_model_folder, ignore_errors=True)
-                        return
-                    try:
-                        await self.sync_status_with_learning_loop()
-                    except Exception:
-                        pass
-                    # self.reload(reason='new model installed')
-
-        except Exception as e:
-            self.log.exception('check_for_update failed')
-            msg = e.cause if isinstance(e, DownloadError) else str(e)
-            self.status.set_error('update_model', f'Could not update model: {msg}')
-            try:
-                await self.sync_status_with_learning_loop()
-            except Exception:
-                pass
-
-    async def sync_status_with_learning_loop(self) -> None:
-        """Sync status of the detector with the Learning Loop.
-        The Learning Loop will respond with the model info of the deployment target.
-        If version_control is set to FollowLoop, the detector will update the target_model.
-        Return if the communication was successful.
-
-        Raises:
-            Exception: If the communication with the Learning Loop failed.
-        """
+    async def on_repeat(self) -> None:
+        """Implementation of the repeat cycle. This method is called every 10 seconds.
+        To avoid too many requests, the status is only synced every 6 cycles (1 minute)."""
+        try:
+            self._repeat_cycles_to_next_sync -= 1
+            if self._repeat_cycles_to_next_sync <= 0:
+                self._repeat_cycles_to_next_sync = self._regular_status_sync_cycles
+                await self._sync_status_with_loop()
+            await self._update_model_if_required()
+        except Exception:
+            self.log.exception("error during '_check_for_update'")
 
-        if not self.sio_client.connected:
-            self.log.info('Status sync failed: not connected')
-            raise Exception('Status sync failed: not connected')
+    async def _sync_status_with_loop(self) -> None:
+        """Sync status of the detector with the Learning Loop."""
 
         if self.detector_logic.model_info is not None:
             current_model = self.detector_logic.model_info.version
@@ -420,8 +352,8 @@ class DetectorNode(Node):
 
         target_model_version = self.target_model.version if self.target_model else None
 
-        status = DetectionStatus(
-            id=self.uuid,
+        status = DetectorStatus(
+            uuid=self.uuid,
             name=self.name,
             state=self.status.state,
             errors=self.status.errors,
@@ -432,49 +364,128 @@ class DetectorNode(Node):
             model_format=self.detector_logic.model_format,
         )
 
-        self.log_status_on_change(status.state or 'None', status)
+        self.log_status_on_change(status.state, status)
+
+        try:
+            response = await self.loop_communicator.post(
+                f'/{self.organization}/projects/{self.project}/detectors', json=jsonable_encoder(asdict(status)))
+        except Exception:
+            self.log.warning('Exception while trying to sync status with loop')
+
+        if response.status_code != 200:
+            self.log.warning('Status update failed: %s', str(response))
 
-        # NOTE: sending organization and project is no longer required!
+    async def _update_model_if_required(self) -> None:
+        """Check if a new model is available and update if necessary.
+        The Learning Loop will respond with the model info of the deployment target.
+        If version_control is set to FollowLoop or the chosen target model is not used, 
+        the detector will update the target_model."""
         try:
-            response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
-        except TimeoutError:
-            self.socket_connection_broken = True
-            self.log.exception('TimeoutError for sending status update (will try to reconnect):')
-            raise Exception('Status update failed due to timeout') from None
-
-        if not response:
-            self.socket_connection_broken = True
-            self.log.error('Status update failed (will try to reconnect): %s', response)
-            raise Exception('Status update failed: Did not receive a response from the learning loop')
-
-        socket_response = from_dict(data_class=SocketResponse, data=response)
-        if not socket_response.success:
-            self.socket_connection_broken = True
-            self.log.error('Status update failed (will try to reconnect): %s', response)
-            raise Exception(f'Status update failed. Response from learning loop: {response}')
-
-        assert socket_response.payload is not None
-
-        deployment_target_model_id = socket_response.payload['target_model_id']
-        deployment_target_model_version = socket_response.payload['target_model_version']
+            if self.operation_mode != OperationMode.Idle:
+                self.log.debug('not checking for updates; operation mode is %s', self.operation_mode)
+                return
+
+            await self._check_for_new_deployment_target()
+
+            self.status.reset_error('update_model')
+            if self.target_model is None:
+                self.log.debug('not running any updates; target model is None')
+                return
+
+            current_version = self.detector_logic.model_info.version \
+                if self.detector_logic.model_info is not None else None
+
+            if current_version != self.target_model.version:
+                self.log.info('Updating model from %s to %s',
+                              current_version or "-", self.target_model.version)
+                await self._update_model(self.target_model)
+
+        except Exception as e:
+            self.log.exception('check_for_update failed')
+            msg = e.cause if isinstance(e, DownloadError) else str(e)
+            self.status.set_error('update_model', f'Could not update model: {msg}')
+            await self._sync_status_with_loop()
+
+    async def _check_for_new_deployment_target(self) -> None:
+        """Ask the learning loop for the current deployment target and update self.loop_deployment_target.
+        If version_control is set to FollowLoop, also update target_model."""
+        try:
+            response = await self.loop_communicator.get(
+                f'/{self.organization}/projects/{self.project}/deployment/target')
+        except Exception:
+            self.log.warning('Exception while trying to check for new deployment target')
+            return
+
+        if response.status_code != 200:
+            self.log.warning('Failed to check for new deployment target: %s', str(response))
+            return
+
+        response_data = response.json()
+
+        deployment_target_uuid = response_data['model_uuid']
+        deployment_target_version = response_data['version']
         self.loop_deployment_target = ModelInformation(organization=self.organization, project=self.project,
                                                        host="", categories=[],
-                                                       id=deployment_target_model_id,
-                                                       version=deployment_target_model_version)
+                                                       id=deployment_target_uuid,
+                                                       version=deployment_target_version)
 
         if (self.version_control == VersionMode.FollowLoop and
                 self.target_model != self.loop_deployment_target):
-            old_target_model_version = self.target_model.version if self.target_model else None
+            previous_version = self.target_model.version if self.target_model else None
             self.target_model = self.loop_deployment_target
-            self.log.info('After sending status. Target_model changed from %s to %s',
-                          old_target_model_version, self.target_model.version)
+            self.log.info('Deployment target changed from %s to %s',
+                          previous_version, self.target_model.version)
+
+    async def _update_model(self, target_model: ModelInformation) -> None:
+        """Download and install the target model.
+        On failure, the target_model will be set to None which will trigger a retry on the next check."""
+
+        with step_into(GLOBALS.data_folder):
+            target_model_folder = f'models/{target_model.version}'
+            if os.path.exists(target_model_folder) and len(os.listdir(target_model_folder)) > 0:
+                self.log.info('No need to download model. %s (already exists)', target_model.version)
+            else:
+                os.makedirs(target_model_folder, exist_ok=True)
+                try:
+                    await self.data_exchanger.download_model(target_model_folder,
+                                                             Context(organization=self.organization,
+                                                                     project=self.project),
+                                                             target_model.id, self.detector_logic.model_format)
+                    self.log.info('Downloaded model %s', target_model.version)
+                except Exception:
+                    self.log.exception('Could not download model %s', target_model.version)
+                    shutil.rmtree(target_model_folder, ignore_errors=True)
+                    self.target_model = None
+                    return
+
+            model_symlink = 'model'
+            try:
+                os.unlink(model_symlink)
+                os.remove(model_symlink)
+            except Exception:
+                pass
+            os.symlink(target_model_folder, model_symlink)
+            self.log.info('Updated symlink for model to %s', os.readlink(model_symlink))
+
+            try:
+                self.detector_logic.load_model_info_and_init_model()
+            except NodeNeedsRestartError:
+                self.log.error('Node needs restart')
+                sys.exit(0)
+            except Exception:
+                self.log.exception('Could not load model, will retry download on next check')
+                shutil.rmtree(target_model_folder, ignore_errors=True)
+                self.target_model = None
+                return
+
+            await self._sync_status_with_loop()
+            # self.reload(reason='new model installed')
+
+# ================================== API Implementations ==================================
 
     async def set_operation_mode(self, mode: OperationMode):
         self.operation_mode = mode
-        try:
-            await self.sync_status_with_learning_loop()
-        except Exception as e:
-            self.log.warning('Operation mode set to %s, but sync failed: %s', mode, e)
+        await self._sync_status_with_loop()
 
     def reload(self, reason: str):
         """provide a cause for the reload"""

{learning_loop_node-0.15.0 → learning_loop_node-0.16.1}/learning_loop_node/detector/outbox.py

@@ -189,7 +189,7 @@ class Outbox():
 
     async def _continuous_upload(self) -> None:
         self.log.info('continuous upload started')
-        assert self.shutdown_event is not None
+        assert self.shutdown_event is not None, 'shutdown_event is None'
         while not self.shutdown_event.is_set():
             await self.upload()
             await asyncio.sleep(self.UPLOAD_INTERVAL_S)
@@ -287,7 +287,7 @@ class Outbox():
             return True
 
         try:
-            assert self.shutdown_event is not None
+            assert self.shutdown_event is not None, 'shutdown_event is None'
             self.shutdown_event.set()
             await asyncio.wait_for(self.upload_task, timeout=self.UPLOAD_TIMEOUT_S + 1)
         except asyncio.TimeoutError:

{learning_loop_node-0.15.0 → learning_loop_node-0.16.1}/learning_loop_node/loop_communication.py

@@ -8,7 +8,8 @@ from httpx import Cookies, Timeout
 
 from .helpers import environment_reader
 
-logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger('loop_communication')
+logging.getLogger("httpx").setLevel(logging.WARNING)
 
 SLEEP_TIME_ON_429 = 5
 MAX_RETRIES_ON_429 = 20
@@ -37,9 +38,9 @@ class LoopCommunicator():
         host: str = environment_reader.host(default='learning-loop.ai')
         self.ssl_cert_path = environment_reader.ssl_certificate_path()
         if self.ssl_cert_path:
-            logging.info('Using SSL certificate at %s', self.ssl_cert_path)
+            logger.info('Using SSL certificate at %s', self.ssl_cert_path)
         else:
-            logging.info('No SSL certificate path set')
+            logger.info('No SSL certificate path set')
         self.host: str = host
         self.username: str = environment_reader.username()
         self.password: str = environment_reader.password()
@@ -52,7 +53,7 @@ class LoopCommunicator():
         else:
             self.async_client = httpx.AsyncClient(base_url=self.base_url, timeout=Timeout(60.0))
 
-        logging.info('Loop interface initialized with base_url: %s / user: %s', self.base_url, self.username)
+        logger.info('Loop interface initialized with base_url: %s / user: %s', self.base_url, self.username)
 
     def websocket_url(self) -> str:
         return f'ws{"s" if "learning-loop.ai" in self.host else ""}://' + self.host
@@ -65,7 +66,7 @@ class LoopCommunicator():
             self.async_client.cookies.clear()
             response = await self.async_client.post('/api/login', data={'username': self.username, 'password': self.password})
             if response.status_code != 200:
-                logging.info('Login failed with response: %s', response)
+                logger.info('Login failed with response: %s', response)
                 raise LoopCommunicationException('Login failed with response: ' + str(response))
             self.async_client.cookies.update(response.cookies)
 
@@ -74,7 +75,7 @@ class LoopCommunicator():
 
         response = await self.async_client.post('/api/logout')
         if response.status_code != 200:
-            logging.info('Logout failed with response: %s', response)
+            logger.info('Logout failed with response: %s', response)
             raise LoopCommunicationException('Logout failed with response: ' + str(response))
         self.async_client.cookies.clear()
 
@@ -90,12 +91,12 @@ class LoopCommunicator():
         start_time = time.time()
         while True:
             try:
-                logging.info('Checking if backend is ready')
+                logger.info('Checking if backend is ready')
                 response = await self.get('/status', requires_login=False)
                 if response.status_code == 200:
                     return True
             except Exception:
-                logging.info('backend not ready yet.')
+                logger.info('backend not ready yet.')
             if timeout is not None and time.time() + 10 - start_time > timeout:
                 raise TimeoutError('Backend not ready within timeout')
             await asyncio.sleep(10)