learning-loop-node 0.18.4__tar.gz → 0.19.0__tar.gz

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.18.4
+Version: 0.19.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
 Project-URL: Repository, https://github.com/zauberzeug/learning_loop_node
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: fastapi<1.0,>=0.128
 Requires-Dist: uvicorn[standard]<0.23.0,>=0.22.0
@@ -17,7 +17,7 @@ Requires-Dist: icecream<3.0.0,>=2.1.0
 Requires-Dist: aiofiles<0.8.0,>=0.7.0
 Requires-Dist: async_generator<2.0.0,>=1.10
 Requires-Dist: pytest-watch<5.0.0,>=4.2.0
-Requires-Dist: python-multipart<0.0.19,>=0.0.18
+Requires-Dist: python-multipart>=0.0.22
 Requires-Dist: psutil<6.0.0,>=5.8.0
 Requires-Dist: numpy<3.0,>=2.0
 Requires-Dist: Pillow<12.0.0,>=11.0.0
@@ -53,21 +53,22 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                     | Alias        | Purpose                                                      | Required by               |
-| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- |
-| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       |
-| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
-| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
-| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  |
-| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector (opt.)           |
-| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
-| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
-| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
-| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            |
-| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            |
-| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
-| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
+| Name                     | Alias        | Purpose                                                      | Required by               | Default      |
+| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- | ------------ |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       | -            |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      | -            |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      | -            |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                | -            |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  | -            |
+| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector                  | -            |
+| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           | 0.3          |
+| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           | 0.6          |
+| EXCLUSIVE_MODEL_BUILD    | -            | Reject detections during update to save VRAM (set to 1)      | Detector (opt.)           | 0            |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            | 10           |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            | 0            |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            | 0            |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            | 0 (disabled) |
+| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) | 0            |
 
 Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
 
@@ -274,5 +275,5 @@ After a short time the converted model should be available as well.
   - `id`: the model UUID (currently not needed by anyone, since host, org, project, version clearly identify the model)
   - `format`: the format e.g. yolo, tkdnn, yolor etc.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
-  - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, ` resolution_x`` resolution_y ` would also be conceivable or `resolutions` to give a list of possible resolutions)
+  - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/README.md

@@ -17,21 +17,22 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                     | Alias        | Purpose                                                      | Required by               |
-| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- |
-| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       |
-| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
-| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
-| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  |
-| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector (opt.)           |
-| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
-| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
-| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
-| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            |
-| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            |
-| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
-| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
+| Name                     | Alias        | Purpose                                                      | Required by               | Default      |
+| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- | ------------ |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       | -            |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      | -            |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      | -            |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                | -            |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  | -            |
+| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector                  | -            |
+| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           | 0.3          |
+| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           | 0.6          |
+| EXCLUSIVE_MODEL_BUILD    | -            | Reject detections during update to save VRAM (set to 1)      | Detector (opt.)           | 0            |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            | 10           |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            | 0            |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            | 0            |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            | 0 (disabled) |
+| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) | 0            |
 
 Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
 
@@ -238,5 +239,5 @@ After a short time the converted model should be available as well.
   - `id`: the model UUID (currently not needed by anyone, since host, org, project, version clearly identify the model)
   - `format`: the format e.g. yolo, tkdnn, yolor etc.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
-  - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, ` resolution_x`` resolution_y ` would also be conceivable or `resolutions` to give a list of possible resolutions)
+  - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node/__init__.py

@@ -1,11 +1,11 @@
 import logging
 
 # from . import log_conf
-from .detector.detector_logic import DetectorLogic
+from .detector.detector_logic import DetectorLogic, DetectorLogicFactory
 from .detector.detector_node import DetectorNode
 from .globals import GLOBALS
 from .trainer.trainer_node import TrainerNode
 
-__all__ = ['TrainerNode', 'DetectorNode', 'DetectorLogic', 'GLOBALS']
+__all__ = ['TrainerNode', 'DetectorNode', 'DetectorLogic', 'DetectorLogicFactory', 'GLOBALS']
 
 logging.info('>>>>>>>>>>>>>>>>>> LOOP INITIALIZED <<<<<<<<<<<<<<<<<<<<<<<')

learning_loop_node-0.19.0/learning_loop_node/detector/__init__.py

@@ -0,0 +1,6 @@
+from .detector_logic import DetectorLogic, DetectorLogicFactory
+
+__all__ = [
+    'DetectorLogic',
+    'DetectorLogicFactory',
+]

learning_loop_node-0.19.0/learning_loop_node/detector/detector_logic.py

@@ -0,0 +1,42 @@
+from typing import List, Protocol
+
+import numpy as np
+
+from ..data_classes import ImageMetadata, ImagesMetadata, ModelInformation
+
+
+class DetectorLogicFactory(Protocol):
+    """Protocol for building DetectorLogic instances.
+
+    The factory controls how the detector is constructed — implementations
+    can build synchronously or offload heavy work to a thread pool.
+    """
+    @property
+    def model_format(self) -> str: ...
+
+    async def build(self, model_info: ModelInformation) -> 'DetectorLogic': ...
+
+
+class DetectorLogic(Protocol):
+    """Protocol for detector implementations.
+
+    Implementations receive the ModelInformation via their constructor (called by the
+    DetectorLogic factory in DetectorNode) and are free to store or ignore it.
+    """
+
+    def evaluate(self, image: np.ndarray) -> ImageMetadata:
+        """Evaluate the image and return the detections.
+
+        Called by the detector node when an image should be evaluated (REST or SocketIO).
+        The resulting detections should be stored in the ImageMetadata.
+        Tags stored in the ImageMetadata will be uploaded to the learning loop.
+        """
+        ...
+
+    def batch_evaluate(self, images: List[np.ndarray]) -> ImagesMetadata:
+        """Evaluate a batch of images and return the detections.
+
+        The resulting detections per image should be stored in the ImagesMetadata.
+        Tags stored in the ImagesMetadata will be uploaded to the learning loop.
+        """
+        ...

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node/detector/detector_node.py

@@ -1,15 +1,16 @@
 import asyncio
 import contextlib
+import gc
+import logging
 import os
 import shutil
 import subprocess
 import sys
-from dataclasses import asdict
+from dataclasses import asdict, dataclass
 from datetime import datetime
-from typing import Dict, List, Literal, Optional
+from typing import Dict, List, Literal, Optional, Union
 
 import numpy as np
-
 import socketio
 from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
@@ -32,7 +33,7 @@ from ..globals import GLOBALS
 from ..helpers import background_tasks, environment_reader, run
 from ..helpers.misc import numpy_image_from_dict
 from ..node import Node
-from .detector_logic import DetectorLogic
+from .detector_logic import DetectorLogic, DetectorLogicFactory
 from .exceptions import NodeNeedsRestartError
 from .inbox_filter.relevance_filter import RelevanceFilter
 from .outbox import Outbox
@@ -47,9 +48,13 @@ from .rest import upload as rest_upload
 
 class DetectorNode(Node):
 
-    def __init__(self, name: str, detector: DetectorLogic, uuid: Optional[str] = None, use_backdoor_controls: bool = False) -> None:
+    def __init__(self, name: str, detector_factory: DetectorLogicFactory,
+                 uuid: Optional[str] = None, use_backdoor_controls: bool = False) -> None:
         super().__init__(name, uuid=uuid, node_type='detector', needs_login=False, needs_sio=False)
-        self.detector_logic = detector
+        self._detector_factory = detector_factory
+        self._detector: _DetectorState = _Initializing()
+        self._exclusive_model_build: bool = os.environ.get('EXCLUSIVE_MODEL_BUILD', '0').lower() in ('1', 'true')
+        self._remaining_init_attempts: int = 2
         self.organization = environment_reader.organization()
         self.project = environment_reader.project()
         assert self.organization and self.project, 'Detector node needs an organization and an project'
@@ -95,13 +100,13 @@ class DetectorNode(Node):
         return AboutResponse(
             operation_mode=self.operation_mode.value,
             state=self.status.state,
-            model_info=self.detector_logic.model_info,  # pylint: disable=protected-access
+            model_info=self._detector.model_info if isinstance(self._detector, _ActiveDetector) else None,
             target_model=self.target_model.version if self.target_model else None,
             version_control=self.version_control.value
         )
 
     def get_model_version_response(self) -> ModelVersionResponse:
-        current_version = self.detector_logic.model_info.version if self.detector_logic.model_info is not None else 'None'  # pylint: disable=protected-access
+        current_version = self._detector.model_info.version if isinstance(self._detector, _ActiveDetector) else 'None'
         target_version = self.target_model.version if self.target_model is not None else 'None'
         loop_version = self.loop_deployment_target.version if self.loop_deployment_target is not None else 'None'
 
@@ -183,15 +188,12 @@ class DetectorNode(Node):
         self.socket_connection_broken = True
         await self.on_startup()
 
-        # simulate startup
-        await self.detector_logic.soft_reload()
-        await self._load_model_info_and_init_model()
-        self.operation_mode = OperationMode.Idle
-
     async def on_startup(self) -> None:
         try:
             self.outbox.ensure_continuous_upload()
-            await self._load_model_info_and_init_model()
+            current_model_dir = self._current_model_path()
+            if current_model_dir and os.path.isdir(current_model_dir):
+                await self._build_and_swap_detector(current_model_dir)
         except Exception:
             self.log.exception("error during 'startup'")
         self.operation_mode = OperationMode.Idle
@@ -251,14 +253,11 @@ class DetectorNode(Node):
                     autoupload=data.get('autoupload', 'filtered'),
                     creation_date=data.get('creation_date', None)
                 )
-                if det is None:
-                    return {'error': 'no model loaded'}
-                detection_dict = jsonable_encoder(asdict(det))
-                return detection_dict
+                return jsonable_encoder(asdict(det))
+            except DetectorUnavailableError as e:
+                return {'error': str(e)}
             except Exception as e:
                 self.log.exception('could not detect via socketio')
-                # with open('/tmp/bad_img_from_socket_io.jpg', 'wb') as f:
-                #     f.write(data['image'])
                 return {'error': str(e)}
 
         @self.sio.event
@@ -266,7 +265,7 @@ class DetectorNode(Node):
             """
             Detect objects in a batch of images sent via SocketIO.
 
-            Data dict follows the schema of the detect endpoint, 
+            Data dict follows the schema of the detect endpoint,
             but 'images' is a list of image dicts.
             """
             try:
@@ -285,21 +284,22 @@ class DetectorNode(Node):
                     autoupload=data.get('autoupload', 'filtered'),
                     creation_date=data.get('creation_date', None)
                 )
-                if det is None:
-                    return {'error': 'no model loaded'}
-                detection_dict = jsonable_encoder(asdict(det))
-                return detection_dict
+                return jsonable_encoder(asdict(det))
+            except DetectorUnavailableError as e:
+                return {'error': str(e)}
             except Exception as e:
                 self.log.exception('could not detect via socketio')
-                # with open('/tmp/bad_img_from_socket_io.jpg', 'wb') as f:
-                #     f.write(data['image'])
                 return {'error': str(e)}
 
         @self.sio.event
         async def info(sid) -> Dict:
-            if self.detector_logic.model_info is not None:
-                return asdict(self.detector_logic.model_info)
-            return {"status": "No model loaded"}
+            match self._detector:
+                case _ActiveDetector(model_info=info):
+                    return asdict(info)
+                case _Updating() as u:
+                    return {"status": f"Updating model to version {u.version}"}
+                case _Initializing():
+                    return {"status": "No model loaded"}
 
         @self.sio.event
         async def about(sid) -> Dict:
@@ -350,8 +350,11 @@ class DetectorNode(Node):
                 except Exception as e:
                     self.log.exception('could not parse detections')
                     return {'error': str(e)}
-                if self.detector_logic.model_info is not None:
-                    image_metadata = self.add_category_id_to_detections(self.detector_logic.model_info, image_metadata)
+                try:
+                    detector = _unwrap_detector(self._detector)
+                    image_metadata = self.add_category_id_to_detections(detector.model_info, image_metadata)
+                except DetectorUnavailableError as e:
+                    self.log.warning('Cannot add category IDs: %s', e)
             else:
                 image_metadata = ImageMetadata()
 
@@ -393,11 +396,7 @@ class DetectorNode(Node):
     async def _sync_status_with_loop(self) -> None:
         """Sync status of the detector with the Learning Loop."""
 
-        if self.detector_logic.model_info is not None:
-            current_model = self.detector_logic.model_info.version
-        else:
-            current_model = None
-
+        current_model = self._detector.model_info.version if isinstance(self._detector, _ActiveDetector) else None
         target_model_version = self.target_model.version if self.target_model else None
 
         status = DetectorStatus(
@@ -409,7 +408,7 @@ class DetectorNode(Node):
             operation_mode=self.operation_mode,
             current_model=current_model,
             target_model=target_model_version,
-            model_format=self.detector_logic.model_format,
+            model_format=self._detector_factory.model_format,
         )
 
         self.log_status_on_change(status.state, status)
@@ -427,7 +426,7 @@ class DetectorNode(Node):
     async def _update_model_if_required(self) -> None:
         """Check if a new model is available and update if necessary.
         The Learning Loop will respond with the model info of the deployment target.
-        If version_control is set to FollowLoop or the chosen target model is not used, 
+        If version_control is set to FollowLoop or the chosen target model is not used,
         the detector will update the target_model."""
         try:
             if self.operation_mode != OperationMode.Idle:
@@ -441,8 +440,14 @@ class DetectorNode(Node):
                 self.log.debug('not running any updates; target model is None')
                 return
 
-            current_version = self.detector_logic.model_info.version \
-                if self.detector_logic.model_info is not None else None
+            match self._detector:
+                case _ActiveDetector(model_info=info):
+                    current_version = info.version
+                case _Updating():
+                    self.log.debug('not checking for updates; model update already in progress')
+                    return
+                case _Initializing():
+                    current_version = None
 
             if current_version != self.target_model.version:
                 self.log.info('Updating model from %s to %s',
@@ -499,7 +504,7 @@ class DetectorNode(Node):
                     await self.data_exchanger.download_model(target_model_folder,
                                                              Context(organization=self.organization,
                                                                      project=self.project),
-                                                             target_model.id, self.detector_logic.model_format)
+                                                             target_model.id, self._detector_factory.model_format)
                     self.log.info('Downloaded model %s', target_model.version)
                 except Exception:
                     self.log.exception('Could not download model %s', target_model.version)
@@ -507,32 +512,76 @@ class DetectorNode(Node):
                     self.target_model = None
                     return
 
-            model_symlink = 'model'
             try:
-                os.unlink(model_symlink)
-                os.remove(model_symlink)
-            except Exception:
-                pass
-            os.symlink(target_model_folder, model_symlink)
-            self.log.info('Updated symlink for model to %s', os.readlink(model_symlink))
-
-            try:
-                await self._load_model_info_and_init_model()
+                await self._build_and_swap_detector(target_model_folder)
             except NodeNeedsRestartError:
                 self.log.error('Node needs restart')
                 sys.exit(0)
             except Exception:
-                self.log.exception('Could not load model, will retry download on next check')
+                self.log.exception('Could not build detector, will retry download on next check')
                 shutil.rmtree(target_model_folder, ignore_errors=True)
                 self.target_model = None
                 return
 
+            self._update_current_model_symlink(target_model_folder)
             await self._sync_status_with_loop()
-            # self.reload(reason='new model installed')
 
-    async def _load_model_info_and_init_model(self) -> None:
-        async with self.detection_lock:
-            self.detector_logic.load_model_info_and_init_model()
+    async def _build_and_swap_detector(self, model_dir: str) -> None:
+        """Load ModelInformation from model_dir, build a new DetectorLogic via the factory,
+        then atomically swap self._detector when ready.
+        The old detector continues to serve requests until the swap.
+
+        If EXCLUSIVE_MODEL_BUILD is set and a detector is active, the old detector is torn down
+        first (freeing e.g. GPU VRAM) and detections are rejected until the new one is ready.
+        """
+        logging.info('Loading model from %s', model_dir)
+        model_info = ModelInformation.load_from_disk(os.path.abspath(model_dir))
+        if model_info is None:
+            logging.warning('No model.json found in %s', model_dir)
+            raise Exception('model.json not found')
+
+        if self._exclusive_model_build and isinstance(self._detector, _ActiveDetector):
+            async with self.detection_lock:
+                old_logic = self._detector.logic
+                self._detector = _Updating(version=model_info.version)
+
+                # Ensure that we actually delete the old DetectorLogic here to
+                # free resources before building a new detector.
+                refcount = sys.getrefcount(old_logic)  # 2 = old_logic local + getrefcount arg
+                assert refcount == 2, f'expected 2 refs to old detector logic, got {refcount}'
+                del old_logic
+                gc.collect()
+
+        try:
+            new_detector = await self._detector_factory.build(model_info)
+            logging.info('Successfully built detector for model %s', model_info)
+            self._remaining_init_attempts = 2
+        except Exception as e:
+            self._remaining_init_attempts -= 1
+            logging.error('Could not build detector for model %s. Retries left: %s. Error: %s',
+                          model_info, self._remaining_init_attempts, e)
+            if not isinstance(self._detector, _ActiveDetector):
+                self._detector = _Initializing()  # no model to fall back to
+            if self._remaining_init_attempts == 0:
+                raise NodeNeedsRestartError('Could not build detector') from None
+            raise
+        self._detector = _ActiveDetector(new_detector, model_info)
+
+    @staticmethod
+    def _current_model_path() -> Optional[str]:
+        """Return the absolute path the current_model symlink points to, or None."""
+        link = os.path.join(GLOBALS.data_folder, 'current_model')
+        if os.path.islink(link):
+            return os.path.realpath(link)
+        return None
+
+    @staticmethod
+    def _update_current_model_symlink(model_dir: str) -> None:
+        """Atomically update the current_model symlink to point to model_dir."""
+        link = os.path.join(GLOBALS.data_folder, 'current_model')
+        tmp_link = link + '.tmp'
+        os.symlink(model_dir, tmp_link)
+        os.rename(tmp_link, link)
 
 # ================================== API Implementations ==================================
 
@@ -565,12 +614,13 @@ class DetectorNode(Node):
         Main processing function for the detector node.
 
         Used when an image is received via REST or SocketIO.
-        This function infers the detections from the image, 
+        This function infers the detections from the image,
         cares about uploading to the loop and returns the detections as ImageMetadata object.
+        Raises exception if no model is loaded.
         """
-
         async with self.detection_lock:
-            metadata = await run.io_bound(self.detector_logic.evaluate, image)
+            detector = _unwrap_detector(self._detector)
+            metadata = await run.io_bound(detector.logic.evaluate, image)
 
         metadata.tags.extend(tags)
         metadata.source = source
@@ -600,14 +650,15 @@ class DetectorNode(Node):
                                    autoupload: str = 'filtered',
                                    creation_date: Optional[str] = None) -> ImagesMetadata:
         """
-        Processing function for the detector node when a a batch inference is requested via SocketIO.
+        Processing function for the detector node when a batch inference is requested via SocketIO.
 
-        This function infers the detections from all images, 
+        This function infers the detections from all images,
         cares about uploading to the loop and returns the detections as a list of ImageMetadata.
+        Raises exception if no model is loaded.
         """
-
         async with self.detection_lock:
-            all_detections = await run.io_bound(self.detector_logic.batch_evaluate, images)
+            detector = _unwrap_detector(self._detector)
+            all_detections = await run.io_bound(detector.logic.batch_evaluate, images)
 
         for metadata in all_detections.items:
             metadata.tags.extend(tags)
@@ -680,6 +731,40 @@ class DetectorNode(Node):
         pass
 
 
+class _Initializing:
+    """No model ready yet — first build pending or in progress."""
+
+
+@dataclass
+class _Updating:
+    """Exclusive model replacement in progress — detections rejected."""
+    version: str
+
+
+@dataclass
+class _ActiveDetector:
+    logic: DetectorLogic
+    model_info: ModelInformation
+
+
+_DetectorState = Union[_Initializing, _Updating, _ActiveDetector]
+
+
+class DetectorUnavailableError(Exception):
+    pass
+
+
+def _unwrap_detector(state: '_DetectorState') -> _ActiveDetector:
+    """Return the active detector or raise DetectorUnavailableError."""
+    match state:
+        case _ActiveDetector() as d:
+            return d
+        case _Updating() as u:
+            raise DetectorUnavailableError(f'updating model to version {u.version}')
+        case _Initializing():
+            raise DetectorUnavailableError('detector not yet initialized')
+
+
 @contextlib.contextmanager
 def step_into(new_dir):
     previous_dir = os.getcwd()

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node/globals.py

@@ -1,7 +1,9 @@
+import os
+
 
 class Globals():
     def __init__(self) -> None:
-        self.data_folder: str = '/data'
+        self.data_folder: str = os.getenv('DATA_FOLDER', '/data')
         self.detector_port: int = 5004  # NOTE used for tests
 
 

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node/helpers/misc.py

@@ -60,11 +60,22 @@ def _handle_task_result(task: asyncio.Task,
 
 
 def get_free_memory_mb() -> float:  # NOTE used by yolov5
-    pynvml.nvmlInit()
-    h = pynvml.nvmlDeviceGetHandleByIndex(0)
-    info = pynvml.nvmlDeviceGetMemoryInfo(h)
-    free = float(info.free) / 1024 / 1024
-    return free
+    """
+    Get the free memory available to the graphics card (index 0) in MB.
+    If VRAM information is unavailable, for example if the system shares ram
+    between CPU and GPU, the available RAM is returned instead.
+    """
+    try:
+        pynvml.nvmlInit()
+
+        h = pynvml.nvmlDeviceGetHandleByIndex(0)
+        info = pynvml.nvmlDeviceGetMemoryInfo(h)
+        free_mb = float(info.free) / 1024 / 1024
+    except pynvml.nvml.NVMLError:
+        logger = logging.getLogger(__name__)
+        logger.warning('Could not get GPU memory info, returning free CPU memory instead')
+        free_mb = float(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_AVPHYS_PAGES')) / 1024 / 1024
+    return free_mb
 
 
 async def is_valid_image(filename: str, check_jpeg: bool) -> bool:

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node/node.py

@@ -203,7 +203,7 @@ class Node(FastAPI):
 
         if self.sio_client is not None:
             try:
-                await self.sio_client.disconnect()
+                await self.sio_client.shutdown()
                 self.log.info('disconnected from loop via sio')
                 # NOTE: without waiting for the disconnect event, we might disconnect the next connection too early
                 await asyncio.wait_for(self.DISCONNECTED_FROM_LOOP.wait(), timeout=5)

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node/trainer/io_helpers.py

@@ -173,7 +173,7 @@ class ActiveTrainingIO:
         response = await self.loop_communicator.post(
             f'/{context.organization}/projects/{context.project}/detections', json=detections_json)
         if response.status_code != 200:
-            msg = f'could not upload detections. {str(response)}'
+            msg = f'could not upload detections. HTTP {response.status_code}: {response.text}'
             logging.error(msg)
             raise Exception(msg)
 

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node.egg-info/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: learning_loop_node
-Version: 0.18.4
+Version: 0.19.0
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Author-email: Zauberzeug GmbH <info@zauberzeug.com>
 License: MIT
 Project-URL: Repository, https://github.com/zauberzeug/learning_loop_node
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: fastapi<1.0,>=0.128
 Requires-Dist: uvicorn[standard]<0.23.0,>=0.22.0
@@ -17,7 +17,7 @@ Requires-Dist: icecream<3.0.0,>=2.1.0
 Requires-Dist: aiofiles<0.8.0,>=0.7.0
 Requires-Dist: async_generator<2.0.0,>=1.10
 Requires-Dist: pytest-watch<5.0.0,>=4.2.0
-Requires-Dist: python-multipart<0.0.19,>=0.0.18
+Requires-Dist: python-multipart>=0.0.22
 Requires-Dist: psutil<6.0.0,>=5.8.0
 Requires-Dist: numpy<3.0,>=2.0
 Requires-Dist: Pillow<12.0.0,>=11.0.0
@@ -53,21 +53,22 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                     | Alias        | Purpose                                                      | Required by               |
-| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- |
-| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       |
-| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
-| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
-| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  |
-| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector (opt.)           |
-| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
-| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
-| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
-| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            |
-| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            |
-| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
-| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
+| Name                     | Alias        | Purpose                                                      | Required by               | Default      |
+| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- | ------------ |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       | -            |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      | -            |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      | -            |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                | -            |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization ID                                              | Detector                  | -            |
+| LOOP_PROJECT             | PROJECT      | Project ID                                                   | Detector                  | -            |
+| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           | 0.3          |
+| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           | 0.6          |
+| EXCLUSIVE_MODEL_BUILD    | -            | Reject detections during update to save VRAM (set to 1)      | Detector (opt.)           | 0            |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            | 10           |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            | 0            |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            | 0            |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            | 0 (disabled) |
+| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) | 0            |
 
 Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
 
@@ -274,5 +275,5 @@ After a short time the converted model should be available as well.
   - `id`: the model UUID (currently not needed by anyone, since host, org, project, version clearly identify the model)
   - `format`: the format e.g. yolo, tkdnn, yolor etc.
 - Nodes add properties to `model.json`, which contains all the information which are needed by subsequent nodes. These are typically the properties:
-  - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, ` resolution_x`` resolution_y ` would also be conceivable or `resolutions` to give a list of possible resolutions)
+  - `resolution`: resolution in which the model expects images (as `int`, since the resolution is mostly square - later, `resolution_x`` resolution_y` would also be conceivable or `resolutions` to give a list of possible resolutions)
   - `categories`: list of categories with name, id, (later also type), in the order in which they are used by the model -- this is neccessary to be robust about renamings

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/learning_loop_node.egg-info/requires.txt

@@ -8,7 +8,7 @@ icecream<3.0.0,>=2.1.0
 aiofiles<0.8.0,>=0.7.0
 async_generator<2.0.0,>=1.10
 pytest-watch<5.0.0,>=4.2.0
-python-multipart<0.0.19,>=0.0.18
+python-multipart>=0.0.22
 psutil<6.0.0,>=5.8.0
 numpy<3.0,>=2.0
 Pillow<12.0.0,>=11.0.0

{learning_loop_node-0.18.4 → learning_loop_node-0.19.0}/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "learning_loop_node"
-version = "0.18.4"
+version = "0.19.0"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = [{ name = "Zauberzeug GmbH", email = "info@zauberzeug.com" }]
 license = { text = "MIT" }
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "fastapi>=0.128,<1.0",
     "uvicorn[standard]>=0.22.0,<0.23.0",
@@ -17,7 +17,7 @@ dependencies = [
     "aiofiles>=0.7.0,<0.8.0",
     "async_generator>=1.10,<2.0.0",
     "pytest-watch>=4.2.0,<5.0.0",
-    "python-multipart>=0.0.18,<0.0.19",
+    "python-multipart>=0.0.22",
     "psutil>=5.8.0,<6.0.0",
     "numpy>=2.0,<3.0",
     "Pillow>=11.0.0,<12.0.0",
@@ -48,3 +48,45 @@ exclude = ["learning_loop_node.tests*", "learning_loop_node.examples*"]
 
 [tool.mypy]
 ignore_missing_imports = true
+
+
+[tool.ruff]
+indent-width = 4
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.lint]
+select = [
+    "I",   # isort
+    "E",   # pycodestyle
+    "W",   # pycodestyle
+    "B",   # bugbear
+    "F",   # pyflakes
+    "UP",  # pyupgrade
+    "RUF", # ruff
+    "PL",  # pylint
+    "NPY201", # NumPy 2.0
+    "Q",   # flake8-quotes
+]
+fixable = [
+    "I",  # isort
+    "RUF022", # `__all__` is not sorted
+    "Q",  # flake8-quotes
+]
+ignore = [
+    "E501", # line too long
+    "E741",
+    "PLR2004", # magic value comparison
+    "Q001",  # multi line quotes
+    "Q002",  # docstring quotes
+    "PLR0911", # too many
+    "PLR0912", # too many
+    "PLR0913", # too many arguments
+    "PLR0915", # too many arguments
+]
+
+[tool.ruff.lint.flake8-quotes]
+inline-quotes = "single"
+multiline-quotes = "single"
+docstring-quotes = "double"
+avoid-escape = true

learning_loop_node-0.18.4/learning_loop_node/detector/detector_logic.py

@@ -1,79 +0,0 @@
-import logging
-from abc import abstractmethod
-from typing import List, Optional
-
-import numpy as np
-
-from ..data_classes import ImageMetadata, ImagesMetadata, ModelInformation
-from ..globals import GLOBALS
-from .exceptions import NodeNeedsRestartError
-
-
-class DetectorLogic():
-
-    def __init__(self, model_format: str) -> None:
-        self.model_format: str = model_format
-        self.model_info: Optional[ModelInformation] = None
-
-        self._remaining_init_attempts: int = 2
-
-    async def soft_reload(self):
-        self.model_info = None
-
-    def load_model_info_and_init_model(self):
-        """
-        Load model information from disk and initialize the model.
-
-        The detector node uses a lock to make sure that this is not called 
-        concurrently with evaluate() or batch_evaluate(). 
-        """
-        logging.info('Loading model from %s', GLOBALS.data_folder)
-        self.model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
-        if self.model_info is None:
-            logging.error('No model found')
-            self.model_info = None
-            return
-
-        try:
-            self.init()
-            logging.info('Successfully loaded model %s', self.model_info)
-            self._remaining_init_attempts = 2
-        except Exception:
-            self._remaining_init_attempts -= 1
-            self.model_info = None
-            logging.error('Could not init model %s. Retries left: %s', self.model_info, self._remaining_init_attempts)
-            if self._remaining_init_attempts == 0:
-                raise NodeNeedsRestartError('Could not init model') from None
-            raise
-
-    @abstractmethod
-    def init(self):
-        """
-        Initialize the model.
-
-        Called when a (new) model was loaded. 
-        Model information available via `self.model_info`
-        The detector node uses a lock to make sure that this is not called
-        concurrently with evaluate() or batch_evaluate(). 
-        """
-
-    @abstractmethod
-    def evaluate(self, image: np.ndarray) -> ImageMetadata:
-        """
-        Evaluate the image and return the detections.
-
-        Called by the detector node when an image should be evaluated (REST or SocketIO).
-        The resulting detections should be stored in the ImageMetadata.
-        Tags stored in the ImageMetadata will be uploaded to the learning loop.
-        The function should return empty metadata if the detector is not initialized.
-        """
-
-    @abstractmethod
-    def batch_evaluate(self, images: List[np.ndarray]) -> ImagesMetadata:
-        """
-        Evaluate a batch of images and return the detections.
-
-        The resulting detections per image should be stored in the ImagesMetadata.
-        Tags stored in the ImagesMetadata will be uploaded to the learning loop.
-        The function should return empty metadata if the detector is not initialized.
-        """