learning-loop-node 0.8.5__tar.gz → 0.8.7__tar.gz

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.8.5
+Version: 0.8.7
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT
@@ -18,8 +18,8 @@ Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
 Requires-Dist: aiohttp (>=3.8.4,<4.0.0)
 Requires-Dist: async_generator (>=1.10,<2.0)
 Requires-Dist: dacite (>=1.8.1,<2.0.0)
-Requires-Dist: fastapi (>=0.70.1,<0.71.0)
-Requires-Dist: fastapi-socketio (>=0.0.6,<0.0.7)
+Requires-Dist: fastapi (>=0.93,<1.0.0)
+Requires-Dist: fastapi-socketio (>=0.0.10,<0.0.11)
 Requires-Dist: fastapi-utils (>=0.2.1,<0.3.0)
 Requires-Dist: httpx (>=0.24.1,<0.25.0)
 Requires-Dist: icecream (>=2.1.0,<3.0.0)
@@ -33,7 +33,7 @@ Requires-Dist: python-socketio (>=5.7.2,<6.0.0)
 Requires-Dist: requests (>=2.25.1,<3.0.0)
 Requires-Dist: simplejson (>=3.17.2,<4.0.0)
 Requires-Dist: tqdm (>=4.63.0,<5.0.0)
-Requires-Dist: uvicorn (>=0.13.3,<0.14.0)
+Requires-Dist: uvicorn[standard] (>=0.22.0,<0.23.0)
 Requires-Dist: werkzeug (>=2.0.1,<3.0.0)
 Project-URL: Repository, https://github.com/zauberzeug/learning_loop_node
 Description-Content-Type: text/markdown
@@ -57,16 +57,18 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                   | Alias        | Purpose                                           | Required by          |
-| ---------------------- | ------------ | ------------------------------------------------- | -------------------- |
-| LOOP_HOST              | HOST         | Learning Loop address (e.g. learning-loop.ai)     | all                  |
-| LOOP_USERNAME          | USERNAME     | Learning Loop user name                           | all besides Detector |
-| LOOP_PASSWORD          | PASSWORD     | Learning Loop password                            | all besides Detector |
-| LOOP_ORGANIZATION      | ORGANIZATION | Organization name                                 | Detector             |
-| LOOP_PROJECT           | PROJECT      | Project name                                      | Detector             |
-| INFERENCE_BATCH_SIZE   | -            | Batch size of trainer when calculating detections | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING | -            | Restart the trainer after training (set to 1)     | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS     | -            | Do not delete old trainings (set to 1)            | Trainer (opt.)       |
+| Name                    | Alias        | Purpose                                                      | Required by          |
+| ----------------------- | ------------ | ------------------------------------------------------------ | -------------------- |
+| LOOP_HOST               | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
+| LOOP_USERNAME           | USERNAME     | Learning Loop user name                                      | all besides Detector |
+| LOOP_PASSWORD           | PASSWORD     | Learning Loop password                                       | all besides Detector |
+| LOOP_ORGANIZATION       | ORGANIZATION | Organization name                                            | Detector             |
+| LOOP_PROJECT            | PROJECT      | Project name                                                 | Detector             |
+| MIN_UNCERTAIN_THRESHOLD | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
+| MAX_UNCERTAIN_THRESHOLD | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
+| INFERENCE_BATCH_SIZE    | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
+| RESTART_AFTER_TRAINING  | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
+| KEEP_OLD_TRAININGS      | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
 
 #### Testing
 

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/README.md

@@ -17,16 +17,18 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                   | Alias        | Purpose                                           | Required by          |
-| ---------------------- | ------------ | ------------------------------------------------- | -------------------- |
-| LOOP_HOST              | HOST         | Learning Loop address (e.g. learning-loop.ai)     | all                  |
-| LOOP_USERNAME          | USERNAME     | Learning Loop user name                           | all besides Detector |
-| LOOP_PASSWORD          | PASSWORD     | Learning Loop password                            | all besides Detector |
-| LOOP_ORGANIZATION      | ORGANIZATION | Organization name                                 | Detector             |
-| LOOP_PROJECT           | PROJECT      | Project name                                      | Detector             |
-| INFERENCE_BATCH_SIZE   | -            | Batch size of trainer when calculating detections | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING | -            | Restart the trainer after training (set to 1)     | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS     | -            | Do not delete old trainings (set to 1)            | Trainer (opt.)       |
+| Name                    | Alias        | Purpose                                                      | Required by          |
+| ----------------------- | ------------ | ------------------------------------------------------------ | -------------------- |
+| LOOP_HOST               | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
+| LOOP_USERNAME           | USERNAME     | Learning Loop user name                                      | all besides Detector |
+| LOOP_PASSWORD           | PASSWORD     | Learning Loop password                                       | all besides Detector |
+| LOOP_ORGANIZATION       | ORGANIZATION | Organization name                                            | Detector             |
+| LOOP_PROJECT            | PROJECT      | Project name                                                 | Detector             |
+| MIN_UNCERTAIN_THRESHOLD | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
+| MAX_UNCERTAIN_THRESHOLD | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
+| INFERENCE_BATCH_SIZE    | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
+| RESTART_AFTER_TRAINING  | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
+| KEEP_OLD_TRAININGS      | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
 
 #### Testing
 

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/learning_loop_node/data_exchanger.py

@@ -59,10 +59,23 @@ class DataExchanger():
             logging.warning('context was not set yet')
             return
 
+        await self.delete_corrupt_images(image_folder)
         new_image_ids = await asyncio.get_event_loop().run_in_executor(None, DataExchanger.filter_existing_images, image_ids, image_folder)
         paths, ids = create_resource_paths(self.context.organization, self.context.project, new_image_ids)
         await self._download_images(paths, ids, image_folder)
 
+    @staticmethod
+    async def delete_corrupt_images(image_folder: str) -> None:
+        logging.info('deleting corrupt images')
+        n_deleted = 0
+        for image in glob(f'{image_folder}/*.jpg'):
+            if not await DataExchanger.is_valid_image(image):
+                logging.debug(f'  deleting image {image}')
+                os.remove(image)
+                n_deleted += 1
+
+        logging.info(f'deleted {n_deleted} images')
+
     @staticmethod
     def filter_existing_images(all_image_ids, image_folder) -> List[str]:
         logging.info(f'### Going to filter {len(all_image_ids)} images ids')
@@ -131,8 +144,9 @@ class DataExchanger():
         if not await self.is_valid_image(filename):
             os.remove(filename)
 
-    async def is_valid_image(self, filename: str) -> bool:
-        if not os.path.isfile(filename):
+    @staticmethod
+    async def is_valid_image(filename: str) -> bool:
+        if not os.path.isfile(filename) or os.path.getsize(filename) == 0:
             return False
         if not check_jpeg:
             return True

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/learning_loop_node/detector/detector_logic.py

@@ -46,7 +46,7 @@ class DetectorLogic():
 
     @abstractmethod
     def init(self):
-        """Initialize the model. Note that `model_info` is available as `self.model_info`"""
+        """Called when a (new) model was loaded. Initialize the model. Model information available via `self.model_info`"""
 
     @abstractmethod
     def evaluate(self, image: np.ndarray) -> Detections:

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/learning_loop_node/detector/detector_node.py

@@ -142,10 +142,13 @@ class DetectorNode(Node):
 
             detection_data = data.get('detections', {})
             if detection_data and self.detector_logic.is_initialized:
-                detections = from_dict(data_class=Detections, data=detection_data)
+                try:
+                    detections = from_dict(data_class=Detections, data=detection_data)
+                except Exception as e:
+                    self.log.exception('could not parse detections')
+                    return {'error': str(e)}
                 detections = self.add_category_id_to_detections(self.detector_logic.model_info, detections)
             else:
-                print(f'No detections: {detection_data}', flush=True)
                 detections = Detections()
 
             tags = data.get('tags', [])
@@ -292,6 +295,8 @@ class DetectorNode(Node):
             self.log.error('could not reload app')
 
     async def get_detections(self, raw_image: np.ndarray, camera_id: Optional[str], tags: List[str], autoupload: Optional[str] = None) -> Optional[Dict]:
+        """Note: raw_image is a numpy array of type uint8, but not in the correrct shape!
+        It can be converted e.g. using cv2.imdecode(raw_image, cv2.IMREAD_COLOR)"""
         loop = asyncio.get_event_loop()
         detections: Detections = await loop.run_in_executor(None, self.detector_logic.evaluate, raw_image)
 

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/learning_loop_node/detector/inbox_filter/cam_observation_history.py

@@ -1,3 +1,4 @@
+import os
 from typing import List, Union
 
 from learning_loop_node.data_classes import (BoxDetection,
@@ -39,7 +40,7 @@ class CamObservationHistory:
                 continue
 
             self.recent_observations.append(Observation(detection))
-            if 0.3 <= detection.confidence <= 0.6:
+            if float(os.environ.get('MIN_UNCERTAIN_THRESHOLD', '0.3')) <= detection.confidence <= float(os.environ.get('MAX_UNCERTAIN_THRESHOLD', '0.6')):
                 causes.add('uncertain')
 
         return list(causes)

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/learning_loop_node/tests/test_downloader.py

@@ -1,5 +1,8 @@
+import os
+import shutil
+
 from learning_loop_node.data_classes import Context
-from learning_loop_node.data_exchanger import DataExchanger
+from learning_loop_node.data_exchanger import DataExchanger, check_jpeg
 from learning_loop_node.globals import GLOBALS
 
 from . import test_helper
@@ -46,3 +49,23 @@ async def test_download_training_data(data_exchanger: DataExchanger):
     image_ids = await data_exchanger.fetch_image_ids()
     image_data = await data_exchanger.download_images_data(image_ids)
     assert len(image_data) == 3
+
+
+async def test_removal_of_corrupted_images(data_exchanger: DataExchanger):
+    image_ids = await data_exchanger.fetch_image_ids()
+
+    shutil.rmtree('/tmp/img_folder', ignore_errors=True)
+    os.makedirs('/tmp/img_folder', exist_ok=True)
+    await data_exchanger.download_images(image_ids, '/tmp/img_folder')
+    num_images = len(os.listdir('/tmp/img_folder'))
+
+    # Generate two corrupted images
+    with open('/tmp/img_folder/c0.jpg', 'w') as f:
+        f.write('')
+    with open('/tmp/img_folder/c1.jpg', 'w') as f:
+        f.write('I am no image')
+
+    await data_exchanger.delete_corrupt_images('/tmp/img_folder')
+
+    assert len(os.listdir('/tmp/img_folder')) == num_images if check_jpeg else num_images - 1
+    shutil.rmtree('/tmp/img_folder', ignore_errors=True)

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/learning_loop_node/trainer/trainer_logic.py

@@ -54,8 +54,8 @@ class TrainerLogic():
         self._training: Optional[Training] = None
         self._active_training_io: Optional[ActiveTrainingIO] = None
         self._node: Optional[TrainerNode] = None
-        self.restart_after_training = bool(int(os.environ.get('RESTART_AFTER_TRAINING', '0')))
-        self.keep_old_trainings = bool(int(os.environ.get('KEEP_OLD_TRAININGS', '0')))
+        self.restart_after_training = os.environ.get('RESTART_AFTER_TRAINING', 'FALSE').lower() in ['true', '1']
+        self.keep_old_trainings = os.environ.get('KEEP_OLD_TRAININGS', 'FALSE').lower() in ['true', '1']
         self.inference_batch_size = int(os.environ.get('INFERENCE_BATCH_SIZE', '10'))
         logging.info(f'INFERENCE_BATCH_SIZE: {self.inference_batch_size}')
 
@@ -335,7 +335,7 @@ class TrainerLogic():
         try:
             new_model_id = await self._upload_model_return_new_id(self.training.context)
             if new_model_id is None:
-                raise Exception('could not upload model')
+                raise Exception('could not upload model - maybe training failed')
             assert new_model_id is not None, 'uploaded_model must be set'
             logging.info(f'successfully uploaded model and received new model id: {new_model_id}')
             self.training.model_id_for_detecting = new_model_id

{learning_loop_node-0.8.5 → learning_loop_node-0.8.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "learning_loop_node"
-version = "v0.8.5"
+version = "v0.8.7"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = ["Zauberzeug GmbH <info@zauberzeug.com>"]
 license = "MIT"
@@ -9,12 +9,12 @@ repository = "https://github.com/zauberzeug/learning_loop_node"
 
 [tool.poetry.dependencies]
 python = "^3.8"
+fastapi = ">=0.93,<1.0.0"
+fastapi-socketio = "^0.0.10"
+uvicorn = {extras = ["standard"], version = "^0.22.0"}
+fastapi-utils = "^0.2.1"
 requests = "^2.25.1"
 python-socketio = "^5.7.2"
-fastapi-socketio = "^0.0.6"
-fastapi = "^0.70.1"
-uvicorn = "^0.13.3"
-fastapi-utils = "^0.2.1"
 simplejson = "^3.17.2"
 icecream = "^2.1.0"
 aiofiles = "^0.7.0"