learning-loop-node 0.8.5__py3-none-any.whl → 0.8.7__py3-none-any.whl

learning_loop_node/data_exchanger.py

@@ -59,10 +59,23 @@ class DataExchanger():
             logging.warning('context was not set yet')
             return
 
+        await self.delete_corrupt_images(image_folder)
         new_image_ids = await asyncio.get_event_loop().run_in_executor(None, DataExchanger.filter_existing_images, image_ids, image_folder)
         paths, ids = create_resource_paths(self.context.organization, self.context.project, new_image_ids)
         await self._download_images(paths, ids, image_folder)
 
+    @staticmethod
+    async def delete_corrupt_images(image_folder: str) -> None:
+        logging.info('deleting corrupt images')
+        n_deleted = 0
+        for image in glob(f'{image_folder}/*.jpg'):
+            if not await DataExchanger.is_valid_image(image):
+                logging.debug(f'  deleting image {image}')
+                os.remove(image)
+                n_deleted += 1
+
+        logging.info(f'deleted {n_deleted} images')
+
     @staticmethod
     def filter_existing_images(all_image_ids, image_folder) -> List[str]:
         logging.info(f'### Going to filter {len(all_image_ids)} images ids')
@@ -131,8 +144,9 @@ class DataExchanger():
         if not await self.is_valid_image(filename):
             os.remove(filename)
 
-    async def is_valid_image(self, filename: str) -> bool:
-        if not os.path.isfile(filename):
+    @staticmethod
+    async def is_valid_image(filename: str) -> bool:
+        if not os.path.isfile(filename) or os.path.getsize(filename) == 0:
             return False
         if not check_jpeg:
             return True

learning_loop_node/detector/detector_logic.py

@@ -46,7 +46,7 @@ class DetectorLogic():
 
     @abstractmethod
     def init(self):
-        """Initialize the model. Note that `model_info` is available as `self.model_info`"""
+        """Called when a (new) model was loaded. Initialize the model. Model information available via `self.model_info`"""
 
     @abstractmethod
     def evaluate(self, image: np.ndarray) -> Detections:

learning_loop_node/detector/detector_node.py

@@ -142,10 +142,13 @@ class DetectorNode(Node):
 
             detection_data = data.get('detections', {})
             if detection_data and self.detector_logic.is_initialized:
-                detections = from_dict(data_class=Detections, data=detection_data)
+                try:
+                    detections = from_dict(data_class=Detections, data=detection_data)
+                except Exception as e:
+                    self.log.exception('could not parse detections')
+                    return {'error': str(e)}
                 detections = self.add_category_id_to_detections(self.detector_logic.model_info, detections)
             else:
-                print(f'No detections: {detection_data}', flush=True)
                 detections = Detections()
 
             tags = data.get('tags', [])
@@ -292,6 +295,8 @@ class DetectorNode(Node):
             self.log.error('could not reload app')
 
     async def get_detections(self, raw_image: np.ndarray, camera_id: Optional[str], tags: List[str], autoupload: Optional[str] = None) -> Optional[Dict]:
+        """Note: raw_image is a numpy array of type uint8, but not in the correrct shape!
+        It can be converted e.g. using cv2.imdecode(raw_image, cv2.IMREAD_COLOR)"""
         loop = asyncio.get_event_loop()
         detections: Detections = await loop.run_in_executor(None, self.detector_logic.evaluate, raw_image)
 

learning_loop_node/detector/inbox_filter/cam_observation_history.py

@@ -1,3 +1,4 @@
+import os
 from typing import List, Union
 
 from learning_loop_node.data_classes import (BoxDetection,
@@ -39,7 +40,7 @@ class CamObservationHistory:
                 continue
 
             self.recent_observations.append(Observation(detection))
-            if 0.3 <= detection.confidence <= 0.6:
+            if float(os.environ.get('MIN_UNCERTAIN_THRESHOLD', '0.3')) <= detection.confidence <= float(os.environ.get('MAX_UNCERTAIN_THRESHOLD', '0.6')):
                 causes.add('uncertain')
 
         return list(causes)

learning_loop_node/tests/test_downloader.py

@@ -1,5 +1,8 @@
+import os
+import shutil
+
 from learning_loop_node.data_classes import Context
-from learning_loop_node.data_exchanger import DataExchanger
+from learning_loop_node.data_exchanger import DataExchanger, check_jpeg
 from learning_loop_node.globals import GLOBALS
 
 from . import test_helper
@@ -46,3 +49,23 @@ async def test_download_training_data(data_exchanger: DataExchanger):
     image_ids = await data_exchanger.fetch_image_ids()
     image_data = await data_exchanger.download_images_data(image_ids)
     assert len(image_data) == 3
+
+
+async def test_removal_of_corrupted_images(data_exchanger: DataExchanger):
+    image_ids = await data_exchanger.fetch_image_ids()
+
+    shutil.rmtree('/tmp/img_folder', ignore_errors=True)
+    os.makedirs('/tmp/img_folder', exist_ok=True)
+    await data_exchanger.download_images(image_ids, '/tmp/img_folder')
+    num_images = len(os.listdir('/tmp/img_folder'))
+
+    # Generate two corrupted images
+    with open('/tmp/img_folder/c0.jpg', 'w') as f:
+        f.write('')
+    with open('/tmp/img_folder/c1.jpg', 'w') as f:
+        f.write('I am no image')
+
+    await data_exchanger.delete_corrupt_images('/tmp/img_folder')
+
+    assert len(os.listdir('/tmp/img_folder')) == num_images if check_jpeg else num_images - 1
+    shutil.rmtree('/tmp/img_folder', ignore_errors=True)

learning_loop_node/trainer/trainer_logic.py

@@ -54,8 +54,8 @@ class TrainerLogic():
         self._training: Optional[Training] = None
         self._active_training_io: Optional[ActiveTrainingIO] = None
         self._node: Optional[TrainerNode] = None
-        self.restart_after_training = bool(int(os.environ.get('RESTART_AFTER_TRAINING', '0')))
-        self.keep_old_trainings = bool(int(os.environ.get('KEEP_OLD_TRAININGS', '0')))
+        self.restart_after_training = os.environ.get('RESTART_AFTER_TRAINING', 'FALSE').lower() in ['true', '1']
+        self.keep_old_trainings = os.environ.get('KEEP_OLD_TRAININGS', 'FALSE').lower() in ['true', '1']
         self.inference_batch_size = int(os.environ.get('INFERENCE_BATCH_SIZE', '10'))
         logging.info(f'INFERENCE_BATCH_SIZE: {self.inference_batch_size}')
 
@@ -335,7 +335,7 @@ class TrainerLogic():
         try:
             new_model_id = await self._upload_model_return_new_id(self.training.context)
             if new_model_id is None:
-                raise Exception('could not upload model')
+                raise Exception('could not upload model - maybe training failed')
             assert new_model_id is not None, 'uploaded_model must be set'
             logging.info(f'successfully uploaded model and received new model id: {new_model_id}')
             self.training.model_id_for_detecting = new_model_id

{learning_loop_node-0.8.5.dist-info → learning_loop_node-0.8.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.8.5
+Version: 0.8.7
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT
@@ -18,8 +18,8 @@ Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
 Requires-Dist: aiohttp (>=3.8.4,<4.0.0)
 Requires-Dist: async_generator (>=1.10,<2.0)
 Requires-Dist: dacite (>=1.8.1,<2.0.0)
-Requires-Dist: fastapi (>=0.70.1,<0.71.0)
-Requires-Dist: fastapi-socketio (>=0.0.6,<0.0.7)
+Requires-Dist: fastapi (>=0.93,<1.0.0)
+Requires-Dist: fastapi-socketio (>=0.0.10,<0.0.11)
 Requires-Dist: fastapi-utils (>=0.2.1,<0.3.0)
 Requires-Dist: httpx (>=0.24.1,<0.25.0)
 Requires-Dist: icecream (>=2.1.0,<3.0.0)
@@ -33,7 +33,7 @@ Requires-Dist: python-socketio (>=5.7.2,<6.0.0)
 Requires-Dist: requests (>=2.25.1,<3.0.0)
 Requires-Dist: simplejson (>=3.17.2,<4.0.0)
 Requires-Dist: tqdm (>=4.63.0,<5.0.0)
-Requires-Dist: uvicorn (>=0.13.3,<0.14.0)
+Requires-Dist: uvicorn[standard] (>=0.22.0,<0.23.0)
 Requires-Dist: werkzeug (>=2.0.1,<3.0.0)
 Project-URL: Repository, https://github.com/zauberzeug/learning_loop_node
 Description-Content-Type: text/markdown
@@ -57,16 +57,18 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                   | Alias        | Purpose                                           | Required by          |
-| ---------------------- | ------------ | ------------------------------------------------- | -------------------- |
-| LOOP_HOST              | HOST         | Learning Loop address (e.g. learning-loop.ai)     | all                  |
-| LOOP_USERNAME          | USERNAME     | Learning Loop user name                           | all besides Detector |
-| LOOP_PASSWORD          | PASSWORD     | Learning Loop password                            | all besides Detector |
-| LOOP_ORGANIZATION      | ORGANIZATION | Organization name                                 | Detector             |
-| LOOP_PROJECT           | PROJECT      | Project name                                      | Detector             |
-| INFERENCE_BATCH_SIZE   | -            | Batch size of trainer when calculating detections | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING | -            | Restart the trainer after training (set to 1)     | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS     | -            | Do not delete old trainings (set to 1)            | Trainer (opt.)       |
+| Name                    | Alias        | Purpose                                                      | Required by          |
+| ----------------------- | ------------ | ------------------------------------------------------------ | -------------------- |
+| LOOP_HOST               | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
+| LOOP_USERNAME           | USERNAME     | Learning Loop user name                                      | all besides Detector |
+| LOOP_PASSWORD           | PASSWORD     | Learning Loop password                                       | all besides Detector |
+| LOOP_ORGANIZATION       | ORGANIZATION | Organization name                                            | Detector             |
+| LOOP_PROJECT            | PROJECT      | Project name                                                 | Detector             |
+| MIN_UNCERTAIN_THRESHOLD | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
+| MAX_UNCERTAIN_THRESHOLD | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
+| INFERENCE_BATCH_SIZE    | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
+| RESTART_AFTER_TRAINING  | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
+| KEEP_OLD_TRAININGS      | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
 
 #### Testing
 

{learning_loop_node-0.8.5.dist-info → learning_loop_node-0.8.7.dist-info}/RECORD

@@ -14,12 +14,12 @@ learning_loop_node/data_classes/detections.py,sha256=J-8z6OrUw_QMyaUOdgODxMF8Z97
 learning_loop_node/data_classes/general.py,sha256=3LYglqOMvQIfNn0_UUwbmdyb905Fzna1yf5p55cnPmE,4635
 learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
 learning_loop_node/data_classes/training.py,sha256=VLGnZTJXh6p7K16lO41lPNlMl6EAIO_kVPfBu8w96ug,4870
-learning_loop_node/data_exchanger.py,sha256=6KUrc290fDeT5dnca-eT-WmTowXfZjjS3sSFgCgesbo,9332
+learning_loop_node/data_exchanger.py,sha256=YAGn7laOZpVIwcK6O0WX-6ec36ypvxlW-u7UfCgblDQ,9876
 learning_loop_node/detector/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-learning_loop_node/detector/detector_logic.py,sha256=vVEJsGK6LuJLJxBk-Rfwa3KBbm5LDKrJa2CH9hSbDWc,1710
-learning_loop_node/detector/detector_node.py,sha256=syZd2hBRTAUoz5QC8afsQr7u-STKPu6i62JPXxMEPmg,16277
+learning_loop_node/detector/detector_logic.py,sha256=qjnFzrVO020-kvA1dcI5zoZLANynMx3RR61zEIS0udo,1741
+learning_loop_node/detector/detector_node.py,sha256=1kUFLIa_S1mZ0PqfeqsNdcrGfJYcR09mhLSp6XciDZY,16559
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=9zJo3ow_NjjWBC8ZvfLFR2Cgwbr5IsYD3d2mIlP-A70,3323
+learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=5iNR2cEPujjY83sfQug97-hZ-T8iFc44gi1s_yd7GO0,3437
 learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=s2FuwZ-tD_5obkSutstjc8pE_hLGbrv9WjrEO9t8rJ8,1011
 learning_loop_node/detector/inbox_filter/tests/test_observation.py,sha256=ORN08yjprqmgmtU25RsVysniyrWX-qGvqFN8ZkkYxow,1385
 learning_loop_node/detector/inbox_filter/tests/test_relevance_group.py,sha256=RUgsk1CnKSOCRZBzNjE7AZTqk06-yelgUqvHFRLH7_I,7865
@@ -53,7 +53,7 @@ learning_loop_node/tests/test_data/file_1.txt,sha256=Lis06nfvbFPVCBZyEgQlfI_Nle2
 learning_loop_node/tests/test_data/file_2.txt,sha256=Xp8EETGhZBdVAgb4URowSSpOytwwwJdV0Renkdur7R8,19
 learning_loop_node/tests/test_data/model.json,sha256=_xNDucGOWila8gWnu8yFfrqmQ45Xq-_39eLKzjRtvpE,516
 learning_loop_node/tests/test_data_classes.py,sha256=m8LEk1quGErxuPzNdW_ExqQjkwE4u7ribwnTdyeiHR8,788
-learning_loop_node/tests/test_downloader.py,sha256=SxFMAvK1y0ul_TuMl5oP-TDIMTu6ChRO-2Xa5Fkc1mU,1944
+learning_loop_node/tests/test_downloader.py,sha256=sMhpQspkzw3VrYG2ws064SRCucf_9ceD_e3qkgzJxtk,2755
 learning_loop_node/tests/test_executor.py,sha256=ILBKfX-A2zlxfot0xoz3jpG3XcEMo3kxQKkPgoz_aT4,1948
 learning_loop_node/tests/test_helper.py,sha256=fFcmTqgw1RJtuQHWk5_Mn9hc-wXIFcQow2JphLFwrlI,2412
 learning_loop_node/tests/test_learning_loop_node.py,sha256=4qWi1ovBzebUAbvw8ecSa-TBGKYuJvlKe2AMnMZ-Qs8,701
@@ -79,9 +79,9 @@ learning_loop_node/trainer/tests/states/test_state_upload_model.py,sha256=3BzIpA
 learning_loop_node/trainer/tests/test_errors.py,sha256=QHZK1YVSXqQskUZbl0cU1sjJeDv3dz_2iF0LFhMwe58,1918
 learning_loop_node/trainer/tests/test_trainer_states.py,sha256=HeIFdAStf6KBT9lQGpVdeLQbLCgb0cqDO2Z_FJ1IO8Y,1144
 learning_loop_node/trainer/tests/testing_trainer_logic.py,sha256=DCKNhBFsV2uN-MNCpkKx8XNoAwWc_7Kfrj1kRpJEEVk,3995
-learning_loop_node/trainer/trainer_logic.py,sha256=ggwlJTv4EXhxy1KEtnZ2tv-NvnIhNOTDeUWFuPFCmUE,31804
+learning_loop_node/trainer/trainer_logic.py,sha256=_Z5LAVRRL9PFmWgWng3zqdrOymFQuv8AeZHGxLHcSno,31864
 learning_loop_node/trainer/trainer_node.py,sha256=-bJ6xoOdqG9aZL5Ok2Z5IHGrQIbnZ4KRy323CngYnNI,7780
 learning_loop_node/trainer/training_syncronizer.py,sha256=qzJbv9WqApbM1aNQNg98IjTWed2z__UKocQ_MJP9gz8,1884
-learning_loop_node-0.8.5.dist-info/METADATA,sha256=H-FTRWa7eN9eH0R7nMjBS-r_WmhuHG5csB3g00Jz8zs,8770
-learning_loop_node-0.8.5.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.8.5.dist-info/RECORD,,
+learning_loop_node-0.8.7.dist-info/METADATA,sha256=cQ7e5nHbiuoTU_pnRCHTaLEUeYOMF5YOlk5GyBQPvoU,9157
+learning_loop_node-0.8.7.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.8.7.dist-info/RECORD,,