learning-loop-node 0.10.3__tar.gz → 0.10.5__tar.gz

{learning_loop_node-0.10.3 → learning_loop_node-0.10.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.3
+Version: 0.10.5
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.10.3 → learning_loop_node-0.10.5}/learning_loop_node/detector/detector_node.py

@@ -43,6 +43,8 @@ class DetectorNode(Node):
         self.operation_mode: OperationMode = OperationMode.Startup
         self.connected_clients: List[str] = []
 
+        self.detection_lock = asyncio.Lock()
+
         self.outbox: Outbox = Outbox()
         self.data_exchanger = DataExchanger(
             Context(organization=self.organization, project=self.project),
@@ -300,8 +302,9 @@ class DetectorNode(Node):
         """Note: raw_image is a numpy array of type uint8, but not in the correrct shape!
         It can be converted e.g. using cv2.imdecode(raw_image, cv2.IMREAD_COLOR)"""
         loop = asyncio.get_event_loop()
+        await self.detection_lock.acquire()
         detections: Detections = await loop.run_in_executor(None, self.detector_logic.evaluate, raw_image)
-
+        self.detection_lock.release()
         for seg_detection in detections.segmentation_detections:
             if isinstance(seg_detection.shape, Shape):
                 shapes = ','.join([str(value) for p in seg_detection.shape.points for _,

{learning_loop_node-0.10.3 → learning_loop_node-0.10.5}/learning_loop_node/trainer/io_helpers.py

@@ -159,18 +159,20 @@ class ActiveTrainingIO:
     async def _upload_detections_batched(self, context: Context, detections: List[Detections]):
         batch_size = 100
         skip_detections = self.load_detection_upload_progress()
-        up_count = 0
         for i in range(skip_detections, len(detections), batch_size):
-            up_count += 1
-            up_progress = i+batch_size
-            batch_detections = detections[i:up_progress]
-            await self._upload_detections(context, batch_detections, up_progress)
+            up_progress = i + batch_size if i + batch_size < len(detections) else 0
+            batch_detections = detections[i:i + batch_size]
+            await self._upload_detections_and_save_progress(context, batch_detections, up_progress)
             skip_detections = up_progress
 
         logging.info('uploaded %d detections', len(detections))
 
-    async def _upload_detections(self, context: Context, batch_detections: List[Detections], up_progress: int):
+    async def _upload_detections_and_save_progress(self, context: Context, batch_detections: List[Detections], up_progress: int):
+        if len(batch_detections) == 0:
+            print('skipping empty batch', flush=True)
+            return
         detections_json = [jsonable_encoder(asdict(detections)) for detections in batch_detections]
+        print(f'uploading {len(detections_json)} detections', flush=True)
         response = await self.loop_communicator.post(
             f'/{context.organization}/projects/{context.project}/detections', json=detections_json)
         if response.status_code != 200:
@@ -179,7 +181,4 @@ class ActiveTrainingIO:
             raise Exception(msg)
 
         logging.info('successfully uploaded detections')
-        if up_progress >= len(batch_detections):
-            self.save_detection_upload_progress(0)
-        else:
-            self.save_detection_upload_progress(up_progress)
+        self.save_detection_upload_progress(up_progress)

{learning_loop_node-0.10.3 → learning_loop_node-0.10.5}/learning_loop_node/trainer/tests/states/test_state_upload_detections.py

@@ -37,7 +37,7 @@ async def create_valid_detection_file(trainer: TrainerLogic, number_of_entries:
                                                              'point_detections': [], 'segmentation_detections': []})
     detections = [detection_entry] * number_of_entries
 
-    assert trainer.active_training_io is not None  # pylint: disable=protected-access
+    assert trainer.active_training_io is not None
     trainer.active_training_io.save_detections(detections, file_index)
 
 
@@ -80,21 +80,21 @@ async def test_ensure_all_detections_are_uploaded(test_initialized_trainer: Test
     create_active_training_file(trainer, training_state=TrainerState.Detected)
     trainer._init_from_last_training()
 
-    await create_valid_detection_file(trainer, 2, 0)
-    await create_valid_detection_file(trainer, 2, 1)
+    await create_valid_detection_file(trainer, 4, 0)
+    await create_valid_detection_file(trainer, 4, 1)
 
     assert trainer.active_training_io.load_detections_upload_file_index() == 0
     detections = trainer.active_training_io.load_detections(0)
-    assert len(detections) == 2
+    assert len(detections) == 4
 
-    batch_size = 1
+    batch_size = 2
     skip_detections = trainer.active_training_io.load_detection_upload_progress()
     for i in range(skip_detections, len(detections), batch_size):
         batch_detections = detections[i:i+batch_size]
-        # pylint: disable=protected-access
-        await trainer.active_training_io._upload_detections(trainer.training.context, batch_detections, i + batch_size)
+        progress = i + batch_size if i + batch_size < len(detections) else 0
+        await trainer.active_training_io._upload_detections_and_save_progress(trainer.training.context, batch_detections, progress)
 
-        expected_value = i + batch_size if i + batch_size < len(detections) else 0  # Progress is reset for every file
+        expected_value = progress  # Progress is reset for every file
         assert trainer.active_training_io.load_detection_upload_progress() == expected_value
 
     assert trainer.active_training_io.load_detections_upload_file_index() == 0
@@ -107,10 +107,11 @@ async def test_ensure_all_detections_are_uploaded(test_initialized_trainer: Test
     skip_detections = trainer.active_training_io.load_detection_upload_progress()
     for i in range(skip_detections, len(detections), batch_size):
         batch_detections = detections[i:i+batch_size]
-        # pylint: disable=protected-access
-        await trainer.active_training_io._upload_detections(trainer.training.context, batch_detections, i + batch_size)
 
-        expected_value = i + batch_size if i + batch_size < len(detections) else 0  # Progress is reset for every file
+        progress = i + batch_size if i + batch_size < len(detections) else 0
+        await trainer.active_training_io._upload_detections_and_save_progress(trainer.training.context, batch_detections, progress)
+
+        expected_value = progress  # Progress is reset for every file
         assert trainer.active_training_io.load_detection_upload_progress() == expected_value
         assert trainer.active_training_io.load_detections_upload_file_index() == 1
 
@@ -160,5 +161,5 @@ async def test_abort_uploading(test_initialized_trainer: TestingTrainerLogic):
     await trainer.stop()
     await asyncio.sleep(0.1)
 
-    assert trainer._training is None  # pylint: disable=protected-access
+    assert trainer._training is None
     assert trainer.node.last_training_io.exists() is False

{learning_loop_node-0.10.3 → learning_loop_node-0.10.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "learning_loop_node"
-version = "v0.10.3"
+version = "v0.10.5"
 description = "Python Library for Nodes which connect to the Zauberzeug Learning Loop"
 authors = ["Zauberzeug GmbH <info@zauberzeug.com>"]
 license = "MIT"