learning-loop-node 0.10.4__py3-none-any.whl → 0.10.5__py3-none-any.whl

learning_loop_node/trainer/io_helpers.py

@@ -159,18 +159,20 @@ class ActiveTrainingIO:
     async def _upload_detections_batched(self, context: Context, detections: List[Detections]):
         batch_size = 100
         skip_detections = self.load_detection_upload_progress()
-        up_count = 0
         for i in range(skip_detections, len(detections), batch_size):
-            up_count += 1
-            up_progress = i+batch_size
-            batch_detections = detections[i:up_progress]
-            await self._upload_detections(context, batch_detections, up_progress)
+            up_progress = i + batch_size if i + batch_size < len(detections) else 0
+            batch_detections = detections[i:i + batch_size]
+            await self._upload_detections_and_save_progress(context, batch_detections, up_progress)
             skip_detections = up_progress
 
         logging.info('uploaded %d detections', len(detections))
 
-    async def _upload_detections(self, context: Context, batch_detections: List[Detections], up_progress: int):
+    async def _upload_detections_and_save_progress(self, context: Context, batch_detections: List[Detections], up_progress: int):
+        if len(batch_detections) == 0:
+            print('skipping empty batch', flush=True)
+            return
         detections_json = [jsonable_encoder(asdict(detections)) for detections in batch_detections]
+        print(f'uploading {len(detections_json)} detections', flush=True)
         response = await self.loop_communicator.post(
             f'/{context.organization}/projects/{context.project}/detections', json=detections_json)
         if response.status_code != 200:
@@ -179,7 +181,4 @@ class ActiveTrainingIO:
             raise Exception(msg)
 
         logging.info('successfully uploaded detections')
-        if up_progress >= len(batch_detections):
-            self.save_detection_upload_progress(0)
-        else:
-            self.save_detection_upload_progress(up_progress)
+        self.save_detection_upload_progress(up_progress)

learning_loop_node/trainer/tests/states/test_state_upload_detections.py

@@ -37,7 +37,7 @@ async def create_valid_detection_file(trainer: TrainerLogic, number_of_entries:
                                                              'point_detections': [], 'segmentation_detections': []})
     detections = [detection_entry] * number_of_entries
 
-    assert trainer.active_training_io is not None  # pylint: disable=protected-access
+    assert trainer.active_training_io is not None
     trainer.active_training_io.save_detections(detections, file_index)
 
 
@@ -80,21 +80,21 @@ async def test_ensure_all_detections_are_uploaded(test_initialized_trainer: Test
     create_active_training_file(trainer, training_state=TrainerState.Detected)
     trainer._init_from_last_training()
 
-    await create_valid_detection_file(trainer, 2, 0)
-    await create_valid_detection_file(trainer, 2, 1)
+    await create_valid_detection_file(trainer, 4, 0)
+    await create_valid_detection_file(trainer, 4, 1)
 
     assert trainer.active_training_io.load_detections_upload_file_index() == 0
     detections = trainer.active_training_io.load_detections(0)
-    assert len(detections) == 2
+    assert len(detections) == 4
 
-    batch_size = 1
+    batch_size = 2
     skip_detections = trainer.active_training_io.load_detection_upload_progress()
     for i in range(skip_detections, len(detections), batch_size):
         batch_detections = detections[i:i+batch_size]
-        # pylint: disable=protected-access
-        await trainer.active_training_io._upload_detections(trainer.training.context, batch_detections, i + batch_size)
+        progress = i + batch_size if i + batch_size < len(detections) else 0
+        await trainer.active_training_io._upload_detections_and_save_progress(trainer.training.context, batch_detections, progress)
 
-        expected_value = i + batch_size if i + batch_size < len(detections) else 0  # Progress is reset for every file
+        expected_value = progress  # Progress is reset for every file
         assert trainer.active_training_io.load_detection_upload_progress() == expected_value
 
     assert trainer.active_training_io.load_detections_upload_file_index() == 0
@@ -107,10 +107,11 @@ async def test_ensure_all_detections_are_uploaded(test_initialized_trainer: Test
     skip_detections = trainer.active_training_io.load_detection_upload_progress()
     for i in range(skip_detections, len(detections), batch_size):
         batch_detections = detections[i:i+batch_size]
-        # pylint: disable=protected-access
-        await trainer.active_training_io._upload_detections(trainer.training.context, batch_detections, i + batch_size)
 
-        expected_value = i + batch_size if i + batch_size < len(detections) else 0  # Progress is reset for every file
+        progress = i + batch_size if i + batch_size < len(detections) else 0
+        await trainer.active_training_io._upload_detections_and_save_progress(trainer.training.context, batch_detections, progress)
+
+        expected_value = progress  # Progress is reset for every file
         assert trainer.active_training_io.load_detection_upload_progress() == expected_value
         assert trainer.active_training_io.load_detections_upload_file_index() == 1
 
@@ -160,5 +161,5 @@ async def test_abort_uploading(test_initialized_trainer: TestingTrainerLogic):
     await trainer.stop()
     await asyncio.sleep(0.1)
 
-    assert trainer._training is None  # pylint: disable=protected-access
+    assert trainer._training is None
     assert trainer.node.last_training_io.exists() is False

{learning_loop_node-0.10.4.dist-info → learning_loop_node-0.10.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.4
+Version: 0.10.5
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.10.4.dist-info → learning_loop_node-0.10.5.dist-info}/RECORD

@@ -58,7 +58,7 @@ learning_loop_node/tests/test_learning_loop_node.py,sha256=4qWi1ovBzebUAbvw8ecSa
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/downloader.py,sha256=qzx7zzObcFEvRVQFe8gi8KJNIapASi1_XssbspXD1Rw,1469
 learning_loop_node/trainer/executor.py,sha256=j-1pkc_5b1Y0Bh8KiQthAK7SoOfJelr5Qi7L0p1sTxE,3933
-learning_loop_node/trainer/io_helpers.py,sha256=fyjImT1NY10gLV0KM4Ns7sRrtKFswVJU_pV9-c3IVEY,7152
+learning_loop_node/trainer/io_helpers.py,sha256=Ylxz8HAId0Jlz95So5kXdJEp1yKQuwroDKIhbTUscF4,7257
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/rest/backdoor_controls.py,sha256=YQcG0KwxzKDNYeMtHrSwr26q__N7ty0o6Kar6CLWAd0,5869
 learning_loop_node/trainer/rest/controls.py,sha256=XF37i2edeMHKdSXyJc4ZqaTZ38u6d3u3Sb3C-Mwyfko,934
@@ -72,7 +72,7 @@ learning_loop_node/trainer/tests/states/test_state_download_train_model.py,sha25
 learning_loop_node/trainer/tests/states/test_state_prepare.py,sha256=fx9_bgPTaR5ANVB8n_hW8dXcaJIh_iKEnInmhzamZ9E,2432
 learning_loop_node/trainer/tests/states/test_state_sync_confusion_matrix.py,sha256=zfNbHB3GFSJXXoEkW-8PYtmX62md3feWp4oisyzs8A4,4773
 learning_loop_node/trainer/tests/states/test_state_train.py,sha256=j1vedjH2EwLTgHhon6eR9ttp-Sw9ozR9-9QgAKlFO-M,3248
-learning_loop_node/trainer/tests/states/test_state_upload_detections.py,sha256=38MumBw_N6Pt5cXI8x-xZjczmWUc7BIqyw4TSfim9h4,7751
+learning_loop_node/trainer/tests/states/test_state_upload_detections.py,sha256=u31gC0-Z2EVTnia1dyY2yNGDGAeyIXPfObBgrEWHhVQ,7674
 learning_loop_node/trainer/tests/states/test_state_upload_model.py,sha256=gAXJJcoxj2EVdFJcPAeqxGAjeL1a4ofLtytnM4joi-Q,3808
 learning_loop_node/trainer/tests/test_errors.py,sha256=8H-kjs9kEBoHWcQVJIZvW5zcwCs1VQI5Tf5I0VSbCUA,2245
 learning_loop_node/trainer/tests/test_trainer_states.py,sha256=OBrClH6srAM2hqqel2xTtfHCeTKYZlG_S4KO2G2GrS4,1147
@@ -80,6 +80,6 @@ learning_loop_node/trainer/tests/testing_trainer_logic.py,sha256=7sQ6okiOhM4IhvR
 learning_loop_node/trainer/trainer_logic.py,sha256=PJxiO1chPdvpq8UTtzv_nVam9CouCswX9b1FnRwT2Tw,8411
 learning_loop_node/trainer/trainer_logic_generic.py,sha256=KFDuxgzrGITHQaJoGvhjHxWzhbb4Q7HBxSpks4CeGBg,24801
 learning_loop_node/trainer/trainer_node.py,sha256=bcyOMeLXrLuLgsPqS8lwEOSZ6vCjGLgT0pLXgaylI1Q,4155
-learning_loop_node-0.10.4.dist-info/METADATA,sha256=bOfPBBjEdohsOEBpAD7grcYOQpo5d7akcNrbxmA3hdY,9287
-learning_loop_node-0.10.4.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.10.4.dist-info/RECORD,,
+learning_loop_node-0.10.5.dist-info/METADATA,sha256=91R8hxfoYIBgEeAqmzV5Durgrseq70TEsJRgD243pVs,9287
+learning_loop_node-0.10.5.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.10.5.dist-info/RECORD,,