learning-loop-node 0.10.4__py3-none-any.whl → 0.10.6__py3-none-any.whl

learning_loop_node/annotation/tests/test_annotator_node.py

@@ -1,3 +1,4 @@
+import asyncio
 import os
 from dataclasses import asdict
 from typing import Dict
@@ -7,10 +8,8 @@ from fastapi.encoders import jsonable_encoder
 
 from learning_loop_node.annotation.annotator_logic import AnnotatorLogic
 from learning_loop_node.annotation.annotator_node import AnnotatorNode
-from learning_loop_node.data_classes import (AnnotationData,
-                                             AnnotationEventType, Category,
-                                             CategoryType, Context, Point,
-                                             ToolOutput, UserInput)
+from learning_loop_node.data_classes import (AnnotationData, AnnotationEventType, Category, CategoryType, Context,
+                                             Point, ToolOutput, UserInput)
 
 
 class MockedAnnotatatorLogic(AnnotatorLogic):
@@ -29,7 +28,7 @@ def default_user_input() -> UserInput:
         coordinate=Point(x=0, y=0),
         event_type=AnnotationEventType.LeftMouseDown,
         context=Context(organization='zauberzeug', project='pytest_p'),
-        image_uuid='501205a9-9b64-3df0-3785-507a677b7f05',
+        image_uuid='f786350c-89ca-9424-9b00-720a9a85fe09',
         category=Category(id='some_id', name='category_1', description='',
                           hotkey='', color='', type=CategoryType.Segmentation)
     )
@@ -40,7 +39,8 @@ def default_user_input() -> UserInput:
 
 @pytest.mark.asyncio
 async def test_image_download(setup_test_project):  # pylint: disable=unused-argument
-    image_path = '/tmp/learning_loop_lib_data/zauberzeug/pytest_p/images/501205a9-9b64-3df0-3785-507a677b7f05.jpg'
+    # TODO: This test depends on a pseudo-random uuid..
+    image_path = '/tmp/learning_loop_lib_data/zauberzeug/pytest_p/images/f786350c-89ca-9424-9b00-720a9a85fe09.jpg'
 
     assert os.path.exists(image_path) is False
 

learning_loop_node/detector/detector_node.py

@@ -156,9 +156,6 @@ class DetectorNode(Node):
 
             tags = data.get('tags', [])
             tags.append('picked_by_system')
-            camera_id = data.get('camera-id', None) or data.get('mac', None)
-            if camera_id is not None:
-                tags.append(camera_id)
 
             loop = asyncio.get_event_loop()
             try:
@@ -315,8 +312,6 @@ class DetectorNode(Node):
         n_po, n_se = len(detections.point_detections), len(detections.segmentation_detections)
         self.log.info(f'detected:{n_bo} boxes, {n_po} points, {n_se} segs, {n_cl} classes')
 
-        if camera_id is not None:
-            tags.append(camera_id)
         if autoupload is None or autoupload == 'filtered':  # NOTE default is filtered
             Thread(target=self.relevance_filter.may_upload_detections,
                    args=(detections, camera_id, raw_image, tags)).start()

learning_loop_node/tests/test_executor.py

@@ -44,6 +44,7 @@ async def test_executor_lifecycle():
 
     assert not executor.is_running()
     sleep(1)
+    # NOTE: It happend that this process became a zombie process which leads to repeated test failures -> restart machine required
     assert_process_is_running('some_executable.sh', False)
 
 

learning_loop_node/trainer/executor.py

@@ -3,7 +3,7 @@ import logging
 import os
 import shlex
 from io import BufferedWriter
-from typing import List, Optional
+from typing import List, Optional, Dict
 
 
 class Executor:
@@ -16,7 +16,7 @@ class Executor:
 
         self.path = base_path
         self.log_file_path = f'{self.path}/{log_name}'
-        self.log_file: None | BufferedWriter = None
+        self.log_file: Optional[BufferedWriter] = None
         self._process: Optional[asyncio.subprocess.Process] = None  # pylint: disable=no-member
         os.makedirs(self.path, exist_ok=True)
 
@@ -26,7 +26,7 @@ class Executor:
             return self._process
         return None
 
-    async def start(self, cmd: str, env: Optional[dict[str, str]] = None) -> None:
+    async def start(self, cmd: str, env: Optional[Dict[str, str]] = None) -> None:
         """Start the process with the given command and environment variables."""
 
         full_env = os.environ.copy()

learning_loop_node/trainer/io_helpers.py

@@ -159,18 +159,20 @@ class ActiveTrainingIO:
     async def _upload_detections_batched(self, context: Context, detections: List[Detections]):
         batch_size = 100
         skip_detections = self.load_detection_upload_progress()
-        up_count = 0
         for i in range(skip_detections, len(detections), batch_size):
-            up_count += 1
-            up_progress = i+batch_size
-            batch_detections = detections[i:up_progress]
-            await self._upload_detections(context, batch_detections, up_progress)
+            up_progress = i + batch_size if i + batch_size < len(detections) else 0
+            batch_detections = detections[i:i + batch_size]
+            await self._upload_detections_and_save_progress(context, batch_detections, up_progress)
             skip_detections = up_progress
 
         logging.info('uploaded %d detections', len(detections))
 
-    async def _upload_detections(self, context: Context, batch_detections: List[Detections], up_progress: int):
+    async def _upload_detections_and_save_progress(self, context: Context, batch_detections: List[Detections], up_progress: int):
+        if len(batch_detections) == 0:
+            print('skipping empty batch', flush=True)
+            return
         detections_json = [jsonable_encoder(asdict(detections)) for detections in batch_detections]
+        print(f'uploading {len(detections_json)} detections', flush=True)
         response = await self.loop_communicator.post(
             f'/{context.organization}/projects/{context.project}/detections', json=detections_json)
         if response.status_code != 200:
@@ -179,7 +181,4 @@ class ActiveTrainingIO:
             raise Exception(msg)
 
         logging.info('successfully uploaded detections')
-        if up_progress >= len(batch_detections):
-            self.save_detection_upload_progress(0)
-        else:
-            self.save_detection_upload_progress(up_progress)
+        self.save_detection_upload_progress(up_progress)

learning_loop_node/trainer/tests/conftest.py

@@ -29,7 +29,7 @@ async def test_initialized_trainer_node():
     trainer._node = node
     trainer._init_new_training(context=Context(organization='zauberzeug', project='demo'),
                                details={'categories': [],
-                                        'id': '917d5c7f-403d-7e92-f95f-577f79c2273a',  # version 1.2 of demo project
+                                        'id': '00000000-0000-0000-0000-000000000012',  # version 1.2 of demo project
                                         'training_number': 0,
                                         'resolution': 800,
                                         'flip_rl': False,
@@ -49,7 +49,7 @@ async def test_initialized_trainer():
     trainer._node = node
     trainer._init_new_training(context=Context(organization='zauberzeug', project='demo'),
                                details={'categories': [],
-                                        'id': '917d5c7f-403d-7e92-f95f-577f79c2273a',  # version 1.2 of demo project
+                                        'id': '00000000-0000-0000-0000-000000000012',  # version 1.2 of demo project
                                         'training_number': 0,
                                         'resolution': 800,
                                         'flip_rl': False,

learning_loop_node/trainer/tests/states/test_state_detecting.py

@@ -14,11 +14,11 @@ def trainer_has_error(trainer: TrainerLogic):
     return trainer.errors.has_error_for(error_key)
 
 
-async def test_successful_detecting(test_initialized_trainer: TestingTrainerLogic):  # NOTE was a flaky test
+async def test_successful_detecting(test_initialized_trainer: TestingTrainerLogic):
     trainer = test_initialized_trainer
     create_active_training_file(trainer, training_state='train_model_uploaded',
-                                model_uuid_for_detecting='917d5c7f-403d-7e92-f95f-577f79c2273a')
-    # trainer.load_active_training()
+                                model_uuid_for_detecting='00000000-0000-0000-0000-000000000011')  # NOTE: this is the hard coded model uuid for zauberzeug/demo (model version 1.1)
+
     _ = asyncio.get_running_loop().create_task(
         trainer._perform_state('do_detections', TrainerState.Detecting, TrainerState.Detected, trainer._do_detections))
 

learning_loop_node/trainer/tests/states/test_state_upload_detections.py

@@ -37,7 +37,7 @@ async def create_valid_detection_file(trainer: TrainerLogic, number_of_entries:
                                                              'point_detections': [], 'segmentation_detections': []})
     detections = [detection_entry] * number_of_entries
 
-    assert trainer.active_training_io is not None  # pylint: disable=protected-access
+    assert trainer.active_training_io is not None
     trainer.active_training_io.save_detections(detections, file_index)
 
 
@@ -80,21 +80,21 @@ async def test_ensure_all_detections_are_uploaded(test_initialized_trainer: Test
     create_active_training_file(trainer, training_state=TrainerState.Detected)
     trainer._init_from_last_training()
 
-    await create_valid_detection_file(trainer, 2, 0)
-    await create_valid_detection_file(trainer, 2, 1)
+    await create_valid_detection_file(trainer, 4, 0)
+    await create_valid_detection_file(trainer, 4, 1)
 
     assert trainer.active_training_io.load_detections_upload_file_index() == 0
     detections = trainer.active_training_io.load_detections(0)
-    assert len(detections) == 2
+    assert len(detections) == 4
 
-    batch_size = 1
+    batch_size = 2
     skip_detections = trainer.active_training_io.load_detection_upload_progress()
     for i in range(skip_detections, len(detections), batch_size):
         batch_detections = detections[i:i+batch_size]
-        # pylint: disable=protected-access
-        await trainer.active_training_io._upload_detections(trainer.training.context, batch_detections, i + batch_size)
+        progress = i + batch_size if i + batch_size < len(detections) else 0
+        await trainer.active_training_io._upload_detections_and_save_progress(trainer.training.context, batch_detections, progress)
 
-        expected_value = i + batch_size if i + batch_size < len(detections) else 0  # Progress is reset for every file
+        expected_value = progress  # Progress is reset for every file
         assert trainer.active_training_io.load_detection_upload_progress() == expected_value
 
     assert trainer.active_training_io.load_detections_upload_file_index() == 0
@@ -107,10 +107,11 @@ async def test_ensure_all_detections_are_uploaded(test_initialized_trainer: Test
     skip_detections = trainer.active_training_io.load_detection_upload_progress()
     for i in range(skip_detections, len(detections), batch_size):
         batch_detections = detections[i:i+batch_size]
-        # pylint: disable=protected-access
-        await trainer.active_training_io._upload_detections(trainer.training.context, batch_detections, i + batch_size)
 
-        expected_value = i + batch_size if i + batch_size < len(detections) else 0  # Progress is reset for every file
+        progress = i + batch_size if i + batch_size < len(detections) else 0
+        await trainer.active_training_io._upload_detections_and_save_progress(trainer.training.context, batch_detections, progress)
+
+        expected_value = progress  # Progress is reset for every file
         assert trainer.active_training_io.load_detection_upload_progress() == expected_value
         assert trainer.active_training_io.load_detections_upload_file_index() == 1
 
@@ -160,5 +161,5 @@ async def test_abort_uploading(test_initialized_trainer: TestingTrainerLogic):
     await trainer.stop()
     await asyncio.sleep(0.1)
 
-    assert trainer._training is None  # pylint: disable=protected-access
+    assert trainer._training is None
     assert trainer.node.last_training_io.exists() is False

{learning_loop_node-0.10.4.dist-info → learning_loop_node-0.10.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.4
+Version: 0.10.6
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.10.4.dist-info → learning_loop_node-0.10.6.dist-info}/RECORD

@@ -2,7 +2,7 @@ learning_loop_node/__init__.py,sha256=onN5s8-x_xBsCM6NLmJO0Ym1sJHeCFaGw8qb0oQZmz
 learning_loop_node/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/annotation/annotator_logic.py,sha256=BTaopkJZkIf1CI5lfsVKsxbxoUIbDJrevavuQUT5e_c,1000
 learning_loop_node/annotation/annotator_node.py,sha256=wk11CQtM3A0Dr7efCn_Mw2X7ql5xn2sgEJzrIeSBC6Q,4043
-learning_loop_node/annotation/tests/test_annotator_node.py,sha256=yCox-ohCXOpCbcTit7PwtE4iLHud8qiZ6UOqqycwPxg,2065
+learning_loop_node/annotation/tests/test_annotator_node.py,sha256=VdwM7Z0vqmwewmijmUPduo9hxZYn5y1nlFauxnjxoDs,2046
 learning_loop_node/conftest.py,sha256=Q_8Dbl3RwI8X_Y5Zvzstr6NJEqY3KldZXVx618OLdb4,3492
 learning_loop_node/data_classes/__init__.py,sha256=wCX88lDgbb8V-gtVCVe9i-NvvZuMe5FX7eD_UJgYYXw,1305
 learning_loop_node/data_classes/annotations.py,sha256=iInU0Nuy_oYT_sj4k_n-W0UShCBI2cHQYrt8imymbtM,1211
@@ -13,7 +13,7 @@ learning_loop_node/data_classes/training.py,sha256=hnMHZMk-WNRERyo7U97qL09v1tIdh
 learning_loop_node/data_exchanger.py,sha256=hxF0zANA35f5EV8tkQ4yjelrKuvafMaKUya0CCjVrK0,8221
 learning_loop_node/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/detector_logic.py,sha256=se0jRFbV7BfTvCuCI3gcUllSYIZ5dxTkvdISe6pPTRg,1660
-learning_loop_node/detector/detector_node.py,sha256=k05ZXOK0OiKg4U7IMSFz-N-w1DmLvB3WF5MzMkkByP4,16776
+learning_loop_node/detector/detector_node.py,sha256=BStenBbtRuvuDxg6lscxvfz_lV_Am5MAAr0bRbTP50o,16553
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=TD346I9ymtIP0_CJXCIKMRuiXbfVVanXNu_iHAwDd7Q,3318
 learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=s2FuwZ-tD_5obkSutstjc8pE_hLGbrv9WjrEO9t8rJ8,1011
@@ -52,27 +52,27 @@ learning_loop_node/tests/test_data/file_2.txt,sha256=Xp8EETGhZBdVAgb4URowSSpOytw
 learning_loop_node/tests/test_data/model.json,sha256=_xNDucGOWila8gWnu8yFfrqmQ45Xq-_39eLKzjRtvpE,516
 learning_loop_node/tests/test_data_classes.py,sha256=m8LEk1quGErxuPzNdW_ExqQjkwE4u7ribwnTdyeiHR8,788
 learning_loop_node/tests/test_downloader.py,sha256=RkXNVQJkfPddGzxfJgSF7kc-V6Qp1OOvYYuhQ9g_ls4,2806
-learning_loop_node/tests/test_executor.py,sha256=-QLylzigUEoUzeQssPR3ECFkD2tWH-fsA_lEGXy993c,2119
+learning_loop_node/tests/test_executor.py,sha256=DuwYv-27R0fUqkcMmIiArXhiPhDnLzgNC4vp2r6Jyj4,2250
 learning_loop_node/tests/test_helper.py,sha256=AjOrTu3dHIlJLYI0mxcNx8MCmFF6IjLhHtPzuai4O9E,2334
 learning_loop_node/tests/test_learning_loop_node.py,sha256=4qWi1ovBzebUAbvw8ecSa-TBGKYuJvlKe2AMnMZ-Qs8,701
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/downloader.py,sha256=qzx7zzObcFEvRVQFe8gi8KJNIapASi1_XssbspXD1Rw,1469
-learning_loop_node/trainer/executor.py,sha256=j-1pkc_5b1Y0Bh8KiQthAK7SoOfJelr5Qi7L0p1sTxE,3933
-learning_loop_node/trainer/io_helpers.py,sha256=fyjImT1NY10gLV0KM4Ns7sRrtKFswVJU_pV9-c3IVEY,7152
+learning_loop_node/trainer/executor.py,sha256=-0BxDqmAI1NCiISi7Rw8McJQfgxxVy1gSa1epYuL3U0,3942
+learning_loop_node/trainer/io_helpers.py,sha256=Ylxz8HAId0Jlz95So5kXdJEp1yKQuwroDKIhbTUscF4,7257
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/rest/backdoor_controls.py,sha256=YQcG0KwxzKDNYeMtHrSwr26q__N7ty0o6Kar6CLWAd0,5869
 learning_loop_node/trainer/rest/controls.py,sha256=XF37i2edeMHKdSXyJc4ZqaTZ38u6d3u3Sb3C-Mwyfko,934
 learning_loop_node/trainer/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/trainer/tests/conftest.py,sha256=UIVmnZZVe1DigOxeraiLh89cyw1QNQFvNJWmPUY1uFI,2455
+learning_loop_node/trainer/tests/conftest.py,sha256=qUmcHPme19AD6K6sQektX63iZecdqPRHKQ2CVNoBzNg,2455
 learning_loop_node/trainer/tests/state_helper.py,sha256=igoGqTBqcqqFcDng2i7ctC67bYR1hLPDl4G_mNRG6r8,934
 learning_loop_node/trainer/tests/states/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/tests/states/test_state_cleanup.py,sha256=tiL31hSjg1Bl2obzg2ufAVpil5qW0YkpDCpSHPqXQrk,1312
-learning_loop_node/trainer/tests/states/test_state_detecting.py,sha256=NU065RhpUwhExldGp-79YD_P9bG4O9DNcLjBfW0qmik,3811
+learning_loop_node/trainer/tests/states/test_state_detecting.py,sha256=heJS9KO9GzIgFiKMvrNWPrboujY6hcR-KjztZ6fivxk,3833
 learning_loop_node/trainer/tests/states/test_state_download_train_model.py,sha256=rNRXIyqyHzHz4fXY1Lsf7WKg8FFVFYfFPIevMCBBcCY,2940
 learning_loop_node/trainer/tests/states/test_state_prepare.py,sha256=fx9_bgPTaR5ANVB8n_hW8dXcaJIh_iKEnInmhzamZ9E,2432
 learning_loop_node/trainer/tests/states/test_state_sync_confusion_matrix.py,sha256=zfNbHB3GFSJXXoEkW-8PYtmX62md3feWp4oisyzs8A4,4773
 learning_loop_node/trainer/tests/states/test_state_train.py,sha256=j1vedjH2EwLTgHhon6eR9ttp-Sw9ozR9-9QgAKlFO-M,3248
-learning_loop_node/trainer/tests/states/test_state_upload_detections.py,sha256=38MumBw_N6Pt5cXI8x-xZjczmWUc7BIqyw4TSfim9h4,7751
+learning_loop_node/trainer/tests/states/test_state_upload_detections.py,sha256=u31gC0-Z2EVTnia1dyY2yNGDGAeyIXPfObBgrEWHhVQ,7674
 learning_loop_node/trainer/tests/states/test_state_upload_model.py,sha256=gAXJJcoxj2EVdFJcPAeqxGAjeL1a4ofLtytnM4joi-Q,3808
 learning_loop_node/trainer/tests/test_errors.py,sha256=8H-kjs9kEBoHWcQVJIZvW5zcwCs1VQI5Tf5I0VSbCUA,2245
 learning_loop_node/trainer/tests/test_trainer_states.py,sha256=OBrClH6srAM2hqqel2xTtfHCeTKYZlG_S4KO2G2GrS4,1147
@@ -80,6 +80,6 @@ learning_loop_node/trainer/tests/testing_trainer_logic.py,sha256=7sQ6okiOhM4IhvR
 learning_loop_node/trainer/trainer_logic.py,sha256=PJxiO1chPdvpq8UTtzv_nVam9CouCswX9b1FnRwT2Tw,8411
 learning_loop_node/trainer/trainer_logic_generic.py,sha256=KFDuxgzrGITHQaJoGvhjHxWzhbb4Q7HBxSpks4CeGBg,24801
 learning_loop_node/trainer/trainer_node.py,sha256=bcyOMeLXrLuLgsPqS8lwEOSZ6vCjGLgT0pLXgaylI1Q,4155
-learning_loop_node-0.10.4.dist-info/METADATA,sha256=bOfPBBjEdohsOEBpAD7grcYOQpo5d7akcNrbxmA3hdY,9287
-learning_loop_node-0.10.4.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.10.4.dist-info/RECORD,,
+learning_loop_node-0.10.6.dist-info/METADATA,sha256=fxypCq0VtpapGOz_5Ao8NhcAtN-Xp-1aQb-qGWWXM7M,9287
+learning_loop_node-0.10.6.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.10.6.dist-info/RECORD,,