learning-loop-node 0.10.13__py3-none-any.whl → 0.10.15__py3-none-any.whl

learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py

@@ -11,11 +11,9 @@ from ..testing_trainer_logic import TestingTrainerLogic
 
 # pylint: disable=protected-access
 
-error_key = 'sync_confusion_matrix'
 
-
-def trainer_has_error(trainer: TrainerLogic):
-    return trainer.errors.has_error_for(error_key)
+def trainer_has_sync_confusion_matrix_error(trainer: TrainerLogic):
+    return trainer.errors.has_error_for('sync_confusion_matrix')
 
 
 async def test_nothing_to_sync(test_initialized_trainer: TestingTrainerLogic):
@@ -26,10 +24,10 @@ async def test_nothing_to_sync(test_initialized_trainer: TestingTrainerLogic):
     create_active_training_file(trainer, training_state=TrainerState.TrainingFinished)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.ConfusionMatrixSynced, timeout=1, interval=0.001)
-    assert trainer_has_error(trainer) is False
+    assert trainer_has_sync_confusion_matrix_error(trainer) is False
     assert trainer.training.training_state == TrainerState.ConfusionMatrixSynced
     assert trainer.node.last_training_io.load() == trainer.training
 
@@ -38,16 +36,16 @@ async def test_unsynced_model_available__sync_successful(test_initialized_traine
     trainer = test_initialized_trainer_node.trainer_logic
     assert isinstance(trainer, TestingTrainerLogic)
 
-    await mock_socket_io_call(mocker, test_initialized_trainer_node, {'success': True})
+    await mock_socket_io_call(mocker, test_initialized_trainer_node, return_value={'success': True})
     create_active_training_file(trainer, training_state=TrainerState.TrainingFinished)
 
     trainer._init_from_last_training()
     trainer.has_new_model = True
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
     await assert_training_state(trainer.training, TrainerState.ConfusionMatrixSynced, timeout=1, interval=0.001)
 
-    assert trainer_has_error(trainer) is False
+    assert trainer_has_sync_confusion_matrix_error(trainer) is False
 #    assert trainer.training.training_state == TrainerState.ConfusionMatrixSynced
     assert trainer.node.last_training_io.load() == trainer.training
 
@@ -56,17 +54,19 @@ async def test_unsynced_model_available__sio_not_connected(test_initialized_trai
     trainer = test_initialized_trainer_node.trainer_logic
     assert isinstance(trainer, TestingTrainerLogic)
 
+    await test_initialized_trainer_node.sio_client.disconnect()
+    test_initialized_trainer_node.set_skip_repeat_loop(True)
     create_active_training_file(trainer, training_state=TrainerState.TrainingFinished)
 
     assert test_initialized_trainer_node.sio_client.connected is False
     trainer.has_new_model = True
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
-    await assert_training_state(trainer.training, 'confusion_matrix_syncing', timeout=1, interval=0.001)
-    await assert_training_state(trainer.training, TrainerState.TrainingFinished, timeout=1, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.ConfusionMatrixSyncing, timeout=1, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.TrainingFinished, timeout=10, interval=0.001)
 
-    assert trainer_has_error(trainer)
+    assert trainer_has_sync_confusion_matrix_error(trainer)  # Due to sio not being connected, the request will fail
     assert trainer.training.training_state == TrainerState.TrainingFinished
     assert trainer.node.last_training_io.load() == trainer.training
 
@@ -75,17 +75,17 @@ async def test_unsynced_model_available__request_is_not_successful(test_initiali
     trainer = test_initialized_trainer_node.trainer_logic
     assert isinstance(trainer, TestingTrainerLogic)
 
-    await mock_socket_io_call(mocker, test_initialized_trainer_node, {'success': False})
+    await mock_socket_io_call(mocker, test_initialized_trainer_node, return_value={'success': False})
 
     create_active_training_file(trainer, training_state=TrainerState.TrainingFinished)
 
     trainer.has_new_model = True
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
-    await assert_training_state(trainer.training, 'confusion_matrix_syncing', timeout=1, interval=0.001)
-    await assert_training_state(trainer.training, TrainerState.TrainingFinished, timeout=1, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.ConfusionMatrixSyncing, timeout=1, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.TrainingFinished, timeout=10, interval=0.001)
 
-    assert trainer_has_error(trainer)
+    assert trainer_has_sync_confusion_matrix_error(trainer)  # Due to sio call failure, the error will be set
     assert trainer.training.training_state == TrainerState.TrainingFinished
     assert trainer.node.last_training_io.load() == trainer.training
 
@@ -100,6 +100,9 @@ async def test_basic_mock(test_initialized_trainer_node: TrainerNode, mocker: Mo
 
 
 async def mock_socket_io_call(mocker, trainer_node: TrainerNode, return_value):
+    '''
+    Patch the socketio call function to always return the given return_value
+    '''
     for _ in range(10):
         if trainer_node.sio_client is None:
             await asyncio.sleep(0.1)

learning_loop_node/tests/trainer/states/test_state_train.py

@@ -14,10 +14,10 @@ async def test_successful_training(test_initialized_trainer: TestingTrainerLogic
     create_active_training_file(trainer, training_state=TrainerState.TrainModelDownloaded)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await condition(lambda: trainer._executor and trainer._executor.is_running(), timeout=1, interval=0.01)
-    await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=1, interval=0.01)
+    await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=10, interval=0.01)
     assert trainer.start_training_task is not None
 
     assert trainer._executor is not None
@@ -34,16 +34,15 @@ async def test_stop_running_training(test_initialized_trainer: TestingTrainerLog
     create_active_training_file(trainer, training_state=TrainerState.TrainModelDownloaded)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await condition(lambda: trainer._executor and trainer._executor.is_running(), timeout=1, interval=0.01)
-    await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=1, interval=0.01)
+    await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=10, interval=0.01)
     assert trainer.start_training_task is not None
 
     await trainer.stop()
     await assert_training_state(trainer.training, TrainerState.TrainingFinished, timeout=2, interval=0.01)
 
-    assert trainer.training.training_state == TrainerState.TrainingFinished
     assert trainer.node.last_training_io.load() == trainer.training
 
 
@@ -55,15 +54,14 @@ async def test_training_can_maybe_resumed(test_initialized_trainer: TestingTrain
     trainer._init_from_last_training()
     trainer._can_resume_flag = True
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await condition(lambda: trainer._executor and trainer._executor.is_running(), timeout=1, interval=0.01)
-    await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=1, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=10, interval=0.001)
     assert trainer.start_training_task is not None
 
     assert trainer._executor is not None
     await trainer._executor.stop_and_wait()  # NOTE normally a training terminates itself e.g
     await assert_training_state(trainer.training, TrainerState.TrainingFinished, timeout=1, interval=0.001)
 
-    assert trainer.training.training_state == TrainerState.TrainingFinished
     assert trainer.node.last_training_io.load() == trainer.training

learning_loop_node/tests/trainer/states/test_state_upload_detections.py

@@ -11,11 +11,10 @@ from ..state_helper import assert_training_state, create_active_training_file
 from ..testing_trainer_logic import TestingTrainerLogic
 
 # pylint: disable=protected-access
-error_key = 'upload_detections'
 
 
-def trainer_has_error(trainer: TrainerLogic):
-    return trainer.errors.has_error_for(error_key)
+def trainer_has_upload_detections_error(trainer: TrainerLogic):
+    return trainer.errors.has_error_for('upload_detections')
 
 
 async def create_valid_detection_file(trainer: TrainerLogic, number_of_entries: int = 1, file_index: int = 0):
@@ -125,12 +124,11 @@ async def test_bad_status_from_LearningLoop(test_initialized_trainer: TestingTra
     trainer._init_from_last_training()
     trainer.active_training_io.save_detections([get_dummy_detections()])
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
     await assert_training_state(trainer.training, TrainerState.DetectionUploading, timeout=1, interval=0.001)
-    await assert_training_state(trainer.training, TrainerState.Detected, timeout=1, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.Detected, timeout=10, interval=0.001)
 
-    assert trainer_has_error(trainer)
-    assert trainer.training.training_state == TrainerState.Detected
+    assert trainer_has_upload_detections_error(trainer)
     assert trainer.node.last_training_io.load() == trainer.training
 
 
@@ -143,7 +141,7 @@ async def test_go_to_cleanup_if_no_detections_exist(test_initialized_trainer: Te
     create_active_training_file(trainer, training_state=TrainerState.Detected)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
     await assert_training_state(trainer.training, TrainerState.ReadyForCleanup, timeout=1, interval=0.001)
 
 
@@ -154,7 +152,7 @@ async def test_abort_uploading(test_initialized_trainer: TestingTrainerLogic):
     trainer._init_from_last_training()
     await create_valid_detection_file(trainer)
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.DetectionUploading, timeout=1, interval=0.001)
 

learning_loop_node/tests/trainer/states/test_state_upload_model.py

@@ -8,11 +8,10 @@ from ..state_helper import assert_training_state, create_active_training_file
 from ..testing_trainer_logic import TestingTrainerLogic
 
 # pylint: disable=protected-access
-error_key = 'upload_model'
 
 
-def trainer_has_error(trainer: TrainerLogic):
-    return trainer.errors.has_error_for(error_key)
+def trainer_has_upload_model_error(trainer: TrainerLogic):
+    return trainer.errors.has_error_for('upload_model')
 
 
 async def test_successful_upload(mocker: MockerFixture, test_initialized_trainer: TestingTrainerLogic):
@@ -28,7 +27,7 @@ async def test_successful_upload(mocker: MockerFixture, test_initialized_trainer
     await assert_training_state(trainer.training, TrainerState.TrainModelUploading, timeout=1, interval=0.001)
     await train_task
 
-    assert trainer_has_error(trainer) is False
+    assert trainer_has_upload_model_error(trainer) is False
     assert trainer.training.training_state == TrainerState.TrainModelUploaded
     assert trainer.training.model_uuid_for_detecting is not None
     assert trainer.node.last_training_io.load() == trainer.training
@@ -40,7 +39,7 @@ async def test_abort_upload_model(test_initialized_trainer: TestingTrainerLogic)
     create_active_training_file(trainer, training_state=TrainerState.ConfusionMatrixSynced)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainModelUploading, timeout=1, interval=0.001)
 
@@ -60,13 +59,13 @@ async def test_bad_server_response_content(test_initialized_trainer: TestingTrai
     create_active_training_file(trainer, training_state=TrainerState.ConfusionMatrixSynced)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainModelUploading, timeout=1, interval=0.001)
     # TODO goes to finished because of the error
-    await assert_training_state(trainer.training, TrainerState.ReadyForCleanup, timeout=2, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.ReadyForCleanup, timeout=10, interval=0.001)
 
-    assert trainer_has_error(trainer)
+    assert trainer_has_upload_model_error(trainer)
     assert trainer.training.training_state == TrainerState.ReadyForCleanup
     assert trainer.training.model_uuid_for_detecting is None
     assert trainer.node.last_training_io.load() == trainer.training

learning_loop_node/tests/trainer/test_errors.py

@@ -14,7 +14,7 @@ async def test_training_process_is_stopped_when_trainer_reports_error(test_initi
     trainer = test_initialized_trainer
     create_active_training_file(trainer, training_state=TrainerState.TrainModelDownloaded)
     trainer._init_from_last_training()
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=1, interval=0.001)
     trainer.error_msg = 'some_error'
@@ -26,7 +26,7 @@ async def test_log_can_provide_only_data_for_current_run(test_initialized_traine
     trainer = test_initialized_trainer
     create_active_training_file(trainer, training_state=TrainerState.TrainModelDownloaded)
     trainer._init_from_last_training()
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=1, interval=0.001)
     await asyncio.sleep(0.1)  # give tests a bit time to to check for the state

learning_loop_node/trainer/exceptions.py

@@ -1,2 +1,12 @@
 class CriticalError(Exception):
-    pass
+    '''
+    CriticalError is raised when the training cannot be continued.
+    In this case the trainer jumps to the TrainerState.ReadyForCleanup and tries to upload the latest model.
+    '''
+
+
+class NodeNeedsRestartError(Exception):
+    '''
+    NodeNeedsRestartError is raised when the node needs to be restarted.
+    This is e.g. the case when the GPU is not available anymore.
+    '''

learning_loop_node/trainer/rest/backdoor_controls.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Dict
 
 from fastapi import APIRouter, HTTPException, Request
 
-from ...data_classes import ErrorConfiguration, NodeState
+from ...data_classes import ErrorConfiguration
 from ..trainer_logic import TrainerLogic
 
 if TYPE_CHECKING:
@@ -16,32 +16,10 @@ if TYPE_CHECKING:
 router = APIRouter()
 
 
-@router.put("/socketio")
-async def switch_socketio(request: Request):
-    '''
-    Example Usage
-
-        curl -X PUT -d "on" http://localhost:8001/socketio
-    '''
-    state = str(await request.body(), 'utf-8')
-    await _switch_socketio(state, request.app)
-
-
-async def _switch_socketio(state: str, trainer_node: TrainerNode):
-    if state == 'off':
-        if trainer_node.status.state != NodeState.Offline:
-            logging.debug('turning socketio off')
-            await trainer_node.sio_client.disconnect()  # pylint: disable=protected-access
-    if state == 'on':
-        if trainer_node.status.state == NodeState.Offline:
-            logging.debug('turning socketio on')
-            await trainer_node.connect_sio()
-
-
 @router.put("/provide_new_model")
 async def provide_new_model(request: Request):
     value = str(await request.body(), 'utf-8')
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     # trainer_logic is MockTrainerLogic which has a property provide_new_model
     assert hasattr(trainer_node.trainer_logic,
                    'provide_new_model'), 'trainer_logic does not have property provide_new_model'
@@ -56,17 +34,22 @@ async def provide_new_model(request: Request):
 
 @router.post("/reset")
 async def reset(request: Request):
-    trainer_node = trainer_node_from_request(request)
-    await _switch_socketio('on', trainer_node)
+    logging.info('BC: reset')
+    trainer_node: 'TrainerNode' = request.app
+    async with trainer_node.repeat_loop_lock:
 
-    await trainer_node.trainer_logic.stop()  # NOTE first stop may only kill running training process
-    await trainer_node.trainer_logic.stop()
+        await trainer_node.trainer_logic.stop()  # NOTE first stop may only kill running training process
+        await trainer_node.trainer_logic.stop()
+        trainer_node.last_training_io.delete()
+        trainer_node.status.reset_all_errors()
 
-    trainer_node.last_training_io.delete()
+        try:
+            await trainer_node.reconnect_to_loop()
+        except Exception:
+            logging.exception('Could not reset sio connection to loop')
 
-    trainer_node.status.reset_all_errors()
-    logging.info('training should be killed, sending new state to LearningLoop')
-    await trainer_node.send_status()
+        logging.info('training should be killed, sending new state to LearningLoop')
+        await trainer_node.send_status()
 
 
 @router.put("/error_configuration")
@@ -82,7 +65,7 @@ def set_error_configuration(msg: Dict, request: Request):
                                              save_model=msg.get('save_model', None), )
 
     logging.info(f'setting error configuration to: {asdict(error_configuration)}')
-    trainer_logic = trainer_node_from_request(request).trainer_logic
+    trainer_logic = request.app.trainer_logic
 
     # NOTE: trainer_logic is MockTrainerLogic which has a property error_configuration
     assert hasattr(trainer_logic, 'error_configuration'), 'trainer_logic does not have property error_configuration'
@@ -92,7 +75,7 @@ def set_error_configuration(msg: Dict, request: Request):
 @router.post("/steps")
 async def add_steps(request: Request):
     logging.warning('Steps was called')
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     trainer_logic = trainer_node.trainer_logic  # NOTE: is MockTrainerLogic which has 'provide_new_model' and 'current_iteration'
 
     assert isinstance(trainer_logic, TrainerLogic), 'trainer_logic is not TrainerLogic'
@@ -123,7 +106,7 @@ async def add_steps(request: Request):
 async def kill_process(request: Request):
 
     # pylint: disable=protected-access
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     trainer_logic = trainer_node.trainer_logic
     assert isinstance(trainer_logic, TrainerLogic), 'trainer_logic is not TrainerLogic'
     if not trainer_logic._executor or not trainer_logic._executor.is_running():
@@ -133,9 +116,5 @@ async def kill_process(request: Request):
 
 @router.post("/force_status_update")
 async def force_status_update(request: Request):
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     await trainer_node.send_status()
-
-
-def trainer_node_from_request(request: Request) -> TrainerNode:
-    return request.app

learning_loop_node/trainer/trainer_logic_generic.py

@@ -14,7 +14,7 @@ from ..data_classes import (Context, Errors, Hyperparameter, PretrainedModel, Tr
                             TrainingOut, TrainingStateData)
 from ..helpers.misc import create_project_folder, delete_all_training_folders, generate_training, is_valid_uuid4
 from .downloader import TrainingsDownloader
-from .exceptions import CriticalError
+from .exceptions import CriticalError, NodeNeedsRestartError
 from .io_helpers import ActiveTrainingIO, EnvironmentVars, LastTrainingIO
 
 if TYPE_CHECKING:
@@ -179,7 +179,7 @@ class TrainerLogicGeneric(ABC):
         if not self.training_active and self.last_training_io.exists():
             self._init_from_last_training()
             logger.info('found incomplete training, continuing now.')
-            asyncio.get_event_loop().create_task(self._run())
+            self._begin_training_task()
             return True
         return False
 
@@ -195,7 +195,11 @@ class TrainerLogicGeneric(ABC):
         """Called on `begin_training` event from the Learning Loop.
         """
         self._init_new_training(Context(organization=organization, project=project), details)
-        asyncio.get_event_loop().create_task(self._run())
+        self._begin_training_task()
+
+    def _begin_training_task(self) -> None:
+        # NOTE: Task object is used to potentially cancel the task
+        self.training_task = asyncio.get_event_loop().create_task(self._run())
 
     def _init_new_training(self, context: Context, details: Dict) -> None:
         """Called on `begin_training` event from the Learning Loop.
@@ -218,8 +222,7 @@ class TrainerLogicGeneric(ABC):
         """
         self.errors.reset_all()
         try:
-            self.training_task = asyncio.get_running_loop().create_task(self._training_loop())
-            await self.training_task  # NOTE: Task object is used to potentially cancel the task
+            await self._training_loop()
         except asyncio.CancelledError:
             if not self.shutdown_event.is_set():
                 logger.info('CancelledError in _run - training task was cancelled but not by shutdown event')
@@ -291,6 +294,9 @@ class TrainerLogicGeneric(ABC):
             logger.error('CriticalError in %s - Exception: %s', state_during, e)
             self.errors.set(error_key, str(e))
             self.training.training_state = TrainerState.ReadyForCleanup
+        except NodeNeedsRestartError:
+            logger.error('Node Restart Requested')
+            sys.exit(0)
         except Exception as e:
             self.errors.set(error_key, str(e))
             logger.exception('Error in %s - Exception: %s', state_during, e)

learning_loop_node/trainer/trainer_node.py

@@ -77,7 +77,7 @@ class TrainerNode(Node):
 
     async def send_status(self):
         if not self.sio_client.connected:
-            self.log.warning('cannot send status - not connected to the Learning Loop')
+            self.log.debug('cannot send status - not connected to the Learning Loop')
             return
 
         status = TrainingStatus(id=self.uuid,
@@ -98,10 +98,11 @@ class TrainerNode(Node):
             status.errors = self.trainer_logic.errors.errors
             status.context = self.trainer_logic.training_context
 
-        self.log.info(f'sending status: {status.short_str()}')
+        self.log.debug('sending status: %s', status.short_str())
         result = await self.sio_client.call('update_trainer', jsonable_encoder(asdict(status)), timeout=30)
         if isinstance(result, Dict) and not result['success']:
-            self.log.error(f'Error when sending status update: Response from loop was:\n {result}')
+            self.socket_connection_broken = True
+            self.log.error('Error when sending status update: Response from loop was:\n %s', result)
 
     def check_idle_timeout(self):
         if not self.idle_timeout:

{learning_loop_node-0.10.13.dist-info → learning_loop_node-0.10.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.13
+Version: 0.10.15
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

{learning_loop_node-0.10.13.dist-info → learning_loop_node-0.10.15.dist-info}/RECORD

@@ -1,28 +1,28 @@
 learning_loop_node/__init__.py,sha256=onN5s8-x_xBsCM6NLmJO0Ym1sJHeCFaGw8qb0oQZmz8,364
 learning_loop_node/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/annotation/annotator_logic.py,sha256=BTaopkJZkIf1CI5lfsVKsxbxoUIbDJrevavuQUT5e_c,1000
-learning_loop_node/annotation/annotator_node.py,sha256=wk11CQtM3A0Dr7efCn_Mw2X7ql5xn2sgEJzrIeSBC6Q,4043
+learning_loop_node/annotation/annotator_node.py,sha256=UrJ8MpZ44UhsjmVuSHr2BhHyLC-kIMDi3IuBBMKzN1g,4117
 learning_loop_node/data_classes/__init__.py,sha256=wCX88lDgbb8V-gtVCVe9i-NvvZuMe5FX7eD_UJgYYXw,1305
 learning_loop_node/data_classes/annotations.py,sha256=iInU0Nuy_oYT_sj4k_n-W0UShCBI2cHQYrt8imymbtM,1211
-learning_loop_node/data_classes/detections.py,sha256=iFSrnbvYvwhL8k9niz1BACn2QJV4RyW0DKezpmlUn9M,4398
-learning_loop_node/data_classes/general.py,sha256=Bd0ngYhYvS_9OYOO6lAKEnDzLuSdPmR4I2YV-0DRsxs,4694
+learning_loop_node/data_classes/detections.py,sha256=hifsGz2LbmeKLZdHxG7cnlOYNEqDmtJd2gxhyU-Xjjs,5811
+learning_loop_node/data_classes/general.py,sha256=usXokcTOVqTuaKJtBf0ffFWfzZhMrQtF7puKfwi6A5k,6195
 learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
 learning_loop_node/data_classes/training.py,sha256=hnMHZMk-WNRERyo7U97qL09v1tIdhnzPfTH-JgifLwU,6164
-learning_loop_node/data_exchanger.py,sha256=U_MrBKSq1MbBwBmjrjxoIo_7xV4Lcwtk6uZDIgmhT_4,8914
+learning_loop_node/data_exchanger.py,sha256=mwZvJf8L1-r6Wi2ZvDfFf4kFjkqxsThXj7AGLUANGlU,8979
 learning_loop_node/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/detector/detector_logic.py,sha256=Xlwc5yRUSe62gqcxdeLH7KXMujaW3JrCjfZo4iCWdHI,2087
-learning_loop_node/detector/detector_node.py,sha256=xuNwXwJYdWZhe3IzP3oz8WUhREXF8k3XI97LvvZqFLQ,19181
+learning_loop_node/detector/detector_logic.py,sha256=IG1s9RF_cCBcNQ8WW1rAS37QKdGzlVoVkuO_CrLGvYs,2084
+learning_loop_node/detector/detector_node.py,sha256=TeChzkpVVmEiZbnfWmtkjuUXNwCZvIWfingmcuAz2cs,19567
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=TD346I9ymtIP0_CJXCIKMRuiXbfVVanXNu_iHAwDd7Q,3318
 learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=7_-x8D8Zf6KJeJXmiC2VrRHU8Ig_R98uhdXVwwX0N4M,1240
 learning_loop_node/detector/outbox.py,sha256=HrYeS6XJLC-1kqq2hDufxXLRmOYGiBlz-m9B6HG5Ie8,8227
 learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/detector/rest/about.py,sha256=VVICeSHONk0VneUvbkOYV-Wbey1sXSej3pJFl57rQig,677
-learning_loop_node/detector/rest/backdoor_controls.py,sha256=ZtQAjxYpeskCHHQGjZd994eyq-u8LSrjObFtxLbU-Ds,1654
-learning_loop_node/detector/rest/detect.py,sha256=a_0A_65GQhuI-9eIgwj6gVyNu_CC9lqSMfAAN9HWFhQ,2065
-learning_loop_node/detector/rest/model_version_control.py,sha256=PKG7foFyNSvjoMhWCDb7w3mq-2e0bx5gq3ov7Rao8HU,3703
-learning_loop_node/detector/rest/operation_mode.py,sha256=eIo6_56qyZECftf4AEN8wJMABIojC0TRazvWeg0Uj_s,1664
-learning_loop_node/detector/rest/outbox_mode.py,sha256=anSZHB6jliz1t3fxrmEzgwNB62UHNdWNc9ZYOc5Nn9s,1018
+learning_loop_node/detector/rest/about.py,sha256=COYgmYO1tXGSIwjF__P79mVZUfSDZoHsW0GUarQ2rv0,1686
+learning_loop_node/detector/rest/backdoor_controls.py,sha256=ZNaFOvC0OLWNtcLiG-NIqS_y1kkLP4csgk3CHhp8Gis,885
+learning_loop_node/detector/rest/detect.py,sha256=KSYUOuTxuc2q3RgV37cBBuCvCJf11BFL7QtnPAM4XbU,2343
+learning_loop_node/detector/rest/model_version_control.py,sha256=jLp3rvCYq8T_QC3KK7uLDYpbDjydwazWkQCUXvkxl-c,4654
+learning_loop_node/detector/rest/operation_mode.py,sha256=RAzVLtGzy4n9-LSIq_XSwMfXDehU4XmorgWAWbQ6BW8,1804
+learning_loop_node/detector/rest/outbox_mode.py,sha256=H8coDNbgLGEfXmKQrhtXWeUHBAHpnrdZktuHXQz0xis,1148
 learning_loop_node/detector/rest/upload.py,sha256=IPzxJPayD7_Gx5uYC1lVJwWxdnQgM8MYGa5NugXVosY,544
 learning_loop_node/examples/novelty_score_updater.py,sha256=1DRgM9lxjFV-q2JvGDDsNLz_ic_rhEZ9wc6ZdjcxwPE,2038
 learning_loop_node/globals.py,sha256=tgw_8RYOipPV9aYlyUhYtXfUxvJKRvfUk6u-qVAtZmY,174
@@ -31,25 +31,25 @@ learning_loop_node/helpers/environment_reader.py,sha256=OtCTDc0KT9r-SMygkZB_Mw-Z
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
 learning_loop_node/helpers/log_conf.py,sha256=z_0PHh7U7DkJbSbKoSPyUfS7NhBHtRxXHdNcj67Hpbc,951
 learning_loop_node/helpers/misc.py,sha256=j4is8Rv0ttnCqF-R-wP3xwEi67OI6IBJav5Woo5lyDk,7701
-learning_loop_node/loop_communication.py,sha256=rG5MdavSTaREZ6OWfAUIT_qkkYPw3is2_FujLmHQeIc,6576
-learning_loop_node/node.py,sha256=M846fxhutHU2djpd2Kz512iMscLJFk4DVUVO8fe2xF4,8051
+learning_loop_node/loop_communication.py,sha256=xkoZtHRgxq1arusHQtC_lEBculFBLeCijyfVSYIEchY,6755
+learning_loop_node/node.py,sha256=vbMR_6QsruB2IYYKUWx4--9Ywjf_vuBQb4jyzLRqpRQ,10300
 learning_loop_node/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/rest.py,sha256=5jxdfNJKbGklaVKufmprHclZF-_EKG67BeEV1XPW7mc,783
+learning_loop_node/rest.py,sha256=o1dl4Mtznd5duyEQtCYSGlK04l1Y-p_YRjG40Q4l31c,1491
 learning_loop_node/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/annotator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/annotator/conftest.py,sha256=G4ZvdZUdvPp9bYCzg3eEVkGCeXn9INZ3AcN7d5CyLkU,1931
 learning_loop_node/tests/annotator/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/annotator/test_annotator_node.py,sha256=TPNPPrQAxQ_zEecQcH7hlczgD3ABtTCNtUvWD1_oApk,1985
 learning_loop_node/tests/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/tests/detector/conftest.py,sha256=kBg4ujWtKI03IRrftOcMly17CVFPgxWvYFpMA40OkU4,5394
+learning_loop_node/tests/detector/conftest.py,sha256=Q14KHTSuSCsASVIxY9CttdVJm5FC7_JH-W5Q4CdDqoM,5414
 learning_loop_node/tests/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/detector/inbox_filter/test_observation.py,sha256=k4WYdvnuV7d_r7zI4M2aA8WuBjm0aycQ0vj1rGE2q4w,1370
 learning_loop_node/tests/detector/inbox_filter/test_relevance_group.py,sha256=XjiMsS0LgvM0OkPf5-s2rjFbG7C42LTmz_rDVMGHKoY,7603
 learning_loop_node/tests/detector/inbox_filter/test_unexpected_observations_count.py,sha256=MWC7PbaCy14jjRw0_oilkXj6gymAsUZXHJdzNW5m2D4,1639
 learning_loop_node/tests/detector/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/detector/test.jpg,sha256=msA-vHPmvPiro_D102Qmn1fn4vNfooqYYEXPxZUmYpk,161390
-learning_loop_node/tests/detector/test_client_communication.py,sha256=2gJARodJSDuJHgeN1_xLMbvDcPQkXpBXEefu7MOyePk,8998
-learning_loop_node/tests/detector/test_detector_node.py,sha256=oLmmMu0EEiatAlpEA5rqPXA4VGYctAAspgVHdxV05_k,2924
+learning_loop_node/tests/detector/test_client_communication.py,sha256=NAOUrHWxoI4yG6oy3BGxWWXX794IOODEj9QBKF3CyrY,9375
+learning_loop_node/tests/detector/test_detector_node.py,sha256=KX2RcFpdIbpPEmcyYM0YMs-6wwTpbOOZONoiwIWryUI,2922
 learning_loop_node/tests/detector/test_outbox.py,sha256=5RMKQfuu1-rvpVCpEtt_D70bYgma-sIrTHWxHdTdU9Y,3001
 learning_loop_node/tests/detector/test_relevance_filter.py,sha256=3VLhHKaxPzLYmiNZagvgg9ZHkPhWk4_-qpmkJw36wBU,2046
 learning_loop_node/tests/detector/testing_detector.py,sha256=FeQroV85IvsT8dmalQBqf1FLNt_buCtZK3-lgtmbrBI,542
@@ -64,32 +64,32 @@ learning_loop_node/tests/general/test_downloader.py,sha256=C6b_wG3TfQX53lmuanpH1
 learning_loop_node/tests/general/test_learning_loop_node.py,sha256=SZd-VChpWnnsPN46pr4E_LL3ZevYx6psU-AWdVeOFpQ,770
 learning_loop_node/tests/test_helper.py,sha256=nTynYtuUaK2hKh87pk7t7AIJaOiD3wJ5d6nCPqnwRMk,3012
 learning_loop_node/tests/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/tests/trainer/conftest.py,sha256=7nncnC2ZApakdyta9TMUfk0sgr_nyZhGFYCHuUlBlFI,3356
+learning_loop_node/tests/trainer/conftest.py,sha256=E3SQL_CGFJ_sNjEfVJbxbvH0g6hjI5753ndAFUbnkQk,3366
 learning_loop_node/tests/trainer/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/trainer/state_helper.py,sha256=MDe9opeKruip74FoRFff8MSWGiQNFqDpPtIEIbgPnFc,919
 learning_loop_node/tests/trainer/states/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/trainer/states/test_state_cleanup.py,sha256=gZNxSSwnj9f0esExNnQzqadM6-sE3IsF5sNbD0bZNu8,1250
-learning_loop_node/tests/trainer/states/test_state_detecting.py,sha256=nvhkjBnGZgGVyomvPDcXuBR_WLqC4-QD2qE5enWiJ-g,3724
-learning_loop_node/tests/trainer/states/test_state_download_train_model.py,sha256=aJBh0ksz9g2kQ3qqk8yrX-50kdRS4I7ZaeeFkr9CR3c,2863
-learning_loop_node/tests/trainer/states/test_state_prepare.py,sha256=yaDhT-1676asGfgqhK2mfdoktAgeyWUwc2h0xyOoWzQ,2340
-learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py,sha256=0Ak5bU2uiqRDSEzVSTBEoT9GN5UgL7SqBqJRyyxjmWE,4727
-learning_loop_node/tests/trainer/states/test_state_train.py,sha256=dQDUvVI_FUYdLWm11F_7WxGge30l0OvML40Hil9r--k,3149
-learning_loop_node/tests/trainer/states/test_state_upload_detections.py,sha256=U2qK1Vbzb0yBMBYNvB815AYBD6MsII7wxzYJrwFnUHQ,7554
-learning_loop_node/tests/trainer/states/test_state_upload_model.py,sha256=u_YXELOWOXoaU4v41jgr3jHPWVFeVbe330k8GCEL9K0,3659
-learning_loop_node/tests/trainer/test_errors.py,sha256=Z3BWvUkVKxMGe_RNYeVbrhPps7ylek3qS1zK2FpdMmU,2165
+learning_loop_node/tests/trainer/states/test_state_detecting.py,sha256=KGXTR69J_1pJoT8S0ceC3vSyHLw52mIpjbawH4c-8JA,3696
+learning_loop_node/tests/trainer/states/test_state_download_train_model.py,sha256=AuTY63sgrlKT0awS6o38fF3mTkDguAFJtcX7J7WhjgQ,2855
+learning_loop_node/tests/trainer/states/test_state_prepare.py,sha256=3hzRo9ycM802QUZO2Zs_rJRZ23hxTi3XjRCwL9M9m9o,2315
+learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py,sha256=6s4A2d5ahD9UJJiHGK1VtmOrhumzuqTlOwyc_8Oc1vk,5073
+learning_loop_node/tests/trainer/states/test_state_train.py,sha256=HYe1O6zcdtD4dnmfX3cyM1_iF7eGig7dQI4M4Xat7YU,2916
+learning_loop_node/tests/trainer/states/test_state_upload_detections.py,sha256=0Qkavl4i2tZmCOxKkNsQUqa1JWhAgcOsbrW3_eYHfxo,7417
+learning_loop_node/tests/trainer/states/test_state_upload_model.py,sha256=y2o4WBo7kBG_JWSWmt4icjrwya5hQ30zCWC-YMVEwEk,3621
+learning_loop_node/tests/trainer/test_errors.py,sha256=khWCTzi-JW4nSz9QnsRh9wDPmiuE_zdxXukh59qixuY,2109
 learning_loop_node/tests/trainer/test_trainer_states.py,sha256=djYCs5ieajQHRjk8QcUVBUkQEG8UGYFoNGwSX0z2oGk,1067
 learning_loop_node/tests/trainer/testing_trainer_logic.py,sha256=KslqDJDntkgH4Yd_z-guiVPvzi5Q-l-Bqc3fUjT5N7U,3883
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/downloader.py,sha256=Qk-oBcrGCVuWTVs3hvAJzQSqCIHPGZ7NXLJ_fAqvCoY,1469
-learning_loop_node/trainer/exceptions.py,sha256=hLLDGncC6PLZjKg4lZBpu-QA8itQIxiuxExz1uptgnw,40
+learning_loop_node/trainer/exceptions.py,sha256=vbuoE6kssLQuA8zd3LiDHmZglP6E2IJJwEi5AZtWXxY,420
 learning_loop_node/trainer/executor.py,sha256=-0BxDqmAI1NCiISi7Rw8McJQfgxxVy1gSa1epYuL3U0,3942
 learning_loop_node/trainer/io_helpers.py,sha256=hGEtNAQBSBbVB56U1ndwfP8qK5K4YIwMQrjCDcaMy9I,7218
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/trainer/rest/backdoor_controls.py,sha256=YQcG0KwxzKDNYeMtHrSwr26q__N7ty0o6Kar6CLWAd0,5869
+learning_loop_node/trainer/rest/backdoor_controls.py,sha256=-pU4iHheBWf0SW2QzBVBsLiCMZBRz9CDdVZv6414Ts8,5134
 learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
 learning_loop_node/trainer/trainer_logic.py,sha256=PlYExIskU9pWJO0e9m_0KJnUdOI10GtW0oDOevYmg1o,8461
-learning_loop_node/trainer/trainer_logic_generic.py,sha256=7ueRSkiViIRfX2T4RM7yIqrzKkqFICCyDjuj40-Y-LE,25826
-learning_loop_node/trainer/trainer_node.py,sha256=0baQKXVUJjf8KRu0pH8i2o_01bRQWmUGooTptC-ZSsE,5334
-learning_loop_node-0.10.13.dist-info/METADATA,sha256=kV-BUra2MEWVvJlQ_tW0-JbHzdrdXTZFEV4AWC5bx4A,11907
-learning_loop_node-0.10.13.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.10.13.dist-info/RECORD,,
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=ERfuGhHGNvIPRyd_QOGavylPDXTCC8qCOO1eJXAwEO8,25957
+learning_loop_node/trainer/trainer_node.py,sha256=8ANS9iy-swdTLvt9wEFixE6YlmqvqBl17A-R4tVYD-I,5384
+learning_loop_node-0.10.15.dist-info/METADATA,sha256=lc7PJ83IwA8dU5N6UFWw_W35eKT21nevCsVU8bzaeU8,11907
+learning_loop_node-0.10.15.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.10.15.dist-info/RECORD,,