learning-loop-node 0.10.12__py3-none-any.whl → 0.10.14__py3-none-any.whl

learning_loop_node/tests/trainer/states/test_state_upload_model.py

@@ -8,11 +8,10 @@ from ..state_helper import assert_training_state, create_active_training_file
 from ..testing_trainer_logic import TestingTrainerLogic
 
 # pylint: disable=protected-access
-error_key = 'upload_model'
 
 
-def trainer_has_error(trainer: TrainerLogic):
-    return trainer.errors.has_error_for(error_key)
+def trainer_has_upload_model_error(trainer: TrainerLogic):
+    return trainer.errors.has_error_for('upload_model')
 
 
 async def test_successful_upload(mocker: MockerFixture, test_initialized_trainer: TestingTrainerLogic):
@@ -28,7 +27,7 @@ async def test_successful_upload(mocker: MockerFixture, test_initialized_trainer
     await assert_training_state(trainer.training, TrainerState.TrainModelUploading, timeout=1, interval=0.001)
     await train_task
 
-    assert trainer_has_error(trainer) is False
+    assert trainer_has_upload_model_error(trainer) is False
     assert trainer.training.training_state == TrainerState.TrainModelUploaded
     assert trainer.training.model_uuid_for_detecting is not None
     assert trainer.node.last_training_io.load() == trainer.training
@@ -40,7 +39,7 @@ async def test_abort_upload_model(test_initialized_trainer: TestingTrainerLogic)
     create_active_training_file(trainer, training_state=TrainerState.ConfusionMatrixSynced)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainModelUploading, timeout=1, interval=0.001)
 
@@ -60,13 +59,13 @@ async def test_bad_server_response_content(test_initialized_trainer: TestingTrai
     create_active_training_file(trainer, training_state=TrainerState.ConfusionMatrixSynced)
     trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainModelUploading, timeout=1, interval=0.001)
     # TODO goes to finished because of the error
-    await assert_training_state(trainer.training, TrainerState.ReadyForCleanup, timeout=2, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.ReadyForCleanup, timeout=10, interval=0.001)
 
-    assert trainer_has_error(trainer)
+    assert trainer_has_upload_model_error(trainer)
     assert trainer.training.training_state == TrainerState.ReadyForCleanup
     assert trainer.training.model_uuid_for_detecting is None
     assert trainer.node.last_training_io.load() == trainer.training

learning_loop_node/tests/trainer/test_errors.py

@@ -14,7 +14,7 @@ async def test_training_process_is_stopped_when_trainer_reports_error(test_initi
     trainer = test_initialized_trainer
     create_active_training_file(trainer, training_state=TrainerState.TrainModelDownloaded)
     trainer._init_from_last_training()
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=1, interval=0.001)
     trainer.error_msg = 'some_error'
@@ -26,7 +26,7 @@ async def test_log_can_provide_only_data_for_current_run(test_initialized_traine
     trainer = test_initialized_trainer
     create_active_training_file(trainer, training_state=TrainerState.TrainModelDownloaded)
     trainer._init_from_last_training()
-    _ = asyncio.get_running_loop().create_task(trainer._run())
+    trainer._begin_training_task()
 
     await assert_training_state(trainer.training, TrainerState.TrainingRunning, timeout=1, interval=0.001)
     await asyncio.sleep(0.1)  # give tests a bit time to to check for the state

learning_loop_node/trainer/io_helpers.py

@@ -16,12 +16,9 @@ from ..loop_communication import LoopCommunicator
 
 class EnvironmentVars:
     def __init__(self) -> None:
-        self.restart_after_training = os.environ.get(
-            'RESTART_AFTER_TRAINING', 'FALSE').lower() in ['true', '1']
-        self.keep_old_trainings = os.environ.get(
-            'KEEP_OLD_TRAININGS', 'FALSE').lower() in ['true', '1']
-        self.inference_batch_size = int(
-            os.environ.get('INFERENCE_BATCH_SIZE', '10'))
+        self.restart_after_training = os.environ.get('RESTART_AFTER_TRAINING', 'FALSE').lower() in ['true', '1']
+        self.keep_old_trainings = os.environ.get('KEEP_OLD_TRAININGS', 'FALSE').lower() in ['true', '1']
+        self.inference_batch_size = int(os.environ.get('INFERENCE_BATCH_SIZE', '10'))
 
 
 class LastTrainingIO:

learning_loop_node/trainer/rest/backdoor_controls.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Dict
 
 from fastapi import APIRouter, HTTPException, Request
 
-from ...data_classes import ErrorConfiguration, NodeState
+from ...data_classes import ErrorConfiguration
 from ..trainer_logic import TrainerLogic
 
 if TYPE_CHECKING:
@@ -16,32 +16,10 @@ if TYPE_CHECKING:
 router = APIRouter()
 
 
-@router.put("/socketio")
-async def switch_socketio(request: Request):
-    '''
-    Example Usage
-
-        curl -X PUT -d "on" http://localhost:8001/socketio
-    '''
-    state = str(await request.body(), 'utf-8')
-    await _switch_socketio(state, request.app)
-
-
-async def _switch_socketio(state: str, trainer_node: TrainerNode):
-    if state == 'off':
-        if trainer_node.status.state != NodeState.Offline:
-            logging.debug('turning socketio off')
-            await trainer_node.sio_client.disconnect()  # pylint: disable=protected-access
-    if state == 'on':
-        if trainer_node.status.state == NodeState.Offline:
-            logging.debug('turning socketio on')
-            await trainer_node.connect_sio()
-
-
 @router.put("/provide_new_model")
 async def provide_new_model(request: Request):
     value = str(await request.body(), 'utf-8')
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     # trainer_logic is MockTrainerLogic which has a property provide_new_model
     assert hasattr(trainer_node.trainer_logic,
                    'provide_new_model'), 'trainer_logic does not have property provide_new_model'
@@ -56,17 +34,22 @@ async def provide_new_model(request: Request):
 
 @router.post("/reset")
 async def reset(request: Request):
-    trainer_node = trainer_node_from_request(request)
-    await _switch_socketio('on', trainer_node)
+    logging.info('BC: reset')
+    trainer_node: 'TrainerNode' = request.app
+    async with trainer_node.repeat_loop_lock:
 
-    await trainer_node.trainer_logic.stop()  # NOTE first stop may only kill running training process
-    await trainer_node.trainer_logic.stop()
+        await trainer_node.trainer_logic.stop()  # NOTE first stop may only kill running training process
+        await trainer_node.trainer_logic.stop()
+        trainer_node.last_training_io.delete()
+        trainer_node.status.reset_all_errors()
 
-    trainer_node.last_training_io.delete()
+        try:
+            await trainer_node.reconnect_to_loop()
+        except Exception:
+            logging.exception('Could not reset sio connection to loop')
 
-    trainer_node.status.reset_all_errors()
-    logging.info('training should be killed, sending new state to LearningLoop')
-    await trainer_node.send_status()
+        logging.info('training should be killed, sending new state to LearningLoop')
+        await trainer_node.send_status()
 
 
 @router.put("/error_configuration")
@@ -82,7 +65,7 @@ def set_error_configuration(msg: Dict, request: Request):
                                              save_model=msg.get('save_model', None), )
 
     logging.info(f'setting error configuration to: {asdict(error_configuration)}')
-    trainer_logic = trainer_node_from_request(request).trainer_logic
+    trainer_logic = request.app.trainer_logic
 
     # NOTE: trainer_logic is MockTrainerLogic which has a property error_configuration
     assert hasattr(trainer_logic, 'error_configuration'), 'trainer_logic does not have property error_configuration'
@@ -92,7 +75,7 @@ def set_error_configuration(msg: Dict, request: Request):
 @router.post("/steps")
 async def add_steps(request: Request):
     logging.warning('Steps was called')
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     trainer_logic = trainer_node.trainer_logic  # NOTE: is MockTrainerLogic which has 'provide_new_model' and 'current_iteration'
 
     assert isinstance(trainer_logic, TrainerLogic), 'trainer_logic is not TrainerLogic'
@@ -123,7 +106,7 @@ async def add_steps(request: Request):
 async def kill_process(request: Request):
 
     # pylint: disable=protected-access
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     trainer_logic = trainer_node.trainer_logic
     assert isinstance(trainer_logic, TrainerLogic), 'trainer_logic is not TrainerLogic'
     if not trainer_logic._executor or not trainer_logic._executor.is_running():
@@ -133,9 +116,5 @@ async def kill_process(request: Request):
 
 @router.post("/force_status_update")
 async def force_status_update(request: Request):
-    trainer_node = trainer_node_from_request(request)
+    trainer_node = request.app
     await trainer_node.send_status()
-
-
-def trainer_node_from_request(request: Request) -> TrainerNode:
-    return request.app

learning_loop_node/trainer/trainer_logic.py

@@ -25,7 +25,7 @@ class TrainerLogic(TrainerLogicGeneric):
         self._detection_progress: Optional[float] = None
         self._executor: Optional[Executor] = None
         self.start_training_task: Optional[Coroutine] = None
-        self.inference_batch_size = 10
+        self.inference_batch_size = self._environment_vars.inference_batch_size
 
     # ---------------------------------------- IMPLEMENTED ABSTRACT PROPERTIES ----------------------------------------
 
@@ -92,7 +92,7 @@ class TrainerLogic(TrainerLogicGeneric):
 
         shutil.rmtree(tmp_folder, ignore_errors=True)
         os.makedirs(tmp_folder)
-        logging.info(f'downloading detection model to {tmp_folder}')
+        logging.info('downloading detection model to %s', tmp_folder)
 
         await self.node.data_exchanger.download_model(tmp_folder, context, model_id, self.model_format)
         with open(f'{tmp_folder}/model.json', 'r') as f:
@@ -104,10 +104,10 @@ class TrainerLogic(TrainerLogicGeneric):
         image_ids = []
         for state, p in zip(['inbox', 'annotate', 'review', 'complete'], [0.1, 0.2, 0.3, 0.4]):
             self._detection_progress = p
-            logging.info(f'fetching image ids of {state}')
+            logging.info('fetching image ids of state %s', state)
             new_ids = await self.node.data_exchanger.fetch_image_uuids(query_params=f'state={state}')
             image_ids += new_ids
-            logging.info(f'downloading {len(new_ids)} images')
+            logging.info('downloading %d images', len(new_ids))
             await self.node.data_exchanger.download_images(new_ids, image_folder)
         self._detection_progress = 0.42
         # await delete_corrupt_images(image_folder)

learning_loop_node/trainer/trainer_logic_generic.py

@@ -179,7 +179,7 @@ class TrainerLogicGeneric(ABC):
         if not self.training_active and self.last_training_io.exists():
             self._init_from_last_training()
             logger.info('found incomplete training, continuing now.')
-            asyncio.get_event_loop().create_task(self._run())
+            self._begin_training_task()
             return True
         return False
 
@@ -195,7 +195,11 @@ class TrainerLogicGeneric(ABC):
         """Called on `begin_training` event from the Learning Loop.
         """
         self._init_new_training(Context(organization=organization, project=project), details)
-        asyncio.get_event_loop().create_task(self._run())
+        self._begin_training_task()
+
+    def _begin_training_task(self) -> None:
+        # NOTE: Task object is used to potentially cancel the task
+        self.training_task = asyncio.get_event_loop().create_task(self._run())
 
     def _init_new_training(self, context: Context, details: Dict) -> None:
         """Called on `begin_training` event from the Learning Loop.
@@ -210,7 +214,7 @@ class TrainerLogicGeneric(ABC):
 
         self._active_training_io = ActiveTrainingIO(
             self._training.training_folder, self.node.loop_communicator, context)
-        logger.info(f'new training initialized: {self._training}')
+        logger.info('new training initialized: %s', self._training)
 
     async def _run(self) -> None:
         """Called on `begin_training` event from the Learning Loop. 
@@ -218,8 +222,7 @@ class TrainerLogicGeneric(ABC):
         """
         self.errors.reset_all()
         try:
-            self.training_task = asyncio.get_running_loop().create_task(self._training_loop())
-            await self.training_task  # NOTE: Task object is used to potentially cancel the task
+            await self._training_loop()
         except asyncio.CancelledError:
             if not self.shutdown_event.is_set():
                 logger.info('CancelledError in _run - training task was cancelled but not by shutdown event')
@@ -229,8 +232,8 @@ class TrainerLogicGeneric(ABC):
                 self._may_restart()
             else:
                 logger.info('CancelledError in _run - shutting down')
-        except Exception as e:
-            logger.exception(f'Error in train: {e}')
+        except Exception:
+            logger.exception('(Ignored) exception in trainer_logic._run:')
 
     # ---------------------------------------- TRAINING STATES ----------------------------------------
 
@@ -271,7 +274,7 @@ class TrainerLogicGeneric(ABC):
         '''
 
         await asyncio.sleep(0.1)
-        logger.info(f'Performing state: {state_during}')
+        logger.info('Performing state: %s', state_during)
         previous_state = self.training.training_state
         self.training.training_state = state_during
         await asyncio.sleep(0.1)
@@ -283,12 +286,12 @@ class TrainerLogicGeneric(ABC):
 
         except asyncio.CancelledError:
             if self.shutdown_event.is_set():
-                logger.info(f'CancelledError in {state_during} - shutdown event set')
+                logger.info('CancelledError in %s - shutdown event set', state_during)
                 raise
-            logger.info(f'CancelledError in {state_during} - cleaning up')
+            logger.info('CancelledError in %s - cleaning up', state_during)
             self.training.training_state = TrainerState.ReadyForCleanup
         except CriticalError as e:
-            logger.error(f'CriticalError in {state_during} - Exception: {e}')
+            logger.error('CriticalError in %s - Exception: %s', state_during, e)
             self.errors.set(error_key, str(e))
             self.training.training_state = TrainerState.ReadyForCleanup
         except Exception as e:
@@ -297,7 +300,7 @@ class TrainerLogicGeneric(ABC):
             self.training.training_state = previous_state
             return
         else:
-            logger.info(f'Successfully finished state: {state_during}')
+            logger.info('Successfully finished state: %s', state_during)
             if not reset_early:
                 self.errors.reset(error_key)
             self.training.training_state = state_after

learning_loop_node/trainer/trainer_node.py

@@ -32,7 +32,7 @@ class TrainerNode(Node):
             self.log.info(
                 f'Trainer started with an idle_timeout of {self.idle_timeout} seconds. Note that shutdown does not work if docker container has the restart policy set to always')
 
-        if use_backdoor_controls:
+        if use_backdoor_controls or os.environ.get('USE_BACKDOOR_CONTROLS', '0').lower() in ('1', 'true'):
             self.include_router(backdoor_controls.router, tags=["controls"])
 
     # ----------------------------------- NODE LIVECYCLE METHODS --------------------------
@@ -77,7 +77,7 @@ class TrainerNode(Node):
 
     async def send_status(self):
         if not self.sio_client.connected:
-            self.log.warning('cannot send status - not connected to the Learning Loop')
+            self.log.debug('cannot send status - not connected to the Learning Loop')
             return
 
         status = TrainingStatus(id=self.uuid,
@@ -98,10 +98,11 @@ class TrainerNode(Node):
             status.errors = self.trainer_logic.errors.errors
             status.context = self.trainer_logic.training_context
 
-        self.log.info(f'sending status: {status.short_str()}')
+        self.log.debug('sending status: %s', status.short_str())
         result = await self.sio_client.call('update_trainer', jsonable_encoder(asdict(status)), timeout=30)
         if isinstance(result, Dict) and not result['success']:
-            self.log.error(f'Error when sending status update: Response from loop was:\n {result}')
+            self.socket_connection_broken = True
+            self.log.error('Error when sending status update: Response from loop was:\n %s', result)
 
     def check_idle_timeout(self):
         if not self.idle_timeout:

{learning_loop_node-0.10.12.dist-info → learning_loop_node-0.10.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.10.12
+Version: 0.10.14
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT
@@ -57,20 +57,21 @@ To start a node you have to implement the logic by inheriting from the correspon
 
 You can configure connection to our Learning Loop by specifying the following environment variables before starting:
 
-| Name                     | Alias        | Purpose                                                      | Required by          |
-| ------------------------ | ------------ | ------------------------------------------------------------ | -------------------- |
-| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                  |
-| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector |
-| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector |
-| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)           |
-| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector             |
-| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector             |
-| MIN_UNCERTAIN_THRESHOLD  | PROJECT      | smallest confidence (float) at which auto-upload will happen | Detector             |
-| MAX_UNCERTAIN_THRESHOLD  | PROJECT      | largest confidence (float) at which auto-upload will happen  | Detector             |
-| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)       |
-| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)       |
-| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)       |
-| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)       |
+| Name                     | Alias        | Purpose                                                      | Required by               |
+| ------------------------ | ------------ | ------------------------------------------------------------ | ------------------------- |
+| LOOP_HOST                | HOST         | Learning Loop address (e.g. learning-loop.ai)                | all                       |
+| LOOP_USERNAME            | USERNAME     | Learning Loop user name                                      | all besides Detector      |
+| LOOP_PASSWORD            | PASSWORD     | Learning Loop password                                       | all besides Detector      |
+| LOOP_SSL_CERT_PATH       | -            | Path to the SSL certificate                                  | all (opt.)                |
+| LOOP_ORGANIZATION        | ORGANIZATION | Organization name                                            | Detector                  |
+| LOOP_PROJECT             | PROJECT      | Project name                                                 | Detector (opt.)           |
+| MIN_UNCERTAIN_THRESHOLD  | -            | smallest confidence (float) at which auto-upload will happen | Detector (opt.)           |
+| MAX_UNCERTAIN_THRESHOLD  | -            | largest confidence (float) at which auto-upload will happen  | Detector (opt.)           |
+| INFERENCE_BATCH_SIZE     | -            | Batch size of trainer when calculating detections            | Trainer (opt.)            |
+| RESTART_AFTER_TRAINING   | -            | Restart the trainer after training (set to 1)                | Trainer (opt.)            |
+| KEEP_OLD_TRAININGS       | -            | Do not delete old trainings (set to 1)                       | Trainer (opt.)            |
+| TRAINER_IDLE_TIMEOUT_SEC | -            | Automatically shutdown trainer after timeout (in seconds)    | Trainer (opt.)            |
+| USE_BACKDOOR_CONTROLS    | -            | Always enable backdoor controls (set to 1)                   | Trainer / Detector (opt.) |
 
 #### Testing
 

{learning_loop_node-0.10.12.dist-info → learning_loop_node-0.10.14.dist-info}/RECORD

@@ -1,53 +1,55 @@
 learning_loop_node/__init__.py,sha256=onN5s8-x_xBsCM6NLmJO0Ym1sJHeCFaGw8qb0oQZmz8,364
 learning_loop_node/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/annotation/annotator_logic.py,sha256=BTaopkJZkIf1CI5lfsVKsxbxoUIbDJrevavuQUT5e_c,1000
-learning_loop_node/annotation/annotator_node.py,sha256=wk11CQtM3A0Dr7efCn_Mw2X7ql5xn2sgEJzrIeSBC6Q,4043
+learning_loop_node/annotation/annotator_node.py,sha256=UrJ8MpZ44UhsjmVuSHr2BhHyLC-kIMDi3IuBBMKzN1g,4117
 learning_loop_node/data_classes/__init__.py,sha256=wCX88lDgbb8V-gtVCVe9i-NvvZuMe5FX7eD_UJgYYXw,1305
 learning_loop_node/data_classes/annotations.py,sha256=iInU0Nuy_oYT_sj4k_n-W0UShCBI2cHQYrt8imymbtM,1211
-learning_loop_node/data_classes/detections.py,sha256=1BcU5PNzIbryWcj2xJ6ysLBTBwGOdv9SxSJiUG8WEmw,4349
-learning_loop_node/data_classes/general.py,sha256=Bd0ngYhYvS_9OYOO6lAKEnDzLuSdPmR4I2YV-0DRsxs,4694
+learning_loop_node/data_classes/detections.py,sha256=hifsGz2LbmeKLZdHxG7cnlOYNEqDmtJd2gxhyU-Xjjs,5811
+learning_loop_node/data_classes/general.py,sha256=usXokcTOVqTuaKJtBf0ffFWfzZhMrQtF7puKfwi6A5k,6195
 learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
 learning_loop_node/data_classes/training.py,sha256=hnMHZMk-WNRERyo7U97qL09v1tIdhnzPfTH-JgifLwU,6164
-learning_loop_node/data_exchanger.py,sha256=U_MrBKSq1MbBwBmjrjxoIo_7xV4Lcwtk6uZDIgmhT_4,8914
+learning_loop_node/data_exchanger.py,sha256=mwZvJf8L1-r6Wi2ZvDfFf4kFjkqxsThXj7AGLUANGlU,8979
 learning_loop_node/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/detector/detector_logic.py,sha256=se0jRFbV7BfTvCuCI3gcUllSYIZ5dxTkvdISe6pPTRg,1660
-learning_loop_node/detector/detector_node.py,sha256=v7hX06HuBtS4LiqlqOxDBaJlqkJanHQ4mrdjc4fVwDQ,18282
+learning_loop_node/detector/detector_logic.py,sha256=IG1s9RF_cCBcNQ8WW1rAS37QKdGzlVoVkuO_CrLGvYs,2084
+learning_loop_node/detector/detector_node.py,sha256=TeChzkpVVmEiZbnfWmtkjuUXNwCZvIWfingmcuAz2cs,19567
 learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=TD346I9ymtIP0_CJXCIKMRuiXbfVVanXNu_iHAwDd7Q,3318
-learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=s2FuwZ-tD_5obkSutstjc8pE_hLGbrv9WjrEO9t8rJ8,1011
-learning_loop_node/detector/outbox.py,sha256=MHHP4rnGaV8JxDSig96KZN4hSQ3i9z6-7WYhTTrMtp0,8082
+learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=7_-x8D8Zf6KJeJXmiC2VrRHU8Ig_R98uhdXVwwX0N4M,1240
+learning_loop_node/detector/outbox.py,sha256=HrYeS6XJLC-1kqq2hDufxXLRmOYGiBlz-m9B6HG5Ie8,8227
 learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/detector/rest/about.py,sha256=pNnIllLBJYc_edFCJGFTCBT55pIj7GI-zYsCZyjQ8EA,623
-learning_loop_node/detector/rest/backdoor_controls.py,sha256=38axRG66Z3_Q6bYKa7Hw-ldChEAu-dJcBM_Sl_17Ozo,1725
-learning_loop_node/detector/rest/detect.py,sha256=8Rl1swANKgHc42P1z75t_PErQxpCKKPdAsKqDIZgdNU,1873
-learning_loop_node/detector/rest/model_version_control.py,sha256=PKG7foFyNSvjoMhWCDb7w3mq-2e0bx5gq3ov7Rao8HU,3703
-learning_loop_node/detector/rest/operation_mode.py,sha256=eIo6_56qyZECftf4AEN8wJMABIojC0TRazvWeg0Uj_s,1664
-learning_loop_node/detector/rest/outbox_mode.py,sha256=anSZHB6jliz1t3fxrmEzgwNB62UHNdWNc9ZYOc5Nn9s,1018
+learning_loop_node/detector/rest/about.py,sha256=COYgmYO1tXGSIwjF__P79mVZUfSDZoHsW0GUarQ2rv0,1686
+learning_loop_node/detector/rest/backdoor_controls.py,sha256=ZNaFOvC0OLWNtcLiG-NIqS_y1kkLP4csgk3CHhp8Gis,885
+learning_loop_node/detector/rest/detect.py,sha256=KSYUOuTxuc2q3RgV37cBBuCvCJf11BFL7QtnPAM4XbU,2343
+learning_loop_node/detector/rest/model_version_control.py,sha256=jLp3rvCYq8T_QC3KK7uLDYpbDjydwazWkQCUXvkxl-c,4654
+learning_loop_node/detector/rest/operation_mode.py,sha256=RAzVLtGzy4n9-LSIq_XSwMfXDehU4XmorgWAWbQ6BW8,1804
+learning_loop_node/detector/rest/outbox_mode.py,sha256=H8coDNbgLGEfXmKQrhtXWeUHBAHpnrdZktuHXQz0xis,1148
 learning_loop_node/detector/rest/upload.py,sha256=IPzxJPayD7_Gx5uYC1lVJwWxdnQgM8MYGa5NugXVosY,544
 learning_loop_node/examples/novelty_score_updater.py,sha256=1DRgM9lxjFV-q2JvGDDsNLz_ic_rhEZ9wc6ZdjcxwPE,2038
 learning_loop_node/globals.py,sha256=tgw_8RYOipPV9aYlyUhYtXfUxvJKRvfUk6u-qVAtZmY,174
 learning_loop_node/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/helpers/environment_reader.py,sha256=OtCTDc0KT9r-SMygkZB_Mw-ZIJPfUZVyUzHJoDCgJP8,1658
 learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
-learning_loop_node/helpers/log_conf.py,sha256=3yd-jaMOeD5cRIgA5w_BH2L5odf8c4-ZjD89Bdqwe44,824
+learning_loop_node/helpers/log_conf.py,sha256=z_0PHh7U7DkJbSbKoSPyUfS7NhBHtRxXHdNcj67Hpbc,951
 learning_loop_node/helpers/misc.py,sha256=j4is8Rv0ttnCqF-R-wP3xwEi67OI6IBJav5Woo5lyDk,7701
 learning_loop_node/loop_communication.py,sha256=rG5MdavSTaREZ6OWfAUIT_qkkYPw3is2_FujLmHQeIc,6576
-learning_loop_node/node.py,sha256=mohvJGGfrMSGPTDvQOlX7TWs4QZ8YZHqr1YkyyGQphc,7983
+learning_loop_node/node.py,sha256=7Xc6VM_Rk1yDKDjKoUjoY4gSsC1zokjt5YQJu0PP5ao,9931
 learning_loop_node/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/rest.py,sha256=o1dl4Mtznd5duyEQtCYSGlK04l1Y-p_YRjG40Q4l31c,1491
 learning_loop_node/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/annotator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/annotator/conftest.py,sha256=G4ZvdZUdvPp9bYCzg3eEVkGCeXn9INZ3AcN7d5CyLkU,1931
 learning_loop_node/tests/annotator/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/annotator/test_annotator_node.py,sha256=TPNPPrQAxQ_zEecQcH7hlczgD3ABtTCNtUvWD1_oApk,1985
 learning_loop_node/tests/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/tests/detector/conftest.py,sha256=_zNEYIuiRmXj4qPsQli82JbqRR5CzhChkyo3dP8WWaU,4161
+learning_loop_node/tests/detector/conftest.py,sha256=Q14KHTSuSCsASVIxY9CttdVJm5FC7_JH-W5Q4CdDqoM,5414
 learning_loop_node/tests/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/detector/inbox_filter/test_observation.py,sha256=k4WYdvnuV7d_r7zI4M2aA8WuBjm0aycQ0vj1rGE2q4w,1370
 learning_loop_node/tests/detector/inbox_filter/test_relevance_group.py,sha256=XjiMsS0LgvM0OkPf5-s2rjFbG7C42LTmz_rDVMGHKoY,7603
 learning_loop_node/tests/detector/inbox_filter/test_unexpected_observations_count.py,sha256=MWC7PbaCy14jjRw0_oilkXj6gymAsUZXHJdzNW5m2D4,1639
 learning_loop_node/tests/detector/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/detector/test.jpg,sha256=msA-vHPmvPiro_D102Qmn1fn4vNfooqYYEXPxZUmYpk,161390
-learning_loop_node/tests/detector/test_client_communication.py,sha256=2gJARodJSDuJHgeN1_xLMbvDcPQkXpBXEefu7MOyePk,8998
+learning_loop_node/tests/detector/test_client_communication.py,sha256=NAOUrHWxoI4yG6oy3BGxWWXX794IOODEj9QBKF3CyrY,9375
+learning_loop_node/tests/detector/test_detector_node.py,sha256=KX2RcFpdIbpPEmcyYM0YMs-6wwTpbOOZONoiwIWryUI,2922
 learning_loop_node/tests/detector/test_outbox.py,sha256=5RMKQfuu1-rvpVCpEtt_D70bYgma-sIrTHWxHdTdU9Y,3001
 learning_loop_node/tests/detector/test_relevance_filter.py,sha256=3VLhHKaxPzLYmiNZagvgg9ZHkPhWk4_-qpmkJw36wBU,2046
 learning_loop_node/tests/detector/testing_detector.py,sha256=FeQroV85IvsT8dmalQBqf1FLNt_buCtZK3-lgtmbrBI,542
@@ -62,32 +64,32 @@ learning_loop_node/tests/general/test_downloader.py,sha256=C6b_wG3TfQX53lmuanpH1
 learning_loop_node/tests/general/test_learning_loop_node.py,sha256=SZd-VChpWnnsPN46pr4E_LL3ZevYx6psU-AWdVeOFpQ,770
 learning_loop_node/tests/test_helper.py,sha256=nTynYtuUaK2hKh87pk7t7AIJaOiD3wJ5d6nCPqnwRMk,3012
 learning_loop_node/tests/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/tests/trainer/conftest.py,sha256=7nncnC2ZApakdyta9TMUfk0sgr_nyZhGFYCHuUlBlFI,3356
+learning_loop_node/tests/trainer/conftest.py,sha256=E3SQL_CGFJ_sNjEfVJbxbvH0g6hjI5753ndAFUbnkQk,3366
 learning_loop_node/tests/trainer/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
 learning_loop_node/tests/trainer/state_helper.py,sha256=MDe9opeKruip74FoRFff8MSWGiQNFqDpPtIEIbgPnFc,919
 learning_loop_node/tests/trainer/states/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/tests/trainer/states/test_state_cleanup.py,sha256=gZNxSSwnj9f0esExNnQzqadM6-sE3IsF5sNbD0bZNu8,1250
-learning_loop_node/tests/trainer/states/test_state_detecting.py,sha256=nvhkjBnGZgGVyomvPDcXuBR_WLqC4-QD2qE5enWiJ-g,3724
-learning_loop_node/tests/trainer/states/test_state_download_train_model.py,sha256=aJBh0ksz9g2kQ3qqk8yrX-50kdRS4I7ZaeeFkr9CR3c,2863
-learning_loop_node/tests/trainer/states/test_state_prepare.py,sha256=yaDhT-1676asGfgqhK2mfdoktAgeyWUwc2h0xyOoWzQ,2340
-learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py,sha256=0Ak5bU2uiqRDSEzVSTBEoT9GN5UgL7SqBqJRyyxjmWE,4727
-learning_loop_node/tests/trainer/states/test_state_train.py,sha256=dQDUvVI_FUYdLWm11F_7WxGge30l0OvML40Hil9r--k,3149
-learning_loop_node/tests/trainer/states/test_state_upload_detections.py,sha256=U2qK1Vbzb0yBMBYNvB815AYBD6MsII7wxzYJrwFnUHQ,7554
-learning_loop_node/tests/trainer/states/test_state_upload_model.py,sha256=u_YXELOWOXoaU4v41jgr3jHPWVFeVbe330k8GCEL9K0,3659
-learning_loop_node/tests/trainer/test_errors.py,sha256=Z3BWvUkVKxMGe_RNYeVbrhPps7ylek3qS1zK2FpdMmU,2165
+learning_loop_node/tests/trainer/states/test_state_detecting.py,sha256=KGXTR69J_1pJoT8S0ceC3vSyHLw52mIpjbawH4c-8JA,3696
+learning_loop_node/tests/trainer/states/test_state_download_train_model.py,sha256=AuTY63sgrlKT0awS6o38fF3mTkDguAFJtcX7J7WhjgQ,2855
+learning_loop_node/tests/trainer/states/test_state_prepare.py,sha256=3hzRo9ycM802QUZO2Zs_rJRZ23hxTi3XjRCwL9M9m9o,2315
+learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py,sha256=6s4A2d5ahD9UJJiHGK1VtmOrhumzuqTlOwyc_8Oc1vk,5073
+learning_loop_node/tests/trainer/states/test_state_train.py,sha256=HYe1O6zcdtD4dnmfX3cyM1_iF7eGig7dQI4M4Xat7YU,2916
+learning_loop_node/tests/trainer/states/test_state_upload_detections.py,sha256=0Qkavl4i2tZmCOxKkNsQUqa1JWhAgcOsbrW3_eYHfxo,7417
+learning_loop_node/tests/trainer/states/test_state_upload_model.py,sha256=y2o4WBo7kBG_JWSWmt4icjrwya5hQ30zCWC-YMVEwEk,3621
+learning_loop_node/tests/trainer/test_errors.py,sha256=khWCTzi-JW4nSz9QnsRh9wDPmiuE_zdxXukh59qixuY,2109
 learning_loop_node/tests/trainer/test_trainer_states.py,sha256=djYCs5ieajQHRjk8QcUVBUkQEG8UGYFoNGwSX0z2oGk,1067
 learning_loop_node/tests/trainer/testing_trainer_logic.py,sha256=KslqDJDntkgH4Yd_z-guiVPvzi5Q-l-Bqc3fUjT5N7U,3883
 learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 learning_loop_node/trainer/downloader.py,sha256=Qk-oBcrGCVuWTVs3hvAJzQSqCIHPGZ7NXLJ_fAqvCoY,1469
 learning_loop_node/trainer/exceptions.py,sha256=hLLDGncC6PLZjKg4lZBpu-QA8itQIxiuxExz1uptgnw,40
 learning_loop_node/trainer/executor.py,sha256=-0BxDqmAI1NCiISi7Rw8McJQfgxxVy1gSa1epYuL3U0,3942
-learning_loop_node/trainer/io_helpers.py,sha256=Ylxz8HAId0Jlz95So5kXdJEp1yKQuwroDKIhbTUscF4,7257
+learning_loop_node/trainer/io_helpers.py,sha256=hGEtNAQBSBbVB56U1ndwfP8qK5K4YIwMQrjCDcaMy9I,7218
 learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-learning_loop_node/trainer/rest/backdoor_controls.py,sha256=YQcG0KwxzKDNYeMtHrSwr26q__N7ty0o6Kar6CLWAd0,5869
+learning_loop_node/trainer/rest/backdoor_controls.py,sha256=-pU4iHheBWf0SW2QzBVBsLiCMZBRz9CDdVZv6414Ts8,5134
 learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
-learning_loop_node/trainer/trainer_logic.py,sha256=PJxiO1chPdvpq8UTtzv_nVam9CouCswX9b1FnRwT2Tw,8411
-learning_loop_node/trainer/trainer_logic_generic.py,sha256=AzllMMiUPP_CMkjIVqse8wY50Cg5RDnk5y5ERVUjtZg,25801
-learning_loop_node/trainer/trainer_node.py,sha256=dIptgXTwBC_bnPo-U2sG_zDaaqZixY7FA0dPt7P2Lps,5261
-learning_loop_node-0.10.12.dist-info/METADATA,sha256=CskPsSnBXkMeNHiPOewq2UAN0qAMazghb7ReZvlBKGI,11702
-learning_loop_node-0.10.12.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
-learning_loop_node-0.10.12.dist-info/RECORD,,
+learning_loop_node/trainer/trainer_logic.py,sha256=PlYExIskU9pWJO0e9m_0KJnUdOI10GtW0oDOevYmg1o,8461
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=dVsvl1LC1w3svl4O8HCW49R7vSpVpC9Ermu4C20sFoA,25821
+learning_loop_node/trainer/trainer_node.py,sha256=8ANS9iy-swdTLvt9wEFixE6YlmqvqBl17A-R4tVYD-I,5384
+learning_loop_node-0.10.14.dist-info/METADATA,sha256=RHHVhHfTIPjxY5lO3UsR6L7BbHzwXs9wch4WJzePXJQ,11907
+learning_loop_node-0.10.14.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.10.14.dist-info/RECORD,,