learning-loop-node 0.9.3__py3-none-any.whl → 0.10.0__py3-none-any.whl

learning_loop_node/tests/test_executor.py

@@ -21,26 +21,28 @@ def cleanup():
     cleanup_process.communicate()
 
 
-def test_executor_lifecycle():
+@pytest.mark.asyncio
+async def test_executor_lifecycle():
     assert_process_is_running('some_executable.sh', False)
 
-    executor = Executor('/tmp/test_executor/' + str(uuid4()))
-    cmd = executor.path + '/some_executable.sh'
-    with open(cmd, 'w') as f:
-        f.write('while true; do echo "some output"; sleep 1; done')
-    os.chmod(cmd, 0o755)
+    executor = Executor('/tmp/test_executor/' + str(uuid4())+'/')
+    cmd = 'bash some_executable.sh'
+    executable_path = executor.path+'some_executable.sh'
+    with open(executable_path, 'w') as f:
+        f.write('/bin/bash -c "while true; do sleep 1; echo some output; done"')
+    os.chmod(executable_path, 0o755)
 
-    executor.start(cmd)
+    await executor.start(cmd)
 
-    assert executor.is_process_running()
+    assert executor.is_running()
     assert_process_is_running('some_executable.sh')
 
-    sleep(1)
+    sleep(5)
     assert 'some output' in executor.get_log()
 
-    executor.stop()
+    await executor.stop_and_wait()
 
-    assert not executor.is_process_running()
+    assert not executor.is_running()
     sleep(1)
     assert_process_is_running('some_executable.sh', False)
 
@@ -48,6 +50,7 @@ def test_executor_lifecycle():
 def assert_process_is_running(process_name, running=True):
     if running:
         for process in psutil.process_iter():
+            print(process.name(), process.cmdline())
             process_name_match = process_name in process.name()
             process_cmd_match = process_name in str(process.cmdline())
             if process_name_match or process_cmd_match:

learning_loop_node/tests/test_helper.py

@@ -7,10 +7,8 @@ from glob import glob
 from typing import Callable
 
 from learning_loop_node.data_classes import Context
-from learning_loop_node.helpers.misc import create_image_folder
+from learning_loop_node.helpers.misc import create_image_folder, create_project_folder, create_training_folder
 from learning_loop_node.loop_communication import LoopCommunicator
-from learning_loop_node.node import Node
-from learning_loop_node.trainer.trainer_logic import TrainerLogic
 
 
 def get_files_in_folder(folder: str):
@@ -65,8 +63,8 @@ def _update_attribute_dict(obj: dict, **kwargs) -> None:
 
 
 def create_needed_folders(training_uuid: str = 'some_uuid'):  # pylint: disable=unused-argument
-    project_folder = Node.create_project_folder(
+    project_folder = create_project_folder(
         Context(organization='zauberzeug', project='pytest'))
     image_folder = create_image_folder(project_folder)
-    training_folder = TrainerLogic.create_training_folder(project_folder, training_uuid)
+    training_folder = create_training_folder(project_folder, training_uuid)
     return project_folder, image_folder, training_folder

learning_loop_node/trainer/downloader.py

@@ -12,7 +12,7 @@ class TrainingsDownloader():
         self.data_exchanger = data_exchanger
 
     async def download_training_data(self, image_folder: str) -> Tuple[List[Dict], int]:
-        image_ids = await self.data_exchanger.fetch_image_ids(query_params=self.data_query_params)
+        image_ids = await self.data_exchanger.fetch_image_uuids(query_params=self.data_query_params)
         image_data, skipped_image_count = await self.download_images_and_annotations(image_ids, image_folder)
         return (image_data, skipped_image_count)
 

learning_loop_node/trainer/executor.py

@@ -1,105 +1,109 @@
-
-import ctypes
+import asyncio
 import logging
 import os
-import signal
-import subprocess
-from sys import platform
+import shlex
+from io import BufferedWriter
 from typing import List, Optional
 
-import psutil
 
+class Executor:
+    def __init__(self, base_path: str, log_name='last_training.log') -> None:
+        """An executor that runs a command in a separate async subprocess.
+        The log of the process is written to 'last_training.log' in the base_path.
+        Tthe process is executed in the base_path directory.
+        The process should be awaited to finish using `wait` or stopped using `stop` to 
+        avoid zombie processes and close the log file."""
 
-def create_signal_handler(sig=signal.SIGTERM):
-    if platform == "linux" or platform == "linux2":
-        # "The system will send a signal to the child once the parent exits for any reason (even sigkill)."
-        # https://stackoverflow.com/a/19448096
-        libc = ctypes.CDLL("libc.so.6")
+        self.path = base_path
+        self.log_file_path = f'{self.path}/{log_name}'
+        self.log_file: None | BufferedWriter = None
+        self._process: Optional[asyncio.subprocess.Process] = None  # pylint: disable=no-member
+        os.makedirs(self.path, exist_ok=True)
 
-        def callable_():
-            os.setsid()
-            return libc.prctl(1, sig)
+    def _get_running_process(self) -> Optional[asyncio.subprocess.Process]:  # pylint: disable=no-member
+        """Get the running process if available."""
+        if self._process is not None and self._process.returncode is None:
+            return self._process
+        return None
 
-        return callable_
-    return os.setsid
+    async def start(self, cmd: str, env: Optional[dict[str, str]] = None) -> None:
+        """Start the process with the given command and environment variables."""
 
+        full_env = os.environ.copy()
+        if env is not None:
+            full_env.update(env)
 
-class Executor:
-    def __init__(self, base_path: str) -> None:
-        self.path = base_path
-        os.makedirs(self.path, exist_ok=True)
-        self.process: Optional[subprocess.Popen[bytes]] = None
-
-    def start(self, cmd: str):
-        with open(f'{self.path}/last_training.log', 'a') as f:
-            f.write(f'\nStarting executor with command: {cmd}\n')
-        # pylint: disable=subprocess-popen-preexec-fn
-        self.process = subprocess.Popen(
-            f'cd {self.path}; {cmd} >> last_training.log 2>&1',
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            executable='/bin/bash',
-            preexec_fn=create_signal_handler(),
-        )
+        logging.info(f'Starting executor with command: {cmd} in {self.path} - logging to {self.log_file_path}')
+        self.log_file = open(self.log_file_path, 'ab')
 
-    def is_process_running(self):
-        if self.process is None:
-            return False
+        self._process = await asyncio.create_subprocess_exec(
+            *shlex.split(cmd),
+            cwd=self.path,
+            stdout=self.log_file,
+            stderr=asyncio.subprocess.STDOUT,  # Merge stderr with stdout
+            env=full_env
+        )
 
-        if self.process.poll() is not None:
-            return False
+    def is_running(self) -> bool:
+        """Check if the process is still running."""
+        return self._process is not None and self._process.returncode is None
 
-        try:
-            psutil.Process(self.process.pid)
-        except psutil.NoSuchProcess:
-            # self.process.terminate() # TODO does this make sense?
-            # self.process = None
-            return False
+    def terminate(self) -> None:
+        """Terminate the process."""
 
-        return True
+        if process := self._get_running_process():
+            try:
+                process.terminate()
+                return
+            except ProcessLookupError:
+                logging.error('No process to terminate')
+        self._process = None
 
-    def get_log(self) -> str:
-        try:
-            with open(f'{self.path}/last_training.log') as f:
-                return f.read()
-        except Exception:
-            return ''
+    async def wait(self) -> Optional[int]:
+        """Wait for the process to finish. Returns the return code of the process or None if no process is running."""
 
-    def get_log_by_lines(self, since_last_start=False) -> List[str]:  # TODO do not read whole log again
-        try:
-            with open(f'{self.path}/last_training.log') as f:
-                lines = f.readlines()
-            if since_last_start:
-                lines_since_last_start = []
-                for line in reversed(lines):
-                    lines_since_last_start.append(line)
-                    if line.startswith('Starting executor with command:'):
-                        break
-                return list(reversed(lines_since_last_start))
-            return lines
-        except Exception:
-            return []
+        if not self._process:
+            logging.info('No process to wait for')
+            return None
 
-    def stop(self):
-        if self.process is None:
-            logging.info('no process running ... nothing to stop')
-            return
+        return_code = await self._process.wait()
 
-        logging.info('terminating process')
+        self.close_log()
+        self._process = None
 
-        try:
-            os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
-        except ProcessLookupError:
-            pass
+        return return_code
 
-        self.process.terminate()
-        _, _ = self.process.communicate(timeout=3)
+    async def stop_and_wait(self) -> Optional[int]:
+        """Terminate the process and wait for it to finish. Returns the return code of the process."""
 
-    @property
-    def return_code(self):
-        if not self.process:
-            return None
-        if self.is_process_running():
+        if not self.is_running():
+            logging.info('No process to stop')
             return None
-        return self.process.poll()
+
+        self.terminate()
+        return await self.wait()
+
+    # -------------------------------------------------------------------------------------------- LOGGING
+
+    def get_log(self) -> str:
+        """Get the log of the process as a string."""
+        if not os.path.exists(self.log_file_path):
+            return ''
+        with open(self.log_file_path, 'r') as f:
+            return f.read()
+
+    def get_log_by_lines(self, tail: Optional[int] = None) -> List[str]:
+        """Get the log of the process as a list of lines."""
+        if not os.path.exists(self.log_file_path):
+            return []
+        with open(self.log_file_path) as f:
+            lines = f.readlines()
+        if tail is not None:
+            lines = lines[-tail:]
+        return lines
+
+    def close_log(self):
+        """Close the log file."""
+        if self.log_file is not None:
+            self.log_file.close()
+            self.log_file = None

learning_loop_node/trainer/io_helpers.py

@@ -1,5 +1,6 @@
 
 import json
+import logging
 import os
 from dataclasses import asdict
 from pathlib import Path
@@ -8,8 +9,19 @@ from typing import List
 from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
 
-from ..data_classes import Detections, Training
+from ..data_classes import Context, Detections, Training
 from ..globals import GLOBALS
+from ..loop_communication import LoopCommunicator
+
+
+class EnvironmentVars:
+    def __init__(self) -> None:
+        self.restart_after_training = os.environ.get(
+            'RESTART_AFTER_TRAINING', 'FALSE').lower() in ['true', '1']
+        self.keep_old_trainings = os.environ.get(
+            'KEEP_OLD_TRAININGS', 'FALSE').lower() in ['true', '1']
+        self.inference_batch_size = int(
+            os.environ.get('INFERENCE_BATCH_SIZE', '10'))
 
 
 class LastTrainingIO:
@@ -35,13 +47,16 @@ class LastTrainingIO:
 
 class ActiveTrainingIO:
 
-    @staticmethod
-    def create_mocked_training_io() -> 'ActiveTrainingIO':
-        training_folder = ''
-        return ActiveTrainingIO(training_folder)
+    # @staticmethod
+    # def create_mocked_training_io() -> 'ActiveTrainingIO':
+    #     training_folder = ''
+    #     return ActiveTrainingIO(training_folder)
 
-    def __init__(self, training_folder: str):
+    def __init__(self, training_folder: str, loop_communicator: LoopCommunicator, context: Context) -> None:
         self.training_folder = training_folder
+        self.loop_communicator = loop_communicator
+        self.context = context
+
         self.mup_path = f'{training_folder}/model_uploading_progress.txt'
         # string with placeholder gor index
         self.det_path = f'{training_folder}' + '/detections_{0}.json'
@@ -63,13 +78,16 @@ class ActiveTrainingIO:
 
     # detections
 
-    def get_detection_file_names(self) -> List[Path]:
+    def _get_detection_file_names(self) -> List[Path]:
         files = [f for f in Path(self.training_folder).iterdir()
                  if f.is_file() and f.name.startswith('detections_')]
         if not files:
             return []
         return files
 
+    def get_number_of_detection_files(self) -> int:
+        return len(self._get_detection_file_names())
+
     # TODO: saving and uploading multiple files is not tested!
     def save_detections(self, detections: List[Detections], index: int = 0) -> None:
         with open(self.det_path.format(index), 'w') as f:
@@ -81,11 +99,11 @@ class ActiveTrainingIO:
             return [from_dict(data_class=Detections, data=d) for d in dict_list]
 
     def delete_detections(self) -> None:
-        for file in self.get_detection_file_names():
+        for file in self._get_detection_file_names():
             os.remove(Path(self.training_folder) / file)
 
     def detections_exist(self) -> bool:
-        return bool(self.get_detection_file_names())
+        return bool(self._get_detection_file_names())
 
     # detections upload file index
 
@@ -124,3 +142,42 @@ class ActiveTrainingIO:
 
     def detection_upload_progress_exist(self) -> bool:
         return os.path.exists(self.dup_path)
+
+    async def upload_detetions(self):
+        num_files = self.get_number_of_detection_files()
+        print(f'num_files: {num_files}', flush=True)
+        if not num_files:
+            logging.error('no detection files found')
+            return
+        current_json_file_index = self.load_detections_upload_file_index()
+        for i in range(current_json_file_index, num_files):
+            detections = self.load_detections(i)
+            logging.info(f'uploading detections {i}/{num_files}')
+            await self._upload_detections_batched(self.context, detections)
+            self.save_detections_upload_file_index(i+1)
+
+    async def _upload_detections_batched(self, context: Context, detections: List[Detections]):
+        batch_size = 10
+        skip_detections = self.load_detection_upload_progress()
+        for i in range(skip_detections, len(detections), batch_size):
+            up_progress = i+batch_size
+            batch_detections = detections[i:up_progress]
+            dict_detections = [jsonable_encoder(asdict(detection)) for detection in batch_detections]
+            logging.info(f'uploading detections. File size : {len(json.dumps(dict_detections))}')
+            await self._upload_detections(context, batch_detections, up_progress)
+            skip_detections = up_progress
+
+    async def _upload_detections(self, context: Context, batch_detections: List[Detections], up_progress: int):
+        detections_json = [jsonable_encoder(asdict(detections)) for detections in batch_detections]
+        response = await self.loop_communicator.post(
+            f'/{context.organization}/projects/{context.project}/detections', json=detections_json)
+        if response.status_code != 200:
+            msg = f'could not upload detections. {str(response)}'
+            logging.error(msg)
+            raise Exception(msg)
+
+        logging.info('successfully uploaded detections')
+        if up_progress > len(batch_detections):
+            self.save_detection_upload_progress(0)
+        else:
+            self.save_detection_upload_progress(up_progress)

learning_loop_node/trainer/rest/backdoor_controls.py

@@ -5,10 +5,10 @@ import logging
 from dataclasses import asdict
 from typing import TYPE_CHECKING, Dict
 
-from dacite import from_dict
 from fastapi import APIRouter, HTTPException, Request
 
 from ...data_classes import ErrorConfiguration, NodeState
+from ..trainer_logic import TrainerLogic
 
 if TYPE_CHECKING:
     from ..trainer_node import TrainerNode
@@ -95,7 +95,9 @@ async def add_steps(request: Request):
     trainer_node = trainer_node_from_request(request)
     trainer_logic = trainer_node.trainer_logic  # NOTE: is MockTrainerLogic which has 'provide_new_model' and 'current_iteration'
 
-    if not trainer_logic._executor or not trainer_logic._executor.is_process_running():  # pylint: disable=protected-access
+    assert isinstance(trainer_logic, TrainerLogic), 'trainer_logic is not TrainerLogic'
+
+    if not trainer_logic._executor or not trainer_logic._executor.is_running():  # pylint: disable=protected-access
         training = trainer_logic._training  # pylint: disable=protected-access
         logging.error(f'cannot add steps when not running, state: {training.training_state if training else "None"}')
         raise HTTPException(status_code=409, detail="trainer is not running")
@@ -109,7 +111,7 @@ async def add_steps(request: Request):
     for _ in range(steps):
         try:
             logging.warning('calling sync_confusion_matrix')
-            await trainer_logic.sync_confusion_matrix()
+            await trainer_logic._sync_confusion_matrix()  # pylint: disable=protected-access
         except Exception:
             pass  # Tests can force synchroniation to fail, error state is reported to backend
     trainer_logic.provide_new_model = previous_state  # type: ignore
@@ -119,11 +121,14 @@ async def add_steps(request: Request):
 
 @router.post("/kill_training_process")
 async def kill_process(request: Request):
+
     # pylint: disable=protected-access
     trainer_node = trainer_node_from_request(request)
-    if not trainer_node.trainer_logic._executor or not trainer_node.trainer_logic._executor.is_process_running():
+    trainer_logic = trainer_node.trainer_logic
+    assert isinstance(trainer_logic, TrainerLogic), 'trainer_logic is not TrainerLogic'
+    if not trainer_logic._executor or not trainer_logic._executor.is_running():
         raise HTTPException(status_code=409, detail="trainer is not running")
-    trainer_node.trainer_logic._executor.stop()
+    await trainer_logic._executor.stop_and_wait()
 
 
 @router.post("/force_status_update")

learning_loop_node/trainer/rest/controls.py

@@ -7,6 +7,8 @@ from learning_loop_node.trainer.trainer_logic import TrainerLogic
 
 router = APIRouter()
 
+# pylint: disable=protected-access
+
 
 @router.post("/controls/detect/{organization}/{project}/{version}")
 async def operation_mode(organization: str, project: str, version: str, request: Request):
@@ -22,5 +24,5 @@ async def operation_mode(organization: str, project: str, version: str, request:
     model_id = next(m for m in models if m['version'] == version)['id']
     logging.info(model_id)
     trainer: TrainerLogic = request.app.trainer
-    await trainer.do_detections()
+    await trainer._do_detections()
     return "OK"

learning_loop_node/trainer/tests/conftest.py

@@ -10,6 +10,8 @@ from learning_loop_node.data_classes import Context
 from learning_loop_node.trainer.tests.testing_trainer_logic import TestingTrainerLogic
 from learning_loop_node.trainer.trainer_node import TrainerNode
 
+# pylint: disable=protected-access
+
 logging.basicConfig(level=logging.INFO)
 # show ouptut from uvicorn server https://stackoverflow.com/a/66132186/364388
 log_to_stderr(logging.INFO)
@@ -24,16 +26,14 @@ async def test_initialized_trainer_node():
 
     trainer = TestingTrainerLogic()
     node = TrainerNode(name='test', trainer_logic=trainer, uuid='NOD30000-0000-0000-0000-000000000000')
-    trainer._node = node  # pylint: disable=protected-access
-    trainer.init_new_training(context=Context(organization='zauberzeug', project='demo'),
-                              details={'categories': [],
-                                       'id': '917d5c7f-403d-7e92-f95f-577f79c2273a',  # version 1.2 of demo project
-                                       'training_number': 0,
-                                       'resolution': 800,
-                                       'flip_rl': False,
-                                       'flip_ud': False})
-
-    # pylint: disable=protected-access
+    trainer._node = node
+    trainer._init_new_training(context=Context(organization='zauberzeug', project='demo'),
+                               details={'categories': [],
+                                        'id': '917d5c7f-403d-7e92-f95f-577f79c2273a',  # version 1.2 of demo project
+                                        'training_number': 0,
+                                        'resolution': 800,
+                                        'flip_rl': False,
+                                        'flip_ud': False})
     await node._on_startup()
     yield node
     await node._on_shutdown()
@@ -44,19 +44,17 @@ async def test_initialized_trainer():
 
     trainer = TestingTrainerLogic()
     node = TrainerNode(name='test', trainer_logic=trainer, uuid='NODE-000-0000-0000-0000-000000000000')
-    # pylint: disable=protected-access
-    await node._on_startup()
-    trainer._node = node  # pylint: disable=protected-access
-    trainer.init_new_training(context=Context(organization='zauberzeug', project='demo'),
-                              details={'categories': [],
-                                       'id': '917d5c7f-403d-7e92-f95f-577f79c2273a',  # version 1.2 of demo project
-                                       'training_number': 0,
-                                       'resolution': 800,
-                                       'flip_rl': False,
-                                       'flip_ud': False})
 
+    await node._on_startup()
+    trainer._node = node
+    trainer._init_new_training(context=Context(organization='zauberzeug', project='demo'),
+                               details={'categories': [],
+                                        'id': '917d5c7f-403d-7e92-f95f-577f79c2273a',  # version 1.2 of demo project
+                                        'training_number': 0,
+                                        'resolution': 800,
+                                        'flip_rl': False,
+                                        'flip_ud': False})
     yield trainer
-    # await node._on_shutdown()
     try:
         await node._on_shutdown()
     except Exception:
@@ -66,10 +64,3 @@ async def test_initialized_trainer():
 def is_port_in_use(port):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         return s.connect_ex(('localhost', port)) == 0
-
-
-# @pytest.fixture(autouse=True, scope='session')
-# def initialize_active_training():
-#     from learning_loop_node.trainer import active_training_module
-#     active_training_module.init('00000000-0000-0000-0000-000000000000')
-#     yield

learning_loop_node/trainer/tests/states/test_state_cleanup.py

@@ -1,11 +1,13 @@
 from learning_loop_node.trainer.tests.state_helper import create_active_training_file
 from learning_loop_node.trainer.tests.testing_trainer_logic import TestingTrainerLogic
 
+# pylint: disable=protected-access
+
 
 async def test_cleanup_successfull(test_initialized_trainer: TestingTrainerLogic):
     trainer = test_initialized_trainer
     create_active_training_file(trainer, training_state='ready_for_cleanup')
-    trainer.init_from_last_training()
+    trainer._init_from_last_training()
     trainer.active_training_io.save_detections(detections=[])
 
     trainer.active_training_io.save_detection_upload_progress(count=42)
@@ -16,9 +18,9 @@ async def test_cleanup_successfull(test_initialized_trainer: TestingTrainerLogic
     assert trainer.active_training_io.detection_upload_progress_exist() is True
     assert trainer.active_training_io.detections_upload_file_index_exists() is True
 
-    await trainer.clear_training()
+    await trainer._clear_training()
 
-    assert trainer._training is None  # pylint: disable=protected-access
+    assert trainer._training is None
     assert trainer.node.last_training_io.exists() is False
     assert trainer.active_training_io.detections_exist() is False
     assert trainer.active_training_io.detection_upload_progress_exist() is False

learning_loop_node/trainer/tests/states/test_state_detecting.py

@@ -1,11 +1,12 @@
 import asyncio
 
 from learning_loop_node.conftest import get_dummy_detections
-from learning_loop_node.data_classes import TrainingState
+from learning_loop_node.data_classes import TrainerState
 from learning_loop_node.trainer.tests.state_helper import assert_training_state, create_active_training_file
 from learning_loop_node.trainer.tests.testing_trainer_logic import TestingTrainerLogic
 from learning_loop_node.trainer.trainer_logic import TrainerLogic
 
+# pylint: disable=protected-access
 error_key = 'detecting'
 
 
@@ -13,60 +14,62 @@ def trainer_has_error(trainer: TrainerLogic):
     return trainer.errors.has_error_for(error_key)
 
 
-async def test_successful_detecting(test_initialized_trainer: TestingTrainerLogic):  # TODO Flaky test
+async def test_successful_detecting(test_initialized_trainer: TestingTrainerLogic):  # NOTE was a flaky test
     trainer = test_initialized_trainer
     create_active_training_file(trainer, training_state='train_model_uploaded',
-                                model_id_for_detecting='917d5c7f-403d-7e92-f95f-577f79c2273a')
+                                model_uuid_for_detecting='917d5c7f-403d-7e92-f95f-577f79c2273a')
     # trainer.load_active_training()
-    _ = asyncio.get_running_loop().create_task(trainer.do_detections())
+    _ = asyncio.get_running_loop().create_task(
+        trainer._perform_state('do_detections', TrainerState.Detecting, TrainerState.Detected, trainer._do_detections))
 
-    await assert_training_state(trainer.training, 'detecting', timeout=1, interval=0.001)
-    await assert_training_state(trainer.training, 'detected', timeout=10, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.Detecting, timeout=1, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.Detected, timeout=10, interval=0.001)
 
     assert trainer_has_error(trainer) is False
-    assert trainer.training.training_state == 'detected'
+    assert trainer.training.training_state == TrainerState.Detected
     assert trainer.node.last_training_io.load() == trainer.training
     assert trainer.active_training_io.detections_exist()
 
 
 async def test_detecting_can_be_aborted(test_initialized_trainer: TestingTrainerLogic):
     trainer = test_initialized_trainer
-    create_active_training_file(trainer, training_state=TrainingState.TrainModelUploaded)
-    trainer.init_from_last_training()
-    trainer.training.model_id_for_detecting = '12345678-bobo-7e92-f95f-424242424242'
+    create_active_training_file(trainer, training_state=TrainerState.TrainModelUploaded)
+    trainer._init_from_last_training()
+    trainer.training.model_uuid_for_detecting = '12345678-bobo-7e92-f95f-424242424242'
 
-    _ = asyncio.get_running_loop().create_task(trainer.run())
+    _ = asyncio.get_running_loop().create_task(trainer._run())
 
-    await assert_training_state(trainer.training, 'detecting', timeout=5, interval=0.001)
+    await assert_training_state(trainer.training, TrainerState.Detecting, timeout=5, interval=0.001)
     await trainer.stop()
     await asyncio.sleep(0.1)
 
-    assert trainer._training is None  # pylint: disable=protected-access
+    assert trainer._training is None
     assert trainer.active_training_io.detections_exist() is False
     assert trainer.node.last_training_io.exists() is False
 
 
 async def test_model_not_downloadable_error(test_initialized_trainer: TestingTrainerLogic):
     trainer = test_initialized_trainer
-    create_active_training_file(trainer, training_state='train_model_uploaded',
-                                model_id_for_detecting='00000000-0000-0000-0000-000000000000')  # bad model id
-    trainer.init_from_last_training()
+    create_active_training_file(trainer, training_state=TrainerState.TrainModelUploaded,
+                                model_uuid_for_detecting='00000000-0000-0000-0000-000000000000')  # bad model id
+    trainer._init_from_last_training()
 
-    _ = asyncio.get_running_loop().create_task(trainer.run())
+    _ = asyncio.get_running_loop().create_task(trainer._run())
 
     await assert_training_state(trainer.training, 'detecting', timeout=1, interval=0.001)
     await assert_training_state(trainer.training, 'train_model_uploaded', timeout=1, interval=0.001)
+    await asyncio.sleep(0.1)
 
     assert trainer_has_error(trainer)
-    assert trainer.training.training_state == 'train_model_uploaded'
-    assert trainer.training.model_id_for_detecting == '00000000-0000-0000-0000-000000000000'
+    assert trainer.training.training_state == TrainerState.TrainModelUploaded
+    assert trainer.training.model_uuid_for_detecting == '00000000-0000-0000-0000-000000000000'
     assert trainer.node.last_training_io.load() == trainer.training
 
 
 def test_save_load_detections(test_initialized_trainer: TestingTrainerLogic):
     trainer = test_initialized_trainer
     create_active_training_file(trainer)
-    trainer.init_from_last_training()
+    trainer._init_from_last_training()
 
     detections = [get_dummy_detections(), get_dummy_detections()]