learning-loop-node 0.9.3__py3-none-any.whl → 0.10.1__py3-none-any.whl

learning_loop_node/trainer/trainer_node.py

@@ -1,74 +1,59 @@
 import asyncio
-import time
 from dataclasses import asdict
-from typing import Dict, Optional, Union
+from typing import Dict, Optional
 
-from dacite import from_dict
 from fastapi.encoders import jsonable_encoder
 from socketio import AsyncClient
 
-from ..data_classes import Context, NodeState, TrainingState, TrainingStatus
-from ..data_classes.socket_response import SocketResponse
+from ..data_classes import TrainingStatus
 from ..node import Node
 from .io_helpers import LastTrainingIO
 from .rest import backdoor_controls, controls
-from .trainer_logic import TrainerLogic
+from .trainer_logic_generic import TrainerLogicGeneric
 
 
 class TrainerNode(Node):
 
-    def __init__(self, name: str, trainer_logic: TrainerLogic, uuid: Optional[str] = None, use_backdoor_controls: bool = False):
-        super().__init__(name, uuid)
-        trainer_logic._node = self  # pylint: disable=protected-access
+    def __init__(self, name: str, trainer_logic: TrainerLogicGeneric, uuid: Optional[str] = None, use_backdoor_controls: bool = False):
+        super().__init__(name, uuid, 'trainer')
+        trainer_logic._node = self
         self.trainer_logic = trainer_logic
         self.last_training_io = LastTrainingIO(self.uuid)
+        self.trainer_logic._last_training_io = self.last_training_io
+
         self.include_router(controls.router, tags=["controls"])
         if use_backdoor_controls:
             self.include_router(backdoor_controls.router, tags=["controls"])
 
-    # --------------------------------------------------- STATUS ---------------------------------------------------
-
-    @property
-    def progress(self) -> Union[float, None]:
-        return self.trainer_logic.general_progress if (self.trainer_logic is not None and
-                                                       hasattr(self.trainer_logic, 'general_progress')) else None
-
-    @property
-    def training_uptime(self) -> Union[float, None]:
-        return time.time() - self.trainer_logic.start_time if self.trainer_logic.start_time else None
-
-    # ----------------------------------- LIVECYCLE: ABSTRACT NODE METHODS --------------------------
+    # ----------------------------------- NODE LIVECYCLE METHODS --------------------------
 
     async def on_startup(self):
         pass
 
     async def on_shutdown(self):
         self.log.info('shutdown detected, stopping training')
-        await self.trainer_logic.shutdown()
+        await self.trainer_logic.on_shutdown()
 
     async def on_repeat(self):
         try:
-            if await self.continue_run_if_incomplete():
+            if await self.trainer_logic.try_continue_run_if_incomplete():
                 return  # NOTE: we prevent sending idle status after starting a continuation
             await self.send_status()
         except Exception as e:
             if isinstance(e, asyncio.TimeoutError):
                 self.log.warning('timeout when sending status to learning loop, reconnecting sio_client')
-                await self.sio_client.disconnect()
-                # NOTE: reconnect happens in node._on_repeat
+                await self.sio_client.disconnect()  # NOTE: reconnect happens in node._on_repeat
             else:
                 self.log.exception(f'could not send status state: {e}')
 
-    # ---------------------------------------------- NODE ABSTRACT METHODS ---------------------------------------------------
+    # ---------------------------------------------- NODE METHODS ---------------------------------------------------
 
     def register_sio_events(self, sio_client: AsyncClient):
 
         @sio_client.event
         async def begin_training(organization: str, project: str, details: Dict):
-            assert self._sio_client is not None
             self.log.info('received begin_training from server')
-            self.trainer_logic.init_new_training(Context(organization=organization, project=project), details)
-            asyncio.get_event_loop().create_task(self.trainer_logic.run())
+            await self.trainer_logic.begin_training(organization, project, details)
             return True
 
         @sio_client.event
@@ -81,93 +66,29 @@ class TrainerNode(Node):
             return True
 
     async def send_status(self):
-        if self._sio_client is None or not self._sio_client.connected:
+        if not self.sio_client.connected:
             self.log.warning('cannot send status - not connected to the Learning Loop')
             return
 
-        if not self.trainer_logic.is_initialized:
-            state_for_learning_loop = str(NodeState.Idle.value)
-        else:
-            assert self.trainer_logic.training.training_state is not None
-            state_for_learning_loop = TrainerNode.state_for_learning_loop(
-                self.trainer_logic.training.training_state)
-
         status = TrainingStatus(id=self.uuid,
                                 name=self.name,
-                                state=state_for_learning_loop,
+                                state=self.trainer_logic.state,
                                 errors={},
-                                uptime=self.training_uptime,
-                                progress=self.progress)
+                                uptime=self.trainer_logic.training_uptime,
+                                progress=self.trainer_logic.general_progress)
 
         status.pretrained_models = self.trainer_logic.provided_pretrained_models
         status.architecture = self.trainer_logic.model_architecture
 
-        if self.trainer_logic.is_initialized and self.trainer_logic.training.data:
-            status.train_image_count = self.trainer_logic.training.data.train_image_count()
-            status.test_image_count = self.trainer_logic.training.data.test_image_count()
-            status.skipped_image_count = self.trainer_logic.training.data.skipped_image_count
-            status.hyperparameters = self.trainer_logic.hyperparameters
+        if data := self.trainer_logic.training_data:
+            status.train_image_count = data.train_image_count()
+            status.test_image_count = data.test_image_count()
+            status.skipped_image_count = data.skipped_image_count
+            status.hyperparameters = self.trainer_logic.hyperparameters_for_state_sync
             status.errors = self.trainer_logic.errors.errors
-            status.context = self.trainer_logic.training.context
+            status.context = self.trainer_logic.training_context
 
         self.log.info(f'sending status: {status.short_str()}')
-        result = await self._sio_client.call('update_trainer', jsonable_encoder(asdict(status)), timeout=30)
-        assert isinstance(result, Dict)
-        response = from_dict(data_class=SocketResponse, data=result)
-
-        if not response.success:
-            self.log.error(f'Error when sending status update: Response from loop was:\n {asdict(response)}')
-
-    async def continue_run_if_incomplete(self) -> bool:
-        if not self.trainer_logic.is_initialized and self.last_training_io.exists():
-            self.log.info('found incomplete training, continuing now.')
-            self.trainer_logic.init_from_last_training()
-            asyncio.get_event_loop().create_task(self.trainer_logic.run())
-            return True
-        return False
-
-    async def get_state(self):
-        if self.trainer_logic._executor is not None and self.trainer_logic._executor.is_process_running():  # pylint: disable=protected-access
-            return NodeState.Running
-        return NodeState.Idle
-
-    def get_node_type(self):
-        return 'trainer'
-
-    # --------------------------------------------------- HELPER ---------------------------------------------------
-
-    @staticmethod
-    def state_for_learning_loop(trainer_state: Union[TrainingState, str]) -> str:
-        if trainer_state == TrainingState.Initialized:
-            return 'Training is initialized'
-        if trainer_state == TrainingState.DataDownloading:
-            return 'Downloading data'
-        if trainer_state == TrainingState.DataDownloaded:
-            return 'Data downloaded'
-        if trainer_state == TrainingState.TrainModelDownloading:
-            return 'Downloading model'
-        if trainer_state == TrainingState.TrainModelDownloaded:
-            return 'Model downloaded'
-        if trainer_state == TrainingState.TrainingRunning:
-            return NodeState.Running
-        if trainer_state == TrainingState.TrainingFinished:
-            return 'Training finished'
-        if trainer_state == TrainingState.Detecting:
-            return NodeState.Detecting
-        if trainer_state == TrainingState.ConfusionMatrixSyncing:
-            return 'Syncing confusion matrix'
-        if trainer_state == TrainingState.ConfusionMatrixSynced:
-            return 'Confusion matrix synced'
-        if trainer_state == TrainingState.TrainModelUploading:
-            return 'Uploading trained model'
-        if trainer_state == TrainingState.TrainModelUploaded:
-            return 'Trained model uploaded'
-        if trainer_state == TrainingState.Detecting:
-            return 'calculating detections'
-        if trainer_state == TrainingState.Detected:
-            return 'Detections calculated'
-        if trainer_state == TrainingState.DetectionUploading:
-            return 'Uploading detections'
-        if trainer_state == TrainingState.ReadyForCleanup:
-            return 'Cleaning training'
-        return 'unknown state'
+        result = await self.sio_client.call('update_trainer', jsonable_encoder(asdict(status)), timeout=30)
+        if isinstance(result, Dict) and not result['success']:
+            self.log.error(f'Error when sending status update: Response from loop was:\n {result}')

{learning_loop_node-0.9.3.dist-info → learning_loop_node-0.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: learning-loop-node
-Version: 0.9.3
+Version: 0.10.1
 Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
 Home-page: https://github.com/zauberzeug/learning_loop_node
 License: MIT

learning_loop_node-0.10.1.dist-info/RECORD

@@ -0,0 +1,85 @@
+learning_loop_node/__init__.py,sha256=onN5s8-x_xBsCM6NLmJO0Ym1sJHeCFaGw8qb0oQZmz8,364
+learning_loop_node/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/annotation/annotator_logic.py,sha256=BTaopkJZkIf1CI5lfsVKsxbxoUIbDJrevavuQUT5e_c,1000
+learning_loop_node/annotation/annotator_node.py,sha256=wk11CQtM3A0Dr7efCn_Mw2X7ql5xn2sgEJzrIeSBC6Q,4043
+learning_loop_node/annotation/tests/test_annotator_node.py,sha256=yCox-ohCXOpCbcTit7PwtE4iLHud8qiZ6UOqqycwPxg,2065
+learning_loop_node/conftest.py,sha256=Q_8Dbl3RwI8X_Y5Zvzstr6NJEqY3KldZXVx618OLdb4,3492
+learning_loop_node/data_classes/__init__.py,sha256=wCX88lDgbb8V-gtVCVe9i-NvvZuMe5FX7eD_UJgYYXw,1305
+learning_loop_node/data_classes/annotations.py,sha256=iInU0Nuy_oYT_sj4k_n-W0UShCBI2cHQYrt8imymbtM,1211
+learning_loop_node/data_classes/detections.py,sha256=1BcU5PNzIbryWcj2xJ6ysLBTBwGOdv9SxSJiUG8WEmw,4349
+learning_loop_node/data_classes/general.py,sha256=44GJrJvGfPwDUmRsS7If9uSlE6KPP50LGUX91VzesLw,4664
+learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
+learning_loop_node/data_classes/training.py,sha256=hnMHZMk-WNRERyo7U97qL09v1tIdhnzPfTH-JgifLwU,6164
+learning_loop_node/data_exchanger.py,sha256=hxF0zANA35f5EV8tkQ4yjelrKuvafMaKUya0CCjVrK0,8221
+learning_loop_node/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/detector/detector_logic.py,sha256=se0jRFbV7BfTvCuCI3gcUllSYIZ5dxTkvdISe6pPTRg,1660
+learning_loop_node/detector/detector_node.py,sha256=Qmj87e5-mmS5SnT_VlfZ1I6vhS6XWzt60H9w82LiZbk,16649
+learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=TD346I9ymtIP0_CJXCIKMRuiXbfVVanXNu_iHAwDd7Q,3318
+learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=s2FuwZ-tD_5obkSutstjc8pE_hLGbrv9WjrEO9t8rJ8,1011
+learning_loop_node/detector/inbox_filter/tests/test_observation.py,sha256=ORN08yjprqmgmtU25RsVysniyrWX-qGvqFN8ZkkYxow,1385
+learning_loop_node/detector/inbox_filter/tests/test_relevance_group.py,sha256=RUgsk1CnKSOCRZBzNjE7AZTqk06-yelgUqvHFRLH7_I,7865
+learning_loop_node/detector/inbox_filter/tests/test_unexpected_observations_count.py,sha256=y_dFUV21h6uZc90Q43s0u4oivJfuCNWlk5iXAWiXGgc,1804
+learning_loop_node/detector/outbox.py,sha256=kxOzIhffTbrBCvZqGQsrwDS68zWJe00mN12O_qNwwVI,4176
+learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/detector/rest/about.py,sha256=-PNqlQI_tzRvoSI_UR9rX8-5GeiENNpRDQ4Ylw3wYVs,607
+learning_loop_node/detector/rest/backdoor_controls.py,sha256=38axRG66Z3_Q6bYKa7Hw-ldChEAu-dJcBM_Sl_17Ozo,1725
+learning_loop_node/detector/rest/detect.py,sha256=8Rl1swANKgHc42P1z75t_PErQxpCKKPdAsKqDIZgdNU,1873
+learning_loop_node/detector/rest/operation_mode.py,sha256=eIo6_56qyZECftf4AEN8wJMABIojC0TRazvWeg0Uj_s,1664
+learning_loop_node/detector/rest/upload.py,sha256=MifNhban7GeaCjwa39lDhTQWyRuVyvyGFGscoszplH0,435
+learning_loop_node/detector/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/detector/tests/conftest.py,sha256=v08N5_jvQyeqQkOzkTaanCwwVd75Rf-tc1M2-fgiv54,3254
+learning_loop_node/detector/tests/test.jpg,sha256=msA-vHPmvPiro_D102Qmn1fn4vNfooqYYEXPxZUmYpk,161390
+learning_loop_node/detector/tests/test_client_communication.py,sha256=QjoES3qMqcsfZgNHI825_bfvjdxxX8NNncCRKIeV9Wo,4603
+learning_loop_node/detector/tests/test_outbox.py,sha256=bXvbxBVSF2O0M3V9gSDEqjMQ1g12_hwckq7rziG3i1c,2051
+learning_loop_node/detector/tests/test_relevance_filter.py,sha256=FzeOU6k17VIQvAHR8fjHbcPeAE7D7C-2Yxol0lDrMEM,1981
+learning_loop_node/detector/tests/testing_detector.py,sha256=2DSwIYJDOG4ixOGU8OxjsZQgaOdVU7_d3ASKsSkf8qc,564
+learning_loop_node/examples/novelty_score_updater.py,sha256=1DRgM9lxjFV-q2JvGDDsNLz_ic_rhEZ9wc6ZdjcxwPE,2038
+learning_loop_node/globals.py,sha256=tgw_8RYOipPV9aYlyUhYtXfUxvJKRvfUk6u-qVAtZmY,174
+learning_loop_node/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/helpers/environment_reader.py,sha256=1GNTXRBcv6QKolbJ50TpsbWoVyw1oO0kvKrt9yjJ0nA,1543
+learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa6hU1d71anqEBb564Rk,1145
+learning_loop_node/helpers/log_conf.py,sha256=3yd-jaMOeD5cRIgA5w_BH2L5odf8c4-ZjD89Bdqwe44,824
+learning_loop_node/helpers/misc.py,sha256=j4is8Rv0ttnCqF-R-wP3xwEi67OI6IBJav5Woo5lyDk,7701
+learning_loop_node/loop_communication.py,sha256=w_CF1Ynl6GdpL1cdZ6wbb-X0-5gmGlPLjSfDQDRpwLE,6157
+learning_loop_node/node.py,sha256=jpTP3gnGxFDF7tq9H7MVfVIIHm8e7bYA5euttHNJAEg,7252
+learning_loop_node/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
+learning_loop_node/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/tests/conftest.py,sha256=vFj8jjwSBzt6IAIxEo7y0rN0I94SPkK2pZsMvrvET5Q,920
+learning_loop_node/tests/test_data/file_1.txt,sha256=Lis06nfvbFPVCBZyEgQlfI_Nle2YDq1GQBlYvEfFtxw,19
+learning_loop_node/tests/test_data/file_2.txt,sha256=Xp8EETGhZBdVAgb4URowSSpOytwwwJdV0Renkdur7R8,19
+learning_loop_node/tests/test_data/model.json,sha256=_xNDucGOWila8gWnu8yFfrqmQ45Xq-_39eLKzjRtvpE,516
+learning_loop_node/tests/test_data_classes.py,sha256=m8LEk1quGErxuPzNdW_ExqQjkwE4u7ribwnTdyeiHR8,788
+learning_loop_node/tests/test_downloader.py,sha256=RkXNVQJkfPddGzxfJgSF7kc-V6Qp1OOvYYuhQ9g_ls4,2806
+learning_loop_node/tests/test_executor.py,sha256=-QLylzigUEoUzeQssPR3ECFkD2tWH-fsA_lEGXy993c,2119
+learning_loop_node/tests/test_helper.py,sha256=AjOrTu3dHIlJLYI0mxcNx8MCmFF6IjLhHtPzuai4O9E,2334
+learning_loop_node/tests/test_learning_loop_node.py,sha256=4qWi1ovBzebUAbvw8ecSa-TBGKYuJvlKe2AMnMZ-Qs8,701
+learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/trainer/downloader.py,sha256=qzx7zzObcFEvRVQFe8gi8KJNIapASi1_XssbspXD1Rw,1469
+learning_loop_node/trainer/executor.py,sha256=j-1pkc_5b1Y0Bh8KiQthAK7SoOfJelr5Qi7L0p1sTxE,3933
+learning_loop_node/trainer/io_helpers.py,sha256=fyjImT1NY10gLV0KM4Ns7sRrtKFswVJU_pV9-c3IVEY,7152
+learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/trainer/rest/backdoor_controls.py,sha256=YQcG0KwxzKDNYeMtHrSwr26q__N7ty0o6Kar6CLWAd0,5869
+learning_loop_node/trainer/rest/controls.py,sha256=XF37i2edeMHKdSXyJc4ZqaTZ38u6d3u3Sb3C-Mwyfko,934
+learning_loop_node/trainer/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/trainer/tests/conftest.py,sha256=UIVmnZZVe1DigOxeraiLh89cyw1QNQFvNJWmPUY1uFI,2455
+learning_loop_node/trainer/tests/state_helper.py,sha256=igoGqTBqcqqFcDng2i7ctC67bYR1hLPDl4G_mNRG6r8,934
+learning_loop_node/trainer/tests/states/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+learning_loop_node/trainer/tests/states/test_state_cleanup.py,sha256=tiL31hSjg1Bl2obzg2ufAVpil5qW0YkpDCpSHPqXQrk,1312
+learning_loop_node/trainer/tests/states/test_state_detecting.py,sha256=NU065RhpUwhExldGp-79YD_P9bG4O9DNcLjBfW0qmik,3811
+learning_loop_node/trainer/tests/states/test_state_download_train_model.py,sha256=rNRXIyqyHzHz4fXY1Lsf7WKg8FFVFYfFPIevMCBBcCY,2940
+learning_loop_node/trainer/tests/states/test_state_prepare.py,sha256=fx9_bgPTaR5ANVB8n_hW8dXcaJIh_iKEnInmhzamZ9E,2432
+learning_loop_node/trainer/tests/states/test_state_sync_confusion_matrix.py,sha256=zfNbHB3GFSJXXoEkW-8PYtmX62md3feWp4oisyzs8A4,4773
+learning_loop_node/trainer/tests/states/test_state_train.py,sha256=j1vedjH2EwLTgHhon6eR9ttp-Sw9ozR9-9QgAKlFO-M,3248
+learning_loop_node/trainer/tests/states/test_state_upload_detections.py,sha256=38MumBw_N6Pt5cXI8x-xZjczmWUc7BIqyw4TSfim9h4,7751
+learning_loop_node/trainer/tests/states/test_state_upload_model.py,sha256=gAXJJcoxj2EVdFJcPAeqxGAjeL1a4ofLtytnM4joi-Q,3808
+learning_loop_node/trainer/tests/test_errors.py,sha256=8H-kjs9kEBoHWcQVJIZvW5zcwCs1VQI5Tf5I0VSbCUA,2245
+learning_loop_node/trainer/tests/test_trainer_states.py,sha256=OBrClH6srAM2hqqel2xTtfHCeTKYZlG_S4KO2G2GrS4,1147
+learning_loop_node/trainer/tests/testing_trainer_logic.py,sha256=7sQ6okiOhM4IhvRRo4XvLPjxnBrqFu9SPbuDX2LLwRs,3925
+learning_loop_node/trainer/trainer_logic.py,sha256=PJxiO1chPdvpq8UTtzv_nVam9CouCswX9b1FnRwT2Tw,8411
+learning_loop_node/trainer/trainer_logic_generic.py,sha256=KFDuxgzrGITHQaJoGvhjHxWzhbb4Q7HBxSpks4CeGBg,24801
+learning_loop_node/trainer/trainer_node.py,sha256=bcyOMeLXrLuLgsPqS8lwEOSZ6vCjGLgT0pLXgaylI1Q,4155
+learning_loop_node-0.10.1.dist-info/METADATA,sha256=j9ZrDeZGnT8IuYUAmt53KSvVtjEhsn3kxBAlPkbX2cE,9158
+learning_loop_node-0.10.1.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
+learning_loop_node-0.10.1.dist-info/RECORD,,

learning_loop_node/converter/converter_logic.py

@@ -1,68 +0,0 @@
-import json
-import os
-import shutil
-from abc import abstractmethod
-from typing import List, Optional
-
-from ..data_classes import ModelInformation
-from ..node import Node
-
-
-class ConverterLogic():
-
-    def __init__(
-            self, source_format: str, target_format: str):
-        self.source_format = source_format
-        self.target_format = target_format
-        self._node: Optional[Node] = None
-        self.model_folder: Optional[str] = None
-
-    def init(self, node: Node) -> None:
-        self._node = node
-
-    @property
-    def node(self) -> Node:
-        if self._node is None:
-            raise Exception('ConverterLogic not initialized')
-        return self._node
-
-    async def convert(self, model_information: ModelInformation) -> None:
-        project_folder = Node.create_project_folder(model_information.context)
-
-        self.model_folder = ConverterLogic.create_model_folder(project_folder, model_information.id)
-        await self.node.data_exchanger.download_model(self.model_folder,
-                                                      model_information.context,
-                                                      model_information.id,
-                                                      self.source_format)
-
-        with open(f'{self.model_folder}/model.json', 'r') as f:
-            content = json.load(f)
-            if 'resolution' in content:
-                model_information.resolution = content['resolution']
-
-        await self._convert(model_information)
-
-    async def upload_model(self, context, model_id: str) -> None:
-        files = self.get_converted_files(model_id)
-        await self.node.data_exchanger.upload_model(context, files, model_id, self.target_format)
-
-    @abstractmethod
-    async def _convert(self, model_information: ModelInformation) -> None:
-        """Converts the model in self.model_folder to the target format."""
-
-    @abstractmethod
-    def get_converted_files(self, model_id) -> List[str]:
-        """Returns a list of files that should be uploaded to the server."""
-
-    @staticmethod
-    def create_convert_folder(project_folder: str) -> str:
-        image_folder = f'{project_folder}/images'
-        os.makedirs(image_folder, exist_ok=True)
-        return image_folder
-
-    @staticmethod
-    def create_model_folder(project_folder: str, model_id: str) -> str:
-        model_folder = f'{project_folder}/{model_id}'
-        shutil.rmtree(model_folder, ignore_errors=True)  # cleanup
-        os.makedirs(model_folder, exist_ok=True)
-        return model_folder

learning_loop_node/converter/converter_node.py

@@ -1,125 +0,0 @@
-import logging
-from dataclasses import asdict
-from http import HTTPStatus
-from typing import List, Optional
-
-from dacite import from_dict
-from fastapi.encoders import jsonable_encoder
-from fastapi_utils.tasks import repeat_every
-from socketio import AsyncClient
-
-from ..data_classes import Category, ModelInformation, NodeState
-from ..node import Node
-from .converter_logic import ConverterLogic
-
-
-class ConverterNode(Node):
-    converter: ConverterLogic
-    skip_check_state: bool = False
-    bad_model_ids: List[str] = []
-
-    def __init__(self, name: str, converter: ConverterLogic, uuid: Optional[str] = None):
-        super().__init__(name, uuid)
-        self.converter = converter
-        converter.init(self)
-
-        @self.on_event("startup")
-        @repeat_every(seconds=60, raise_exceptions=True, wait_first=False)
-        async def check_state():
-            if not self.skip_check_state:
-                try:
-                    await self.check_state()
-                except Exception:
-                    logging.error('could not check state. Is loop reachable?')
-
-    async def convert_model(self, model_information: ModelInformation):
-        if model_information.id in self.bad_model_ids:
-            logging.info(
-                f'skipping bad model model {model_information.id} for {model_information.context.organization}/{model_information.context.project}.')
-            return
-        try:
-            logging.info(
-                f'converting model {jsonable_encoder(asdict(model_information))}')
-            await self.converter.convert(model_information)
-            logging.info('uploading model ')
-            await self.converter.upload_model(model_information.context, model_information.id)
-        except Exception as e:
-            self.bad_model_ids.append(model_information.id)
-            logging.error(
-                f'could not convert model {model_information.id} for {model_information.context.organization}/{model_information.context.project}. Details: {str(e)}.')
-
-    async def check_state(self):
-        logging.info(f'checking state: {self.status.state}')
-
-        if self.status.state == NodeState.Running:
-            return
-        self.status.state = NodeState.Running
-        try:
-            await self.convert_models()
-        except Exception as exc:
-            logging.error(str(exc))
-
-        self.status.state = NodeState.Idle
-
-    async def convert_models(self) -> None:
-        try:
-            response = await self.loop_communicator.get('/projects')
-            assert response.status_code == 200, f'Assert statuscode 200, but was {response.status_code}.'
-            content = response.json()
-            projects = content['projects']
-
-            for project in projects:
-                organization_id = project['organization_id']
-                project_id = project['project_id']
-
-                response = await self.loop_communicator.get(f'{project["resource"]}')
-                if response.status_code != HTTPStatus.OK:
-                    logging.error(f'got bad response for {response.url}: {str(response.status_code)}')
-                    continue
-
-                project_categories = [from_dict(data_class=Category, data=c) for c in response.json()['categories']]
-
-                path = f'{project["resource"]}/models'
-                models_response = await self.loop_communicator.get(path)
-                assert models_response.status_code == 200
-                content = models_response.json()
-                models = content['models']
-
-                for model in models:
-                    if (model['version']
-                                and self.converter.source_format in model['formats']
-                                and self.converter.target_format not in model['formats']
-                            ):
-                        # if self.converter.source_format in model['formats'] and project_id == 'drawingbot' and model['version'] == "6.0":
-                        model_information = ModelInformation(
-                            host=self.loop_communicator.base_url,
-                            organization=organization_id,
-                            project=project_id,
-                            id=model['id'],
-                            categories=project_categories,
-                            version=model['version'],
-                        )
-                        await self.convert_model(model_information)
-        except Exception:
-            logging.exception('could not convert models')
-
-    async def send_status(self):
-        pass
-
-    async def on_startup(self):
-        pass
-
-    async def on_shutdown(self):
-        pass
-
-    async def on_repeat(self):
-        pass
-
-    def register_sio_events(self, sio_client: AsyncClient):
-        pass
-
-    async def get_state(self):
-        return NodeState.Idle  # NOTE unused for this node type
-
-    def get_node_type(self):
-        return 'converter'

learning_loop_node/converter/tests/test_converter.py

@@ -1,55 +0,0 @@
-import logging
-from typing import List
-
-import pytest
-
-from learning_loop_node.converter.converter_logic import ConverterLogic
-from learning_loop_node.converter.converter_node import ConverterNode
-from learning_loop_node.data_classes import ModelInformation
-from learning_loop_node.loop_communication import LoopCommunicator
-from learning_loop_node.tests import test_helper
-
-
-class TestConverter(ConverterLogic):
-    __test__ = False  # hint for pytest
-
-    def __init__(self, source_format: str, target_format: str,  models: List[ModelInformation]):
-        super().__init__(source_format, target_format)
-        self.models = models
-
-    async def _convert(self, model_information: ModelInformation) -> None:
-        self.models.append(model_information)
-
-    def get_converted_files(self, model_id) -> List[str]:
-        return []  # test: test_meta_information fails because model cannot be uploaded
-
-
-@pytest.mark.asyncio
-@pytest.fixture()
-async def setup_converter_test_project(glc: LoopCommunicator):
-    await glc.delete("/zauberzeug/projects/pytest_conv?keep_images=true")
-    project_configuration = {
-        'project_name': 'pytest_conv', 'box_categories': 1, 'point_categories': 1, 'inbox': 0, 'annotate': 0, 'review': 0,
-        'complete': 0, 'image_style': 'plain', 'thumbs': False, 'trainings': 1}
-    r = await glc.post("/zauberzeug/projects/generator", json=project_configuration)
-    assert r.status_code == 200
-    yield
-    await glc.delete("/zauberzeug/projects/pytest?keep_images=true")
-
-
-# pylint: disable=redefined-outer-name, unused-argument
-@pytest.mark.asyncio
-async def test_meta_information(setup_converter_test_project):
-    model_id = await test_helper.get_latest_model_id(project='pytest_conv')
-
-    converter = TestConverter(source_format='mocked', target_format='test', models=[])
-    node = ConverterNode(name='test', converter=converter)
-    await node.convert_models()
-
-    pytest_project_model = [m for m in converter.models if m.id == model_id][0]
-
-    categories = pytest_project_model.categories
-    assert len(categories) == 2
-    category_types = [category.type for category in categories]
-    assert 'box' in category_types
-    assert 'point' in category_types

learning_loop_node/trainer/training_syncronizer.py

@@ -1,52 +0,0 @@
-
-import asyncio
-import logging
-from dataclasses import asdict
-from typing import TYPE_CHECKING, Any
-
-import socketio
-from dacite import from_dict
-from fastapi.encoders import jsonable_encoder
-
-from ..data_classes import TrainingOut
-from ..data_classes.socket_response import SocketResponse
-
-if TYPE_CHECKING:
-    from .trainer_logic import TrainerLogic
-
-
-async def try_sync_model(trainer: 'TrainerLogic', trainer_node_uuid: str, sio_client: socketio.AsyncClient):
-    try:
-        model = trainer.get_new_model()
-    except Exception as exc:
-        logging.exception('error while getting new model')
-        raise Exception(f'Could not get new model: {str(exc)}') from exc
-    logging.debug(f'new model {model}')
-
-    if model:
-        response = await sync_model(trainer, trainer_node_uuid, sio_client, model)
-
-        if not response.success:
-            error_msg = f'Error for update_training: Response from loop was : {asdict(response)}'
-            logging.error(error_msg)
-            raise Exception(error_msg)
-
-
-async def sync_model(trainer, trainer_node_uuid, sio_client, model):
-    current_training = trainer.training
-    new_training = TrainingOut(
-        trainer_id=trainer_node_uuid,
-        confusion_matrix=model.confusion_matrix,
-        train_image_count=current_training.data.train_image_count(),
-        test_image_count=current_training.data.test_image_count(),
-        hyperparameters=trainer.hyperparameters)
-
-    await asyncio.sleep(0.1)  # NOTE needed for tests.
-
-    result = await sio_client.call('update_training', (current_training.context.organization, current_training.context.project, jsonable_encoder(new_training)))
-    response = from_dict(data_class=SocketResponse, data=result)
-
-    if response.success:
-        logging.info(f'successfully updated training {asdict(new_training)}')
-        trainer.on_model_published(model)
-    return response