learning-loop-node 0.9.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

learning_loop_node/__init__.py

@@ -1,12 +1,11 @@
 import logging
-import os
-import sys
 
-from .converter.converter_node import ConverterNode
 # from . import log_conf
 from .detector.detector_logic import DetectorLogic
 from .detector.detector_node import DetectorNode
 from .globals import GLOBALS
 from .trainer.trainer_node import TrainerNode
 
+__all__ = ['TrainerNode', 'DetectorNode', 'DetectorLogic', 'GLOBALS']
+
 logging.info('>>>>>>>>>>>>>>>>>> LOOP INITIALIZED <<<<<<<<<<<<<<<<<<<<<<<')

learning_loop_node/annotation/annotator_logic.py

@@ -7,10 +7,10 @@ from ..node import Node
 
 class AnnotatorLogic():
 
-    def __init__(self):
+    def __init__(self) -> None:
         self._node: Optional[Node] = None
 
-    def init(self, node: Node):
+    def init(self, node: Node) -> None:
         self._node = node
 
     @abstractmethod

learning_loop_node/annotation/annotator_node.py

@@ -8,7 +8,7 @@ from socketio import AsyncClient
 from ..data_classes import AnnotationNodeStatus, Context, NodeState, UserInput
 from ..data_classes.socket_response import SocketResponse
 from ..data_exchanger import DataExchanger
-from ..helpers.misc import create_image_folder
+from ..helpers.misc import create_image_folder, create_project_folder
 from ..node import Node
 from .annotator_logic import AnnotatorLogic
 
@@ -18,10 +18,11 @@ from .annotator_logic import AnnotatorLogic
 class AnnotatorNode(Node):
 
     def __init__(self, name: str, annotator_logic: AnnotatorLogic, uuid: Optional[str] = None):
-        super().__init__(name, uuid)
+        super().__init__(name, uuid, 'annotation_node')
         self.tool = annotator_logic
         self.histories: Dict = {}
         annotator_logic.init(self)
+        self.status_sent = False
 
     def register_sio_events(self, sio_client: AsyncClient):
 
@@ -50,8 +51,6 @@ class AnnotatorNode(Node):
             raise
 
         if tool_result.annotation:
-            if not self.sio_is_initialized():
-                raise Exception('Socket client waas not initialized')
             await self.sio_client.call('update_segmentation_annotation', (user_input.data.context.organization,
                                                                           user_input.data.context.project,
                                                                           jsonable_encoder(asdict(tool_result.annotation))), timeout=30)
@@ -67,6 +66,9 @@ class AnnotatorNode(Node):
         return self.histories.setdefault(frontend_id, self.tool.create_empty_history())
 
     async def send_status(self):
+        if self.status_sent:
+            return
+
         status = AnnotationNodeStatus(
             id=self.uuid,
             name=self.name,
@@ -75,28 +77,27 @@ class AnnotatorNode(Node):
         )
 
         self.log.info(f'Sending status {status}')
-        if self._sio_client is None:
-            raise Exception('No socket client')
-        result = await self._sio_client.call('update_annotation_node', jsonable_encoder(asdict(status)), timeout=10)
+        try:
+            result = await self.sio_client.call('update_annotation_node', jsonable_encoder(asdict(status)), timeout=10)
+        except Exception as e:
+            self.log.error(f'Error for updating: {str(e)}')
+            return
+
         assert isinstance(result, Dict)
         response = from_dict(data_class=SocketResponse, data=result)
 
         if not response.success:
             self.log.error(f'Error for updating: Response from loop was : {asdict(response)}')
+        else:
+            self.status_sent = True
 
     async def download_image(self, context: Context, uuid: str):
-        project_folder = Node.create_project_folder(context)
+        project_folder = create_project_folder(context)
         images_folder = create_image_folder(project_folder)
 
         downloader = DataExchanger(context=context, loop_communicator=self.loop_communicator)
         await downloader.download_images([uuid], images_folder)
 
-    async def get_state(self):
-        return NodeState.Online
-
-    def get_node_type(self):
-        return 'annotation_node'
-
     async def on_startup(self):
         pass
 
@@ -104,4 +105,4 @@ class AnnotatorNode(Node):
         pass
 
     async def on_repeat(self):
-        pass
+        await self.send_status()

learning_loop_node/data_classes/__init__.py

@@ -1,12 +1,19 @@
-from .annotations import (AnnotationData, AnnotationEventType,
-                          SegmentationAnnotation, ToolOutput, UserInput)
-from .detections import (BoxDetection, ClassificationDetection, Detections,
-                         Observation, Point, PointDetection,
+from .annotations import AnnotationData, AnnotationEventType, SegmentationAnnotation, ToolOutput, UserInput
+from .detections import (BoxDetection, ClassificationDetection, Detections, Observation, Point, PointDetection,
                          SegmentationDetection, Shape)
-from .general import (AnnotationNodeStatus, Category, CategoryType, Context,
-                      DetectionStatus, ErrorConfiguration, ModelInformation,
-                      NodeState, NodeStatus)
+from .general import (AnnotationNodeStatus, Category, CategoryType, Context, DetectionStatus, ErrorConfiguration,
+                      ModelInformation, NodeState, NodeStatus)
 from .socket_response import SocketResponse
-from .training import (BasicModel, Errors, Hyperparameter, Model,
-                       PretrainedModel, Training, TrainingData, TrainingError,
-                       TrainingOut, TrainingState, TrainingStatus)
+from .training import (Errors, Hyperparameter, Model, PretrainedModel, TrainerState, Training, TrainingData,
+                       TrainingError, TrainingOut, TrainingStateData, TrainingStatus)
+
+__all__ = [
+    'AnnotationData', 'AnnotationEventType', 'SegmentationAnnotation', 'ToolOutput', 'UserInput',
+    'BoxDetection', 'ClassificationDetection', 'Detections', 'Observation', 'Point', 'PointDetection',
+    'SegmentationDetection', 'Shape',
+    'AnnotationNodeStatus', 'Category', 'CategoryType', 'Context', 'DetectionStatus', 'ErrorConfiguration',
+    'ModelInformation', 'NodeState', 'NodeStatus',
+    'SocketResponse',
+    'Errors', 'Hyperparameter', 'Model', 'PretrainedModel', 'TrainerState', 'Training', 'TrainingData',
+    'TrainingError', 'TrainingOut', 'TrainingStateData', 'TrainingStatus',
+]

learning_loop_node/data_classes/detections.py

@@ -13,8 +13,11 @@ KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) e
 
 @dataclass(**KWONLY_SLOTS)
 class BoxDetection():
+    """Coordinates according to COCO format. x,y is the top left corner of the box.
+    x increases to the right, y increases downwards.
+    """
     category_name: str
-    x: int  # TODO add definition of x,y,w,h
+    x: int
     y: int
     width: int
     height: int
@@ -47,6 +50,8 @@ class BoxDetection():
 
 @dataclass(**KWONLY_SLOTS)
 class PointDetection():
+    """Coordinates according to COCO format. x,y is the center of the point.
+    x increases to the right, y increases downwards."""
     category_name: str
     x: float
     y: float
@@ -111,7 +116,7 @@ class Detections():
     point_detections: List[PointDetection] = field(default_factory=list)
     segmentation_detections: List[SegmentationDetection] = field(default_factory=list)
     classification_detections: List[ClassificationDetection] = field(default_factory=list)
-    tags: Optional[List[str]] = field(default_factory=list)
+    tags: List[str] = field(default_factory=list)
     date: Optional[str] = field(default_factory=current_datetime)
     image_id: Optional[str] = None  # used for detection of trainers
 

learning_loop_node/data_classes/general.py

@@ -34,10 +34,6 @@ class Category():
         return [from_dict(data_class=Category, data=value) for value in values]
 
 
-def create_category(identifier: str, name: str, ctype: Union[CategoryType, str]):  # TODO: This is probably unused
-    return Category(id=identifier, name=name, description='', hotkey='', color='', type=ctype, point_size=None)
-
-
 @dataclass(**KWONLY_SLOTS)
 class Context():
     organization: str
@@ -57,6 +53,7 @@ class ModelInformation():
     categories: List[Category]
     resolution: Optional[int] = None
     model_root_path: Optional[str] = None
+    model_size: Optional[str] = None
 
     @property
     def context(self):
@@ -64,6 +61,8 @@ class ModelInformation():
 
     @staticmethod
     def load_from_disk(model_root_path: str) -> Optional['ModelInformation']:
+        """Load model.json from model_root_path and return ModelInformation object.
+        """
         model_info_file_path = f'{model_root_path}/model.json'
         if not os.path.exists(model_info_file_path):
             logging.warning(f"could not find model information file '{model_info_file_path}'")
@@ -90,6 +89,10 @@ class ModelInformation():
             del self_as_dict['model_root_path']
             f.write(json.dumps(self_as_dict))
 
+    @staticmethod
+    def from_dict(data: Dict) -> 'ModelInformation':
+        return from_dict(ModelInformation, data=data)
+
 
 @dataclass(**KWONLY_SLOTS)
 class ErrorConfiguration():
@@ -117,7 +120,7 @@ class NodeState(str, Enum):
 class NodeStatus():
     id: str
     name: str
-    state: Optional[NodeState] = NodeState.Offline
+    state: Optional[NodeState] = NodeState.Online
     uptime: Optional[int] = 0
     errors: Dict = field(default_factory=dict)
     capabilities: List[str] = field(default_factory=list)

learning_loop_node/data_classes/training.py

@@ -1,8 +1,10 @@
 
 import sys
+import time
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Dict, List, Optional, Union
+from pathlib import Path
+from typing import Dict, List, Optional
 
 # pylint: disable=no-name-in-module
 from .general import Category, Context
@@ -16,6 +18,14 @@ class Hyperparameter():
     flip_rl: bool
     flip_ud: bool
 
+    @staticmethod
+    def from_data(data: Dict):
+        return Hyperparameter(
+            resolution=data['resolution'],
+            flip_rl=data.get('flip_rl', False),
+            flip_ud=data.get('flip_ud', False)
+        )
+
 
 @dataclass(**KWONLY_SLOTS)
 class TrainingData():
@@ -41,14 +51,15 @@ class PretrainedModel():
     description: str
 
 
-class TrainingState(str, Enum):
+class TrainerState(str, Enum):
+    Idle = 'idle'
     Initialized = 'initialized'
     Preparing = 'preparing'
     DataDownloading = 'data_downloading'
     DataDownloaded = 'data_downloaded'
     TrainModelDownloading = 'train_model_downloading'
     TrainModelDownloaded = 'train_model_downloaded'
-    TrainingRunning = 'training_running'
+    TrainingRunning = 'running'
     TrainingFinished = 'training_finished'
     ConfusionMatrixSyncing = 'confusion_matrix_syncing'
     ConfusionMatrixSynced = 'confusion_matrix_synced'
@@ -62,9 +73,9 @@ class TrainingState(str, Enum):
 
 @dataclass(**KWONLY_SLOTS)
 class TrainingStatus():
-    id: str  # TODO this must not be changed, but tests wont detect it -> update tests!
+    id: str  # NOTE this must not be changed, but tests wont detect a change -> update tests!
     name: str
-    state: Union[Optional[TrainingState], str]
+    state: Optional[str]
     errors: Optional[Dict]
     uptime: Optional[float]
     progress: Optional[float]
@@ -77,13 +88,13 @@ class TrainingStatus():
     architecture: Optional[str] = None
     context: Optional[Context] = None
 
-    def short_str(self):
+    def short_str(self) -> str:
         prgr = f'{self.progress * 100:.0f}%' if self.progress else ''
         trtesk = f'{self.train_image_count}/{self.test_image_count}/{self.skipped_image_count}' if self.train_image_count else 'n.a.'
         cntxt = f'{self.context.organization}/{self.context.project}' if self.context else ''
         hyps = f'({self.hyperparameters})' if self.hyperparameters else ''
         arch = f'.{self.architecture} - ' if self.architecture else ''
-        return f'[{str(self.state)} {prgr}. {self.name}({self.id}). Tr/Ts/Tsk: {trtesk} {cntxt}{arch}{hyps}]'
+        return f'[{str(self.state).rsplit(".", maxsplit=1)[-1]} {prgr}. {self.name}({self.id}). Tr/Ts/Tsk: {trtesk} {cntxt}{arch}{hyps}]'
 
 
 @dataclass(**KWONLY_SLOTS)
@@ -91,21 +102,35 @@ class Training():
     id: str
     context: Context
 
-    project_folder: str
-    images_folder: str
-    training_folder: str
+    project_folder: str  # f'{GLOBALS.data_folder}/{context.organization}/{context.project}'
+    images_folder: str  # f'{project_folder}/images'
+    training_folder: str  # f'{project_folder}/trainings/{trainings_id}'
+    start_time: float = field(default_factory=time.time)
+
+    # model uuid to download (to continue training) | is not a uuid when training from scratch (blank or pt-name from provided_pretrained_models->name)
+    base_model_uuid_or_name: Optional[str] = None
 
-    base_model_id: Optional[str] = None
     data: Optional[TrainingData] = None
     training_number: Optional[int] = None
-    training_state: Optional[Union[TrainingState, str]] = None
-    model_id_for_detecting: Optional[str] = None
+    training_state: Optional[str] = None
+    model_uuid_for_detecting: Optional[str] = None
     hyperparameters: Optional[Dict] = None
 
+    @property
+    def training_folder_path(self) -> Path:
+        return Path(self.training_folder)
+
+    def set_values_from_data(self, data: Dict) -> None:
+        self.data = TrainingData(categories=Category.from_list(data['categories']))
+        self.data.hyperparameter = Hyperparameter.from_data(data=data)
+        self.training_number = data['training_number']
+        self.base_model_uuid_or_name = data['id']
+        self.training_state = TrainerState.Initialized
+
 
 @dataclass(**KWONLY_SLOTS)
 class TrainingOut():
-    confusion_matrix: Optional[Dict] = None
+    confusion_matrix: Optional[Dict] = None  # This is actually just class-wise metrics
     train_image_count: Optional[int] = None
     test_image_count: Optional[int] = None
     trainer_id: Optional[str] = None
@@ -113,9 +138,9 @@ class TrainingOut():
 
 
 @dataclass(**KWONLY_SLOTS)
-class BasicModel():
-    confusion_matrix: Optional[Dict] = None
-    meta_information: Optional[Dict] = None
+class TrainingStateData():
+    confusion_matrix: Dict = field(default_factory=dict)
+    meta_information: Dict = field(default_factory=dict)
 
 
 @dataclass(**KWONLY_SLOTS)
@@ -130,8 +155,8 @@ class Model():
 
 
 class Errors():
-    def __init__(self):
-        self._errors: Dict = {}
+    def __init__(self) -> None:
+        self._errors: Dict[str, str] = {}
 
     def set(self, key: str, value: str):
         self._errors[key] = value
@@ -140,7 +165,7 @@ class Errors():
     def errors(self) -> Dict:
         return self._errors
 
-    def reset(self, key: str):
+    def reset(self, key: str) -> None:
         try:
             del self._errors[key]
         except AttributeError:
@@ -148,7 +173,7 @@ class Errors():
         except KeyError:
             pass
 
-    def reset_all(self):
+    def reset_all(self) -> None:
         self._errors = {}
 
     def has_error_for(self, key: str) -> bool:
@@ -162,3 +187,6 @@ class TrainingError(Exception):
     def __init__(self, cause: str, *args: object) -> None:
         super().__init__(*args)
         self.cause = cause
+
+    def __str__(self) -> str:
+        return f'TrainingError: {self.cause}'