learning-loop-node 0.18.3__py3-none-any.whl → 0.18.4__py3-none-any.whl

learning_loop_node/annotation/annotator_logic.py

@@ -0,0 +1,29 @@
+from abc import abstractmethod
+from typing import Dict, Optional
+
+from ..data_classes import ToolOutput, UserInput
+from ..node import Node
+
+
+class AnnotatorLogic():
+
+    def __init__(self) -> None:
+        self._node: Optional[Node] = None
+
+    def init(self, node: Node) -> None:
+        self._node = node
+
+    @abstractmethod
+    async def handle_user_input(self, user_input: UserInput, history: Dict) -> ToolOutput:
+        """ This method is called when a user input is received from the client.
+            The function should return a ToolOutput object."""
+
+    @abstractmethod
+    def create_empty_history(self) -> Dict:
+        """ This method is called when a new annotation session is started.
+            The function should return an empty history object."""
+
+    @abstractmethod
+    def logout_user(self, sid: str) -> bool:
+        """ This method is called when a user disconnects from the server.
+            The function should return True if the user was logged out successfully."""

learning_loop_node/annotation/annotator_node.py

@@ -0,0 +1,116 @@
+from dataclasses import asdict
+from typing import Dict, Optional
+
+from dacite import from_dict
+from fastapi.encoders import jsonable_encoder
+from socketio import AsyncClient
+
+from ..data_classes import AnnotationNodeStatus, Context, NodeState, UserInput
+from ..data_classes.socket_response import SocketResponse
+from ..data_exchanger import DataExchanger
+from ..helpers.misc import create_image_folder, create_project_folder
+from ..node import Node
+from .annotator_logic import AnnotatorLogic
+
+# TODO: The use case 'segmentation' is hardcoded here. This should be more flexible.
+
+
+class AnnotatorNode(Node):
+
+    def __init__(self, name: str, annotator_logic: AnnotatorLogic, uuid: Optional[str] = None):
+        super().__init__(name, uuid=uuid, node_type='annotation_node')
+        self.tool = annotator_logic
+        self.histories: Dict = {}
+        annotator_logic.init(self)
+
+    def register_sio_events(self, sio_client: AsyncClient):
+
+        @sio_client.event
+        async def handle_user_input(user_input_dict):
+            return await self._handle_user_input(user_input_dict)
+
+        @sio_client.event
+        async def user_logout(sid):
+            self.reset_history(sid)
+            return self.tool.logout_user(sid)
+
+    async def _handle_user_input(self, user_input_dict: Dict) -> str:
+        if not self.sio_client or not self.sio_client.connected:
+            raise ConnectionError('SocketIO client is not connected')
+
+        user_input = from_dict(data_class=UserInput, data=user_input_dict)
+
+        if user_input.data.key_up == 'Escape':
+            self.reset_history(user_input.frontend_id)
+            return ''
+
+        await self.download_image(user_input.data.context, user_input.data.image_uuid)
+
+        try:
+            tool_result = await self.tool.handle_user_input(user_input, self.get_history(user_input.frontend_id))
+        except Exception:
+            self.reset_history(user_input.frontend_id)
+            raise
+
+        if tool_result.annotation:
+            await self.sio_client.call('update_segmentation_annotation', (user_input.data.context.organization,
+                                                                          user_input.data.context.project,
+                                                                          jsonable_encoder(asdict(tool_result.annotation))), timeout=30)
+        return tool_result.svg
+
+    def reset_history(self, frontend_id: str) -> None:
+        """Reset the history for a given frontend_id."""
+        if frontend_id in self.histories:
+            del self.histories[frontend_id]
+
+    def get_history(self, frontend_id: str) -> Dict:
+        """Get the history for a given frontend_id. If no history exists, create a new one."""
+        return self.histories.setdefault(frontend_id, self.tool.create_empty_history())
+
+    async def send_status(self):
+
+        if not self.sio_client or not self.sio_client.connected:
+            raise ConnectionError('SocketIO client is not connected')
+
+        status = AnnotationNodeStatus(
+            id=self.uuid,
+            name=self.name,
+            state=NodeState.Online,
+            capabilities=['segmentation']
+        )
+
+        self.log_status_on_change(status.state.value if status.state else 'None', status)
+
+        try:
+            result = await self.sio_client.call('update_annotation_node', jsonable_encoder(asdict(status)), timeout=10)
+        except Exception:
+            self.socket_connection_broken = True
+            self.log.exception('Error for updating:')
+            return
+
+        if not isinstance(result, Dict):
+            self.socket_connection_broken = True
+            self.log.error('Expected Dict, got %s', type(result))
+            return
+
+        response = from_dict(data_class=SocketResponse, data=result)
+
+        if not response.success:
+            self.socket_connection_broken = True
+            self.log.error('Response from loop was: %s', asdict(response))
+
+    async def download_image(self, context: Context, uuid: str):
+        project_folder = create_project_folder(context)
+        images_folder = create_image_folder(project_folder)
+
+        downloader = DataExchanger(context=context, loop_communicator=self.loop_communicator)
+        await downloader.download_images([uuid], images_folder)
+
+    async def on_startup(self):
+        pass
+
+    async def on_shutdown(self):
+        pass
+
+    async def on_repeat(self):
+        await self.send_status()

learning_loop_node/data_classes/__init__.py

@@ -0,0 +1,50 @@
+from .annotation_data import (
+    AnnotationData,
+    SegmentationAnnotation,
+    ToolOutput,
+    UserInput,
+)
+from .detections import (
+    BoxDetection,
+    ClassificationDetection,
+    Detections,
+    Observation,
+    Point,
+    PointDetection,
+    SegmentationDetection,
+    Shape,
+)
+from .general import (
+    AboutResponse,
+    AnnotationNodeStatus,
+    Category,
+    Context,
+    DetectorStatus,
+    ErrorConfiguration,
+    ModelInformation,
+    ModelVersionResponse,
+    NodeState,
+    NodeStatus,
+)
+from .image_metadata import ImageMetadata, ImagesMetadata
+from .socket_response import SocketResponse
+from .training import (
+    Errors,
+    PretrainedModel,
+    Training,
+    TrainingError,
+    TrainingOut,
+    TrainingStateData,
+    TrainingStatus,
+)
+
+__all__ = [
+    'AboutResponse', 'AnnotationData', 'SegmentationAnnotation', 'ToolOutput', 'UserInput',
+    'BoxDetection', 'ClassificationDetection', 'ImageMetadata', 'Observation', 'Point', 'PointDetection',
+    'SegmentationDetection', 'Shape', 'Detections',
+    'AnnotationNodeStatus', 'Category', 'Context', 'DetectorStatus', 'ErrorConfiguration',
+    'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse', 'ImagesMetadata',
+    'SocketResponse',
+    'Errors', 'PretrainedModel', 'Training',
+    'TrainingError', 'TrainingOut', 'TrainingStateData', 'TrainingStatus',
+]

learning_loop_node/data_classes/annotation_data.py

@@ -0,0 +1,43 @@
+import sys
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from ..enums import AnnotationEventType
+from .detections import Point, Shape
+from .general import Category, Context
+
+KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
+
+
+@dataclass(**KWONLY_SLOTS)
+class AnnotationData():
+    coordinate: Point
+    event_type: Union[AnnotationEventType, str]
+    context: Context
+    image_uuid: str
+    category: Category
+
+    key_up: Optional[str] = None
+    key_down: Optional[str] = None
+    epsilon: Optional[float] = None
+    is_shift_key_pressed: Optional[bool] = None
+
+
+@dataclass(**KWONLY_SLOTS)
+class SegmentationAnnotation():
+    id: str
+    shape: Shape
+    image_id: str
+    category_id: str
+
+
+@dataclass(**KWONLY_SLOTS)
+class UserInput():
+    frontend_id: str
+    data: AnnotationData
+
+
+@dataclass(**KWONLY_SLOTS)
+class ToolOutput():
+    svg: str
+    annotation: Optional[SegmentationAnnotation] = None

learning_loop_node/data_classes/annotations.py

@@ -0,0 +1,44 @@
+
+import sys
+from dataclasses import dataclass, field
+from typing import Optional
+
+KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
+
+
+@dataclass(**KWONLY_SLOTS)
+class BoxAnnotation():
+    """Coordinates according to COCO format. x,y is the top left corner of the box.
+    x increases to the right, y increases downwards.
+    """
+    category_name: str = field(metadata={'description': 'Category name'})
+    x: int = field(metadata={'description': 'X coordinate (left to right)'})
+    y: int = field(metadata={'description': 'Y coordinate (top to bottom)'})
+    width: int = field(metadata={'description': 'Width'})
+    height: int = field(metadata={'description': 'Height'})
+    category_id: Optional[str] = field(default=None, metadata={'description': 'Category UUID'})
+
+    def __str__(self):
+        return f'x:{int(self.x)} y: {int(self.y)}, w: {int(self.width)} h: {int(self.height)} -> {self.category_name}'
+
+
+@dataclass(**KWONLY_SLOTS)
+class PointAnnotation():
+    """Coordinates according to COCO format. x,y is the center of the point.
+    x increases to the right, y increases downwards."""
+    category_name: str = field(metadata={'description': 'Category name'})
+    x: float = field(metadata={'description': 'X coordinate (right)'})
+    y: float = field(metadata={'description': 'Y coordinate (down)'})
+    category_id: Optional[str] = field(default=None, metadata={'description': 'Category UUID'})
+
+    def __str__(self):
+        return f'x:{int(self.x)} y: {int(self.y)}, -> {self.category_name}'
+
+
+@dataclass(**KWONLY_SLOTS)
+class ClassificationAnnotation():
+    category_name: str = field(metadata={'description': 'Category name'})
+    category_id: Optional[str] = field(default=None, metadata={'description': 'Category UUID'})
+
+    def __str__(self):
+        return f'-> {self.category_name}'

learning_loop_node/data_classes/detections.py

@@ -0,0 +1,136 @@
+
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import List, Optional, Union
+
+import numpy as np
+
+KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
+
+
+@dataclass(**KWONLY_SLOTS)
+class BoxDetection():
+    """Coordinates according to COCO format. x,y is the top left corner of the box.
+    x increases to the right, y increases downwards.
+    """
+    category_name: str = field(metadata={'description': 'Category name'})
+    x: int = field(metadata={'description': 'X coordinate (left to right)'})
+    y: int = field(metadata={'description': 'Y coordinate (top to bottom)'})
+    width: int = field(metadata={'description': 'Width'})
+    height: int = field(metadata={'description': 'Height'})
+    model_name: str = field(metadata={'description': 'Model name'})
+    confidence: float = field(metadata={'description': 'Confidence'})
+    category_id: Optional[str] = field(default=None, metadata={'description': 'Category UUID'})
+
+    def intersection_over_union(self, other_detection: 'BoxDetection') -> float:
+        # https://www.pyimagesearch.com/2016/11/07/intersection-over-union-iou-for-object-detection/
+        xA = max(self.x, other_detection.x)
+        yA = max(self.y, other_detection.y)
+        xB = min(self.x + self.width, other_detection.x + other_detection.width)
+        yB = min(self.y + self.height, other_detection.y +
+                 other_detection.height)
+
+        interArea = max(xB - xA, 0) * max(yB - yA, 0)
+        union = float(self._get_area() + other_detection._get_area() - interArea)  # pylint: disable=protected-access
+        if union == 0:
+            print("WARNING: something went wrong while calculating iou")
+            return 0
+
+        return interArea / union
+
+    def _get_area(self) -> int:
+        return self.width * self.height
+
+    def __str__(self):
+        return f'x:{int(self.x)} y: {int(self.y)}, w: {int(self.width)} h: {int(self.height)} c: {self.confidence:.2f} -> {self.category_name}'
+
+
+@dataclass(**KWONLY_SLOTS)
+class PointDetection():
+    """Coordinates according to COCO format. x,y is the center of the point.
+    x increases to the right, y increases downwards."""
+    category_name: str = field(metadata={'description': 'Category name'})
+    x: float = field(metadata={'description': 'X coordinate (right)'})
+    y: float = field(metadata={'description': 'Y coordinate (down)'})
+    model_name: str = field(metadata={'description': 'Model name'})
+    confidence: float = field(metadata={'description': 'Confidence'})
+    category_id: Optional[str] = field(default=None, metadata={'description': 'Category UUID'})
+
+    def distance(self, other: 'PointDetection') -> float:
+        return np.sqrt((other.x - self.x)**2 + (other.y - self.y)**2)
+
+    def __str__(self):
+        return f'x:{int(self.x)} y: {int(self.y)}, c: {self.confidence:.2f} -> {self.category_name}'
+
+
+@dataclass(**KWONLY_SLOTS)
+class ClassificationDetection():
+    category_name: str = field(metadata={'description': 'Category name'})
+    model_name: str = field(metadata={'description': 'Model name'})
+    confidence: float = field(metadata={'description': 'Confidence'})
+    category_id: Optional[str] = field(default=None, metadata={'description': 'Category UUID'})
+
+    def __str__(self):
+        return f'c: {self.confidence:.2f} -> {self.category_name}'
+
+
+@dataclass(**KWONLY_SLOTS)
+class Point():
+    x: int
+    y: int
+
+    def __str__(self):
+        return f'x:{self.x} y: {self.y}'
+
+
+@dataclass(**KWONLY_SLOTS)
+class Shape():
+    points: List[Point]
+
+    def __str__(self):
+        return ', '. join([str(s) for s in self.points])
+
+
+@dataclass(**KWONLY_SLOTS)
+class SegmentationDetection():
+    category_name: str
+    shape: Union[Shape, str]
+    model_name: str
+    confidence: float
+    category_id: Optional[str] = None
+
+    def __str__(self):
+        return f'shape:{str(self.shape)}, c: {self.confidence:.2f} -> {self.category_name}'
+
+
+@dataclass(**KWONLY_SLOTS)
+class Detections():
+    box_detections: List[BoxDetection] = field(default_factory=list, metadata={
+        'description': 'List of box detections'})
+    point_detections: List[PointDetection] = field(default_factory=list, metadata={
+        'description': 'List of point detections'})
+    segmentation_detections: List[SegmentationDetection] = field(default_factory=list, metadata={
+        'description': 'List of segmentation detections'})
+    classification_detections: List[ClassificationDetection] = field(default_factory=list, metadata={
+        'description': 'List of classification detections'})
+
+    image_id: Optional[str] = field(default=None, metadata={
+        'description': 'Image uuid'})
+
+    def __len__(self):
+        return len(self.box_detections) + len(self.point_detections) + len(self.segmentation_detections) + len(self.classification_detections)
+
+
+# TODO make dataclass
+class Observation():
+
+    def __init__(self, detection: Union[BoxDetection, PointDetection, SegmentationDetection, ClassificationDetection]):
+        self.detection = detection
+        self.last_seen = datetime.now()
+
+    def update_last_seen(self):
+        self.last_seen = datetime.now()
+
+    def is_older_than(self, forget_time_in_seconds):
+        return self.last_seen < datetime.now() - timedelta(seconds=forget_time_in_seconds)

learning_loop_node/data_classes/general.py

@@ -0,0 +1,187 @@
+import json
+import logging
+import os
+import sys
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from typing import Dict, List, Optional, Union
+
+from dacite import from_dict
+
+from ..enums import CategoryType
+
+KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
+
+
+@dataclass(**KWONLY_SLOTS)
+class Category():
+    id: str = field(metadata={"description": "The uuid of the category."})
+    name: str = field(metadata={"description": "The name of the category."})
+    description: Optional[str] = field(default=None, metadata={
+        "description": "An optional description of the category."})
+    hotkey: Optional[str] = field(default=None, metadata={
+        "description": "The key shortcut of the category when annotating in the Learning Loop UI."})
+    color: Optional[str] = field(default=None, metadata={
+        "description": "The color of the category when displayed in the Learning Loop UI."})
+    point_size: Optional[int] = field(default=None, metadata={
+        "description": "The point size of the category in pixels. Represents the uncertainty of the category."})
+    type: Optional[Union[CategoryType, str]] = field(default=None, metadata={
+        "description": "The type of the category",
+        "example": "box, point, segmentation, classification"})
+
+    @staticmethod
+    def from_list(values: List[dict]) -> List['Category']:
+        return [from_dict(data_class=Category, data=value) for value in values]
+
+
+@dataclass(**KWONLY_SLOTS)
+class Context():
+    organization: str
+    project: str
+
+
+# pylint: disable=no-name-in-module
+
+
+@dataclass(**KWONLY_SLOTS)
+class ModelInformation():
+    id: str = field(metadata={"description": "The uuid of the model."})
+    host: Optional[str] = field(metadata={"description": "The hostname that started the training.",
+                                          "example": "learning-loop.ai"})
+    organization: str = field(metadata={"description": "The owner organization of the model."})
+    project: str = field(metadata={"description": "The project of the model."})
+    version: str = field(metadata={"description": "The version of the model."})
+    categories: List[Category] = field(default_factory=list, metadata={
+                                       "description": "The categories used in the model."})
+    resolution: Optional[int] = field(default=None, metadata={
+        "description": "The resolution of the model (width and height of the image after preprocessing in pixels)."})
+    model_root_path: Optional[str] = field(
+        default=None, metadata={"description": "The path of the parent directory of the model in the file system."})
+    model_size: Optional[str] = field(default=None, metadata={
+                                      "description": "The size of the model (i.e. the specification or variant of the model architecture)."})
+
+    @property
+    def context(self):
+        return Context(organization=self.organization, project=self.project)
+
+    @staticmethod
+    def load_from_disk(model_root_path: str) -> Optional['ModelInformation']:
+        """Load model.json from model_root_path and return ModelInformation object.
+        """
+        model_info_file_path = f'{model_root_path}/model.json'
+        if not os.path.exists(model_info_file_path):
+            logging.warning('could not find model information file %s', model_info_file_path)
+            return None
+        with open(model_info_file_path, 'r') as f:
+            try:
+                content = json.load(f)
+            except Exception as exc:
+                raise Exception(f"could not read model information from file '{model_info_file_path}'") from exc
+            try:
+                model_information = from_dict(data_class=ModelInformation, data=content)
+                model_information.model_root_path = model_root_path
+            except Exception as exc:
+                raise Exception(
+                    f"could not parse model information from file '{model_info_file_path}'. \n {str(exc)}") from exc
+
+        return model_information
+
+    def save(self):
+        if not self.model_root_path:
+            raise Exception("model_root_path is not set")
+        with open(self.model_root_path + '/model.json', 'w') as f:
+            self_as_dict = asdict(self)
+            del self_as_dict['model_root_path']
+            f.write(json.dumps(self_as_dict))
+
+    @staticmethod
+    def from_dict(data: Dict) -> 'ModelInformation':
+        return from_dict(ModelInformation, data=data)
+
+
+@dataclass(**KWONLY_SLOTS)
+class AboutResponse:
+    operation_mode: str = field(metadata={"description": "The operation mode of the detector node"})
+    state: Optional[str] = field(metadata={
+        "description": "The state of the detector node",
+        "example": "idle, online, detecting"})
+    model_info: Optional[ModelInformation] = field(metadata={
+        "description": "Information about the model of the detector node"})
+    target_model: Optional[str] = field(metadata={"description": "The target model of the detector node"})
+    version_control: str = field(metadata={
+        "description": "The version control mode of the detector node",
+        "example": "follow_loop, specific_version, pause"})
+
+
+@dataclass(**KWONLY_SLOTS)
+class ModelVersionResponse:
+    current_version: str = field(metadata={"description": "The version of the model currently used by the detector."})
+    target_version: str = field(metadata={"description": "The target model version set in the detector."})
+    loop_version: str = field(metadata={"description": "The target model version specified by the loop."})
+    local_versions: List[str] = field(metadata={"description": "The locally available versions of the model."})
+    version_control: str = field(metadata={"description": "The version control mode."})
+
+
+@dataclass(**KWONLY_SLOTS)
+class ErrorConfiguration():
+    begin_training: Optional[bool] = False
+    save_model: Optional[bool] = False
+    get_new_model: Optional[bool] = False
+    crash_training: Optional[bool] = False
+
+
+# pylint: disable=no-name-in-module
+
+
+class NodeState(str, Enum):
+    Idle = "idle"
+    Offline = "offline"
+    Online = "online"
+    Preparing = "preparing"
+    Running = "running"
+    Stopping = "stopping"
+    Detecting = 'detecting'
+    Uploading = 'uploading'
+
+
+@dataclass(**KWONLY_SLOTS)
+class NodeStatus():
+    id: str
+    name: str
+    state: NodeState = NodeState.Online
+    uptime: int = 0
+    errors: Dict = field(default_factory=dict)
+    capabilities: List[str] = field(default_factory=list)
+
+    def set_error(self, key: str, value: str):
+        self.errors[key] = value
+
+    def reset_error(self, key: str):
+        try:
+            del self.errors[key]
+        except AttributeError:
+            pass
+        except KeyError:
+            pass
+
+    def reset_all_errors(self):
+        for key in list(self.errors.keys()):
+            self.reset_error(key)
+
+
+@dataclass(**KWONLY_SLOTS)
+class AnnotationNodeStatus(NodeStatus):
+    capabilities: List[str] = field(default_factory=list)
+
+
+@dataclass(**KWONLY_SLOTS)
+class DetectorStatus():
+    uuid: str
+    name: str
+    state: NodeState
+    uptime: int
+    model_format: str
+    current_model: Optional[str]
+    target_model: Optional[str]
+    errors: Dict
+    operation_mode: str

learning_loop_node/data_classes/image_metadata.py

@@ -0,0 +1,56 @@
+
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Optional
+
+from .annotations import BoxAnnotation, ClassificationAnnotation, PointAnnotation
+from .detections import (
+    BoxDetection,
+    ClassificationDetection,
+    PointDetection,
+    SegmentationDetection,
+)
+
+# pylint: disable=too-many-instance-attributes
+
+KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
+
+
+def current_datetime():
+    return datetime.now().isoformat(sep='_', timespec='milliseconds')
+
+
+@dataclass(**KWONLY_SLOTS)
+class ImageMetadata():
+    box_detections: List[BoxDetection] = field(default_factory=list, metadata={
+        'description': 'List of box detections'})
+    point_detections: List[PointDetection] = field(default_factory=list, metadata={
+        'description': 'List of point detections'})
+    segmentation_detections: List[SegmentationDetection] = field(default_factory=list, metadata={
+        'description': 'List of segmentation detections'})
+    classification_detections: List[ClassificationDetection] = field(default_factory=list, metadata={
+        'description': 'List of classification detections'})
+
+    box_annotations: List[BoxAnnotation] = field(default_factory=list, metadata={
+        'description': 'List of box annotations'})
+    point_annotations: List[PointAnnotation] = field(default_factory=list, metadata={
+        'description': 'List of point annotations'})
+    classification_annotation: Optional[ClassificationAnnotation] = field(default=None, metadata={
+        'description': 'Classification annotation'})
+
+    tags: List[str] = field(default_factory=list, metadata={
+        'description': 'List of tags'})
+
+    created: Optional[str] = field(default_factory=current_datetime, metadata={
+        'description': 'Creation date of the image'})
+    source: Optional[str] = field(default=None, metadata={
+        'description': 'Source of the image'})
+
+    def __len__(self):
+        return len(self.box_detections) + len(self.point_detections) + len(self.segmentation_detections) + len(self.classification_detections)
+
+
+@dataclass(**KWONLY_SLOTS)
+class ImagesMetadata():
+    items: List[ImageMetadata] = field(default_factory=list, metadata={'description': 'List of image metadata'})