learning-loop-node 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of learning-loop-node might be problematic. Click here for more details.
- learning_loop_node/annotation/annotator_node.py +2 -1
- learning_loop_node/data_classes/__init__.py +8 -9
- learning_loop_node/data_classes/annotations.py +1 -11
- learning_loop_node/data_classes/general.py +25 -7
- learning_loop_node/data_classes/training.py +1 -21
- learning_loop_node/detector/detector_logic.py +3 -2
- learning_loop_node/detector/detector_node.py +128 -21
- learning_loop_node/detector/outbox.py +1 -6
- learning_loop_node/detector/rest/about.py +3 -27
- learning_loop_node/detector/rest/model_version_control.py +9 -81
- learning_loop_node/detector/rest/operation_mode.py +2 -9
- learning_loop_node/enums/__init__.py +6 -0
- learning_loop_node/enums/annotator.py +11 -0
- learning_loop_node/enums/detector.py +18 -0
- learning_loop_node/enums/general.py +9 -0
- learning_loop_node/enums/trainer.py +22 -0
- learning_loop_node/node.py +9 -0
- learning_loop_node/tests/annotator/test_annotator_node.py +2 -2
- learning_loop_node/tests/general/test_data_classes.py +2 -1
- learning_loop_node/tests/trainer/states/test_state_detecting.py +1 -1
- learning_loop_node/tests/trainer/states/test_state_download_train_model.py +1 -1
- learning_loop_node/tests/trainer/states/test_state_prepare.py +2 -1
- learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py +1 -1
- learning_loop_node/tests/trainer/states/test_state_train.py +1 -1
- learning_loop_node/tests/trainer/states/test_state_upload_detections.py +2 -1
- learning_loop_node/tests/trainer/states/test_state_upload_model.py +2 -1
- learning_loop_node/tests/trainer/test_errors.py +1 -1
- learning_loop_node/tests/trainer/test_trainer_states.py +2 -1
- learning_loop_node/trainer/trainer_logic.py +2 -1
- learning_loop_node/trainer/trainer_logic_generic.py +3 -2
- learning_loop_node/trainer/trainer_node.py +2 -1
- {learning_loop_node-0.12.1.dist-info → learning_loop_node-0.13.1.dist-info}/METADATA +22 -3
- {learning_loop_node-0.12.1.dist-info → learning_loop_node-0.13.1.dist-info}/RECORD +34 -29
- {learning_loop_node-0.12.1.dist-info → learning_loop_node-0.13.1.dist-info}/WHEEL +0 -0
|
@@ -73,7 +73,8 @@ class AnnotatorNode(Node):
|
|
|
73
73
|
capabilities=['segmentation']
|
|
74
74
|
)
|
|
75
75
|
|
|
76
|
-
self.
|
|
76
|
+
self.log_status_on_change(status.state.value if status.state else 'None', status)
|
|
77
|
+
|
|
77
78
|
try:
|
|
78
79
|
result = await self.sio_client.call('update_annotation_node', jsonable_encoder(asdict(status)), timeout=10)
|
|
79
80
|
except Exception:
|
|
@@ -1,20 +1,19 @@
|
|
|
1
|
-
from .annotations import AnnotationData,
|
|
1
|
+
from .annotations import AnnotationData, SegmentationAnnotation, ToolOutput, UserInput
|
|
2
2
|
from .detections import (BoxDetection, ClassificationDetection, Detections, Observation, Point, PointDetection,
|
|
3
3
|
SegmentationDetection, Shape)
|
|
4
|
-
from .general import (AnnotationNodeStatus, Category,
|
|
5
|
-
ModelInformation, NodeState, NodeStatus)
|
|
4
|
+
from .general import (AboutResponse, AnnotationNodeStatus, Category, Context, DetectionStatus, ErrorConfiguration,
|
|
5
|
+
ModelInformation, ModelVersionResponse, NodeState, NodeStatus)
|
|
6
6
|
from .image_metadata import ImageMetadata
|
|
7
7
|
from .socket_response import SocketResponse
|
|
8
|
-
from .training import
|
|
9
|
-
TrainingStatus)
|
|
8
|
+
from .training import Errors, PretrainedModel, Training, TrainingError, TrainingOut, TrainingStateData, TrainingStatus
|
|
10
9
|
|
|
11
10
|
__all__ = [
|
|
12
|
-
'
|
|
11
|
+
'AboutResponse', 'AnnotationData', 'SegmentationAnnotation', 'ToolOutput', 'UserInput',
|
|
13
12
|
'BoxDetection', 'ClassificationDetection', 'ImageMetadata', 'Observation', 'Point', 'PointDetection',
|
|
14
13
|
'SegmentationDetection', 'Shape', 'Detections',
|
|
15
|
-
'AnnotationNodeStatus', 'Category', '
|
|
16
|
-
'ModelInformation', 'NodeState', 'NodeStatus',
|
|
14
|
+
'AnnotationNodeStatus', 'Category', 'Context', 'DetectionStatus', 'ErrorConfiguration',
|
|
15
|
+
'ModelInformation', 'NodeState', 'NodeStatus', 'ModelVersionResponse',
|
|
17
16
|
'SocketResponse',
|
|
18
|
-
'Errors', 'PretrainedModel', '
|
|
17
|
+
'Errors', 'PretrainedModel', 'Training',
|
|
19
18
|
'TrainingError', 'TrainingOut', 'TrainingStateData', 'TrainingStatus',
|
|
20
19
|
]
|
|
@@ -1,24 +1,14 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from dataclasses import dataclass
|
|
3
|
-
from enum import Enum
|
|
4
3
|
from typing import Optional, Union
|
|
5
4
|
|
|
5
|
+
from ..enums import AnnotationEventType
|
|
6
6
|
from .detections import Point, Shape
|
|
7
7
|
from .general import Category, Context
|
|
8
8
|
|
|
9
9
|
KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
class AnnotationEventType(str, Enum):
|
|
13
|
-
LeftMouseDown = 'left_mouse_down'
|
|
14
|
-
RightMouseDown = 'right_mouse_down'
|
|
15
|
-
MouseMove = 'mouse_move'
|
|
16
|
-
LeftMouseUp = 'left_mouse_up'
|
|
17
|
-
RightMouseUp = 'right_mouse_up'
|
|
18
|
-
KeyUp = 'key_up'
|
|
19
|
-
KeyDown = 'key_down'
|
|
20
|
-
|
|
21
|
-
|
|
22
12
|
@dataclass(**KWONLY_SLOTS)
|
|
23
13
|
class AnnotationData():
|
|
24
14
|
coordinate: Point
|
|
@@ -8,14 +8,9 @@ from typing import Dict, List, Optional, Union
|
|
|
8
8
|
|
|
9
9
|
from dacite import from_dict
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
from ..enums import CategoryType
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
Box = 'box'
|
|
16
|
-
Point = 'point'
|
|
17
|
-
Segmentation = 'segmentation'
|
|
18
|
-
Classification = 'classification'
|
|
13
|
+
KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) else {}
|
|
19
14
|
|
|
20
15
|
|
|
21
16
|
@dataclass(**KWONLY_SLOTS)
|
|
@@ -104,6 +99,29 @@ class ModelInformation():
|
|
|
104
99
|
return from_dict(ModelInformation, data=data)
|
|
105
100
|
|
|
106
101
|
|
|
102
|
+
@dataclass(**KWONLY_SLOTS)
|
|
103
|
+
class AboutResponse:
|
|
104
|
+
operation_mode: str = field(metadata={"description": "The operation mode of the detector node"})
|
|
105
|
+
state: Optional[str] = field(metadata={
|
|
106
|
+
"description": "The state of the detector node",
|
|
107
|
+
"example": "idle, online, detecting"})
|
|
108
|
+
model_info: Optional[ModelInformation] = field(metadata={
|
|
109
|
+
"description": "Information about the model of the detector node"})
|
|
110
|
+
target_model: Optional[str] = field(metadata={"description": "The target model of the detector node"})
|
|
111
|
+
version_control: str = field(metadata={
|
|
112
|
+
"description": "The version control mode of the detector node",
|
|
113
|
+
"example": "follow_loop, specific_version, pause"})
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass(**KWONLY_SLOTS)
|
|
117
|
+
class ModelVersionResponse:
|
|
118
|
+
current_version: str = field(metadata={"description": "The version of the model currently used by the detector."})
|
|
119
|
+
target_version: str = field(metadata={"description": "The target model version set in the detector."})
|
|
120
|
+
loop_version: str = field(metadata={"description": "The target model version specified by the loop."})
|
|
121
|
+
local_versions: List[str] = field(metadata={"description": "The locally available versions of the model."})
|
|
122
|
+
version_control: str = field(metadata={"description": "The version control mode."})
|
|
123
|
+
|
|
124
|
+
|
|
107
125
|
@dataclass(**KWONLY_SLOTS)
|
|
108
126
|
class ErrorConfiguration():
|
|
109
127
|
begin_training: Optional[bool] = False
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
import sys
|
|
3
3
|
import time
|
|
4
4
|
from dataclasses import dataclass, field
|
|
5
|
-
from enum import Enum
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from typing import Any, Dict, List, Optional
|
|
8
7
|
from uuid import uuid4
|
|
9
8
|
|
|
9
|
+
from ..enums import TrainerState
|
|
10
10
|
from ..helpers.misc import create_image_folder, create_training_folder
|
|
11
11
|
# pylint: disable=no-name-in-module
|
|
12
12
|
from .general import Category, Context
|
|
@@ -21,26 +21,6 @@ class PretrainedModel():
|
|
|
21
21
|
description: str
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
class TrainerState(str, Enum):
|
|
25
|
-
Idle = 'idle'
|
|
26
|
-
Initialized = 'initialized'
|
|
27
|
-
Preparing = 'preparing'
|
|
28
|
-
DataDownloading = 'data_downloading'
|
|
29
|
-
DataDownloaded = 'data_downloaded'
|
|
30
|
-
TrainModelDownloading = 'train_model_downloading'
|
|
31
|
-
TrainModelDownloaded = 'train_model_downloaded'
|
|
32
|
-
TrainingRunning = 'running'
|
|
33
|
-
TrainingFinished = 'training_finished'
|
|
34
|
-
ConfusionMatrixSyncing = 'confusion_matrix_syncing'
|
|
35
|
-
ConfusionMatrixSynced = 'confusion_matrix_synced'
|
|
36
|
-
TrainModelUploading = 'train_model_uploading'
|
|
37
|
-
TrainModelUploaded = 'train_model_uploaded'
|
|
38
|
-
Detecting = 'detecting'
|
|
39
|
-
Detected = 'detected'
|
|
40
|
-
DetectionUploading = 'detection_uploading'
|
|
41
|
-
ReadyForCleanup = 'ready_for_cleanup'
|
|
42
|
-
|
|
43
|
-
|
|
44
24
|
@dataclass(**KWONLY_SLOTS)
|
|
45
25
|
class TrainingStatus():
|
|
46
26
|
id: str # NOTE this must not be changed, but tests wont detect a change -> update tests!
|
|
@@ -31,14 +31,15 @@ class DetectorLogic():
|
|
|
31
31
|
logging.info('Loading model from %s', GLOBALS.data_folder)
|
|
32
32
|
model_info = ModelInformation.load_from_disk(f'{GLOBALS.data_folder}/model')
|
|
33
33
|
if model_info is None:
|
|
34
|
-
logging.
|
|
34
|
+
logging.error('No model found')
|
|
35
35
|
self._model_info = None
|
|
36
|
-
|
|
36
|
+
raise Exception('No model found')
|
|
37
37
|
try:
|
|
38
38
|
self._model_info = model_info
|
|
39
39
|
self.init()
|
|
40
40
|
logging.info('Successfully loaded model %s', self._model_info)
|
|
41
41
|
except Exception:
|
|
42
|
+
self._model_info = None
|
|
42
43
|
logging.error('Could not init model %s', model_info)
|
|
43
44
|
raise
|
|
44
45
|
|
|
@@ -6,7 +6,7 @@ import subprocess
|
|
|
6
6
|
from dataclasses import asdict
|
|
7
7
|
from datetime import datetime
|
|
8
8
|
from threading import Thread
|
|
9
|
-
from typing import Dict, List, Optional
|
|
9
|
+
from typing import Dict, List, Optional
|
|
10
10
|
|
|
11
11
|
import numpy as np
|
|
12
12
|
import socketio
|
|
@@ -14,9 +14,11 @@ from dacite import from_dict
|
|
|
14
14
|
from fastapi.encoders import jsonable_encoder
|
|
15
15
|
from socketio import AsyncClient
|
|
16
16
|
|
|
17
|
-
from ..data_classes import Category, Context, DetectionStatus, ImageMetadata, ModelInformation,
|
|
17
|
+
from ..data_classes import (AboutResponse, Category, Context, DetectionStatus, ImageMetadata, ModelInformation,
|
|
18
|
+
ModelVersionResponse, Shape)
|
|
18
19
|
from ..data_classes.socket_response import SocketResponse
|
|
19
20
|
from ..data_exchanger import DataExchanger, DownloadError
|
|
21
|
+
from ..enums import OperationMode, VersionMode
|
|
20
22
|
from ..globals import GLOBALS
|
|
21
23
|
from ..helpers import environment_reader
|
|
22
24
|
from ..node import Node
|
|
@@ -30,7 +32,6 @@ from .rest import model_version_control as rest_version_control
|
|
|
30
32
|
from .rest import operation_mode as rest_mode
|
|
31
33
|
from .rest import outbox_mode as rest_outbox_mode
|
|
32
34
|
from .rest import upload as rest_upload
|
|
33
|
-
from .rest.operation_mode import OperationMode
|
|
34
35
|
|
|
35
36
|
|
|
36
37
|
class DetectorNode(Node):
|
|
@@ -58,8 +59,8 @@ class DetectorNode(Node):
|
|
|
58
59
|
# FollowLoop: the detector node will follow the loop and update the model if necessary
|
|
59
60
|
# SpecificVersion: the detector node will update to a specific version, set via the /model_version endpoint
|
|
60
61
|
# Pause: the detector node will not update the model
|
|
61
|
-
self.version_control:
|
|
62
|
-
'VERSION_CONTROL_DEFAULT', 'follow_loop').lower() == 'pause' else
|
|
62
|
+
self.version_control: VersionMode = VersionMode.Pause if os.environ.get(
|
|
63
|
+
'VERSION_CONTROL_DEFAULT', 'follow_loop').lower() == 'pause' else VersionMode.FollowLoop
|
|
63
64
|
self.target_model: Optional[ModelInformation] = None
|
|
64
65
|
self.loop_deployment_target: Optional[ModelInformation] = None
|
|
65
66
|
|
|
@@ -75,6 +76,76 @@ class DetectorNode(Node):
|
|
|
75
76
|
|
|
76
77
|
self.setup_sio_server()
|
|
77
78
|
|
|
79
|
+
def get_about_response(self) -> AboutResponse:
|
|
80
|
+
return AboutResponse(
|
|
81
|
+
operation_mode=self.operation_mode.value,
|
|
82
|
+
state=self.status.state,
|
|
83
|
+
model_info=self.detector_logic._model_info, # pylint: disable=protected-access
|
|
84
|
+
target_model=self.target_model.version if self.target_model else None,
|
|
85
|
+
version_control=self.version_control.value
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def get_model_version_response(self) -> ModelVersionResponse:
|
|
89
|
+
current_version = self.detector_logic._model_info.version if self.detector_logic._model_info is not None else 'None' # pylint: disable=protected-access
|
|
90
|
+
target_version = self.target_model.version if self.target_model is not None else 'None'
|
|
91
|
+
loop_version = self.loop_deployment_target.version if self.loop_deployment_target is not None else 'None'
|
|
92
|
+
|
|
93
|
+
local_versions: list[str] = []
|
|
94
|
+
models_path = os.path.join(GLOBALS.data_folder, 'models')
|
|
95
|
+
local_models = os.listdir(models_path) if os.path.exists(models_path) else []
|
|
96
|
+
for model in local_models:
|
|
97
|
+
if model.replace('.', '').isdigit():
|
|
98
|
+
local_versions.append(model)
|
|
99
|
+
|
|
100
|
+
return ModelVersionResponse(
|
|
101
|
+
current_version=current_version,
|
|
102
|
+
target_version=target_version,
|
|
103
|
+
loop_version=loop_version,
|
|
104
|
+
local_versions=local_versions,
|
|
105
|
+
version_control=self.version_control.value,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
async def set_model_version_mode(self, version_control_mode: str) -> None:
|
|
109
|
+
|
|
110
|
+
self.log.info('Setting model version mode to %s', version_control_mode)
|
|
111
|
+
|
|
112
|
+
if version_control_mode == 'follow_loop':
|
|
113
|
+
self.version_control = VersionMode.FollowLoop
|
|
114
|
+
elif version_control_mode == 'pause':
|
|
115
|
+
self.version_control = VersionMode.Pause
|
|
116
|
+
else:
|
|
117
|
+
self.version_control = VersionMode.SpecificVersion
|
|
118
|
+
if not version_control_mode or not version_control_mode.replace('.', '').isdigit():
|
|
119
|
+
raise Exception('Invalid version number')
|
|
120
|
+
target_version = version_control_mode
|
|
121
|
+
|
|
122
|
+
if self.target_model is not None and self.target_model.version == target_version:
|
|
123
|
+
return
|
|
124
|
+
|
|
125
|
+
# Fetch the model uuid by version from the loop
|
|
126
|
+
uri = f'/{self.organization}/projects/{self.project}/models'
|
|
127
|
+
response = await self.loop_communicator.get(uri)
|
|
128
|
+
if response.status_code != 200:
|
|
129
|
+
self.version_control = VersionMode.Pause
|
|
130
|
+
raise Exception('Failed to load models from learning loop')
|
|
131
|
+
|
|
132
|
+
models = response.json()['models']
|
|
133
|
+
models_with_target_version = [m for m in models if m['version'] == target_version]
|
|
134
|
+
if len(models_with_target_version) == 0:
|
|
135
|
+
self.version_control = VersionMode.Pause
|
|
136
|
+
raise Exception(f'No Model with version {target_version}')
|
|
137
|
+
if len(models_with_target_version) > 1:
|
|
138
|
+
self.version_control = VersionMode.Pause
|
|
139
|
+
raise Exception(f'Multiple models with version {target_version}')
|
|
140
|
+
|
|
141
|
+
model_id = models_with_target_version[0]['id']
|
|
142
|
+
model_host = models_with_target_version[0].get('host', 'unknown')
|
|
143
|
+
|
|
144
|
+
self.target_model = ModelInformation(organization=self.organization, project=self.project,
|
|
145
|
+
host=model_host, categories=[],
|
|
146
|
+
id=model_id,
|
|
147
|
+
version=target_version)
|
|
148
|
+
|
|
78
149
|
async def soft_reload(self) -> None:
|
|
79
150
|
# simulate init
|
|
80
151
|
self.organization = environment_reader.organization()
|
|
@@ -85,8 +156,8 @@ class DetectorNode(Node):
|
|
|
85
156
|
Context(organization=self.organization, project=self.project),
|
|
86
157
|
self.loop_communicator)
|
|
87
158
|
self.relevance_filter = RelevanceFilter(self.outbox)
|
|
88
|
-
self.version_control =
|
|
89
|
-
'VERSION_CONTROL_DEFAULT', 'follow_loop').lower() == 'pause' else
|
|
159
|
+
self.version_control = VersionMode.Pause if os.environ.get(
|
|
160
|
+
'VERSION_CONTROL_DEFAULT', 'follow_loop').lower() == 'pause' else VersionMode.FollowLoop
|
|
90
161
|
self.target_model = None
|
|
91
162
|
# self.setup_sio_server()
|
|
92
163
|
|
|
@@ -141,9 +212,8 @@ class DetectorNode(Node):
|
|
|
141
212
|
@self.sio.event
|
|
142
213
|
async def detect(sid, data: Dict) -> Dict:
|
|
143
214
|
try:
|
|
144
|
-
np_image = np.frombuffer(data['image'], np.uint8)
|
|
145
215
|
det = await self.get_detections(
|
|
146
|
-
raw_image=
|
|
216
|
+
raw_image=np.frombuffer(data['image'], np.uint8),
|
|
147
217
|
camera_id=data.get('camera-id', None) or data.get('mac', None),
|
|
148
218
|
tags=data.get('tags', []),
|
|
149
219
|
source=data.get('source', None),
|
|
@@ -160,10 +230,38 @@ class DetectorNode(Node):
|
|
|
160
230
|
return {'error': str(e)}
|
|
161
231
|
|
|
162
232
|
@self.sio.event
|
|
163
|
-
async def info(sid) ->
|
|
233
|
+
async def info(sid) -> Dict:
|
|
164
234
|
if self.detector_logic.is_initialized:
|
|
165
235
|
return asdict(self.detector_logic.model_info)
|
|
166
|
-
return
|
|
236
|
+
return {"status": "No model loaded"}
|
|
237
|
+
|
|
238
|
+
@self.sio.event
|
|
239
|
+
async def about(sid) -> Dict:
|
|
240
|
+
return asdict(self.get_about_response())
|
|
241
|
+
|
|
242
|
+
@self.sio.event
|
|
243
|
+
async def get_model_version(sid) -> Dict:
|
|
244
|
+
return asdict(self.get_model_version_response())
|
|
245
|
+
|
|
246
|
+
@self.sio.event
|
|
247
|
+
async def set_model_version_mode(sid, data: str) -> Dict:
|
|
248
|
+
try:
|
|
249
|
+
await self.set_model_version_mode(data)
|
|
250
|
+
return {"status": "OK"}
|
|
251
|
+
except Exception as e:
|
|
252
|
+
return {'error': str(e)}
|
|
253
|
+
|
|
254
|
+
@self.sio.event
|
|
255
|
+
async def get_outbox_mode(sid) -> Dict:
|
|
256
|
+
return {'outbox_mode': self.outbox.get_mode().value}
|
|
257
|
+
|
|
258
|
+
@self.sio.event
|
|
259
|
+
async def set_outbox_mode(sid, data: str) -> Dict:
|
|
260
|
+
try:
|
|
261
|
+
await self.outbox.set_mode(data)
|
|
262
|
+
return {"status": "OK"}
|
|
263
|
+
except Exception as e:
|
|
264
|
+
return {'error': str(e)}
|
|
167
265
|
|
|
168
266
|
@self.sio.event
|
|
169
267
|
async def upload(sid, data: Dict) -> Optional[Dict]:
|
|
@@ -228,13 +326,16 @@ class DetectorNode(Node):
|
|
|
228
326
|
with step_into(GLOBALS.data_folder):
|
|
229
327
|
model_symlink = 'model'
|
|
230
328
|
target_model_folder = f'models/{self.target_model.version}'
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
329
|
+
if not os.path.exists(target_model_folder):
|
|
330
|
+
os.makedirs(target_model_folder)
|
|
331
|
+
await self.data_exchanger.download_model(target_model_folder,
|
|
332
|
+
Context(organization=self.organization,
|
|
333
|
+
project=self.project),
|
|
334
|
+
self.target_model.id,
|
|
335
|
+
self.detector_logic.model_format)
|
|
336
|
+
self.log.info('Downloaded model %s', self.target_model.version)
|
|
337
|
+
else:
|
|
338
|
+
self.log.info('No need to download model %s (already exists)', self.target_model.version)
|
|
238
339
|
try:
|
|
239
340
|
os.unlink(model_symlink)
|
|
240
341
|
os.remove(model_symlink)
|
|
@@ -243,7 +344,12 @@ class DetectorNode(Node):
|
|
|
243
344
|
os.symlink(target_model_folder, model_symlink)
|
|
244
345
|
self.log.info('Updated symlink for model to %s', os.readlink(model_symlink))
|
|
245
346
|
|
|
246
|
-
|
|
347
|
+
try:
|
|
348
|
+
self.detector_logic.load_model()
|
|
349
|
+
except Exception:
|
|
350
|
+
self.log.exception('Could not load model, will retry download on next check')
|
|
351
|
+
shutil.rmtree(target_model_folder, ignore_errors=True)
|
|
352
|
+
return
|
|
247
353
|
try:
|
|
248
354
|
await self.sync_status_with_learning_loop()
|
|
249
355
|
except Exception:
|
|
@@ -292,7 +398,8 @@ class DetectorNode(Node):
|
|
|
292
398
|
model_format=self.detector_logic.model_format,
|
|
293
399
|
)
|
|
294
400
|
|
|
295
|
-
self.
|
|
401
|
+
self.log_status_on_change(status.state or 'None', status)
|
|
402
|
+
|
|
296
403
|
response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
|
|
297
404
|
if not response:
|
|
298
405
|
self.socket_connection_broken = True
|
|
@@ -313,7 +420,7 @@ class DetectorNode(Node):
|
|
|
313
420
|
id=deployment_target_model_id,
|
|
314
421
|
version=deployment_target_model_version)
|
|
315
422
|
|
|
316
|
-
if (self.version_control ==
|
|
423
|
+
if (self.version_control == VersionMode.FollowLoop and
|
|
317
424
|
self.target_model != self.loop_deployment_target):
|
|
318
425
|
old_target_model_version = self.target_model.version if self.target_model else None
|
|
319
426
|
self.target_model = self.loop_deployment_target
|
|
@@ -7,7 +7,6 @@ import shutil
|
|
|
7
7
|
from asyncio import Task
|
|
8
8
|
from dataclasses import asdict
|
|
9
9
|
from datetime import datetime
|
|
10
|
-
from enum import Enum
|
|
11
10
|
from glob import glob
|
|
12
11
|
from io import BufferedReader, TextIOWrapper
|
|
13
12
|
from multiprocessing import Event
|
|
@@ -20,15 +19,11 @@ import PIL.Image # type: ignore
|
|
|
20
19
|
from fastapi.encoders import jsonable_encoder
|
|
21
20
|
|
|
22
21
|
from ..data_classes import ImageMetadata
|
|
22
|
+
from ..enums import OutboxMode
|
|
23
23
|
from ..globals import GLOBALS
|
|
24
24
|
from ..helpers import environment_reader
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
class OutboxMode(Enum):
|
|
28
|
-
CONTINUOUS_UPLOAD = 'continuous_upload'
|
|
29
|
-
STOPPED = 'stopped'
|
|
30
|
-
|
|
31
|
-
|
|
32
27
|
class Outbox():
|
|
33
28
|
def __init__(self) -> None:
|
|
34
29
|
self.log = logging.getLogger()
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
|
|
2
2
|
import sys
|
|
3
|
-
from
|
|
4
|
-
from typing import TYPE_CHECKING, Optional
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
5
4
|
|
|
6
5
|
from fastapi import APIRouter, Request
|
|
7
6
|
|
|
8
|
-
from ...data_classes import
|
|
7
|
+
from ...data_classes import AboutResponse
|
|
9
8
|
|
|
10
9
|
if TYPE_CHECKING:
|
|
11
10
|
from ..detector_node import DetectorNode
|
|
@@ -14,20 +13,6 @@ KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) e
|
|
|
14
13
|
router = APIRouter()
|
|
15
14
|
|
|
16
15
|
|
|
17
|
-
@dataclass(**KWONLY_SLOTS)
|
|
18
|
-
class AboutResponse:
|
|
19
|
-
operation_mode: str = field(metadata={"description": "The operation mode of the detector node"})
|
|
20
|
-
state: Optional[str] = field(metadata={
|
|
21
|
-
"description": "The state of the detector node",
|
|
22
|
-
"example": "idle, online, detecting"})
|
|
23
|
-
model_info: Optional[ModelInformation] = field(metadata={
|
|
24
|
-
"description": "Information about the model of the detector node"})
|
|
25
|
-
target_model: Optional[str] = field(metadata={"description": "The target model of the detector node"})
|
|
26
|
-
version_control: str = field(metadata={
|
|
27
|
-
"description": "The version control mode of the detector node",
|
|
28
|
-
"example": "follow_loop, specific_version, pause"})
|
|
29
|
-
|
|
30
|
-
|
|
31
16
|
@router.get("/about", response_model=AboutResponse)
|
|
32
17
|
async def get_about(request: Request):
|
|
33
18
|
'''
|
|
@@ -38,13 +23,4 @@ async def get_about(request: Request):
|
|
|
38
23
|
curl http://hosturl/about
|
|
39
24
|
'''
|
|
40
25
|
app: 'DetectorNode' = request.app
|
|
41
|
-
|
|
42
|
-
response = AboutResponse(
|
|
43
|
-
operation_mode=app.operation_mode.value,
|
|
44
|
-
state=app.status.state,
|
|
45
|
-
model_info=app.detector_logic._model_info, # pylint: disable=protected-access
|
|
46
|
-
target_model=app.target_model.version if app.target_model is not None else None,
|
|
47
|
-
version_control=app.version_control.value
|
|
48
|
-
)
|
|
49
|
-
|
|
50
|
-
return response
|
|
26
|
+
return app.get_about_response()
|
|
@@ -1,14 +1,10 @@
|
|
|
1
1
|
|
|
2
|
-
import os
|
|
3
2
|
import sys
|
|
4
|
-
from
|
|
5
|
-
from enum import Enum
|
|
6
|
-
from typing import TYPE_CHECKING, List
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
7
4
|
|
|
8
5
|
from fastapi import APIRouter, HTTPException, Request
|
|
9
6
|
|
|
10
|
-
from ...data_classes import
|
|
11
|
-
from ...globals import GLOBALS
|
|
7
|
+
from ...data_classes import ModelVersionResponse
|
|
12
8
|
|
|
13
9
|
if TYPE_CHECKING:
|
|
14
10
|
from ..detector_node import DetectorNode
|
|
@@ -17,22 +13,7 @@ KWONLY_SLOTS = {'kw_only': True, 'slots': True} if sys.version_info >= (3, 10) e
|
|
|
17
13
|
router = APIRouter()
|
|
18
14
|
|
|
19
15
|
|
|
20
|
-
|
|
21
|
-
FollowLoop = 'follow_loop' # will follow the loop
|
|
22
|
-
SpecificVersion = 'specific_version' # will follow the specific version
|
|
23
|
-
Pause = 'pause' # will pause the updates
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
@dataclass(**KWONLY_SLOTS)
|
|
27
|
-
class ModelVersionResponse:
|
|
28
|
-
current_version: str = field(metadata={"description": "The version of the model currently used by the detector."})
|
|
29
|
-
target_version: str = field(metadata={"description": "The target model version set in the detector."})
|
|
30
|
-
loop_version: str = field(metadata={"description": "The target model version specified by the loop."})
|
|
31
|
-
local_versions: List[str] = field(metadata={"description": "The locally available versions of the model."})
|
|
32
|
-
version_control: str = field(metadata={"description": "The version control mode."})
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
@router.get("/model_version")
|
|
16
|
+
@router.get("/model_version", name='Get model version information', response_model=ModelVersionResponse)
|
|
36
17
|
async def get_version(request: Request):
|
|
37
18
|
'''
|
|
38
19
|
Get information about the model version control and the current model version.
|
|
@@ -41,31 +22,11 @@ async def get_version(request: Request):
|
|
|
41
22
|
curl http://localhost/model_version
|
|
42
23
|
'''
|
|
43
24
|
# pylint: disable=protected-access
|
|
44
|
-
|
|
45
25
|
app: 'DetectorNode' = request.app
|
|
26
|
+
return app.get_model_version_response()
|
|
46
27
|
|
|
47
|
-
current_version = app.detector_logic._model_info.version if app.detector_logic._model_info is not None else 'None'
|
|
48
|
-
target_version = app.target_model.version if app.target_model is not None else 'None'
|
|
49
|
-
loop_version = app.loop_deployment_target.version if app.loop_deployment_target is not None else 'None'
|
|
50
|
-
|
|
51
|
-
local_versions: list[str] = []
|
|
52
|
-
models_path = os.path.join(GLOBALS.data_folder, 'models')
|
|
53
|
-
local_models = os.listdir(models_path) if os.path.exists(models_path) else []
|
|
54
|
-
for model in local_models:
|
|
55
|
-
if model.replace('.', '').isdigit():
|
|
56
|
-
local_versions.append(model)
|
|
57
|
-
|
|
58
|
-
response = ModelVersionResponse(
|
|
59
|
-
current_version=current_version,
|
|
60
|
-
target_version=target_version,
|
|
61
|
-
loop_version=loop_version,
|
|
62
|
-
local_versions=local_versions,
|
|
63
|
-
version_control=app.version_control.value,
|
|
64
|
-
)
|
|
65
|
-
return response
|
|
66
28
|
|
|
67
|
-
|
|
68
|
-
@router.put("/model_version")
|
|
29
|
+
@router.put("/model_version", name='Set model version control mode')
|
|
69
30
|
async def put_version(request: Request):
|
|
70
31
|
'''
|
|
71
32
|
Set the model version control mode.
|
|
@@ -77,42 +38,9 @@ async def put_version(request: Request):
|
|
|
77
38
|
'''
|
|
78
39
|
app: 'DetectorNode' = request.app
|
|
79
40
|
content = str(await request.body(), 'utf-8')
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
app.version_control = VersionMode.Pause
|
|
85
|
-
else:
|
|
86
|
-
app.version_control = VersionMode.SpecificVersion
|
|
87
|
-
if not content or not content.replace('.', '').isdigit():
|
|
88
|
-
raise HTTPException(400, 'Invalid version number')
|
|
89
|
-
target_version = content
|
|
90
|
-
|
|
91
|
-
if app.target_model is not None and app.target_model.version == target_version:
|
|
92
|
-
return "OK"
|
|
93
|
-
|
|
94
|
-
# Fetch the model uuid by version from the loop
|
|
95
|
-
uri = f'/{app.organization}/projects/{app.project}/models'
|
|
96
|
-
response = await app.loop_communicator.get(uri)
|
|
97
|
-
if response.status_code != 200:
|
|
98
|
-
app.version_control = VersionMode.Pause
|
|
99
|
-
raise HTTPException(500, 'Failed to load models from learning loop')
|
|
100
|
-
|
|
101
|
-
models = response.json()['models']
|
|
102
|
-
models_with_target_version = [m for m in models if m['version'] == target_version]
|
|
103
|
-
if len(models_with_target_version) == 0:
|
|
104
|
-
app.version_control = VersionMode.Pause
|
|
105
|
-
raise HTTPException(400, f'No Model with version {target_version}')
|
|
106
|
-
if len(models_with_target_version) > 1:
|
|
107
|
-
app.version_control = VersionMode.Pause
|
|
108
|
-
raise HTTPException(500, f'Multiple models with version {target_version}')
|
|
109
|
-
|
|
110
|
-
model_id = models_with_target_version[0]['id']
|
|
111
|
-
model_host = models_with_target_version[0].get('host', 'unknown')
|
|
112
|
-
|
|
113
|
-
app.target_model = ModelInformation(organization=app.organization, project=app.project,
|
|
114
|
-
host=model_host, categories=[],
|
|
115
|
-
id=model_id,
|
|
116
|
-
version=target_version)
|
|
41
|
+
try:
|
|
42
|
+
await app.set_model_version_mode(content)
|
|
43
|
+
except Exception as exc:
|
|
44
|
+
raise HTTPException(400, str(exc)) from exc
|
|
117
45
|
|
|
118
46
|
return "OK"
|
|
@@ -1,24 +1,17 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from enum import Enum
|
|
3
2
|
from typing import TYPE_CHECKING
|
|
4
3
|
|
|
5
4
|
from fastapi import APIRouter, HTTPException, Request
|
|
6
5
|
from fastapi.responses import PlainTextResponse
|
|
7
6
|
|
|
7
|
+
from ...enums import OperationMode
|
|
8
|
+
|
|
8
9
|
if TYPE_CHECKING:
|
|
9
10
|
from learning_loop_node.detector.detector_node import DetectorNode
|
|
10
11
|
|
|
11
12
|
router = APIRouter()
|
|
12
13
|
|
|
13
14
|
|
|
14
|
-
class OperationMode(str, Enum):
|
|
15
|
-
Startup = 'startup' # used until model is loaded
|
|
16
|
-
Idle = 'idle' # will check and perform updates
|
|
17
|
-
Detecting = 'detecting' # Blocks updates
|
|
18
|
-
|
|
19
|
-
# NOTE: This is only ment to be used by a detector node
|
|
20
|
-
|
|
21
|
-
|
|
22
15
|
@router.put("/operation_mode")
|
|
23
16
|
async def put_operation_mode(request: Request):
|
|
24
17
|
'''
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from .annotator import AnnotationEventType
|
|
2
|
+
from .detector import OperationMode, OutboxMode, VersionMode
|
|
3
|
+
from .general import CategoryType
|
|
4
|
+
from .trainer import TrainerState
|
|
5
|
+
|
|
6
|
+
__all__ = ['VersionMode', 'OperationMode', 'OutboxMode', 'AnnotationEventType', 'CategoryType', 'TrainerState']
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class AnnotationEventType(str, Enum):
|
|
5
|
+
LeftMouseDown = 'left_mouse_down'
|
|
6
|
+
RightMouseDown = 'right_mouse_down'
|
|
7
|
+
MouseMove = 'mouse_move'
|
|
8
|
+
LeftMouseUp = 'left_mouse_up'
|
|
9
|
+
RightMouseUp = 'right_mouse_up'
|
|
10
|
+
KeyUp = 'key_up'
|
|
11
|
+
KeyDown = 'key_down'
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class VersionMode(str, Enum):
|
|
5
|
+
FollowLoop = 'follow_loop' # will follow the loop
|
|
6
|
+
SpecificVersion = 'specific_version' # will follow the specific version
|
|
7
|
+
Pause = 'pause' # will pause the updates
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class OperationMode(str, Enum):
|
|
11
|
+
Startup = 'startup' # used until model is loaded
|
|
12
|
+
Idle = 'idle' # will check and perform updates
|
|
13
|
+
Detecting = 'detecting' # Blocks updates
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class OutboxMode(Enum):
|
|
17
|
+
CONTINUOUS_UPLOAD = 'continuous_upload'
|
|
18
|
+
STOPPED = 'stopped'
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
|
|
2
|
+
from enum import Enum
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class TrainerState(str, Enum):
|
|
6
|
+
Idle = 'idle'
|
|
7
|
+
Initialized = 'initialized'
|
|
8
|
+
Preparing = 'preparing'
|
|
9
|
+
DataDownloading = 'data_downloading'
|
|
10
|
+
DataDownloaded = 'data_downloaded'
|
|
11
|
+
TrainModelDownloading = 'train_model_downloading'
|
|
12
|
+
TrainModelDownloaded = 'train_model_downloaded'
|
|
13
|
+
TrainingRunning = 'running'
|
|
14
|
+
TrainingFinished = 'training_finished'
|
|
15
|
+
ConfusionMatrixSyncing = 'confusion_matrix_syncing'
|
|
16
|
+
ConfusionMatrixSynced = 'confusion_matrix_synced'
|
|
17
|
+
TrainModelUploading = 'train_model_uploading'
|
|
18
|
+
TrainModelUploaded = 'train_model_uploaded'
|
|
19
|
+
Detecting = 'detecting'
|
|
20
|
+
Detected = 'detected'
|
|
21
|
+
DetectionUploading = 'detection_uploading'
|
|
22
|
+
ReadyForCleanup = 'ready_for_cleanup'
|
learning_loop_node/node.py
CHANGED
|
@@ -73,6 +73,15 @@ class Node(FastAPI):
|
|
|
73
73
|
|
|
74
74
|
self.repeat_loop_lock = asyncio.Lock()
|
|
75
75
|
|
|
76
|
+
self.previous_state: Optional[str] = None
|
|
77
|
+
|
|
78
|
+
def log_status_on_change(self, current_state_str: str, full_status: Any):
|
|
79
|
+
if self.previous_state != current_state_str:
|
|
80
|
+
self.previous_state = current_state_str
|
|
81
|
+
self.log.info('Status changed to %s', full_status)
|
|
82
|
+
else:
|
|
83
|
+
self.log.debug('sending status %s', full_status)
|
|
84
|
+
|
|
76
85
|
def init_loop_communicator(self):
|
|
77
86
|
self.loop_communicator = LoopCommunicator()
|
|
78
87
|
self.websocket_url = self.loop_communicator.websocket_url()
|
|
@@ -7,8 +7,8 @@ from fastapi.encoders import jsonable_encoder
|
|
|
7
7
|
|
|
8
8
|
from ...annotation.annotator_logic import AnnotatorLogic
|
|
9
9
|
from ...annotation.annotator_node import AnnotatorNode
|
|
10
|
-
from ...data_classes import
|
|
11
|
-
|
|
10
|
+
from ...data_classes import AnnotationData, Category, Context, Point, ToolOutput, UserInput
|
|
11
|
+
from ...enums import AnnotationEventType, CategoryType
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class MockedAnnotatatorLogic(AnnotatorLogic):
|
|
@@ -3,7 +3,8 @@ from dataclasses import asdict
|
|
|
3
3
|
from dacite import from_dict
|
|
4
4
|
from fastapi.encoders import jsonable_encoder
|
|
5
5
|
|
|
6
|
-
from ...data_classes import AnnotationData,
|
|
6
|
+
from ...data_classes import AnnotationData, Category, Context, Point
|
|
7
|
+
from ...enums import AnnotationEventType
|
|
7
8
|
|
|
8
9
|
# Used by all Nodes
|
|
9
10
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
|
|
3
|
-
from ....
|
|
3
|
+
from ....enums import TrainerState
|
|
4
4
|
from ....trainer.trainer_logic import TrainerLogic
|
|
5
5
|
from ...test_helper import get_dummy_detections
|
|
6
6
|
from ..state_helper import assert_training_state, create_active_training_file
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import asyncio
|
|
3
3
|
import os
|
|
4
4
|
|
|
5
|
-
from ....
|
|
5
|
+
from ....enums import TrainerState
|
|
6
6
|
from ... import test_helper
|
|
7
7
|
from ..state_helper import assert_training_state, create_active_training_file
|
|
8
8
|
from ..testing_trainer_logic import TestingTrainerLogic
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
|
|
3
|
-
from ....data_classes import Context
|
|
3
|
+
from ....data_classes import Context
|
|
4
|
+
from ....enums import TrainerState
|
|
4
5
|
from ....trainer.trainer_logic import TrainerLogic
|
|
5
6
|
from ..state_helper import assert_training_state, create_active_training_file
|
|
6
7
|
from ..testing_trainer_logic import TestingTrainerLogic
|
|
@@ -3,7 +3,7 @@ import asyncio
|
|
|
3
3
|
|
|
4
4
|
from pytest_mock import MockerFixture # pip install pytest-mock
|
|
5
5
|
|
|
6
|
-
from ....
|
|
6
|
+
from ....enums import TrainerState
|
|
7
7
|
from ....trainer.trainer_logic import TrainerLogic
|
|
8
8
|
from ....trainer.trainer_node import TrainerNode
|
|
9
9
|
from ..state_helper import assert_training_state, create_active_training_file
|
|
@@ -3,7 +3,8 @@ import asyncio
|
|
|
3
3
|
import pytest
|
|
4
4
|
from dacite import from_dict
|
|
5
5
|
|
|
6
|
-
from ....data_classes import BoxDetection, Context, Detections
|
|
6
|
+
from ....data_classes import BoxDetection, Context, Detections
|
|
7
|
+
from ....enums import TrainerState
|
|
7
8
|
from ....loop_communication import LoopCommunicator
|
|
8
9
|
from ....trainer.trainer_logic import TrainerLogic
|
|
9
10
|
from ...test_helper import get_dummy_detections
|
|
@@ -2,7 +2,8 @@ import asyncio
|
|
|
2
2
|
|
|
3
3
|
from pytest_mock import MockerFixture
|
|
4
4
|
|
|
5
|
-
from ....data_classes import Context
|
|
5
|
+
from ....data_classes import Context
|
|
6
|
+
from ....enums import TrainerState
|
|
6
7
|
from ....trainer.trainer_logic import TrainerLogic
|
|
7
8
|
from ..state_helper import assert_training_state, create_active_training_file
|
|
8
9
|
from ..testing_trainer_logic import TestingTrainerLogic
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
|
|
2
2
|
from uuid import uuid4
|
|
3
3
|
|
|
4
|
-
from ...data_classes import Context,
|
|
4
|
+
from ...data_classes import Context, Training
|
|
5
|
+
from ...enums import TrainerState
|
|
5
6
|
from ...trainer.io_helpers import LastTrainingIO
|
|
6
7
|
from ...trainer.trainer_node import TrainerNode
|
|
7
8
|
from .testing_trainer_logic import TestingTrainerLogic
|
|
@@ -9,7 +9,8 @@ from typing import Coroutine, List, Optional
|
|
|
9
9
|
|
|
10
10
|
from dacite import from_dict
|
|
11
11
|
|
|
12
|
-
from ..data_classes import Detections, ModelInformation,
|
|
12
|
+
from ..data_classes import Detections, ModelInformation, TrainingError
|
|
13
|
+
from ..enums import TrainerState
|
|
13
14
|
from ..helpers.misc import create_image_folder, create_project_folder, images_for_ids, is_valid_uuid4
|
|
14
15
|
from .executor import Executor
|
|
15
16
|
from .trainer_logic_generic import TrainerLogicGeneric
|
|
@@ -10,8 +10,8 @@ from typing import TYPE_CHECKING, Callable, Coroutine, Dict, List, Optional
|
|
|
10
10
|
|
|
11
11
|
from fastapi.encoders import jsonable_encoder
|
|
12
12
|
|
|
13
|
-
from ..data_classes import
|
|
14
|
-
|
|
13
|
+
from ..data_classes import Context, Errors, PretrainedModel, Training, TrainingOut, TrainingStateData, TrainingStatus
|
|
14
|
+
from ..enums import TrainerState
|
|
15
15
|
from ..helpers.misc import create_project_folder, delete_all_training_folders, is_valid_uuid4
|
|
16
16
|
from .downloader import TrainingsDownloader
|
|
17
17
|
from .exceptions import CriticalError, NodeNeedsRestartError
|
|
@@ -348,6 +348,7 @@ class TrainerLogicGeneric(ABC):
|
|
|
348
348
|
|
|
349
349
|
logger.info('loading model from Learning Loop')
|
|
350
350
|
logger.info('downloading model %s as %s', base_model_uuid, self.model_format)
|
|
351
|
+
assert base_model_uuid is not None
|
|
351
352
|
await self.node.data_exchanger.download_model(self.training.training_folder, self.training.context, base_model_uuid, self.model_format)
|
|
352
353
|
shutil.move(f'{self.training.training_folder}/model.json',
|
|
353
354
|
f'{self.training.training_folder}/base_model.json')
|
|
@@ -81,7 +81,8 @@ class TrainerNode(Node):
|
|
|
81
81
|
return
|
|
82
82
|
|
|
83
83
|
status = self.trainer_logic.generate_status_for_loop(self.uuid, self.name)
|
|
84
|
-
self.
|
|
84
|
+
self.log_status_on_change(status.state or 'None', status.short_str())
|
|
85
|
+
|
|
85
86
|
result = await self.sio_client.call('update_trainer', jsonable_encoder(asdict(status)), timeout=30)
|
|
86
87
|
if isinstance(result, Dict) and not result['success']:
|
|
87
88
|
self.socket_connection_broken = True
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: learning-loop-node
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.1
|
|
4
4
|
Summary: Python Library for Nodes which connect to the Zauberzeug Learning Loop
|
|
5
5
|
Home-page: https://github.com/zauberzeug/learning_loop_node
|
|
6
6
|
License: MIT
|
|
@@ -106,11 +106,11 @@ The detector also has a sio **upload endpoint** that can be used to upload image
|
|
|
106
106
|
|
|
107
107
|
The endpoint returns None if the upload was successful and an error message otherwise.
|
|
108
108
|
|
|
109
|
-
### Changing the model
|
|
109
|
+
### Changing the model versioning mode
|
|
110
110
|
|
|
111
111
|
The detector can be configured to one of the following behaviors:
|
|
112
112
|
|
|
113
|
-
-
|
|
113
|
+
- use a specific model version
|
|
114
114
|
- automatically update the model version according to the learning loop deployment target
|
|
115
115
|
- pause the model updates and use the version that was last loaded
|
|
116
116
|
|
|
@@ -124,6 +124,15 @@ The model versioning configuration can be accessed/changed via a REST endpoint.
|
|
|
124
124
|
Note that the configuration is not persistent, however, the default behavior on startup can be configured via the environment variable `VERSION_CONTROL_DEFAULT`.
|
|
125
125
|
If the environment variable is set to `VERSION_CONTROL_DEFAULT=PAUSE`, the detector will pause the model updates on startup. Otherwise, the detector will automatically follow the loop deployment target.
|
|
126
126
|
|
|
127
|
+
The model versioning configuration can also be changed via a socketio event:
|
|
128
|
+
|
|
129
|
+
- Configure the detector to use a specific model version: `sio.emit('set_model_version_mode', '1.0')`
|
|
130
|
+
- Configure the detector to automatically update the model version: `sio.emit('set_model_version_mode', 'follow_loop')`
|
|
131
|
+
- Pause the model updates: `sio.emit('set_model_version_mode', 'pause')`
|
|
132
|
+
|
|
133
|
+
There is also a GET endpoint to fetch the current model versioning configuration:
|
|
134
|
+
`sio.emit('get_model_version')` or `curl http://localhost/model_version`
|
|
135
|
+
|
|
127
136
|
### Changing the outbox mode
|
|
128
137
|
|
|
129
138
|
If the autoupload is set to `all` or `filtered` (selected) images and the corresponding detections are saved on HDD (the outbox). A background thread will upload the images and detections to the Learning Loop. The outbox is located in the `outbox` folder in the root directory of the node. The outbox can be cleared by deleting the files in the folder.
|
|
@@ -138,6 +147,16 @@ Example Usage:
|
|
|
138
147
|
The current state can be queried via a GET request:
|
|
139
148
|
`curl http://localhost/outbox_mode`
|
|
140
149
|
|
|
150
|
+
Alternatively, the outbox mode can be changed via a socketio event:
|
|
151
|
+
|
|
152
|
+
- Enable upload: `sio.emit('set_outbox_mode', 'continuous_upload')`
|
|
153
|
+
- Disable upload: `sio.emit('set_outbox_mode', 'stopped')`
|
|
154
|
+
|
|
155
|
+
The outbox mode can also be queried via:
|
|
156
|
+
|
|
157
|
+
- HTTP: `curl http://localhost/outbox_mode`
|
|
158
|
+
- SocketIO: `sio.emit('get_outbox_mode')`
|
|
159
|
+
|
|
141
160
|
### Explicit upload
|
|
142
161
|
|
|
143
162
|
The detector has a REST endpoint to upload images (and detections) to the Learning Loop. The endpoint takes a POST request with the image and optionally the detections. The image is expected to be in jpg format. The detections are expected to be a json dictionary. Example:
|
|
@@ -1,30 +1,35 @@
|
|
|
1
1
|
learning_loop_node/__init__.py,sha256=onN5s8-x_xBsCM6NLmJO0Ym1sJHeCFaGw8qb0oQZmz8,364
|
|
2
2
|
learning_loop_node/annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
learning_loop_node/annotation/annotator_logic.py,sha256=BTaopkJZkIf1CI5lfsVKsxbxoUIbDJrevavuQUT5e_c,1000
|
|
4
|
-
learning_loop_node/annotation/annotator_node.py,sha256=
|
|
5
|
-
learning_loop_node/data_classes/__init__.py,sha256=
|
|
6
|
-
learning_loop_node/data_classes/annotations.py,sha256=
|
|
4
|
+
learning_loop_node/annotation/annotator_node.py,sha256=Ac6tuiO-Ne8m8XLDBK_ucRhofiudRUIjL1nM-rkUkGE,4156
|
|
5
|
+
learning_loop_node/data_classes/__init__.py,sha256=GUFbVDT8ywGUpEMhV4WZHwvD2yM9A0eAvPydHeq6gyA,1233
|
|
6
|
+
learning_loop_node/data_classes/annotations.py,sha256=NfMlTv2_5AfVY_JDM4tbjETFjSN2S2I2LJJPMMcDT50,966
|
|
7
7
|
learning_loop_node/data_classes/detections.py,sha256=7vqcS0EK8cmDjRDckHlpSZDZ9YO6qajRmYvx-oxatFc,5425
|
|
8
|
-
learning_loop_node/data_classes/general.py,sha256=
|
|
8
|
+
learning_loop_node/data_classes/general.py,sha256=r7fVfuQvbo8qOTT7zylgfM45TbIvYu8bkDIAZ3wszqA,7397
|
|
9
9
|
learning_loop_node/data_classes/image_metadata.py,sha256=56nNSf_7aMlvKsJOG8vKCzJHcqKGHVRoULp85pJ2imA,1598
|
|
10
10
|
learning_loop_node/data_classes/socket_response.py,sha256=tIdt-oYf6ULoJIDYQCecNM9OtWR6_wJ9tL0Ksu83Vko,655
|
|
11
|
-
learning_loop_node/data_classes/training.py,sha256=
|
|
11
|
+
learning_loop_node/data_classes/training.py,sha256=FFPsr2AA7ynYz39MLZaFJ0sF_9Axll5HHbAA8nnirp0,5726
|
|
12
12
|
learning_loop_node/data_exchanger.py,sha256=IG5ki3f3IsVuXbyw6q_gUIakgv-GMT6e9nhOhzjKgW4,9055
|
|
13
13
|
learning_loop_node/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
learning_loop_node/detector/detector_logic.py,sha256=
|
|
15
|
-
learning_loop_node/detector/detector_node.py,sha256=
|
|
14
|
+
learning_loop_node/detector/detector_logic.py,sha256=FhGbu0mdF0tW0Gg8cr8xM6ZmZzEigGf0IcAWBjoxFrs,2191
|
|
15
|
+
learning_loop_node/detector/detector_node.py,sha256=ud5jFhazk_8wKzrwtpWRwVsLQkWzskzCnymAKbJW7H0,24921
|
|
16
16
|
learning_loop_node/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
17
|
learning_loop_node/detector/inbox_filter/cam_observation_history.py,sha256=8gzxYPD3t1OS9wBHXfIvNV2xTTMo0B70O1b50iaH2D8,3344
|
|
18
18
|
learning_loop_node/detector/inbox_filter/relevance_filter.py,sha256=NPEmrAtuGjIWCtHS0B3zDmnYWkhVFCLbd_7RUp08_AM,1372
|
|
19
|
-
learning_loop_node/detector/outbox.py,sha256=
|
|
19
|
+
learning_loop_node/detector/outbox.py,sha256=CYuXI6uXQ_mrVPZ0qvNgdSZH1nLarcs7Fj_7Mb3k7MA,8558
|
|
20
20
|
learning_loop_node/detector/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
-
learning_loop_node/detector/rest/about.py,sha256=
|
|
21
|
+
learning_loop_node/detector/rest/about.py,sha256=evHJ2svUZY_DFz0FSef5u9c5KW4Uc3GL7EbPinG9-dg,583
|
|
22
22
|
learning_loop_node/detector/rest/backdoor_controls.py,sha256=ZNaFOvC0OLWNtcLiG-NIqS_y1kkLP4csgk3CHhp8Gis,885
|
|
23
23
|
learning_loop_node/detector/rest/detect.py,sha256=ofJ3ysTarbCpiH1YAD6gSJbrDOzAcsLRuGxhr57dtk0,2503
|
|
24
|
-
learning_loop_node/detector/rest/model_version_control.py,sha256=
|
|
25
|
-
learning_loop_node/detector/rest/operation_mode.py,sha256=
|
|
24
|
+
learning_loop_node/detector/rest/model_version_control.py,sha256=P4FOG0U9HT6QtCoNt-1s1pT6drtgdVjGZWEuCAyuNmA,1370
|
|
25
|
+
learning_loop_node/detector/rest/operation_mode.py,sha256=1_xfutA_6nzdb4Q_jZiHQ5m_wA83bcG5jSIy-sfNIvk,1575
|
|
26
26
|
learning_loop_node/detector/rest/outbox_mode.py,sha256=H8coDNbgLGEfXmKQrhtXWeUHBAHpnrdZktuHXQz0xis,1148
|
|
27
27
|
learning_loop_node/detector/rest/upload.py,sha256=5YWY0Ku4duZqKd6tjyJzq-Ga83o2UYb1VmzuxBIgo0w,1061
|
|
28
|
+
learning_loop_node/enums/__init__.py,sha256=tjSrhztIQ8W656_QuXfTbbVNtH_wDXP5hpYZgzfgRhc,285
|
|
29
|
+
learning_loop_node/enums/annotator.py,sha256=mtTAw-8LJIrHcYkBjYHCZuhYEEHS6QzSK8k6BhLusvQ,285
|
|
30
|
+
learning_loop_node/enums/detector.py,sha256=Qvm5LWWR9BfsDxHEQ8YzaPaUuSmp4BescYuV4X4ikwE,512
|
|
31
|
+
learning_loop_node/enums/general.py,sha256=OSpcIpdFjuRByON2xBoiIXl1SgDREIqkwnaxkTFjnHU,164
|
|
32
|
+
learning_loop_node/enums/trainer.py,sha256=VaD63guLO4aKgVfXT0EryPlXKQGegSET3Cp4R0uC_Ew,746
|
|
28
33
|
learning_loop_node/examples/novelty_score_updater.py,sha256=1DRgM9lxjFV-q2JvGDDsNLz_ic_rhEZ9wc6ZdjcxwPE,2038
|
|
29
34
|
learning_loop_node/globals.py,sha256=tgw_8RYOipPV9aYlyUhYtXfUxvJKRvfUk6u-qVAtZmY,174
|
|
30
35
|
learning_loop_node/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -33,14 +38,14 @@ learning_loop_node/helpers/gdrive_downloader.py,sha256=zeYJciTAJVRpu_eFjwgYLCpIa
|
|
|
33
38
|
learning_loop_node/helpers/log_conf.py,sha256=hqVAa_9NnYEU6N0dcOKmph82p7MpgKqeF_eomTLYzWY,961
|
|
34
39
|
learning_loop_node/helpers/misc.py,sha256=J29iBmsEUAraKKDN1m1NKiHQ3QrP5ub5HBU6cllSP2g,7384
|
|
35
40
|
learning_loop_node/loop_communication.py,sha256=kc7GrkUS14Ka5OICaaOd_LZ61D-6O19GcyDEwckTxvM,7286
|
|
36
|
-
learning_loop_node/node.py,sha256=
|
|
41
|
+
learning_loop_node/node.py,sha256=rBdZiBzp7VqQkIt9GJ8b4nK2Y54lUJFzK-gc7rb14cA,10966
|
|
37
42
|
learning_loop_node/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
43
|
learning_loop_node/rest.py,sha256=omwlRHLnyG-kgCBVnZDk5_SAPobL9g7slWeX21wsPGw,1551
|
|
39
44
|
learning_loop_node/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
40
45
|
learning_loop_node/tests/annotator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
46
|
learning_loop_node/tests/annotator/conftest.py,sha256=G4ZvdZUdvPp9bYCzg3eEVkGCeXn9INZ3AcN7d5CyLkU,1931
|
|
42
47
|
learning_loop_node/tests/annotator/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0W5UOnDfGGs_igU,262
|
|
43
|
-
learning_loop_node/tests/annotator/test_annotator_node.py,sha256=
|
|
48
|
+
learning_loop_node/tests/annotator/test_annotator_node.py,sha256=UWRXRSBc1e795ftkp7xrEXbyR4LYvFDDHRpZGqC3vr8,1974
|
|
44
49
|
learning_loop_node/tests/detector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
50
|
learning_loop_node/tests/detector/conftest.py,sha256=4zNW8dnwj3CDKCkFNVCPbHgFTYtDvdaqnUM4s_I-cq4,5328
|
|
46
51
|
learning_loop_node/tests/detector/inbox_filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -60,7 +65,7 @@ learning_loop_node/tests/general/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0
|
|
|
60
65
|
learning_loop_node/tests/general/test_data/file_1.txt,sha256=Lis06nfvbFPVCBZyEgQlfI_Nle2YDq1GQBlYvEfFtxw,19
|
|
61
66
|
learning_loop_node/tests/general/test_data/file_2.txt,sha256=Xp8EETGhZBdVAgb4URowSSpOytwwwJdV0Renkdur7R8,19
|
|
62
67
|
learning_loop_node/tests/general/test_data/model.json,sha256=_xNDucGOWila8gWnu8yFfrqmQ45Xq-_39eLKzjRtvpE,516
|
|
63
|
-
learning_loop_node/tests/general/test_data_classes.py,sha256=
|
|
68
|
+
learning_loop_node/tests/general/test_data_classes.py,sha256=RnDzRtB-eRfWnaaA6qAzC1W8wurFzJ4xt1Q5pd7ZCS0,721
|
|
64
69
|
learning_loop_node/tests/general/test_downloader.py,sha256=y4GcUyR0OAfrwltd6eyQgopwTt3DwjzX0Sr8yrooLec,3347
|
|
65
70
|
learning_loop_node/tests/general/test_learning_loop_node.py,sha256=SZd-VChpWnnsPN46pr4E_LL3ZevYx6psU-AWdVeOFpQ,770
|
|
66
71
|
learning_loop_node/tests/test_helper.py,sha256=Xajn6BWJqeD36YAETwdcJd6awY2NPmaOis3gWgFc97k,2909
|
|
@@ -70,15 +75,15 @@ learning_loop_node/tests/trainer/pytest.ini,sha256=8QdjmawLy1zAzXrJ88or1kpFDhJw0
|
|
|
70
75
|
learning_loop_node/tests/trainer/state_helper.py,sha256=MDe9opeKruip74FoRFff8MSWGiQNFqDpPtIEIbgPnFc,919
|
|
71
76
|
learning_loop_node/tests/trainer/states/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
72
77
|
learning_loop_node/tests/trainer/states/test_state_cleanup.py,sha256=gZNxSSwnj9f0esExNnQzqadM6-sE3IsF5sNbD0bZNu8,1250
|
|
73
|
-
learning_loop_node/tests/trainer/states/test_state_detecting.py,sha256
|
|
74
|
-
learning_loop_node/tests/trainer/states/test_state_download_train_model.py,sha256
|
|
75
|
-
learning_loop_node/tests/trainer/states/test_state_prepare.py,sha256=
|
|
76
|
-
learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py,sha256=
|
|
77
|
-
learning_loop_node/tests/trainer/states/test_state_train.py,sha256=
|
|
78
|
-
learning_loop_node/tests/trainer/states/test_state_upload_detections.py,sha256=
|
|
79
|
-
learning_loop_node/tests/trainer/states/test_state_upload_model.py,sha256=
|
|
80
|
-
learning_loop_node/tests/trainer/test_errors.py,sha256=
|
|
81
|
-
learning_loop_node/tests/trainer/test_trainer_states.py,sha256=
|
|
78
|
+
learning_loop_node/tests/trainer/states/test_state_detecting.py,sha256=-NLR5se7_OY_X8_Gf-BWw7X6dS_Pzsnkz84J5aTbqFU,3689
|
|
79
|
+
learning_loop_node/tests/trainer/states/test_state_download_train_model.py,sha256=-T8iAutBliv0MV5bV5lPvn2aNjF3vMBCj8iAZTC-Q7g,2992
|
|
80
|
+
learning_loop_node/tests/trainer/states/test_state_prepare.py,sha256=boCU93Bv2VWbW73MC_suTbwCcuR7RWn-6dgVvdiJ9tA,2291
|
|
81
|
+
learning_loop_node/tests/trainer/states/test_state_sync_confusion_matrix.py,sha256=qygblvDhsC7gcteo2puaR16Mqr4d8W2QgfGr6gUPI7s,5104
|
|
82
|
+
learning_loop_node/tests/trainer/states/test_state_train.py,sha256=ovRs8EepQjy0yQJssK0TdcZcraBhmUkbMWeNKdHS114,2893
|
|
83
|
+
learning_loop_node/tests/trainer/states/test_state_upload_detections.py,sha256=oFQGTeRZhW7MBISAfpe65KphZNxFUsZu3-5hD9_LS6k,7438
|
|
84
|
+
learning_loop_node/tests/trainer/states/test_state_upload_model.py,sha256=jHWLa48tNljZwIiqI-1z71ENRGnn7Z0BsVcDBVWVBj4,3642
|
|
85
|
+
learning_loop_node/tests/trainer/test_errors.py,sha256=LdIky-SuR4Owml8FlByD0GXdv3vGh6JuqDQgJfFdgkw,2102
|
|
86
|
+
learning_loop_node/tests/trainer/test_trainer_states.py,sha256=OFrHSjpTIcwDp-QRe8sdseTZr0Z5dVeCt_ani9pfCqI,1238
|
|
82
87
|
learning_loop_node/tests/trainer/testing_trainer_logic.py,sha256=vaz7EbsVRcSMyAfEGpXgNFj1yQc9dYLCYDrjjmYTZ1o,3765
|
|
83
88
|
learning_loop_node/trainer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
84
89
|
learning_loop_node/trainer/downloader.py,sha256=AIyEKM4XYtd6VgmXrP3VayV9DpJzdURK1Brx81ePNSM,1470
|
|
@@ -88,9 +93,9 @@ learning_loop_node/trainer/io_helpers.py,sha256=ZnAPVqhq8XCHe1NoiOQJ_w0B-estcc8C
|
|
|
88
93
|
learning_loop_node/trainer/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
89
94
|
learning_loop_node/trainer/rest/backdoor_controls.py,sha256=ZnK8ypY5r_q0-YZbtaOxhQThzuZvMsQHM5gJGESd_dE,5131
|
|
90
95
|
learning_loop_node/trainer/test_executor.py,sha256=6BVGDN_6f5GEMMEvDLSG1yzMybSvgXaP5uYpSfsVPP0,2224
|
|
91
|
-
learning_loop_node/trainer/trainer_logic.py,sha256=
|
|
92
|
-
learning_loop_node/trainer/trainer_logic_generic.py,sha256=
|
|
93
|
-
learning_loop_node/trainer/trainer_node.py,sha256=
|
|
94
|
-
learning_loop_node-0.
|
|
95
|
-
learning_loop_node-0.
|
|
96
|
-
learning_loop_node-0.
|
|
96
|
+
learning_loop_node/trainer/trainer_logic.py,sha256=eK-01qZzi10UjLMCQX8vy5eW2FoghPj3rzzDC-s3Si4,8792
|
|
97
|
+
learning_loop_node/trainer/trainer_logic_generic.py,sha256=JGH2IClpte8WqO_Pmh7Epa-328Pyl5RYYZlWgOdygvs,26827
|
|
98
|
+
learning_loop_node/trainer/trainer_node.py,sha256=Dl4ZQAjjXQggibeBjvhXAoFClw1ZX2Kkt3v_fjrJnCI,4508
|
|
99
|
+
learning_loop_node-0.13.1.dist-info/METADATA,sha256=uR7jIEJgo8EPnOimjLXTu2IM58cgLiHUVfPTD78xc9o,12761
|
|
100
|
+
learning_loop_node-0.13.1.dist-info/WHEEL,sha256=WGfLGfLX43Ei_YORXSnT54hxFygu34kMpcQdmgmEwCQ,88
|
|
101
|
+
learning_loop_node-0.13.1.dist-info/RECORD,,
|
|
File without changes
|