synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synapse_sdk/__init__.py +24 -0
- synapse_sdk/cli/code_server.py +305 -33
- synapse_sdk/clients/agent/__init__.py +2 -1
- synapse_sdk/clients/agent/container.py +143 -0
- synapse_sdk/clients/agent/ray.py +296 -38
- synapse_sdk/clients/backend/annotation.py +1 -1
- synapse_sdk/clients/backend/core.py +31 -4
- synapse_sdk/clients/backend/data_collection.py +82 -7
- synapse_sdk/clients/backend/hitl.py +1 -1
- synapse_sdk/clients/backend/ml.py +1 -1
- synapse_sdk/clients/base.py +211 -61
- synapse_sdk/loggers.py +46 -0
- synapse_sdk/plugins/README.md +1340 -0
- synapse_sdk/plugins/categories/base.py +59 -9
- synapse_sdk/plugins/categories/export/actions/__init__.py +3 -0
- synapse_sdk/plugins/categories/export/actions/export/__init__.py +28 -0
- synapse_sdk/plugins/categories/export/actions/export/action.py +165 -0
- synapse_sdk/plugins/categories/export/actions/export/enums.py +113 -0
- synapse_sdk/plugins/categories/export/actions/export/exceptions.py +53 -0
- synapse_sdk/plugins/categories/export/actions/export/models.py +74 -0
- synapse_sdk/plugins/categories/export/actions/export/run.py +195 -0
- synapse_sdk/plugins/categories/export/actions/export/utils.py +187 -0
- synapse_sdk/plugins/categories/export/templates/config.yaml +19 -1
- synapse_sdk/plugins/categories/export/templates/plugin/__init__.py +390 -0
- synapse_sdk/plugins/categories/export/templates/plugin/export.py +153 -177
- synapse_sdk/plugins/categories/neural_net/actions/train.py +1130 -32
- synapse_sdk/plugins/categories/neural_net/actions/tune.py +157 -4
- synapse_sdk/plugins/categories/neural_net/templates/config.yaml +7 -4
- synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py +4 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/__init__.py +3 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/action.py +10 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/__init__.py +28 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/action.py +148 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/enums.py +269 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/exceptions.py +14 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/factory.py +76 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/models.py +100 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/orchestrator.py +248 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/run.py +64 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/__init__.py +17 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/annotation.py +265 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/base.py +170 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/extraction.py +83 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/metrics.py +92 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/preprocessor.py +243 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/validation.py +143 -0
- synapse_sdk/plugins/categories/upload/actions/upload/__init__.py +19 -0
- synapse_sdk/plugins/categories/upload/actions/upload/action.py +236 -0
- synapse_sdk/plugins/categories/upload/actions/upload/context.py +185 -0
- synapse_sdk/plugins/categories/upload/actions/upload/enums.py +493 -0
- synapse_sdk/plugins/categories/upload/actions/upload/exceptions.py +36 -0
- synapse_sdk/plugins/categories/upload/actions/upload/factory.py +138 -0
- synapse_sdk/plugins/categories/upload/actions/upload/models.py +214 -0
- synapse_sdk/plugins/categories/upload/actions/upload/orchestrator.py +183 -0
- synapse_sdk/plugins/categories/upload/actions/upload/registry.py +113 -0
- synapse_sdk/plugins/categories/upload/actions/upload/run.py +179 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/base.py +107 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py +62 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/collection.py +63 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py +91 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py +82 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py +235 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py +201 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py +104 -0
- synapse_sdk/plugins/categories/upload/actions/upload/steps/validate.py +71 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/base.py +82 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/batch.py +39 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/single.py +29 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py +300 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/recursive.py +287 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/excel.py +174 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/none.py +16 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/sync.py +84 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/__init__.py +1 -0
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py +60 -0
- synapse_sdk/plugins/categories/upload/actions/upload/utils.py +250 -0
- synapse_sdk/plugins/categories/upload/templates/README.md +470 -0
- synapse_sdk/plugins/categories/upload/templates/config.yaml +28 -2
- synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py +310 -0
- synapse_sdk/plugins/categories/upload/templates/plugin/upload.py +82 -20
- synapse_sdk/plugins/models.py +111 -9
- synapse_sdk/plugins/templates/plugin-config-schema.json +7 -0
- synapse_sdk/plugins/templates/schema.json +7 -0
- synapse_sdk/plugins/utils/__init__.py +3 -0
- synapse_sdk/plugins/utils/ray_gcs.py +66 -0
- synapse_sdk/shared/__init__.py +25 -0
- synapse_sdk/utils/converters/dm/__init__.py +42 -41
- synapse_sdk/utils/converters/dm/base.py +137 -0
- synapse_sdk/utils/converters/dm/from_v1.py +208 -562
- synapse_sdk/utils/converters/dm/to_v1.py +258 -304
- synapse_sdk/utils/converters/dm/tools/__init__.py +214 -0
- synapse_sdk/utils/converters/dm/tools/answer.py +95 -0
- synapse_sdk/utils/converters/dm/tools/bounding_box.py +132 -0
- synapse_sdk/utils/converters/dm/tools/bounding_box_3d.py +121 -0
- synapse_sdk/utils/converters/dm/tools/classification.py +75 -0
- synapse_sdk/utils/converters/dm/tools/keypoint.py +117 -0
- synapse_sdk/utils/converters/dm/tools/named_entity.py +111 -0
- synapse_sdk/utils/converters/dm/tools/polygon.py +122 -0
- synapse_sdk/utils/converters/dm/tools/polyline.py +124 -0
- synapse_sdk/utils/converters/dm/tools/prompt.py +94 -0
- synapse_sdk/utils/converters/dm/tools/relation.py +86 -0
- synapse_sdk/utils/converters/dm/tools/segmentation.py +141 -0
- synapse_sdk/utils/converters/dm/tools/segmentation_3d.py +83 -0
- synapse_sdk/utils/converters/dm/types.py +168 -0
- synapse_sdk/utils/converters/dm/utils.py +162 -0
- synapse_sdk/utils/converters/dm_legacy/__init__.py +56 -0
- synapse_sdk/utils/converters/dm_legacy/from_v1.py +627 -0
- synapse_sdk/utils/converters/dm_legacy/to_v1.py +367 -0
- synapse_sdk/utils/file/__init__.py +58 -0
- synapse_sdk/utils/file/archive.py +32 -0
- synapse_sdk/utils/file/checksum.py +56 -0
- synapse_sdk/utils/file/chunking.py +31 -0
- synapse_sdk/utils/file/download.py +385 -0
- synapse_sdk/utils/file/encoding.py +40 -0
- synapse_sdk/utils/file/io.py +22 -0
- synapse_sdk/utils/file/upload.py +165 -0
- synapse_sdk/utils/file/video/__init__.py +29 -0
- synapse_sdk/utils/file/video/transcode.py +307 -0
- synapse_sdk/utils/{file.py → file.py.backup} +77 -0
- synapse_sdk/utils/network.py +272 -0
- synapse_sdk/utils/storage/__init__.py +6 -2
- synapse_sdk/utils/storage/providers/file_system.py +6 -0
- {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/METADATA +19 -2
- {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/RECORD +134 -74
- synapse_sdk/devtools/docs/.gitignore +0 -20
- synapse_sdk/devtools/docs/README.md +0 -41
- synapse_sdk/devtools/docs/blog/2019-05-28-first-blog-post.md +0 -12
- synapse_sdk/devtools/docs/blog/2019-05-29-long-blog-post.md +0 -44
- synapse_sdk/devtools/docs/blog/2021-08-01-mdx-blog-post.mdx +0 -24
- synapse_sdk/devtools/docs/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg +0 -0
- synapse_sdk/devtools/docs/blog/2021-08-26-welcome/index.md +0 -29
- synapse_sdk/devtools/docs/blog/authors.yml +0 -25
- synapse_sdk/devtools/docs/blog/tags.yml +0 -19
- synapse_sdk/devtools/docs/docusaurus.config.ts +0 -138
- synapse_sdk/devtools/docs/package-lock.json +0 -17455
- synapse_sdk/devtools/docs/package.json +0 -47
- synapse_sdk/devtools/docs/sidebars.ts +0 -44
- synapse_sdk/devtools/docs/src/components/HomepageFeatures/index.tsx +0 -71
- synapse_sdk/devtools/docs/src/components/HomepageFeatures/styles.module.css +0 -11
- synapse_sdk/devtools/docs/src/css/custom.css +0 -30
- synapse_sdk/devtools/docs/src/pages/index.module.css +0 -23
- synapse_sdk/devtools/docs/src/pages/index.tsx +0 -21
- synapse_sdk/devtools/docs/src/pages/markdown-page.md +0 -7
- synapse_sdk/devtools/docs/static/.nojekyll +0 -0
- synapse_sdk/devtools/docs/static/img/docusaurus-social-card.jpg +0 -0
- synapse_sdk/devtools/docs/static/img/docusaurus.png +0 -0
- synapse_sdk/devtools/docs/static/img/favicon.ico +0 -0
- synapse_sdk/devtools/docs/static/img/logo.png +0 -0
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_mountain.svg +0 -171
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_react.svg +0 -170
- synapse_sdk/devtools/docs/static/img/undraw_docusaurus_tree.svg +0 -40
- synapse_sdk/devtools/docs/tsconfig.json +0 -8
- synapse_sdk/plugins/categories/export/actions/export.py +0 -346
- synapse_sdk/plugins/categories/export/enums.py +0 -7
- synapse_sdk/plugins/categories/neural_net/actions/gradio.py +0 -151
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py +0 -943
- synapse_sdk/plugins/categories/upload/actions/upload.py +0 -954
- {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/WHEEL +0 -0
- {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/entry_points.txt +0 -0
- {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/licenses/LICENSE +0 -0
- {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/top_level.txt +0 -0
|
@@ -1,943 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from datetime import datetime
|
|
3
|
-
from enum import Enum
|
|
4
|
-
from typing import Annotated, Any, Dict, List, Optional, Tuple
|
|
5
|
-
|
|
6
|
-
import requests
|
|
7
|
-
from pydantic import AfterValidator, BaseModel, field_validator
|
|
8
|
-
from pydantic_core import PydanticCustomError
|
|
9
|
-
|
|
10
|
-
from synapse_sdk.clients.backend import BackendClient
|
|
11
|
-
from synapse_sdk.clients.backend.models import JobStatus
|
|
12
|
-
from synapse_sdk.clients.exceptions import ClientError
|
|
13
|
-
from synapse_sdk.plugins.categories.base import Action
|
|
14
|
-
from synapse_sdk.plugins.categories.decorators import register_action
|
|
15
|
-
from synapse_sdk.plugins.enums import PluginCategory, RunMethod
|
|
16
|
-
from synapse_sdk.plugins.models import Run
|
|
17
|
-
from synapse_sdk.shared.enums import Context
|
|
18
|
-
from synapse_sdk.utils.pydantic.validators import non_blank
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
class AnnotationMethod(str, Enum):
|
|
22
|
-
FILE = 'file'
|
|
23
|
-
INFERENCE = 'inference'
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
class AnnotateTaskDataStatus(str, Enum):
|
|
27
|
-
SUCCESS = 'success'
|
|
28
|
-
FAILED = 'failed'
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class CriticalError(Exception):
|
|
32
|
-
"""Critical error."""
|
|
33
|
-
|
|
34
|
-
def __init__(self, message: str = 'Critical error occured while processing task'):
|
|
35
|
-
self.message = message
|
|
36
|
-
super().__init__(self.message)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
class PreAnnotationToTaskFailed(Exception):
|
|
40
|
-
"""Pre-annotation to task failed."""
|
|
41
|
-
|
|
42
|
-
def __init__(self, message: str = 'Pre-annotation to task failed'):
|
|
43
|
-
self.message = message
|
|
44
|
-
super().__init__(self.message)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class ToTaskRun(Run):
|
|
48
|
-
class AnnotateTaskEventLog(BaseModel):
|
|
49
|
-
"""Annotate task event log model."""
|
|
50
|
-
|
|
51
|
-
info: Optional[str] = None
|
|
52
|
-
status: Context
|
|
53
|
-
created: str
|
|
54
|
-
|
|
55
|
-
class AnnotateTaskDataLog(BaseModel):
|
|
56
|
-
"""Log model for annotate task data."""
|
|
57
|
-
|
|
58
|
-
task_info: Optional[str] = None
|
|
59
|
-
status: AnnotateTaskDataStatus
|
|
60
|
-
created: str
|
|
61
|
-
|
|
62
|
-
class MetricsRecord(BaseModel):
|
|
63
|
-
"""Metrics record model."""
|
|
64
|
-
|
|
65
|
-
stand_by: int
|
|
66
|
-
failed: int
|
|
67
|
-
success: int
|
|
68
|
-
|
|
69
|
-
LOG_MESSAGES = {
|
|
70
|
-
'INVALID_PROJECT_RESPONSE': {
|
|
71
|
-
'message': 'Invalid project response received.',
|
|
72
|
-
'level': Context.DANGER,
|
|
73
|
-
},
|
|
74
|
-
'NO_DATA_COLLECTION': {
|
|
75
|
-
'message': 'Project does not have a data collection.',
|
|
76
|
-
'level': Context.DANGER,
|
|
77
|
-
},
|
|
78
|
-
'INVALID_DATA_COLLECTION_RESPONSE': {
|
|
79
|
-
'message': 'Invalid data collection response received.',
|
|
80
|
-
'level': Context.DANGER,
|
|
81
|
-
},
|
|
82
|
-
'NO_TASKS_FOUND': {
|
|
83
|
-
'message': 'Tasks to annotate not found.',
|
|
84
|
-
'level': Context.WARNING,
|
|
85
|
-
},
|
|
86
|
-
'TARGET_SPEC_REQUIRED': {
|
|
87
|
-
'message': 'Target specification name is required for file annotation method.',
|
|
88
|
-
'level': Context.DANGER,
|
|
89
|
-
},
|
|
90
|
-
'TARGET_SPEC_NOT_FOUND': {
|
|
91
|
-
'message': 'Target specification name "{}" not found in file specifications',
|
|
92
|
-
'level': Context.DANGER,
|
|
93
|
-
},
|
|
94
|
-
'UNSUPPORTED_METHOD': {
|
|
95
|
-
'message': 'Unsupported annotation method: {}',
|
|
96
|
-
'level': Context.DANGER,
|
|
97
|
-
},
|
|
98
|
-
'ANNOTATING_DATA': {
|
|
99
|
-
'message': 'Annotating data to tasks...',
|
|
100
|
-
'level': None,
|
|
101
|
-
},
|
|
102
|
-
'CRITICAL_ERROR': {
|
|
103
|
-
'message': 'Critical error occured while processing task. Stopping the job.',
|
|
104
|
-
'level': Context.DANGER,
|
|
105
|
-
},
|
|
106
|
-
'TASK_PROCESSING_FAILED': {
|
|
107
|
-
'message': 'Failed to process task {}: {}',
|
|
108
|
-
'level': Context.DANGER,
|
|
109
|
-
},
|
|
110
|
-
'ANNOTATION_COMPLETED': {
|
|
111
|
-
'message': 'Annotation completed. Success: {}, Failed: {}',
|
|
112
|
-
'level': None,
|
|
113
|
-
},
|
|
114
|
-
'INVALID_TASK_RESPONSE': {
|
|
115
|
-
'message': 'Invalid task response received for task {}',
|
|
116
|
-
'level': Context.DANGER,
|
|
117
|
-
},
|
|
118
|
-
'TARGET_SPEC_REQUIRED_FOR_TASK': {
|
|
119
|
-
'message': 'Target specification name is required for file annotation method for task {}',
|
|
120
|
-
'level': Context.DANGER,
|
|
121
|
-
},
|
|
122
|
-
'UNSUPPORTED_METHOD_FOR_TASK': {
|
|
123
|
-
'message': 'Unsupported annotation method: {} for task {}',
|
|
124
|
-
'level': Context.DANGER,
|
|
125
|
-
},
|
|
126
|
-
'PRIMARY_IMAGE_URL_NOT_FOUND': {
|
|
127
|
-
'message': 'Primary image URL not found in task data for task {}',
|
|
128
|
-
'level': Context.DANGER,
|
|
129
|
-
},
|
|
130
|
-
'FILE_SPEC_NOT_FOUND': {
|
|
131
|
-
'message': 'File specification not found for task {}',
|
|
132
|
-
'level': Context.DANGER,
|
|
133
|
-
},
|
|
134
|
-
'FILE_ORIGINAL_NAME_NOT_FOUND': {
|
|
135
|
-
'message': 'File original name not found for task {}',
|
|
136
|
-
'level': Context.DANGER,
|
|
137
|
-
},
|
|
138
|
-
'URL_NOT_FOUND': {
|
|
139
|
-
'message': 'URL not found for task {}',
|
|
140
|
-
'level': Context.DANGER,
|
|
141
|
-
},
|
|
142
|
-
'FETCH_DATA_FAILED': {
|
|
143
|
-
'message': 'Failed to fetch data from URL: {} for task {}',
|
|
144
|
-
'level': Context.DANGER,
|
|
145
|
-
},
|
|
146
|
-
'CONVERT_DATA_FAILED': {
|
|
147
|
-
'message': 'Failed to convert data to task object: {} for task {}',
|
|
148
|
-
'level': Context.DANGER,
|
|
149
|
-
},
|
|
150
|
-
'PREPROCESSOR_ID_REQUIRED': {
|
|
151
|
-
'message': 'Pre-processor ID is required for inference annotation method for task {}',
|
|
152
|
-
'level': Context.DANGER,
|
|
153
|
-
},
|
|
154
|
-
'INFERENCE_PROCESSING_FAILED': {
|
|
155
|
-
'message': 'Failed to process inference for task {}: {}',
|
|
156
|
-
'level': Context.DANGER,
|
|
157
|
-
},
|
|
158
|
-
'ANNOTATING_INFERENCE_DATA': {
|
|
159
|
-
'message': 'Annotating data to tasks using inference...',
|
|
160
|
-
'level': None,
|
|
161
|
-
},
|
|
162
|
-
'INFERENCE_ANNOTATION_COMPLETED': {
|
|
163
|
-
'message': 'Inference annotation completed. Success: {}, Failed: {}',
|
|
164
|
-
'level': None,
|
|
165
|
-
},
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
def log_message_with_code(self, code: str, *args, level: Optional[Context] = None):
|
|
169
|
-
"""Log message using predefined code and optional level override."""
|
|
170
|
-
if code not in self.LOG_MESSAGES:
|
|
171
|
-
self.log_message(f'Unknown log code: {code}')
|
|
172
|
-
return
|
|
173
|
-
|
|
174
|
-
log_config = self.LOG_MESSAGES[code]
|
|
175
|
-
message = log_config['message'].format(*args) if args else log_config['message']
|
|
176
|
-
log_level = level or log_config['level']
|
|
177
|
-
|
|
178
|
-
if log_level:
|
|
179
|
-
self.log_message(message, context=log_level.value)
|
|
180
|
-
else:
|
|
181
|
-
self.log_message(message, context=Context.INFO.value)
|
|
182
|
-
|
|
183
|
-
def log_annotate_task_event(self, code: str, *args, level: Optional[Context] = None):
|
|
184
|
-
"""Log annotate task event using predefined code."""
|
|
185
|
-
if code not in self.LOG_MESSAGES:
|
|
186
|
-
now = datetime.now().isoformat()
|
|
187
|
-
self.log(
|
|
188
|
-
'annotate_task_event',
|
|
189
|
-
self.AnnotateTaskEventLog(
|
|
190
|
-
info=f'Unknown log code: {code}', status=Context.DANGER, created=now
|
|
191
|
-
).model_dump(),
|
|
192
|
-
)
|
|
193
|
-
return
|
|
194
|
-
|
|
195
|
-
log_config = self.LOG_MESSAGES[code]
|
|
196
|
-
message = log_config['message'].format(*args) if args else log_config['message']
|
|
197
|
-
log_level = level or log_config['level'] or Context.INFO
|
|
198
|
-
|
|
199
|
-
now = datetime.now().isoformat()
|
|
200
|
-
self.log(
|
|
201
|
-
'annotate_task_event',
|
|
202
|
-
self.AnnotateTaskEventLog(info=message, status=log_level, created=now).model_dump(),
|
|
203
|
-
)
|
|
204
|
-
|
|
205
|
-
def log_annotate_task_data(self, task_info: Dict[str, Any], status: AnnotateTaskDataStatus):
|
|
206
|
-
"""Log annotate task data."""
|
|
207
|
-
now = datetime.now().isoformat()
|
|
208
|
-
self.log(
|
|
209
|
-
'annotate_task_data',
|
|
210
|
-
self.AnnotateTaskDataLog(task_info=json.dumps(task_info), status=status, created=now).model_dump(),
|
|
211
|
-
)
|
|
212
|
-
|
|
213
|
-
def log_metrics(self, record: MetricsRecord, category: str):
|
|
214
|
-
"""Log FileToTask metrics.
|
|
215
|
-
|
|
216
|
-
Args:
|
|
217
|
-
record (MetricsRecord): The metrics record to log.
|
|
218
|
-
category (str): The category of the metrics.
|
|
219
|
-
"""
|
|
220
|
-
record = self.MetricsRecord.model_validate(record)
|
|
221
|
-
self.set_metrics(value=record.model_dump(), category=category)
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
class ToTaskParams(BaseModel):
|
|
225
|
-
"""ToTask action parameters.
|
|
226
|
-
|
|
227
|
-
Args:
|
|
228
|
-
name (str): The name of the action.
|
|
229
|
-
description (str | None): The description of the action.
|
|
230
|
-
project (int): The project ID.
|
|
231
|
-
agent (int): The agent ID.
|
|
232
|
-
task_filters (dict): The filters of tasks.
|
|
233
|
-
method (AnnotationMethod): The method of annotation.
|
|
234
|
-
target_specification_name (str | None): The name of the target specification.
|
|
235
|
-
model (int): The model ID.
|
|
236
|
-
pre_processor (int | None): The pre processor ID.
|
|
237
|
-
pre_processor_params (dict): The params of the pre processor.
|
|
238
|
-
"""
|
|
239
|
-
|
|
240
|
-
name: Annotated[str, AfterValidator(non_blank)]
|
|
241
|
-
description: Optional[str] = None
|
|
242
|
-
project: int
|
|
243
|
-
agent: int
|
|
244
|
-
task_filters: Dict[str, Any]
|
|
245
|
-
method: Optional[AnnotationMethod] = None
|
|
246
|
-
target_specification_name: Optional[str] = None
|
|
247
|
-
model: Optional[int] = None
|
|
248
|
-
pre_processor: Optional[int] = None
|
|
249
|
-
pre_processor_params: Dict[str, Any]
|
|
250
|
-
|
|
251
|
-
@field_validator('project', mode='before')
|
|
252
|
-
@classmethod
|
|
253
|
-
def check_project_exists(cls, value: int, info) -> int:
|
|
254
|
-
"""Validate synapse-backend project exists."""
|
|
255
|
-
if not value:
|
|
256
|
-
return value
|
|
257
|
-
|
|
258
|
-
action = info.context['action']
|
|
259
|
-
client = action.client
|
|
260
|
-
try:
|
|
261
|
-
client.get_project(value)
|
|
262
|
-
except ClientError:
|
|
263
|
-
raise PydanticCustomError('client_error', 'Error occurred while checking project exists.')
|
|
264
|
-
return value
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
class ToTaskResult(BaseModel):
|
|
268
|
-
"""Result model for ToTaskAction.start method.
|
|
269
|
-
|
|
270
|
-
Args:
|
|
271
|
-
status (JobStatus): The job status from the action execution.
|
|
272
|
-
message (str): A descriptive message about the action result.
|
|
273
|
-
"""
|
|
274
|
-
|
|
275
|
-
status: JobStatus
|
|
276
|
-
message: str
|
|
277
|
-
|
|
278
|
-
def model_dump(self, **kwargs):
|
|
279
|
-
"""Override model_dump to return status as enum value."""
|
|
280
|
-
data = super().model_dump(**kwargs)
|
|
281
|
-
if 'status' in data and isinstance(data['status'], JobStatus):
|
|
282
|
-
data['status'] = data['status'].value
|
|
283
|
-
return data
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
@register_action
|
|
287
|
-
class ToTaskAction(Action):
|
|
288
|
-
"""ToTask action for pre-annotation data processing.
|
|
289
|
-
|
|
290
|
-
This action handles the process of annotating data to tasks in a project. It supports
|
|
291
|
-
two annotation methods: file-based annotation and inference-based annotation.
|
|
292
|
-
|
|
293
|
-
File-based annotation fetches data from file URLs specified in task data units,
|
|
294
|
-
downloads and processes JSON data, and updates task data with the processed information.
|
|
295
|
-
It also validates target specification names against file specifications.
|
|
296
|
-
|
|
297
|
-
Inference-based annotation is currently not supported but will support model inference
|
|
298
|
-
for automatic data annotation in future implementations.
|
|
299
|
-
|
|
300
|
-
Attrs:
|
|
301
|
-
name (str): Action name, set to 'to_task'.
|
|
302
|
-
category (PluginCategory): Plugin category, set to PRE_ANNOTATION.
|
|
303
|
-
method (RunMethod): Execution method, set to JOB.
|
|
304
|
-
run_class (Type[ToTaskRun]): Run class for this action.
|
|
305
|
-
params_model (Type[ToTaskParams]): Parameter validation model.
|
|
306
|
-
progress_categories (Dict): Progress tracking configuration.
|
|
307
|
-
metrics_categories (Set[str]): Metrics categories for this action.
|
|
308
|
-
|
|
309
|
-
Note:
|
|
310
|
-
This action requires a valid project with an associated data collection.
|
|
311
|
-
For file-based annotation, the target_specification_name must exist in the
|
|
312
|
-
project's file specifications.
|
|
313
|
-
|
|
314
|
-
Raises:
|
|
315
|
-
ValueError: If run instance or parameters are not properly initialized.
|
|
316
|
-
ClientError: If there are issues with backend API calls.
|
|
317
|
-
"""
|
|
318
|
-
|
|
319
|
-
name = 'to_task'
|
|
320
|
-
category = PluginCategory.PRE_ANNOTATION
|
|
321
|
-
method = RunMethod.JOB
|
|
322
|
-
run_class = ToTaskRun
|
|
323
|
-
params_model = ToTaskParams
|
|
324
|
-
progress_categories = {
|
|
325
|
-
'annotate_task_data': {
|
|
326
|
-
'proportion': 100,
|
|
327
|
-
},
|
|
328
|
-
}
|
|
329
|
-
metrics_categories = {
|
|
330
|
-
'annotate_task_data': {
|
|
331
|
-
'stand_by': 0,
|
|
332
|
-
'failed': 0,
|
|
333
|
-
'success': 0,
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
def start(self) -> dict:
|
|
338
|
-
"""Start to_task action.
|
|
339
|
-
|
|
340
|
-
* Generate tasks.
|
|
341
|
-
* Annotate data to tasks.
|
|
342
|
-
|
|
343
|
-
Returns:
|
|
344
|
-
dict: Validated result with status and message.
|
|
345
|
-
"""
|
|
346
|
-
if not self.run or not self.params:
|
|
347
|
-
result = ToTaskResult(
|
|
348
|
-
status=JobStatus.FAILED, message='Run instance or parameters not properly initialized'
|
|
349
|
-
)
|
|
350
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
351
|
-
|
|
352
|
-
# Type assertion to help the linter
|
|
353
|
-
assert isinstance(self.run, ToTaskRun)
|
|
354
|
-
assert isinstance(self.run.client, BackendClient)
|
|
355
|
-
|
|
356
|
-
client: BackendClient = self.run.client
|
|
357
|
-
project_id = self.params['project']
|
|
358
|
-
project_response = client.get_project(project_id)
|
|
359
|
-
if isinstance(project_response, str):
|
|
360
|
-
self.run.log_message_with_code('INVALID_PROJECT_RESPONSE')
|
|
361
|
-
self.run.end_log()
|
|
362
|
-
result = ToTaskResult(status=JobStatus.FAILED, message='Invalid project response received')
|
|
363
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
364
|
-
project: Dict[str, Any] = project_response
|
|
365
|
-
|
|
366
|
-
data_collection_id = project.get('data_collection')
|
|
367
|
-
if not data_collection_id:
|
|
368
|
-
self.run.log_message_with_code('NO_DATA_COLLECTION')
|
|
369
|
-
self.run.end_log()
|
|
370
|
-
result = ToTaskResult(status=JobStatus.FAILED, message='Project does not have a data collection')
|
|
371
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
372
|
-
|
|
373
|
-
data_collection_response = client.get_data_collection(data_collection_id)
|
|
374
|
-
if isinstance(data_collection_response, str):
|
|
375
|
-
self.run.log_message_with_code('INVALID_DATA_COLLECTION_RESPONSE')
|
|
376
|
-
self.run.end_log()
|
|
377
|
-
result = ToTaskResult(status=JobStatus.FAILED, message='Invalid data collection response received')
|
|
378
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
379
|
-
data_collection: Dict[str, Any] = data_collection_response
|
|
380
|
-
|
|
381
|
-
# Generate tasks if provided project is empty.
|
|
382
|
-
task_ids_query_params = {
|
|
383
|
-
'project': self.params['project'],
|
|
384
|
-
'fields': 'id',
|
|
385
|
-
}
|
|
386
|
-
if self.params['task_filters']:
|
|
387
|
-
task_ids_query_params.update(self.params['task_filters'])
|
|
388
|
-
task_ids_generator, task_ids_count = client.list_tasks(params=task_ids_query_params, list_all=True)
|
|
389
|
-
task_ids = [int(item.get('id', 0)) for item in task_ids_generator if isinstance(item, dict) and item.get('id')]
|
|
390
|
-
|
|
391
|
-
# If no tasks found, break the job.
|
|
392
|
-
if not task_ids_count:
|
|
393
|
-
self.run.log_message_with_code('NO_TASKS_FOUND')
|
|
394
|
-
self.run.end_log()
|
|
395
|
-
result = ToTaskResult(status=JobStatus.FAILED, message='No tasks found to annotate')
|
|
396
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
397
|
-
|
|
398
|
-
# Annotate data to tasks.
|
|
399
|
-
method = self.params.get('method')
|
|
400
|
-
if method == AnnotationMethod.FILE:
|
|
401
|
-
# Check if target specification name exists in file specifications.
|
|
402
|
-
target_specification_name = self.params.get('target_specification_name')
|
|
403
|
-
if not target_specification_name:
|
|
404
|
-
self.run.log_message_with_code('TARGET_SPEC_REQUIRED')
|
|
405
|
-
self.run.end_log()
|
|
406
|
-
result = ToTaskResult(
|
|
407
|
-
status=JobStatus.FAILED, message='Target specification name is required for file annotation method'
|
|
408
|
-
)
|
|
409
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
410
|
-
|
|
411
|
-
file_specifications = data_collection.get('file_specifications', [])
|
|
412
|
-
target_spec_exists = any(spec.get('name') == target_specification_name for spec in file_specifications)
|
|
413
|
-
if not target_spec_exists:
|
|
414
|
-
self.run.log_message_with_code('TARGET_SPEC_NOT_FOUND', target_specification_name)
|
|
415
|
-
self.run.end_log()
|
|
416
|
-
result = ToTaskResult(
|
|
417
|
-
status=JobStatus.FAILED,
|
|
418
|
-
message=f"Target specification name '{target_specification_name}' not found in file specifications",
|
|
419
|
-
)
|
|
420
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
421
|
-
self._handle_annotate_data_from_files(task_ids, target_specification_name)
|
|
422
|
-
elif method == AnnotationMethod.INFERENCE:
|
|
423
|
-
self._handle_annotate_data_with_inference(task_ids)
|
|
424
|
-
else:
|
|
425
|
-
self.run.log_message_with_code('UNSUPPORTED_METHOD', method)
|
|
426
|
-
self.run.end_log()
|
|
427
|
-
result = ToTaskResult(status=JobStatus.FAILED, message=f'Unsupported annotation method: {method}')
|
|
428
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
429
|
-
|
|
430
|
-
current_progress = self.run.logger.get_current_progress()
|
|
431
|
-
if current_progress['overall'] != 100:
|
|
432
|
-
result = ToTaskResult(
|
|
433
|
-
status=JobStatus.FAILED, message='Pre-annotation to task failed. Current progress is not 100%'
|
|
434
|
-
)
|
|
435
|
-
raise PreAnnotationToTaskFailed(result.message)
|
|
436
|
-
|
|
437
|
-
result = ToTaskResult(status=JobStatus.SUCCEEDED, message='Pre-annotation to task completed successfully')
|
|
438
|
-
return result.model_dump()
|
|
439
|
-
|
|
440
|
-
def _handle_annotate_data_from_files(self, task_ids: List[int], target_specification_name: str):
|
|
441
|
-
"""Handle annotate data from files to tasks.
|
|
442
|
-
|
|
443
|
-
Args:
|
|
444
|
-
task_ids (List[int]): List of task IDs to annotate data to.
|
|
445
|
-
target_specification_name (str): The name of the target specification.
|
|
446
|
-
"""
|
|
447
|
-
if not self.run or not self.params:
|
|
448
|
-
raise ValueError('Run instance or parameters not properly initialized')
|
|
449
|
-
|
|
450
|
-
# Type assertion to help the linter
|
|
451
|
-
assert isinstance(self.run, ToTaskRun)
|
|
452
|
-
assert isinstance(self.run.client, BackendClient)
|
|
453
|
-
|
|
454
|
-
client: BackendClient = self.run.client
|
|
455
|
-
task_params = {
|
|
456
|
-
'fields': 'id,data,data_unit',
|
|
457
|
-
'expand': 'data_unit',
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
total_tasks = len(task_ids)
|
|
461
|
-
success_count = 0
|
|
462
|
-
failed_count = 0
|
|
463
|
-
current_progress = 0
|
|
464
|
-
|
|
465
|
-
# Initialize metrics and progress
|
|
466
|
-
self._update_metrics(total_tasks, success_count, failed_count)
|
|
467
|
-
self.run.set_progress(0, total_tasks, category='annotate_task_data')
|
|
468
|
-
self.run.log_message_with_code('ANNOTATING_DATA')
|
|
469
|
-
|
|
470
|
-
# Process each task
|
|
471
|
-
for task_id in task_ids:
|
|
472
|
-
try:
|
|
473
|
-
result = self._process_single_task(
|
|
474
|
-
client, task_id, task_params, target_specification_name, AnnotationMethod.FILE
|
|
475
|
-
)
|
|
476
|
-
if result['success']:
|
|
477
|
-
success_count += 1
|
|
478
|
-
else:
|
|
479
|
-
failed_count += 1
|
|
480
|
-
|
|
481
|
-
current_progress += 1
|
|
482
|
-
self._update_metrics(total_tasks, success_count, failed_count)
|
|
483
|
-
self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
|
|
484
|
-
|
|
485
|
-
except CriticalError:
|
|
486
|
-
self.run.log_message_with_code('CRITICAL_ERROR')
|
|
487
|
-
return
|
|
488
|
-
|
|
489
|
-
except Exception as e:
|
|
490
|
-
self.run.log_annotate_task_event('TASK_PROCESSING_FAILED', task_id, str(e))
|
|
491
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': str(e)}, AnnotateTaskDataStatus.FAILED)
|
|
492
|
-
failed_count += 1
|
|
493
|
-
current_progress += 1
|
|
494
|
-
self._update_metrics(total_tasks, success_count, failed_count)
|
|
495
|
-
self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
|
|
496
|
-
|
|
497
|
-
# Finalize progress
|
|
498
|
-
self.run.set_progress(total_tasks, total_tasks, category='annotate_task_data')
|
|
499
|
-
self.run.log_message_with_code('ANNOTATION_COMPLETED', success_count, failed_count)
|
|
500
|
-
|
|
501
|
-
def _process_single_task(
|
|
502
|
-
self,
|
|
503
|
-
client: BackendClient,
|
|
504
|
-
task_id: int,
|
|
505
|
-
task_params: Dict[str, Any],
|
|
506
|
-
target_specification_name: Optional[str],
|
|
507
|
-
method: AnnotationMethod,
|
|
508
|
-
) -> Dict[str, Any]:
|
|
509
|
-
"""Process a single task for annotation.
|
|
510
|
-
|
|
511
|
-
Args:
|
|
512
|
-
client (BackendClient): The backend client instance.
|
|
513
|
-
task_id (int): The task ID to process.
|
|
514
|
-
task_params (Dict[str, Any]): Parameters for getting task data.
|
|
515
|
-
target_specification_name (Optional[str]): The name of the target specification.
|
|
516
|
-
method (AnnotationMethod): The annotation method to use.
|
|
517
|
-
|
|
518
|
-
Returns:
|
|
519
|
-
Dict[str, Any]: Result dictionary with 'success' boolean and optional 'error' message.
|
|
520
|
-
"""
|
|
521
|
-
if not self.run:
|
|
522
|
-
raise ValueError('Run instance not properly initialized')
|
|
523
|
-
|
|
524
|
-
# Type assertion to help the linter
|
|
525
|
-
assert isinstance(self.run, ToTaskRun)
|
|
526
|
-
|
|
527
|
-
# Get task data
|
|
528
|
-
task_response = client.get_task(task_id, params=task_params)
|
|
529
|
-
if isinstance(task_response, str):
|
|
530
|
-
error_msg = 'Invalid task response'
|
|
531
|
-
self.run.log_annotate_task_event('INVALID_TASK_RESPONSE', task_id)
|
|
532
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
533
|
-
return {'success': False, 'error': error_msg}
|
|
534
|
-
|
|
535
|
-
task: Dict[str, Any] = task_response
|
|
536
|
-
|
|
537
|
-
if method == AnnotationMethod.FILE:
|
|
538
|
-
if not target_specification_name:
|
|
539
|
-
error_msg = 'Target specification name is required for file annotation method'
|
|
540
|
-
self.run.log_message_with_code('TARGET_SPEC_REQUIRED_FOR_TASK', task_id)
|
|
541
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
542
|
-
return {'success': False, 'error': error_msg}
|
|
543
|
-
return self._process_single_task_with_file(client, task_id, task, target_specification_name)
|
|
544
|
-
elif method == AnnotationMethod.INFERENCE:
|
|
545
|
-
return self._process_single_task_with_inference(client, task_id, task)
|
|
546
|
-
else:
|
|
547
|
-
error_msg = f'Unsupported annotation method: {method}'
|
|
548
|
-
self.run.log_annotate_task_event('UNSUPPORTED_METHOD_FOR_TASK', method, task_id)
|
|
549
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
550
|
-
return {'success': False, 'error': error_msg}
|
|
551
|
-
|
|
552
|
-
def _process_single_task_with_file(
|
|
553
|
-
self, client: BackendClient, task_id: int, task: Dict[str, Any], target_specification_name: str
|
|
554
|
-
) -> Dict[str, Any]:
|
|
555
|
-
"""Process a single task for file-based annotation.
|
|
556
|
-
|
|
557
|
-
Args:
|
|
558
|
-
client (BackendClient): The backend client instance.
|
|
559
|
-
task_id (int): The task ID to process.
|
|
560
|
-
task (Dict[str, Any]): The task data.
|
|
561
|
-
target_specification_name (str): The name of the target specification.
|
|
562
|
-
|
|
563
|
-
Returns:
|
|
564
|
-
Dict[str, Any]: Result dictionary with 'success' boolean and optional 'error' message.
|
|
565
|
-
"""
|
|
566
|
-
if not self.run:
|
|
567
|
-
raise ValueError('Run instance not properly initialized')
|
|
568
|
-
|
|
569
|
-
# Type assertion to help the linter
|
|
570
|
-
assert isinstance(self.run, ToTaskRun)
|
|
571
|
-
|
|
572
|
-
# Extract data file information
|
|
573
|
-
data_unit = task.get('data_unit', {})
|
|
574
|
-
files = data_unit.get('files', {})
|
|
575
|
-
data_file = files.get(target_specification_name)
|
|
576
|
-
|
|
577
|
-
# Extract primary file URL from task data
|
|
578
|
-
primary_file_url, primary_file_original_name = self._extract_primary_file_url(task)
|
|
579
|
-
if not primary_file_url:
|
|
580
|
-
error_msg = 'Primary image URL not found in task data'
|
|
581
|
-
self.run.log_annotate_task_event('PRIMARY_IMAGE_URL_NOT_FOUND', task_id)
|
|
582
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
583
|
-
return {'success': False, 'error': error_msg}
|
|
584
|
-
|
|
585
|
-
if not data_file:
|
|
586
|
-
error_msg = 'File specification not found'
|
|
587
|
-
self.run.log_annotate_task_event('FILE_SPEC_NOT_FOUND', task_id)
|
|
588
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
589
|
-
return {'success': False, 'error': error_msg}
|
|
590
|
-
|
|
591
|
-
data_file_url = data_file.get('url')
|
|
592
|
-
data_file_original_name = data_file.get('file_name_original')
|
|
593
|
-
if not data_file_original_name:
|
|
594
|
-
error_msg = 'File original name not found'
|
|
595
|
-
self.run.log_annotate_task_event('FILE_ORIGINAL_NAME_NOT_FOUND', task_id)
|
|
596
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
597
|
-
return {'success': False, 'error': error_msg}
|
|
598
|
-
|
|
599
|
-
if not data_file_url:
|
|
600
|
-
error_msg = 'URL not found'
|
|
601
|
-
self.run.log_annotate_task_event('URL_NOT_FOUND', task_id)
|
|
602
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
603
|
-
return {'success': False, 'error': error_msg}
|
|
604
|
-
|
|
605
|
-
# Fetch and process the data
|
|
606
|
-
try:
|
|
607
|
-
# Convert data to task object
|
|
608
|
-
annotation_to_task = self.entrypoint(self.run)
|
|
609
|
-
converted_data = annotation_to_task.convert_data_from_file(
|
|
610
|
-
primary_file_url, primary_file_original_name, data_file_url, data_file_original_name
|
|
611
|
-
)
|
|
612
|
-
except requests.RequestException as e:
|
|
613
|
-
error_msg = f'Failed to fetch data from URL: {str(e)}'
|
|
614
|
-
self.run.log_annotate_task_event('FETCH_DATA_FAILED', str(e), task_id)
|
|
615
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
616
|
-
return {'success': False, 'error': error_msg}
|
|
617
|
-
except Exception as e:
|
|
618
|
-
error_msg = f'Failed to convert data to task object: {str(e)}'
|
|
619
|
-
self.run.log_annotate_task_event('CONVERT_DATA_FAILED', str(e), task_id)
|
|
620
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
621
|
-
return {'success': False, 'error': error_msg}
|
|
622
|
-
|
|
623
|
-
# Submit annotation data
|
|
624
|
-
client.annotate_task_data(task_id, data={'action': 'submit', 'data': converted_data})
|
|
625
|
-
|
|
626
|
-
# Log success
|
|
627
|
-
self.run.log_annotate_task_data({'task_id': task_id}, AnnotateTaskDataStatus.SUCCESS)
|
|
628
|
-
return {'success': True}
|
|
629
|
-
|
|
630
|
-
def _process_single_task_with_inference(
|
|
631
|
-
self, client: BackendClient, task_id: int, task: Dict[str, Any]
|
|
632
|
-
) -> Dict[str, Any]:
|
|
633
|
-
"""Process a single task for inference-based annotation.
|
|
634
|
-
|
|
635
|
-
Args:
|
|
636
|
-
client (BackendClient): The backend client instance.
|
|
637
|
-
task_id (int): The task ID to process.
|
|
638
|
-
task (Dict[str, Any]): The task data.
|
|
639
|
-
|
|
640
|
-
Returns:
|
|
641
|
-
Dict[str, Any]: Result dictionary with 'success' boolean and optional 'error' message.
|
|
642
|
-
"""
|
|
643
|
-
if not self.run or not self.params:
|
|
644
|
-
raise ValueError('Run instance or parameters not properly initialized')
|
|
645
|
-
|
|
646
|
-
# Type assertion to help the linter
|
|
647
|
-
assert isinstance(self.run, ToTaskRun)
|
|
648
|
-
|
|
649
|
-
try:
|
|
650
|
-
# Validate pre-processor ID
|
|
651
|
-
pre_processor_id = self.params.get('pre_processor')
|
|
652
|
-
if not pre_processor_id:
|
|
653
|
-
error_msg = 'Pre-processor ID is required for inference annotation method'
|
|
654
|
-
self.run.log_message_with_code('PREPROCESSOR_ID_REQUIRED', task_id)
|
|
655
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
656
|
-
return {'success': False, 'error': error_msg}
|
|
657
|
-
|
|
658
|
-
# Get pre-processor information
|
|
659
|
-
pre_processor_info = self._get_pre_processor_info(client, pre_processor_id)
|
|
660
|
-
if not pre_processor_info['success']:
|
|
661
|
-
return pre_processor_info
|
|
662
|
-
|
|
663
|
-
pre_processor_code = pre_processor_info['code']
|
|
664
|
-
pre_processor_version = pre_processor_info['version']
|
|
665
|
-
|
|
666
|
-
# Ensure pre-processor is running
|
|
667
|
-
pre_processor_status = self._ensure_pre_processor_running(client, pre_processor_code)
|
|
668
|
-
if not pre_processor_status['success']:
|
|
669
|
-
return pre_processor_status
|
|
670
|
-
|
|
671
|
-
# Extract primary file URL from task data
|
|
672
|
-
primary_file_url, _ = self._extract_primary_file_url(task)
|
|
673
|
-
if not primary_file_url:
|
|
674
|
-
error_msg = 'Primary image URL not found in task data'
|
|
675
|
-
self.run.log_annotate_task_event('PRIMARY_IMAGE_URL_NOT_FOUND', task_id)
|
|
676
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
677
|
-
return {'success': False, 'error': error_msg}
|
|
678
|
-
|
|
679
|
-
# Run inference
|
|
680
|
-
inference_result = self._run_inference(client, pre_processor_code, pre_processor_version, primary_file_url)
|
|
681
|
-
if not inference_result['success']:
|
|
682
|
-
return inference_result
|
|
683
|
-
|
|
684
|
-
# Convert and submit inference data
|
|
685
|
-
annotation_to_task = self.entrypoint(self.run)
|
|
686
|
-
converted_result = annotation_to_task.convert_data_from_inference(inference_result['data'])
|
|
687
|
-
|
|
688
|
-
# Submit inference annotation data
|
|
689
|
-
client.annotate_task_data(task_id, data={'action': 'submit', 'data': converted_result})
|
|
690
|
-
|
|
691
|
-
# Log success
|
|
692
|
-
self.run.log_annotate_task_data(
|
|
693
|
-
{'task_id': task_id, 'pre_processor_id': pre_processor_id}, AnnotateTaskDataStatus.SUCCESS
|
|
694
|
-
)
|
|
695
|
-
return {'success': True}
|
|
696
|
-
|
|
697
|
-
except Exception as e:
|
|
698
|
-
error_msg = f'Failed to process inference for task {task_id}: {str(e)}'
|
|
699
|
-
self.run.log_message_with_code('INFERENCE_PROCESSING_FAILED', task_id, str(e))
|
|
700
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
|
|
701
|
-
return {'success': False, 'error': error_msg}
|
|
702
|
-
|
|
703
|
-
def _get_pre_processor_info(self, client: BackendClient, pre_processor_id: int) -> Dict[str, Any]:
|
|
704
|
-
"""Get pre-processor information from the backend.
|
|
705
|
-
|
|
706
|
-
Args:
|
|
707
|
-
client (BackendClient): The backend client instance.
|
|
708
|
-
pre_processor_id (int): The pre-processor ID.
|
|
709
|
-
|
|
710
|
-
Returns:
|
|
711
|
-
Dict[str, Any]: Result dictionary with pre-processor info or error.
|
|
712
|
-
"""
|
|
713
|
-
try:
|
|
714
|
-
pre_processor_response = client.get_plugin_release(pre_processor_id)
|
|
715
|
-
if isinstance(pre_processor_response, str):
|
|
716
|
-
return {'success': False, 'error': 'Invalid pre-processor response received'}
|
|
717
|
-
|
|
718
|
-
pre_processor: Dict[str, Any] = pre_processor_response
|
|
719
|
-
config = pre_processor.get('config', {})
|
|
720
|
-
code = config.get('code')
|
|
721
|
-
version = pre_processor.get('version')
|
|
722
|
-
|
|
723
|
-
if not code or not version:
|
|
724
|
-
return {'success': False, 'error': 'Invalid pre-processor configuration'}
|
|
725
|
-
|
|
726
|
-
return {'success': True, 'code': code, 'version': version}
|
|
727
|
-
except Exception as e:
|
|
728
|
-
return {'success': False, 'error': f'Failed to get pre-processor info: {str(e)}'}
|
|
729
|
-
|
|
730
|
-
def _ensure_pre_processor_running(self, client: BackendClient, pre_processor_code: str) -> Dict[str, Any]:
|
|
731
|
-
"""Ensure the pre-processor is running, restart if necessary.
|
|
732
|
-
|
|
733
|
-
Args:
|
|
734
|
-
client (BackendClient): The backend client instance.
|
|
735
|
-
pre_processor_code (str): The pre-processor code.
|
|
736
|
-
|
|
737
|
-
Returns:
|
|
738
|
-
Dict[str, Any]: Result dictionary indicating success or failure.
|
|
739
|
-
"""
|
|
740
|
-
try:
|
|
741
|
-
# Check if pre-processor is running
|
|
742
|
-
serve_applications_response = client.list_serve_applications(params={'plugin_code': pre_processor_code})
|
|
743
|
-
if isinstance(serve_applications_response, str):
|
|
744
|
-
return {'success': False, 'error': 'Invalid serve applications response'}
|
|
745
|
-
|
|
746
|
-
# Handle the response properly - it should be a dict with 'results' key
|
|
747
|
-
if not isinstance(serve_applications_response, dict):
|
|
748
|
-
return {'success': False, 'error': 'Unexpected serve applications response format'}
|
|
749
|
-
|
|
750
|
-
serve_applications: Dict[str, Any] = serve_applications_response
|
|
751
|
-
results = serve_applications.get('results', [])
|
|
752
|
-
running_serve_apps = [app for app in results if isinstance(app, dict) and app.get('status') == 'RUNNING']
|
|
753
|
-
|
|
754
|
-
# If not running, restart the pre-processor
|
|
755
|
-
if not running_serve_apps:
|
|
756
|
-
restart_result = self._restart_pre_processor(client, pre_processor_code)
|
|
757
|
-
if not restart_result['success']:
|
|
758
|
-
return restart_result
|
|
759
|
-
|
|
760
|
-
# Verify restart was successful
|
|
761
|
-
serve_applications_response = client.list_serve_applications(params={'plugin_code': pre_processor_code})
|
|
762
|
-
if isinstance(serve_applications_response, str):
|
|
763
|
-
return {'success': False, 'error': 'Failed to verify pre-processor restart'}
|
|
764
|
-
|
|
765
|
-
if not isinstance(serve_applications_response, dict):
|
|
766
|
-
return {'success': False, 'error': 'Unexpected serve applications response format after restart'}
|
|
767
|
-
|
|
768
|
-
serve_applications: Dict[str, Any] = serve_applications_response
|
|
769
|
-
results = serve_applications.get('results', [])
|
|
770
|
-
running_serve_apps = [
|
|
771
|
-
app for app in results if isinstance(app, dict) and app.get('status') == 'RUNNING'
|
|
772
|
-
]
|
|
773
|
-
|
|
774
|
-
if not running_serve_apps:
|
|
775
|
-
return {'success': False, 'error': 'Failed to restart pre-processor'}
|
|
776
|
-
|
|
777
|
-
return {'success': True}
|
|
778
|
-
except Exception as e:
|
|
779
|
-
return {'success': False, 'error': f'Failed to ensure pre-processor running: {str(e)}'}
|
|
780
|
-
|
|
781
|
-
def _restart_pre_processor(self, client: BackendClient, pre_processor_code: str) -> Dict[str, Any]:
|
|
782
|
-
"""Restart the pre-processor.
|
|
783
|
-
|
|
784
|
-
Args:
|
|
785
|
-
client (BackendClient): The backend client instance.
|
|
786
|
-
pre_processor_code (str): The pre-processor code.
|
|
787
|
-
|
|
788
|
-
Returns:
|
|
789
|
-
Dict[str, Any]: Result dictionary indicating success or failure.
|
|
790
|
-
"""
|
|
791
|
-
try:
|
|
792
|
-
if not self.config:
|
|
793
|
-
return {'success': False, 'error': 'Configuration not available'}
|
|
794
|
-
|
|
795
|
-
inference_options = self.config.get('inference_options', {})
|
|
796
|
-
serve_application_deployment_payload = {
|
|
797
|
-
'agent': self.params.get('agent') if self.params else None,
|
|
798
|
-
'action': 'deployment',
|
|
799
|
-
'params': {
|
|
800
|
-
'num_cpus': inference_options.get('required_cpu_count', 2),
|
|
801
|
-
'num_gpus': inference_options.get('required_gpu_count', 1),
|
|
802
|
-
},
|
|
803
|
-
'debug': True,
|
|
804
|
-
}
|
|
805
|
-
|
|
806
|
-
deployment_result = client.run_plugin(pre_processor_code, serve_application_deployment_payload)
|
|
807
|
-
if not deployment_result:
|
|
808
|
-
return {'success': False, 'error': 'Failed to restart pre-processor'}
|
|
809
|
-
|
|
810
|
-
return {'success': True}
|
|
811
|
-
except Exception as e:
|
|
812
|
-
return {'success': False, 'error': f'Failed to restart pre-processor: {str(e)}'}
|
|
813
|
-
|
|
814
|
-
def _extract_primary_file_url(self, task: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
|
|
815
|
-
"""Extract the primary file URL from task data.
|
|
816
|
-
|
|
817
|
-
Args:
|
|
818
|
-
task (Dict[str, Any]): The task data.
|
|
819
|
-
|
|
820
|
-
Returns:
|
|
821
|
-
Tuple[Optional[str], Optional[str]]: The primary file URL and original name.
|
|
822
|
-
"""
|
|
823
|
-
data_unit = task.get('data_unit', {})
|
|
824
|
-
files = data_unit.get('files', {})
|
|
825
|
-
|
|
826
|
-
for file_info in files.values():
|
|
827
|
-
if isinstance(file_info, dict) and file_info.get('is_primary') and file_info.get('url'):
|
|
828
|
-
return file_info['url'], file_info.get('file_name_original')
|
|
829
|
-
|
|
830
|
-
return None, None
|
|
831
|
-
|
|
832
|
-
def _run_inference(
|
|
833
|
-
self, client: BackendClient, pre_processor_code: str, pre_processor_version: str, primary_file_url: str
|
|
834
|
-
) -> Dict[str, Any]:
|
|
835
|
-
"""Run inference using the pre-processor.
|
|
836
|
-
|
|
837
|
-
Args:
|
|
838
|
-
client (BackendClient): The backend client instance.
|
|
839
|
-
pre_processor_code (str): The pre-processor code.
|
|
840
|
-
pre_processor_version (str): The pre-processor version.
|
|
841
|
-
primary_file_url (str): The primary image URL.
|
|
842
|
-
|
|
843
|
-
Returns:
|
|
844
|
-
Dict[str, Any]: Result dictionary with inference data or error.
|
|
845
|
-
"""
|
|
846
|
-
try:
|
|
847
|
-
if not self.params:
|
|
848
|
-
return {'success': False, 'error': 'Parameters not available'}
|
|
849
|
-
|
|
850
|
-
pre_processor_params = self.params.get('pre_processor_params', {})
|
|
851
|
-
pre_processor_params['image_path'] = primary_file_url
|
|
852
|
-
|
|
853
|
-
inference_payload = {
|
|
854
|
-
'agent': 1,
|
|
855
|
-
'action': 'inference',
|
|
856
|
-
'version': pre_processor_version,
|
|
857
|
-
'params': {
|
|
858
|
-
'model': self.params['model'],
|
|
859
|
-
'method': 'post',
|
|
860
|
-
'json': pre_processor_params,
|
|
861
|
-
},
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
inference_data = client.run_plugin(pre_processor_code, inference_payload)
|
|
865
|
-
return {'success': True, 'data': inference_data}
|
|
866
|
-
except Exception as e:
|
|
867
|
-
return {'success': False, 'error': f'Failed to run inference: {str(e)}'}
|
|
868
|
-
|
|
869
|
-
def _update_metrics(self, total_tasks: int, success_count: int, failed_count: int):
|
|
870
|
-
"""Update metrics for task annotation progress.
|
|
871
|
-
|
|
872
|
-
Args:
|
|
873
|
-
total_tasks (int): Total number of tasks to process.
|
|
874
|
-
success_count (int): Number of successfully processed tasks.
|
|
875
|
-
failed_count (int): Number of failed tasks.
|
|
876
|
-
"""
|
|
877
|
-
if not self.run:
|
|
878
|
-
raise ValueError('Run instance not properly initialized')
|
|
879
|
-
|
|
880
|
-
# Type assertion to help the linter
|
|
881
|
-
assert isinstance(self.run, ToTaskRun)
|
|
882
|
-
|
|
883
|
-
metrics = self.run.MetricsRecord(
|
|
884
|
-
stand_by=total_tasks - success_count - failed_count, failed=failed_count, success=success_count
|
|
885
|
-
)
|
|
886
|
-
self.run.log_metrics(metrics, 'annotate_task_data')
|
|
887
|
-
|
|
888
|
-
def _handle_annotate_data_with_inference(self, task_ids: List[int]):
|
|
889
|
-
"""Handle annotate data with inference to tasks.
|
|
890
|
-
|
|
891
|
-
Args:
|
|
892
|
-
task_ids (List[int]): List of task IDs to annotate data to.
|
|
893
|
-
"""
|
|
894
|
-
if not self.run or not self.params:
|
|
895
|
-
raise ValueError('Run instance or parameters not properly initialized')
|
|
896
|
-
|
|
897
|
-
if not self.params.get('model'):
|
|
898
|
-
raise ValueError('Model is required for inference annotation method')
|
|
899
|
-
|
|
900
|
-
# Type assertion to help the linter
|
|
901
|
-
assert isinstance(self.run, ToTaskRun)
|
|
902
|
-
assert isinstance(self.run.client, BackendClient)
|
|
903
|
-
|
|
904
|
-
client: BackendClient = self.run.client
|
|
905
|
-
task_params = {
|
|
906
|
-
'fields': 'id,data,data_unit',
|
|
907
|
-
'expand': 'data_unit',
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
total_tasks = len(task_ids)
|
|
911
|
-
success_count = 0
|
|
912
|
-
failed_count = 0
|
|
913
|
-
current_progress = 0
|
|
914
|
-
|
|
915
|
-
# Initialize metrics and progress
|
|
916
|
-
self._update_metrics(total_tasks, success_count, failed_count)
|
|
917
|
-
self.run.set_progress(0, total_tasks, category='annotate_task_data')
|
|
918
|
-
self.run.log_message_with_code('ANNOTATING_INFERENCE_DATA')
|
|
919
|
-
|
|
920
|
-
# Process each task
|
|
921
|
-
for task_id in task_ids:
|
|
922
|
-
try:
|
|
923
|
-
result = self._process_single_task(client, task_id, task_params, None, AnnotationMethod.INFERENCE)
|
|
924
|
-
if result['success']:
|
|
925
|
-
success_count += 1
|
|
926
|
-
else:
|
|
927
|
-
failed_count += 1
|
|
928
|
-
|
|
929
|
-
current_progress += 1
|
|
930
|
-
self._update_metrics(total_tasks, success_count, failed_count)
|
|
931
|
-
self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
|
|
932
|
-
|
|
933
|
-
except Exception as e:
|
|
934
|
-
self.run.log_annotate_task_event('TASK_PROCESSING_FAILED', task_id, str(e))
|
|
935
|
-
self.run.log_annotate_task_data({'task_id': task_id, 'error': str(e)}, AnnotateTaskDataStatus.FAILED)
|
|
936
|
-
failed_count += 1
|
|
937
|
-
current_progress += 1
|
|
938
|
-
self._update_metrics(total_tasks, success_count, failed_count)
|
|
939
|
-
self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
|
|
940
|
-
|
|
941
|
-
# Finalize progress
|
|
942
|
-
self.run.set_progress(total_tasks, total_tasks, category='annotate_task_data')
|
|
943
|
-
self.run.log_message_with_code('INFERENCE_ANNOTATION_COMPLETED', success_count, failed_count)
|