synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. synapse_sdk/__init__.py +24 -0
  2. synapse_sdk/cli/code_server.py +305 -33
  3. synapse_sdk/clients/agent/__init__.py +2 -1
  4. synapse_sdk/clients/agent/container.py +143 -0
  5. synapse_sdk/clients/agent/ray.py +296 -38
  6. synapse_sdk/clients/backend/annotation.py +1 -1
  7. synapse_sdk/clients/backend/core.py +31 -4
  8. synapse_sdk/clients/backend/data_collection.py +82 -7
  9. synapse_sdk/clients/backend/hitl.py +1 -1
  10. synapse_sdk/clients/backend/ml.py +1 -1
  11. synapse_sdk/clients/base.py +211 -61
  12. synapse_sdk/loggers.py +46 -0
  13. synapse_sdk/plugins/README.md +1340 -0
  14. synapse_sdk/plugins/categories/base.py +59 -9
  15. synapse_sdk/plugins/categories/export/actions/__init__.py +3 -0
  16. synapse_sdk/plugins/categories/export/actions/export/__init__.py +28 -0
  17. synapse_sdk/plugins/categories/export/actions/export/action.py +165 -0
  18. synapse_sdk/plugins/categories/export/actions/export/enums.py +113 -0
  19. synapse_sdk/plugins/categories/export/actions/export/exceptions.py +53 -0
  20. synapse_sdk/plugins/categories/export/actions/export/models.py +74 -0
  21. synapse_sdk/plugins/categories/export/actions/export/run.py +195 -0
  22. synapse_sdk/plugins/categories/export/actions/export/utils.py +187 -0
  23. synapse_sdk/plugins/categories/export/templates/config.yaml +19 -1
  24. synapse_sdk/plugins/categories/export/templates/plugin/__init__.py +390 -0
  25. synapse_sdk/plugins/categories/export/templates/plugin/export.py +153 -177
  26. synapse_sdk/plugins/categories/neural_net/actions/train.py +1130 -32
  27. synapse_sdk/plugins/categories/neural_net/actions/tune.py +157 -4
  28. synapse_sdk/plugins/categories/neural_net/templates/config.yaml +7 -4
  29. synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py +4 -0
  30. synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/__init__.py +3 -0
  31. synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/action.py +10 -0
  32. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/__init__.py +28 -0
  33. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/action.py +148 -0
  34. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/enums.py +269 -0
  35. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/exceptions.py +14 -0
  36. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/factory.py +76 -0
  37. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/models.py +100 -0
  38. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/orchestrator.py +248 -0
  39. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/run.py +64 -0
  40. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/__init__.py +17 -0
  41. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/annotation.py +265 -0
  42. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/base.py +170 -0
  43. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/extraction.py +83 -0
  44. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/metrics.py +92 -0
  45. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/preprocessor.py +243 -0
  46. synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/validation.py +143 -0
  47. synapse_sdk/plugins/categories/upload/actions/upload/__init__.py +19 -0
  48. synapse_sdk/plugins/categories/upload/actions/upload/action.py +236 -0
  49. synapse_sdk/plugins/categories/upload/actions/upload/context.py +185 -0
  50. synapse_sdk/plugins/categories/upload/actions/upload/enums.py +493 -0
  51. synapse_sdk/plugins/categories/upload/actions/upload/exceptions.py +36 -0
  52. synapse_sdk/plugins/categories/upload/actions/upload/factory.py +138 -0
  53. synapse_sdk/plugins/categories/upload/actions/upload/models.py +214 -0
  54. synapse_sdk/plugins/categories/upload/actions/upload/orchestrator.py +183 -0
  55. synapse_sdk/plugins/categories/upload/actions/upload/registry.py +113 -0
  56. synapse_sdk/plugins/categories/upload/actions/upload/run.py +179 -0
  57. synapse_sdk/plugins/categories/upload/actions/upload/steps/__init__.py +1 -0
  58. synapse_sdk/plugins/categories/upload/actions/upload/steps/base.py +107 -0
  59. synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py +62 -0
  60. synapse_sdk/plugins/categories/upload/actions/upload/steps/collection.py +63 -0
  61. synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py +91 -0
  62. synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py +82 -0
  63. synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py +235 -0
  64. synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py +201 -0
  65. synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py +104 -0
  66. synapse_sdk/plugins/categories/upload/actions/upload/steps/validate.py +71 -0
  67. synapse_sdk/plugins/categories/upload/actions/upload/strategies/__init__.py +1 -0
  68. synapse_sdk/plugins/categories/upload/actions/upload/strategies/base.py +82 -0
  69. synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/__init__.py +1 -0
  70. synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/batch.py +39 -0
  71. synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/single.py +29 -0
  72. synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/__init__.py +1 -0
  73. synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py +300 -0
  74. synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/recursive.py +287 -0
  75. synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/__init__.py +1 -0
  76. synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/excel.py +174 -0
  77. synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/none.py +16 -0
  78. synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/__init__.py +1 -0
  79. synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/sync.py +84 -0
  80. synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/__init__.py +1 -0
  81. synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py +60 -0
  82. synapse_sdk/plugins/categories/upload/actions/upload/utils.py +250 -0
  83. synapse_sdk/plugins/categories/upload/templates/README.md +470 -0
  84. synapse_sdk/plugins/categories/upload/templates/config.yaml +28 -2
  85. synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py +310 -0
  86. synapse_sdk/plugins/categories/upload/templates/plugin/upload.py +82 -20
  87. synapse_sdk/plugins/models.py +111 -9
  88. synapse_sdk/plugins/templates/plugin-config-schema.json +7 -0
  89. synapse_sdk/plugins/templates/schema.json +7 -0
  90. synapse_sdk/plugins/utils/__init__.py +3 -0
  91. synapse_sdk/plugins/utils/ray_gcs.py +66 -0
  92. synapse_sdk/shared/__init__.py +25 -0
  93. synapse_sdk/utils/converters/dm/__init__.py +42 -41
  94. synapse_sdk/utils/converters/dm/base.py +137 -0
  95. synapse_sdk/utils/converters/dm/from_v1.py +208 -562
  96. synapse_sdk/utils/converters/dm/to_v1.py +258 -304
  97. synapse_sdk/utils/converters/dm/tools/__init__.py +214 -0
  98. synapse_sdk/utils/converters/dm/tools/answer.py +95 -0
  99. synapse_sdk/utils/converters/dm/tools/bounding_box.py +132 -0
  100. synapse_sdk/utils/converters/dm/tools/bounding_box_3d.py +121 -0
  101. synapse_sdk/utils/converters/dm/tools/classification.py +75 -0
  102. synapse_sdk/utils/converters/dm/tools/keypoint.py +117 -0
  103. synapse_sdk/utils/converters/dm/tools/named_entity.py +111 -0
  104. synapse_sdk/utils/converters/dm/tools/polygon.py +122 -0
  105. synapse_sdk/utils/converters/dm/tools/polyline.py +124 -0
  106. synapse_sdk/utils/converters/dm/tools/prompt.py +94 -0
  107. synapse_sdk/utils/converters/dm/tools/relation.py +86 -0
  108. synapse_sdk/utils/converters/dm/tools/segmentation.py +141 -0
  109. synapse_sdk/utils/converters/dm/tools/segmentation_3d.py +83 -0
  110. synapse_sdk/utils/converters/dm/types.py +168 -0
  111. synapse_sdk/utils/converters/dm/utils.py +162 -0
  112. synapse_sdk/utils/converters/dm_legacy/__init__.py +56 -0
  113. synapse_sdk/utils/converters/dm_legacy/from_v1.py +627 -0
  114. synapse_sdk/utils/converters/dm_legacy/to_v1.py +367 -0
  115. synapse_sdk/utils/file/__init__.py +58 -0
  116. synapse_sdk/utils/file/archive.py +32 -0
  117. synapse_sdk/utils/file/checksum.py +56 -0
  118. synapse_sdk/utils/file/chunking.py +31 -0
  119. synapse_sdk/utils/file/download.py +385 -0
  120. synapse_sdk/utils/file/encoding.py +40 -0
  121. synapse_sdk/utils/file/io.py +22 -0
  122. synapse_sdk/utils/file/upload.py +165 -0
  123. synapse_sdk/utils/file/video/__init__.py +29 -0
  124. synapse_sdk/utils/file/video/transcode.py +307 -0
  125. synapse_sdk/utils/{file.py → file.py.backup} +77 -0
  126. synapse_sdk/utils/network.py +272 -0
  127. synapse_sdk/utils/storage/__init__.py +6 -2
  128. synapse_sdk/utils/storage/providers/file_system.py +6 -0
  129. {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/METADATA +19 -2
  130. {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/RECORD +134 -74
  131. synapse_sdk/devtools/docs/.gitignore +0 -20
  132. synapse_sdk/devtools/docs/README.md +0 -41
  133. synapse_sdk/devtools/docs/blog/2019-05-28-first-blog-post.md +0 -12
  134. synapse_sdk/devtools/docs/blog/2019-05-29-long-blog-post.md +0 -44
  135. synapse_sdk/devtools/docs/blog/2021-08-01-mdx-blog-post.mdx +0 -24
  136. synapse_sdk/devtools/docs/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg +0 -0
  137. synapse_sdk/devtools/docs/blog/2021-08-26-welcome/index.md +0 -29
  138. synapse_sdk/devtools/docs/blog/authors.yml +0 -25
  139. synapse_sdk/devtools/docs/blog/tags.yml +0 -19
  140. synapse_sdk/devtools/docs/docusaurus.config.ts +0 -138
  141. synapse_sdk/devtools/docs/package-lock.json +0 -17455
  142. synapse_sdk/devtools/docs/package.json +0 -47
  143. synapse_sdk/devtools/docs/sidebars.ts +0 -44
  144. synapse_sdk/devtools/docs/src/components/HomepageFeatures/index.tsx +0 -71
  145. synapse_sdk/devtools/docs/src/components/HomepageFeatures/styles.module.css +0 -11
  146. synapse_sdk/devtools/docs/src/css/custom.css +0 -30
  147. synapse_sdk/devtools/docs/src/pages/index.module.css +0 -23
  148. synapse_sdk/devtools/docs/src/pages/index.tsx +0 -21
  149. synapse_sdk/devtools/docs/src/pages/markdown-page.md +0 -7
  150. synapse_sdk/devtools/docs/static/.nojekyll +0 -0
  151. synapse_sdk/devtools/docs/static/img/docusaurus-social-card.jpg +0 -0
  152. synapse_sdk/devtools/docs/static/img/docusaurus.png +0 -0
  153. synapse_sdk/devtools/docs/static/img/favicon.ico +0 -0
  154. synapse_sdk/devtools/docs/static/img/logo.png +0 -0
  155. synapse_sdk/devtools/docs/static/img/undraw_docusaurus_mountain.svg +0 -171
  156. synapse_sdk/devtools/docs/static/img/undraw_docusaurus_react.svg +0 -170
  157. synapse_sdk/devtools/docs/static/img/undraw_docusaurus_tree.svg +0 -40
  158. synapse_sdk/devtools/docs/tsconfig.json +0 -8
  159. synapse_sdk/plugins/categories/export/actions/export.py +0 -346
  160. synapse_sdk/plugins/categories/export/enums.py +0 -7
  161. synapse_sdk/plugins/categories/neural_net/actions/gradio.py +0 -151
  162. synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py +0 -943
  163. synapse_sdk/plugins/categories/upload/actions/upload.py +0 -954
  164. {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/WHEEL +0 -0
  165. {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/entry_points.txt +0 -0
  166. {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/licenses/LICENSE +0 -0
  167. {synapse_sdk-1.0.0b5.dist-info → synapse_sdk-2025.12.3.dist-info}/top_level.txt +0 -0
@@ -1,943 +0,0 @@
1
- import json
2
- from datetime import datetime
3
- from enum import Enum
4
- from typing import Annotated, Any, Dict, List, Optional, Tuple
5
-
6
- import requests
7
- from pydantic import AfterValidator, BaseModel, field_validator
8
- from pydantic_core import PydanticCustomError
9
-
10
- from synapse_sdk.clients.backend import BackendClient
11
- from synapse_sdk.clients.backend.models import JobStatus
12
- from synapse_sdk.clients.exceptions import ClientError
13
- from synapse_sdk.plugins.categories.base import Action
14
- from synapse_sdk.plugins.categories.decorators import register_action
15
- from synapse_sdk.plugins.enums import PluginCategory, RunMethod
16
- from synapse_sdk.plugins.models import Run
17
- from synapse_sdk.shared.enums import Context
18
- from synapse_sdk.utils.pydantic.validators import non_blank
19
-
20
-
21
- class AnnotationMethod(str, Enum):
22
- FILE = 'file'
23
- INFERENCE = 'inference'
24
-
25
-
26
- class AnnotateTaskDataStatus(str, Enum):
27
- SUCCESS = 'success'
28
- FAILED = 'failed'
29
-
30
-
31
- class CriticalError(Exception):
32
- """Critical error."""
33
-
34
- def __init__(self, message: str = 'Critical error occured while processing task'):
35
- self.message = message
36
- super().__init__(self.message)
37
-
38
-
39
- class PreAnnotationToTaskFailed(Exception):
40
- """Pre-annotation to task failed."""
41
-
42
- def __init__(self, message: str = 'Pre-annotation to task failed'):
43
- self.message = message
44
- super().__init__(self.message)
45
-
46
-
47
- class ToTaskRun(Run):
48
- class AnnotateTaskEventLog(BaseModel):
49
- """Annotate task event log model."""
50
-
51
- info: Optional[str] = None
52
- status: Context
53
- created: str
54
-
55
- class AnnotateTaskDataLog(BaseModel):
56
- """Log model for annotate task data."""
57
-
58
- task_info: Optional[str] = None
59
- status: AnnotateTaskDataStatus
60
- created: str
61
-
62
- class MetricsRecord(BaseModel):
63
- """Metrics record model."""
64
-
65
- stand_by: int
66
- failed: int
67
- success: int
68
-
69
- LOG_MESSAGES = {
70
- 'INVALID_PROJECT_RESPONSE': {
71
- 'message': 'Invalid project response received.',
72
- 'level': Context.DANGER,
73
- },
74
- 'NO_DATA_COLLECTION': {
75
- 'message': 'Project does not have a data collection.',
76
- 'level': Context.DANGER,
77
- },
78
- 'INVALID_DATA_COLLECTION_RESPONSE': {
79
- 'message': 'Invalid data collection response received.',
80
- 'level': Context.DANGER,
81
- },
82
- 'NO_TASKS_FOUND': {
83
- 'message': 'Tasks to annotate not found.',
84
- 'level': Context.WARNING,
85
- },
86
- 'TARGET_SPEC_REQUIRED': {
87
- 'message': 'Target specification name is required for file annotation method.',
88
- 'level': Context.DANGER,
89
- },
90
- 'TARGET_SPEC_NOT_FOUND': {
91
- 'message': 'Target specification name "{}" not found in file specifications',
92
- 'level': Context.DANGER,
93
- },
94
- 'UNSUPPORTED_METHOD': {
95
- 'message': 'Unsupported annotation method: {}',
96
- 'level': Context.DANGER,
97
- },
98
- 'ANNOTATING_DATA': {
99
- 'message': 'Annotating data to tasks...',
100
- 'level': None,
101
- },
102
- 'CRITICAL_ERROR': {
103
- 'message': 'Critical error occured while processing task. Stopping the job.',
104
- 'level': Context.DANGER,
105
- },
106
- 'TASK_PROCESSING_FAILED': {
107
- 'message': 'Failed to process task {}: {}',
108
- 'level': Context.DANGER,
109
- },
110
- 'ANNOTATION_COMPLETED': {
111
- 'message': 'Annotation completed. Success: {}, Failed: {}',
112
- 'level': None,
113
- },
114
- 'INVALID_TASK_RESPONSE': {
115
- 'message': 'Invalid task response received for task {}',
116
- 'level': Context.DANGER,
117
- },
118
- 'TARGET_SPEC_REQUIRED_FOR_TASK': {
119
- 'message': 'Target specification name is required for file annotation method for task {}',
120
- 'level': Context.DANGER,
121
- },
122
- 'UNSUPPORTED_METHOD_FOR_TASK': {
123
- 'message': 'Unsupported annotation method: {} for task {}',
124
- 'level': Context.DANGER,
125
- },
126
- 'PRIMARY_IMAGE_URL_NOT_FOUND': {
127
- 'message': 'Primary image URL not found in task data for task {}',
128
- 'level': Context.DANGER,
129
- },
130
- 'FILE_SPEC_NOT_FOUND': {
131
- 'message': 'File specification not found for task {}',
132
- 'level': Context.DANGER,
133
- },
134
- 'FILE_ORIGINAL_NAME_NOT_FOUND': {
135
- 'message': 'File original name not found for task {}',
136
- 'level': Context.DANGER,
137
- },
138
- 'URL_NOT_FOUND': {
139
- 'message': 'URL not found for task {}',
140
- 'level': Context.DANGER,
141
- },
142
- 'FETCH_DATA_FAILED': {
143
- 'message': 'Failed to fetch data from URL: {} for task {}',
144
- 'level': Context.DANGER,
145
- },
146
- 'CONVERT_DATA_FAILED': {
147
- 'message': 'Failed to convert data to task object: {} for task {}',
148
- 'level': Context.DANGER,
149
- },
150
- 'PREPROCESSOR_ID_REQUIRED': {
151
- 'message': 'Pre-processor ID is required for inference annotation method for task {}',
152
- 'level': Context.DANGER,
153
- },
154
- 'INFERENCE_PROCESSING_FAILED': {
155
- 'message': 'Failed to process inference for task {}: {}',
156
- 'level': Context.DANGER,
157
- },
158
- 'ANNOTATING_INFERENCE_DATA': {
159
- 'message': 'Annotating data to tasks using inference...',
160
- 'level': None,
161
- },
162
- 'INFERENCE_ANNOTATION_COMPLETED': {
163
- 'message': 'Inference annotation completed. Success: {}, Failed: {}',
164
- 'level': None,
165
- },
166
- }
167
-
168
- def log_message_with_code(self, code: str, *args, level: Optional[Context] = None):
169
- """Log message using predefined code and optional level override."""
170
- if code not in self.LOG_MESSAGES:
171
- self.log_message(f'Unknown log code: {code}')
172
- return
173
-
174
- log_config = self.LOG_MESSAGES[code]
175
- message = log_config['message'].format(*args) if args else log_config['message']
176
- log_level = level or log_config['level']
177
-
178
- if log_level:
179
- self.log_message(message, context=log_level.value)
180
- else:
181
- self.log_message(message, context=Context.INFO.value)
182
-
183
- def log_annotate_task_event(self, code: str, *args, level: Optional[Context] = None):
184
- """Log annotate task event using predefined code."""
185
- if code not in self.LOG_MESSAGES:
186
- now = datetime.now().isoformat()
187
- self.log(
188
- 'annotate_task_event',
189
- self.AnnotateTaskEventLog(
190
- info=f'Unknown log code: {code}', status=Context.DANGER, created=now
191
- ).model_dump(),
192
- )
193
- return
194
-
195
- log_config = self.LOG_MESSAGES[code]
196
- message = log_config['message'].format(*args) if args else log_config['message']
197
- log_level = level or log_config['level'] or Context.INFO
198
-
199
- now = datetime.now().isoformat()
200
- self.log(
201
- 'annotate_task_event',
202
- self.AnnotateTaskEventLog(info=message, status=log_level, created=now).model_dump(),
203
- )
204
-
205
- def log_annotate_task_data(self, task_info: Dict[str, Any], status: AnnotateTaskDataStatus):
206
- """Log annotate task data."""
207
- now = datetime.now().isoformat()
208
- self.log(
209
- 'annotate_task_data',
210
- self.AnnotateTaskDataLog(task_info=json.dumps(task_info), status=status, created=now).model_dump(),
211
- )
212
-
213
- def log_metrics(self, record: MetricsRecord, category: str):
214
- """Log FileToTask metrics.
215
-
216
- Args:
217
- record (MetricsRecord): The metrics record to log.
218
- category (str): The category of the metrics.
219
- """
220
- record = self.MetricsRecord.model_validate(record)
221
- self.set_metrics(value=record.model_dump(), category=category)
222
-
223
-
224
- class ToTaskParams(BaseModel):
225
- """ToTask action parameters.
226
-
227
- Args:
228
- name (str): The name of the action.
229
- description (str | None): The description of the action.
230
- project (int): The project ID.
231
- agent (int): The agent ID.
232
- task_filters (dict): The filters of tasks.
233
- method (AnnotationMethod): The method of annotation.
234
- target_specification_name (str | None): The name of the target specification.
235
- model (int): The model ID.
236
- pre_processor (int | None): The pre processor ID.
237
- pre_processor_params (dict): The params of the pre processor.
238
- """
239
-
240
- name: Annotated[str, AfterValidator(non_blank)]
241
- description: Optional[str] = None
242
- project: int
243
- agent: int
244
- task_filters: Dict[str, Any]
245
- method: Optional[AnnotationMethod] = None
246
- target_specification_name: Optional[str] = None
247
- model: Optional[int] = None
248
- pre_processor: Optional[int] = None
249
- pre_processor_params: Dict[str, Any]
250
-
251
- @field_validator('project', mode='before')
252
- @classmethod
253
- def check_project_exists(cls, value: int, info) -> int:
254
- """Validate synapse-backend project exists."""
255
- if not value:
256
- return value
257
-
258
- action = info.context['action']
259
- client = action.client
260
- try:
261
- client.get_project(value)
262
- except ClientError:
263
- raise PydanticCustomError('client_error', 'Error occurred while checking project exists.')
264
- return value
265
-
266
-
267
- class ToTaskResult(BaseModel):
268
- """Result model for ToTaskAction.start method.
269
-
270
- Args:
271
- status (JobStatus): The job status from the action execution.
272
- message (str): A descriptive message about the action result.
273
- """
274
-
275
- status: JobStatus
276
- message: str
277
-
278
- def model_dump(self, **kwargs):
279
- """Override model_dump to return status as enum value."""
280
- data = super().model_dump(**kwargs)
281
- if 'status' in data and isinstance(data['status'], JobStatus):
282
- data['status'] = data['status'].value
283
- return data
284
-
285
-
286
- @register_action
287
- class ToTaskAction(Action):
288
- """ToTask action for pre-annotation data processing.
289
-
290
- This action handles the process of annotating data to tasks in a project. It supports
291
- two annotation methods: file-based annotation and inference-based annotation.
292
-
293
- File-based annotation fetches data from file URLs specified in task data units,
294
- downloads and processes JSON data, and updates task data with the processed information.
295
- It also validates target specification names against file specifications.
296
-
297
- Inference-based annotation is currently not supported but will support model inference
298
- for automatic data annotation in future implementations.
299
-
300
- Attrs:
301
- name (str): Action name, set to 'to_task'.
302
- category (PluginCategory): Plugin category, set to PRE_ANNOTATION.
303
- method (RunMethod): Execution method, set to JOB.
304
- run_class (Type[ToTaskRun]): Run class for this action.
305
- params_model (Type[ToTaskParams]): Parameter validation model.
306
- progress_categories (Dict): Progress tracking configuration.
307
- metrics_categories (Set[str]): Metrics categories for this action.
308
-
309
- Note:
310
- This action requires a valid project with an associated data collection.
311
- For file-based annotation, the target_specification_name must exist in the
312
- project's file specifications.
313
-
314
- Raises:
315
- ValueError: If run instance or parameters are not properly initialized.
316
- ClientError: If there are issues with backend API calls.
317
- """
318
-
319
- name = 'to_task'
320
- category = PluginCategory.PRE_ANNOTATION
321
- method = RunMethod.JOB
322
- run_class = ToTaskRun
323
- params_model = ToTaskParams
324
- progress_categories = {
325
- 'annotate_task_data': {
326
- 'proportion': 100,
327
- },
328
- }
329
- metrics_categories = {
330
- 'annotate_task_data': {
331
- 'stand_by': 0,
332
- 'failed': 0,
333
- 'success': 0,
334
- }
335
- }
336
-
337
- def start(self) -> dict:
338
- """Start to_task action.
339
-
340
- * Generate tasks.
341
- * Annotate data to tasks.
342
-
343
- Returns:
344
- dict: Validated result with status and message.
345
- """
346
- if not self.run or not self.params:
347
- result = ToTaskResult(
348
- status=JobStatus.FAILED, message='Run instance or parameters not properly initialized'
349
- )
350
- raise PreAnnotationToTaskFailed(result.message)
351
-
352
- # Type assertion to help the linter
353
- assert isinstance(self.run, ToTaskRun)
354
- assert isinstance(self.run.client, BackendClient)
355
-
356
- client: BackendClient = self.run.client
357
- project_id = self.params['project']
358
- project_response = client.get_project(project_id)
359
- if isinstance(project_response, str):
360
- self.run.log_message_with_code('INVALID_PROJECT_RESPONSE')
361
- self.run.end_log()
362
- result = ToTaskResult(status=JobStatus.FAILED, message='Invalid project response received')
363
- raise PreAnnotationToTaskFailed(result.message)
364
- project: Dict[str, Any] = project_response
365
-
366
- data_collection_id = project.get('data_collection')
367
- if not data_collection_id:
368
- self.run.log_message_with_code('NO_DATA_COLLECTION')
369
- self.run.end_log()
370
- result = ToTaskResult(status=JobStatus.FAILED, message='Project does not have a data collection')
371
- raise PreAnnotationToTaskFailed(result.message)
372
-
373
- data_collection_response = client.get_data_collection(data_collection_id)
374
- if isinstance(data_collection_response, str):
375
- self.run.log_message_with_code('INVALID_DATA_COLLECTION_RESPONSE')
376
- self.run.end_log()
377
- result = ToTaskResult(status=JobStatus.FAILED, message='Invalid data collection response received')
378
- raise PreAnnotationToTaskFailed(result.message)
379
- data_collection: Dict[str, Any] = data_collection_response
380
-
381
- # Generate tasks if provided project is empty.
382
- task_ids_query_params = {
383
- 'project': self.params['project'],
384
- 'fields': 'id',
385
- }
386
- if self.params['task_filters']:
387
- task_ids_query_params.update(self.params['task_filters'])
388
- task_ids_generator, task_ids_count = client.list_tasks(params=task_ids_query_params, list_all=True)
389
- task_ids = [int(item.get('id', 0)) for item in task_ids_generator if isinstance(item, dict) and item.get('id')]
390
-
391
- # If no tasks found, break the job.
392
- if not task_ids_count:
393
- self.run.log_message_with_code('NO_TASKS_FOUND')
394
- self.run.end_log()
395
- result = ToTaskResult(status=JobStatus.FAILED, message='No tasks found to annotate')
396
- raise PreAnnotationToTaskFailed(result.message)
397
-
398
- # Annotate data to tasks.
399
- method = self.params.get('method')
400
- if method == AnnotationMethod.FILE:
401
- # Check if target specification name exists in file specifications.
402
- target_specification_name = self.params.get('target_specification_name')
403
- if not target_specification_name:
404
- self.run.log_message_with_code('TARGET_SPEC_REQUIRED')
405
- self.run.end_log()
406
- result = ToTaskResult(
407
- status=JobStatus.FAILED, message='Target specification name is required for file annotation method'
408
- )
409
- raise PreAnnotationToTaskFailed(result.message)
410
-
411
- file_specifications = data_collection.get('file_specifications', [])
412
- target_spec_exists = any(spec.get('name') == target_specification_name for spec in file_specifications)
413
- if not target_spec_exists:
414
- self.run.log_message_with_code('TARGET_SPEC_NOT_FOUND', target_specification_name)
415
- self.run.end_log()
416
- result = ToTaskResult(
417
- status=JobStatus.FAILED,
418
- message=f"Target specification name '{target_specification_name}' not found in file specifications",
419
- )
420
- raise PreAnnotationToTaskFailed(result.message)
421
- self._handle_annotate_data_from_files(task_ids, target_specification_name)
422
- elif method == AnnotationMethod.INFERENCE:
423
- self._handle_annotate_data_with_inference(task_ids)
424
- else:
425
- self.run.log_message_with_code('UNSUPPORTED_METHOD', method)
426
- self.run.end_log()
427
- result = ToTaskResult(status=JobStatus.FAILED, message=f'Unsupported annotation method: {method}')
428
- raise PreAnnotationToTaskFailed(result.message)
429
-
430
- current_progress = self.run.logger.get_current_progress()
431
- if current_progress['overall'] != 100:
432
- result = ToTaskResult(
433
- status=JobStatus.FAILED, message='Pre-annotation to task failed. Current progress is not 100%'
434
- )
435
- raise PreAnnotationToTaskFailed(result.message)
436
-
437
- result = ToTaskResult(status=JobStatus.SUCCEEDED, message='Pre-annotation to task completed successfully')
438
- return result.model_dump()
439
-
440
- def _handle_annotate_data_from_files(self, task_ids: List[int], target_specification_name: str):
441
- """Handle annotate data from files to tasks.
442
-
443
- Args:
444
- task_ids (List[int]): List of task IDs to annotate data to.
445
- target_specification_name (str): The name of the target specification.
446
- """
447
- if not self.run or not self.params:
448
- raise ValueError('Run instance or parameters not properly initialized')
449
-
450
- # Type assertion to help the linter
451
- assert isinstance(self.run, ToTaskRun)
452
- assert isinstance(self.run.client, BackendClient)
453
-
454
- client: BackendClient = self.run.client
455
- task_params = {
456
- 'fields': 'id,data,data_unit',
457
- 'expand': 'data_unit',
458
- }
459
-
460
- total_tasks = len(task_ids)
461
- success_count = 0
462
- failed_count = 0
463
- current_progress = 0
464
-
465
- # Initialize metrics and progress
466
- self._update_metrics(total_tasks, success_count, failed_count)
467
- self.run.set_progress(0, total_tasks, category='annotate_task_data')
468
- self.run.log_message_with_code('ANNOTATING_DATA')
469
-
470
- # Process each task
471
- for task_id in task_ids:
472
- try:
473
- result = self._process_single_task(
474
- client, task_id, task_params, target_specification_name, AnnotationMethod.FILE
475
- )
476
- if result['success']:
477
- success_count += 1
478
- else:
479
- failed_count += 1
480
-
481
- current_progress += 1
482
- self._update_metrics(total_tasks, success_count, failed_count)
483
- self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
484
-
485
- except CriticalError:
486
- self.run.log_message_with_code('CRITICAL_ERROR')
487
- return
488
-
489
- except Exception as e:
490
- self.run.log_annotate_task_event('TASK_PROCESSING_FAILED', task_id, str(e))
491
- self.run.log_annotate_task_data({'task_id': task_id, 'error': str(e)}, AnnotateTaskDataStatus.FAILED)
492
- failed_count += 1
493
- current_progress += 1
494
- self._update_metrics(total_tasks, success_count, failed_count)
495
- self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
496
-
497
- # Finalize progress
498
- self.run.set_progress(total_tasks, total_tasks, category='annotate_task_data')
499
- self.run.log_message_with_code('ANNOTATION_COMPLETED', success_count, failed_count)
500
-
501
- def _process_single_task(
502
- self,
503
- client: BackendClient,
504
- task_id: int,
505
- task_params: Dict[str, Any],
506
- target_specification_name: Optional[str],
507
- method: AnnotationMethod,
508
- ) -> Dict[str, Any]:
509
- """Process a single task for annotation.
510
-
511
- Args:
512
- client (BackendClient): The backend client instance.
513
- task_id (int): The task ID to process.
514
- task_params (Dict[str, Any]): Parameters for getting task data.
515
- target_specification_name (Optional[str]): The name of the target specification.
516
- method (AnnotationMethod): The annotation method to use.
517
-
518
- Returns:
519
- Dict[str, Any]: Result dictionary with 'success' boolean and optional 'error' message.
520
- """
521
- if not self.run:
522
- raise ValueError('Run instance not properly initialized')
523
-
524
- # Type assertion to help the linter
525
- assert isinstance(self.run, ToTaskRun)
526
-
527
- # Get task data
528
- task_response = client.get_task(task_id, params=task_params)
529
- if isinstance(task_response, str):
530
- error_msg = 'Invalid task response'
531
- self.run.log_annotate_task_event('INVALID_TASK_RESPONSE', task_id)
532
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
533
- return {'success': False, 'error': error_msg}
534
-
535
- task: Dict[str, Any] = task_response
536
-
537
- if method == AnnotationMethod.FILE:
538
- if not target_specification_name:
539
- error_msg = 'Target specification name is required for file annotation method'
540
- self.run.log_message_with_code('TARGET_SPEC_REQUIRED_FOR_TASK', task_id)
541
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
542
- return {'success': False, 'error': error_msg}
543
- return self._process_single_task_with_file(client, task_id, task, target_specification_name)
544
- elif method == AnnotationMethod.INFERENCE:
545
- return self._process_single_task_with_inference(client, task_id, task)
546
- else:
547
- error_msg = f'Unsupported annotation method: {method}'
548
- self.run.log_annotate_task_event('UNSUPPORTED_METHOD_FOR_TASK', method, task_id)
549
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
550
- return {'success': False, 'error': error_msg}
551
-
552
- def _process_single_task_with_file(
553
- self, client: BackendClient, task_id: int, task: Dict[str, Any], target_specification_name: str
554
- ) -> Dict[str, Any]:
555
- """Process a single task for file-based annotation.
556
-
557
- Args:
558
- client (BackendClient): The backend client instance.
559
- task_id (int): The task ID to process.
560
- task (Dict[str, Any]): The task data.
561
- target_specification_name (str): The name of the target specification.
562
-
563
- Returns:
564
- Dict[str, Any]: Result dictionary with 'success' boolean and optional 'error' message.
565
- """
566
- if not self.run:
567
- raise ValueError('Run instance not properly initialized')
568
-
569
- # Type assertion to help the linter
570
- assert isinstance(self.run, ToTaskRun)
571
-
572
- # Extract data file information
573
- data_unit = task.get('data_unit', {})
574
- files = data_unit.get('files', {})
575
- data_file = files.get(target_specification_name)
576
-
577
- # Extract primary file URL from task data
578
- primary_file_url, primary_file_original_name = self._extract_primary_file_url(task)
579
- if not primary_file_url:
580
- error_msg = 'Primary image URL not found in task data'
581
- self.run.log_annotate_task_event('PRIMARY_IMAGE_URL_NOT_FOUND', task_id)
582
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
583
- return {'success': False, 'error': error_msg}
584
-
585
- if not data_file:
586
- error_msg = 'File specification not found'
587
- self.run.log_annotate_task_event('FILE_SPEC_NOT_FOUND', task_id)
588
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
589
- return {'success': False, 'error': error_msg}
590
-
591
- data_file_url = data_file.get('url')
592
- data_file_original_name = data_file.get('file_name_original')
593
- if not data_file_original_name:
594
- error_msg = 'File original name not found'
595
- self.run.log_annotate_task_event('FILE_ORIGINAL_NAME_NOT_FOUND', task_id)
596
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
597
- return {'success': False, 'error': error_msg}
598
-
599
- if not data_file_url:
600
- error_msg = 'URL not found'
601
- self.run.log_annotate_task_event('URL_NOT_FOUND', task_id)
602
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
603
- return {'success': False, 'error': error_msg}
604
-
605
- # Fetch and process the data
606
- try:
607
- # Convert data to task object
608
- annotation_to_task = self.entrypoint(self.run)
609
- converted_data = annotation_to_task.convert_data_from_file(
610
- primary_file_url, primary_file_original_name, data_file_url, data_file_original_name
611
- )
612
- except requests.RequestException as e:
613
- error_msg = f'Failed to fetch data from URL: {str(e)}'
614
- self.run.log_annotate_task_event('FETCH_DATA_FAILED', str(e), task_id)
615
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
616
- return {'success': False, 'error': error_msg}
617
- except Exception as e:
618
- error_msg = f'Failed to convert data to task object: {str(e)}'
619
- self.run.log_annotate_task_event('CONVERT_DATA_FAILED', str(e), task_id)
620
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
621
- return {'success': False, 'error': error_msg}
622
-
623
- # Submit annotation data
624
- client.annotate_task_data(task_id, data={'action': 'submit', 'data': converted_data})
625
-
626
- # Log success
627
- self.run.log_annotate_task_data({'task_id': task_id}, AnnotateTaskDataStatus.SUCCESS)
628
- return {'success': True}
629
-
630
- def _process_single_task_with_inference(
631
- self, client: BackendClient, task_id: int, task: Dict[str, Any]
632
- ) -> Dict[str, Any]:
633
- """Process a single task for inference-based annotation.
634
-
635
- Args:
636
- client (BackendClient): The backend client instance.
637
- task_id (int): The task ID to process.
638
- task (Dict[str, Any]): The task data.
639
-
640
- Returns:
641
- Dict[str, Any]: Result dictionary with 'success' boolean and optional 'error' message.
642
- """
643
- if not self.run or not self.params:
644
- raise ValueError('Run instance or parameters not properly initialized')
645
-
646
- # Type assertion to help the linter
647
- assert isinstance(self.run, ToTaskRun)
648
-
649
- try:
650
- # Validate pre-processor ID
651
- pre_processor_id = self.params.get('pre_processor')
652
- if not pre_processor_id:
653
- error_msg = 'Pre-processor ID is required for inference annotation method'
654
- self.run.log_message_with_code('PREPROCESSOR_ID_REQUIRED', task_id)
655
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
656
- return {'success': False, 'error': error_msg}
657
-
658
- # Get pre-processor information
659
- pre_processor_info = self._get_pre_processor_info(client, pre_processor_id)
660
- if not pre_processor_info['success']:
661
- return pre_processor_info
662
-
663
- pre_processor_code = pre_processor_info['code']
664
- pre_processor_version = pre_processor_info['version']
665
-
666
- # Ensure pre-processor is running
667
- pre_processor_status = self._ensure_pre_processor_running(client, pre_processor_code)
668
- if not pre_processor_status['success']:
669
- return pre_processor_status
670
-
671
- # Extract primary file URL from task data
672
- primary_file_url, _ = self._extract_primary_file_url(task)
673
- if not primary_file_url:
674
- error_msg = 'Primary image URL not found in task data'
675
- self.run.log_annotate_task_event('PRIMARY_IMAGE_URL_NOT_FOUND', task_id)
676
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
677
- return {'success': False, 'error': error_msg}
678
-
679
- # Run inference
680
- inference_result = self._run_inference(client, pre_processor_code, pre_processor_version, primary_file_url)
681
- if not inference_result['success']:
682
- return inference_result
683
-
684
- # Convert and submit inference data
685
- annotation_to_task = self.entrypoint(self.run)
686
- converted_result = annotation_to_task.convert_data_from_inference(inference_result['data'])
687
-
688
- # Submit inference annotation data
689
- client.annotate_task_data(task_id, data={'action': 'submit', 'data': converted_result})
690
-
691
- # Log success
692
- self.run.log_annotate_task_data(
693
- {'task_id': task_id, 'pre_processor_id': pre_processor_id}, AnnotateTaskDataStatus.SUCCESS
694
- )
695
- return {'success': True}
696
-
697
- except Exception as e:
698
- error_msg = f'Failed to process inference for task {task_id}: {str(e)}'
699
- self.run.log_message_with_code('INFERENCE_PROCESSING_FAILED', task_id, str(e))
700
- self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
701
- return {'success': False, 'error': error_msg}
702
-
703
- def _get_pre_processor_info(self, client: BackendClient, pre_processor_id: int) -> Dict[str, Any]:
704
- """Get pre-processor information from the backend.
705
-
706
- Args:
707
- client (BackendClient): The backend client instance.
708
- pre_processor_id (int): The pre-processor ID.
709
-
710
- Returns:
711
- Dict[str, Any]: Result dictionary with pre-processor info or error.
712
- """
713
- try:
714
- pre_processor_response = client.get_plugin_release(pre_processor_id)
715
- if isinstance(pre_processor_response, str):
716
- return {'success': False, 'error': 'Invalid pre-processor response received'}
717
-
718
- pre_processor: Dict[str, Any] = pre_processor_response
719
- config = pre_processor.get('config', {})
720
- code = config.get('code')
721
- version = pre_processor.get('version')
722
-
723
- if not code or not version:
724
- return {'success': False, 'error': 'Invalid pre-processor configuration'}
725
-
726
- return {'success': True, 'code': code, 'version': version}
727
- except Exception as e:
728
- return {'success': False, 'error': f'Failed to get pre-processor info: {str(e)}'}
729
-
730
- def _ensure_pre_processor_running(self, client: BackendClient, pre_processor_code: str) -> Dict[str, Any]:
731
- """Ensure the pre-processor is running, restart if necessary.
732
-
733
- Args:
734
- client (BackendClient): The backend client instance.
735
- pre_processor_code (str): The pre-processor code.
736
-
737
- Returns:
738
- Dict[str, Any]: Result dictionary indicating success or failure.
739
- """
740
- try:
741
- # Check if pre-processor is running
742
- serve_applications_response = client.list_serve_applications(params={'plugin_code': pre_processor_code})
743
- if isinstance(serve_applications_response, str):
744
- return {'success': False, 'error': 'Invalid serve applications response'}
745
-
746
- # Handle the response properly - it should be a dict with 'results' key
747
- if not isinstance(serve_applications_response, dict):
748
- return {'success': False, 'error': 'Unexpected serve applications response format'}
749
-
750
- serve_applications: Dict[str, Any] = serve_applications_response
751
- results = serve_applications.get('results', [])
752
- running_serve_apps = [app for app in results if isinstance(app, dict) and app.get('status') == 'RUNNING']
753
-
754
- # If not running, restart the pre-processor
755
- if not running_serve_apps:
756
- restart_result = self._restart_pre_processor(client, pre_processor_code)
757
- if not restart_result['success']:
758
- return restart_result
759
-
760
- # Verify restart was successful
761
- serve_applications_response = client.list_serve_applications(params={'plugin_code': pre_processor_code})
762
- if isinstance(serve_applications_response, str):
763
- return {'success': False, 'error': 'Failed to verify pre-processor restart'}
764
-
765
- if not isinstance(serve_applications_response, dict):
766
- return {'success': False, 'error': 'Unexpected serve applications response format after restart'}
767
-
768
- serve_applications: Dict[str, Any] = serve_applications_response
769
- results = serve_applications.get('results', [])
770
- running_serve_apps = [
771
- app for app in results if isinstance(app, dict) and app.get('status') == 'RUNNING'
772
- ]
773
-
774
- if not running_serve_apps:
775
- return {'success': False, 'error': 'Failed to restart pre-processor'}
776
-
777
- return {'success': True}
778
- except Exception as e:
779
- return {'success': False, 'error': f'Failed to ensure pre-processor running: {str(e)}'}
780
-
781
- def _restart_pre_processor(self, client: BackendClient, pre_processor_code: str) -> Dict[str, Any]:
782
- """Restart the pre-processor.
783
-
784
- Args:
785
- client (BackendClient): The backend client instance.
786
- pre_processor_code (str): The pre-processor code.
787
-
788
- Returns:
789
- Dict[str, Any]: Result dictionary indicating success or failure.
790
- """
791
- try:
792
- if not self.config:
793
- return {'success': False, 'error': 'Configuration not available'}
794
-
795
- inference_options = self.config.get('inference_options', {})
796
- serve_application_deployment_payload = {
797
- 'agent': self.params.get('agent') if self.params else None,
798
- 'action': 'deployment',
799
- 'params': {
800
- 'num_cpus': inference_options.get('required_cpu_count', 2),
801
- 'num_gpus': inference_options.get('required_gpu_count', 1),
802
- },
803
- 'debug': True,
804
- }
805
-
806
- deployment_result = client.run_plugin(pre_processor_code, serve_application_deployment_payload)
807
- if not deployment_result:
808
- return {'success': False, 'error': 'Failed to restart pre-processor'}
809
-
810
- return {'success': True}
811
- except Exception as e:
812
- return {'success': False, 'error': f'Failed to restart pre-processor: {str(e)}'}
813
-
814
- def _extract_primary_file_url(self, task: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
815
- """Extract the primary file URL from task data.
816
-
817
- Args:
818
- task (Dict[str, Any]): The task data.
819
-
820
- Returns:
821
- Tuple[Optional[str], Optional[str]]: The primary file URL and original name.
822
- """
823
- data_unit = task.get('data_unit', {})
824
- files = data_unit.get('files', {})
825
-
826
- for file_info in files.values():
827
- if isinstance(file_info, dict) and file_info.get('is_primary') and file_info.get('url'):
828
- return file_info['url'], file_info.get('file_name_original')
829
-
830
- return None, None
831
-
832
- def _run_inference(
833
- self, client: BackendClient, pre_processor_code: str, pre_processor_version: str, primary_file_url: str
834
- ) -> Dict[str, Any]:
835
- """Run inference using the pre-processor.
836
-
837
- Args:
838
- client (BackendClient): The backend client instance.
839
- pre_processor_code (str): The pre-processor code.
840
- pre_processor_version (str): The pre-processor version.
841
- primary_file_url (str): The primary image URL.
842
-
843
- Returns:
844
- Dict[str, Any]: Result dictionary with inference data or error.
845
- """
846
- try:
847
- if not self.params:
848
- return {'success': False, 'error': 'Parameters not available'}
849
-
850
- pre_processor_params = self.params.get('pre_processor_params', {})
851
- pre_processor_params['image_path'] = primary_file_url
852
-
853
- inference_payload = {
854
- 'agent': 1,
855
- 'action': 'inference',
856
- 'version': pre_processor_version,
857
- 'params': {
858
- 'model': self.params['model'],
859
- 'method': 'post',
860
- 'json': pre_processor_params,
861
- },
862
- }
863
-
864
- inference_data = client.run_plugin(pre_processor_code, inference_payload)
865
- return {'success': True, 'data': inference_data}
866
- except Exception as e:
867
- return {'success': False, 'error': f'Failed to run inference: {str(e)}'}
868
-
869
- def _update_metrics(self, total_tasks: int, success_count: int, failed_count: int):
870
- """Update metrics for task annotation progress.
871
-
872
- Args:
873
- total_tasks (int): Total number of tasks to process.
874
- success_count (int): Number of successfully processed tasks.
875
- failed_count (int): Number of failed tasks.
876
- """
877
- if not self.run:
878
- raise ValueError('Run instance not properly initialized')
879
-
880
- # Type assertion to help the linter
881
- assert isinstance(self.run, ToTaskRun)
882
-
883
- metrics = self.run.MetricsRecord(
884
- stand_by=total_tasks - success_count - failed_count, failed=failed_count, success=success_count
885
- )
886
- self.run.log_metrics(metrics, 'annotate_task_data')
887
-
888
- def _handle_annotate_data_with_inference(self, task_ids: List[int]):
889
- """Handle annotate data with inference to tasks.
890
-
891
- Args:
892
- task_ids (List[int]): List of task IDs to annotate data to.
893
- """
894
- if not self.run or not self.params:
895
- raise ValueError('Run instance or parameters not properly initialized')
896
-
897
- if not self.params.get('model'):
898
- raise ValueError('Model is required for inference annotation method')
899
-
900
- # Type assertion to help the linter
901
- assert isinstance(self.run, ToTaskRun)
902
- assert isinstance(self.run.client, BackendClient)
903
-
904
- client: BackendClient = self.run.client
905
- task_params = {
906
- 'fields': 'id,data,data_unit',
907
- 'expand': 'data_unit',
908
- }
909
-
910
- total_tasks = len(task_ids)
911
- success_count = 0
912
- failed_count = 0
913
- current_progress = 0
914
-
915
- # Initialize metrics and progress
916
- self._update_metrics(total_tasks, success_count, failed_count)
917
- self.run.set_progress(0, total_tasks, category='annotate_task_data')
918
- self.run.log_message_with_code('ANNOTATING_INFERENCE_DATA')
919
-
920
- # Process each task
921
- for task_id in task_ids:
922
- try:
923
- result = self._process_single_task(client, task_id, task_params, None, AnnotationMethod.INFERENCE)
924
- if result['success']:
925
- success_count += 1
926
- else:
927
- failed_count += 1
928
-
929
- current_progress += 1
930
- self._update_metrics(total_tasks, success_count, failed_count)
931
- self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
932
-
933
- except Exception as e:
934
- self.run.log_annotate_task_event('TASK_PROCESSING_FAILED', task_id, str(e))
935
- self.run.log_annotate_task_data({'task_id': task_id, 'error': str(e)}, AnnotateTaskDataStatus.FAILED)
936
- failed_count += 1
937
- current_progress += 1
938
- self._update_metrics(total_tasks, success_count, failed_count)
939
- self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
940
-
941
- # Finalize progress
942
- self.run.set_progress(total_tasks, total_tasks, category='annotate_task_data')
943
- self.run.log_message_with_code('INFERENCE_ANNOTATION_COMPLETED', success_count, failed_count)