synapse-sdk 2025.9.5__py3-none-any.whl → 2025.10.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse-sdk might be problematic. Click here for more details.
- synapse_sdk/clients/base.py +129 -9
- synapse_sdk/devtools/docs/docs/api/clients/base.md +230 -8
- synapse_sdk/devtools/docs/docs/api/plugins/models.md +58 -3
- synapse_sdk/devtools/docs/docs/plugins/categories/neural-net-plugins/train-action-overview.md +663 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-action.md +934 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-overview.md +585 -0
- synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-template.md +715 -0
- synapse_sdk/devtools/docs/docs/plugins/export-plugins.md +39 -0
- synapse_sdk/devtools/docs/docs/plugins/plugins.md +12 -5
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/base.md +230 -8
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/plugins/models.md +114 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/neural-net-plugins/train-action-overview.md +621 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md +198 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-action-development.md +1645 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-overview.md +717 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-template-development.md +1380 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-action.md +934 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-overview.md +585 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-template.md +715 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md +39 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current.json +16 -4
- synapse_sdk/devtools/docs/sidebars.ts +45 -1
- synapse_sdk/plugins/README.md +487 -80
- synapse_sdk/plugins/categories/base.py +1 -0
- synapse_sdk/plugins/categories/export/actions/export/action.py +8 -3
- synapse_sdk/plugins/categories/export/actions/export/utils.py +108 -8
- synapse_sdk/plugins/categories/export/templates/config.yaml +18 -0
- synapse_sdk/plugins/categories/export/templates/plugin/export.py +97 -0
- synapse_sdk/plugins/categories/neural_net/actions/train.py +592 -22
- synapse_sdk/plugins/categories/neural_net/actions/tune.py +150 -3
- synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py +4 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/__init__.py +3 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation/action.py +10 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/__init__.py +28 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/action.py +145 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/enums.py +269 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/exceptions.py +14 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/factory.py +76 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/models.py +97 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/orchestrator.py +250 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/run.py +64 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/__init__.py +17 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/annotation.py +284 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/base.py +170 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/extraction.py +83 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/metrics.py +87 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/preprocessor.py +127 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/validation.py +143 -0
- synapse_sdk/plugins/categories/upload/actions/upload/__init__.py +2 -1
- synapse_sdk/plugins/categories/upload/actions/upload/action.py +8 -1
- synapse_sdk/plugins/categories/upload/actions/upload/context.py +0 -1
- synapse_sdk/plugins/categories/upload/actions/upload/models.py +134 -94
- synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py +2 -2
- synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py +6 -2
- synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py +24 -9
- synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py +130 -18
- synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py +147 -37
- synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py +10 -5
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py +31 -6
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/recursive.py +65 -37
- synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py +17 -2
- synapse_sdk/plugins/categories/upload/templates/README.md +394 -0
- synapse_sdk/plugins/models.py +62 -0
- synapse_sdk/utils/file/download.py +261 -0
- {synapse_sdk-2025.9.5.dist-info → synapse_sdk-2025.10.6.dist-info}/METADATA +15 -2
- {synapse_sdk-2025.9.5.dist-info → synapse_sdk-2025.10.6.dist-info}/RECORD +74 -43
- synapse_sdk/devtools/docs/docs/plugins/developing-upload-template.md +0 -1463
- synapse_sdk/devtools/docs/docs/plugins/upload-plugins.md +0 -1964
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/developing-upload-template.md +0 -1463
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/upload-plugins.md +0 -2077
- {synapse_sdk-2025.9.5.dist-info → synapse_sdk-2025.10.6.dist-info}/WHEEL +0 -0
- {synapse_sdk-2025.9.5.dist-info → synapse_sdk-2025.10.6.dist-info}/entry_points.txt +0 -0
- {synapse_sdk-2025.9.5.dist-info → synapse_sdk-2025.10.6.dist-info}/licenses/LICENSE +0 -0
- {synapse_sdk-2025.9.5.dist-info → synapse_sdk-2025.10.6.dist-info}/top_level.txt +0 -0
|
@@ -23,16 +23,86 @@ class TuneRun(TrainRun):
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
class SearchAlgo(BaseModel):
|
|
26
|
+
"""
|
|
27
|
+
Configuration for Ray Tune search algorithms.
|
|
28
|
+
|
|
29
|
+
Supported algorithms:
|
|
30
|
+
- 'bayesoptsearch': Bayesian optimization using Gaussian Processes
|
|
31
|
+
- 'hyperoptsearch': Tree-structured Parzen Estimator (TPE)
|
|
32
|
+
- 'basicvariantgenerator': Random search (default)
|
|
33
|
+
|
|
34
|
+
Attributes:
|
|
35
|
+
name (str): Name of the search algorithm (case-insensitive)
|
|
36
|
+
points_to_evaluate (Optional[dict]): Optional initial hyperparameter
|
|
37
|
+
configurations to evaluate before starting optimization
|
|
38
|
+
|
|
39
|
+
Example:
|
|
40
|
+
{
|
|
41
|
+
"name": "hyperoptsearch",
|
|
42
|
+
"points_to_evaluate": [
|
|
43
|
+
{"learning_rate": 0.001, "batch_size": 32}
|
|
44
|
+
]
|
|
45
|
+
}
|
|
46
|
+
"""
|
|
47
|
+
|
|
26
48
|
name: str
|
|
27
49
|
points_to_evaluate: Optional[dict] = None
|
|
28
50
|
|
|
29
51
|
|
|
30
52
|
class Scheduler(BaseModel):
|
|
53
|
+
"""
|
|
54
|
+
Configuration for Ray Tune schedulers.
|
|
55
|
+
|
|
56
|
+
Supported schedulers:
|
|
57
|
+
- 'fifo': First-In-First-Out scheduler (default, runs all trials)
|
|
58
|
+
- 'hyperband': HyperBand early stopping scheduler
|
|
59
|
+
|
|
60
|
+
Attributes:
|
|
61
|
+
name (str): Name of the scheduler (case-insensitive)
|
|
62
|
+
options (Optional[str]): Optional scheduler-specific configuration parameters
|
|
63
|
+
|
|
64
|
+
Example:
|
|
65
|
+
{
|
|
66
|
+
"name": "hyperband",
|
|
67
|
+
"options": {
|
|
68
|
+
"max_t": 100,
|
|
69
|
+
"reduction_factor": 3
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
"""
|
|
73
|
+
|
|
31
74
|
name: str
|
|
32
75
|
options: Optional[str] = None
|
|
33
76
|
|
|
34
77
|
|
|
35
78
|
class TuneConfig(BaseModel):
|
|
79
|
+
"""
|
|
80
|
+
Configuration for Ray Tune hyperparameter optimization.
|
|
81
|
+
|
|
82
|
+
Attributes:
|
|
83
|
+
mode (Optional[str]): Optimization mode - 'max' or 'min'
|
|
84
|
+
metric (Optional[str]): Name of the metric to optimize
|
|
85
|
+
num_samples (int): Number of hyperparameter configurations to try (default: 1)
|
|
86
|
+
max_concurrent_trials (Optional[int]): Maximum number of trials to run in parallel
|
|
87
|
+
search_alg (Optional[SearchAlgo]): Search algorithm configuration
|
|
88
|
+
scheduler (Optional[Scheduler]): Trial scheduler configuration
|
|
89
|
+
|
|
90
|
+
Example:
|
|
91
|
+
{
|
|
92
|
+
"mode": "max",
|
|
93
|
+
"metric": "accuracy",
|
|
94
|
+
"num_samples": 20,
|
|
95
|
+
"max_concurrent_trials": 4,
|
|
96
|
+
"search_alg": {
|
|
97
|
+
"name": "hyperoptsearch"
|
|
98
|
+
},
|
|
99
|
+
"scheduler": {
|
|
100
|
+
"name": "hyperband",
|
|
101
|
+
"options": {"max_t": 100}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
"""
|
|
105
|
+
|
|
36
106
|
mode: Optional[str] = None
|
|
37
107
|
metric: Optional[str] = None
|
|
38
108
|
num_samples: int = 1
|
|
@@ -42,10 +112,51 @@ class TuneConfig(BaseModel):
|
|
|
42
112
|
|
|
43
113
|
|
|
44
114
|
class TuneParams(BaseModel):
|
|
115
|
+
"""
|
|
116
|
+
Parameters for TuneAction (DEPRECATED - use TrainAction with is_tune=True instead).
|
|
117
|
+
|
|
118
|
+
Attributes:
|
|
119
|
+
name (str): Name for the tuning job
|
|
120
|
+
description (str): Description of the job
|
|
121
|
+
checkpoint (int | None): Optional checkpoint ID to resume from
|
|
122
|
+
dataset (int): Dataset ID to use for training
|
|
123
|
+
hyperparameter (list): Hyperparameter search space
|
|
124
|
+
tune_config (TuneConfig): Tune configuration
|
|
125
|
+
|
|
126
|
+
Hyperparameter format:
|
|
127
|
+
Each item in hyperparameter list must have:
|
|
128
|
+
- 'name': Parameter name (string)
|
|
129
|
+
- 'type': Distribution type (string)
|
|
130
|
+
- Type-specific parameters:
|
|
131
|
+
- uniform/quniform: 'min', 'max'
|
|
132
|
+
- loguniform/qloguniform: 'min', 'max', 'base'
|
|
133
|
+
- randn/qrandn: 'mean', 'sd'
|
|
134
|
+
- randint/qrandint: 'min', 'max'
|
|
135
|
+
- lograndint/qlograndint: 'min', 'max', 'base'
|
|
136
|
+
- choice/grid_search: 'options'
|
|
137
|
+
|
|
138
|
+
Example:
|
|
139
|
+
{
|
|
140
|
+
"name": "my_tuning",
|
|
141
|
+
"dataset": 123,
|
|
142
|
+
"hyperparameter": [
|
|
143
|
+
{"name": "batch_size", "type": "choice", "options": [16, 32, 64]},
|
|
144
|
+
{"name": "learning_rate", "type": "loguniform", "min": 0.0001, "max": 0.01, "base": 10},
|
|
145
|
+
{"name": "epochs", "type": "randint", "min": 5, "max": 15}
|
|
146
|
+
],
|
|
147
|
+
"tune_config": {
|
|
148
|
+
"mode": "max",
|
|
149
|
+
"metric": "accuracy",
|
|
150
|
+
"num_samples": 10
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
"""
|
|
154
|
+
|
|
45
155
|
name: Annotated[str, AfterValidator(non_blank)]
|
|
46
156
|
description: str
|
|
47
157
|
checkpoint: int | None
|
|
48
158
|
dataset: int
|
|
159
|
+
hyperparameter: list
|
|
49
160
|
tune_config: TuneConfig
|
|
50
161
|
|
|
51
162
|
@field_validator('name')
|
|
@@ -73,6 +184,23 @@ class TuneParams(BaseModel):
|
|
|
73
184
|
@register_action
|
|
74
185
|
class TuneAction(TrainAction):
|
|
75
186
|
"""
|
|
187
|
+
**DEPRECATED**: This action is deprecated. Please use TrainAction with is_tune=True instead.
|
|
188
|
+
|
|
189
|
+
To migrate from tune to train with tuning:
|
|
190
|
+
- Change action from "tune" to "train"
|
|
191
|
+
- Add "is_tune": true to params
|
|
192
|
+
- Keep tune_config and hyperparameter as they are
|
|
193
|
+
|
|
194
|
+
Example:
|
|
195
|
+
{
|
|
196
|
+
"action": "train",
|
|
197
|
+
"params": {
|
|
198
|
+
"is_tune": true,
|
|
199
|
+
"tune_config": { ... },
|
|
200
|
+
"hyperparameter": [ ... ]
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
76
204
|
**Must read** Important notes before using Tune:
|
|
77
205
|
|
|
78
206
|
1. Path to the model output (which is the return value of your train function)
|
|
@@ -256,6 +384,10 @@ class TuneAction(TrainAction):
|
|
|
256
384
|
|
|
257
385
|
Returns:
|
|
258
386
|
object: Ray Tune scheduler instance.
|
|
387
|
+
|
|
388
|
+
Supported schedulers:
|
|
389
|
+
- 'fifo': FIFOScheduler (default)
|
|
390
|
+
- 'hyperband': HyperBandScheduler
|
|
259
391
|
"""
|
|
260
392
|
|
|
261
393
|
from ray.tune.schedulers import (
|
|
@@ -278,7 +410,12 @@ class TuneAction(TrainAction):
|
|
|
278
410
|
}
|
|
279
411
|
|
|
280
412
|
scheduler_type = tune_config['scheduler'].get('name', 'fifo').lower()
|
|
281
|
-
scheduler_class = scheduler_map.get(scheduler_type
|
|
413
|
+
scheduler_class = scheduler_map.get(scheduler_type)
|
|
414
|
+
|
|
415
|
+
if scheduler_class is None:
|
|
416
|
+
raise ValueError(
|
|
417
|
+
f'Unsupported scheduler: {scheduler_type}. Supported schedulers are: {", ".join(scheduler_map.keys())}'
|
|
418
|
+
)
|
|
282
419
|
|
|
283
420
|
# 옵션이 있는 경우 전달하고, 없으면 기본 생성자 호출
|
|
284
421
|
options = tune_config['scheduler'].get('options')
|
|
@@ -291,13 +428,18 @@ class TuneAction(TrainAction):
|
|
|
291
428
|
@staticmethod
|
|
292
429
|
def convert_tune_search_alg(tune_config):
|
|
293
430
|
"""
|
|
294
|
-
Convert YAML hyperparameter configuration to Ray Tune search algorithm
|
|
431
|
+
Convert YAML hyperparameter configuration to Ray Tune search algorithm.
|
|
295
432
|
|
|
296
433
|
Args:
|
|
297
434
|
tune_config (dict): Hyperparameter configuration.
|
|
298
435
|
|
|
299
436
|
Returns:
|
|
300
|
-
|
|
437
|
+
object: Ray Tune search algorithm instance or None
|
|
438
|
+
|
|
439
|
+
Supported search algorithms:
|
|
440
|
+
- 'bayesoptsearch': Bayesian optimization
|
|
441
|
+
- 'hyperoptsearch': Tree-structured Parzen Estimator
|
|
442
|
+
- 'basicvariantgenerator': Random search (default)
|
|
301
443
|
"""
|
|
302
444
|
|
|
303
445
|
if tune_config.get('search_alg') is None:
|
|
@@ -328,6 +470,11 @@ class TuneAction(TrainAction):
|
|
|
328
470
|
from ray.tune.search.basic_variant import BasicVariantGenerator
|
|
329
471
|
|
|
330
472
|
search_alg = BasicVariantGenerator(points_to_evaluate=points_to_evaluate)
|
|
473
|
+
else:
|
|
474
|
+
raise ValueError(
|
|
475
|
+
f'Unsupported search algorithm: {search_alg_name}. '
|
|
476
|
+
f'Supported algorithms are: bayesoptsearch, hyperoptsearch, basicvariantgenerator'
|
|
477
|
+
)
|
|
331
478
|
|
|
332
479
|
return search_alg
|
|
333
480
|
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import PluginCategory, RunMethod
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class PreAnnotationAction(Action):
|
|
8
|
+
name = 'pre_annotation'
|
|
9
|
+
category = PluginCategory.PRE_ANNOTATION
|
|
10
|
+
method = RunMethod.TASK
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from .action import ToTaskAction
|
|
2
|
+
from .enums import AnnotateTaskDataStatus, AnnotationMethod, LogCode
|
|
3
|
+
from .exceptions import CriticalError, PreAnnotationToTaskFailed
|
|
4
|
+
|
|
5
|
+
# Advanced imports for extending the system
|
|
6
|
+
from .factory import ToTaskStrategyFactory
|
|
7
|
+
from .models import MetricsRecord, ToTaskParams, ToTaskResult
|
|
8
|
+
from .orchestrator import ToTaskOrchestrator
|
|
9
|
+
from .run import ToTaskRun
|
|
10
|
+
from .strategies.base import ToTaskContext
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
# Core public API (maintains backward compatibility)
|
|
14
|
+
'ToTaskAction',
|
|
15
|
+
'ToTaskRun',
|
|
16
|
+
'ToTaskParams',
|
|
17
|
+
'ToTaskResult',
|
|
18
|
+
'AnnotationMethod',
|
|
19
|
+
'AnnotateTaskDataStatus',
|
|
20
|
+
'LogCode',
|
|
21
|
+
'CriticalError',
|
|
22
|
+
'PreAnnotationToTaskFailed',
|
|
23
|
+
'MetricsRecord',
|
|
24
|
+
# Advanced components for customization and testing
|
|
25
|
+
'ToTaskOrchestrator',
|
|
26
|
+
'ToTaskContext',
|
|
27
|
+
'ToTaskStrategyFactory',
|
|
28
|
+
]
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Refactored ToTask action using Strategy and Facade patterns."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict
|
|
4
|
+
|
|
5
|
+
from synapse_sdk.clients.backend import BackendClient
|
|
6
|
+
from synapse_sdk.clients.backend.models import JobStatus
|
|
7
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
8
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
9
|
+
from synapse_sdk.plugins.enums import PluginCategory, RunMethod
|
|
10
|
+
|
|
11
|
+
from .enums import LogCode
|
|
12
|
+
from .exceptions import PreAnnotationToTaskFailed
|
|
13
|
+
from .models import ToTaskParams, ToTaskResult
|
|
14
|
+
from .orchestrator import ToTaskOrchestrator
|
|
15
|
+
from .run import ToTaskRun
|
|
16
|
+
from .strategies.base import ToTaskContext
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@register_action
|
|
20
|
+
class ToTaskAction(Action):
|
|
21
|
+
"""ToTask action for pre-annotation data processing using Strategy and Facade patterns.
|
|
22
|
+
|
|
23
|
+
This action handles the process of annotating data to tasks in a project. It supports
|
|
24
|
+
two annotation methods: file-based annotation and inference-based annotation.
|
|
25
|
+
|
|
26
|
+
The action uses a Strategy pattern to handle different annotation methods and validation
|
|
27
|
+
approaches, coordinated by an Orchestrator (Facade pattern) that manages the complete
|
|
28
|
+
workflow with rollback capabilities.
|
|
29
|
+
|
|
30
|
+
File-based annotation fetches data from file URLs specified in task data units,
|
|
31
|
+
downloads and processes JSON data, and updates task data with the processed information.
|
|
32
|
+
It also validates target specification names against file specifications.
|
|
33
|
+
|
|
34
|
+
Inference-based annotation uses pre-processor plugins for model inference
|
|
35
|
+
for automatic data annotation.
|
|
36
|
+
|
|
37
|
+
Attrs:
|
|
38
|
+
name (str): Action name, set to 'to_task'.
|
|
39
|
+
category (PluginCategory): Plugin category, set to PRE_ANNOTATION.
|
|
40
|
+
method (RunMethod): Execution method, set to JOB.
|
|
41
|
+
run_class (Type[ToTaskRun]): Run class for this action.
|
|
42
|
+
params_model (Type[ToTaskParams]): Parameter validation model.
|
|
43
|
+
progress_categories (Dict): Progress tracking configuration.
|
|
44
|
+
metrics_categories (Set[str]): Metrics categories for this action.
|
|
45
|
+
|
|
46
|
+
Note:
|
|
47
|
+
This action requires a valid project with an associated data collection.
|
|
48
|
+
For file-based annotation, the target_specification_name must exist in the
|
|
49
|
+
project's file specifications.
|
|
50
|
+
|
|
51
|
+
Raises:
|
|
52
|
+
ValueError: If run instance or parameters are not properly initialized.
|
|
53
|
+
PreAnnotationToTaskFailed: If the annotation workflow fails.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
name = 'to_task'
|
|
57
|
+
category = PluginCategory.PRE_ANNOTATION
|
|
58
|
+
method = RunMethod.JOB
|
|
59
|
+
run_class = ToTaskRun
|
|
60
|
+
params_model = ToTaskParams
|
|
61
|
+
progress_categories = {
|
|
62
|
+
'annotate_task_data': {
|
|
63
|
+
'proportion': 100,
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
metrics_categories = {
|
|
67
|
+
'annotate_task_data': {
|
|
68
|
+
'stand_by': 0,
|
|
69
|
+
'failed': 0,
|
|
70
|
+
'success': 0,
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
def __init__(self, *args, **kwargs):
|
|
75
|
+
"""Initialize the action with orchestrator context."""
|
|
76
|
+
super().__init__(*args, **kwargs)
|
|
77
|
+
self.context = None
|
|
78
|
+
|
|
79
|
+
def start(self) -> Dict:
|
|
80
|
+
"""Start to_task action using orchestrator facade.
|
|
81
|
+
|
|
82
|
+
The action now uses a simplified workflow:
|
|
83
|
+
1. Validate initialization
|
|
84
|
+
2. Create execution context
|
|
85
|
+
3. Execute workflow through orchestrator
|
|
86
|
+
4. Handle results and errors
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
dict: Validated result with status and message.
|
|
90
|
+
"""
|
|
91
|
+
# Validate initialization
|
|
92
|
+
if not self.run or not self.params:
|
|
93
|
+
result = ToTaskResult(
|
|
94
|
+
status=JobStatus.FAILED, message='Run instance or parameters not properly initialized'
|
|
95
|
+
)
|
|
96
|
+
raise PreAnnotationToTaskFailed(result.message)
|
|
97
|
+
|
|
98
|
+
# Type assertions for better IDE support
|
|
99
|
+
assert isinstance(self.run, ToTaskRun)
|
|
100
|
+
assert isinstance(self.run.client, BackendClient)
|
|
101
|
+
|
|
102
|
+
# Log action start
|
|
103
|
+
self.run.log_message_with_code(LogCode.TO_TASK_STARTED)
|
|
104
|
+
|
|
105
|
+
try:
|
|
106
|
+
# Create execution context
|
|
107
|
+
self.context = ToTaskContext(
|
|
108
|
+
params=self.params,
|
|
109
|
+
client=self.run.client,
|
|
110
|
+
logger=self.run,
|
|
111
|
+
entrypoint=self.entrypoint,
|
|
112
|
+
config=self.config,
|
|
113
|
+
plugin_config=self.plugin_config,
|
|
114
|
+
job_id=self.job_id,
|
|
115
|
+
progress_categories=self.progress_categories,
|
|
116
|
+
metrics_categories=self.metrics_categories,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# Create and execute orchestrator
|
|
120
|
+
orchestrator = ToTaskOrchestrator(self.context)
|
|
121
|
+
result = orchestrator.execute_workflow()
|
|
122
|
+
|
|
123
|
+
# Log successful completion
|
|
124
|
+
self.run.log_message_with_code(LogCode.TO_TASK_COMPLETED)
|
|
125
|
+
return result
|
|
126
|
+
|
|
127
|
+
except PreAnnotationToTaskFailed as e:
|
|
128
|
+
# Re-raise pre-annotation specific errors
|
|
129
|
+
self.run.log_message_with_code(LogCode.TO_TASK_FAILED, str(e))
|
|
130
|
+
raise e
|
|
131
|
+
|
|
132
|
+
except Exception as e:
|
|
133
|
+
# Handle unexpected errors
|
|
134
|
+
error_msg = f'ToTask action failed: {str(e)}'
|
|
135
|
+
self.run.log_message_with_code(LogCode.TO_TASK_FAILED, error_msg)
|
|
136
|
+
result = ToTaskResult(status=JobStatus.FAILED, message=error_msg)
|
|
137
|
+
raise PreAnnotationToTaskFailed(result.message)
|
|
138
|
+
|
|
139
|
+
def get_context(self) -> ToTaskContext:
|
|
140
|
+
"""Get the current execution context for testing/debugging.
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
ToTaskContext: The current execution context, or None if not initialized.
|
|
144
|
+
"""
|
|
145
|
+
return self.context
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
from synapse_sdk.shared.enums import Context
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class AnnotationMethod(str, Enum):
|
|
7
|
+
FILE = 'file'
|
|
8
|
+
INFERENCE = 'inference'
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AnnotateTaskDataStatus(str, Enum):
|
|
12
|
+
SUCCESS = 'success'
|
|
13
|
+
FAILED = 'failed'
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class LogCode(str, Enum):
|
|
17
|
+
"""Type-safe logging codes for to_task operations.
|
|
18
|
+
|
|
19
|
+
Enumeration of all possible log events during to_task processing. Each code
|
|
20
|
+
corresponds to a specific event or error state with predefined message
|
|
21
|
+
templates and log levels.
|
|
22
|
+
|
|
23
|
+
The codes are organized by category:
|
|
24
|
+
- Validation codes (INVALID_PROJECT_RESPONSE, NO_DATA_COLLECTION, etc.)
|
|
25
|
+
- Processing codes (ANNOTATING_DATA, ANNOTATION_COMPLETED, etc.)
|
|
26
|
+
- Error codes (CRITICAL_ERROR, TASK_PROCESSING_FAILED, etc.)
|
|
27
|
+
- Inference codes (ANNOTATING_INFERENCE_DATA, INFERENCE_PROCESSING_FAILED, etc.)
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
INVALID_PROJECT_RESPONSE = 'INVALID_PROJECT_RESPONSE'
|
|
31
|
+
NO_DATA_COLLECTION = 'NO_DATA_COLLECTION'
|
|
32
|
+
INVALID_DATA_COLLECTION_RESPONSE = 'INVALID_DATA_COLLECTION_RESPONSE'
|
|
33
|
+
NO_TASKS_FOUND = 'NO_TASKS_FOUND'
|
|
34
|
+
TARGET_SPEC_REQUIRED = 'TARGET_SPEC_REQUIRED'
|
|
35
|
+
TARGET_SPEC_NOT_FOUND = 'TARGET_SPEC_NOT_FOUND'
|
|
36
|
+
UNSUPPORTED_METHOD = 'UNSUPPORTED_METHOD'
|
|
37
|
+
ANNOTATING_DATA = 'ANNOTATING_DATA'
|
|
38
|
+
CRITICAL_ERROR = 'CRITICAL_ERROR'
|
|
39
|
+
TASK_PROCESSING_FAILED = 'TASK_PROCESSING_FAILED'
|
|
40
|
+
ANNOTATION_COMPLETED = 'ANNOTATION_COMPLETED'
|
|
41
|
+
INVALID_TASK_RESPONSE = 'INVALID_TASK_RESPONSE'
|
|
42
|
+
TARGET_SPEC_REQUIRED_FOR_TASK = 'TARGET_SPEC_REQUIRED_FOR_TASK'
|
|
43
|
+
UNSUPPORTED_METHOD_FOR_TASK = 'UNSUPPORTED_METHOD_FOR_TASK'
|
|
44
|
+
PRIMARY_IMAGE_URL_NOT_FOUND = 'PRIMARY_IMAGE_URL_NOT_FOUND'
|
|
45
|
+
FILE_SPEC_NOT_FOUND = 'FILE_SPEC_NOT_FOUND'
|
|
46
|
+
FILE_ORIGINAL_NAME_NOT_FOUND = 'FILE_ORIGINAL_NAME_NOT_FOUND'
|
|
47
|
+
URL_NOT_FOUND = 'URL_NOT_FOUND'
|
|
48
|
+
FETCH_DATA_FAILED = 'FETCH_DATA_FAILED'
|
|
49
|
+
CONVERT_DATA_FAILED = 'CONVERT_DATA_FAILED'
|
|
50
|
+
PREPROCESSOR_ID_REQUIRED = 'PREPROCESSOR_ID_REQUIRED'
|
|
51
|
+
INFERENCE_PROCESSING_FAILED = 'INFERENCE_PROCESSING_FAILED'
|
|
52
|
+
ANNOTATING_INFERENCE_DATA = 'ANNOTATING_INFERENCE_DATA'
|
|
53
|
+
INFERENCE_ANNOTATION_COMPLETED = 'INFERENCE_ANNOTATION_COMPLETED'
|
|
54
|
+
INFERENCE_PREPROCESSOR_FAILED = 'INFERENCE_PREPROCESSOR_FAILED'
|
|
55
|
+
|
|
56
|
+
# Orchestrator workflow codes
|
|
57
|
+
TO_TASK_STARTED = 'TO_TASK_STARTED'
|
|
58
|
+
TO_TASK_COMPLETED = 'TO_TASK_COMPLETED'
|
|
59
|
+
TO_TASK_FAILED = 'TO_TASK_FAILED'
|
|
60
|
+
STEP_STARTED = 'STEP_STARTED'
|
|
61
|
+
STEP_COMPLETED = 'STEP_COMPLETED'
|
|
62
|
+
STEP_FAILED = 'STEP_FAILED'
|
|
63
|
+
ROLLBACK_FAILED = 'ROLLBACK_FAILED'
|
|
64
|
+
ROLLBACK_ACTION_FAILED = 'ROLLBACK_ACTION_FAILED'
|
|
65
|
+
|
|
66
|
+
# Additional strategy codes
|
|
67
|
+
VALIDATION_FAILED = 'VALIDATION_FAILED'
|
|
68
|
+
NO_DATA_UNIT = 'NO_DATA_UNIT'
|
|
69
|
+
NO_DATA_UNIT_FILES = 'NO_DATA_UNIT_FILES'
|
|
70
|
+
TARGET_SPEC_URL_NOT_FOUND = 'TARGET_SPEC_URL_NOT_FOUND'
|
|
71
|
+
DATA_DOWNLOAD_FAILED = 'DATA_DOWNLOAD_FAILED'
|
|
72
|
+
JSON_DECODE_FAILED = 'JSON_DECODE_FAILED'
|
|
73
|
+
ANNOTATION_SUBMISSION_FAILED = 'ANNOTATION_SUBMISSION_FAILED'
|
|
74
|
+
NO_PREPROCESSOR_ID = 'NO_PREPROCESSOR_ID'
|
|
75
|
+
DATA_EXTRACTION_FAILED = 'DATA_EXTRACTION_FAILED'
|
|
76
|
+
PROGRESS_UPDATE_FAILED = 'PROGRESS_UPDATE_FAILED'
|
|
77
|
+
METRICS_RECORDING_FAILED = 'METRICS_RECORDING_FAILED'
|
|
78
|
+
METRICS_UPDATE_FAILED = 'METRICS_UPDATE_FAILED'
|
|
79
|
+
METRICS_FINALIZATION_FAILED = 'METRICS_FINALIZATION_FAILED'
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
LOG_MESSAGES = {
|
|
83
|
+
LogCode.INVALID_PROJECT_RESPONSE: {
|
|
84
|
+
'message': 'Invalid project response received.',
|
|
85
|
+
'level': Context.DANGER,
|
|
86
|
+
},
|
|
87
|
+
LogCode.NO_DATA_COLLECTION: {
|
|
88
|
+
'message': 'Project does not have a data collection.',
|
|
89
|
+
'level': Context.DANGER,
|
|
90
|
+
},
|
|
91
|
+
LogCode.INVALID_DATA_COLLECTION_RESPONSE: {
|
|
92
|
+
'message': 'Invalid data collection response received.',
|
|
93
|
+
'level': Context.DANGER,
|
|
94
|
+
},
|
|
95
|
+
LogCode.NO_TASKS_FOUND: {
|
|
96
|
+
'message': 'Tasks to annotate not found.',
|
|
97
|
+
'level': Context.WARNING,
|
|
98
|
+
},
|
|
99
|
+
LogCode.TARGET_SPEC_REQUIRED: {
|
|
100
|
+
'message': 'Target specification name is required for file annotation method.',
|
|
101
|
+
'level': Context.DANGER,
|
|
102
|
+
},
|
|
103
|
+
LogCode.TARGET_SPEC_NOT_FOUND: {
|
|
104
|
+
'message': 'Target specification name "{}" not found in file specifications',
|
|
105
|
+
'level': Context.DANGER,
|
|
106
|
+
},
|
|
107
|
+
LogCode.UNSUPPORTED_METHOD: {
|
|
108
|
+
'message': 'Unsupported annotation method: {}',
|
|
109
|
+
'level': Context.DANGER,
|
|
110
|
+
},
|
|
111
|
+
LogCode.ANNOTATING_DATA: {
|
|
112
|
+
'message': 'Annotating data to tasks...',
|
|
113
|
+
'level': None,
|
|
114
|
+
},
|
|
115
|
+
LogCode.CRITICAL_ERROR: {
|
|
116
|
+
'message': 'Critical error occured while processing task. Stopping the job.',
|
|
117
|
+
'level': Context.DANGER,
|
|
118
|
+
},
|
|
119
|
+
LogCode.TASK_PROCESSING_FAILED: {
|
|
120
|
+
'message': 'Failed to process task {}: {}',
|
|
121
|
+
'level': Context.DANGER,
|
|
122
|
+
},
|
|
123
|
+
LogCode.ANNOTATION_COMPLETED: {
|
|
124
|
+
'message': 'Annotation completed. Success: {}, Failed: {}',
|
|
125
|
+
'level': None,
|
|
126
|
+
},
|
|
127
|
+
LogCode.INVALID_TASK_RESPONSE: {
|
|
128
|
+
'message': 'Invalid task response received for task {}',
|
|
129
|
+
'level': Context.DANGER,
|
|
130
|
+
},
|
|
131
|
+
LogCode.TARGET_SPEC_REQUIRED_FOR_TASK: {
|
|
132
|
+
'message': 'Target specification name is required for file annotation method for task {}',
|
|
133
|
+
'level': Context.DANGER,
|
|
134
|
+
},
|
|
135
|
+
LogCode.UNSUPPORTED_METHOD_FOR_TASK: {
|
|
136
|
+
'message': 'Unsupported annotation method: {} for task {}',
|
|
137
|
+
'level': Context.DANGER,
|
|
138
|
+
},
|
|
139
|
+
LogCode.PRIMARY_IMAGE_URL_NOT_FOUND: {
|
|
140
|
+
'message': 'Primary image URL not found in task data for task {}',
|
|
141
|
+
'level': Context.DANGER,
|
|
142
|
+
},
|
|
143
|
+
LogCode.FILE_SPEC_NOT_FOUND: {
|
|
144
|
+
'message': 'File specification not found for task {}',
|
|
145
|
+
'level': Context.DANGER,
|
|
146
|
+
},
|
|
147
|
+
LogCode.FILE_ORIGINAL_NAME_NOT_FOUND: {
|
|
148
|
+
'message': 'File original name not found for task {}',
|
|
149
|
+
'level': Context.DANGER,
|
|
150
|
+
},
|
|
151
|
+
LogCode.URL_NOT_FOUND: {
|
|
152
|
+
'message': 'URL not found for task {}',
|
|
153
|
+
'level': Context.DANGER,
|
|
154
|
+
},
|
|
155
|
+
LogCode.FETCH_DATA_FAILED: {
|
|
156
|
+
'message': 'Failed to fetch data from URL: {} for task {}',
|
|
157
|
+
'level': Context.DANGER,
|
|
158
|
+
},
|
|
159
|
+
LogCode.CONVERT_DATA_FAILED: {
|
|
160
|
+
'message': 'Failed to convert data to task object: {} for task {}',
|
|
161
|
+
'level': Context.DANGER,
|
|
162
|
+
},
|
|
163
|
+
LogCode.PREPROCESSOR_ID_REQUIRED: {
|
|
164
|
+
'message': 'Pre-processor ID is required for inference annotation method for task {}',
|
|
165
|
+
'level': Context.DANGER,
|
|
166
|
+
},
|
|
167
|
+
LogCode.INFERENCE_PROCESSING_FAILED: {
|
|
168
|
+
'message': 'Failed to process inference for task {}: {}',
|
|
169
|
+
'level': Context.DANGER,
|
|
170
|
+
},
|
|
171
|
+
LogCode.ANNOTATING_INFERENCE_DATA: {
|
|
172
|
+
'message': 'Annotating data to tasks using inference...',
|
|
173
|
+
'level': None,
|
|
174
|
+
},
|
|
175
|
+
LogCode.INFERENCE_ANNOTATION_COMPLETED: {
|
|
176
|
+
'message': 'Inference annotation completed. Success: {}, Failed: {}',
|
|
177
|
+
'level': None,
|
|
178
|
+
},
|
|
179
|
+
LogCode.INFERENCE_PREPROCESSOR_FAILED: {
|
|
180
|
+
'message': 'Inference pre processor failed for task {}: {}',
|
|
181
|
+
'level': Context.DANGER,
|
|
182
|
+
},
|
|
183
|
+
# Orchestrator workflow messages
|
|
184
|
+
LogCode.TO_TASK_STARTED: {
|
|
185
|
+
'message': 'ToTask action started.',
|
|
186
|
+
'level': Context.INFO,
|
|
187
|
+
},
|
|
188
|
+
LogCode.TO_TASK_COMPLETED: {
|
|
189
|
+
'message': 'ToTask action completed successfully.',
|
|
190
|
+
'level': Context.SUCCESS,
|
|
191
|
+
},
|
|
192
|
+
LogCode.TO_TASK_FAILED: {
|
|
193
|
+
'message': 'ToTask action failed: {}',
|
|
194
|
+
'level': Context.DANGER,
|
|
195
|
+
},
|
|
196
|
+
LogCode.STEP_STARTED: {
|
|
197
|
+
'message': 'Starting workflow step: {}',
|
|
198
|
+
'level': Context.INFO,
|
|
199
|
+
},
|
|
200
|
+
LogCode.STEP_COMPLETED: {
|
|
201
|
+
'message': 'Completed workflow step: {}',
|
|
202
|
+
'level': Context.INFO,
|
|
203
|
+
},
|
|
204
|
+
LogCode.STEP_FAILED: {
|
|
205
|
+
'message': 'Failed workflow step {}: {}',
|
|
206
|
+
'level': Context.DANGER,
|
|
207
|
+
},
|
|
208
|
+
LogCode.ROLLBACK_FAILED: {
|
|
209
|
+
'message': 'Failed to rollback step {}: {}',
|
|
210
|
+
'level': Context.WARNING,
|
|
211
|
+
},
|
|
212
|
+
LogCode.ROLLBACK_ACTION_FAILED: {
|
|
213
|
+
'message': 'Failed to execute rollback action: {}',
|
|
214
|
+
'level': Context.WARNING,
|
|
215
|
+
},
|
|
216
|
+
# Additional strategy messages
|
|
217
|
+
LogCode.VALIDATION_FAILED: {
|
|
218
|
+
'message': 'Validation failed: {}',
|
|
219
|
+
'level': Context.DANGER,
|
|
220
|
+
},
|
|
221
|
+
LogCode.NO_DATA_UNIT: {
|
|
222
|
+
'message': 'Task does not have a data unit',
|
|
223
|
+
'level': Context.DANGER,
|
|
224
|
+
},
|
|
225
|
+
LogCode.NO_DATA_UNIT_FILES: {
|
|
226
|
+
'message': 'Data unit does not have files',
|
|
227
|
+
'level': Context.DANGER,
|
|
228
|
+
},
|
|
229
|
+
LogCode.TARGET_SPEC_URL_NOT_FOUND: {
|
|
230
|
+
'message': 'Target specification URL not found for {} in task {}',
|
|
231
|
+
'level': Context.DANGER,
|
|
232
|
+
},
|
|
233
|
+
LogCode.DATA_DOWNLOAD_FAILED: {
|
|
234
|
+
'message': 'Failed to download data for task {}: {}',
|
|
235
|
+
'level': Context.DANGER,
|
|
236
|
+
},
|
|
237
|
+
LogCode.JSON_DECODE_FAILED: {
|
|
238
|
+
'message': 'Failed to decode JSON data for task {}: {}',
|
|
239
|
+
'level': Context.DANGER,
|
|
240
|
+
},
|
|
241
|
+
LogCode.ANNOTATION_SUBMISSION_FAILED: {
|
|
242
|
+
'message': 'Failed to submit annotation data for task {}: {}',
|
|
243
|
+
'level': Context.DANGER,
|
|
244
|
+
},
|
|
245
|
+
LogCode.NO_PREPROCESSOR_ID: {
|
|
246
|
+
'message': 'Pre-processor ID is required for inference annotation method',
|
|
247
|
+
'level': Context.DANGER,
|
|
248
|
+
},
|
|
249
|
+
LogCode.DATA_EXTRACTION_FAILED: {
|
|
250
|
+
'message': 'Data extraction failed: {}',
|
|
251
|
+
'level': Context.DANGER,
|
|
252
|
+
},
|
|
253
|
+
LogCode.PROGRESS_UPDATE_FAILED: {
|
|
254
|
+
'message': 'Progress update failed: {}',
|
|
255
|
+
'level': Context.WARNING,
|
|
256
|
+
},
|
|
257
|
+
LogCode.METRICS_RECORDING_FAILED: {
|
|
258
|
+
'message': 'Metrics recording failed: {}',
|
|
259
|
+
'level': Context.WARNING,
|
|
260
|
+
},
|
|
261
|
+
LogCode.METRICS_UPDATE_FAILED: {
|
|
262
|
+
'message': 'Metrics update failed: {}',
|
|
263
|
+
'level': Context.WARNING,
|
|
264
|
+
},
|
|
265
|
+
LogCode.METRICS_FINALIZATION_FAILED: {
|
|
266
|
+
'message': 'Metrics finalization failed: {}',
|
|
267
|
+
'level': Context.WARNING,
|
|
268
|
+
},
|
|
269
|
+
}
|