synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse-sdk might be problematic. Click here for more details.
- synapse_sdk/__init__.py +24 -0
- synapse_sdk/cli/__init__.py +9 -8
- synapse_sdk/cli/agent/__init__.py +25 -0
- synapse_sdk/cli/agent/config.py +104 -0
- synapse_sdk/cli/agent/select.py +197 -0
- synapse_sdk/cli/auth.py +104 -0
- synapse_sdk/cli/main.py +1025 -0
- synapse_sdk/cli/plugin/__init__.py +58 -0
- synapse_sdk/cli/plugin/create.py +566 -0
- synapse_sdk/cli/plugin/job.py +196 -0
- synapse_sdk/cli/plugin/publish.py +322 -0
- synapse_sdk/cli/plugin/run.py +131 -0
- synapse_sdk/cli/plugin/test.py +200 -0
- synapse_sdk/clients/README.md +239 -0
- synapse_sdk/clients/__init__.py +5 -0
- synapse_sdk/clients/_template.py +266 -0
- synapse_sdk/clients/agent/__init__.py +84 -29
- synapse_sdk/clients/agent/async_ray.py +289 -0
- synapse_sdk/clients/agent/container.py +83 -0
- synapse_sdk/clients/agent/plugin.py +101 -0
- synapse_sdk/clients/agent/ray.py +296 -39
- synapse_sdk/clients/backend/__init__.py +152 -12
- synapse_sdk/clients/backend/annotation.py +164 -22
- synapse_sdk/clients/backend/core.py +101 -0
- synapse_sdk/clients/backend/data_collection.py +292 -0
- synapse_sdk/clients/backend/hitl.py +87 -0
- synapse_sdk/clients/backend/integration.py +374 -46
- synapse_sdk/clients/backend/ml.py +134 -22
- synapse_sdk/clients/backend/models.py +247 -0
- synapse_sdk/clients/base.py +538 -59
- synapse_sdk/clients/exceptions.py +35 -7
- synapse_sdk/clients/pipeline/__init__.py +5 -0
- synapse_sdk/clients/pipeline/client.py +636 -0
- synapse_sdk/clients/protocols.py +178 -0
- synapse_sdk/clients/utils.py +86 -8
- synapse_sdk/clients/validation.py +58 -0
- synapse_sdk/enums.py +76 -0
- synapse_sdk/exceptions.py +168 -0
- synapse_sdk/integrations/__init__.py +74 -0
- synapse_sdk/integrations/_base.py +119 -0
- synapse_sdk/integrations/_context.py +53 -0
- synapse_sdk/integrations/ultralytics/__init__.py +78 -0
- synapse_sdk/integrations/ultralytics/_callbacks.py +126 -0
- synapse_sdk/integrations/ultralytics/_patches.py +124 -0
- synapse_sdk/loggers.py +476 -95
- synapse_sdk/mcp/MCP.md +69 -0
- synapse_sdk/mcp/__init__.py +48 -0
- synapse_sdk/mcp/__main__.py +6 -0
- synapse_sdk/mcp/config.py +349 -0
- synapse_sdk/mcp/prompts/__init__.py +4 -0
- synapse_sdk/mcp/resources/__init__.py +4 -0
- synapse_sdk/mcp/server.py +1352 -0
- synapse_sdk/mcp/tools/__init__.py +6 -0
- synapse_sdk/plugins/__init__.py +133 -9
- synapse_sdk/plugins/action.py +229 -0
- synapse_sdk/plugins/actions/__init__.py +82 -0
- synapse_sdk/plugins/actions/dataset/__init__.py +37 -0
- synapse_sdk/plugins/actions/dataset/action.py +471 -0
- synapse_sdk/plugins/actions/export/__init__.py +55 -0
- synapse_sdk/plugins/actions/export/action.py +183 -0
- synapse_sdk/plugins/actions/export/context.py +59 -0
- synapse_sdk/plugins/actions/inference/__init__.py +84 -0
- synapse_sdk/plugins/actions/inference/action.py +285 -0
- synapse_sdk/plugins/actions/inference/context.py +81 -0
- synapse_sdk/plugins/actions/inference/deployment.py +322 -0
- synapse_sdk/plugins/actions/inference/serve.py +252 -0
- synapse_sdk/plugins/actions/train/__init__.py +54 -0
- synapse_sdk/plugins/actions/train/action.py +326 -0
- synapse_sdk/plugins/actions/train/context.py +57 -0
- synapse_sdk/plugins/actions/upload/__init__.py +49 -0
- synapse_sdk/plugins/actions/upload/action.py +165 -0
- synapse_sdk/plugins/actions/upload/context.py +61 -0
- synapse_sdk/plugins/config.py +98 -0
- synapse_sdk/plugins/context/__init__.py +109 -0
- synapse_sdk/plugins/context/env.py +113 -0
- synapse_sdk/plugins/datasets/__init__.py +113 -0
- synapse_sdk/plugins/datasets/converters/__init__.py +76 -0
- synapse_sdk/plugins/datasets/converters/base.py +347 -0
- synapse_sdk/plugins/datasets/converters/yolo/__init__.py +9 -0
- synapse_sdk/plugins/datasets/converters/yolo/from_dm.py +468 -0
- synapse_sdk/plugins/datasets/converters/yolo/to_dm.py +381 -0
- synapse_sdk/plugins/datasets/formats/__init__.py +82 -0
- synapse_sdk/plugins/datasets/formats/dm.py +351 -0
- synapse_sdk/plugins/datasets/formats/yolo.py +240 -0
- synapse_sdk/plugins/decorators.py +83 -0
- synapse_sdk/plugins/discovery.py +790 -0
- synapse_sdk/plugins/docs/ACTION_DEV_GUIDE.md +933 -0
- synapse_sdk/plugins/docs/ARCHITECTURE.md +1225 -0
- synapse_sdk/plugins/docs/LOGGING_SYSTEM.md +683 -0
- synapse_sdk/plugins/docs/OVERVIEW.md +531 -0
- synapse_sdk/plugins/docs/PIPELINE_GUIDE.md +145 -0
- synapse_sdk/plugins/docs/README.md +513 -0
- synapse_sdk/plugins/docs/STEP.md +656 -0
- synapse_sdk/plugins/enums.py +70 -10
- synapse_sdk/plugins/errors.py +92 -0
- synapse_sdk/plugins/executors/__init__.py +43 -0
- synapse_sdk/plugins/executors/local.py +99 -0
- synapse_sdk/plugins/executors/ray/__init__.py +18 -0
- synapse_sdk/plugins/executors/ray/base.py +282 -0
- synapse_sdk/plugins/executors/ray/job.py +298 -0
- synapse_sdk/plugins/executors/ray/jobs_api.py +511 -0
- synapse_sdk/plugins/executors/ray/packaging.py +137 -0
- synapse_sdk/plugins/executors/ray/pipeline.py +792 -0
- synapse_sdk/plugins/executors/ray/task.py +257 -0
- synapse_sdk/plugins/models/__init__.py +26 -0
- synapse_sdk/plugins/models/logger.py +173 -0
- synapse_sdk/plugins/models/pipeline.py +25 -0
- synapse_sdk/plugins/pipelines/__init__.py +81 -0
- synapse_sdk/plugins/pipelines/action_pipeline.py +417 -0
- synapse_sdk/plugins/pipelines/context.py +107 -0
- synapse_sdk/plugins/pipelines/display.py +311 -0
- synapse_sdk/plugins/runner.py +114 -0
- synapse_sdk/plugins/schemas/__init__.py +19 -0
- synapse_sdk/plugins/schemas/results.py +152 -0
- synapse_sdk/plugins/steps/__init__.py +63 -0
- synapse_sdk/plugins/steps/base.py +128 -0
- synapse_sdk/plugins/steps/context.py +90 -0
- synapse_sdk/plugins/steps/orchestrator.py +128 -0
- synapse_sdk/plugins/steps/registry.py +103 -0
- synapse_sdk/plugins/steps/utils/__init__.py +20 -0
- synapse_sdk/plugins/steps/utils/logging.py +85 -0
- synapse_sdk/plugins/steps/utils/timing.py +71 -0
- synapse_sdk/plugins/steps/utils/validation.py +68 -0
- synapse_sdk/plugins/templates/__init__.py +50 -0
- synapse_sdk/plugins/templates/base/.gitignore.j2 +26 -0
- synapse_sdk/plugins/templates/base/.synapseignore.j2 +11 -0
- synapse_sdk/plugins/templates/base/README.md.j2 +26 -0
- synapse_sdk/plugins/templates/base/plugin/__init__.py.j2 +1 -0
- synapse_sdk/plugins/templates/base/pyproject.toml.j2 +14 -0
- synapse_sdk/plugins/templates/base/requirements.txt.j2 +1 -0
- synapse_sdk/plugins/templates/custom/plugin/main.py.j2 +18 -0
- synapse_sdk/plugins/templates/data_validation/plugin/validate.py.j2 +32 -0
- synapse_sdk/plugins/templates/export/plugin/export.py.j2 +36 -0
- synapse_sdk/plugins/templates/neural_net/plugin/inference.py.j2 +36 -0
- synapse_sdk/plugins/templates/neural_net/plugin/train.py.j2 +33 -0
- synapse_sdk/plugins/templates/post_annotation/plugin/post_annotate.py.j2 +32 -0
- synapse_sdk/plugins/templates/pre_annotation/plugin/pre_annotate.py.j2 +32 -0
- synapse_sdk/plugins/templates/smart_tool/plugin/auto_label.py.j2 +44 -0
- synapse_sdk/plugins/templates/upload/plugin/upload.py.j2 +35 -0
- synapse_sdk/plugins/testing/__init__.py +25 -0
- synapse_sdk/plugins/testing/sample_actions.py +98 -0
- synapse_sdk/plugins/types.py +206 -0
- synapse_sdk/plugins/upload.py +595 -64
- synapse_sdk/plugins/utils.py +325 -37
- synapse_sdk/shared/__init__.py +25 -0
- synapse_sdk/utils/__init__.py +1 -0
- synapse_sdk/utils/auth.py +74 -0
- synapse_sdk/utils/file/__init__.py +58 -0
- synapse_sdk/utils/file/archive.py +449 -0
- synapse_sdk/utils/file/checksum.py +167 -0
- synapse_sdk/utils/file/download.py +286 -0
- synapse_sdk/utils/file/io.py +129 -0
- synapse_sdk/utils/file/requirements.py +36 -0
- synapse_sdk/utils/network.py +168 -0
- synapse_sdk/utils/storage/__init__.py +238 -0
- synapse_sdk/utils/storage/config.py +188 -0
- synapse_sdk/utils/storage/errors.py +52 -0
- synapse_sdk/utils/storage/providers/__init__.py +13 -0
- synapse_sdk/utils/storage/providers/base.py +76 -0
- synapse_sdk/utils/storage/providers/gcs.py +168 -0
- synapse_sdk/utils/storage/providers/http.py +250 -0
- synapse_sdk/utils/storage/providers/local.py +126 -0
- synapse_sdk/utils/storage/providers/s3.py +177 -0
- synapse_sdk/utils/storage/providers/sftp.py +208 -0
- synapse_sdk/utils/storage/registry.py +125 -0
- synapse_sdk/utils/websocket.py +99 -0
- synapse_sdk-2026.1.1b2.dist-info/METADATA +715 -0
- synapse_sdk-2026.1.1b2.dist-info/RECORD +172 -0
- {synapse_sdk-1.0.0a11.dist-info → synapse_sdk-2026.1.1b2.dist-info}/WHEEL +1 -1
- synapse_sdk-2026.1.1b2.dist-info/licenses/LICENSE +201 -0
- locale/en/LC_MESSAGES/messages.mo +0 -0
- locale/en/LC_MESSAGES/messages.po +0 -39
- locale/ko/LC_MESSAGES/messages.mo +0 -0
- locale/ko/LC_MESSAGES/messages.po +0 -34
- synapse_sdk/cli/create_plugin.py +0 -10
- synapse_sdk/clients/agent/core.py +0 -7
- synapse_sdk/clients/agent/service.py +0 -15
- synapse_sdk/clients/backend/dataset.py +0 -51
- synapse_sdk/clients/ray/__init__.py +0 -6
- synapse_sdk/clients/ray/core.py +0 -22
- synapse_sdk/clients/ray/serve.py +0 -20
- synapse_sdk/i18n.py +0 -35
- synapse_sdk/plugins/categories/__init__.py +0 -0
- synapse_sdk/plugins/categories/base.py +0 -235
- synapse_sdk/plugins/categories/data_validation/__init__.py +0 -0
- synapse_sdk/plugins/categories/data_validation/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/data_validation/actions/validation.py +0 -10
- synapse_sdk/plugins/categories/data_validation/templates/config.yaml +0 -3
- synapse_sdk/plugins/categories/data_validation/templates/plugin/__init__.py +0 -0
- synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py +0 -5
- synapse_sdk/plugins/categories/decorators.py +0 -13
- synapse_sdk/plugins/categories/export/__init__.py +0 -0
- synapse_sdk/plugins/categories/export/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/export/actions/export.py +0 -10
- synapse_sdk/plugins/categories/import/__init__.py +0 -0
- synapse_sdk/plugins/categories/import/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/import/actions/import.py +0 -10
- synapse_sdk/plugins/categories/neural_net/__init__.py +0 -0
- synapse_sdk/plugins/categories/neural_net/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/neural_net/actions/deployment.py +0 -45
- synapse_sdk/plugins/categories/neural_net/actions/inference.py +0 -18
- synapse_sdk/plugins/categories/neural_net/actions/test.py +0 -10
- synapse_sdk/plugins/categories/neural_net/actions/train.py +0 -143
- synapse_sdk/plugins/categories/neural_net/templates/config.yaml +0 -12
- synapse_sdk/plugins/categories/neural_net/templates/plugin/__init__.py +0 -0
- synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py +0 -4
- synapse_sdk/plugins/categories/neural_net/templates/plugin/test.py +0 -2
- synapse_sdk/plugins/categories/neural_net/templates/plugin/train.py +0 -14
- synapse_sdk/plugins/categories/post_annotation/__init__.py +0 -0
- synapse_sdk/plugins/categories/post_annotation/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py +0 -10
- synapse_sdk/plugins/categories/post_annotation/templates/config.yaml +0 -3
- synapse_sdk/plugins/categories/post_annotation/templates/plugin/__init__.py +0 -0
- synapse_sdk/plugins/categories/post_annotation/templates/plugin/post_annotation.py +0 -3
- synapse_sdk/plugins/categories/pre_annotation/__init__.py +0 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py +0 -10
- synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml +0 -3
- synapse_sdk/plugins/categories/pre_annotation/templates/plugin/__init__.py +0 -0
- synapse_sdk/plugins/categories/pre_annotation/templates/plugin/pre_annotation.py +0 -3
- synapse_sdk/plugins/categories/registry.py +0 -16
- synapse_sdk/plugins/categories/smart_tool/__init__.py +0 -0
- synapse_sdk/plugins/categories/smart_tool/actions/__init__.py +0 -0
- synapse_sdk/plugins/categories/smart_tool/actions/auto_label.py +0 -37
- synapse_sdk/plugins/categories/smart_tool/templates/config.yaml +0 -7
- synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py +0 -0
- synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py +0 -11
- synapse_sdk/plugins/categories/templates.py +0 -32
- synapse_sdk/plugins/cli/__init__.py +0 -21
- synapse_sdk/plugins/cli/publish.py +0 -37
- synapse_sdk/plugins/cli/run.py +0 -67
- synapse_sdk/plugins/exceptions.py +0 -22
- synapse_sdk/plugins/models.py +0 -121
- synapse_sdk/plugins/templates/cookiecutter.json +0 -11
- synapse_sdk/plugins/templates/hooks/post_gen_project.py +0 -3
- synapse_sdk/plugins/templates/hooks/pre_prompt.py +0 -21
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/.env +0 -24
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/.env.dist +0 -24
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/.gitignore +0 -27
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/.pre-commit-config.yaml +0 -7
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/README.md +0 -5
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/config.yaml +0 -6
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/main.py +0 -4
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/plugin/__init__.py +0 -0
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/pyproject.toml +0 -13
- synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/requirements.txt +0 -1
- synapse_sdk/shared/enums.py +0 -8
- synapse_sdk/utils/debug.py +0 -5
- synapse_sdk/utils/file.py +0 -87
- synapse_sdk/utils/module_loading.py +0 -29
- synapse_sdk/utils/pydantic/__init__.py +0 -0
- synapse_sdk/utils/pydantic/config.py +0 -4
- synapse_sdk/utils/pydantic/errors.py +0 -33
- synapse_sdk/utils/pydantic/validators.py +0 -7
- synapse_sdk/utils/storage.py +0 -91
- synapse_sdk/utils/string.py +0 -11
- synapse_sdk-1.0.0a11.dist-info/LICENSE +0 -21
- synapse_sdk-1.0.0a11.dist-info/METADATA +0 -43
- synapse_sdk-1.0.0a11.dist-info/RECORD +0 -111
- {synapse_sdk-1.0.0a11.dist-info → synapse_sdk-2026.1.1b2.dist-info}/entry_points.txt +0 -0
- {synapse_sdk-1.0.0a11.dist-info → synapse_sdk-2026.1.1b2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,933 @@
|
|
|
1
|
+
# Action Development Guide
|
|
2
|
+
|
|
3
|
+
Comprehensive guide for developing plugin actions in the Synapse SDK, including synchronous and asynchronous patterns.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
Actions are the fundamental units of work in the plugin system. This guide covers:
|
|
8
|
+
- Class-based and function-based action development
|
|
9
|
+
- Specialized action base classes
|
|
10
|
+
- Parameter validation and result schemas
|
|
11
|
+
- Progress tracking and logging
|
|
12
|
+
- Async execution patterns
|
|
13
|
+
- Step-based workflows within actions
|
|
14
|
+
|
|
15
|
+
## Architecture
|
|
16
|
+
|
|
17
|
+
```mermaid
|
|
18
|
+
flowchart TB
|
|
19
|
+
subgraph ActionTypes["Action Types"]
|
|
20
|
+
base["BaseAction[P]<br/>(Core)"]
|
|
21
|
+
train["BaseTrainAction[P]<br/>(Training)"]
|
|
22
|
+
export["BaseExportAction[P]<br/>(Export)"]
|
|
23
|
+
inference["BaseInferenceAction[P]<br/>(Inference)"]
|
|
24
|
+
upload["BaseUploadAction[P]<br/>(Upload)"]
|
|
25
|
+
decorator["@action decorator<br/>(Function-based)"]
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
subgraph Execution["Execution"]
|
|
29
|
+
local["LocalExecutor<br/>- Sync, in-process"]
|
|
30
|
+
actor["RayActorExecutor<br/>- Sync, remote"]
|
|
31
|
+
job["RayJobExecutor<br/>- Async, isolated"]
|
|
32
|
+
serve["Ray Serve<br/>- Async, HTTP"]
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
subgraph Components["Components"]
|
|
36
|
+
params["Pydantic Params"]
|
|
37
|
+
context["RuntimeContext"]
|
|
38
|
+
steps["StepRegistry<br/>+ Orchestrator"]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
base --> train
|
|
42
|
+
base --> export
|
|
43
|
+
base --> inference
|
|
44
|
+
base --> upload
|
|
45
|
+
|
|
46
|
+
params --> base
|
|
47
|
+
context --> base
|
|
48
|
+
steps --> train
|
|
49
|
+
steps --> export
|
|
50
|
+
steps --> inference
|
|
51
|
+
|
|
52
|
+
base --> local
|
|
53
|
+
base --> actor
|
|
54
|
+
base --> job
|
|
55
|
+
inference --> serve
|
|
56
|
+
|
|
57
|
+
style ActionTypes fill:#e1f5fe
|
|
58
|
+
style Execution fill:#f3e5f5
|
|
59
|
+
style Components fill:#fff3e0
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Class-Based Actions
|
|
65
|
+
|
|
66
|
+
### Basic Structure
|
|
67
|
+
|
|
68
|
+
Use `BaseAction[P]` for class-based actions with typed parameters:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from synapse_sdk.plugins import BaseAction
|
|
72
|
+
from synapse_sdk.plugins.enums import PluginCategory
|
|
73
|
+
from pydantic import BaseModel, Field
|
|
74
|
+
|
|
75
|
+
class TrainParams(BaseModel):
|
|
76
|
+
"""Training parameters with validation."""
|
|
77
|
+
epochs: int = Field(default=10, ge=1, le=1000)
|
|
78
|
+
learning_rate: float = Field(default=0.001, gt=0, lt=1)
|
|
79
|
+
batch_size: int = Field(default=32, ge=1)
|
|
80
|
+
|
|
81
|
+
class TrainResult(BaseModel):
|
|
82
|
+
"""Typed result schema."""
|
|
83
|
+
weights_path: str
|
|
84
|
+
final_loss: float
|
|
85
|
+
epochs_completed: int
|
|
86
|
+
|
|
87
|
+
class TrainAction(BaseAction[TrainParams]):
|
|
88
|
+
"""Train a model with progress tracking."""
|
|
89
|
+
|
|
90
|
+
# Optional metadata
|
|
91
|
+
action_name = 'train'
|
|
92
|
+
category = PluginCategory.NEURAL_NET
|
|
93
|
+
|
|
94
|
+
# Optional: Enable result validation
|
|
95
|
+
result_model = TrainResult
|
|
96
|
+
|
|
97
|
+
def execute(self) -> TrainResult:
|
|
98
|
+
# Access validated params
|
|
99
|
+
epochs = self.params.epochs
|
|
100
|
+
lr = self.params.learning_rate
|
|
101
|
+
|
|
102
|
+
# Log start event
|
|
103
|
+
self.log('train_start', {'epochs': epochs, 'lr': lr})
|
|
104
|
+
|
|
105
|
+
# Training loop with progress
|
|
106
|
+
for epoch in range(epochs):
|
|
107
|
+
self.set_progress(epoch + 1, epochs, category='train')
|
|
108
|
+
|
|
109
|
+
# Training logic...
|
|
110
|
+
loss = train_epoch(epoch, lr)
|
|
111
|
+
|
|
112
|
+
# Record metrics
|
|
113
|
+
self.set_metrics({'loss': loss, 'epoch': epoch}, category='train')
|
|
114
|
+
|
|
115
|
+
# Return typed result
|
|
116
|
+
return TrainResult(
|
|
117
|
+
weights_path='/model/weights.pt',
|
|
118
|
+
final_loss=loss,
|
|
119
|
+
epochs_completed=epochs,
|
|
120
|
+
)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### BaseAction Class Attributes
|
|
124
|
+
|
|
125
|
+
| Attribute | Type | Description |
|
|
126
|
+
|-----------|------|-------------|
|
|
127
|
+
| `action_name` | `str | None` | Action identifier (from config if not set) |
|
|
128
|
+
| `category` | `PluginCategory | None` | Grouping category |
|
|
129
|
+
| `input_type` | `type[DataType] | None` | Semantic input type |
|
|
130
|
+
| `output_type` | `type[DataType] | None` | Semantic output type |
|
|
131
|
+
| `params_model` | `type[BaseModel]` | Auto-extracted from generic |
|
|
132
|
+
| `result_model` | `type[BaseModel] | NoResult` | Optional result schema |
|
|
133
|
+
|
|
134
|
+
### Instance Properties and Methods
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
class MyAction(BaseAction[MyParams]):
|
|
138
|
+
def execute(self) -> dict:
|
|
139
|
+
# Params access
|
|
140
|
+
value = self.params.some_field
|
|
141
|
+
|
|
142
|
+
# Context access
|
|
143
|
+
ctx = self.ctx
|
|
144
|
+
logger = self.logger # Shortcut to ctx.logger
|
|
145
|
+
|
|
146
|
+
# Logging shortcuts
|
|
147
|
+
self.log('event', {'data': value})
|
|
148
|
+
self.set_progress(50, 100, category='process')
|
|
149
|
+
self.set_metrics({'accuracy': 0.95}, category='eval')
|
|
150
|
+
|
|
151
|
+
# ML framework autolog (ultralytics supported)
|
|
152
|
+
self.autolog('ultralytics')
|
|
153
|
+
|
|
154
|
+
return {'status': 'done'}
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## Function-Based Actions
|
|
160
|
+
|
|
161
|
+
Use the `@action` decorator for simple, stateless operations:
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
from synapse_sdk.plugins import action
|
|
165
|
+
from synapse_sdk.plugins.context import RuntimeContext
|
|
166
|
+
from pydantic import BaseModel
|
|
167
|
+
|
|
168
|
+
class ConvertParams(BaseModel):
|
|
169
|
+
input_format: str
|
|
170
|
+
output_format: str
|
|
171
|
+
file_path: str
|
|
172
|
+
|
|
173
|
+
class ConvertResult(BaseModel):
|
|
174
|
+
output_path: str
|
|
175
|
+
converted: bool
|
|
176
|
+
|
|
177
|
+
@action(
|
|
178
|
+
name='convert',
|
|
179
|
+
description='Convert file between formats',
|
|
180
|
+
params=ConvertParams,
|
|
181
|
+
result=ConvertResult,
|
|
182
|
+
)
|
|
183
|
+
def convert(params: ConvertParams, context: RuntimeContext) -> ConvertResult:
|
|
184
|
+
"""Convert a file from one format to another."""
|
|
185
|
+
|
|
186
|
+
context.set_progress(0, 100)
|
|
187
|
+
|
|
188
|
+
# Conversion logic
|
|
189
|
+
output_path = convert_file(
|
|
190
|
+
params.file_path,
|
|
191
|
+
params.input_format,
|
|
192
|
+
params.output_format,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
context.set_progress(100, 100)
|
|
196
|
+
|
|
197
|
+
return ConvertResult(output_path=output_path, converted=True)
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
### Decorator Parameters
|
|
201
|
+
|
|
202
|
+
| Parameter | Type | Description |
|
|
203
|
+
|-----------|------|-------------|
|
|
204
|
+
| `name` | `str | None` | Action name (defaults to function name) |
|
|
205
|
+
| `description` | `str` | Human-readable description |
|
|
206
|
+
| `params` | `type[BaseModel] | None` | Parameter validation model |
|
|
207
|
+
| `result` | `type[BaseModel] | None` | Result validation model |
|
|
208
|
+
| `category` | `PluginCategory | None` | Plugin category |
|
|
209
|
+
|
|
210
|
+
### When to Use Each Style
|
|
211
|
+
|
|
212
|
+
| Aspect | Class-Based | Function-Based |
|
|
213
|
+
|--------|-------------|----------------|
|
|
214
|
+
| Complexity | Complex, multi-method | Simple, single function |
|
|
215
|
+
| State | Instance state allowed | Stateless |
|
|
216
|
+
| Type declarations | `input_type`, `output_type` | Not supported |
|
|
217
|
+
| Autolog | `self.autolog()` | Not available |
|
|
218
|
+
| Step workflows | Via `setup_steps()` | Not supported |
|
|
219
|
+
| Testing | Easier mocking | Direct function calls |
|
|
220
|
+
|
|
221
|
+
---
|
|
222
|
+
|
|
223
|
+
## Specialized Base Actions
|
|
224
|
+
|
|
225
|
+
### BaseTrainAction
|
|
226
|
+
|
|
227
|
+
For ML training workflows with dataset fetching and model upload:
|
|
228
|
+
|
|
229
|
+
```python
|
|
230
|
+
from synapse_sdk.plugins.actions.train import BaseTrainAction, BaseTrainParams
|
|
231
|
+
|
|
232
|
+
class MyTrainParams(BaseTrainParams):
|
|
233
|
+
"""Extends base with custom fields."""
|
|
234
|
+
dataset_id: int
|
|
235
|
+
epochs: int = 100
|
|
236
|
+
|
|
237
|
+
class MyTrainAction(BaseTrainAction[MyTrainParams]):
|
|
238
|
+
action_name = 'train'
|
|
239
|
+
|
|
240
|
+
def execute(self) -> dict:
|
|
241
|
+
# Use built-in helpers
|
|
242
|
+
dataset = self.get_dataset() # Fetches params.dataset_id
|
|
243
|
+
checkpoint = self.get_checkpoint() # Optional resume
|
|
244
|
+
|
|
245
|
+
# Standard progress categories
|
|
246
|
+
self.set_progress(1, 3, self.progress.DATASET)
|
|
247
|
+
self.set_progress(2, 3, self.progress.TRAIN)
|
|
248
|
+
self.set_progress(3, 3, self.progress.MODEL_UPLOAD)
|
|
249
|
+
|
|
250
|
+
# Upload model
|
|
251
|
+
model = self.create_model('./model', name='trained-model')
|
|
252
|
+
|
|
253
|
+
return {'model_id': model['id']}
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
**Built-in Methods:**
|
|
257
|
+
|
|
258
|
+
| Method | Description |
|
|
259
|
+
|--------|-------------|
|
|
260
|
+
| `get_dataset()` | Fetch dataset via `params.dataset_id` |
|
|
261
|
+
| `get_checkpoint()` | Get training checkpoint (base model or resume) |
|
|
262
|
+
| `create_model(path, **kwargs)` | Upload trained model to backend |
|
|
263
|
+
| `get_model(model_id)` | Retrieve model metadata |
|
|
264
|
+
| `setup_steps(registry)` | Register step-based workflow |
|
|
265
|
+
|
|
266
|
+
### BaseExportAction
|
|
267
|
+
|
|
268
|
+
For data export workflows:
|
|
269
|
+
|
|
270
|
+
```python
|
|
271
|
+
from synapse_sdk.plugins.actions.export import BaseExportAction
|
|
272
|
+
|
|
273
|
+
class ExportParams(BaseModel):
|
|
274
|
+
filter: dict
|
|
275
|
+
output_format: str
|
|
276
|
+
|
|
277
|
+
class MyExportAction(BaseExportAction[ExportParams]):
|
|
278
|
+
action_name = 'export'
|
|
279
|
+
|
|
280
|
+
def get_filtered_results(self, filters: dict) -> tuple[Any, int]:
|
|
281
|
+
# Override for your target type
|
|
282
|
+
return self.client.get_assignments(filters)
|
|
283
|
+
|
|
284
|
+
def execute(self) -> dict:
|
|
285
|
+
results, count = self.get_filtered_results(self.params.filter)
|
|
286
|
+
|
|
287
|
+
self.set_progress(0, count, self.progress.DATASET_CONVERSION)
|
|
288
|
+
|
|
289
|
+
for i, item in enumerate(results, 1):
|
|
290
|
+
# Process and export item
|
|
291
|
+
self.set_progress(i, count)
|
|
292
|
+
|
|
293
|
+
return {'exported': count}
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
### BaseInferenceAction
|
|
297
|
+
|
|
298
|
+
For model inference workflows:
|
|
299
|
+
|
|
300
|
+
```python
|
|
301
|
+
from synapse_sdk.plugins.actions.inference import BaseInferenceAction
|
|
302
|
+
|
|
303
|
+
class InferParams(BaseModel):
|
|
304
|
+
model_id: int
|
|
305
|
+
inputs: list[dict]
|
|
306
|
+
|
|
307
|
+
class MyInferenceAction(BaseInferenceAction[InferParams]):
|
|
308
|
+
action_name = 'infer'
|
|
309
|
+
|
|
310
|
+
def infer(self, model: Any, inputs: list[dict]) -> list[dict]:
|
|
311
|
+
# Override with your inference logic
|
|
312
|
+
return [{'prediction': model.predict(inp)} for inp in inputs]
|
|
313
|
+
|
|
314
|
+
def execute(self) -> dict:
|
|
315
|
+
# Use built-in model loading
|
|
316
|
+
model_info = self.load_model(self.params.model_id)
|
|
317
|
+
model_path = model_info['path']
|
|
318
|
+
|
|
319
|
+
self.set_progress(1, 3, self.progress.MODEL_LOAD)
|
|
320
|
+
|
|
321
|
+
# Load your specific model
|
|
322
|
+
import torch
|
|
323
|
+
model = torch.load(f'{model_path}/model.pt')
|
|
324
|
+
|
|
325
|
+
self.set_progress(2, 3, self.progress.INFERENCE)
|
|
326
|
+
|
|
327
|
+
results = self.infer(model, self.params.inputs)
|
|
328
|
+
|
|
329
|
+
return {'results': results}
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
**Built-in Methods:**
|
|
333
|
+
|
|
334
|
+
| Method | Description |
|
|
335
|
+
|--------|-------------|
|
|
336
|
+
| `get_model(model_id)` | Retrieve model metadata |
|
|
337
|
+
| `download_model(model_id, output_dir)` | Download and extract model artifacts |
|
|
338
|
+
| `load_model(model_id)` | Download + return model info with path |
|
|
339
|
+
| `infer(model, inputs)` | Override for inference logic |
|
|
340
|
+
|
|
341
|
+
---
|
|
342
|
+
|
|
343
|
+
## Step-Based Workflows
|
|
344
|
+
|
|
345
|
+
Specialized actions support step-based execution via `setup_steps()`:
|
|
346
|
+
|
|
347
|
+
```python
|
|
348
|
+
from synapse_sdk.plugins.actions.train import BaseTrainAction, TrainContext
|
|
349
|
+
from synapse_sdk.plugins.steps import BaseStep, StepResult, StepRegistry
|
|
350
|
+
|
|
351
|
+
class LoadDatasetStep(BaseStep[TrainContext]):
|
|
352
|
+
@property
|
|
353
|
+
def name(self) -> str:
|
|
354
|
+
return 'load_dataset'
|
|
355
|
+
|
|
356
|
+
@property
|
|
357
|
+
def progress_weight(self) -> float:
|
|
358
|
+
return 0.2
|
|
359
|
+
|
|
360
|
+
def execute(self, context: TrainContext) -> StepResult:
|
|
361
|
+
# Load dataset into context
|
|
362
|
+
context.dataset = load_dataset(context.params['dataset_id'])
|
|
363
|
+
return StepResult(success=True, data={'count': len(context.dataset)})
|
|
364
|
+
|
|
365
|
+
class TrainStep(BaseStep[TrainContext]):
|
|
366
|
+
@property
|
|
367
|
+
def name(self) -> str:
|
|
368
|
+
return 'train'
|
|
369
|
+
|
|
370
|
+
@property
|
|
371
|
+
def progress_weight(self) -> float:
|
|
372
|
+
return 0.7
|
|
373
|
+
|
|
374
|
+
def execute(self, context: TrainContext) -> StepResult:
|
|
375
|
+
epochs = context.params.get('epochs', 100)
|
|
376
|
+
|
|
377
|
+
for epoch in range(epochs):
|
|
378
|
+
context.set_progress(epoch + 1, epochs)
|
|
379
|
+
# Training logic...
|
|
380
|
+
|
|
381
|
+
return StepResult(success=True)
|
|
382
|
+
|
|
383
|
+
class MyTrainAction(BaseTrainAction[MyParams]):
|
|
384
|
+
def setup_steps(self, registry: StepRegistry[TrainContext]) -> None:
|
|
385
|
+
"""Register workflow steps."""
|
|
386
|
+
registry.register(LoadDatasetStep())
|
|
387
|
+
registry.register(TrainStep())
|
|
388
|
+
# Step-based execution replaces execute()
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
**Execution Flow:**
|
|
392
|
+
|
|
393
|
+
```mermaid
|
|
394
|
+
flowchart TD
|
|
395
|
+
A["action.run()"] --> B{"setup_steps()<br/>registered steps?"}
|
|
396
|
+
|
|
397
|
+
B -->|Yes| C["Create context via<br/>create_context()"]
|
|
398
|
+
C --> D["Orchestrator.execute()"]
|
|
399
|
+
D --> E["Run steps in order"]
|
|
400
|
+
E --> F["Return orchestrator result"]
|
|
401
|
+
|
|
402
|
+
B -->|No| G["action.execute()"]
|
|
403
|
+
G --> H["Return execute() result"]
|
|
404
|
+
|
|
405
|
+
style A fill:#e8f5e9
|
|
406
|
+
style F fill:#e8f5e9
|
|
407
|
+
style H fill:#e8f5e9
|
|
408
|
+
```
|
|
409
|
+
|
|
410
|
+
---
|
|
411
|
+
|
|
412
|
+
## Async Execution Patterns
|
|
413
|
+
|
|
414
|
+
### Ray Serve Deployments
|
|
415
|
+
|
|
416
|
+
For HTTP inference endpoints with async handling:
|
|
417
|
+
|
|
418
|
+
```python
|
|
419
|
+
from ray import serve
|
|
420
|
+
from fastapi import FastAPI
|
|
421
|
+
from synapse_sdk.plugins.actions.inference import BaseServeDeployment
|
|
422
|
+
|
|
423
|
+
app = FastAPI()
|
|
424
|
+
|
|
425
|
+
@serve.deployment(num_replicas=2)
|
|
426
|
+
@serve.ingress(app)
|
|
427
|
+
class MyInferenceService(BaseServeDeployment):
|
|
428
|
+
"""Async inference service with model multiplexing."""
|
|
429
|
+
|
|
430
|
+
async def _get_model(self, model_info: dict) -> Any:
|
|
431
|
+
"""Load model from extracted artifacts."""
|
|
432
|
+
import torch
|
|
433
|
+
model_path = model_info['path'] / 'model.pt'
|
|
434
|
+
return torch.load(model_path)
|
|
435
|
+
|
|
436
|
+
async def infer(self, inputs: list[dict]) -> list[dict]:
|
|
437
|
+
"""Run inference on inputs."""
|
|
438
|
+
model = await self.get_model() # Multiplexed model loading
|
|
439
|
+
results = []
|
|
440
|
+
for inp in inputs:
|
|
441
|
+
prediction = model.predict(inp['data'])
|
|
442
|
+
results.append({'prediction': prediction.tolist()})
|
|
443
|
+
return results
|
|
444
|
+
|
|
445
|
+
@app.post('/predict')
|
|
446
|
+
async def predict(self, request: dict) -> dict:
|
|
447
|
+
results = await self.infer(request['inputs'])
|
|
448
|
+
return {'results': results}
|
|
449
|
+
|
|
450
|
+
# Deploy
|
|
451
|
+
deployment = MyInferenceService.bind(backend_url='https://api.example.com')
|
|
452
|
+
serve.run(deployment)
|
|
453
|
+
```
|
|
454
|
+
|
|
455
|
+
### Async Progress Streaming
|
|
456
|
+
|
|
457
|
+
For async progress monitoring:
|
|
458
|
+
|
|
459
|
+
```python
|
|
460
|
+
from synapse_sdk.plugins.pipelines import ActionPipeline
|
|
461
|
+
from synapse_sdk.plugins.pipelines.display import display_progress_async
|
|
462
|
+
from synapse_sdk.plugins.executors.ray import RayPipelineExecutor
|
|
463
|
+
|
|
464
|
+
async def run_pipeline_async():
|
|
465
|
+
pipeline = ActionPipeline([DownloadAction, ProcessAction, UploadAction])
|
|
466
|
+
executor = RayPipelineExecutor(
|
|
467
|
+
ray_address='auto',
|
|
468
|
+
pipeline_service_url='http://localhost:8100',
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
# Submit pipeline
|
|
472
|
+
run_id = pipeline.submit({'dataset_id': 123}, executor)
|
|
473
|
+
|
|
474
|
+
# Async progress streaming with Rich display
|
|
475
|
+
final = await display_progress_async(
|
|
476
|
+
executor.stream_progress_async(run_id)
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
print(f"Pipeline completed: {final.status}")
|
|
480
|
+
return final
|
|
481
|
+
```
|
|
482
|
+
|
|
483
|
+
### Async Iteration Pattern
|
|
484
|
+
|
|
485
|
+
Manual async progress handling:
|
|
486
|
+
|
|
487
|
+
```python
|
|
488
|
+
async def monitor_progress(executor, run_id):
|
|
489
|
+
"""Custom async progress monitoring."""
|
|
490
|
+
async for progress in executor.stream_progress_async(run_id):
|
|
491
|
+
print(f"Status: {progress.status}")
|
|
492
|
+
print(f"Progress: {progress.overall_progress}%")
|
|
493
|
+
|
|
494
|
+
if progress.current_action:
|
|
495
|
+
action = progress.actions[progress.current_action]
|
|
496
|
+
print(f"Current: {action.name} - {action.progress * 100}%")
|
|
497
|
+
|
|
498
|
+
# Custom handling
|
|
499
|
+
if progress.status in ('COMPLETED', 'FAILED'):
|
|
500
|
+
break
|
|
501
|
+
|
|
502
|
+
return progress
|
|
503
|
+
```
|
|
504
|
+
|
|
505
|
+
---
|
|
506
|
+
|
|
507
|
+
## Execution Modes
|
|
508
|
+
|
|
509
|
+
### Local Execution
|
|
510
|
+
|
|
511
|
+
Synchronous, in-process execution for development:
|
|
512
|
+
|
|
513
|
+
```python
|
|
514
|
+
from synapse_sdk.plugins import run_plugin
|
|
515
|
+
|
|
516
|
+
result = run_plugin(
|
|
517
|
+
plugin_code='my_plugin',
|
|
518
|
+
action='train',
|
|
519
|
+
params={'epochs': 10},
|
|
520
|
+
mode='local',
|
|
521
|
+
)
|
|
522
|
+
```
|
|
523
|
+
|
|
524
|
+
Or with explicit executor:
|
|
525
|
+
|
|
526
|
+
```python
|
|
527
|
+
from synapse_sdk.plugins.executors import LocalExecutor
|
|
528
|
+
|
|
529
|
+
executor = LocalExecutor(
|
|
530
|
+
env={'DEBUG': 'true'},
|
|
531
|
+
job_id='dev-123',
|
|
532
|
+
)
|
|
533
|
+
result = executor.execute(TrainAction, {'epochs': 10})
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
### Ray Task Execution
|
|
537
|
+
|
|
538
|
+
Fast startup with persistent actor:
|
|
539
|
+
|
|
540
|
+
```python
|
|
541
|
+
from synapse_sdk.plugins.executors.ray import RayActorExecutor
|
|
542
|
+
|
|
543
|
+
executor = RayActorExecutor(
|
|
544
|
+
ray_address='auto',
|
|
545
|
+
working_dir='/path/to/plugin',
|
|
546
|
+
num_gpus=1,
|
|
547
|
+
include_sdk=True, # For local SDK development
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
result = executor.execute(TrainAction, {'epochs': 100})
|
|
551
|
+
executor.shutdown() # Clean up actor
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
### Ray Job Execution
|
|
555
|
+
|
|
556
|
+
Full isolation for production workloads:
|
|
557
|
+
|
|
558
|
+
```python
|
|
559
|
+
from synapse_sdk.plugins.executors.ray import RayJobExecutor
|
|
560
|
+
|
|
561
|
+
executor = RayJobExecutor(
|
|
562
|
+
dashboard_url='http://ray-cluster:8265',
|
|
563
|
+
working_dir='/path/to/plugin',
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
# Async submission
|
|
567
|
+
job_id = executor.submit('train', {'epochs': 100})
|
|
568
|
+
|
|
569
|
+
# Monitor status
|
|
570
|
+
status = executor.get_status(job_id)
|
|
571
|
+
logs = executor.get_logs(job_id)
|
|
572
|
+
|
|
573
|
+
# Wait for completion
|
|
574
|
+
result = executor.wait(job_id, timeout_seconds=3600)
|
|
575
|
+
```
|
|
576
|
+
|
|
577
|
+
### Execution Mode Comparison
|
|
578
|
+
|
|
579
|
+
| Mode | Startup | Isolation | Use Case |
|
|
580
|
+
|------|---------|-----------|----------|
|
|
581
|
+
| `local` | Instant | None | Development, testing |
|
|
582
|
+
| `task` | <1s | Process | Fast parallel tasks |
|
|
583
|
+
| `job` | ~30s | Full | Production, heavy workloads |
|
|
584
|
+
| Ray Serve | Variable | Container | HTTP inference endpoints |
|
|
585
|
+
|
|
586
|
+
---
|
|
587
|
+
|
|
588
|
+
## Parameter Validation
|
|
589
|
+
|
|
590
|
+
### Pydantic Models
|
|
591
|
+
|
|
592
|
+
Use Pydantic v2 for comprehensive validation:
|
|
593
|
+
|
|
594
|
+
```python
|
|
595
|
+
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
596
|
+
|
|
597
|
+
class TrainParams(BaseModel):
|
|
598
|
+
"""Training parameters with validation."""
|
|
599
|
+
|
|
600
|
+
# Required with constraints
|
|
601
|
+
dataset_id: int = Field(..., gt=0, description='Dataset identifier')
|
|
602
|
+
|
|
603
|
+
# Optional with defaults
|
|
604
|
+
epochs: int = Field(default=100, ge=1, le=10000)
|
|
605
|
+
learning_rate: float = Field(default=0.001, gt=0, lt=1)
|
|
606
|
+
batch_size: int = Field(default=32, ge=1, le=512)
|
|
607
|
+
|
|
608
|
+
# String validation
|
|
609
|
+
model_name: str = Field(default='yolov8n', pattern=r'^yolov8[nsmlx]$')
|
|
610
|
+
|
|
611
|
+
# List validation
|
|
612
|
+
augmentations: list[str] = Field(default_factory=list)
|
|
613
|
+
|
|
614
|
+
@field_validator('augmentations')
|
|
615
|
+
@classmethod
|
|
616
|
+
def validate_augmentations(cls, v):
|
|
617
|
+
valid = {'flip', 'rotate', 'scale', 'mosaic'}
|
|
618
|
+
invalid = set(v) - valid
|
|
619
|
+
if invalid:
|
|
620
|
+
raise ValueError(f"Invalid augmentations: {invalid}")
|
|
621
|
+
return v
|
|
622
|
+
|
|
623
|
+
@model_validator(mode='after')
|
|
624
|
+
def validate_batch_epochs(self):
|
|
625
|
+
if self.batch_size > 256 and self.epochs > 1000:
|
|
626
|
+
raise ValueError("Large batch + many epochs may OOM")
|
|
627
|
+
return self
|
|
628
|
+
```
|
|
629
|
+
|
|
630
|
+
### Accessing Validated Params
|
|
631
|
+
|
|
632
|
+
```python
|
|
633
|
+
class MyAction(BaseAction[TrainParams]):
|
|
634
|
+
def execute(self) -> dict:
|
|
635
|
+
# Params are already validated
|
|
636
|
+
dataset_id = self.params.dataset_id # int, > 0
|
|
637
|
+
epochs = self.params.epochs # int, 1-10000
|
|
638
|
+
|
|
639
|
+
# Type hints work correctly
|
|
640
|
+
lr: float = self.params.learning_rate
|
|
641
|
+
|
|
642
|
+
return {'trained': True}
|
|
643
|
+
```
|
|
644
|
+
|
|
645
|
+
---
|
|
646
|
+
|
|
647
|
+
## Result Validation
|
|
648
|
+
|
|
649
|
+
### Defining Result Models
|
|
650
|
+
|
|
651
|
+
```python
|
|
652
|
+
from pydantic import BaseModel
|
|
653
|
+
|
|
654
|
+
class TrainResult(BaseModel):
|
|
655
|
+
"""Typed training result."""
|
|
656
|
+
weights_path: str
|
|
657
|
+
final_loss: float
|
|
658
|
+
epochs_completed: int
|
|
659
|
+
metrics: dict[str, float] = {}
|
|
660
|
+
|
|
661
|
+
class MyAction(BaseAction[TrainParams]):
|
|
662
|
+
result_model = TrainResult # Enable validation
|
|
663
|
+
|
|
664
|
+
def execute(self) -> TrainResult:
|
|
665
|
+
# Return typed result
|
|
666
|
+
return TrainResult(
|
|
667
|
+
weights_path='/model/weights.pt',
|
|
668
|
+
final_loss=0.05,
|
|
669
|
+
epochs_completed=100,
|
|
670
|
+
metrics={'accuracy': 0.95},
|
|
671
|
+
)
|
|
672
|
+
```
|
|
673
|
+
|
|
674
|
+
### Warning-Only Validation
|
|
675
|
+
|
|
676
|
+
Result validation is warning-only by default:
|
|
677
|
+
|
|
678
|
+
```python
|
|
679
|
+
class MyAction(BaseAction[MyParams]):
|
|
680
|
+
result_model = TrainResult
|
|
681
|
+
|
|
682
|
+
def execute(self) -> dict:
|
|
683
|
+
# This returns dict, not TrainResult
|
|
684
|
+
# Warning logged, but execution continues
|
|
685
|
+
return {'weights_path': '/model.pt'}
|
|
686
|
+
# Warning: Result validation warning for TrainResult: ...
|
|
687
|
+
```
|
|
688
|
+
|
|
689
|
+
---
|
|
690
|
+
|
|
691
|
+
## Progress Tracking
|
|
692
|
+
|
|
693
|
+
### Basic Progress
|
|
694
|
+
|
|
695
|
+
```python
|
|
696
|
+
def execute(self) -> dict:
|
|
697
|
+
total = 100
|
|
698
|
+
|
|
699
|
+
for i in range(total):
|
|
700
|
+
self.set_progress(i + 1, total)
|
|
701
|
+
# Process item...
|
|
702
|
+
|
|
703
|
+
return {'processed': total}
|
|
704
|
+
```
|
|
705
|
+
|
|
706
|
+
### Multi-Phase Progress
|
|
707
|
+
|
|
708
|
+
```python
|
|
709
|
+
def execute(self) -> dict:
|
|
710
|
+
# Phase 1: Download
|
|
711
|
+
self.set_progress(0, 100, category='download')
|
|
712
|
+
for i in range(100):
|
|
713
|
+
self.set_progress(i + 1, 100, category='download')
|
|
714
|
+
# Download chunk...
|
|
715
|
+
|
|
716
|
+
# Phase 2: Process
|
|
717
|
+
self.set_progress(0, 50, category='process')
|
|
718
|
+
for i in range(50):
|
|
719
|
+
self.set_progress(i + 1, 50, category='process')
|
|
720
|
+
# Process item...
|
|
721
|
+
|
|
722
|
+
return {'status': 'done'}
|
|
723
|
+
```
|
|
724
|
+
|
|
725
|
+
### Standard Progress Categories
|
|
726
|
+
|
|
727
|
+
Specialized actions provide standard category names:
|
|
728
|
+
|
|
729
|
+
```python
|
|
730
|
+
class MyTrainAction(BaseTrainAction[MyParams]):
|
|
731
|
+
def execute(self) -> dict:
|
|
732
|
+
# Use standard categories
|
|
733
|
+
self.set_progress(1, 3, self.progress.DATASET) # 'dataset'
|
|
734
|
+
self.set_progress(2, 3, self.progress.TRAIN) # 'train'
|
|
735
|
+
self.set_progress(3, 3, self.progress.MODEL_UPLOAD) # 'model_upload'
|
|
736
|
+
```
|
|
737
|
+
|
|
738
|
+
---
|
|
739
|
+
|
|
740
|
+
## Logging and Metrics
|
|
741
|
+
|
|
742
|
+
### Event Logging
|
|
743
|
+
|
|
744
|
+
```python
|
|
745
|
+
def execute(self) -> dict:
|
|
746
|
+
# Structured event logging
|
|
747
|
+
self.log('train_start', {
|
|
748
|
+
'epochs': self.params.epochs,
|
|
749
|
+
'learning_rate': self.params.learning_rate,
|
|
750
|
+
})
|
|
751
|
+
|
|
752
|
+
# Log with file association
|
|
753
|
+
self.log('checkpoint_saved', {'epoch': 50}, file='/model/ckpt_50.pt')
|
|
754
|
+
|
|
755
|
+
# Final event
|
|
756
|
+
self.log('train_complete', {'final_loss': 0.05})
|
|
757
|
+
|
|
758
|
+
return {'status': 'done'}
|
|
759
|
+
```
|
|
760
|
+
|
|
761
|
+
### Metrics Recording
|
|
762
|
+
|
|
763
|
+
```python
|
|
764
|
+
def execute(self) -> dict:
|
|
765
|
+
for epoch in range(self.params.epochs):
|
|
766
|
+
loss = train_epoch(epoch)
|
|
767
|
+
|
|
768
|
+
# Record metrics per category
|
|
769
|
+
self.set_metrics({
|
|
770
|
+
'loss': loss,
|
|
771
|
+
'accuracy': compute_accuracy(),
|
|
772
|
+
'epoch': epoch,
|
|
773
|
+
}, category='train')
|
|
774
|
+
|
|
775
|
+
return {'final_loss': loss}
|
|
776
|
+
```
|
|
777
|
+
|
|
778
|
+
### Context Methods
|
|
779
|
+
|
|
780
|
+
Access via `self.ctx` for additional logging:
|
|
781
|
+
|
|
782
|
+
```python
|
|
783
|
+
def execute(self) -> dict:
|
|
784
|
+
# User-facing message
|
|
785
|
+
self.ctx.log_message('Starting training...', context='info')
|
|
786
|
+
self.ctx.log_message('Training complete!', context='success')
|
|
787
|
+
self.ctx.log_message('Low accuracy detected', context='warning')
|
|
788
|
+
|
|
789
|
+
# Developer debug events (not shown to end users)
|
|
790
|
+
self.ctx.log_dev_event('Checkpoint saved', {'path': '/ckpt/model.pt'})
|
|
791
|
+
|
|
792
|
+
return {'status': 'done'}
|
|
793
|
+
```
|
|
794
|
+
|
|
795
|
+
---
|
|
796
|
+
|
|
797
|
+
## Config.yaml Integration
|
|
798
|
+
|
|
799
|
+
Define actions in `config.yaml` for discovery:
|
|
800
|
+
|
|
801
|
+
```yaml
|
|
802
|
+
# my_plugin/config.yaml
|
|
803
|
+
name: My Plugin
|
|
804
|
+
code: my_plugin
|
|
805
|
+
version: 0.1.0
|
|
806
|
+
category: neural_net
|
|
807
|
+
|
|
808
|
+
actions:
|
|
809
|
+
train:
|
|
810
|
+
name: Train Model
|
|
811
|
+
description: Train a model with custom parameters
|
|
812
|
+
entrypoint: my_plugin.actions:TrainAction
|
|
813
|
+
method: task
|
|
814
|
+
|
|
815
|
+
export:
|
|
816
|
+
name: Export Model
|
|
817
|
+
description: Export model to various formats
|
|
818
|
+
entrypoint: my_plugin.actions:ExportAction
|
|
819
|
+
method: local
|
|
820
|
+
|
|
821
|
+
infer:
|
|
822
|
+
name: Run Inference
|
|
823
|
+
description: Run batch inference
|
|
824
|
+
entrypoint: my_plugin.actions:InferAction
|
|
825
|
+
method: job
|
|
826
|
+
```
|
|
827
|
+
|
|
828
|
+
**Discovery:**
|
|
829
|
+
|
|
830
|
+
```python
|
|
831
|
+
from synapse_sdk.plugins.discovery import PluginDiscovery
|
|
832
|
+
|
|
833
|
+
# From config path
|
|
834
|
+
discovery = PluginDiscovery.from_path('/path/to/plugin')
|
|
835
|
+
|
|
836
|
+
# List actions
|
|
837
|
+
actions = discovery.list_actions() # ['train', 'export', 'infer']
|
|
838
|
+
|
|
839
|
+
# Get action class
|
|
840
|
+
TrainAction = discovery.get_action_class('train')
|
|
841
|
+
```
|
|
842
|
+
|
|
843
|
+
---
|
|
844
|
+
|
|
845
|
+
## Best Practices
|
|
846
|
+
|
|
847
|
+
### Action Design
|
|
848
|
+
|
|
849
|
+
1. **Single Responsibility**: One action = one task
|
|
850
|
+
2. **Idempotent Operations**: Safe to retry on failure
|
|
851
|
+
3. **Progress Updates**: Update frequently for user feedback
|
|
852
|
+
4. **Structured Logging**: Use consistent event names
|
|
853
|
+
|
|
854
|
+
### Error Handling
|
|
855
|
+
|
|
856
|
+
```python
|
|
857
|
+
from synapse_sdk.plugins.errors import ExecutionError, ValidationError
|
|
858
|
+
|
|
859
|
+
def execute(self) -> dict:
|
|
860
|
+
# Validate early
|
|
861
|
+
if not Path(self.params.input_path).exists():
|
|
862
|
+
raise ValidationError(f"Input not found: {self.params.input_path}")
|
|
863
|
+
|
|
864
|
+
try:
|
|
865
|
+
result = process_data(self.params.input_path)
|
|
866
|
+
except IOError as e:
|
|
867
|
+
raise ExecutionError(f"Processing failed: {e}") from e
|
|
868
|
+
|
|
869
|
+
return result
|
|
870
|
+
```
|
|
871
|
+
|
|
872
|
+
### Resource Cleanup
|
|
873
|
+
|
|
874
|
+
```python
|
|
875
|
+
def execute(self) -> dict:
|
|
876
|
+
temp_dir = None
|
|
877
|
+
try:
|
|
878
|
+
temp_dir = Path(tempfile.mkdtemp())
|
|
879
|
+
# Use temp_dir...
|
|
880
|
+
return {'status': 'done'}
|
|
881
|
+
finally:
|
|
882
|
+
if temp_dir and temp_dir.exists():
|
|
883
|
+
shutil.rmtree(temp_dir)
|
|
884
|
+
```
|
|
885
|
+
|
|
886
|
+
### Testing Actions
|
|
887
|
+
|
|
888
|
+
```python
|
|
889
|
+
import pytest
|
|
890
|
+
from synapse_sdk.plugins.executors import LocalExecutor
|
|
891
|
+
from synapse_sdk.loggers import NoOpLogger
|
|
892
|
+
from synapse_sdk.plugins.context import RuntimeContext, PluginEnvironment
|
|
893
|
+
|
|
894
|
+
def test_train_action():
|
|
895
|
+
"""Test action with mock context."""
|
|
896
|
+
executor = LocalExecutor(
|
|
897
|
+
env={'DEBUG': 'true'},
|
|
898
|
+
job_id='test-123',
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
result = executor.execute(
|
|
902
|
+
TrainAction,
|
|
903
|
+
{'epochs': 5, 'learning_rate': 0.01},
|
|
904
|
+
)
|
|
905
|
+
|
|
906
|
+
assert result['epochs_completed'] == 5
|
|
907
|
+
assert 'weights_path' in result
|
|
908
|
+
```
|
|
909
|
+
|
|
910
|
+
---
|
|
911
|
+
|
|
912
|
+
## Module Reference
|
|
913
|
+
|
|
914
|
+
| Module | Classes | Purpose |
|
|
915
|
+
|--------|---------|---------|
|
|
916
|
+
| `synapse_sdk.plugins` | `BaseAction`, `action`, `run_plugin` | Core action API |
|
|
917
|
+
| `synapse_sdk.plugins.actions.train` | `BaseTrainAction`, `BaseTrainParams` | Training workflows |
|
|
918
|
+
| `synapse_sdk.plugins.actions.export` | `BaseExportAction` | Export workflows |
|
|
919
|
+
| `synapse_sdk.plugins.actions.inference` | `BaseInferenceAction`, `BaseServeDeployment` | Inference workflows |
|
|
920
|
+
| `synapse_sdk.plugins.actions.upload` | `BaseUploadAction` | Upload workflows |
|
|
921
|
+
| `synapse_sdk.plugins.context` | `RuntimeContext`, `PluginEnvironment` | Action context |
|
|
922
|
+
| `synapse_sdk.plugins.executors` | `LocalExecutor` | Local execution |
|
|
923
|
+
| `synapse_sdk.plugins.executors.ray` | `RayActorExecutor`, `RayJobExecutor` | Ray execution |
|
|
924
|
+
|
|
925
|
+
---
|
|
926
|
+
|
|
927
|
+
## Related Documentation
|
|
928
|
+
|
|
929
|
+
- **[OVERVIEW.md](OVERVIEW.md)** - Plugin system introduction
|
|
930
|
+
- **[ARCHITECTURE.md](ARCHITECTURE.md)** - Technical architecture
|
|
931
|
+
- **[STEP.md](STEP.md)** - Step implementations
|
|
932
|
+
- **[LOGGING_SYSTEM.md](LOGGING_SYSTEM.md)** - Logging and progress
|
|
933
|
+
- **[PIPELINE_GUIDE.md](PIPELINE_GUIDE.md)** - Multi-action pipelines
|