synapse-sdk 1.0.0a0__py3-none-any.whl → 1.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse-sdk might be problematic. Click here for more details.
- synapse_sdk/clients/agent/__init__.py +21 -0
- synapse_sdk/clients/agent/core.py +11 -0
- synapse_sdk/clients/agent/service.py +15 -0
- synapse_sdk/clients/backend/__init__.py +21 -0
- synapse_sdk/clients/backend/annotation.py +29 -0
- {synapse/client/mixins → synapse_sdk/clients/backend}/dataset.py +6 -4
- synapse_sdk/clients/backend/integration.py +36 -0
- synapse_sdk/clients/backend/ml.py +24 -0
- synapse/client/__init__.py → synapse_sdk/clients/base.py +22 -30
- {synapse → synapse_sdk}/loggers.py +1 -1
- synapse_sdk/plugins/__init__.py +13 -0
- {synapse → synapse_sdk}/plugins/categories/base.py +43 -45
- synapse_sdk/plugins/categories/data_validation/actions/validation.py +10 -0
- {synapse → synapse_sdk}/plugins/categories/decorators.py +2 -2
- synapse_sdk/plugins/categories/export/actions/export.py +10 -0
- synapse_sdk/plugins/categories/import/actions/import.py +10 -0
- {synapse → synapse_sdk}/plugins/categories/neural_net/actions/deployment.py +3 -3
- synapse_sdk/plugins/categories/neural_net/actions/inference.py +10 -0
- synapse_sdk/plugins/categories/neural_net/actions/test.py +10 -0
- {synapse → synapse_sdk}/plugins/categories/neural_net/actions/train.py +5 -5
- synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py +10 -0
- synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py +10 -0
- {synapse → synapse_sdk}/plugins/categories/registry.py +2 -2
- synapse_sdk/plugins/cli/__init__.py +19 -0
- synapse_sdk/plugins/cli/publish.py +36 -0
- synapse_sdk/plugins/cli/run.py +63 -0
- synapse_sdk/plugins/upload.py +82 -0
- {synapse → synapse_sdk}/plugins/utils.py +8 -4
- {synapse → synapse_sdk}/utils/file.py +13 -1
- synapse_sdk/utils/storage.py +91 -0
- {synapse → synapse_sdk}/utils/string.py +4 -0
- {synapse_sdk-1.0.0a0.dist-info → synapse_sdk-1.0.0a2.dist-info}/METADATA +2 -1
- synapse_sdk-1.0.0a2.dist-info/RECORD +58 -0
- {synapse_sdk-1.0.0a0.dist-info → synapse_sdk-1.0.0a2.dist-info}/WHEEL +1 -1
- synapse_sdk-1.0.0a2.dist-info/top_level.txt +1 -0
- synapse/client/mixins/annotation.py +0 -28
- synapse/client/mixins/integration.py +0 -12
- synapse/client/mixins/ml.py +0 -23
- synapse/config.py +0 -8
- synapse/plugins/__init__.py +0 -39
- synapse/plugins/categories/data_validation/actions/validation.py +0 -10
- synapse/plugins/categories/export/actions/export.py +0 -10
- synapse/plugins/categories/import/actions/import.py +0 -10
- synapse/plugins/categories/neural_net/actions/inference.py +0 -10
- synapse/plugins/categories/neural_net/actions/test.py +0 -10
- synapse/plugins/categories/post_annotation/actions/post_annotation.py +0 -10
- synapse/plugins/categories/pre_annotation/actions/pre_annotation.py +0 -10
- synapse/plugins/upload.py +0 -79
- synapse_sdk-1.0.0a0.dist-info/RECORD +0 -51
- synapse_sdk-1.0.0a0.dist-info/top_level.txt +0 -1
- {synapse → synapse_sdk}/__init__.py +0 -0
- {synapse/client/mixins → synapse_sdk/clients}/__init__.py +0 -0
- {synapse/client → synapse_sdk/clients}/exceptions.py +0 -0
- {synapse/client → synapse_sdk/clients}/utils.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/data_validation/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/data_validation/actions/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/export/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/export/actions/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/import/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/import/actions/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/neural_net/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/neural_net/actions/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/post_annotation/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/post_annotation/actions/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/pre_annotation/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/categories/pre_annotation/actions/__init__.py +0 -0
- {synapse → synapse_sdk}/plugins/enums.py +0 -0
- {synapse → synapse_sdk}/plugins/job.py +0 -0
- {synapse → synapse_sdk}/utils/__init__.py +0 -0
- {synapse → synapse_sdk}/utils/debug.py +0 -0
- {synapse → synapse_sdk}/utils/module_loading.py +0 -0
- {synapse_sdk-1.0.0a0.dist-info → synapse_sdk-1.0.0a2.dist-info}/LICENSE +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from synapse_sdk.clients.agent.core import CoreClientMixin
|
|
2
|
+
from synapse_sdk.clients.agent.service import ServiceClientMixin
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class AgentClient(CoreClientMixin, ServiceClientMixin):
|
|
6
|
+
agent_token = None
|
|
7
|
+
user_token = None
|
|
8
|
+
tenant = None
|
|
9
|
+
|
|
10
|
+
def __init__(self, base_url, agent_token, user_token, tenant):
|
|
11
|
+
super().__init__(base_url)
|
|
12
|
+
self.agent_token = agent_token
|
|
13
|
+
self.user_token = user_token
|
|
14
|
+
self.tenant = tenant
|
|
15
|
+
|
|
16
|
+
def _get_headers(self):
|
|
17
|
+
return {
|
|
18
|
+
'Authorization': self.agent_token,
|
|
19
|
+
'SYNAPSE-User': f'Token {self.user_token}',
|
|
20
|
+
'SYNAPSE-Tenant': f'Token {self.tenant}',
|
|
21
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from synapse_sdk.clients.base import BaseClient
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class CoreClientMixin(BaseClient):
|
|
5
|
+
def get_agent_system(self):
|
|
6
|
+
path = 'agents/system/'
|
|
7
|
+
return self._get(path)
|
|
8
|
+
|
|
9
|
+
def get_jobs_progress(self, data):
|
|
10
|
+
path = 'integration/jobs/progress/'
|
|
11
|
+
return self._post(path, data=data, timeout=1)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from synapse_sdk.clients.base import BaseClient
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ServiceClientMixin(BaseClient):
|
|
5
|
+
def run_plugin_release(self, code, data):
|
|
6
|
+
path = f'plugin_releases/{code}/run/'
|
|
7
|
+
return self._post(path, data=data)
|
|
8
|
+
|
|
9
|
+
def run_debug_plugin_release(self, data):
|
|
10
|
+
path = 'plugin_releases/run_debug/'
|
|
11
|
+
return self._post(path, data=data)
|
|
12
|
+
|
|
13
|
+
def create_plugin_release(self, data):
|
|
14
|
+
path = 'plugin_releases/'
|
|
15
|
+
return self._post(path, data=data)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
|
|
2
|
+
from synapse_sdk.clients.backend.dataset import DatasetClientMixin
|
|
3
|
+
from synapse_sdk.clients.backend.integration import IntegrationClientMixin
|
|
4
|
+
from synapse_sdk.clients.backend.ml import MLClientMixin
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class BackendClient(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
|
|
8
|
+
token = None
|
|
9
|
+
tenant = None
|
|
10
|
+
|
|
11
|
+
def __init__(self, base_url, token, tenant=None):
|
|
12
|
+
super().__init__(base_url)
|
|
13
|
+
self.token = token
|
|
14
|
+
if tenant:
|
|
15
|
+
self.tenant = tenant
|
|
16
|
+
|
|
17
|
+
def _get_headers(self):
|
|
18
|
+
headers = {'Authorization': f'Token {self.token}'}
|
|
19
|
+
if self.tenant:
|
|
20
|
+
headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
|
|
21
|
+
return headers
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from synapse_sdk.clients.base import BaseClient
|
|
2
|
+
from synapse_sdk.clients.utils import get_default_url_conversion
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class AnnotationClientMixin(BaseClient):
|
|
6
|
+
def get_project(self, pk):
|
|
7
|
+
path = f'projects/{pk}/'
|
|
8
|
+
return self._get(path)
|
|
9
|
+
|
|
10
|
+
def get_label_tag(self, pk):
|
|
11
|
+
path = f'label_tags/{pk}/'
|
|
12
|
+
return self._get(path)
|
|
13
|
+
|
|
14
|
+
def list_label_tags(self, data):
|
|
15
|
+
path = 'label_tags/'
|
|
16
|
+
return self._list(path, data=data)
|
|
17
|
+
|
|
18
|
+
def list_labels(self, data, url_conversion=None, list_all=False):
|
|
19
|
+
path = 'labels/'
|
|
20
|
+
url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
|
|
21
|
+
return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
|
|
22
|
+
|
|
23
|
+
def create_labels(self, data):
|
|
24
|
+
path = 'labels/'
|
|
25
|
+
return self._post(path, data=data)
|
|
26
|
+
|
|
27
|
+
def set_tags_labels(self, data, params=None):
|
|
28
|
+
path = 'labels/set_tags/'
|
|
29
|
+
return self._post(path, data=data, params=params)
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from multiprocessing import Pool
|
|
2
2
|
from tqdm import tqdm
|
|
3
|
-
from ..utils import get_batched_list
|
|
4
3
|
|
|
4
|
+
from synapse_sdk.clients.base import BaseClient
|
|
5
|
+
from synapse_sdk.clients.utils import get_batched_list
|
|
5
6
|
|
|
6
|
-
|
|
7
|
+
|
|
8
|
+
class DatasetClientMixin(BaseClient):
|
|
7
9
|
def list_dataset(self):
|
|
8
10
|
path = 'datasets/'
|
|
9
11
|
return self._get(path)
|
|
@@ -14,10 +16,10 @@ class DatasetClientMixin:
|
|
|
14
16
|
|
|
15
17
|
def create_data_units(self, data):
|
|
16
18
|
path = 'data_units/'
|
|
17
|
-
return self._post(path,
|
|
19
|
+
return self._post(path, data=data)
|
|
18
20
|
|
|
19
21
|
def import_dataset(self, dataset_id, dataset, project_id=None, batch_size=1000, process_pool=10):
|
|
20
|
-
# TODO validate
|
|
22
|
+
# TODO validate dataset with schema
|
|
21
23
|
|
|
22
24
|
params = [(data, dataset_id) for data in dataset]
|
|
23
25
|
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from synapse_sdk.clients.base import BaseClient
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class IntegrationClientMixin(BaseClient):
|
|
5
|
+
def get_plugin(self, pk):
|
|
6
|
+
path = f'plugins/{pk}/'
|
|
7
|
+
return self._get(path)
|
|
8
|
+
|
|
9
|
+
def create_plugin(self, data):
|
|
10
|
+
path = 'plugins/'
|
|
11
|
+
return self._post(path, data=data)
|
|
12
|
+
|
|
13
|
+
def update_plugin(self, pk, data):
|
|
14
|
+
path = f'plugins/{pk}/'
|
|
15
|
+
return self._put(path, data=data)
|
|
16
|
+
|
|
17
|
+
def run_plugin(self, pk, data):
|
|
18
|
+
path = f'plugins/{pk}/run/'
|
|
19
|
+
return self._post(path, data=data)
|
|
20
|
+
|
|
21
|
+
def get_plugin_release(self, pk, params=None):
|
|
22
|
+
path = f'plugin_releases/{pk}/'
|
|
23
|
+
return self._get(path, params=params)
|
|
24
|
+
|
|
25
|
+
def create_plugin_release(self, data):
|
|
26
|
+
path = 'plugin_releases/'
|
|
27
|
+
files = {'file': data.pop('file')}
|
|
28
|
+
return self._post(path, data=data, files=files)
|
|
29
|
+
|
|
30
|
+
def create_logs(self, data):
|
|
31
|
+
path = 'logs/'
|
|
32
|
+
return self._post(path, data=data)
|
|
33
|
+
|
|
34
|
+
def create_task(self, data):
|
|
35
|
+
path = 'agent_tasks/'
|
|
36
|
+
return self._post(path, data=data)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from synapse_sdk.clients.base import BaseClient
|
|
2
|
+
from synapse_sdk.clients.utils import get_default_url_conversion
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class MLClientMixin(BaseClient):
|
|
6
|
+
def get_model(self, pk, params=None, url_conversion=None):
|
|
7
|
+
path = f'models/{pk}/'
|
|
8
|
+
url_conversion = get_default_url_conversion(
|
|
9
|
+
url_conversion, files_fields=['files', 'parent.files'], is_list=False
|
|
10
|
+
)
|
|
11
|
+
return self._get(path, params=params, url_conversion=url_conversion)
|
|
12
|
+
|
|
13
|
+
def create_model(self, data):
|
|
14
|
+
path = 'models/'
|
|
15
|
+
return self._post(path, data=data)
|
|
16
|
+
|
|
17
|
+
def update_model(self, pk, data, files=None):
|
|
18
|
+
path = f'models/{pk}/'
|
|
19
|
+
return self._patch(path, data=data, files=files)
|
|
20
|
+
|
|
21
|
+
def list_train_dataset(self, params=None, url_conversion=None, list_all=False):
|
|
22
|
+
path = 'train_dataset/'
|
|
23
|
+
url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
|
|
24
|
+
return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
|
|
@@ -4,24 +4,16 @@ from pathlib import Path
|
|
|
4
4
|
|
|
5
5
|
import requests
|
|
6
6
|
|
|
7
|
-
from .exceptions import ClientError
|
|
8
|
-
from .
|
|
9
|
-
from .mixins.dataset import DatasetClientMixin
|
|
10
|
-
from .mixins.integration import IntegrationClientMixin
|
|
11
|
-
from .mixins.ml import MLClientMixin
|
|
12
|
-
from ..utils.file import files_url_to_path_from_objs
|
|
7
|
+
from synapse_sdk.clients.exceptions import ClientError
|
|
8
|
+
from synapse_sdk.utils.file import files_url_to_path_from_objs
|
|
13
9
|
|
|
14
10
|
|
|
15
|
-
class
|
|
11
|
+
class BaseClient:
|
|
12
|
+
name = None
|
|
16
13
|
base_url = None
|
|
17
|
-
token = None
|
|
18
|
-
workspace_code = None
|
|
19
14
|
|
|
20
|
-
def __init__(self, base_url
|
|
15
|
+
def __init__(self, base_url):
|
|
21
16
|
self.base_url = base_url
|
|
22
|
-
self.token = token
|
|
23
|
-
if workspace_code:
|
|
24
|
-
self.workspace_code = workspace_code
|
|
25
17
|
requests_session = requests.Session()
|
|
26
18
|
self.requests_session = requests_session
|
|
27
19
|
|
|
@@ -31,10 +23,7 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
|
|
|
31
23
|
return path
|
|
32
24
|
|
|
33
25
|
def _get_headers(self):
|
|
34
|
-
|
|
35
|
-
if self.workspace_code:
|
|
36
|
-
headers['DATAMAKER-Workspace'] = f'Token {self.workspace_code}'
|
|
37
|
-
return headers
|
|
26
|
+
return {}
|
|
38
27
|
|
|
39
28
|
def _request(self, method, path, **kwargs):
|
|
40
29
|
url = self._get_url(path)
|
|
@@ -56,12 +45,12 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
|
|
|
56
45
|
response.status_code, response.json() if response.status_code == 400 else response.reason
|
|
57
46
|
)
|
|
58
47
|
except requests.ConnectionError:
|
|
59
|
-
raise ClientError(408, '
|
|
48
|
+
raise ClientError(408, f'{self.name} is not responding')
|
|
60
49
|
|
|
61
50
|
return response.json()
|
|
62
51
|
|
|
63
|
-
def _get(self, path,
|
|
64
|
-
response = self._request('get', path,
|
|
52
|
+
def _get(self, path, url_conversion=None, **kwargs):
|
|
53
|
+
response = self._request('get', path, **kwargs)
|
|
65
54
|
if url_conversion:
|
|
66
55
|
if url_conversion['is_list']:
|
|
67
56
|
files_url_to_path_from_objs(response['results'], **url_conversion)
|
|
@@ -69,21 +58,24 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
|
|
|
69
58
|
files_url_to_path_from_objs(response, **url_conversion)
|
|
70
59
|
return response
|
|
71
60
|
|
|
72
|
-
def _post(self, path,
|
|
73
|
-
return self._request('post', path,
|
|
61
|
+
def _post(self, path, **kwargs):
|
|
62
|
+
return self._request('post', path, **kwargs)
|
|
74
63
|
|
|
75
|
-
def
|
|
76
|
-
return self._request('
|
|
64
|
+
def _put(self, path, **kwargs):
|
|
65
|
+
return self._request('put', path, **kwargs)
|
|
77
66
|
|
|
78
|
-
def
|
|
79
|
-
|
|
67
|
+
def _patch(self, path, **kwargs):
|
|
68
|
+
return self._request('patch', path, **kwargs)
|
|
69
|
+
|
|
70
|
+
def _list(self, path, url_conversion=None, list_all=False, **kwargs):
|
|
71
|
+
response = self._get(path, url_conversion, **kwargs)
|
|
80
72
|
if list_all:
|
|
81
|
-
return self._list_all(path,
|
|
73
|
+
return self._list_all(path, url_conversion, **kwargs), response['count']
|
|
82
74
|
else:
|
|
83
75
|
return response
|
|
84
76
|
|
|
85
|
-
def _list_all(self, path,
|
|
86
|
-
response = self._get(path,
|
|
77
|
+
def _list_all(self, path, url_conversion=None, **kwargs):
|
|
78
|
+
response = self._get(path, url_conversion, **kwargs)
|
|
87
79
|
yield from response['results']
|
|
88
80
|
if response['next']:
|
|
89
|
-
yield from self._list_all(response['next'],
|
|
81
|
+
yield from self._list_all(response['next'], url_conversion, **kwargs)
|
|
@@ -8,11 +8,12 @@ import ray
|
|
|
8
8
|
import requests
|
|
9
9
|
from ray.dashboard.modules.job.sdk import JobSubmissionClient
|
|
10
10
|
|
|
11
|
-
from
|
|
12
|
-
from
|
|
13
|
-
from
|
|
14
|
-
from
|
|
15
|
-
from
|
|
11
|
+
from synapse_sdk.loggers import ConsoleLogger
|
|
12
|
+
from synapse_sdk.plugins.enums import RunMethod
|
|
13
|
+
from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload, download_and_upload
|
|
14
|
+
from synapse_sdk.plugins.utils import get_plugin_checksum
|
|
15
|
+
from synapse_sdk.utils.module_loading import import_string
|
|
16
|
+
from synapse_sdk.utils.storage import get_storage
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
class Action:
|
|
@@ -27,15 +28,10 @@ class Action:
|
|
|
27
28
|
debug = False
|
|
28
29
|
|
|
29
30
|
default_envs = [
|
|
31
|
+
'RAY_DASHBOARD_URL',
|
|
30
32
|
'RAY_SERVE_ADDRESS',
|
|
31
|
-
'
|
|
32
|
-
'
|
|
33
|
-
'SYNAPSE_PLUGIN_BASE_URL',
|
|
34
|
-
'SYNAPSE_PLUGIN_UPLOAD_S3_ENDPOINT_URL',
|
|
35
|
-
'SYNAPSE_PLUGIN_UPLOAD_S3_BUCKET_NAME',
|
|
36
|
-
'SYNAPSE_PLUGIN_UPLOAD_S3_ACCESS_KEY',
|
|
37
|
-
'SYNAPSE_PLUGIN_UPLOAD_S3_SECRET_KEY',
|
|
38
|
-
'SYNAPSE_PLUGIN_UPLOAD_S3_BASE_URL',
|
|
33
|
+
'SYNAPSE_PLUGIN_STORAGE',
|
|
34
|
+
'SYNAPSE_DEBUG_PLUGIN_PATH',
|
|
39
35
|
'SYNAPSE_DEBUG_MODULES',
|
|
40
36
|
]
|
|
41
37
|
|
|
@@ -64,45 +60,44 @@ class Action:
|
|
|
64
60
|
def plugin_checksum(self):
|
|
65
61
|
return get_plugin_checksum(self.plugin_id)
|
|
66
62
|
|
|
67
|
-
@cached_property
|
|
68
|
-
def plugin_url(self):
|
|
69
|
-
if self.debug:
|
|
70
|
-
plugin_url = self.envs.get('SYNAPSE_PLUGIN_URL')
|
|
71
|
-
if not plugin_url:
|
|
72
|
-
plugin_url = upload_path(self.envs.get('SYNAPSE_PLUGIN_PATH', '.'), **self.get_kwargs_upload_path())
|
|
73
|
-
self.envs['SYNAPSE_PLUGIN_URL'] = plugin_url
|
|
74
|
-
return plugin_url
|
|
75
|
-
base_url = self.envs['SYNAPSE_PLUGIN_BASE_URL']
|
|
76
|
-
return str(os.path.join(base_url, f'{self.plugin_checksum}.zip'))
|
|
77
|
-
|
|
78
63
|
@cached_property
|
|
79
64
|
def entrypoint(self):
|
|
80
65
|
return import_string(self.config['entrypoint'])
|
|
81
66
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
return
|
|
85
|
-
'endpoint_url': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_ENDPOINT_URL'],
|
|
86
|
-
'bucket_name': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_BUCKET_NAME'],
|
|
87
|
-
'access_key': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_ACCESS_KEY'],
|
|
88
|
-
'secret_key': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_SECRET_KEY'],
|
|
89
|
-
'base_url': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_BASE_URL'],
|
|
90
|
-
}
|
|
67
|
+
@property
|
|
68
|
+
def plugin_storage_url(self):
|
|
69
|
+
return self.envs['SYNAPSE_PLUGIN_STORAGE']
|
|
91
70
|
|
|
92
|
-
|
|
93
|
-
|
|
71
|
+
@property
|
|
72
|
+
def plugin_url(self):
|
|
73
|
+
if self.debug:
|
|
74
|
+
plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH', '.')
|
|
75
|
+
if plugin_path.startswith('https://'): # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
|
|
76
|
+
plugin_url = plugin_path
|
|
77
|
+
elif plugin_path.startswith('http://'):
|
|
78
|
+
plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
|
|
79
|
+
else:
|
|
80
|
+
plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
|
|
81
|
+
self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
|
|
82
|
+
return plugin_url
|
|
83
|
+
storage = get_storage(self.plugin_storage_url)
|
|
84
|
+
return storage.get_url(f'{self.plugin_checksum}.zip')
|
|
94
85
|
|
|
95
|
-
|
|
86
|
+
@property
|
|
87
|
+
def debug_modules(self):
|
|
96
88
|
debug_modules = []
|
|
97
89
|
for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
|
|
98
|
-
if module_path.startswith('
|
|
90
|
+
if module_path.startswith('https://'): # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
|
|
99
91
|
module_url = module_path
|
|
100
92
|
else:
|
|
101
|
-
module_url = build_and_upload(module_path,
|
|
93
|
+
module_url = build_and_upload(module_path, self.plugin_storage_url)
|
|
102
94
|
debug_modules.append(module_url)
|
|
103
95
|
self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
|
|
104
96
|
return debug_modules
|
|
105
97
|
|
|
98
|
+
def get_default_envs(self):
|
|
99
|
+
return {env: os.environ[env] for env in self.default_envs if env in os.environ}
|
|
100
|
+
|
|
106
101
|
def get_runtime_env(self):
|
|
107
102
|
runtime_env = {
|
|
108
103
|
'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
|
|
@@ -110,10 +105,13 @@ class Action:
|
|
|
110
105
|
}
|
|
111
106
|
|
|
112
107
|
if self.debug:
|
|
113
|
-
runtime_env['pip'] += self.
|
|
108
|
+
runtime_env['pip'] += self.debug_modules
|
|
114
109
|
|
|
110
|
+
# 맨 마지막에 진행되어야 함
|
|
115
111
|
runtime_env['env_vars'] = self.envs
|
|
116
|
-
|
|
112
|
+
|
|
113
|
+
if self.debug:
|
|
114
|
+
pprint(runtime_env)
|
|
117
115
|
return runtime_env
|
|
118
116
|
|
|
119
117
|
def run(self):
|
|
@@ -130,7 +128,7 @@ class Action:
|
|
|
130
128
|
def run_by_task(self):
|
|
131
129
|
@ray.remote(runtime_env=self.get_runtime_env())
|
|
132
130
|
def run_task(category, action, *args, **kwargs):
|
|
133
|
-
from
|
|
131
|
+
from synapse_sdk.plugins.utils import get_action_class
|
|
134
132
|
|
|
135
133
|
action = get_action_class(category, action)(*args, **kwargs)
|
|
136
134
|
return action.run_action()
|
|
@@ -152,14 +150,14 @@ class Action:
|
|
|
152
150
|
return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
|
|
153
151
|
|
|
154
152
|
def run_by_job(self):
|
|
155
|
-
entrypoint_args = [self.name, f'"{json.dumps(self.params)}"']
|
|
153
|
+
entrypoint_args = ['run', self.name, f'"{json.dumps(self.params)}"', '--direct']
|
|
156
154
|
if self.debug:
|
|
157
|
-
entrypoint_args.
|
|
155
|
+
entrypoint_args.insert(0, '--debug')
|
|
158
156
|
|
|
159
|
-
client = JobSubmissionClient()
|
|
157
|
+
client = JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
|
|
160
158
|
return client.submit_job(
|
|
161
159
|
submission_id=self.job_id,
|
|
162
|
-
entrypoint=f'python main.py {" ".join(entrypoint_args)}
|
|
160
|
+
entrypoint=f'python main.py {" ".join(entrypoint_args)}',
|
|
163
161
|
runtime_env=self.get_runtime_env(),
|
|
164
162
|
)
|
|
165
163
|
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class ValidationAction(Action):
|
|
8
|
+
name = 'validation'
|
|
9
|
+
category = PluginCategory.DATA_VALIDATION
|
|
10
|
+
method = RunMethod.TASK
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
from
|
|
2
|
-
from
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
def register_action(action_class):
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class ExportAction(Action):
|
|
8
|
+
name = 'export'
|
|
9
|
+
category = PluginCategory.EXPORT
|
|
10
|
+
method = RunMethod.JOB
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class ImportAction(Action):
|
|
8
|
+
name = 'import'
|
|
9
|
+
category = PluginCategory.IMPORT
|
|
10
|
+
method = RunMethod.JOB
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from ray import serve
|
|
2
2
|
|
|
3
|
-
from
|
|
4
|
-
from
|
|
5
|
-
from
|
|
3
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
4
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
5
|
+
from synapse_sdk.plugins.enums import PluginCategory, RunMethod
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
@register_action
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class InferenceAction(Action):
|
|
8
|
+
name = 'inference'
|
|
9
|
+
category = PluginCategory.NEURAL_NET
|
|
10
|
+
method = RunMethod.RESTAPI
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class TestAction(Action):
|
|
8
|
+
name = 'test'
|
|
9
|
+
category = PluginCategory.NEURAL_NET
|
|
10
|
+
method = RunMethod.TASK
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
from
|
|
2
|
-
from
|
|
3
|
-
from
|
|
4
|
-
from
|
|
5
|
-
from
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
from synapse_sdk.utils.file import get_dict_from_file, files_url_to_path_from_objs
|
|
5
|
+
from synapse_sdk.utils.module_loading import import_string
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
@register_action
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class PostAnnotationAction(Action):
|
|
8
|
+
name = 'post_annotation'
|
|
9
|
+
category = PluginCategory.POST_ANNOTATION
|
|
10
|
+
method = RunMethod.TASK
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from synapse_sdk.plugins.categories.base import Action
|
|
2
|
+
from synapse_sdk.plugins.categories.decorators import register_action
|
|
3
|
+
from synapse_sdk.plugins.enums import RunMethod, PluginCategory
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_action
|
|
7
|
+
class PreAnnotationAction(Action):
|
|
8
|
+
name = 'pre_annotation'
|
|
9
|
+
category = PluginCategory.PRE_ANNOTATION
|
|
10
|
+
method = RunMethod.TASK
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import pkgutil
|
|
2
2
|
from importlib import import_module
|
|
3
3
|
|
|
4
|
-
from
|
|
4
|
+
from synapse_sdk.plugins.enums import PluginCategory
|
|
5
5
|
|
|
6
6
|
_REGISTERED_ACTIONS = {}
|
|
7
7
|
|
|
@@ -9,7 +9,7 @@ _REGISTERED_ACTIONS = {}
|
|
|
9
9
|
def register_actions():
|
|
10
10
|
if not _REGISTERED_ACTIONS:
|
|
11
11
|
for category in PluginCategory:
|
|
12
|
-
plugin_category_module_name = f'
|
|
12
|
+
plugin_category_module_name = f'synapse_sdk.plugins.categories.{category.value}.actions'
|
|
13
13
|
plugin_category_module = import_module(plugin_category_module_name)
|
|
14
14
|
for _, action_name, _ in pkgutil.iter_modules(plugin_category_module.__path__):
|
|
15
15
|
action_module_name = f'{plugin_category_module_name}.{action_name}'
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from .publish import publish
|
|
4
|
+
from .run import run
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@click.group()
|
|
8
|
+
@click.option('--debug/--no-debug', default=False)
|
|
9
|
+
@click.pass_context
|
|
10
|
+
def cli(ctx, debug):
|
|
11
|
+
ctx.ensure_object(dict)
|
|
12
|
+
ctx.obj['DEBUG'] = debug
|
|
13
|
+
|
|
14
|
+
if debug:
|
|
15
|
+
click.echo('Debug mode is "on"')
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
cli.add_command(run)
|
|
19
|
+
cli.add_command(publish)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
|
|
6
|
+
from synapse_sdk.clients.backend import BackendClient
|
|
7
|
+
from synapse_sdk.plugins.upload import archive
|
|
8
|
+
from synapse_sdk.plugins.utils import read_config
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@click.command()
|
|
12
|
+
@click.option('--host', required=True)
|
|
13
|
+
@click.option('--user_token', required=True)
|
|
14
|
+
@click.option('--tenant', required=True)
|
|
15
|
+
@click.option('--debug_modules', default='', envvar='SYNAPSE_DEBUG_MODULES')
|
|
16
|
+
@click.pass_context
|
|
17
|
+
def publish(ctx, host, user_token, tenant, debug_modules):
|
|
18
|
+
debug = ctx.obj['DEBUG']
|
|
19
|
+
|
|
20
|
+
config = read_config()
|
|
21
|
+
|
|
22
|
+
source_path = Path('./')
|
|
23
|
+
archive_path = source_path / 'dist' / 'archive.zip'
|
|
24
|
+
archive(source_path, archive_path)
|
|
25
|
+
|
|
26
|
+
data = {'plugin': config['code'], 'file': str(archive_path), 'debug': debug}
|
|
27
|
+
if debug:
|
|
28
|
+
data['debug_meta'] = json.dumps({'modules': debug_modules.split(',')})
|
|
29
|
+
|
|
30
|
+
client = BackendClient(host, user_token, tenant=tenant)
|
|
31
|
+
client.create_plugin_release(data)
|
|
32
|
+
click.secho(
|
|
33
|
+
f'Successfully published "{config["name"]}" ({config["code"]}@{config["version"]}) to synapse backend!',
|
|
34
|
+
fg='green',
|
|
35
|
+
bold=True,
|
|
36
|
+
)
|