synapse-sdk 1.0.0a0__py3-none-any.whl → 1.0.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse-sdk might be problematic. Click here for more details.

Files changed (73) hide show
  1. synapse_sdk/clients/agent/__init__.py +21 -0
  2. synapse_sdk/clients/agent/core.py +11 -0
  3. synapse_sdk/clients/agent/service.py +15 -0
  4. synapse_sdk/clients/backend/__init__.py +21 -0
  5. synapse_sdk/clients/backend/annotation.py +29 -0
  6. {synapse/client/mixins → synapse_sdk/clients/backend}/dataset.py +6 -4
  7. synapse_sdk/clients/backend/integration.py +36 -0
  8. synapse_sdk/clients/backend/ml.py +24 -0
  9. synapse/client/__init__.py → synapse_sdk/clients/base.py +22 -30
  10. {synapse → synapse_sdk}/loggers.py +1 -1
  11. synapse_sdk/plugins/__init__.py +13 -0
  12. {synapse → synapse_sdk}/plugins/categories/base.py +43 -45
  13. synapse_sdk/plugins/categories/data_validation/actions/validation.py +10 -0
  14. {synapse → synapse_sdk}/plugins/categories/decorators.py +2 -2
  15. synapse_sdk/plugins/categories/export/actions/export.py +10 -0
  16. synapse_sdk/plugins/categories/import/actions/import.py +10 -0
  17. {synapse → synapse_sdk}/plugins/categories/neural_net/actions/deployment.py +3 -3
  18. synapse_sdk/plugins/categories/neural_net/actions/inference.py +10 -0
  19. synapse_sdk/plugins/categories/neural_net/actions/test.py +10 -0
  20. {synapse → synapse_sdk}/plugins/categories/neural_net/actions/train.py +5 -5
  21. synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py +10 -0
  22. synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py +10 -0
  23. {synapse → synapse_sdk}/plugins/categories/registry.py +2 -2
  24. synapse_sdk/plugins/cli/__init__.py +19 -0
  25. synapse_sdk/plugins/cli/publish.py +36 -0
  26. synapse_sdk/plugins/cli/run.py +63 -0
  27. synapse_sdk/plugins/upload.py +82 -0
  28. {synapse → synapse_sdk}/plugins/utils.py +8 -4
  29. {synapse → synapse_sdk}/utils/file.py +13 -1
  30. synapse_sdk/utils/storage.py +91 -0
  31. {synapse → synapse_sdk}/utils/string.py +4 -0
  32. {synapse_sdk-1.0.0a0.dist-info → synapse_sdk-1.0.0a2.dist-info}/METADATA +2 -1
  33. synapse_sdk-1.0.0a2.dist-info/RECORD +58 -0
  34. {synapse_sdk-1.0.0a0.dist-info → synapse_sdk-1.0.0a2.dist-info}/WHEEL +1 -1
  35. synapse_sdk-1.0.0a2.dist-info/top_level.txt +1 -0
  36. synapse/client/mixins/annotation.py +0 -28
  37. synapse/client/mixins/integration.py +0 -12
  38. synapse/client/mixins/ml.py +0 -23
  39. synapse/config.py +0 -8
  40. synapse/plugins/__init__.py +0 -39
  41. synapse/plugins/categories/data_validation/actions/validation.py +0 -10
  42. synapse/plugins/categories/export/actions/export.py +0 -10
  43. synapse/plugins/categories/import/actions/import.py +0 -10
  44. synapse/plugins/categories/neural_net/actions/inference.py +0 -10
  45. synapse/plugins/categories/neural_net/actions/test.py +0 -10
  46. synapse/plugins/categories/post_annotation/actions/post_annotation.py +0 -10
  47. synapse/plugins/categories/pre_annotation/actions/pre_annotation.py +0 -10
  48. synapse/plugins/upload.py +0 -79
  49. synapse_sdk-1.0.0a0.dist-info/RECORD +0 -51
  50. synapse_sdk-1.0.0a0.dist-info/top_level.txt +0 -1
  51. {synapse → synapse_sdk}/__init__.py +0 -0
  52. {synapse/client/mixins → synapse_sdk/clients}/__init__.py +0 -0
  53. {synapse/client → synapse_sdk/clients}/exceptions.py +0 -0
  54. {synapse/client → synapse_sdk/clients}/utils.py +0 -0
  55. {synapse → synapse_sdk}/plugins/categories/__init__.py +0 -0
  56. {synapse → synapse_sdk}/plugins/categories/data_validation/__init__.py +0 -0
  57. {synapse → synapse_sdk}/plugins/categories/data_validation/actions/__init__.py +0 -0
  58. {synapse → synapse_sdk}/plugins/categories/export/__init__.py +0 -0
  59. {synapse → synapse_sdk}/plugins/categories/export/actions/__init__.py +0 -0
  60. {synapse → synapse_sdk}/plugins/categories/import/__init__.py +0 -0
  61. {synapse → synapse_sdk}/plugins/categories/import/actions/__init__.py +0 -0
  62. {synapse → synapse_sdk}/plugins/categories/neural_net/__init__.py +0 -0
  63. {synapse → synapse_sdk}/plugins/categories/neural_net/actions/__init__.py +0 -0
  64. {synapse → synapse_sdk}/plugins/categories/post_annotation/__init__.py +0 -0
  65. {synapse → synapse_sdk}/plugins/categories/post_annotation/actions/__init__.py +0 -0
  66. {synapse → synapse_sdk}/plugins/categories/pre_annotation/__init__.py +0 -0
  67. {synapse → synapse_sdk}/plugins/categories/pre_annotation/actions/__init__.py +0 -0
  68. {synapse → synapse_sdk}/plugins/enums.py +0 -0
  69. {synapse → synapse_sdk}/plugins/job.py +0 -0
  70. {synapse → synapse_sdk}/utils/__init__.py +0 -0
  71. {synapse → synapse_sdk}/utils/debug.py +0 -0
  72. {synapse → synapse_sdk}/utils/module_loading.py +0 -0
  73. {synapse_sdk-1.0.0a0.dist-info → synapse_sdk-1.0.0a2.dist-info}/LICENSE +0 -0
@@ -0,0 +1,21 @@
1
+ from synapse_sdk.clients.agent.core import CoreClientMixin
2
+ from synapse_sdk.clients.agent.service import ServiceClientMixin
3
+
4
+
5
+ class AgentClient(CoreClientMixin, ServiceClientMixin):
6
+ agent_token = None
7
+ user_token = None
8
+ tenant = None
9
+
10
+ def __init__(self, base_url, agent_token, user_token, tenant):
11
+ super().__init__(base_url)
12
+ self.agent_token = agent_token
13
+ self.user_token = user_token
14
+ self.tenant = tenant
15
+
16
+ def _get_headers(self):
17
+ return {
18
+ 'Authorization': self.agent_token,
19
+ 'SYNAPSE-User': f'Token {self.user_token}',
20
+ 'SYNAPSE-Tenant': f'Token {self.tenant}',
21
+ }
@@ -0,0 +1,11 @@
1
+ from synapse_sdk.clients.base import BaseClient
2
+
3
+
4
+ class CoreClientMixin(BaseClient):
5
+ def get_agent_system(self):
6
+ path = 'agents/system/'
7
+ return self._get(path)
8
+
9
+ def get_jobs_progress(self, data):
10
+ path = 'integration/jobs/progress/'
11
+ return self._post(path, data=data, timeout=1)
@@ -0,0 +1,15 @@
1
+ from synapse_sdk.clients.base import BaseClient
2
+
3
+
4
+ class ServiceClientMixin(BaseClient):
5
+ def run_plugin_release(self, code, data):
6
+ path = f'plugin_releases/{code}/run/'
7
+ return self._post(path, data=data)
8
+
9
+ def run_debug_plugin_release(self, data):
10
+ path = 'plugin_releases/run_debug/'
11
+ return self._post(path, data=data)
12
+
13
+ def create_plugin_release(self, data):
14
+ path = 'plugin_releases/'
15
+ return self._post(path, data=data)
@@ -0,0 +1,21 @@
1
+ from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
2
+ from synapse_sdk.clients.backend.dataset import DatasetClientMixin
3
+ from synapse_sdk.clients.backend.integration import IntegrationClientMixin
4
+ from synapse_sdk.clients.backend.ml import MLClientMixin
5
+
6
+
7
+ class BackendClient(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
8
+ token = None
9
+ tenant = None
10
+
11
+ def __init__(self, base_url, token, tenant=None):
12
+ super().__init__(base_url)
13
+ self.token = token
14
+ if tenant:
15
+ self.tenant = tenant
16
+
17
+ def _get_headers(self):
18
+ headers = {'Authorization': f'Token {self.token}'}
19
+ if self.tenant:
20
+ headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
21
+ return headers
@@ -0,0 +1,29 @@
1
+ from synapse_sdk.clients.base import BaseClient
2
+ from synapse_sdk.clients.utils import get_default_url_conversion
3
+
4
+
5
+ class AnnotationClientMixin(BaseClient):
6
+ def get_project(self, pk):
7
+ path = f'projects/{pk}/'
8
+ return self._get(path)
9
+
10
+ def get_label_tag(self, pk):
11
+ path = f'label_tags/{pk}/'
12
+ return self._get(path)
13
+
14
+ def list_label_tags(self, data):
15
+ path = 'label_tags/'
16
+ return self._list(path, data=data)
17
+
18
+ def list_labels(self, data, url_conversion=None, list_all=False):
19
+ path = 'labels/'
20
+ url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
21
+ return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
22
+
23
+ def create_labels(self, data):
24
+ path = 'labels/'
25
+ return self._post(path, data=data)
26
+
27
+ def set_tags_labels(self, data, params=None):
28
+ path = 'labels/set_tags/'
29
+ return self._post(path, data=data, params=params)
@@ -1,9 +1,11 @@
1
1
  from multiprocessing import Pool
2
2
  from tqdm import tqdm
3
- from ..utils import get_batched_list
4
3
 
4
+ from synapse_sdk.clients.base import BaseClient
5
+ from synapse_sdk.clients.utils import get_batched_list
5
6
 
6
- class DatasetClientMixin:
7
+
8
+ class DatasetClientMixin(BaseClient):
7
9
  def list_dataset(self):
8
10
  path = 'datasets/'
9
11
  return self._get(path)
@@ -14,10 +16,10 @@ class DatasetClientMixin:
14
16
 
15
17
  def create_data_units(self, data):
16
18
  path = 'data_units/'
17
- return self._post(path, payload=data)
19
+ return self._post(path, data=data)
18
20
 
19
21
  def import_dataset(self, dataset_id, dataset, project_id=None, batch_size=1000, process_pool=10):
20
- # TODO validate datset with schema
22
+ # TODO validate dataset with schema
21
23
 
22
24
  params = [(data, dataset_id) for data in dataset]
23
25
 
@@ -0,0 +1,36 @@
1
+ from synapse_sdk.clients.base import BaseClient
2
+
3
+
4
+ class IntegrationClientMixin(BaseClient):
5
+ def get_plugin(self, pk):
6
+ path = f'plugins/{pk}/'
7
+ return self._get(path)
8
+
9
+ def create_plugin(self, data):
10
+ path = 'plugins/'
11
+ return self._post(path, data=data)
12
+
13
+ def update_plugin(self, pk, data):
14
+ path = f'plugins/{pk}/'
15
+ return self._put(path, data=data)
16
+
17
+ def run_plugin(self, pk, data):
18
+ path = f'plugins/{pk}/run/'
19
+ return self._post(path, data=data)
20
+
21
+ def get_plugin_release(self, pk, params=None):
22
+ path = f'plugin_releases/{pk}/'
23
+ return self._get(path, params=params)
24
+
25
+ def create_plugin_release(self, data):
26
+ path = 'plugin_releases/'
27
+ files = {'file': data.pop('file')}
28
+ return self._post(path, data=data, files=files)
29
+
30
+ def create_logs(self, data):
31
+ path = 'logs/'
32
+ return self._post(path, data=data)
33
+
34
+ def create_task(self, data):
35
+ path = 'agent_tasks/'
36
+ return self._post(path, data=data)
@@ -0,0 +1,24 @@
1
+ from synapse_sdk.clients.base import BaseClient
2
+ from synapse_sdk.clients.utils import get_default_url_conversion
3
+
4
+
5
+ class MLClientMixin(BaseClient):
6
+ def get_model(self, pk, params=None, url_conversion=None):
7
+ path = f'models/{pk}/'
8
+ url_conversion = get_default_url_conversion(
9
+ url_conversion, files_fields=['files', 'parent.files'], is_list=False
10
+ )
11
+ return self._get(path, params=params, url_conversion=url_conversion)
12
+
13
+ def create_model(self, data):
14
+ path = 'models/'
15
+ return self._post(path, data=data)
16
+
17
+ def update_model(self, pk, data, files=None):
18
+ path = f'models/{pk}/'
19
+ return self._patch(path, data=data, files=files)
20
+
21
+ def list_train_dataset(self, params=None, url_conversion=None, list_all=False):
22
+ path = 'train_dataset/'
23
+ url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
24
+ return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
@@ -4,24 +4,16 @@ from pathlib import Path
4
4
 
5
5
  import requests
6
6
 
7
- from .exceptions import ClientError
8
- from .mixins.annotation import AnnotationClientMixin
9
- from .mixins.dataset import DatasetClientMixin
10
- from .mixins.integration import IntegrationClientMixin
11
- from .mixins.ml import MLClientMixin
12
- from ..utils.file import files_url_to_path_from_objs
7
+ from synapse_sdk.clients.exceptions import ClientError
8
+ from synapse_sdk.utils.file import files_url_to_path_from_objs
13
9
 
14
10
 
15
- class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
11
+ class BaseClient:
12
+ name = None
16
13
  base_url = None
17
- token = None
18
- workspace_code = None
19
14
 
20
- def __init__(self, base_url, token, workspace_code=None):
15
+ def __init__(self, base_url):
21
16
  self.base_url = base_url
22
- self.token = token
23
- if workspace_code:
24
- self.workspace_code = workspace_code
25
17
  requests_session = requests.Session()
26
18
  self.requests_session = requests_session
27
19
 
@@ -31,10 +23,7 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
31
23
  return path
32
24
 
33
25
  def _get_headers(self):
34
- headers = {'Authorization': f'Token {self.token}'}
35
- if self.workspace_code:
36
- headers['DATAMAKER-Workspace'] = f'Token {self.workspace_code}'
37
- return headers
26
+ return {}
38
27
 
39
28
  def _request(self, method, path, **kwargs):
40
29
  url = self._get_url(path)
@@ -56,12 +45,12 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
56
45
  response.status_code, response.json() if response.status_code == 400 else response.reason
57
46
  )
58
47
  except requests.ConnectionError:
59
- raise ClientError(408, 'Server is not responding')
48
+ raise ClientError(408, f'{self.name} is not responding')
60
49
 
61
50
  return response.json()
62
51
 
63
- def _get(self, path, payload=None, url_conversion=None):
64
- response = self._request('get', path, params=payload)
52
+ def _get(self, path, url_conversion=None, **kwargs):
53
+ response = self._request('get', path, **kwargs)
65
54
  if url_conversion:
66
55
  if url_conversion['is_list']:
67
56
  files_url_to_path_from_objs(response['results'], **url_conversion)
@@ -69,21 +58,24 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
69
58
  files_url_to_path_from_objs(response, **url_conversion)
70
59
  return response
71
60
 
72
- def _post(self, path, payload=None, files=None, params=None):
73
- return self._request('post', path, data=payload, files=files, params=params)
61
+ def _post(self, path, **kwargs):
62
+ return self._request('post', path, **kwargs)
74
63
 
75
- def _patch(self, path, payload=None, files=None, params=None):
76
- return self._request('patch', path, data=payload, files=files, params=params)
64
+ def _put(self, path, **kwargs):
65
+ return self._request('put', path, **kwargs)
77
66
 
78
- def _list(self, path, payload=None, url_conversion=None, list_all=False):
79
- response = self._get(path, payload, url_conversion)
67
+ def _patch(self, path, **kwargs):
68
+ return self._request('patch', path, **kwargs)
69
+
70
+ def _list(self, path, url_conversion=None, list_all=False, **kwargs):
71
+ response = self._get(path, url_conversion, **kwargs)
80
72
  if list_all:
81
- return self._list_all(path, payload, url_conversion), response['count']
73
+ return self._list_all(path, url_conversion, **kwargs), response['count']
82
74
  else:
83
75
  return response
84
76
 
85
- def _list_all(self, path, payload=None, url_conversion=None):
86
- response = self._get(path, payload, url_conversion)
77
+ def _list_all(self, path, url_conversion=None, **kwargs):
78
+ response = self._get(path, url_conversion, **kwargs)
87
79
  yield from response['results']
88
80
  if response['next']:
89
- yield from self._list_all(response['next'], payload, url_conversion)
81
+ yield from self._list_all(response['next'], url_conversion, **kwargs)
@@ -1,6 +1,6 @@
1
1
  import datetime
2
2
 
3
- from synapse.client import ClientError
3
+ from synapse_sdk.clients.exceptions import ClientError
4
4
 
5
5
 
6
6
  class BaseLogger:
@@ -0,0 +1,13 @@
1
+ import os
2
+
3
+ from dotenv import load_dotenv
4
+
5
+ from synapse_sdk.plugins.cli import cli
6
+
7
+
8
+ def init():
9
+ load_dotenv(os.path.join(os.getcwd(), '.env'))
10
+ cli(obj={}, auto_envvar_prefix='SYNAPSE_PLUGIN')
11
+
12
+
13
+ __all__ = ['init']
@@ -8,11 +8,12 @@ import ray
8
8
  import requests
9
9
  from ray.dashboard.modules.job.sdk import JobSubmissionClient
10
10
 
11
- from synapse.loggers import ConsoleLogger
12
- from synapse.plugins.enums import RunMethod
13
- from synapse.plugins.upload import upload_path, build_and_upload
14
- from synapse.plugins.utils import get_plugin_checksum
15
- from synapse.utils.module_loading import import_string
11
+ from synapse_sdk.loggers import ConsoleLogger
12
+ from synapse_sdk.plugins.enums import RunMethod
13
+ from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload, download_and_upload
14
+ from synapse_sdk.plugins.utils import get_plugin_checksum
15
+ from synapse_sdk.utils.module_loading import import_string
16
+ from synapse_sdk.utils.storage import get_storage
16
17
 
17
18
 
18
19
  class Action:
@@ -27,15 +28,10 @@ class Action:
27
28
  debug = False
28
29
 
29
30
  default_envs = [
31
+ 'RAY_DASHBOARD_URL',
30
32
  'RAY_SERVE_ADDRESS',
31
- 'SYNAPSE_PLUGIN_URL',
32
- 'SYNAPSE_PLUGIN_PATH',
33
- 'SYNAPSE_PLUGIN_BASE_URL',
34
- 'SYNAPSE_PLUGIN_UPLOAD_S3_ENDPOINT_URL',
35
- 'SYNAPSE_PLUGIN_UPLOAD_S3_BUCKET_NAME',
36
- 'SYNAPSE_PLUGIN_UPLOAD_S3_ACCESS_KEY',
37
- 'SYNAPSE_PLUGIN_UPLOAD_S3_SECRET_KEY',
38
- 'SYNAPSE_PLUGIN_UPLOAD_S3_BASE_URL',
33
+ 'SYNAPSE_PLUGIN_STORAGE',
34
+ 'SYNAPSE_DEBUG_PLUGIN_PATH',
39
35
  'SYNAPSE_DEBUG_MODULES',
40
36
  ]
41
37
 
@@ -64,45 +60,44 @@ class Action:
64
60
  def plugin_checksum(self):
65
61
  return get_plugin_checksum(self.plugin_id)
66
62
 
67
- @cached_property
68
- def plugin_url(self):
69
- if self.debug:
70
- plugin_url = self.envs.get('SYNAPSE_PLUGIN_URL')
71
- if not plugin_url:
72
- plugin_url = upload_path(self.envs.get('SYNAPSE_PLUGIN_PATH', '.'), **self.get_kwargs_upload_path())
73
- self.envs['SYNAPSE_PLUGIN_URL'] = plugin_url
74
- return plugin_url
75
- base_url = self.envs['SYNAPSE_PLUGIN_BASE_URL']
76
- return str(os.path.join(base_url, f'{self.plugin_checksum}.zip'))
77
-
78
63
  @cached_property
79
64
  def entrypoint(self):
80
65
  return import_string(self.config['entrypoint'])
81
66
 
82
- def get_kwargs_upload_path(self):
83
- # TODO upload 관련 env 단일화 후 제거
84
- return {
85
- 'endpoint_url': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_ENDPOINT_URL'],
86
- 'bucket_name': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_BUCKET_NAME'],
87
- 'access_key': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_ACCESS_KEY'],
88
- 'secret_key': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_SECRET_KEY'],
89
- 'base_url': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_BASE_URL'],
90
- }
67
+ @property
68
+ def plugin_storage_url(self):
69
+ return self.envs['SYNAPSE_PLUGIN_STORAGE']
91
70
 
92
- def get_default_envs(self):
93
- return {env: os.environ[env] for env in self.default_envs if env in os.environ}
71
+ @property
72
+ def plugin_url(self):
73
+ if self.debug:
74
+ plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH', '.')
75
+ if plugin_path.startswith('https://'): # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
76
+ plugin_url = plugin_path
77
+ elif plugin_path.startswith('http://'):
78
+ plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
79
+ else:
80
+ plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
81
+ self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
82
+ return plugin_url
83
+ storage = get_storage(self.plugin_storage_url)
84
+ return storage.get_url(f'{self.plugin_checksum}.zip')
94
85
 
95
- def get_debug_modules(self):
86
+ @property
87
+ def debug_modules(self):
96
88
  debug_modules = []
97
89
  for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
98
- if module_path.startswith('http'):
90
+ if module_path.startswith('https://'): # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
99
91
  module_url = module_path
100
92
  else:
101
- module_url = build_and_upload(module_path, **self.get_kwargs_upload_path())
93
+ module_url = build_and_upload(module_path, self.plugin_storage_url)
102
94
  debug_modules.append(module_url)
103
95
  self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
104
96
  return debug_modules
105
97
 
98
+ def get_default_envs(self):
99
+ return {env: os.environ[env] for env in self.default_envs if env in os.environ}
100
+
106
101
  def get_runtime_env(self):
107
102
  runtime_env = {
108
103
  'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
@@ -110,10 +105,13 @@ class Action:
110
105
  }
111
106
 
112
107
  if self.debug:
113
- runtime_env['pip'] += self.get_debug_modules()
108
+ runtime_env['pip'] += self.debug_modules
114
109
 
110
+ # 맨 마지막에 진행되어야 함
115
111
  runtime_env['env_vars'] = self.envs
116
- pprint(runtime_env)
112
+
113
+ if self.debug:
114
+ pprint(runtime_env)
117
115
  return runtime_env
118
116
 
119
117
  def run(self):
@@ -130,7 +128,7 @@ class Action:
130
128
  def run_by_task(self):
131
129
  @ray.remote(runtime_env=self.get_runtime_env())
132
130
  def run_task(category, action, *args, **kwargs):
133
- from synapse.plugins.utils import get_action_class
131
+ from synapse_sdk.plugins.utils import get_action_class
134
132
 
135
133
  action = get_action_class(category, action)(*args, **kwargs)
136
134
  return action.run_action()
@@ -152,14 +150,14 @@ class Action:
152
150
  return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
153
151
 
154
152
  def run_by_job(self):
155
- entrypoint_args = [self.name, f'"{json.dumps(self.params)}"']
153
+ entrypoint_args = ['run', self.name, f'"{json.dumps(self.params)}"', '--direct']
156
154
  if self.debug:
157
- entrypoint_args.append('--debug')
155
+ entrypoint_args.insert(0, '--debug')
158
156
 
159
- client = JobSubmissionClient()
157
+ client = JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
160
158
  return client.submit_job(
161
159
  submission_id=self.job_id,
162
- entrypoint=f'python main.py {" ".join(entrypoint_args)} --direct',
160
+ entrypoint=f'python main.py {" ".join(entrypoint_args)}',
163
161
  runtime_env=self.get_runtime_env(),
164
162
  )
165
163
 
@@ -0,0 +1,10 @@
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+
5
+
6
+ @register_action
7
+ class ValidationAction(Action):
8
+ name = 'validation'
9
+ category = PluginCategory.DATA_VALIDATION
10
+ method = RunMethod.TASK
@@ -1,5 +1,5 @@
1
- from synapse.plugins.categories.base import Action
2
- from synapse.plugins.categories.registry import _REGISTERED_ACTIONS
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS
3
3
 
4
4
 
5
5
  def register_action(action_class):
@@ -0,0 +1,10 @@
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+
5
+
6
+ @register_action
7
+ class ExportAction(Action):
8
+ name = 'export'
9
+ category = PluginCategory.EXPORT
10
+ method = RunMethod.JOB
@@ -0,0 +1,10 @@
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+
5
+
6
+ @register_action
7
+ class ImportAction(Action):
8
+ name = 'import'
9
+ category = PluginCategory.IMPORT
10
+ method = RunMethod.JOB
@@ -1,8 +1,8 @@
1
1
  from ray import serve
2
2
 
3
- from synapse.plugins.categories.base import Action
4
- from synapse.plugins.categories.decorators import register_action
5
- from synapse.plugins.enums import PluginCategory, RunMethod
3
+ from synapse_sdk.plugins.categories.base import Action
4
+ from synapse_sdk.plugins.categories.decorators import register_action
5
+ from synapse_sdk.plugins.enums import PluginCategory, RunMethod
6
6
 
7
7
 
8
8
  @register_action
@@ -0,0 +1,10 @@
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+
5
+
6
+ @register_action
7
+ class InferenceAction(Action):
8
+ name = 'inference'
9
+ category = PluginCategory.NEURAL_NET
10
+ method = RunMethod.RESTAPI
@@ -0,0 +1,10 @@
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+
5
+
6
+ @register_action
7
+ class TestAction(Action):
8
+ name = 'test'
9
+ category = PluginCategory.NEURAL_NET
10
+ method = RunMethod.TASK
@@ -1,8 +1,8 @@
1
- from synapse.plugins.categories.base import Action
2
- from synapse.plugins.categories.decorators import register_action
3
- from synapse.plugins.enums import RunMethod, PluginCategory
4
- from synapse.utils.file import get_dict_from_file, files_url_to_path_from_objs
5
- from synapse.utils.module_loading import import_string
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+ from synapse_sdk.utils.file import get_dict_from_file, files_url_to_path_from_objs
5
+ from synapse_sdk.utils.module_loading import import_string
6
6
 
7
7
 
8
8
  @register_action
@@ -0,0 +1,10 @@
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+
5
+
6
+ @register_action
7
+ class PostAnnotationAction(Action):
8
+ name = 'post_annotation'
9
+ category = PluginCategory.POST_ANNOTATION
10
+ method = RunMethod.TASK
@@ -0,0 +1,10 @@
1
+ from synapse_sdk.plugins.categories.base import Action
2
+ from synapse_sdk.plugins.categories.decorators import register_action
3
+ from synapse_sdk.plugins.enums import RunMethod, PluginCategory
4
+
5
+
6
+ @register_action
7
+ class PreAnnotationAction(Action):
8
+ name = 'pre_annotation'
9
+ category = PluginCategory.PRE_ANNOTATION
10
+ method = RunMethod.TASK
@@ -1,7 +1,7 @@
1
1
  import pkgutil
2
2
  from importlib import import_module
3
3
 
4
- from synapse.plugins.enums import PluginCategory
4
+ from synapse_sdk.plugins.enums import PluginCategory
5
5
 
6
6
  _REGISTERED_ACTIONS = {}
7
7
 
@@ -9,7 +9,7 @@ _REGISTERED_ACTIONS = {}
9
9
  def register_actions():
10
10
  if not _REGISTERED_ACTIONS:
11
11
  for category in PluginCategory:
12
- plugin_category_module_name = f'synapse.plugins.categories.{category.value}.actions'
12
+ plugin_category_module_name = f'synapse_sdk.plugins.categories.{category.value}.actions'
13
13
  plugin_category_module = import_module(plugin_category_module_name)
14
14
  for _, action_name, _ in pkgutil.iter_modules(plugin_category_module.__path__):
15
15
  action_module_name = f'{plugin_category_module_name}.{action_name}'
@@ -0,0 +1,19 @@
1
+ import click
2
+
3
+ from .publish import publish
4
+ from .run import run
5
+
6
+
7
+ @click.group()
8
+ @click.option('--debug/--no-debug', default=False)
9
+ @click.pass_context
10
+ def cli(ctx, debug):
11
+ ctx.ensure_object(dict)
12
+ ctx.obj['DEBUG'] = debug
13
+
14
+ if debug:
15
+ click.echo('Debug mode is "on"')
16
+
17
+
18
+ cli.add_command(run)
19
+ cli.add_command(publish)
@@ -0,0 +1,36 @@
1
+ import json
2
+ from pathlib import Path
3
+
4
+ import click
5
+
6
+ from synapse_sdk.clients.backend import BackendClient
7
+ from synapse_sdk.plugins.upload import archive
8
+ from synapse_sdk.plugins.utils import read_config
9
+
10
+
11
+ @click.command()
12
+ @click.option('--host', required=True)
13
+ @click.option('--user_token', required=True)
14
+ @click.option('--tenant', required=True)
15
+ @click.option('--debug_modules', default='', envvar='SYNAPSE_DEBUG_MODULES')
16
+ @click.pass_context
17
+ def publish(ctx, host, user_token, tenant, debug_modules):
18
+ debug = ctx.obj['DEBUG']
19
+
20
+ config = read_config()
21
+
22
+ source_path = Path('./')
23
+ archive_path = source_path / 'dist' / 'archive.zip'
24
+ archive(source_path, archive_path)
25
+
26
+ data = {'plugin': config['code'], 'file': str(archive_path), 'debug': debug}
27
+ if debug:
28
+ data['debug_meta'] = json.dumps({'modules': debug_modules.split(',')})
29
+
30
+ client = BackendClient(host, user_token, tenant=tenant)
31
+ client.create_plugin_release(data)
32
+ click.secho(
33
+ f'Successfully published "{config["name"]}" ({config["code"]}@{config["version"]}) to synapse backend!',
34
+ fg='green',
35
+ bold=True,
36
+ )