synapse-sdk 1.0.0a1__py3-none-any.whl → 1.0.0a3__py3-none-any.whl

synapse_sdk/clients/agent/__init__.py

@@ -0,0 +1,21 @@
+from synapse_sdk.clients.agent.service import ServiceClientMixin
+
+
+class AgentClient(ServiceClientMixin):
+    name = 'Agent'
+    agent_token = None
+    user_token = None
+    tenant = None
+
+    def __init__(self, base_url, agent_token, user_token, tenant):
+        super().__init__(base_url)
+        self.agent_token = agent_token
+        self.user_token = user_token
+        self.tenant = tenant
+
+    def _get_headers(self):
+        return {
+            'Authorization': self.agent_token,
+            'SYNAPSE-User': f'Token {self.user_token}',
+            'SYNAPSE-Tenant': f'Token {self.tenant}',
+        }

synapse_sdk/clients/agent/service.py

@@ -0,0 +1,54 @@
+from synapse_sdk.clients.base import BaseClient
+
+
+class ServiceClientMixin(BaseClient):
+    def run_plugin_release(self, code, data):
+        path = f'plugin_releases/{code}/run/'
+        return self._post(path, data=data)
+
+    def run_debug_plugin_release(self, data):
+        path = 'plugin_releases/run_debug/'
+        return self._post(path, data=data)
+
+    def create_plugin_release(self, data):
+        path = 'plugin_releases/'
+        return self._post(path, data=data)
+
+    def get_job(self, pk):
+        path = f'jobs/{pk}/'
+        return self._get(path)
+
+    def list_jobs(self):
+        path = 'jobs/'
+        return self._get(path)
+
+    def list_job_logs(self, pk):
+        path = f'jobs/{pk}/logs/'
+        return self._get(path)
+
+    def tail_job_logs(self, pk):
+        path = f'jobs/{pk}/tail_logs/'
+
+        url = self._get_url(path)
+        headers = self._get_headers()
+
+        response = self.requests_session.get(url, headers=headers, stream=True)
+        for line in response.iter_lines():
+            if line:
+                yield line.decode('utf-8')
+
+    def get_node(self, pk):
+        path = f'nodes/{pk}/'
+        return self._get(path)
+
+    def list_nodes(self):
+        path = 'nodes/'
+        return self._get(path)
+
+    def get_task(self, pk):
+        path = f'tasks/{pk}/'
+        return self._get(path)
+
+    def list_tasks(self):
+        path = 'tasks/'
+        return self._get(path)

synapse_sdk/clients/backend/__init__.py

@@ -0,0 +1,22 @@
+from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
+from synapse_sdk.clients.backend.dataset import DatasetClientMixin
+from synapse_sdk.clients.backend.integration import IntegrationClientMixin
+from synapse_sdk.clients.backend.ml import MLClientMixin
+
+
+class BackendClient(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
+    name = 'Backend'
+    token = None
+    tenant = None
+
+    def __init__(self, base_url, token, tenant=None):
+        super().__init__(base_url)
+        self.token = token
+        if tenant:
+            self.tenant = tenant
+
+    def _get_headers(self):
+        headers = {'Authorization': f'Token {self.token}'}
+        if self.tenant:
+            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
+        return headers

synapse_sdk/clients/backend/annotation.py

@@ -0,0 +1,29 @@
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_default_url_conversion
+
+
+class AnnotationClientMixin(BaseClient):
+    def get_project(self, pk):
+        path = f'projects/{pk}/'
+        return self._get(path)
+
+    def get_label_tag(self, pk):
+        path = f'label_tags/{pk}/'
+        return self._get(path)
+
+    def list_label_tags(self, data):
+        path = 'label_tags/'
+        return self._list(path, data=data)
+
+    def list_labels(self, data, url_conversion=None, list_all=False):
+        path = 'labels/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
+
+    def create_labels(self, data):
+        path = 'labels/'
+        return self._post(path, data=data)
+
+    def set_tags_labels(self, data, params=None):
+        path = 'labels/set_tags/'
+        return self._post(path, data=data, params=params)

synapse_sdk/{client/mixins → clients/backend}/dataset.py

@@ -1,9 +1,11 @@
 from multiprocessing import Pool
 from tqdm import tqdm
-from ..utils import get_batched_list
 
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_batched_list
 
-class DatasetClientMixin:
+
+class DatasetClientMixin(BaseClient):
     def list_dataset(self):
         path = 'datasets/'
         return self._get(path)
@@ -14,10 +16,10 @@ class DatasetClientMixin:
 
     def create_data_units(self, data):
         path = 'data_units/'
-        return self._post(path, payload=data)
+        return self._post(path, data=data)
 
     def import_dataset(self, dataset_id, dataset, project_id=None, batch_size=1000, process_pool=10):
-        # TODO validate datset with schema
+        # TODO validate dataset with schema
 
         params = [(data, dataset_id) for data in dataset]
 

synapse_sdk/clients/backend/integration.py

@@ -0,0 +1,47 @@
+from synapse_sdk.clients.base import BaseClient
+
+
+class IntegrationClientMixin(BaseClient):
+    def get_plugin(self, pk):
+        path = f'plugins/{pk}/'
+        return self._get(path)
+
+    def create_plugin(self, data):
+        path = 'plugins/'
+        return self._post(path, data=data)
+
+    def update_plugin(self, pk, data):
+        path = f'plugins/{pk}/'
+        return self._put(path, data=data)
+
+    def run_plugin(self, pk, data):
+        path = f'plugins/{pk}/run/'
+        return self._post(path, data=data)
+
+    def get_plugin_release(self, pk, params=None):
+        path = f'plugin_releases/{pk}/'
+        return self._get(path, params=params)
+
+    def create_plugin_release(self, data):
+        path = 'plugin_releases/'
+        files = {'file': data.pop('file')}
+        return self._post(path, data=data, files=files)
+
+    def list_job_console_logs(self, pk):
+        path = f'jobs/{pk}/console_logs/'
+        return self._get(path)
+
+    def tail_job_console_logs(self, pk):
+        path = f'jobs/{pk}/tail_console_logs/'
+
+        url = self._get_url(path)
+        headers = self._get_headers()
+
+        response = self.requests_session.get(url, headers=headers, stream=True)
+        for line in response.iter_lines():
+            if line:
+                yield line.decode('utf-8')
+
+    def create_logs(self, data):
+        path = 'logs/'
+        return self._post(path, data=data)

synapse_sdk/clients/backend/ml.py

@@ -0,0 +1,24 @@
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_default_url_conversion
+
+
+class MLClientMixin(BaseClient):
+    def get_model(self, pk, params=None, url_conversion=None):
+        path = f'models/{pk}/'
+        url_conversion = get_default_url_conversion(
+            url_conversion, files_fields=['files', 'parent.files'], is_list=False
+        )
+        return self._get(path, params=params, url_conversion=url_conversion)
+
+    def create_model(self, data):
+        path = 'models/'
+        return self._post(path, data=data)
+
+    def update_model(self, pk, data, files=None):
+        path = f'models/{pk}/'
+        return self._patch(path, data=data, files=files)
+
+    def list_train_dataset(self, params=None, url_conversion=None, list_all=False):
+        path = 'train_dataset/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)

synapse_sdk/{client/__init__.py → clients/base.py}

@@ -4,24 +4,16 @@ from pathlib import Path
 
 import requests
 
-from .exceptions import ClientError
-from .mixins.annotation import AnnotationClientMixin
-from .mixins.dataset import DatasetClientMixin
-from .mixins.integration import IntegrationClientMixin
-from .mixins.ml import MLClientMixin
-from ..utils.file import files_url_to_path_from_objs
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.utils.file import files_url_to_path_from_objs
 
 
-class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
+class BaseClient:
+    name = None
     base_url = None
-    token = None
-    tenant = None
 
-    def __init__(self, base_url, token, tenant=None):
+    def __init__(self, base_url):
         self.base_url = base_url
-        self.token = token
-        if tenant:
-            self.tenant = tenant
         requests_session = requests.Session()
         self.requests_session = requests_session
 
@@ -31,10 +23,7 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
         return path
 
     def _get_headers(self):
-        headers = {'Authorization': f'Token {self.token}'}
-        if self.tenant:
-            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
-        return headers
+        return {}
 
     def _request(self, method, path, **kwargs):
         url = self._get_url(path)
@@ -56,12 +45,12 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
                     response.status_code, response.json() if response.status_code == 400 else response.reason
                 )
         except requests.ConnectionError:
-            raise ClientError(408, 'Server is not responding')
+            raise ClientError(408, f'{self.name} is not responding')
 
         return response.json()
 
-    def _get(self, path, payload=None, url_conversion=None):
-        response = self._request('get', path, params=payload)
+    def _get(self, path, url_conversion=None, **kwargs):
+        response = self._request('get', path, **kwargs)
         if url_conversion:
             if url_conversion['is_list']:
                 files_url_to_path_from_objs(response['results'], **url_conversion)
@@ -69,21 +58,24 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
                 files_url_to_path_from_objs(response, **url_conversion)
         return response
 
-    def _post(self, path, payload=None, files=None, params=None):
-        return self._request('post', path, data=payload, files=files, params=params)
+    def _post(self, path, **kwargs):
+        return self._request('post', path, **kwargs)
 
-    def _patch(self, path, payload=None, files=None, params=None):
-        return self._request('patch', path, data=payload, files=files, params=params)
+    def _put(self, path, **kwargs):
+        return self._request('put', path, **kwargs)
 
-    def _list(self, path, payload=None, url_conversion=None, list_all=False):
-        response = self._get(path, payload, url_conversion)
+    def _patch(self, path, **kwargs):
+        return self._request('patch', path, **kwargs)
+
+    def _list(self, path, url_conversion=None, list_all=False, **kwargs):
+        response = self._get(path, url_conversion, **kwargs)
         if list_all:
-            return self._list_all(path, payload, url_conversion), response['count']
+            return self._list_all(path, url_conversion, **kwargs), response['count']
         else:
             return response
 
-    def _list_all(self, path, payload=None, url_conversion=None):
-        response = self._get(path, payload, url_conversion)
+    def _list_all(self, path, url_conversion=None, **kwargs):
+        response = self._get(path, url_conversion, **kwargs)
         yield from response['results']
         if response['next']:
-            yield from self._list_all(response['next'], payload, url_conversion)
+            yield from self._list_all(response['next'], url_conversion, **kwargs)

synapse_sdk/{client → clients}/exceptions.py

@@ -6,11 +6,3 @@ class ClientError(Exception):
         self.status = status
         self.reason = reason
         super().__init__(status, reason, *args)
-
-    def as_validation_error(self):
-        if self.status == 400:
-            error = self.reason
-        else:
-            error = str(self)
-
-        return {'backend_errors': error}

synapse_sdk/loggers.py

@@ -1,16 +1,10 @@
 import datetime
 
-from synapse_sdk.client import ClientError
+from synapse_sdk.clients.exceptions import ClientError
 
 
 class BaseLogger:
     progress_records = {}
-    logs_queue = []
-    client = None
-
-    def __init__(self, client=None, task=None):
-        self.client = client
-        self.task = task
 
     def set_progress(self, current, total, category=''):
         percent = 0
@@ -19,30 +13,40 @@ class BaseLogger:
             percent = float(round(percent, 2))
 
         self.progress_records[category] = {'current': current, 'total': total, 'percent': percent}
-        if self.task:
-            self.task.update_state(state='PROGRESS', meta=self.progress_records)
-        else:
-            print(self.progress_records)
-
-    def log(self, action, data):
-        log = {'action': action, 'data': data, 'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}
-
-        if self.client and self.task:
-            log['task_id'] = self.task.request.id
-            self.logs_queue.append(log)
-            try:
-                self.client.create_logs(self.logs_queue)
-                self.logs_queue.clear()
-            except ClientError as e:
-                print(e)
-        else:
-            print(log)
 
 
 class ConsoleLogger(BaseLogger):
+    def set_progress(self, current, total, category=''):
+        super().set_progress(current, total, category=category)
+        print(self.progress_records)
+
     def log(self, action, data):
         print(action, data)
 
 
-class SynapseLogger(BaseLogger):
-    pass
+class BackendLogger(BaseLogger):
+    logs_queue = []
+    client = None
+    job_id = None
+
+    def __init__(self, client, job_id):
+        self.client = client
+        self.job_id = job_id
+
+    def set_progress(self, current, total, category=''):
+        super().set_progress(current, total, category=category)
+        # TODO set_progress to the job
+
+    def log(self, action, data):
+        log = {
+            'action': action,
+            'data': data,
+            'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
+            'job': self.job_id,
+        }
+        self.logs_queue.append(log)
+        try:
+            self.client.create_logs(self.logs_queue)
+            self.logs_queue.clear()
+        except ClientError as e:
+            print(e)

synapse_sdk/plugins/categories/base.py

@@ -8,11 +8,13 @@ import ray
 import requests
 from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
-from synapse_sdk.loggers import ConsoleLogger
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.loggers import ConsoleLogger, BackendLogger
 from synapse_sdk.plugins.enums import RunMethod
-from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload
+from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload, download_and_upload
 from synapse_sdk.plugins.utils import get_plugin_checksum
 from synapse_sdk.utils.module_loading import import_string
+from synapse_sdk.utils.storage import get_storage
 
 
 class Action:
@@ -29,11 +31,12 @@ class Action:
     default_envs = [
         'RAY_DASHBOARD_URL',
         'RAY_SERVE_ADDRESS',
-        'SYNAPSE_PLUGIN_URL',
-        'SYNAPSE_PLUGIN_PATH',
-        'SYNAPSE_PLUGIN_BASE_URL',
         'SYNAPSE_PLUGIN_STORAGE',
+        'SYNAPSE_DEBUG_PLUGIN_PATH',
         'SYNAPSE_DEBUG_MODULES',
+        'SYNAPSE_PLUGIN_RUN_HOST',
+        'SYNAPSE_PLUGIN_RUN_USER_TOKEN',
+        'SYNAPSE_PLUGIN_RUN_TENANT',
     ]
 
     def __init__(self, params, plugin_config, envs=None, job_id=None, direct=False, debug=False):
@@ -43,13 +46,8 @@ class Action:
         self.job_id = job_id
         self.direct = direct
         self.debug = debug
-        if envs:
-            self.envs = {**envs, **self.get_default_envs()}
-        else:
-            self.envs = self.get_default_envs()
-
-        # TODO logger 지정 방식 개선
-        self.logger = ConsoleLogger()
+        self.envs = {**envs, **self.get_default_envs()} if envs else self.get_default_envs()
+        self.set_logger()
 
     @cached_property
     def plugin_id(self):
@@ -61,38 +59,55 @@ class Action:
     def plugin_checksum(self):
         return get_plugin_checksum(self.plugin_id)
 
-    @cached_property
-    def plugin_url(self):
-        if self.debug:
-            plugin_url = self.envs.get('SYNAPSE_PLUGIN_URL')
-            if not plugin_url:
-                plugin_url = archive_and_upload(
-                    self.envs.get('SYNAPSE_PLUGIN_PATH', '.'),
-                    self.envs['SYNAPSE_PLUGIN_STORAGE'],
-                )
-                self.envs['SYNAPSE_PLUGIN_URL'] = plugin_url
-            return plugin_url
-        base_url = self.envs['SYNAPSE_PLUGIN_BASE_URL']
-        return str(os.path.join(base_url, f'{self.plugin_checksum}.zip'))
-
     @cached_property
     def entrypoint(self):
         return import_string(self.config['entrypoint'])
 
-    def get_default_envs(self):
-        return {env: os.environ[env] for env in self.default_envs if env in os.environ}
+    @property
+    def plugin_storage_url(self):
+        return self.envs['SYNAPSE_PLUGIN_STORAGE']
+
+    @property
+    def plugin_url(self):
+        if self.debug:
+            plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH', '.')
+            if plugin_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
+                plugin_url = plugin_path
+            elif plugin_path.startswith('http://'):
+                plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
+            else:
+                plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
+            self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
+            return plugin_url
+        storage = get_storage(self.plugin_storage_url)
+        return storage.get_url(f'{self.plugin_checksum}.zip')
 
-    def get_debug_modules(self):
+    @property
+    def debug_modules(self):
         debug_modules = []
         for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
-            if module_path.startswith('http'):
+            if module_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
                 module_url = module_path
             else:
-                module_url = build_and_upload(module_path, self.envs['SYNAPSE_PLUGIN_STORAGE'])
+                module_url = build_and_upload(module_path, self.plugin_storage_url)
             debug_modules.append(module_url)
         self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
+    def set_logger(self):
+        if self.job_id:
+            client = BackendClient(
+                self.envs['SYNAPSE_PLUGIN_RUN_HOST'],
+                self.envs['SYNAPSE_PLUGIN_RUN_USER_TOKEN'],
+                self.envs['SYNAPSE_PLUGIN_RUN_TENANT'],
+            )
+            self.logger = BackendLogger(client, self.job_id)
+        else:
+            self.logger = ConsoleLogger()
+
+    def get_default_envs(self):
+        return {env: os.environ[env] for env in self.default_envs if env in os.environ}
+
     def get_runtime_env(self):
         runtime_env = {
             'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
@@ -100,7 +115,7 @@ class Action:
         }
 
         if self.debug:
-            runtime_env['pip'] += self.get_debug_modules()
+            runtime_env['pip'] += self.debug_modules
 
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
@@ -145,14 +160,22 @@ class Action:
         return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
 
     def run_by_job(self):
-        entrypoint_args = ['run', self.name, f'"{json.dumps(self.params)}"', '--direct']
+        main_options = []
+        options = ['run', '--direct']
+        arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
+
         if self.debug:
-            entrypoint_args.insert(0, '--debug')
+            main_options.append('--debug')
+
+        if self.job_id:
+            options.append(f'--job-id={self.job_id}')
+
+        cmd = ' '.join(main_options + options + arguments)
 
         client = JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
         return client.submit_job(
             submission_id=self.job_id,
-            entrypoint=f'python main.py {" ".join(entrypoint_args)}',
+            entrypoint=f'python main.py {cmd}',
             runtime_env=self.get_runtime_env(),
         )
 

synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -1,8 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import RunMethod, PluginCategory
-from synapse_sdk.utils.file import get_dict_from_file, files_url_to_path_from_objs
-from synapse_sdk.utils.module_loading import import_string
 
 
 @register_action
@@ -11,73 +9,20 @@ class TrainAction(Action):
     category = PluginCategory.NEURAL_NET
     method = RunMethod.JOB
 
-    def get_input_dataset_for_training(self, model_id=None):
-        """
-        :return:
-        {
-            "train": [
-                {
-                    "files": {
-                        "image": {
-                            "path": "/path/to/image.jpg",
-                            "meta": {
-                                "width": 265,
-                                "height": 190,
-                                "created": 1651563526.0277045,
-                                "file_size": 5191,
-                                "last_modified": 1651563526.0277045
-                            }
-                        }
-                    },
-                    "ground_truth": {
-                        ...label_data
-                    }
-                },
-                ...
-            ],
-            "validation": ...,
-            "test": ...
-        }
-        """
+    def get_dataset(self):
+        return {}
 
-        client = self.logger.client
-        input_dataset = {}
-        category_int_to_str = {1: 'train', 2: 'validation', 3: 'test'}
-
-        if client:
-            train_dataset, count_dataset = client.list_train_dataset(
-                payload={'fields': ['category', 'files', 'ground_truth'], 'model': model_id}, list_all=True
-            )
-
-            for i, train_data in enumerate(train_dataset, start=1):
-                self.set_progress(i, count_dataset, category='dataset_download')
-                category = category_int_to_str[train_data.pop('category')]
-                try:
-                    input_dataset[category].append(train_data)
-                except KeyError:
-                    input_dataset[category] = [train_data]
-
-        else:
-            for category in category_int_to_str.values():
-                dataset_path = self.task['dataset'].get(category)
-                if dataset_path:
-                    input_dataset[category] = get_dict_from_file(dataset_path)
-                    files_url_to_path_from_objs(input_dataset[category], ['files'], is_list=True)
-
-        return input_dataset
-
-    def run_train(self):
-        hyperparameter = self.task['hyperparameter']
-        train = import_string(self.plugin['train']['entrypoint'])
+    def run(self):
+        hyperparameter = self.params['hyperparameter']
 
         # download dataset
         self.log_event('Preparing dataset for training.')
-        input_dataset = self.get_input_dataset_for_training()
+        input_dataset = self.get_dataset()
 
         # train dataset
         self.log_event('Starting model training.')
 
-        model_files = train(self, input_dataset, hyperparameter)
+        model_files = self.entrypoint(self, input_dataset, hyperparameter)
 
         # upload model_data
         self.log_event('Registering model data.')
@@ -85,10 +30,6 @@ class TrainAction(Action):
         self.end_log()
         return model_files
 
-    def start(self):
-        action = self.task['action']
-        getattr(self, f'run_{action}')()
-
     def log_metric(self, x, i, **kwargs):
         self.log(x, {x: i, **kwargs})
 

synapse_sdk/plugins/cli/publish.py

@@ -1,24 +1,36 @@
+import json
 from pathlib import Path
 
 import click
 
-from synapse_sdk.client import Client
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.plugins.upload import archive
 from synapse_sdk.plugins.utils import read_config
 
-from ..upload import archive
-
 
 @click.command()
-@click.option('-h', '--host', required=True)
-@click.option('-t', '--token', required=True)
-@click.option('-w', '--workspace', required=True)
-def publish(host, token, workspace):
-    client = Client(host, token, tenant=workspace)
+@click.option('--host', required=True)
+@click.option('--user_token', required=True)
+@click.option('--tenant', required=True)
+@click.option('--debug_modules', default='', envvar='SYNAPSE_DEBUG_MODULES')
+@click.pass_context
+def publish(ctx, host, user_token, tenant, debug_modules):
+    debug = ctx.obj['DEBUG']
+
     config = read_config()
 
     source_path = Path('./')
     archive_path = source_path / 'dist' / 'archive.zip'
     archive(source_path, archive_path)
 
-    data = {'plugin': config['code'], 'file': str(archive_path)}
+    data = {'plugin': config['code'], 'file': str(archive_path), 'debug': debug}
+    if debug:
+        data['debug_meta'] = json.dumps({'modules': debug_modules.split(',')})
+
+    client = BackendClient(host, user_token, tenant=tenant)
     client.create_plugin_release(data)
+    click.secho(
+        f'Successfully published "{config["name"]}" ({config["code"]}@{config["version"]}) to synapse backend!',
+        fg='green',
+        bold=True,
+    )

synapse_sdk/plugins/cli/run.py

@@ -1,18 +1,64 @@
+import os
+
 import click
 
-from synapse_sdk.plugins.utils import get_action
+from synapse_sdk.clients.agent import AgentClient
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.plugins.utils import get_action, read_config
 
 
 @click.command()
 @click.argument('action')
 @click.argument('params')
+@click.option('--job-id')
 @click.option('--direct/--no-direct', default=False)
+@click.option('--run-by', type=click.Choice(['script', 'agent', 'backend']), default='script')
+@click.option('--agent-host')
+@click.option('--agent-token')
+@click.option('--host')
+@click.option('--agent')
+@click.option('--user-token')
+@click.option('--tenant')
 @click.pass_context
-def run(ctx, action, params, direct):
+def run(ctx, action, params, job_id, direct, run_by, agent_host, agent_token, host, agent, user_token, tenant):
     debug = ctx.obj['DEBUG']
 
-    action = get_action(action, params, direct=direct, debug=debug)
+    if run_by == 'script':
+        run_by_script(action, params, job_id, direct, debug)
+    elif run_by == 'agent':
+        run_by_agent(action, params, job_id, agent_host, agent_token, user_token, tenant, debug)
+    elif run_by == 'backend':
+        run_by_backend(action, params, agent, host, user_token, tenant)
+
+
+def run_by_script(action, params, job_id, direct, debug):
+    action = get_action(action, params, job_id=job_id, direct=direct, debug=debug)
     result = action.run_action()
 
     if debug:
         click.echo(result)
+
+
+def run_by_agent(action, params, job_id, agent_host, agent_token, user_token, tenant, debug):
+    client = AgentClient(agent_host, agent_token, user_token, tenant)
+    data = {'action': action, 'params': params, 'job_id': job_id}
+    if debug:
+        data.update({
+            'plugin_path': os.getcwd(),
+            'modules': os.getenv('SYNAPSE_DEBUG_MODULES', '').split(','),
+        })
+        result = client.run_debug_plugin_release(data=data)
+    else:
+        config = read_config()
+        result = client.run_plugin_release(code=f'{config["code"]}@{config["version"]}', data=data)
+
+    click.echo(result)
+
+
+def run_by_backend(action, params, agent, host, user_token, tenant):
+    client = BackendClient(host, user_token, tenant=tenant)
+    config = read_config()
+    data = {'agent': agent, 'version': config['version'], 'action': action, 'params': params}
+    result = client.run_plugin(config['code'], data=data)
+
+    click.echo(result)

synapse_sdk/plugins/upload.py

@@ -1,8 +1,9 @@
 import re
 import subprocess
+import tempfile
 from pathlib import Path
 
-from synapse_sdk.utils.file import calculate_checksum
+from synapse_sdk.utils.file import calculate_checksum, download_file
 from synapse_sdk.utils.storage import get_storage
 
 
@@ -12,6 +13,15 @@ def archive(source_path, archive_path):
     subprocess.run(command, cwd=source_path, shell=True, check=True, stdout=subprocess.DEVNULL)
 
 
+def download_and_upload(source_url, url):
+    storage = get_storage(url)
+    with tempfile.TemporaryDirectory() as temp_path:
+        file_path = str(download_file(source_url, temp_path))
+        checksum = calculate_checksum(file_path, prefix='dev')
+        # TODO 중복 체크
+        return storage.upload(file_path, f'{checksum}.zip')
+
+
 def archive_and_upload(source_path, url):
     storage = get_storage(url)
     dist_path = Path(source_path, 'dist')

synapse_sdk/utils/storage.py

@@ -78,8 +78,7 @@ class S3Storage(BaseStorage):
             return False
 
     def get_url(self, target):
-        object_name = os.path.join(self.options['location'], target)
-        return f'{self.options["base_url"]}/{self.options["bucket_name"]}/{object_name}'
+        return os.path.join(self.options['base_url'], self.options['bucket_name'], self.options['location'], target)
 
 
 STORAGE_STORAGES = {

{synapse_sdk-1.0.0a1.dist-info → synapse_sdk-1.0.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a1
+Version: 1.0.0a3
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a1.dist-info → synapse_sdk-1.0.0a3.dist-info}/RECORD

@@ -1,21 +1,23 @@
 synapse_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/config.py,sha256=AHyTHtzkkCZb9DLu1_R99VOwPgWATZj4K7yzHaPijfA,149
-synapse_sdk/loggers.py,sha256=W4KPq-QOK4ECBL0HxxGUGWCNR3FKbHOUMqnVrwz-Ihg,1331
-synapse_sdk/client/__init__.py,sha256=STq4STOQC5hhxKsAxhMYP6q3f4J1yJd1JUlz_db_130,3339
-synapse_sdk/client/exceptions.py,sha256=LzgzPKRPhNUpLh-jBQHKze8c3CQoVA22jSfjUMBZCuU,405
-synapse_sdk/client/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
-synapse_sdk/client/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/client/mixins/annotation.py,sha256=69xblc0RLokKlpSfIzoJ5dVRvw3VpPIVgbQKV3L5Xx4,905
-synapse_sdk/client/mixins/dataset.py,sha256=KXsen7-vdSzdR09uxD88lT604ItJKuCOSLR3fEU5ud0,1666
-synapse_sdk/client/mixins/integration.py,sha256=iCBjoz7GjzerhxGM0cPunauohs9Vq4_98EtTwxBMdYs,880
-synapse_sdk/client/mixins/ml.py,sha256=xkfO95k6p_nQVyai2zBO-z4b9nh6YIryN4iakDGiPsE,891
+synapse_sdk/loggers.py,sha256=Xl_oge_xYJ44cUoh9xBPKrNvDtK2W3Tzh8PKqrSROmQ,1439
+synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/clients/base.py,sha256=IuZY9-me62lPRgmMjXPhqy8cVLbtVSyT9acj19KMQkU,2788
+synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRTAjwk,220
+synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
+synapse_sdk/clients/agent/__init__.py,sha256=n4exZwc8gwuH6zqZUhstcD_sqIdm-ZoPvUcAxnCrnOQ,609
+synapse_sdk/clients/agent/service.py,sha256=esZcipWJkIN-SE279set9PjWrn0923_-zznxkD5hcXg,1423
+synapse_sdk/clients/backend/__init__.py,sha256=850EjxNoGWXJpVZLRIaajrymBuW0MXiP3M3onRHDGbQ,800
+synapse_sdk/clients/backend/annotation.py,sha256=Zt1VA3fScYCxy_Ss1TEzqu7jYdNxlNBRYAjZfuEWOSI,989
+synapse_sdk/clients/backend/dataset.py,sha256=abZr9FyVzgnEpk1POK6m1FfbzWKnqWTo7wyQwwS5VvM,1743
+synapse_sdk/clients/backend/integration.py,sha256=_SO9dyC1dOcYiMMl7e8CIMFm1k1hx6UPT9lzfvWnZqw,1400
+synapse_sdk/clients/backend/ml.py,sha256=l4rGLBZgLUYQOBePvWAoNyz-yZgJuhC-1KCFeZOYDuQ,1012
 synapse_sdk/plugins/__init__.py,sha256=9vsbYhxah4_ofTaG0x0qLFID_raHNkO57Y8A31Ws-lU,222
 synapse_sdk/plugins/enums.py,sha256=lQZqO2bEeBKdk6q-SMjfOLDlgxv7BuIPk3fXeUFfHRs,327
 synapse_sdk/plugins/job.py,sha256=UzFKA8o_F6RzY_PwyI4dlF3kSfmMG0xEYIyKLfdqSP8,91
-synapse_sdk/plugins/upload.py,sha256=lImaecAz5NG8puT5PwVNkV3kT0d6b7DNBD0jUMaVcBI,2600
+synapse_sdk/plugins/upload.py,sha256=9DF-f0Or6ea4mJxuCmPlpYOG4EGeGqIRALc4ulrnQd4,2973
 synapse_sdk/plugins/utils.py,sha256=RFxFtmjj-uBK03wUwLhtUecfn_IOKRJupudmsguc2Sc,1212
 synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/base.py,sha256=4pltsAR7RkCm9vaV-1TkVmJSi9w78KsBlVv9v7fFO6U,5823
+synapse_sdk/plugins/categories/base.py,sha256=PxA8xfy46rT_zvlI1WZjuRsxqa2Mx6TtKVXyolNqjfk,6831
 synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
 synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
 synapse_sdk/plugins/categories/data_validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -32,7 +34,7 @@ synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBS
 synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=afqgD_mzErcWdW3xdbrKIOL13fUPO0biZk8_l4l2j84,767
 synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=erM2z7aUTwyzJZqWBlxhTP8dm8cOraI_vUYAqcXkdSY,334
 synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=dAW1zfodlUhoL-sD17tG-CQT0RBxIcHWJ8f1eeZ00M4,321
-synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=sg-PDU_RnAtQGCwI6XnfC63lyPE2fRY6qEUv-_VKu8A,3283
+synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=ASbqY7ZB0-bVrEOH1uvOkhrYKNt7ZC1aIbPZPfKXZGc,994
 synapse_sdk/plugins/categories/post_annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/post_annotation/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py,sha256=1tutwNDHpnrCPHzMTsMEk29WPajnZikjBE83j7Z-Xt0,347
@@ -40,16 +42,16 @@ synapse_sdk/plugins/categories/pre_annotation/__init__.py,sha256=47DEQpj8HBSa-_T
 synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py,sha256=YYQt9HsgXlBclE4Sn0c7p1zqCxWHkIHAwyA-tbqrmPQ,344
 synapse_sdk/plugins/cli/__init__.py,sha256=8ogaOhN-RbDNYHqziW8nLsNUxKkZwGkHBdKxTahcm3U,334
-synapse_sdk/plugins/cli/publish.py,sha256=ws4vuXPWiwKq6czIjg__eWE2-vBgquXnsYFdGyI42vs,663
-synapse_sdk/plugins/cli/run.py,sha256=_Lb-Jeh3NHUwui_y34-7lRXp7Ek_rP8BgBM5wqxLdeU,418
+synapse_sdk/plugins/cli/publish.py,sha256=v9aMMMyZgftSLW63uO9ZKeGBJEBeVCfOSmsSrfpKOP4,1135
+synapse_sdk/plugins/cli/run.py,sha256=2_iwDCGbPNMCsEWzbeZIo4EGW-cssp3UVZNfkq6ZbHo,2249
 synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/debug.py,sha256=46sMFQLg_JSRUCymnT3wgszG1QsgrocRGiBjVX38r50,53
 synapse_sdk/utils/file.py,sha256=Iptk_DCPsmJzqAABCD3vC6z1yG74fKb5x81LnUCZzYo,1916
 synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
-synapse_sdk/utils/storage.py,sha256=6L3pgno3Luw6GuSRPdiIvS4ZFzjwGP6sLDKMy719pBw,2622
+synapse_sdk/utils/storage.py,sha256=U3TScqQNgHQ89s0kUqQ8hm3npQAznIyRqzWDKR0YA3E,2581
 synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
-synapse_sdk-1.0.0a1.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a1.dist-info/METADATA,sha256=zGbciSdYPvJ1Vl_sk5D0pmTNMqTuemSgkF9QbaVBXec,503
-synapse_sdk-1.0.0a1.dist-info/WHEEL,sha256=a7TGlA-5DaHMRrarXjVbQagU3Man_dCnGIWMJr5kRWo,91
-synapse_sdk-1.0.0a1.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a1.dist-info/RECORD,,
+synapse_sdk-1.0.0a3.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a3.dist-info/METADATA,sha256=hfVF3hav09XmhhOM7v7GibfROb0hdiDFz3E_sm55EO0,503
+synapse_sdk-1.0.0a3.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+synapse_sdk-1.0.0a3.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a3.dist-info/RECORD,,

{synapse_sdk-1.0.0a1.dist-info → synapse_sdk-1.0.0a3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.4.0)
+Generator: setuptools (75.5.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

synapse_sdk/client/mixins/annotation.py

@@ -1,28 +0,0 @@
-from ..utils import get_default_url_conversion
-
-
-class AnnotationClientMixin:
-    def get_project(self, pk):
-        path = f'projects/{pk}/'
-        return self._get(path)
-
-    def get_label_tag(self, pk):
-        path = f'label_tags/{pk}/'
-        return self._get(path)
-
-    def list_label_tags(self, payload=None):
-        path = 'label_tags/'
-        return self._list(path, payload)
-
-    def list_labels(self, payload=None, url_conversion=None, list_all=False):
-        path = 'labels/'
-        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
-        return self._list(path, payload, url_conversion, list_all)
-
-    def create_labels(self, data):
-        path = 'labels/'
-        return self._post(path, payload=data)
-
-    def set_tags_labels(self, data, params=None):
-        path = 'labels/set_tags/'
-        return self._post(path, payload=data, params=params)

synapse_sdk/client/mixins/integration.py

@@ -1,29 +0,0 @@
-class IntegrationClientMixin:
-    def get_plugin(self, pk):
-        path = f'plugins/{pk}/'
-        return self._get(path)
-
-    def create_plugin(self, data):
-        path = 'plugins/'
-        return self._post(path, payload=data)
-
-    def update_plugin(self, pk, data):
-        path = f'plugins/{pk}/'
-        return self._put(path, payload=data)
-
-    def get_plugin_release(self, pk, params=None):
-        path = f'plugin_releases/{pk}/'
-        return self._get(path, payload=params)
-
-    def create_plugin_release(self, data):
-        path = 'plugin_releases/'
-        files = {'file': data.pop('file')}
-        return self._post(path, payload=data, files=files)
-
-    def create_logs(self, data):
-        path = 'logs/'
-        return self._post(path, payload=data)
-
-    def create_task(self, data):
-        path = 'agent_tasks/'
-        return self._post(path, payload=data)

synapse_sdk/client/mixins/ml.py

@@ -1,23 +0,0 @@
-from ..utils import get_default_url_conversion
-
-
-class MLClientMixin:
-    def get_model(self, pk, payload=None, url_conversion=None):
-        path = f'models/{pk}/'
-        url_conversion = get_default_url_conversion(
-            url_conversion, files_fields=['files', 'parent.files'], is_list=False
-        )
-        return self._get(path, payload, url_conversion)
-
-    def create_model(self, data):
-        path = 'models/'
-        return self._post(path, payload=data)
-
-    def update_model(self, pk, data, files=None):
-        path = f'models/{pk}/'
-        return self._patch(path, payload=data, files=files)
-
-    def list_train_dataset(self, payload=None, url_conversion=None, list_all=False):
-        path = 'train_dataset/'
-        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
-        return self._list(path, payload, url_conversion, list_all)

synapse_sdk/config.py

@@ -1,8 +0,0 @@
-try:
-    from constance import config as constance_config
-
-    config = constance_config
-except ImportError:
-    config = None
-
-__all__ = ['config']