synapse-sdk 1.0.0a1__tar.gz → 1.0.0a3__tar.gz

{synapse_sdk-1.0.0a1 → synapse_sdk-1.0.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a1
+Version: 1.0.0a3
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

synapse_sdk-1.0.0a3/synapse_sdk/clients/agent/__init__.py

@@ -0,0 +1,21 @@
+from synapse_sdk.clients.agent.service import ServiceClientMixin
+
+
+class AgentClient(ServiceClientMixin):
+    name = 'Agent'
+    agent_token = None
+    user_token = None
+    tenant = None
+
+    def __init__(self, base_url, agent_token, user_token, tenant):
+        super().__init__(base_url)
+        self.agent_token = agent_token
+        self.user_token = user_token
+        self.tenant = tenant
+
+    def _get_headers(self):
+        return {
+            'Authorization': self.agent_token,
+            'SYNAPSE-User': f'Token {self.user_token}',
+            'SYNAPSE-Tenant': f'Token {self.tenant}',
+        }

synapse_sdk-1.0.0a3/synapse_sdk/clients/agent/service.py

@@ -0,0 +1,54 @@
+from synapse_sdk.clients.base import BaseClient
+
+
+class ServiceClientMixin(BaseClient):
+    def run_plugin_release(self, code, data):
+        path = f'plugin_releases/{code}/run/'
+        return self._post(path, data=data)
+
+    def run_debug_plugin_release(self, data):
+        path = 'plugin_releases/run_debug/'
+        return self._post(path, data=data)
+
+    def create_plugin_release(self, data):
+        path = 'plugin_releases/'
+        return self._post(path, data=data)
+
+    def get_job(self, pk):
+        path = f'jobs/{pk}/'
+        return self._get(path)
+
+    def list_jobs(self):
+        path = 'jobs/'
+        return self._get(path)
+
+    def list_job_logs(self, pk):
+        path = f'jobs/{pk}/logs/'
+        return self._get(path)
+
+    def tail_job_logs(self, pk):
+        path = f'jobs/{pk}/tail_logs/'
+
+        url = self._get_url(path)
+        headers = self._get_headers()
+
+        response = self.requests_session.get(url, headers=headers, stream=True)
+        for line in response.iter_lines():
+            if line:
+                yield line.decode('utf-8')
+
+    def get_node(self, pk):
+        path = f'nodes/{pk}/'
+        return self._get(path)
+
+    def list_nodes(self):
+        path = 'nodes/'
+        return self._get(path)
+
+    def get_task(self, pk):
+        path = f'tasks/{pk}/'
+        return self._get(path)
+
+    def list_tasks(self):
+        path = 'tasks/'
+        return self._get(path)

synapse_sdk-1.0.0a3/synapse_sdk/clients/backend/__init__.py

@@ -0,0 +1,22 @@
+from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
+from synapse_sdk.clients.backend.dataset import DatasetClientMixin
+from synapse_sdk.clients.backend.integration import IntegrationClientMixin
+from synapse_sdk.clients.backend.ml import MLClientMixin
+
+
+class BackendClient(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
+    name = 'Backend'
+    token = None
+    tenant = None
+
+    def __init__(self, base_url, token, tenant=None):
+        super().__init__(base_url)
+        self.token = token
+        if tenant:
+            self.tenant = tenant
+
+    def _get_headers(self):
+        headers = {'Authorization': f'Token {self.token}'}
+        if self.tenant:
+            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
+        return headers

synapse_sdk-1.0.0a3/synapse_sdk/clients/backend/annotation.py

@@ -0,0 +1,29 @@
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_default_url_conversion
+
+
+class AnnotationClientMixin(BaseClient):
+    def get_project(self, pk):
+        path = f'projects/{pk}/'
+        return self._get(path)
+
+    def get_label_tag(self, pk):
+        path = f'label_tags/{pk}/'
+        return self._get(path)
+
+    def list_label_tags(self, data):
+        path = 'label_tags/'
+        return self._list(path, data=data)
+
+    def list_labels(self, data, url_conversion=None, list_all=False):
+        path = 'labels/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
+
+    def create_labels(self, data):
+        path = 'labels/'
+        return self._post(path, data=data)
+
+    def set_tags_labels(self, data, params=None):
+        path = 'labels/set_tags/'
+        return self._post(path, data=data, params=params)

{synapse_sdk-1.0.0a1/synapse_sdk/client/mixins → synapse_sdk-1.0.0a3/synapse_sdk/clients/backend}/dataset.py

@@ -1,9 +1,11 @@
 from multiprocessing import Pool
 from tqdm import tqdm
-from ..utils import get_batched_list
 
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_batched_list
 
-class DatasetClientMixin:
+
+class DatasetClientMixin(BaseClient):
     def list_dataset(self):
         path = 'datasets/'
         return self._get(path)
@@ -14,10 +16,10 @@ class DatasetClientMixin:
 
     def create_data_units(self, data):
         path = 'data_units/'
-        return self._post(path, payload=data)
+        return self._post(path, data=data)
 
     def import_dataset(self, dataset_id, dataset, project_id=None, batch_size=1000, process_pool=10):
-        # TODO validate datset with schema
+        # TODO validate dataset with schema
 
         params = [(data, dataset_id) for data in dataset]
 

synapse_sdk-1.0.0a3/synapse_sdk/clients/backend/integration.py

@@ -0,0 +1,47 @@
+from synapse_sdk.clients.base import BaseClient
+
+
+class IntegrationClientMixin(BaseClient):
+    def get_plugin(self, pk):
+        path = f'plugins/{pk}/'
+        return self._get(path)
+
+    def create_plugin(self, data):
+        path = 'plugins/'
+        return self._post(path, data=data)
+
+    def update_plugin(self, pk, data):
+        path = f'plugins/{pk}/'
+        return self._put(path, data=data)
+
+    def run_plugin(self, pk, data):
+        path = f'plugins/{pk}/run/'
+        return self._post(path, data=data)
+
+    def get_plugin_release(self, pk, params=None):
+        path = f'plugin_releases/{pk}/'
+        return self._get(path, params=params)
+
+    def create_plugin_release(self, data):
+        path = 'plugin_releases/'
+        files = {'file': data.pop('file')}
+        return self._post(path, data=data, files=files)
+
+    def list_job_console_logs(self, pk):
+        path = f'jobs/{pk}/console_logs/'
+        return self._get(path)
+
+    def tail_job_console_logs(self, pk):
+        path = f'jobs/{pk}/tail_console_logs/'
+
+        url = self._get_url(path)
+        headers = self._get_headers()
+
+        response = self.requests_session.get(url, headers=headers, stream=True)
+        for line in response.iter_lines():
+            if line:
+                yield line.decode('utf-8')
+
+    def create_logs(self, data):
+        path = 'logs/'
+        return self._post(path, data=data)

synapse_sdk-1.0.0a3/synapse_sdk/clients/backend/ml.py

@@ -0,0 +1,24 @@
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_default_url_conversion
+
+
+class MLClientMixin(BaseClient):
+    def get_model(self, pk, params=None, url_conversion=None):
+        path = f'models/{pk}/'
+        url_conversion = get_default_url_conversion(
+            url_conversion, files_fields=['files', 'parent.files'], is_list=False
+        )
+        return self._get(path, params=params, url_conversion=url_conversion)
+
+    def create_model(self, data):
+        path = 'models/'
+        return self._post(path, data=data)
+
+    def update_model(self, pk, data, files=None):
+        path = f'models/{pk}/'
+        return self._patch(path, data=data, files=files)
+
+    def list_train_dataset(self, params=None, url_conversion=None, list_all=False):
+        path = 'train_dataset/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)

synapse_sdk-1.0.0a1/synapse_sdk/client/__init__.py → synapse_sdk-1.0.0a3/synapse_sdk/clients/base.py

@@ -4,24 +4,16 @@ from pathlib import Path
 
 import requests
 
-from .exceptions import ClientError
-from .mixins.annotation import AnnotationClientMixin
-from .mixins.dataset import DatasetClientMixin
-from .mixins.integration import IntegrationClientMixin
-from .mixins.ml import MLClientMixin
-from ..utils.file import files_url_to_path_from_objs
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.utils.file import files_url_to_path_from_objs
 
 
-class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
+class BaseClient:
+    name = None
     base_url = None
-    token = None
-    tenant = None
 
-    def __init__(self, base_url, token, tenant=None):
+    def __init__(self, base_url):
         self.base_url = base_url
-        self.token = token
-        if tenant:
-            self.tenant = tenant
         requests_session = requests.Session()
         self.requests_session = requests_session
 
@@ -31,10 +23,7 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
         return path
 
     def _get_headers(self):
-        headers = {'Authorization': f'Token {self.token}'}
-        if self.tenant:
-            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
-        return headers
+        return {}
 
     def _request(self, method, path, **kwargs):
         url = self._get_url(path)
@@ -56,12 +45,12 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
                     response.status_code, response.json() if response.status_code == 400 else response.reason
                 )
         except requests.ConnectionError:
-            raise ClientError(408, 'Server is not responding')
+            raise ClientError(408, f'{self.name} is not responding')
 
         return response.json()
 
-    def _get(self, path, payload=None, url_conversion=None):
-        response = self._request('get', path, params=payload)
+    def _get(self, path, url_conversion=None, **kwargs):
+        response = self._request('get', path, **kwargs)
         if url_conversion:
             if url_conversion['is_list']:
                 files_url_to_path_from_objs(response['results'], **url_conversion)
@@ -69,21 +58,24 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
                 files_url_to_path_from_objs(response, **url_conversion)
         return response
 
-    def _post(self, path, payload=None, files=None, params=None):
-        return self._request('post', path, data=payload, files=files, params=params)
+    def _post(self, path, **kwargs):
+        return self._request('post', path, **kwargs)
 
-    def _patch(self, path, payload=None, files=None, params=None):
-        return self._request('patch', path, data=payload, files=files, params=params)
+    def _put(self, path, **kwargs):
+        return self._request('put', path, **kwargs)
 
-    def _list(self, path, payload=None, url_conversion=None, list_all=False):
-        response = self._get(path, payload, url_conversion)
+    def _patch(self, path, **kwargs):
+        return self._request('patch', path, **kwargs)
+
+    def _list(self, path, url_conversion=None, list_all=False, **kwargs):
+        response = self._get(path, url_conversion, **kwargs)
         if list_all:
-            return self._list_all(path, payload, url_conversion), response['count']
+            return self._list_all(path, url_conversion, **kwargs), response['count']
         else:
             return response
 
-    def _list_all(self, path, payload=None, url_conversion=None):
-        response = self._get(path, payload, url_conversion)
+    def _list_all(self, path, url_conversion=None, **kwargs):
+        response = self._get(path, url_conversion, **kwargs)
         yield from response['results']
         if response['next']:
-            yield from self._list_all(response['next'], payload, url_conversion)
+            yield from self._list_all(response['next'], url_conversion, **kwargs)

{synapse_sdk-1.0.0a1/synapse_sdk/client → synapse_sdk-1.0.0a3/synapse_sdk/clients}/exceptions.py

@@ -6,11 +6,3 @@ class ClientError(Exception):
         self.status = status
         self.reason = reason
         super().__init__(status, reason, *args)
-
-    def as_validation_error(self):
-        if self.status == 400:
-            error = self.reason
-        else:
-            error = str(self)
-
-        return {'backend_errors': error}

synapse_sdk-1.0.0a3/synapse_sdk/loggers.py

@@ -0,0 +1,52 @@
+import datetime
+
+from synapse_sdk.clients.exceptions import ClientError
+
+
+class BaseLogger:
+    progress_records = {}
+
+    def set_progress(self, current, total, category=''):
+        percent = 0
+        if total > 0:
+            percent = (current / total) * 100
+            percent = float(round(percent, 2))
+
+        self.progress_records[category] = {'current': current, 'total': total, 'percent': percent}
+
+
+class ConsoleLogger(BaseLogger):
+    def set_progress(self, current, total, category=''):
+        super().set_progress(current, total, category=category)
+        print(self.progress_records)
+
+    def log(self, action, data):
+        print(action, data)
+
+
+class BackendLogger(BaseLogger):
+    logs_queue = []
+    client = None
+    job_id = None
+
+    def __init__(self, client, job_id):
+        self.client = client
+        self.job_id = job_id
+
+    def set_progress(self, current, total, category=''):
+        super().set_progress(current, total, category=category)
+        # TODO set_progress to the job
+
+    def log(self, action, data):
+        log = {
+            'action': action,
+            'data': data,
+            'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
+            'job': self.job_id,
+        }
+        self.logs_queue.append(log)
+        try:
+            self.client.create_logs(self.logs_queue)
+            self.logs_queue.clear()
+        except ClientError as e:
+            print(e)

{synapse_sdk-1.0.0a1 → synapse_sdk-1.0.0a3}/synapse_sdk/plugins/categories/base.py

@@ -8,11 +8,13 @@ import ray
 import requests
 from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
-from synapse_sdk.loggers import ConsoleLogger
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.loggers import ConsoleLogger, BackendLogger
 from synapse_sdk.plugins.enums import RunMethod
-from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload
+from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload, download_and_upload
 from synapse_sdk.plugins.utils import get_plugin_checksum
 from synapse_sdk.utils.module_loading import import_string
+from synapse_sdk.utils.storage import get_storage
 
 
 class Action:
@@ -29,11 +31,12 @@ class Action:
     default_envs = [
         'RAY_DASHBOARD_URL',
         'RAY_SERVE_ADDRESS',
-        'SYNAPSE_PLUGIN_URL',
-        'SYNAPSE_PLUGIN_PATH',
-        'SYNAPSE_PLUGIN_BASE_URL',
         'SYNAPSE_PLUGIN_STORAGE',
+        'SYNAPSE_DEBUG_PLUGIN_PATH',
         'SYNAPSE_DEBUG_MODULES',
+        'SYNAPSE_PLUGIN_RUN_HOST',
+        'SYNAPSE_PLUGIN_RUN_USER_TOKEN',
+        'SYNAPSE_PLUGIN_RUN_TENANT',
     ]
 
     def __init__(self, params, plugin_config, envs=None, job_id=None, direct=False, debug=False):
@@ -43,13 +46,8 @@ class Action:
         self.job_id = job_id
         self.direct = direct
         self.debug = debug
-        if envs:
-            self.envs = {**envs, **self.get_default_envs()}
-        else:
-            self.envs = self.get_default_envs()
-
-        # TODO logger 지정 방식 개선
-        self.logger = ConsoleLogger()
+        self.envs = {**envs, **self.get_default_envs()} if envs else self.get_default_envs()
+        self.set_logger()
 
     @cached_property
     def plugin_id(self):
@@ -61,38 +59,55 @@ class Action:
     def plugin_checksum(self):
         return get_plugin_checksum(self.plugin_id)
 
-    @cached_property
-    def plugin_url(self):
-        if self.debug:
-            plugin_url = self.envs.get('SYNAPSE_PLUGIN_URL')
-            if not plugin_url:
-                plugin_url = archive_and_upload(
-                    self.envs.get('SYNAPSE_PLUGIN_PATH', '.'),
-                    self.envs['SYNAPSE_PLUGIN_STORAGE'],
-                )
-                self.envs['SYNAPSE_PLUGIN_URL'] = plugin_url
-            return plugin_url
-        base_url = self.envs['SYNAPSE_PLUGIN_BASE_URL']
-        return str(os.path.join(base_url, f'{self.plugin_checksum}.zip'))
-
     @cached_property
     def entrypoint(self):
         return import_string(self.config['entrypoint'])
 
-    def get_default_envs(self):
-        return {env: os.environ[env] for env in self.default_envs if env in os.environ}
+    @property
+    def plugin_storage_url(self):
+        return self.envs['SYNAPSE_PLUGIN_STORAGE']
+
+    @property
+    def plugin_url(self):
+        if self.debug:
+            plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH', '.')
+            if plugin_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
+                plugin_url = plugin_path
+            elif plugin_path.startswith('http://'):
+                plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
+            else:
+                plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
+            self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
+            return plugin_url
+        storage = get_storage(self.plugin_storage_url)
+        return storage.get_url(f'{self.plugin_checksum}.zip')
 
-    def get_debug_modules(self):
+    @property
+    def debug_modules(self):
         debug_modules = []
         for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
-            if module_path.startswith('http'):
+            if module_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
                 module_url = module_path
             else:
-                module_url = build_and_upload(module_path, self.envs['SYNAPSE_PLUGIN_STORAGE'])
+                module_url = build_and_upload(module_path, self.plugin_storage_url)
             debug_modules.append(module_url)
         self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
+    def set_logger(self):
+        if self.job_id:
+            client = BackendClient(
+                self.envs['SYNAPSE_PLUGIN_RUN_HOST'],
+                self.envs['SYNAPSE_PLUGIN_RUN_USER_TOKEN'],
+                self.envs['SYNAPSE_PLUGIN_RUN_TENANT'],
+            )
+            self.logger = BackendLogger(client, self.job_id)
+        else:
+            self.logger = ConsoleLogger()
+
+    def get_default_envs(self):
+        return {env: os.environ[env] for env in self.default_envs if env in os.environ}
+
     def get_runtime_env(self):
         runtime_env = {
             'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
@@ -100,7 +115,7 @@ class Action:
         }
 
         if self.debug:
-            runtime_env['pip'] += self.get_debug_modules()
+            runtime_env['pip'] += self.debug_modules
 
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
@@ -145,14 +160,22 @@ class Action:
         return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
 
     def run_by_job(self):
-        entrypoint_args = ['run', self.name, f'"{json.dumps(self.params)}"', '--direct']
+        main_options = []
+        options = ['run', '--direct']
+        arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
+
         if self.debug:
-            entrypoint_args.insert(0, '--debug')
+            main_options.append('--debug')
+
+        if self.job_id:
+            options.append(f'--job-id={self.job_id}')
+
+        cmd = ' '.join(main_options + options + arguments)
 
         client = JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
         return client.submit_job(
             submission_id=self.job_id,
-            entrypoint=f'python main.py {" ".join(entrypoint_args)}',
+            entrypoint=f'python main.py {cmd}',
             runtime_env=self.get_runtime_env(),
         )
 

synapse_sdk-1.0.0a3/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -0,0 +1,37 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class TrainAction(Action):
+    name = 'train'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
+
+    def get_dataset(self):
+        return {}
+
+    def run(self):
+        hyperparameter = self.params['hyperparameter']
+
+        # download dataset
+        self.log_event('Preparing dataset for training.')
+        input_dataset = self.get_dataset()
+
+        # train dataset
+        self.log_event('Starting model training.')
+
+        model_files = self.entrypoint(self, input_dataset, hyperparameter)
+
+        # upload model_data
+        self.log_event('Registering model data.')
+
+        self.end_log()
+        return model_files
+
+    def log_metric(self, x, i, **kwargs):
+        self.log(x, {x: i, **kwargs})
+
+    def log_model(self, files, status=None):
+        pass

synapse_sdk-1.0.0a3/synapse_sdk/plugins/cli/publish.py

@@ -0,0 +1,36 @@
+import json
+from pathlib import Path
+
+import click
+
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.plugins.upload import archive
+from synapse_sdk.plugins.utils import read_config
+
+
+@click.command()
+@click.option('--host', required=True)
+@click.option('--user_token', required=True)
+@click.option('--tenant', required=True)
+@click.option('--debug_modules', default='', envvar='SYNAPSE_DEBUG_MODULES')
+@click.pass_context
+def publish(ctx, host, user_token, tenant, debug_modules):
+    debug = ctx.obj['DEBUG']
+
+    config = read_config()
+
+    source_path = Path('./')
+    archive_path = source_path / 'dist' / 'archive.zip'
+    archive(source_path, archive_path)
+
+    data = {'plugin': config['code'], 'file': str(archive_path), 'debug': debug}
+    if debug:
+        data['debug_meta'] = json.dumps({'modules': debug_modules.split(',')})
+
+    client = BackendClient(host, user_token, tenant=tenant)
+    client.create_plugin_release(data)
+    click.secho(
+        f'Successfully published "{config["name"]}" ({config["code"]}@{config["version"]}) to synapse backend!',
+        fg='green',
+        bold=True,
+    )

synapse_sdk-1.0.0a3/synapse_sdk/plugins/cli/run.py

@@ -0,0 +1,64 @@
+import os
+
+import click
+
+from synapse_sdk.clients.agent import AgentClient
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.plugins.utils import get_action, read_config
+
+
+@click.command()
+@click.argument('action')
+@click.argument('params')
+@click.option('--job-id')
+@click.option('--direct/--no-direct', default=False)
+@click.option('--run-by', type=click.Choice(['script', 'agent', 'backend']), default='script')
+@click.option('--agent-host')
+@click.option('--agent-token')
+@click.option('--host')
+@click.option('--agent')
+@click.option('--user-token')
+@click.option('--tenant')
+@click.pass_context
+def run(ctx, action, params, job_id, direct, run_by, agent_host, agent_token, host, agent, user_token, tenant):
+    debug = ctx.obj['DEBUG']
+
+    if run_by == 'script':
+        run_by_script(action, params, job_id, direct, debug)
+    elif run_by == 'agent':
+        run_by_agent(action, params, job_id, agent_host, agent_token, user_token, tenant, debug)
+    elif run_by == 'backend':
+        run_by_backend(action, params, agent, host, user_token, tenant)
+
+
+def run_by_script(action, params, job_id, direct, debug):
+    action = get_action(action, params, job_id=job_id, direct=direct, debug=debug)
+    result = action.run_action()
+
+    if debug:
+        click.echo(result)
+
+
+def run_by_agent(action, params, job_id, agent_host, agent_token, user_token, tenant, debug):
+    client = AgentClient(agent_host, agent_token, user_token, tenant)
+    data = {'action': action, 'params': params, 'job_id': job_id}
+    if debug:
+        data.update({
+            'plugin_path': os.getcwd(),
+            'modules': os.getenv('SYNAPSE_DEBUG_MODULES', '').split(','),
+        })
+        result = client.run_debug_plugin_release(data=data)
+    else:
+        config = read_config()
+        result = client.run_plugin_release(code=f'{config["code"]}@{config["version"]}', data=data)
+
+    click.echo(result)
+
+
+def run_by_backend(action, params, agent, host, user_token, tenant):
+    client = BackendClient(host, user_token, tenant=tenant)
+    config = read_config()
+    data = {'agent': agent, 'version': config['version'], 'action': action, 'params': params}
+    result = client.run_plugin(config['code'], data=data)
+
+    click.echo(result)

{synapse_sdk-1.0.0a1 → synapse_sdk-1.0.0a3}/synapse_sdk/plugins/upload.py

@@ -1,8 +1,9 @@
 import re
 import subprocess
+import tempfile
 from pathlib import Path
 
-from synapse_sdk.utils.file import calculate_checksum
+from synapse_sdk.utils.file import calculate_checksum, download_file
 from synapse_sdk.utils.storage import get_storage
 
 
@@ -12,6 +13,15 @@ def archive(source_path, archive_path):
     subprocess.run(command, cwd=source_path, shell=True, check=True, stdout=subprocess.DEVNULL)
 
 
+def download_and_upload(source_url, url):
+    storage = get_storage(url)
+    with tempfile.TemporaryDirectory() as temp_path:
+        file_path = str(download_file(source_url, temp_path))
+        checksum = calculate_checksum(file_path, prefix='dev')
+        # TODO 중복 체크
+        return storage.upload(file_path, f'{checksum}.zip')
+
+
 def archive_and_upload(source_path, url):
     storage = get_storage(url)
     dist_path = Path(source_path, 'dist')

{synapse_sdk-1.0.0a1 → synapse_sdk-1.0.0a3}/synapse_sdk/utils/storage.py

@@ -78,8 +78,7 @@ class S3Storage(BaseStorage):
             return False
 
     def get_url(self, target):
-        object_name = os.path.join(self.options['location'], target)
-        return f'{self.options["base_url"]}/{self.options["bucket_name"]}/{object_name}'
+        return os.path.join(self.options['base_url'], self.options['bucket_name'], self.options['location'], target)
 
 
 STORAGE_STORAGES = {

{synapse_sdk-1.0.0a1 → synapse_sdk-1.0.0a3}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a1
+Version: 1.0.0a3
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a1 → synapse_sdk-1.0.0a3}/synapse_sdk.egg-info/SOURCES.txt

@@ -7,21 +7,23 @@ requirements.txt
 .github/workflows/lint.yml
 .github/workflows/pypi-publish.yml
 synapse_sdk/__init__.py
-synapse_sdk/config.py
 synapse_sdk/loggers.py
 synapse_sdk.egg-info/PKG-INFO
 synapse_sdk.egg-info/SOURCES.txt
 synapse_sdk.egg-info/dependency_links.txt
 synapse_sdk.egg-info/requires.txt
 synapse_sdk.egg-info/top_level.txt
-synapse_sdk/client/__init__.py
-synapse_sdk/client/exceptions.py
-synapse_sdk/client/utils.py
-synapse_sdk/client/mixins/__init__.py
-synapse_sdk/client/mixins/annotation.py
-synapse_sdk/client/mixins/dataset.py
-synapse_sdk/client/mixins/integration.py
-synapse_sdk/client/mixins/ml.py
+synapse_sdk/clients/__init__.py
+synapse_sdk/clients/base.py
+synapse_sdk/clients/exceptions.py
+synapse_sdk/clients/utils.py
+synapse_sdk/clients/agent/__init__.py
+synapse_sdk/clients/agent/service.py
+synapse_sdk/clients/backend/__init__.py
+synapse_sdk/clients/backend/annotation.py
+synapse_sdk/clients/backend/dataset.py
+synapse_sdk/clients/backend/integration.py
+synapse_sdk/clients/backend/ml.py
 synapse_sdk/plugins/__init__.py
 synapse_sdk/plugins/enums.py
 synapse_sdk/plugins/job.py

synapse_sdk-1.0.0a1/synapse_sdk/client/mixins/annotation.py

@@ -1,28 +0,0 @@
-from ..utils import get_default_url_conversion
-
-
-class AnnotationClientMixin:
-    def get_project(self, pk):
-        path = f'projects/{pk}/'
-        return self._get(path)
-
-    def get_label_tag(self, pk):
-        path = f'label_tags/{pk}/'
-        return self._get(path)
-
-    def list_label_tags(self, payload=None):
-        path = 'label_tags/'
-        return self._list(path, payload)
-
-    def list_labels(self, payload=None, url_conversion=None, list_all=False):
-        path = 'labels/'
-        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
-        return self._list(path, payload, url_conversion, list_all)
-
-    def create_labels(self, data):
-        path = 'labels/'
-        return self._post(path, payload=data)
-
-    def set_tags_labels(self, data, params=None):
-        path = 'labels/set_tags/'
-        return self._post(path, payload=data, params=params)

synapse_sdk-1.0.0a1/synapse_sdk/client/mixins/integration.py

@@ -1,29 +0,0 @@
-class IntegrationClientMixin:
-    def get_plugin(self, pk):
-        path = f'plugins/{pk}/'
-        return self._get(path)
-
-    def create_plugin(self, data):
-        path = 'plugins/'
-        return self._post(path, payload=data)
-
-    def update_plugin(self, pk, data):
-        path = f'plugins/{pk}/'
-        return self._put(path, payload=data)
-
-    def get_plugin_release(self, pk, params=None):
-        path = f'plugin_releases/{pk}/'
-        return self._get(path, payload=params)
-
-    def create_plugin_release(self, data):
-        path = 'plugin_releases/'
-        files = {'file': data.pop('file')}
-        return self._post(path, payload=data, files=files)
-
-    def create_logs(self, data):
-        path = 'logs/'
-        return self._post(path, payload=data)
-
-    def create_task(self, data):
-        path = 'agent_tasks/'
-        return self._post(path, payload=data)

synapse_sdk-1.0.0a1/synapse_sdk/client/mixins/ml.py

@@ -1,23 +0,0 @@
-from ..utils import get_default_url_conversion
-
-
-class MLClientMixin:
-    def get_model(self, pk, payload=None, url_conversion=None):
-        path = f'models/{pk}/'
-        url_conversion = get_default_url_conversion(
-            url_conversion, files_fields=['files', 'parent.files'], is_list=False
-        )
-        return self._get(path, payload, url_conversion)
-
-    def create_model(self, data):
-        path = 'models/'
-        return self._post(path, payload=data)
-
-    def update_model(self, pk, data, files=None):
-        path = f'models/{pk}/'
-        return self._patch(path, payload=data, files=files)
-
-    def list_train_dataset(self, payload=None, url_conversion=None, list_all=False):
-        path = 'train_dataset/'
-        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
-        return self._list(path, payload, url_conversion, list_all)

synapse_sdk-1.0.0a1/synapse_sdk/config.py

@@ -1,8 +0,0 @@
-try:
-    from constance import config as constance_config
-
-    config = constance_config
-except ImportError:
-    config = None
-
-__all__ = ['config']

synapse_sdk-1.0.0a1/synapse_sdk/loggers.py

@@ -1,48 +0,0 @@
-import datetime
-
-from synapse_sdk.client import ClientError
-
-
-class BaseLogger:
-    progress_records = {}
-    logs_queue = []
-    client = None
-
-    def __init__(self, client=None, task=None):
-        self.client = client
-        self.task = task
-
-    def set_progress(self, current, total, category=''):
-        percent = 0
-        if total > 0:
-            percent = (current / total) * 100
-            percent = float(round(percent, 2))
-
-        self.progress_records[category] = {'current': current, 'total': total, 'percent': percent}
-        if self.task:
-            self.task.update_state(state='PROGRESS', meta=self.progress_records)
-        else:
-            print(self.progress_records)
-
-    def log(self, action, data):
-        log = {'action': action, 'data': data, 'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}
-
-        if self.client and self.task:
-            log['task_id'] = self.task.request.id
-            self.logs_queue.append(log)
-            try:
-                self.client.create_logs(self.logs_queue)
-                self.logs_queue.clear()
-            except ClientError as e:
-                print(e)
-        else:
-            print(log)
-
-
-class ConsoleLogger(BaseLogger):
-    def log(self, action, data):
-        print(action, data)
-
-
-class SynapseLogger(BaseLogger):
-    pass

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -1,96 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
-from synapse_sdk.utils.file import get_dict_from_file, files_url_to_path_from_objs
-from synapse_sdk.utils.module_loading import import_string
-
-
-@register_action
-class TrainAction(Action):
-    name = 'train'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.JOB
-
-    def get_input_dataset_for_training(self, model_id=None):
-        """
-        :return:
-        {
-            "train": [
-                {
-                    "files": {
-                        "image": {
-                            "path": "/path/to/image.jpg",
-                            "meta": {
-                                "width": 265,
-                                "height": 190,
-                                "created": 1651563526.0277045,
-                                "file_size": 5191,
-                                "last_modified": 1651563526.0277045
-                            }
-                        }
-                    },
-                    "ground_truth": {
-                        ...label_data
-                    }
-                },
-                ...
-            ],
-            "validation": ...,
-            "test": ...
-        }
-        """
-
-        client = self.logger.client
-        input_dataset = {}
-        category_int_to_str = {1: 'train', 2: 'validation', 3: 'test'}
-
-        if client:
-            train_dataset, count_dataset = client.list_train_dataset(
-                payload={'fields': ['category', 'files', 'ground_truth'], 'model': model_id}, list_all=True
-            )
-
-            for i, train_data in enumerate(train_dataset, start=1):
-                self.set_progress(i, count_dataset, category='dataset_download')
-                category = category_int_to_str[train_data.pop('category')]
-                try:
-                    input_dataset[category].append(train_data)
-                except KeyError:
-                    input_dataset[category] = [train_data]
-
-        else:
-            for category in category_int_to_str.values():
-                dataset_path = self.task['dataset'].get(category)
-                if dataset_path:
-                    input_dataset[category] = get_dict_from_file(dataset_path)
-                    files_url_to_path_from_objs(input_dataset[category], ['files'], is_list=True)
-
-        return input_dataset
-
-    def run_train(self):
-        hyperparameter = self.task['hyperparameter']
-        train = import_string(self.plugin['train']['entrypoint'])
-
-        # download dataset
-        self.log_event('Preparing dataset for training.')
-        input_dataset = self.get_input_dataset_for_training()
-
-        # train dataset
-        self.log_event('Starting model training.')
-
-        model_files = train(self, input_dataset, hyperparameter)
-
-        # upload model_data
-        self.log_event('Registering model data.')
-
-        self.end_log()
-        return model_files
-
-    def start(self):
-        action = self.task['action']
-        getattr(self, f'run_{action}')()
-
-    def log_metric(self, x, i, **kwargs):
-        self.log(x, {x: i, **kwargs})
-
-    def log_model(self, files, status=None):
-        pass

synapse_sdk-1.0.0a1/synapse_sdk/plugins/cli/publish.py

@@ -1,24 +0,0 @@
-from pathlib import Path
-
-import click
-
-from synapse_sdk.client import Client
-from synapse_sdk.plugins.utils import read_config
-
-from ..upload import archive
-
-
-@click.command()
-@click.option('-h', '--host', required=True)
-@click.option('-t', '--token', required=True)
-@click.option('-w', '--workspace', required=True)
-def publish(host, token, workspace):
-    client = Client(host, token, tenant=workspace)
-    config = read_config()
-
-    source_path = Path('./')
-    archive_path = source_path / 'dist' / 'archive.zip'
-    archive(source_path, archive_path)
-
-    data = {'plugin': config['code'], 'file': str(archive_path)}
-    client.create_plugin_release(data)

synapse_sdk-1.0.0a1/synapse_sdk/plugins/cli/run.py

@@ -1,18 +0,0 @@
-import click
-
-from synapse_sdk.plugins.utils import get_action
-
-
-@click.command()
-@click.argument('action')
-@click.argument('params')
-@click.option('--direct/--no-direct', default=False)
-@click.pass_context
-def run(ctx, action, params, direct):
-    debug = ctx.obj['DEBUG']
-
-    action = get_action(action, params, direct=direct, debug=debug)
-    result = action.run_action()
-
-    if debug:
-        click.echo(result)