synapse-sdk 0.1.2a0__py3-none-any.whl → 1.0.0a1__py3-none-any.whl

{synapse → synapse_sdk}/client/__init__.py

@@ -15,13 +15,13 @@ from ..utils.file import files_url_to_path_from_objs
 class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
     base_url = None
     token = None
-    workspace_code = None
+    tenant = None
 
-    def __init__(self, base_url, token, workspace_code=None):
+    def __init__(self, base_url, token, tenant=None):
         self.base_url = base_url
         self.token = token
-        if workspace_code:
-            self.workspace_code = workspace_code
+        if tenant:
+            self.tenant = tenant
         requests_session = requests.Session()
         self.requests_session = requests_session
 
@@ -32,8 +32,8 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
 
     def _get_headers(self):
         headers = {'Authorization': f'Token {self.token}'}
-        if self.workspace_code:
-            headers['DATAMAKER-Workspace'] = f'Token {self.workspace_code}'
+        if self.tenant:
+            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
         return headers
 
     def _request(self, method, path, **kwargs):

synapse_sdk/client/mixins/integration.py

@@ -0,0 +1,29 @@
+class IntegrationClientMixin:
+    def get_plugin(self, pk):
+        path = f'plugins/{pk}/'
+        return self._get(path)
+
+    def create_plugin(self, data):
+        path = 'plugins/'
+        return self._post(path, payload=data)
+
+    def update_plugin(self, pk, data):
+        path = f'plugins/{pk}/'
+        return self._put(path, payload=data)
+
+    def get_plugin_release(self, pk, params=None):
+        path = f'plugin_releases/{pk}/'
+        return self._get(path, payload=params)
+
+    def create_plugin_release(self, data):
+        path = 'plugin_releases/'
+        files = {'file': data.pop('file')}
+        return self._post(path, payload=data, files=files)
+
+    def create_logs(self, data):
+        path = 'logs/'
+        return self._post(path, payload=data)
+
+    def create_task(self, data):
+        path = 'agent_tasks/'
+        return self._post(path, payload=data)

{synapse → synapse_sdk}/loggers.py

@@ -1,6 +1,6 @@
 import datetime
 
-from synapse.client import ClientError
+from synapse_sdk.client import ClientError
 
 
 class BaseLogger:

synapse_sdk/plugins/__init__.py

@@ -0,0 +1,13 @@
+import os
+
+from dotenv import load_dotenv
+
+from synapse_sdk.plugins.cli import cli
+
+
+def init():
+    load_dotenv(os.path.join(os.getcwd(), '.env'))
+    cli(obj={}, auto_envvar_prefix='SYNAPSE_PLUGIN')
+
+
+__all__ = ['init']

synapse_sdk/plugins/categories/base.py

@@ -0,0 +1,178 @@
+import inspect
+import json
+import os
+from functools import cached_property
+from pprint import pprint
+
+import ray
+import requests
+from ray.dashboard.modules.job.sdk import JobSubmissionClient
+
+from synapse_sdk.loggers import ConsoleLogger
+from synapse_sdk.plugins.enums import RunMethod
+from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload
+from synapse_sdk.plugins.utils import get_plugin_checksum
+from synapse_sdk.utils.module_loading import import_string
+
+
+class Action:
+    name = None
+    category = None
+    method = None
+    params = None
+    plugin_config = None
+    config = None
+    client = None
+    logger = None
+    debug = False
+
+    default_envs = [
+        'RAY_DASHBOARD_URL',
+        'RAY_SERVE_ADDRESS',
+        'SYNAPSE_PLUGIN_URL',
+        'SYNAPSE_PLUGIN_PATH',
+        'SYNAPSE_PLUGIN_BASE_URL',
+        'SYNAPSE_PLUGIN_STORAGE',
+        'SYNAPSE_DEBUG_MODULES',
+    ]
+
+    def __init__(self, params, plugin_config, envs=None, job_id=None, direct=False, debug=False):
+        self.params = params
+        self.plugin_config = plugin_config
+        self.config = plugin_config['actions'][self.name]
+        self.job_id = job_id
+        self.direct = direct
+        self.debug = debug
+        if envs:
+            self.envs = {**envs, **self.get_default_envs()}
+        else:
+            self.envs = self.get_default_envs()
+
+        # TODO logger 지정 방식 개선
+        self.logger = ConsoleLogger()
+
+    @cached_property
+    def plugin_id(self):
+        code = self.plugin_config['code']
+        version = self.plugin_config['version']
+        return f'{code}@{version}'
+
+    @cached_property
+    def plugin_checksum(self):
+        return get_plugin_checksum(self.plugin_id)
+
+    @cached_property
+    def plugin_url(self):
+        if self.debug:
+            plugin_url = self.envs.get('SYNAPSE_PLUGIN_URL')
+            if not plugin_url:
+                plugin_url = archive_and_upload(
+                    self.envs.get('SYNAPSE_PLUGIN_PATH', '.'),
+                    self.envs['SYNAPSE_PLUGIN_STORAGE'],
+                )
+                self.envs['SYNAPSE_PLUGIN_URL'] = plugin_url
+            return plugin_url
+        base_url = self.envs['SYNAPSE_PLUGIN_BASE_URL']
+        return str(os.path.join(base_url, f'{self.plugin_checksum}.zip'))
+
+    @cached_property
+    def entrypoint(self):
+        return import_string(self.config['entrypoint'])
+
+    def get_default_envs(self):
+        return {env: os.environ[env] for env in self.default_envs if env in os.environ}
+
+    def get_debug_modules(self):
+        debug_modules = []
+        for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
+            if module_path.startswith('http'):
+                module_url = module_path
+            else:
+                module_url = build_and_upload(module_path, self.envs['SYNAPSE_PLUGIN_STORAGE'])
+            debug_modules.append(module_url)
+        self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
+        return debug_modules
+
+    def get_runtime_env(self):
+        runtime_env = {
+            'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
+            'working_dir': self.plugin_url,
+        }
+
+        if self.debug:
+            runtime_env['pip'] += self.get_debug_modules()
+
+        # 맨 마지막에 진행되어야 함
+        runtime_env['env_vars'] = self.envs
+
+        if self.debug:
+            pprint(runtime_env)
+        return runtime_env
+
+    def run(self):
+        return self.entrypoint(self, **self.params)
+
+    def run_action(self):
+        if self.direct:
+            if self.method == RunMethod.RESTAPI:
+                return self.run_by_restapi()
+            else:
+                return self.run()
+        return getattr(self, f'run_by_{self.method.value}')()
+
+    def run_by_task(self):
+        @ray.remote(runtime_env=self.get_runtime_env())
+        def run_task(category, action, *args, **kwargs):
+            from synapse_sdk.plugins.utils import get_action_class
+
+            action = get_action_class(category, action)(*args, **kwargs)
+            return action.run_action()
+
+        init_signature = inspect.signature(self.__class__.__init__)
+
+        args = []
+        kwargs = {}
+
+        for param in init_signature.parameters.values():
+            if param.name == 'self':
+                continue
+            if param.default == param.empty:
+                args.append(getattr(self, param.name))
+            else:
+                kwargs[param.name] = getattr(self, param.name)
+
+        kwargs['direct'] = True
+        return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
+
+    def run_by_job(self):
+        entrypoint_args = ['run', self.name, f'"{json.dumps(self.params)}"', '--direct']
+        if self.debug:
+            entrypoint_args.insert(0, '--debug')
+
+        client = JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
+        return client.submit_job(
+            submission_id=self.job_id,
+            entrypoint=f'python main.py {" ".join(entrypoint_args)}',
+            runtime_env=self.get_runtime_env(),
+        )
+
+    def run_by_restapi(self):
+        path = self.params.pop('path', '')
+        method = self.params.pop('method')
+
+        url = os.path.join(self.envs['RAY_SERVE_ADDRESS'], self.plugin_checksum, path)
+        response = getattr(requests, method)(url, **self.params)
+        # TODO ok response가 아닌 경우 대응하기
+        return response.json()
+
+    def set_progress(self, current, total, category=''):
+        self.logger.set_progress(current, total, category)
+
+    def log(self, action, data):
+        self.logger.log(action, data)
+
+    def log_event(self, message):
+        self.logger.log('event', {'content': message})
+
+    def end_log(self):
+        self.log_event('Plugin run is complete.')

synapse_sdk/plugins/categories/data_validation/actions/validation.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class ValidationAction(Action):
+    name = 'validation'
+    category = PluginCategory.DATA_VALIDATION
+    method = RunMethod.TASK

synapse_sdk/plugins/categories/decorators.py

@@ -0,0 +1,13 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS
+
+
+def register_action(action_class):
+    if not issubclass(action_class, Action):
+        raise ValueError('Wrapped class must subclass Action class.')
+
+    try:
+        _REGISTERED_ACTIONS[action_class.category.value][action_class.name] = action_class
+    except KeyError:
+        _REGISTERED_ACTIONS[action_class.category.value] = {action_class.name: action_class}
+    return action_class

synapse_sdk/plugins/categories/export/actions/export.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class ExportAction(Action):
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB

synapse_sdk/plugins/categories/import/actions/import.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class ImportAction(Action):
+    name = 'import'
+    category = PluginCategory.IMPORT
+    method = RunMethod.JOB

synapse_sdk/plugins/categories/neural_net/actions/deployment.py

@@ -0,0 +1,23 @@
+from ray import serve
+
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+
+
+@register_action
+class DeploymentAction(Action):
+    name = 'deployment'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
+
+    def get_deployment(self):
+        return serve.deployment(ray_actor_options=self.get_actor_options())(self.entrypoint)
+
+    def get_actor_options(self):
+        return {'runtime_env': self.get_runtime_env()}
+
+    def run(self):
+        deployment = self.get_deployment()
+        serve.delete(self.plugin_id)
+        serve.run(deployment.bind(), name=self.plugin_id, route_prefix=f'/{self.plugin_checksum}')

synapse_sdk/plugins/categories/neural_net/actions/inference.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class InferenceAction(Action):
+    name = 'inference'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.RESTAPI

synapse_sdk/plugins/categories/neural_net/actions/test.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class TestAction(Action):
+    name = 'test'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.TASK

{synapse → synapse_sdk}/plugins/categories/neural_net/actions/train.py

@@ -1,10 +1,15 @@
-from synapse.plugins.categories.base import Action
-from synapse.utils.file import get_dict_from_file, files_url_to_path_from_objs
-from synapse.utils.module_loading import import_string
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.utils.file import get_dict_from_file, files_url_to_path_from_objs
+from synapse_sdk.utils.module_loading import import_string
 
 
+@register_action
 class TrainAction(Action):
-    # TODO implement specifying which hardware to use (gpu-n, cpu)
+    name = 'train'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
 
     def get_input_dataset_for_training(self, model_id=None):
         """

synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class PostAnnotationAction(Action):
+    name = 'post_annotation'
+    category = PluginCategory.POST_ANNOTATION
+    method = RunMethod.TASK

synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class PreAnnotationAction(Action):
+    name = 'pre_annotation'
+    category = PluginCategory.PRE_ANNOTATION
+    method = RunMethod.TASK

synapse_sdk/plugins/categories/registry.py

@@ -0,0 +1,16 @@
+import pkgutil
+from importlib import import_module
+
+from synapse_sdk.plugins.enums import PluginCategory
+
+_REGISTERED_ACTIONS = {}
+
+
+def register_actions():
+    if not _REGISTERED_ACTIONS:
+        for category in PluginCategory:
+            plugin_category_module_name = f'synapse_sdk.plugins.categories.{category.value}.actions'
+            plugin_category_module = import_module(plugin_category_module_name)
+            for _, action_name, _ in pkgutil.iter_modules(plugin_category_module.__path__):
+                action_module_name = f'{plugin_category_module_name}.{action_name}'
+                import_module(action_module_name)

synapse_sdk/plugins/cli/__init__.py

@@ -0,0 +1,19 @@
+import click
+
+from .publish import publish
+from .run import run
+
+
+@click.group()
+@click.option('--debug/--no-debug', default=False)
+@click.pass_context
+def cli(ctx, debug):
+    ctx.ensure_object(dict)
+    ctx.obj['DEBUG'] = debug
+
+    if debug:
+        click.echo('Debug mode is "on"')
+
+
+cli.add_command(run)
+cli.add_command(publish)

synapse_sdk/plugins/cli/publish.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+import click
+
+from synapse_sdk.client import Client
+from synapse_sdk.plugins.utils import read_config
+
+from ..upload import archive
+
+
+@click.command()
+@click.option('-h', '--host', required=True)
+@click.option('-t', '--token', required=True)
+@click.option('-w', '--workspace', required=True)
+def publish(host, token, workspace):
+    client = Client(host, token, tenant=workspace)
+    config = read_config()
+
+    source_path = Path('./')
+    archive_path = source_path / 'dist' / 'archive.zip'
+    archive(source_path, archive_path)
+
+    data = {'plugin': config['code'], 'file': str(archive_path)}
+    client.create_plugin_release(data)

synapse_sdk/plugins/cli/run.py

@@ -0,0 +1,18 @@
+import click
+
+from synapse_sdk.plugins.utils import get_action
+
+
+@click.command()
+@click.argument('action')
+@click.argument('params')
+@click.option('--direct/--no-direct', default=False)
+@click.pass_context
+def run(ctx, action, params, direct):
+    debug = ctx.obj['DEBUG']
+
+    action = get_action(action, params, direct=direct, debug=debug)
+    result = action.run_action()
+
+    if debug:
+        click.echo(result)

synapse_sdk/plugins/enums.py

@@ -0,0 +1,16 @@
+from enum import Enum
+
+
+class RunMethod(Enum):
+    JOB = 'job'
+    TASK = 'task'
+    RESTAPI = 'restapi'
+
+
+class PluginCategory(Enum):
+    NEURAL_NET = 'neural_net'
+    EXPORT = 'export'
+    IMPORT = 'import'
+    POST_ANNOTATION = 'post_annotation'
+    PRE_ANNOTATION = 'pre_annotation'
+    DATA_VALIDATION = 'data_validation'

synapse_sdk/plugins/job.py

@@ -0,0 +1,5 @@
+class Job:
+    job_id = None
+
+    def __init__(self, job_id):
+        self.job_id = job_id

synapse_sdk/plugins/upload.py

@@ -0,0 +1,72 @@
+import re
+import subprocess
+from pathlib import Path
+
+from synapse_sdk.utils.file import calculate_checksum
+from synapse_sdk.utils.storage import get_storage
+
+
+def archive(source_path, archive_path):
+    archive_path.parent.mkdir(parents=True, exist_ok=True)
+    command = f'git ls-files --others --exclude-standard --cached  | zip -q --names-stdin {archive_path}'
+    subprocess.run(command, cwd=source_path, shell=True, check=True, stdout=subprocess.DEVNULL)
+
+
+def archive_and_upload(source_path, url):
+    storage = get_storage(url)
+    dist_path = Path(source_path, 'dist')
+    archive_path = dist_path / 'archive.zip'
+
+    archive(source_path, archive_path)
+    checksum = calculate_checksum(archive_path, prefix='dev')
+    checksum_archive_path = dist_path / f'{checksum}.zip'
+
+    if checksum_archive_path.exists():
+        # TODO 실제 스토리지 있는지 확인
+        return storage.get_url(checksum_archive_path.name)
+
+    archive_path.rename(checksum_archive_path)
+    for file_path in dist_path.glob('*.zip'):
+        if file_path.name != checksum_archive_path.name:
+            file_path.unlink()
+    return storage.upload(str(checksum_archive_path), checksum_archive_path.name)
+
+
+def build_and_upload(source_path, url, virtualenv_path='.venv'):
+    storage = get_storage(url)
+    dist_path = Path(source_path, 'dist')
+    archive_path = dist_path / 'archive.zip'
+
+    archive(source_path, archive_path)
+    checksum = calculate_checksum(archive_path, prefix='dev')
+    checksum_archive_path = dist_path / f'{checksum}.zip'
+
+    if checksum_archive_path.exists():
+        # TODO 실제 스토리지 있는지 확인
+        wheel_path = next(dist_path.glob('*.whl'), None)
+        return storage.get_url(wheel_path.name)
+
+    # wheel file 빌드 진행
+    for file_path in dist_path.glob('*.whl'):
+        file_path.unlink()
+
+    print(f'Building {Path(source_path).name}...')
+    subprocess.run(
+        f'{virtualenv_path}/bin/python -m build --wheel',
+        cwd=source_path,
+        shell=True,
+        check=True,
+        stdout=subprocess.DEVNULL,
+    )
+    wheel_path = next(dist_path.glob('*.whl'), None)
+
+    archive_path.rename(checksum_archive_path)
+    for file_path in dist_path.glob('*.zip'):
+        if file_path.name != checksum_archive_path.name:
+            file_path.unlink()
+    return storage.upload(str(wheel_path), wheel_path.name)
+
+
+def change_whl_version(whl_name, new_version):
+    pattern = r'^(?P<distribution>.+?)-(?P<version>[\d\.\w]+(\+[\w\.]+)?)(?P<rest>-.+\.whl)$'
+    return re.sub(pattern, rf'\g<distribution>-{new_version}\g<rest>', whl_name)

synapse_sdk/plugins/utils.py

@@ -0,0 +1,44 @@
+import json
+
+from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS, register_actions
+from synapse_sdk.utils.file import get_dict_from_file
+from synapse_sdk.utils.string import hash_text
+
+
+def get_action(action, params_data, *args, **kwargs):
+    if isinstance(params_data, str):
+        try:
+            params = json.loads(params_data)
+        except json.JSONDecodeError:
+            params = get_dict_from_file(params_data)
+    else:
+        params = params_data
+
+    config_data = kwargs.pop('config', False)
+    if config_data:
+        if isinstance(config_data, str):
+            config = get_dict_from_file(config_data)
+        else:
+            config = config_data
+    else:
+        config = read_config()
+    category = config['category']
+    return get_action_class(category, action)(params, config, *args, **kwargs)
+
+
+def get_action_class(category, action):
+    register_actions()
+    return _REGISTERED_ACTIONS[category][action]
+
+
+def get_available_actions(category):
+    register_actions()
+    return list(_REGISTERED_ACTIONS[category].keys())
+
+
+def get_plugin_checksum(plugin_id):
+    return hash_text(plugin_id)
+
+
+def read_config():
+    return get_dict_from_file('config.yaml')

synapse_sdk/utils/debug.py

@@ -0,0 +1,2 @@
+def get_message():
+    return 'hello world from sdk'

{synapse → synapse_sdk}/utils/file.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import operator
 from functools import reduce
@@ -13,7 +14,7 @@ def download_file(url, path_download, name=None, coerce=None):
     else:
         name = Path(url).name
 
-    path = path_download / name
+    path = Path(path_download) / name
     if not path.is_file():
         r = requests.get(url, allow_redirects=True)
         open(str(path), 'wb').write(r.content)
@@ -56,3 +57,14 @@ def get_dict_from_file(file_path):
             return yaml.safe_load(f)
         else:
             return json.load(f)
+
+
+def calculate_checksum(file_path, prefix=''):
+    md5_hash = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b''):
+            md5_hash.update(byte_block)
+    checksum = md5_hash.hexdigest()
+    if prefix:
+        return f'dev-{checksum}'
+    return checksum

synapse_sdk/utils/storage.py

@@ -0,0 +1,92 @@
+import os
+from urllib.parse import urlparse, parse_qs
+
+import boto3
+from botocore.exceptions import ClientError
+
+
+class BaseStorage:
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, url):
+        self.url = urlparse(url)
+        self.query_params = self.url_querystring_to_dict()
+        self.options = self.get_options()
+
+    def url_querystring_to_dict(self):
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def get_options(self):
+        return None
+
+    def upload(self, source, target):
+        raise NotImplementedError
+
+    def exists(self, target):
+        raise NotImplementedError
+
+    def get_url(self, target):
+        raise NotImplementedError
+
+
+class S3Storage(BaseStorage):
+    def __init__(self, url):
+        super().__init__(url)
+        self.client = boto3.client(
+            's3',
+            endpoint_url=self.options['endpoint_url'],
+            aws_access_key_id=self.options['access_key'],
+            aws_secret_access_key=self.options['secret_key'],
+        )
+
+    def get_options(self):
+        base_url = f'https://{self.url.hostname}'
+        local_endpoint = self.query_params.get('local_endpoint')
+        endpoint_url = f'http://{local_endpoint}' if local_endpoint else base_url
+        return {
+            'base_url': base_url,
+            'endpoint_url': endpoint_url,
+            'bucket_name': self.url.path[1:],
+            'access_key': self.url.username,
+            'secret_key': self.url.password,
+            **self.query_params,
+        }
+
+    def upload(self, source, target):
+        object_name = os.path.join(self.options['location'], target)
+        self.client.upload_file(source, self.options['bucket_name'], object_name)
+        return self.get_url(target)
+
+    def exists(self, target):
+        try:
+            self.client.head_object(Bucket=self.options['bucket_name'], Key=target)
+            return True
+        except ClientError:
+            return False
+
+    def get_url(self, target):
+        object_name = os.path.join(self.options['location'], target)
+        return f'{self.options["base_url"]}/{self.options["bucket_name"]}/{object_name}'
+
+
+STORAGE_STORAGES = {
+    's3': S3Storage,
+}
+
+
+def get_storage(url):
+    storage_scheme = urlparse(url).scheme
+    return STORAGE_STORAGES[storage_scheme](url)

{synapse → synapse_sdk}/utils/string.py

@@ -5,3 +5,7 @@ def hash_text(text):
     md5_hash = hashlib.md5()
     md5_hash.update(text.encode('utf-8'))
     return md5_hash.hexdigest()
+
+
+def str_to_bool(value):
+    return value.lower() in ['true', '1', 'yes']

{synapse_sdk-0.1.2a0.dist-info → synapse_sdk-1.0.0a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 0.1.2a0
+Version: 1.0.0a1
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -8,8 +8,11 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: boto3
+Requires-Dist: click
 Requires-Dist: requests
 Requires-Dist: tqdm
+Requires-Dist: python-dotenv
 Requires-Dist: pyyaml
 Requires-Dist: pydantic
 Requires-Dist: ray[all]

synapse_sdk-1.0.0a1.dist-info/RECORD

@@ -0,0 +1,55 @@
+synapse_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/config.py,sha256=AHyTHtzkkCZb9DLu1_R99VOwPgWATZj4K7yzHaPijfA,149
+synapse_sdk/loggers.py,sha256=W4KPq-QOK4ECBL0HxxGUGWCNR3FKbHOUMqnVrwz-Ihg,1331
+synapse_sdk/client/__init__.py,sha256=STq4STOQC5hhxKsAxhMYP6q3f4J1yJd1JUlz_db_130,3339
+synapse_sdk/client/exceptions.py,sha256=LzgzPKRPhNUpLh-jBQHKze8c3CQoVA22jSfjUMBZCuU,405
+synapse_sdk/client/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
+synapse_sdk/client/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/client/mixins/annotation.py,sha256=69xblc0RLokKlpSfIzoJ5dVRvw3VpPIVgbQKV3L5Xx4,905
+synapse_sdk/client/mixins/dataset.py,sha256=KXsen7-vdSzdR09uxD88lT604ItJKuCOSLR3fEU5ud0,1666
+synapse_sdk/client/mixins/integration.py,sha256=iCBjoz7GjzerhxGM0cPunauohs9Vq4_98EtTwxBMdYs,880
+synapse_sdk/client/mixins/ml.py,sha256=xkfO95k6p_nQVyai2zBO-z4b9nh6YIryN4iakDGiPsE,891
+synapse_sdk/plugins/__init__.py,sha256=9vsbYhxah4_ofTaG0x0qLFID_raHNkO57Y8A31Ws-lU,222
+synapse_sdk/plugins/enums.py,sha256=lQZqO2bEeBKdk6q-SMjfOLDlgxv7BuIPk3fXeUFfHRs,327
+synapse_sdk/plugins/job.py,sha256=UzFKA8o_F6RzY_PwyI4dlF3kSfmMG0xEYIyKLfdqSP8,91
+synapse_sdk/plugins/upload.py,sha256=lImaecAz5NG8puT5PwVNkV3kT0d6b7DNBD0jUMaVcBI,2600
+synapse_sdk/plugins/utils.py,sha256=RFxFtmjj-uBK03wUwLhtUecfn_IOKRJupudmsguc2Sc,1212
+synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/base.py,sha256=4pltsAR7RkCm9vaV-1TkVmJSi9w78KsBlVv9v7fFO6U,5823
+synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
+synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
+synapse_sdk/plugins/categories/data_validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/data_validation/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/data_validation/actions/validation.py,sha256=NbEGFytxjakawglZR_Sf9UAjQyuzRxpdwEI1GDbEBW0,338
+synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/export/actions/export.py,sha256=zpvVB0Dac9ytshJrN2ouDpsGHMuJlap-Ymz4qEw_Hzo,320
+synapse_sdk/plugins/categories/import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/import/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/import/actions/import.py,sha256=bkB8x-x7jJfcCnzbz5bOJJy7mPhTKYfIWUdmCoHvpdM,320
+synapse_sdk/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=afqgD_mzErcWdW3xdbrKIOL13fUPO0biZk8_l4l2j84,767
+synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=erM2z7aUTwyzJZqWBlxhTP8dm8cOraI_vUYAqcXkdSY,334
+synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=dAW1zfodlUhoL-sD17tG-CQT0RBxIcHWJ8f1eeZ00M4,321
+synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=sg-PDU_RnAtQGCwI6XnfC63lyPE2fRY6qEUv-_VKu8A,3283
+synapse_sdk/plugins/categories/post_annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/post_annotation/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py,sha256=1tutwNDHpnrCPHzMTsMEk29WPajnZikjBE83j7Z-Xt0,347
+synapse_sdk/plugins/categories/pre_annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py,sha256=YYQt9HsgXlBclE4Sn0c7p1zqCxWHkIHAwyA-tbqrmPQ,344
+synapse_sdk/plugins/cli/__init__.py,sha256=8ogaOhN-RbDNYHqziW8nLsNUxKkZwGkHBdKxTahcm3U,334
+synapse_sdk/plugins/cli/publish.py,sha256=ws4vuXPWiwKq6czIjg__eWE2-vBgquXnsYFdGyI42vs,663
+synapse_sdk/plugins/cli/run.py,sha256=_Lb-Jeh3NHUwui_y34-7lRXp7Ek_rP8BgBM5wqxLdeU,418
+synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/utils/debug.py,sha256=46sMFQLg_JSRUCymnT3wgszG1QsgrocRGiBjVX38r50,53
+synapse_sdk/utils/file.py,sha256=Iptk_DCPsmJzqAABCD3vC6z1yG74fKb5x81LnUCZzYo,1916
+synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
+synapse_sdk/utils/storage.py,sha256=6L3pgno3Luw6GuSRPdiIvS4ZFzjwGP6sLDKMy719pBw,2622
+synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
+synapse_sdk-1.0.0a1.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a1.dist-info/METADATA,sha256=zGbciSdYPvJ1Vl_sk5D0pmTNMqTuemSgkF9QbaVBXec,503
+synapse_sdk-1.0.0a1.dist-info/WHEEL,sha256=a7TGlA-5DaHMRrarXjVbQagU3Man_dCnGIWMJr5kRWo,91
+synapse_sdk-1.0.0a1.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a1.dist-info/RECORD,,

{synapse_sdk-0.1.2a0.dist-info → synapse_sdk-1.0.0a1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

synapse_sdk-1.0.0a1.dist-info/top_level.txt

@@ -0,0 +1 @@
+synapse_sdk

synapse/client/mixins/integration.py

@@ -1,12 +0,0 @@
-class IntegrationClientMixin:
-    def get_plugin(self, pk):
-        path = f'plugins/{pk}/'
-        return self._get(path)
-
-    def create_logs(self, data):
-        path = 'logs/'
-        return self._post(path, payload=data)
-
-    def create_task(self, data):
-        path = 'agent_tasks/'
-        return self._post(path, payload=data)

synapse/plugins/__init__.py

@@ -1,32 +0,0 @@
-import argparse
-
-from synapse.plugins.utils import get_action
-
-action = None
-
-
-def init():
-    global action
-    parser = argparse.ArgumentParser(description='synapse plugin runner')
-
-    # Add arguments
-    parser.add_argument('action', help='action to run on this plugin')
-    parser.add_argument('params', help='parameter of the action')
-
-    # Parse arguments
-    args = parser.parse_args()
-
-    # Access parsed arguments
-    action = args.action
-    params = args.params
-
-    action = get_action(action, params)
-
-
-def run():
-    global action
-    assert action is not None
-    action.run()
-
-
-__all__ = ['init', 'run']

synapse/plugins/categories/__init__.py

@@ -1,11 +0,0 @@
-from synapse.plugins.categories.neural_net.actions.deployment import DeploymentAction
-from synapse.plugins.categories.neural_net.actions.test import TestAction
-from synapse.plugins.categories.neural_net.actions.train import TrainAction
-
-ACTIONS = {
-    'neural_net': {
-        'deployment': DeploymentAction,
-        'train': TrainAction,
-        'test': TestAction,
-    },
-}

synapse/plugins/categories/base.py

@@ -1,51 +0,0 @@
-import os
-from functools import cached_property
-
-from synapse.loggers import ConsoleLogger
-from synapse.plugins.utils import get_plugin_checksum
-
-
-class Action:
-    params = None
-    config = None
-    client = None
-    logger = None
-
-    def __init__(self, params, config):
-        self.params = params
-        self.config = config
-
-        # TODO logger 지정 방식 개선
-        self.logger = ConsoleLogger()
-
-    @cached_property
-    def plugin_id(self):
-        code = self.config['code']
-        version = self.config['version']
-        return f'{code}@{version}'
-
-    @cached_property
-    def plugin_checksum(self):
-        return get_plugin_checksum(self.plugin_id)
-
-    def get_plugin_url(self):
-        base_url = os.getenv('SYNAPSE_PLUGIN_BASE_URL')
-        return str(os.path.join(base_url, f'{self.plugin_checksum}.zip'))
-
-    def get_runtime_env(self):
-        return {'working_dir': self.get_plugin_url()}
-
-    def run(self):
-        raise NotImplementedError
-
-    def set_progress(self, current, total, category=''):
-        self.logger.set_progress(current, total, category)
-
-    def log(self, action, data):
-        self.logger.log(action, data)
-
-    def log_event(self, message):
-        self.logger.log('event', {'content': message})
-
-    def end_log(self):
-        self.log_event('Plugin run is complete.')

synapse/plugins/categories/neural_net/actions/deployment.py

@@ -1,24 +0,0 @@
-from ray import serve
-
-from synapse.plugins.categories.base import Action
-from synapse.utils.module_loading import import_string
-
-
-class DeploymentAction(Action):
-    deployment = None
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.deployment = self.get_deployment()
-
-    def get_deployment(self):
-        entrypoint = self.config['actions']['deployment']['entrypoint']
-        deployment = import_string(entrypoint)
-        return serve.deployment(ray_actor_options=self.get_actor_options())(deployment)
-
-    def get_actor_options(self):
-        return {'runtime_env': self.get_runtime_env()}
-
-    def run(self):
-        serve.delete(self.plugin_id)
-        serve.run(self.deployment.bind(), name=self.plugin_id, route_prefix=f'/{self.plugin_checksum}')

synapse/plugins/categories/neural_net/actions/test.py

@@ -1,6 +0,0 @@
-from synapse.plugins.categories.base import Action
-
-
-class TestAction(Action):
-    def run(self):
-        pass

synapse/plugins/utils.py

@@ -1,26 +0,0 @@
-import json
-
-from synapse.utils.file import get_dict_from_file
-from synapse.utils.string import hash_text
-
-
-def get_action(action, json_or_path):
-    from synapse.plugins.categories import ACTIONS
-
-    try:
-        params = json.loads(json_or_path)
-    except json.JSONDecodeError:
-        params = get_dict_from_file(json_or_path)
-    config = get_dict_from_file('config.yaml')
-    category = config['category']
-    return ACTIONS[category][action](params, config)
-
-
-def get_available_actions(category):
-    from synapse.plugins.categories import ACTIONS
-
-    return list(ACTIONS[category].keys())
-
-
-def get_plugin_checksum(plugin_id):
-    return hash_text(plugin_id)

synapse_sdk-0.1.2a0.dist-info/RECORD

@@ -1,35 +0,0 @@
-synapse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/config.py,sha256=AHyTHtzkkCZb9DLu1_R99VOwPgWATZj4K7yzHaPijfA,149
-synapse/loggers.py,sha256=xFZl1sHffhRShtZoTwlNG7zAtkrG9FDxZDMz8xdvzyg,1327
-synapse/client/__init__.py,sha256=zX2xGXMkcXLQ0zs96K6oQYvNkMlh4P1Xpmw-fvg6MEE,3400
-synapse/client/exceptions.py,sha256=LzgzPKRPhNUpLh-jBQHKze8c3CQoVA22jSfjUMBZCuU,405
-synapse/client/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
-synapse/client/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/client/mixins/annotation.py,sha256=69xblc0RLokKlpSfIzoJ5dVRvw3VpPIVgbQKV3L5Xx4,905
-synapse/client/mixins/dataset.py,sha256=KXsen7-vdSzdR09uxD88lT604ItJKuCOSLR3fEU5ud0,1666
-synapse/client/mixins/integration.py,sha256=LfMgPsP_OBNCZmSTpEThlLcSnOHv1-H8EhhryPNriCE,336
-synapse/client/mixins/ml.py,sha256=xkfO95k6p_nQVyai2zBO-z4b9nh6YIryN4iakDGiPsE,891
-synapse/plugins/__init__.py,sha256=OCcxzbGsQ68zPbz3Nyd21_oqL1W9ocJU_e5wNbS9wUc,623
-synapse/plugins/utils.py,sha256=G5Ife0tveVbKpMjAI7sBFTLO5Kul8sAS--SFhlMy_1M,669
-synapse/plugins/categories/__init__.py,sha256=jIL3HdMQVliX7IMWUnKW1ttp0kyVL9YwsXAXFMeRK9Q,376
-synapse/plugins/categories/base.py,sha256=cqBGGPLHNqczdvsRNKiLSceQkFVVHH1zYaMschEBBG8,1330
-synapse/plugins/categories/custom_script/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/custom_script/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/import/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/plugins/categories/neural_net/actions/deployment.py,sha256=B86zjVbxLO7WsSqPGNgReTetAilt2wRj4VI3TS38UXo,805
-synapse/plugins/categories/neural_net/actions/test.py,sha256=XX2dcwNAEkn7mFliwPb70cx7wn0LL_QcDQ7EW6n-H2w,111
-synapse/plugins/categories/neural_net/actions/train.py,sha256=lQvqVyC8lqVcs8x6v4bD1vTQyDrUz15gSiodEOPGcQc,3100
-synapse/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse/utils/file.py,sha256=kwNoOAzJqT6v2fBHUhhs9sW2mdziUJmYHOxPzIodhmE,1578
-synapse/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
-synapse/utils/string.py,sha256=FMH1vlh7BsDcawCiov3HtQEcjp3KSHmjcZ2wZ9EeGu0,141
-synapse_sdk-0.1.2a0.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-0.1.2a0.dist-info/METADATA,sha256=I093h4TrY-MzSy5apNzHgxcRAXn6QcLxqHgji2qMtyo,432
-synapse_sdk-0.1.2a0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-synapse_sdk-0.1.2a0.dist-info/top_level.txt,sha256=v_1YsqjmoSCzCKs7oIhzTNmWtSYoORiBMv1TJkOhx8A,8
-synapse_sdk-0.1.2a0.dist-info/RECORD,,

synapse_sdk-0.1.2a0.dist-info/top_level.txt

@@ -1 +0,0 @@
-synapse