synapse-sdk 1.0.0a0__tar.gz → 1.0.0a1__tar.gz

{synapse_sdk-1.0.0a0 → synapse_sdk-1.0.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a0
+Version: 1.0.0a1
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -9,6 +9,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3
+Requires-Dist: click
 Requires-Dist: requests
 Requires-Dist: tqdm
 Requires-Dist: python-dotenv

{synapse_sdk-1.0.0a0 → synapse_sdk-1.0.0a1}/pyproject.toml

@@ -14,6 +14,7 @@ classifiers = [
 ]
 dependencies = [
     'boto3',
+    'click',
     'requests',
     'tqdm',
     'python-dotenv',

{synapse_sdk-1.0.0a0 → synapse_sdk-1.0.0a1}/requirements.txt

@@ -1,4 +1,5 @@
 boto3
+click
 requests
 tqdm
 pyyaml

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/client/__init__.py

@@ -15,13 +15,13 @@ from ..utils.file import files_url_to_path_from_objs
 class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
     base_url = None
     token = None
-    workspace_code = None
+    tenant = None
 
-    def __init__(self, base_url, token, workspace_code=None):
+    def __init__(self, base_url, token, tenant=None):
         self.base_url = base_url
         self.token = token
-        if workspace_code:
-            self.workspace_code = workspace_code
+        if tenant:
+            self.tenant = tenant
         requests_session = requests.Session()
         self.requests_session = requests_session
 
@@ -32,8 +32,8 @@ class Client(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin,
 
     def _get_headers(self):
         headers = {'Authorization': f'Token {self.token}'}
-        if self.workspace_code:
-            headers['DATAMAKER-Workspace'] = f'Token {self.workspace_code}'
+        if self.tenant:
+            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
         return headers
 
     def _request(self, method, path, **kwargs):

synapse_sdk-1.0.0a1/synapse_sdk/client/mixins/integration.py

@@ -0,0 +1,29 @@
+class IntegrationClientMixin:
+    def get_plugin(self, pk):
+        path = f'plugins/{pk}/'
+        return self._get(path)
+
+    def create_plugin(self, data):
+        path = 'plugins/'
+        return self._post(path, payload=data)
+
+    def update_plugin(self, pk, data):
+        path = f'plugins/{pk}/'
+        return self._put(path, payload=data)
+
+    def get_plugin_release(self, pk, params=None):
+        path = f'plugin_releases/{pk}/'
+        return self._get(path, payload=params)
+
+    def create_plugin_release(self, data):
+        path = 'plugin_releases/'
+        files = {'file': data.pop('file')}
+        return self._post(path, payload=data, files=files)
+
+    def create_logs(self, data):
+        path = 'logs/'
+        return self._post(path, payload=data)
+
+    def create_task(self, data):
+        path = 'agent_tasks/'
+        return self._post(path, payload=data)

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/loggers.py

@@ -1,6 +1,6 @@
 import datetime
 
-from synapse.client import ClientError
+from synapse_sdk.client import ClientError
 
 
 class BaseLogger:

synapse_sdk-1.0.0a1/synapse_sdk/plugins/__init__.py

@@ -0,0 +1,13 @@
+import os
+
+from dotenv import load_dotenv
+
+from synapse_sdk.plugins.cli import cli
+
+
+def init():
+    load_dotenv(os.path.join(os.getcwd(), '.env'))
+    cli(obj={}, auto_envvar_prefix='SYNAPSE_PLUGIN')
+
+
+__all__ = ['init']

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/plugins/categories/base.py

@@ -8,11 +8,11 @@ import ray
 import requests
 from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
-from synapse.loggers import ConsoleLogger
-from synapse.plugins.enums import RunMethod
-from synapse.plugins.upload import upload_path, build_and_upload
-from synapse.plugins.utils import get_plugin_checksum
-from synapse.utils.module_loading import import_string
+from synapse_sdk.loggers import ConsoleLogger
+from synapse_sdk.plugins.enums import RunMethod
+from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload
+from synapse_sdk.plugins.utils import get_plugin_checksum
+from synapse_sdk.utils.module_loading import import_string
 
 
 class Action:
@@ -27,15 +27,12 @@ class Action:
     debug = False
 
     default_envs = [
+        'RAY_DASHBOARD_URL',
         'RAY_SERVE_ADDRESS',
         'SYNAPSE_PLUGIN_URL',
         'SYNAPSE_PLUGIN_PATH',
         'SYNAPSE_PLUGIN_BASE_URL',
-        'SYNAPSE_PLUGIN_UPLOAD_S3_ENDPOINT_URL',
-        'SYNAPSE_PLUGIN_UPLOAD_S3_BUCKET_NAME',
-        'SYNAPSE_PLUGIN_UPLOAD_S3_ACCESS_KEY',
-        'SYNAPSE_PLUGIN_UPLOAD_S3_SECRET_KEY',
-        'SYNAPSE_PLUGIN_UPLOAD_S3_BASE_URL',
+        'SYNAPSE_PLUGIN_STORAGE',
         'SYNAPSE_DEBUG_MODULES',
     ]
 
@@ -69,7 +66,10 @@ class Action:
         if self.debug:
             plugin_url = self.envs.get('SYNAPSE_PLUGIN_URL')
             if not plugin_url:
-                plugin_url = upload_path(self.envs.get('SYNAPSE_PLUGIN_PATH', '.'), **self.get_kwargs_upload_path())
+                plugin_url = archive_and_upload(
+                    self.envs.get('SYNAPSE_PLUGIN_PATH', '.'),
+                    self.envs['SYNAPSE_PLUGIN_STORAGE'],
+                )
                 self.envs['SYNAPSE_PLUGIN_URL'] = plugin_url
             return plugin_url
         base_url = self.envs['SYNAPSE_PLUGIN_BASE_URL']
@@ -79,16 +79,6 @@ class Action:
     def entrypoint(self):
         return import_string(self.config['entrypoint'])
 
-    def get_kwargs_upload_path(self):
-        # TODO upload 관련 env 단일화 후 제거
-        return {
-            'endpoint_url': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_ENDPOINT_URL'],
-            'bucket_name': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_BUCKET_NAME'],
-            'access_key': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_ACCESS_KEY'],
-            'secret_key': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_SECRET_KEY'],
-            'base_url': self.envs['SYNAPSE_PLUGIN_UPLOAD_S3_BASE_URL'],
-        }
-
     def get_default_envs(self):
         return {env: os.environ[env] for env in self.default_envs if env in os.environ}
 
@@ -98,7 +88,7 @@ class Action:
             if module_path.startswith('http'):
                 module_url = module_path
             else:
-                module_url = build_and_upload(module_path, **self.get_kwargs_upload_path())
+                module_url = build_and_upload(module_path, self.envs['SYNAPSE_PLUGIN_STORAGE'])
             debug_modules.append(module_url)
         self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
@@ -112,8 +102,11 @@ class Action:
         if self.debug:
             runtime_env['pip'] += self.get_debug_modules()
 
+        # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
-        pprint(runtime_env)
+
+        if self.debug:
+            pprint(runtime_env)
         return runtime_env
 
     def run(self):
@@ -130,7 +123,7 @@ class Action:
     def run_by_task(self):
         @ray.remote(runtime_env=self.get_runtime_env())
         def run_task(category, action, *args, **kwargs):
-            from synapse.plugins.utils import get_action_class
+            from synapse_sdk.plugins.utils import get_action_class
 
             action = get_action_class(category, action)(*args, **kwargs)
             return action.run_action()
@@ -152,14 +145,14 @@ class Action:
         return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
 
     def run_by_job(self):
-        entrypoint_args = [self.name, f'"{json.dumps(self.params)}"']
+        entrypoint_args = ['run', self.name, f'"{json.dumps(self.params)}"', '--direct']
         if self.debug:
-            entrypoint_args.append('--debug')
+            entrypoint_args.insert(0, '--debug')
 
-        client = JobSubmissionClient()
+        client = JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
         return client.submit_job(
             submission_id=self.job_id,
-            entrypoint=f'python main.py {" ".join(entrypoint_args)} --direct',
+            entrypoint=f'python main.py {" ".join(entrypoint_args)}',
             runtime_env=self.get_runtime_env(),
         )
 

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/data_validation/actions/validation.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class ValidationAction(Action):
+    name = 'validation'
+    category = PluginCategory.DATA_VALIDATION
+    method = RunMethod.TASK

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/plugins/categories/decorators.py

@@ -1,5 +1,5 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.registry import _REGISTERED_ACTIONS
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS
 
 
 def register_action(action_class):

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/export/actions/export.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class ExportAction(Action):
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/import/actions/import.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class ImportAction(Action):
+    name = 'import'
+    category = PluginCategory.IMPORT
+    method = RunMethod.JOB

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/plugins/categories/neural_net/actions/deployment.py

@@ -1,8 +1,8 @@
 from ray import serve
 
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/neural_net/actions/inference.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class InferenceAction(Action):
+    name = 'inference'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.RESTAPI

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/neural_net/actions/test.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class TestAction(Action):
+    name = 'test'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.TASK

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/plugins/categories/neural_net/actions/train.py

@@ -1,8 +1,8 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-from synapse.utils.file import get_dict_from_file, files_url_to_path_from_objs
-from synapse.utils.module_loading import import_string
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.utils.file import get_dict_from_file, files_url_to_path_from_objs
+from synapse_sdk.utils.module_loading import import_string
 
 
 @register_action

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class PostAnnotationAction(Action):
+    name = 'post_annotation'
+    category = PluginCategory.POST_ANNOTATION
+    method = RunMethod.TASK

synapse_sdk-1.0.0a1/synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py

@@ -0,0 +1,10 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class PreAnnotationAction(Action):
+    name = 'pre_annotation'
+    category = PluginCategory.PRE_ANNOTATION
+    method = RunMethod.TASK

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/plugins/categories/registry.py

@@ -1,7 +1,7 @@
 import pkgutil
 from importlib import import_module
 
-from synapse.plugins.enums import PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory
 
 _REGISTERED_ACTIONS = {}
 
@@ -9,7 +9,7 @@ _REGISTERED_ACTIONS = {}
 def register_actions():
     if not _REGISTERED_ACTIONS:
         for category in PluginCategory:
-            plugin_category_module_name = f'synapse.plugins.categories.{category.value}.actions'
+            plugin_category_module_name = f'synapse_sdk.plugins.categories.{category.value}.actions'
             plugin_category_module = import_module(plugin_category_module_name)
             for _, action_name, _ in pkgutil.iter_modules(plugin_category_module.__path__):
                 action_module_name = f'{plugin_category_module_name}.{action_name}'

synapse_sdk-1.0.0a1/synapse_sdk/plugins/cli/__init__.py

@@ -0,0 +1,19 @@
+import click
+
+from .publish import publish
+from .run import run
+
+
+@click.group()
+@click.option('--debug/--no-debug', default=False)
+@click.pass_context
+def cli(ctx, debug):
+    ctx.ensure_object(dict)
+    ctx.obj['DEBUG'] = debug
+
+    if debug:
+        click.echo('Debug mode is "on"')
+
+
+cli.add_command(run)
+cli.add_command(publish)

synapse_sdk-1.0.0a1/synapse_sdk/plugins/cli/publish.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+import click
+
+from synapse_sdk.client import Client
+from synapse_sdk.plugins.utils import read_config
+
+from ..upload import archive
+
+
+@click.command()
+@click.option('-h', '--host', required=True)
+@click.option('-t', '--token', required=True)
+@click.option('-w', '--workspace', required=True)
+def publish(host, token, workspace):
+    client = Client(host, token, tenant=workspace)
+    config = read_config()
+
+    source_path = Path('./')
+    archive_path = source_path / 'dist' / 'archive.zip'
+    archive(source_path, archive_path)
+
+    data = {'plugin': config['code'], 'file': str(archive_path)}
+    client.create_plugin_release(data)

synapse_sdk-1.0.0a1/synapse_sdk/plugins/cli/run.py

@@ -0,0 +1,18 @@
+import click
+
+from synapse_sdk.plugins.utils import get_action
+
+
+@click.command()
+@click.argument('action')
+@click.argument('params')
+@click.option('--direct/--no-direct', default=False)
+@click.pass_context
+def run(ctx, action, params, direct):
+    debug = ctx.obj['DEBUG']
+
+    action = get_action(action, params, direct=direct, debug=debug)
+    result = action.run_action()
+
+    if debug:
+        click.echo(result)

synapse_sdk-1.0.0a1/synapse_sdk/plugins/upload.py

@@ -0,0 +1,72 @@
+import re
+import subprocess
+from pathlib import Path
+
+from synapse_sdk.utils.file import calculate_checksum
+from synapse_sdk.utils.storage import get_storage
+
+
+def archive(source_path, archive_path):
+    archive_path.parent.mkdir(parents=True, exist_ok=True)
+    command = f'git ls-files --others --exclude-standard --cached  | zip -q --names-stdin {archive_path}'
+    subprocess.run(command, cwd=source_path, shell=True, check=True, stdout=subprocess.DEVNULL)
+
+
+def archive_and_upload(source_path, url):
+    storage = get_storage(url)
+    dist_path = Path(source_path, 'dist')
+    archive_path = dist_path / 'archive.zip'
+
+    archive(source_path, archive_path)
+    checksum = calculate_checksum(archive_path, prefix='dev')
+    checksum_archive_path = dist_path / f'{checksum}.zip'
+
+    if checksum_archive_path.exists():
+        # TODO 실제 스토리지 있는지 확인
+        return storage.get_url(checksum_archive_path.name)
+
+    archive_path.rename(checksum_archive_path)
+    for file_path in dist_path.glob('*.zip'):
+        if file_path.name != checksum_archive_path.name:
+            file_path.unlink()
+    return storage.upload(str(checksum_archive_path), checksum_archive_path.name)
+
+
+def build_and_upload(source_path, url, virtualenv_path='.venv'):
+    storage = get_storage(url)
+    dist_path = Path(source_path, 'dist')
+    archive_path = dist_path / 'archive.zip'
+
+    archive(source_path, archive_path)
+    checksum = calculate_checksum(archive_path, prefix='dev')
+    checksum_archive_path = dist_path / f'{checksum}.zip'
+
+    if checksum_archive_path.exists():
+        # TODO 실제 스토리지 있는지 확인
+        wheel_path = next(dist_path.glob('*.whl'), None)
+        return storage.get_url(wheel_path.name)
+
+    # wheel file 빌드 진행
+    for file_path in dist_path.glob('*.whl'):
+        file_path.unlink()
+
+    print(f'Building {Path(source_path).name}...')
+    subprocess.run(
+        f'{virtualenv_path}/bin/python -m build --wheel',
+        cwd=source_path,
+        shell=True,
+        check=True,
+        stdout=subprocess.DEVNULL,
+    )
+    wheel_path = next(dist_path.glob('*.whl'), None)
+
+    archive_path.rename(checksum_archive_path)
+    for file_path in dist_path.glob('*.zip'):
+        if file_path.name != checksum_archive_path.name:
+            file_path.unlink()
+    return storage.upload(str(wheel_path), wheel_path.name)
+
+
+def change_whl_version(whl_name, new_version):
+    pattern = r'^(?P<distribution>.+?)-(?P<version>[\d\.\w]+(\+[\w\.]+)?)(?P<rest>-.+\.whl)$'
+    return re.sub(pattern, rf'\g<distribution>-{new_version}\g<rest>', whl_name)

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/plugins/utils.py

@@ -1,8 +1,8 @@
 import json
 
-from synapse.plugins.categories.registry import _REGISTERED_ACTIONS, register_actions
-from synapse.utils.file import get_dict_from_file
-from synapse.utils.string import hash_text
+from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS, register_actions
+from synapse_sdk.utils.file import get_dict_from_file
+from synapse_sdk.utils.string import hash_text
 
 
 def get_action(action, params_data, *args, **kwargs):
@@ -21,7 +21,7 @@ def get_action(action, params_data, *args, **kwargs):
         else:
             config = config_data
     else:
-        config = get_dict_from_file('config.yaml')
+        config = read_config()
     category = config['category']
     return get_action_class(category, action)(params, config, *args, **kwargs)
 
@@ -38,3 +38,7 @@ def get_available_actions(category):
 
 def get_plugin_checksum(plugin_id):
     return hash_text(plugin_id)
+
+
+def read_config():
+    return get_dict_from_file('config.yaml')

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/utils/file.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import operator
 from functools import reduce
@@ -13,7 +14,7 @@ def download_file(url, path_download, name=None, coerce=None):
     else:
         name = Path(url).name
 
-    path = path_download / name
+    path = Path(path_download) / name
     if not path.is_file():
         r = requests.get(url, allow_redirects=True)
         open(str(path), 'wb').write(r.content)
@@ -56,3 +57,14 @@ def get_dict_from_file(file_path):
             return yaml.safe_load(f)
         else:
             return json.load(f)
+
+
+def calculate_checksum(file_path, prefix=''):
+    md5_hash = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b''):
+            md5_hash.update(byte_block)
+    checksum = md5_hash.hexdigest()
+    if prefix:
+        return f'dev-{checksum}'
+    return checksum

synapse_sdk-1.0.0a1/synapse_sdk/utils/storage.py

@@ -0,0 +1,92 @@
+import os
+from urllib.parse import urlparse, parse_qs
+
+import boto3
+from botocore.exceptions import ClientError
+
+
+class BaseStorage:
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, url):
+        self.url = urlparse(url)
+        self.query_params = self.url_querystring_to_dict()
+        self.options = self.get_options()
+
+    def url_querystring_to_dict(self):
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def get_options(self):
+        return None
+
+    def upload(self, source, target):
+        raise NotImplementedError
+
+    def exists(self, target):
+        raise NotImplementedError
+
+    def get_url(self, target):
+        raise NotImplementedError
+
+
+class S3Storage(BaseStorage):
+    def __init__(self, url):
+        super().__init__(url)
+        self.client = boto3.client(
+            's3',
+            endpoint_url=self.options['endpoint_url'],
+            aws_access_key_id=self.options['access_key'],
+            aws_secret_access_key=self.options['secret_key'],
+        )
+
+    def get_options(self):
+        base_url = f'https://{self.url.hostname}'
+        local_endpoint = self.query_params.get('local_endpoint')
+        endpoint_url = f'http://{local_endpoint}' if local_endpoint else base_url
+        return {
+            'base_url': base_url,
+            'endpoint_url': endpoint_url,
+            'bucket_name': self.url.path[1:],
+            'access_key': self.url.username,
+            'secret_key': self.url.password,
+            **self.query_params,
+        }
+
+    def upload(self, source, target):
+        object_name = os.path.join(self.options['location'], target)
+        self.client.upload_file(source, self.options['bucket_name'], object_name)
+        return self.get_url(target)
+
+    def exists(self, target):
+        try:
+            self.client.head_object(Bucket=self.options['bucket_name'], Key=target)
+            return True
+        except ClientError:
+            return False
+
+    def get_url(self, target):
+        object_name = os.path.join(self.options['location'], target)
+        return f'{self.options["base_url"]}/{self.options["bucket_name"]}/{object_name}'
+
+
+STORAGE_STORAGES = {
+    's3': S3Storage,
+}
+
+
+def get_storage(url):
+    storage_scheme = urlparse(url).scheme
+    return STORAGE_STORAGES[storage_scheme](url)

{synapse_sdk-1.0.0a0/synapse → synapse_sdk-1.0.0a1/synapse_sdk}/utils/string.py

@@ -5,3 +5,7 @@ def hash_text(text):
     md5_hash = hashlib.md5()
     md5_hash.update(text.encode('utf-8'))
     return md5_hash.hexdigest()
+
+
+def str_to_bool(value):
+    return value.lower() in ['true', '1', 'yes']

{synapse_sdk-1.0.0a0 → synapse_sdk-1.0.0a1}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a0
+Version: 1.0.0a1
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -9,6 +9,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3
+Requires-Dist: click
 Requires-Dist: requests
 Requires-Dist: tqdm
 Requires-Dist: python-dotenv

synapse_sdk-1.0.0a1/synapse_sdk.egg-info/SOURCES.txt

@@ -0,0 +1,63 @@
+.gitignore
+.pre-commit-config.yaml
+LICENSE
+README.md
+pyproject.toml
+requirements.txt
+.github/workflows/lint.yml
+.github/workflows/pypi-publish.yml
+synapse_sdk/__init__.py
+synapse_sdk/config.py
+synapse_sdk/loggers.py
+synapse_sdk.egg-info/PKG-INFO
+synapse_sdk.egg-info/SOURCES.txt
+synapse_sdk.egg-info/dependency_links.txt
+synapse_sdk.egg-info/requires.txt
+synapse_sdk.egg-info/top_level.txt
+synapse_sdk/client/__init__.py
+synapse_sdk/client/exceptions.py
+synapse_sdk/client/utils.py
+synapse_sdk/client/mixins/__init__.py
+synapse_sdk/client/mixins/annotation.py
+synapse_sdk/client/mixins/dataset.py
+synapse_sdk/client/mixins/integration.py
+synapse_sdk/client/mixins/ml.py
+synapse_sdk/plugins/__init__.py
+synapse_sdk/plugins/enums.py
+synapse_sdk/plugins/job.py
+synapse_sdk/plugins/upload.py
+synapse_sdk/plugins/utils.py
+synapse_sdk/plugins/categories/__init__.py
+synapse_sdk/plugins/categories/base.py
+synapse_sdk/plugins/categories/decorators.py
+synapse_sdk/plugins/categories/registry.py
+synapse_sdk/plugins/categories/data_validation/__init__.py
+synapse_sdk/plugins/categories/data_validation/actions/__init__.py
+synapse_sdk/plugins/categories/data_validation/actions/validation.py
+synapse_sdk/plugins/categories/export/__init__.py
+synapse_sdk/plugins/categories/export/actions/__init__.py
+synapse_sdk/plugins/categories/export/actions/export.py
+synapse_sdk/plugins/categories/import/__init__.py
+synapse_sdk/plugins/categories/import/actions/__init__.py
+synapse_sdk/plugins/categories/import/actions/import.py
+synapse_sdk/plugins/categories/neural_net/__init__.py
+synapse_sdk/plugins/categories/neural_net/actions/__init__.py
+synapse_sdk/plugins/categories/neural_net/actions/deployment.py
+synapse_sdk/plugins/categories/neural_net/actions/inference.py
+synapse_sdk/plugins/categories/neural_net/actions/test.py
+synapse_sdk/plugins/categories/neural_net/actions/train.py
+synapse_sdk/plugins/categories/post_annotation/__init__.py
+synapse_sdk/plugins/categories/post_annotation/actions/__init__.py
+synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py
+synapse_sdk/plugins/categories/pre_annotation/__init__.py
+synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py
+synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py
+synapse_sdk/plugins/cli/__init__.py
+synapse_sdk/plugins/cli/publish.py
+synapse_sdk/plugins/cli/run.py
+synapse_sdk/utils/__init__.py
+synapse_sdk/utils/debug.py
+synapse_sdk/utils/file.py
+synapse_sdk/utils/module_loading.py
+synapse_sdk/utils/storage.py
+synapse_sdk/utils/string.py

{synapse_sdk-1.0.0a0 → synapse_sdk-1.0.0a1}/synapse_sdk.egg-info/requires.txt

@@ -1,4 +1,5 @@
 boto3
+click
 requests
 tqdm
 python-dotenv

synapse_sdk-1.0.0a1/synapse_sdk.egg-info/top_level.txt

@@ -0,0 +1 @@
+synapse_sdk

synapse_sdk-1.0.0a0/synapse/client/mixins/integration.py

@@ -1,12 +0,0 @@
-class IntegrationClientMixin:
-    def get_plugin(self, pk):
-        path = f'plugins/{pk}/'
-        return self._get(path)
-
-    def create_logs(self, data):
-        path = 'logs/'
-        return self._post(path, payload=data)
-
-    def create_task(self, data):
-        path = 'agent_tasks/'
-        return self._post(path, payload=data)

synapse_sdk-1.0.0a0/synapse/plugins/__init__.py

@@ -1,39 +0,0 @@
-import argparse
-import os
-
-from dotenv import load_dotenv
-
-from synapse.plugins.utils import get_action
-
-
-action = None
-
-
-def run():
-    global action
-    parser = argparse.ArgumentParser(description='synapse plugin runner')
-
-    # Add arguments
-    parser.add_argument('action', help='action to run on this plugin')
-    parser.add_argument('params', help='parameter of the action')
-    parser.add_argument('--direct', help='run without using ray', action='store_true')
-    parser.add_argument('--debug', help='run with debug mode', action='store_true')
-
-    # Parse arguments
-    args = parser.parse_args()
-
-    # Access parsed arguments
-    action = args.action
-    params = args.params
-    direct = args.direct
-    debug = args.debug
-
-    load_dotenv(os.path.join(os.getcwd(), '.env'))
-
-    action = get_action(action, params, direct=direct, debug=debug)
-    result = action.run_action()
-    if debug:
-        print(result)
-
-
-__all__ = ['run']

synapse_sdk-1.0.0a0/synapse/plugins/categories/data_validation/actions/validation.py

@@ -1,10 +0,0 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-
-
-@register_action
-class ValidationAction(Action):
-    name = 'validation'
-    category = PluginCategory.DATA_VALIDATION
-    method = RunMethod.TASK

synapse_sdk-1.0.0a0/synapse/plugins/categories/export/actions/export.py

@@ -1,10 +0,0 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-
-
-@register_action
-class ExportAction(Action):
-    name = 'export'
-    category = PluginCategory.EXPORT
-    method = RunMethod.JOB

synapse_sdk-1.0.0a0/synapse/plugins/categories/import/actions/import.py

@@ -1,10 +0,0 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-
-
-@register_action
-class ImportAction(Action):
-    name = 'import'
-    category = PluginCategory.IMPORT
-    method = RunMethod.JOB

synapse_sdk-1.0.0a0/synapse/plugins/categories/neural_net/actions/inference.py

@@ -1,10 +0,0 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-
-
-@register_action
-class InferenceAction(Action):
-    name = 'inference'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.RESTAPI

synapse_sdk-1.0.0a0/synapse/plugins/categories/neural_net/actions/test.py

@@ -1,10 +0,0 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-
-
-@register_action
-class TestAction(Action):
-    name = 'test'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.TASK

synapse_sdk-1.0.0a0/synapse/plugins/categories/post_annotation/actions/post_annotation.py

@@ -1,10 +0,0 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-
-
-@register_action
-class PostAnnotationAction(Action):
-    name = 'post_annotation'
-    category = PluginCategory.POST_ANNOTATION
-    method = RunMethod.TASK

synapse_sdk-1.0.0a0/synapse/plugins/categories/pre_annotation/actions/pre_annotation.py

@@ -1,10 +0,0 @@
-from synapse.plugins.categories.base import Action
-from synapse.plugins.categories.decorators import register_action
-from synapse.plugins.enums import RunMethod, PluginCategory
-
-
-@register_action
-class PreAnnotationAction(Action):
-    name = 'pre_annotation'
-    category = PluginCategory.PRE_ANNOTATION
-    method = RunMethod.TASK

synapse_sdk-1.0.0a0/synapse/plugins/upload.py

@@ -1,79 +0,0 @@
-import os
-import re
-import subprocess
-import tempfile
-import hashlib
-from pathlib import Path
-
-import boto3
-
-
-def calculate_checksum(file_path):
-    md5_hash = hashlib.md5()
-    with open(file_path, 'rb') as f:
-        for byte_block in iter(lambda: f.read(4096), b''):
-            md5_hash.update(byte_block)
-    checksum = md5_hash.hexdigest()
-    return f'dev-{checksum}'
-
-
-def upload_to_s3(file_path, bucket_name, object_name, endpoint_url, access_key, secret_key):
-    s3_client = boto3.client(
-        's3',
-        endpoint_url=endpoint_url,
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-    )
-    s3_client.upload_file(file_path, bucket_name, object_name)
-
-
-def upload_path(
-    source_path,
-    endpoint_url=None,
-    bucket_name=None,
-    access_key=None,
-    secret_key=None,
-    base_url=None,
-):
-    if not endpoint_url:
-        endpoint_url = os.environ['PLUGIN_UPLOAD_S3_ENDPOINT_URL']
-    if not bucket_name:
-        bucket_name = os.environ['PLUGIN_UPLOAD_S3_BUCKET_NAME']
-    if not access_key:
-        access_key = os.environ['PLUGIN_UPLOAD_S3_ACCESS_KEY']
-    if not secret_key:
-        secret_key = os.environ['PLUGIN_UPLOAD_S3_SECRET_KEY']
-    if not base_url:
-        base_url = os.environ['PLUGIN_UPLOAD_S3_BASE_URL']
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        temp_archive_path = os.path.join(temp_dir, 'archive.zip')
-        command = f'git ls-files --others --exclude-standard --cached  | zip -q --names-stdin {temp_archive_path}'
-
-        subprocess.run(command, cwd=source_path, shell=True, check=True)
-
-        checksum = calculate_checksum(temp_archive_path)
-        # TODO subpath param으로 받기
-        s3_object_name = f'assets/{checksum}.zip'
-
-        upload_to_s3(temp_archive_path, bucket_name, s3_object_name, endpoint_url, access_key, secret_key)
-        return f'{base_url}/{bucket_name}/{s3_object_name}'
-
-
-def change_whl_version(whl_name, new_version):
-    pattern = r'^(?P<distribution>.+?)-(?P<version>\d+(\.\d+)*)(?P<rest>-.+\.whl)$'
-    return re.sub(pattern, rf'\g<distribution>-{new_version}\g<rest>', whl_name)
-
-
-def build_and_upload(source_path, endpoint_url, bucket_name, access_key, secret_key, base_url, virtualenv_path='.venv'):
-    # TODO 이미 빌드한 whl이 있으면 skip
-    subprocess.run(f'{virtualenv_path}/bin/python -m build --wheel', cwd=source_path, shell=True, check=True)
-
-    whl_file = next(Path(source_path, 'dist').glob('*.whl'), None)
-    checksum = calculate_checksum(whl_file)
-
-    # TODO subpath param으로 받기
-    s3_object_name = f'assets/{change_whl_version(whl_file.name, checksum)}'
-
-    upload_to_s3(str(whl_file), bucket_name, s3_object_name, endpoint_url, access_key, secret_key)
-    return f'{base_url}/{bucket_name}/{s3_object_name}'

synapse_sdk-1.0.0a0/synapse_sdk.egg-info/SOURCES.txt

@@ -1,59 +0,0 @@
-.gitignore
-.pre-commit-config.yaml
-LICENSE
-README.md
-pyproject.toml
-requirements.txt
-.github/workflows/lint.yml
-.github/workflows/pypi-publish.yml
-synapse/__init__.py
-synapse/config.py
-synapse/loggers.py
-synapse/client/__init__.py
-synapse/client/exceptions.py
-synapse/client/utils.py
-synapse/client/mixins/__init__.py
-synapse/client/mixins/annotation.py
-synapse/client/mixins/dataset.py
-synapse/client/mixins/integration.py
-synapse/client/mixins/ml.py
-synapse/plugins/__init__.py
-synapse/plugins/enums.py
-synapse/plugins/job.py
-synapse/plugins/upload.py
-synapse/plugins/utils.py
-synapse/plugins/categories/__init__.py
-synapse/plugins/categories/base.py
-synapse/plugins/categories/decorators.py
-synapse/plugins/categories/registry.py
-synapse/plugins/categories/data_validation/__init__.py
-synapse/plugins/categories/data_validation/actions/__init__.py
-synapse/plugins/categories/data_validation/actions/validation.py
-synapse/plugins/categories/export/__init__.py
-synapse/plugins/categories/export/actions/__init__.py
-synapse/plugins/categories/export/actions/export.py
-synapse/plugins/categories/import/__init__.py
-synapse/plugins/categories/import/actions/__init__.py
-synapse/plugins/categories/import/actions/import.py
-synapse/plugins/categories/neural_net/__init__.py
-synapse/plugins/categories/neural_net/actions/__init__.py
-synapse/plugins/categories/neural_net/actions/deployment.py
-synapse/plugins/categories/neural_net/actions/inference.py
-synapse/plugins/categories/neural_net/actions/test.py
-synapse/plugins/categories/neural_net/actions/train.py
-synapse/plugins/categories/post_annotation/__init__.py
-synapse/plugins/categories/post_annotation/actions/__init__.py
-synapse/plugins/categories/post_annotation/actions/post_annotation.py
-synapse/plugins/categories/pre_annotation/__init__.py
-synapse/plugins/categories/pre_annotation/actions/__init__.py
-synapse/plugins/categories/pre_annotation/actions/pre_annotation.py
-synapse/utils/__init__.py
-synapse/utils/debug.py
-synapse/utils/file.py
-synapse/utils/module_loading.py
-synapse/utils/string.py
-synapse_sdk.egg-info/PKG-INFO
-synapse_sdk.egg-info/SOURCES.txt
-synapse_sdk.egg-info/dependency_links.txt
-synapse_sdk.egg-info/requires.txt
-synapse_sdk.egg-info/top_level.txt

synapse_sdk-1.0.0a0/synapse_sdk.egg-info/top_level.txt

@@ -1 +0,0 @@
-synapse