synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl

synapse_sdk/plugins/categories/base.py

@@ -1,235 +0,0 @@
-import inspect
-import json
-import os
-from functools import cached_property
-from pprint import pprint
-
-import ray
-import requests
-from pydantic import ValidationError
-from ray.dashboard.modules.job.sdk import JobSubmissionClient
-from ray.exceptions import RayTaskError
-
-from synapse_sdk.clients.ray import RayClient
-from synapse_sdk.plugins.enums import RunMethod
-from synapse_sdk.plugins.exceptions import ActionError
-from synapse_sdk.plugins.models import PluginRelease, Run
-from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
-from synapse_sdk.utils.module_loading import import_string
-from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
-
-
-class Action:
-    # class 변수
-    name = None
-    category = None
-    method = None
-    run_class = Run
-    params_model = None
-    progress_categories = None
-
-    # init 변수
-    params = None
-    plugin_config = None
-    plugin_release = None
-    config = None
-    job_id = None
-    direct = None
-    debug = None
-    envs = None
-    run = None
-
-    default_envs = [
-        'RAY_ADDRESS',
-        'RAY_DASHBOARD_URL',
-        'RAY_SERVE_ADDRESS',
-        'SYNAPSE_PLUGIN_STORAGE',
-        'SYNAPSE_DEBUG_PLUGIN_PATH',
-        'SYNAPSE_DEBUG_MODULES',
-        'SYNAPSE_PLUGIN_RUN_HOST',
-        'SYNAPSE_PLUGIN_RUN_USER_TOKEN',
-        'SYNAPSE_PLUGIN_RUN_TENANT',
-    ]
-
-    def __init__(self, params, plugin_config, envs=None, job_id=None, direct=False, debug=False):
-        self.params = params
-        self.plugin_config = plugin_config
-        self.plugin_release = PluginRelease(config=plugin_config)
-        self.config = self.plugin_release.get_action_config(self.name)
-        self.job_id = job_id
-        self.direct = direct
-        self.debug = debug
-        self.envs = {**self.get_default_envs(), **envs} if envs else self.get_default_envs()
-        self.run = self.get_run()
-
-    @cached_property
-    def entrypoint(self):
-        return import_string(self.config['entrypoint'])
-
-    @property
-    def plugin_storage_url(self):
-        return self.envs['SYNAPSE_PLUGIN_STORAGE']
-
-    @property
-    def client(self):
-        return self.run.client
-
-    @property
-    def ray_client(self):
-        return RayClient(self.envs['RAY_DASHBOARD_URL'])
-
-    @property
-    def plugin_url(self):
-        if self.debug:
-            plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH') or '.'
-            if plugin_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                plugin_url = plugin_path
-            elif plugin_path.startswith('http://'):
-                plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
-            else:
-                plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
-            self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
-            return plugin_url
-        return self.plugin_release.get_url(self.plugin_storage_url)
-
-    @property
-    def debug_modules(self):
-        debug_modules = []
-        for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
-            if module_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                module_url = module_path
-            else:
-                module_url = build_and_upload(module_path, self.plugin_storage_url)
-            debug_modules.append(module_url)
-        self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
-        return debug_modules
-
-    def get_run(self):
-        context = {
-            'plugin_release': self.plugin_release,
-            'progress_categories': self.progress_categories,
-            'params': self.params,
-            'envs': self.envs,
-            'debug': self.debug,
-        }
-        return self.run_class(self.job_id, context)
-
-    def get_default_envs(self):
-        return {env: os.environ[env] for env in self.default_envs if env in os.environ}
-
-    def get_runtime_env(self):
-        runtime_env = {
-            'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
-            'working_dir': self.plugin_url,
-        }
-
-        if self.debug:
-            runtime_env['pip'] += self.debug_modules
-
-        # 맨 마지막에 진행되어야 함
-        runtime_env['env_vars'] = self.envs
-
-        if self.debug:
-            pprint(runtime_env)
-        return runtime_env
-
-    def validate_params(self):
-        if self.params_model:
-            try:
-                self.params_model.model_validate(self.params, context={'action': self})
-            except ValidationError as e:
-                raise ActionError({'params': pydantic_to_drf_error(e)})
-
-    def run_action(self):
-        self.validate_params()
-
-        if not ray.is_initialized():
-            ray.init(address=self.envs['RAY_ADDRESS'], ignore_reinit_error=True)
-
-        if self.direct:
-            if self.method == RunMethod.RESTAPI:
-                return self.start_by_restapi()
-            else:
-                result = self.start()
-                self.post_action_by_job(result)
-                return result
-        return getattr(self, f'start_by_{self.method.value}')()
-
-    def start(self):
-        if self.method == RunMethod.JOB:
-            return self.entrypoint(self.run, **self.params)
-        return self.entrypoint(**self.params)
-
-    def start_by_task(self):
-        @ray.remote(runtime_env=self.get_runtime_env())
-        def run_task(category, action, *args, **kwargs):
-            from synapse_sdk.plugins.utils import get_action_class
-
-            action = get_action_class(category, action)(*args, **kwargs)
-            return action.run_action()
-
-        init_signature = inspect.signature(self.__class__.__init__)
-
-        args = []
-        kwargs = {}
-
-        for param in init_signature.parameters.values():
-            if param.name == 'self':
-                continue
-            if param.default == param.empty:
-                args.append(getattr(self, param.name))
-            else:
-                kwargs[param.name] = getattr(self, param.name)
-
-        kwargs['direct'] = True
-        try:
-            return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
-        except RayTaskError as e:
-            raise ActionError(e.cause)
-
-    def start_by_job(self):
-        main_options = []
-        options = ['run', '--direct']
-        arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
-
-        if self.debug:
-            main_options.append('--debug')
-
-        if self.job_id:
-            options.append(f'--job-id={self.job_id}')
-
-        cmd = ' '.join(main_options + options + arguments)
-
-        client = self.get_job_client()
-        return client.submit_job(
-            submission_id=self.job_id,
-            entrypoint=f'python main.py {cmd}',
-            runtime_env=self.get_runtime_env(),
-        )
-
-    def start_by_restapi(self):
-        path = self.params.pop('path', '')
-        method = self.params.pop('method')
-
-        url = self.plugin_release.get_serve_url(self.envs['RAY_SERVE_ADDRESS'], path)
-        try:
-            response = getattr(requests, method)(url, **self.params)
-            try:
-                response_body = response.json()
-            except ValueError:
-                response_body = response.text
-            if response.ok:
-                return response_body
-            else:
-                raise ActionError({'status': response.status_code, 'reason': response.reason, 'message': response_body})
-        except requests.ConnectionError:
-            raise ActionError('Unable to connect to serve application')
-
-    def post_action_by_job(self, result):
-        if self.client:
-            job_client = self.get_job_client()
-            logs = job_client.get_job_logs(self.job_id).split('\n')
-            self.client.update_job(self.job_id, data={'result': result or {}, 'console_logs': logs})
-
-    def get_job_client(self):
-        return JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))

synapse_sdk/plugins/categories/data_validation/actions/validation.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class ValidationAction(Action):
-    name = 'validation'
-    category = PluginCategory.DATA_VALIDATION
-    method = RunMethod.TASK

synapse_sdk/plugins/categories/data_validation/templates/config.yaml

@@ -1,3 +0,0 @@
-actions:
-  validation:
-    entrypoint: plugin.validation.validate

synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py

@@ -1,5 +0,0 @@
-def validate(data, assignment_id, **kwargs):
-    errors = []
-    if data.get('errors'):
-        errors += data['errors']
-    return errors

synapse_sdk/plugins/categories/decorators.py

@@ -1,13 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS
-
-
-def register_action(action_class):
-    if not issubclass(action_class, Action):
-        raise ValueError('Wrapped class must subclass Action class.')
-
-    try:
-        _REGISTERED_ACTIONS[action_class.category.value][action_class.name] = action_class
-    except KeyError:
-        _REGISTERED_ACTIONS[action_class.category.value] = {action_class.name: action_class}
-    return action_class

synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class ExportAction(Action):
-    name = 'export'
-    category = PluginCategory.EXPORT
-    method = RunMethod.JOB

synapse_sdk/plugins/categories/import/actions/import.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class ImportAction(Action):
-    name = 'import'
-    category = PluginCategory.IMPORT
-    method = RunMethod.JOB

synapse_sdk/plugins/categories/neural_net/actions/deployment.py

@@ -1,45 +0,0 @@
-from ray import serve
-
-from synapse_sdk.clients.exceptions import ClientError
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class DeploymentAction(Action):
-    name = 'deployment'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.JOB
-
-    def get_deployment(self):
-        return serve.deployment(ray_actor_options=self.get_actor_options())(self.entrypoint)
-
-    def get_actor_options(self):
-        return {'runtime_env': self.get_runtime_env()}
-
-    def start(self):
-        deployment = self.get_deployment()
-        serve.delete(self.plugin_release.code)
-        # TODO add run object
-        serve.run(deployment.bind(), name=self.plugin_release.code, route_prefix=f'/{self.plugin_release.checksum}')
-
-        # 백엔드에 ServeApplication 추가
-        serve_application = self.create_serve_application()
-        return {'serve_application': serve_application['id'] if serve_application else None}
-
-    def create_serve_application(self):
-        if self.client:
-            try:
-                job = self.client.get_job(self.job_id)
-                serve_application = self.ray_client.get_serve_application(self.plugin_release.code)
-                return self.client.create_serve_application({
-                    'plugin': self.plugin_release.plugin,
-                    'version': self.plugin_release.version,
-                    'agent': job['agent'],
-                    'status': serve_application['status'],
-                    'data': serve_application,
-                })
-            except ClientError:
-                pass
-        return None

synapse_sdk/plugins/categories/neural_net/actions/inference.py

@@ -1,18 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class InferenceAction(Action):
-    name = 'inference'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.RESTAPI
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        headers = {'serve_multiplexed_model_id': str(self.params.pop('model'))}
-        if 'headers' in self.params:
-            self.params['headers'].update(headers)
-        else:
-            self.params['headers'] = headers

synapse_sdk/plugins/categories/neural_net/actions/test.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class TestAction(Action):
-    name = 'test'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.JOB

synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -1,143 +0,0 @@
-import copy
-import tempfile
-from decimal import Decimal
-from pathlib import Path
-from typing import Annotated
-
-from pydantic import AfterValidator, BaseModel, field_validator
-from pydantic_core import PydanticCustomError
-
-from synapse_sdk.clients.exceptions import ClientError
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-from synapse_sdk.plugins.models import Run
-from synapse_sdk.utils.file import archive
-from synapse_sdk.utils.pydantic.validators import non_blank
-
-
-class TrainRun(Run):
-    def log_metric(self, key, value, **metrics):
-        self.log('metric', {'key': key, 'value': value, 'metrics': metrics})
-
-
-class Hyperparameter(BaseModel):
-    batch_size: int
-    iterations: int
-    learning_rate: Decimal
-
-
-class TrainParams(BaseModel):
-    name: Annotated[str, AfterValidator(non_blank)]
-    description: str
-    checkpoint: int | None
-    dataset: int
-    hyperparameter: Hyperparameter
-
-    @field_validator('name')
-    @staticmethod
-    def unique_name(value, info):
-        action = info.context['action']
-        client = action.client
-        try:
-            model_exists = client.exists('list_models', params={'name': value})
-            job_exists = client.exists(
-                'list_jobs',
-                params={
-                    'ids_ex': action.job_id,
-                    'category': 'neural_net',
-                    'action': 'train',
-                    'is_active': True,
-                    'params': f'name:{value}',
-                },
-            )
-            assert not model_exists and not job_exists, '존재하는 학습 이름입니다.'
-        except ClientError:
-            raise PydanticCustomError('client_error', '')
-        return value
-
-
-@register_action
-class TrainAction(Action):
-    name = 'train'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.JOB
-    run_class = TrainRun
-    params_model = TrainParams
-    progress_categories = {
-        'dataset': {
-            'proportion': 20,
-        },
-        'train': {
-            'proportion': 75,
-        },
-        'model_upload': {
-            'proportion': 5,
-        },
-    }
-
-    def start(self):
-        hyperparameter = self.params['hyperparameter']
-
-        # download dataset
-        self.run.log_message('Preparing dataset for training.')
-        input_dataset = self.get_dataset()
-
-        # train dataset
-        self.run.log_message('Starting model training.')
-
-        result = self.entrypoint(self.run, input_dataset, hyperparameter)
-
-        # upload model_data
-        self.run.log_message('Registering model data.')
-        self.run.set_progress(0, 1, category='model_upload')
-        model = self.create_model(result)
-        self.run.set_progress(1, 1, category='model_upload')
-
-        self.run.end_log()
-        return {'model_id': model['id'] if model else None}
-
-    def get_dataset(self):
-        client = self.run.client
-        assert bool(client)
-
-        input_dataset = {}
-
-        ground_truths, count_dataset = client.list_ground_truth_events(
-            params={
-                'fields': ['category', 'files', 'data'],
-                'ground_truth_dataset_versions': self.params['dataset'],
-            },
-            list_all=True,
-        )
-        self.run.set_progress(0, count_dataset, category='dataset')
-        for i, ground_truth in enumerate(ground_truths, start=1):
-            self.run.set_progress(i, count_dataset, category='dataset')
-            try:
-                input_dataset[ground_truth['category']].append(ground_truth)
-            except KeyError:
-                input_dataset[ground_truth['category']] = [ground_truth]
-
-        return input_dataset
-
-    def create_model(self, path):
-        if not self.client:
-            print(path)
-            return None
-
-        params = copy.deepcopy(self.params)
-        configuration_fields = ['hyperparameter']
-        configuration = {field: params.pop(field) for field in configuration_fields}
-
-        with tempfile.TemporaryDirectory() as temp_path:
-            input_path = Path(path)
-            archive_path = Path(temp_path, 'archive.zip')
-            archive(input_path, archive_path)
-
-            return self.client.create_model({
-                'plugin': self.plugin_release.plugin,
-                'version': self.plugin_release.version,
-                'file': str(archive_path),
-                'configuration': configuration,
-                **params,
-            })

synapse_sdk/plugins/categories/neural_net/templates/config.yaml

@@ -1,12 +0,0 @@
-actions:
-  train:
-    dataset: datamaker
-    entrypoint: plugin.train.train
-  deployment:
-    entrypoint: plugin.inference.InferenceDeployment
-  inference:
-    method: restapi
-    endpoints:
-      - method: get
-  test:
-    entrypoint: plugin.test.test

synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py

@@ -1,4 +0,0 @@
-class InferenceDeployment:
-    def __call__(self):
-        message = 'hello datamaker'
-        return {'message': message}

synapse_sdk/plugins/categories/neural_net/templates/plugin/test.py

@@ -1,2 +0,0 @@
-def test(run):
-    return 'this is neural net test!'

synapse_sdk/plugins/categories/neural_net/templates/plugin/train.py

@@ -1,14 +0,0 @@
-import time
-
-
-def train(run, dataset, hyperparameter, checkpoint=None):
-    count_iterations = hyperparameter['iterations']
-
-    for i in range(1, count_iterations + 1):
-        time.sleep(0.5)
-        loss = float(round((count_iterations - i) / count_iterations, 2))
-        miou = 1 - loss
-        run.log_metric('iteration', i, loss=loss, miou=miou)
-        run.set_progress(i, count_iterations, category='train')
-
-    return {'weight': '/tmp/agent/test/a.txt', 'config': '/tmp/agent/test/b.txt'}

synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class PostAnnotationAction(Action):
-    name = 'post_annotation'
-    category = PluginCategory.POST_ANNOTATION
-    method = RunMethod.TASK

synapse_sdk/plugins/categories/post_annotation/templates/config.yaml

@@ -1,3 +0,0 @@
-actions:
-  post_annotation:
-    entrypoint: plugin.post_annotation.post_annotate

synapse_sdk/plugins/categories/post_annotation/templates/plugin/post_annotation.py

@@ -1,3 +0,0 @@
-def post_annotate(data, assignment_id, **kwargs):
-    data['post_annotate'] = 'hello world'
-    return data

synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class PreAnnotationAction(Action):
-    name = 'pre_annotation'
-    category = PluginCategory.PRE_ANNOTATION
-    method = RunMethod.TASK

synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml

@@ -1,3 +0,0 @@
-actions:
-  pre_annotation:
-    entrypoint: plugin.pre_annotation.pre_annotate

synapse_sdk/plugins/categories/pre_annotation/templates/plugin/pre_annotation.py

@@ -1,3 +0,0 @@
-def pre_annotate(data, assignment_id, **kwargs):
-    data['pre_annotate'] = 'hello world'
-    return data

synapse_sdk/plugins/categories/registry.py

@@ -1,16 +0,0 @@
-import pkgutil
-from importlib import import_module
-
-from synapse_sdk.plugins.enums import PluginCategory
-
-_REGISTERED_ACTIONS = {}
-
-
-def register_actions():
-    if not _REGISTERED_ACTIONS:
-        for category in PluginCategory:
-            plugin_category_module_name = f'synapse_sdk.plugins.categories.{category.value}.actions'
-            plugin_category_module = import_module(plugin_category_module_name)
-            for _, action_name, _ in pkgutil.iter_modules(plugin_category_module.__path__):
-                action_module_name = f'{plugin_category_module_name}.{action_name}'
-                import_module(action_module_name)