synapse-sdk 1.0.0a7__tar.gz → 1.0.0a9__tar.gz

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/.gitignore

@@ -25,4 +25,4 @@ __pycache__/
 .cache
 
 # python
-main.py
+/main.py

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a7
+Version: 1.0.0a9
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/requirements.txt

@@ -6,4 +6,4 @@ tqdm
 pyyaml
 pydantic
 python-dotenv
-ray[all]==2.38.0
+ray[all]>=2.38.0

synapse_sdk-1.0.0a9/synapse_sdk/clients/agent/__init__.py

@@ -0,0 +1,23 @@
+from synapse_sdk.clients.agent.core import CoreClientMixin
+from synapse_sdk.clients.agent.service import ServiceClientMixin
+
+
+class AgentClient(CoreClientMixin, ServiceClientMixin):
+    name = 'Agent'
+    agent_token = None
+    user_token = None
+    tenant = None
+
+    def __init__(self, base_url, agent_token, user_token=None, tenant=None):
+        super().__init__(base_url)
+        self.agent_token = agent_token
+        self.user_token = user_token
+        self.tenant = tenant
+
+    def _get_headers(self):
+        headers = {'Authorization': self.agent_token}
+        if self.user_token:
+            headers['SYNAPSE-User'] = f'Token {self.user_token}'
+        if self.tenant:
+            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
+        return headers

synapse_sdk-1.0.0a9/synapse_sdk/clients/agent/core.py

@@ -0,0 +1,7 @@
+from synapse_sdk.clients.base import BaseClient
+
+
+class CoreClientMixin(BaseClient):
+    def health_check(self):
+        path = 'health/'
+        return self._get(path)

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/clients/backend/__init__.py

@@ -9,14 +9,15 @@ class BackendClient(AnnotationClientMixin, DatasetClientMixin, IntegrationClient
     token = None
     tenant = None
 
-    def __init__(self, base_url, token, tenant=None):
+    def __init__(self, base_url, token=None, tenant=None):
         super().__init__(base_url)
         self.token = token
-        if tenant:
-            self.tenant = tenant
+        self.tenant = tenant
 
     def _get_headers(self):
-        headers = {'Authorization': f'Token {self.token}'}
+        headers = {}
+        if self.token:
+            headers = {'Authorization': f'Token {self.token}'}
         if self.tenant:
             headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
         return headers

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/clients/backend/annotation.py

@@ -7,23 +7,23 @@ class AnnotationClientMixin(BaseClient):
         path = f'projects/{pk}/'
         return self._get(path)
 
-    def get_label_tag(self, pk):
-        path = f'label_tags/{pk}/'
+    def get_task_tag(self, pk):
+        path = f'task_tags/{pk}/'
         return self._get(path)
 
-    def list_label_tags(self, data):
-        path = 'label_tags/'
+    def list_task_tags(self, data):
+        path = 'task_tags/'
         return self._list(path, data=data)
 
-    def list_labels(self, data, url_conversion=None, list_all=False):
-        path = 'labels/'
+    def list_tasks(self, data, url_conversion=None, list_all=False):
+        path = 'tasks/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
 
-    def create_labels(self, data):
-        path = 'labels/'
+    def create_tasks(self, data):
+        path = 'tasks/'
         return self._post(path, data=data)
 
-    def set_tags_labels(self, data, params=None):
-        path = 'labels/set_tags/'
+    def set_tags_tasks(self, data, params=None):
+        path = 'tasks/set_tags/'
         return self._post(path, data=data, params=params)

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/clients/backend/dataset.py

@@ -33,15 +33,15 @@ class DatasetClientMixin(BaseClient):
             data_units = self.create_data_units(batch)
 
             if project_id:
-                labels_data = []
+                tasks_data = []
                 for data, data_unit in zip(batch, data_units):
-                    label_data = {'project': project_id, 'data_unit': data_unit['id']}
+                    task_data = {'project': project_id, 'data_unit': data_unit['id']}
                     if 'ground_truth' in data:
-                        label_data['ground_truth'] = data['ground_truth']
+                        task_data['ground_truth'] = data['ground_truth']
 
-                    labels_data.append(label_data)
+                    tasks_data.append(task_data)
 
-                self.create_labels(labels_data)
+                self.create_tasks(tasks_data)
 
     def import_data_file(self, data, dataset_id):
         for name, path in data['files'].items():

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/clients/backend/integration.py

@@ -2,6 +2,10 @@ from synapse_sdk.clients.base import BaseClient
 
 
 class IntegrationClientMixin(BaseClient):
+    def health_check_agent(self, token):
+        path = f'agents/{token}/connect/'
+        return self._post(path)
+
     def get_plugin(self, pk):
         path = f'plugins/{pk}/'
         return self._get(path)
@@ -27,6 +31,14 @@ class IntegrationClientMixin(BaseClient):
         files = {'file': data.pop('file')}
         return self._post(path, data=data, files=files)
 
+    def list_jobs(self, params=None):
+        path = 'jobs/'
+        return self._get(path, params=params)
+
+    def update_job(self, pk, data):
+        path = f'jobs/{pk}/'
+        return self._patch(path, data=data)
+
     def list_job_console_logs(self, pk):
         path = f'jobs/{pk}/console_logs/'
         return self._get(path)

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/clients/backend/ml.py

@@ -3,6 +3,10 @@ from synapse_sdk.clients.utils import get_default_url_conversion
 
 
 class MLClientMixin(BaseClient):
+    def list_models(self, params=None):
+        path = 'models/'
+        return self._get(path, params=params)
+
     def get_model(self, pk, params=None, url_conversion=None):
         path = f'models/{pk}/'
         url_conversion = get_default_url_conversion(
@@ -12,13 +16,10 @@ class MLClientMixin(BaseClient):
 
     def create_model(self, data):
         path = 'models/'
-        return self._post(path, data=data)
-
-    def update_model(self, pk, data, files=None):
-        path = f'models/{pk}/'
-        return self._patch(path, data=data, files=files)
+        files = {'file': data.pop('file')}
+        return self._post(path, data=data, files=files)
 
-    def list_train_dataset(self, params=None, url_conversion=None, list_all=False):
-        path = 'train_dataset/'
+    def list_ground_truth_events(self, params=None, url_conversion=None, list_all=False):
+        path = 'ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/clients/base.py

@@ -33,6 +33,9 @@ class BaseClient:
             if kwargs.get('files') is not None:
                 for name, file in kwargs['files'].items():
                     kwargs['files'][name] = Path(str(file)).open(mode='rb')
+                for name, value in kwargs['data'].items():
+                    if isinstance(value, dict):
+                        kwargs['data'][name] = json.dumps(value)
             else:
                 headers['Content-Type'] = 'application/json'
                 if 'data' in kwargs:
@@ -47,7 +50,10 @@ class BaseClient:
         except requests.ConnectionError:
             raise ClientError(408, f'{self.name} is not responding')
 
-        return response.json()
+        try:
+            return response.json()
+        except ValueError:
+            return response.text
 
     def _get(self, path, url_conversion=None, **kwargs):
         response = self._request('get', path, **kwargs)
@@ -79,3 +85,6 @@ class BaseClient:
         yield from response['results']
         if response['next']:
             yield from self._list_all(response['next'], url_conversion, **kwargs)
+
+    def exists(self, api, *args, **kwargs):
+        return getattr(self, api)(*args, **kwargs)['count'] > 0

synapse_sdk-1.0.0a9/synapse_sdk/loggers.py

@@ -0,0 +1,116 @@
+import datetime
+import time
+
+from synapse_sdk.clients.exceptions import ClientError
+
+
+class BaseLogger:
+    progress_record = {}
+    progress_categories = None
+    current_category = None
+    time_begin_per_category = {}
+
+    def __init__(self, progress_categories=None):
+        self.progress_categories = progress_categories
+        if progress_categories:
+            self.progress_record['categories'] = progress_categories
+
+    def set_progress(self, current, total, category=None):
+        assert 0 <= current <= total and total > 0
+        assert category is not None or 'categories' not in self.progress_record
+
+        percent = (current / total) * 100
+        percent = round(percent, 2)
+        # TODO current 0 으로 시작하지 않아도 작동되도록 수정
+        if current == 0:
+            self.time_begin_per_category[category] = time.time()
+            time_remaining = None
+        else:
+            seconds_per_item = (time.time() - self.time_begin_per_category[category]) / current
+            time_remaining = round(seconds_per_item * (total - current), 2)
+
+        current_progress = {'percent': percent, 'time_remaining': time_remaining}
+
+        if category:
+            self.current_category = category
+            self.progress_record['categories'][category].update(current_progress)
+        else:
+            self.progress_record.update(current_progress)
+
+    def get_current_progress(self):
+        categories = self.progress_record.get('categories')
+
+        if categories:
+            category_progress = None
+
+            overall = 0
+            for category, category_record in categories.items():
+                if category == self.current_category:
+                    break
+                overall += category_record['proportion']
+
+            category_record = categories[self.current_category]
+            category_percent = category_record.get('percent', 0)
+            if not category_progress and 'percent' in category_record:
+                category_progress = {
+                    'category': self.current_category,
+                    'percent': category_percent,
+                    'time_remaining': category_record.get('time_remaining'),
+                }
+            if category_percent > 0:
+                overall += round(category_record['proportion'] / 100 * category_percent, 2)
+            progress = {'overall': overall, **category_progress}
+        else:
+            progress = {
+                'overall': self.progress_record.get('percent'),
+                'time_remaining': self.progress_record.get('time_remaining'),
+            }
+
+        return progress
+
+
+class ConsoleLogger(BaseLogger):
+    def set_progress(self, current, total, category=None):
+        super().set_progress(current, total, category=category)
+        print(self.get_current_progress())
+
+    def log(self, action, data):
+        print(action, data)
+
+
+class BackendLogger(BaseLogger):
+    logs_queue = []
+    client = None
+    job_id = None
+
+    def __init__(self, client, job_id, **kwargs):
+        super().__init__(**kwargs)
+        self.client = client
+        self.job_id = job_id
+
+    def set_progress(self, current, total, category=None):
+        super().set_progress(current, total, category=category)
+        try:
+            progress_record = {
+                'record': self.progress_record,
+                'current_progress': self.get_current_progress(),
+            }
+            self.client.update_job(self.job_id, data={'progress_record': progress_record})
+        except ClientError:
+            pass
+
+    def log(self, action, data):
+        print(action, data)
+
+        log = {
+            'action': action,
+            'data': data,
+            'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
+            'job': self.job_id,
+        }
+        self.logs_queue.append(log)
+        try:
+            self.client.create_logs(self.logs_queue)
+            self.logs_queue.clear()
+        except ClientError as e:
+            print(e)

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/plugins/categories/base.py

@@ -6,25 +6,35 @@ from pprint import pprint
 
 import ray
 import requests
+from pydantic import ValidationError
 from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
 from synapse_sdk.plugins.enums import RunMethod
+from synapse_sdk.plugins.exceptions import ActionValidationError
 from synapse_sdk.plugins.models import PluginRelease, Run
 from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
 from synapse_sdk.utils.module_loading import import_string
+from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
 
 class Action:
+    # class 변수
     name = None
     category = None
     method = None
+    run_class = Run
+    params_model = None
+    progress_categories = None
+
+    # init 변수
     params = None
     plugin_config = None
     plugin_release = None
     config = None
-    client = None
-    debug = False
-    run_class = Run
+    job_id = None
+    direct = None
+    debug = None
+    envs = None
     run = None
 
     default_envs = [
@@ -57,10 +67,14 @@ class Action:
     def plugin_storage_url(self):
         return self.envs['SYNAPSE_PLUGIN_STORAGE']
 
+    @property
+    def client(self):
+        return self.run.client
+
     @property
     def plugin_url(self):
         if self.debug:
-            plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH', '.')
+            plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH') or '.'
             if plugin_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
                 plugin_url = plugin_path
             elif plugin_path.startswith('http://'):
@@ -85,6 +99,8 @@ class Action:
 
     def get_run(self):
         context = {
+            'plugin_release': self.plugin_release,
+            'progress_categories': self.progress_categories,
             'params': self.params,
             'envs': self.envs,
             'debug': self.debug,
@@ -110,12 +126,22 @@ class Action:
             pprint(runtime_env)
         return runtime_env
 
+    def validate_params(self):
+        if self.params_model:
+            try:
+                self.params_model.model_validate(self.params, context={'action': self})
+            except ValidationError as e:
+                raise ActionValidationError({'params': pydantic_to_drf_error(e)})
+
     def run_action(self):
+        self.validate_params()
         if self.direct:
             if self.method == RunMethod.RESTAPI:
                 return self.start_by_restapi()
             else:
-                return self.start()
+                result = self.start()
+                self.post_action_by_job(result)
+                return result
         return getattr(self, f'start_by_{self.method.value}')()
 
     def start(self):
@@ -175,3 +201,7 @@ class Action:
         response = getattr(requests, method)(url, **self.params)
         # TODO ok response가 아닌 경우 대응하기
         return response.json()
+
+    def post_action_by_job(self, result):
+        if self.client:
+            self.client.update_job(self.job_id, data={'result': result})

synapse_sdk-1.0.0a9/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -0,0 +1,142 @@
+import copy
+import tempfile
+from decimal import Decimal
+from pathlib import Path
+from typing import Annotated
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.utils.file import archive
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+
+class TrainRun(Run):
+    def log_metric(self, x, i, **kwargs):
+        self.log(x, {x: i, **kwargs})
+
+
+class Hyperparameter(BaseModel):
+    batch_size: int
+    iterations: int
+    learning_rate: Decimal
+
+
+class TrainParams(BaseModel):
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str
+    checkpoint: int | None
+    dataset: int
+    hyperparameter: Hyperparameter
+
+    @field_validator('name')
+    @staticmethod
+    def unique_name(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            model_exists = client.exists('list_models', params={'name': value})
+            job_exists = client.exists(
+                'list_jobs',
+                params={
+                    'ids_ex': action.job_id,
+                    'category': 'neural_net',
+                    'action': 'train',
+                    'is_active': True,
+                    'params': f'name:{value}',
+                },
+            )
+            assert not model_exists and not job_exists, '존재하는 학습 이름입니다.'
+        except ClientError:
+            raise PydanticCustomError('client_error', '')
+        return value
+
+
+@register_action
+class TrainAction(Action):
+    name = 'train'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
+    run_class = TrainRun
+    params_model = TrainParams
+    progress_categories = {
+        'dataset': {
+            'proportion': 20,
+        },
+        'train': {
+            'proportion': 75,
+        },
+        'model_upload': {
+            'proportion': 5,
+        },
+    }
+
+    def start(self):
+        hyperparameter = self.params['hyperparameter']
+
+        # download dataset
+        self.run.log_event('Preparing dataset for training.')
+        input_dataset = self.get_dataset()
+
+        # train dataset
+        self.run.log_event('Starting model training.')
+
+        result = self.entrypoint(self.run, input_dataset, hyperparameter)
+
+        # upload model_data
+        self.run.log_event('Registering model data.')
+        self.run.set_progress(0, 1, category='model_upload')
+        model = self.create_model(result)
+        self.run.set_progress(1, 1, category='model_upload')
+
+        self.run.end_log()
+        return {'model_id': model['id']}
+
+    def get_dataset(self):
+        client = self.run.client
+        assert bool(client)
+
+        input_dataset = {}
+
+        ground_truths, count_dataset = client.list_ground_truth_events(
+            params={
+                'fields': ['category', 'files', 'data'],
+                'ground_truth_dataset_versions': self.params['dataset'],
+            },
+            list_all=True,
+        )
+        self.run.set_progress(0, count_dataset, category='dataset')
+        for i, ground_truth in enumerate(ground_truths, start=1):
+            self.run.set_progress(i, count_dataset, category='dataset')
+            try:
+                input_dataset[ground_truth['category']].append(ground_truth)
+            except KeyError:
+                input_dataset[ground_truth['category']] = [ground_truth]
+
+        return input_dataset
+
+    def create_model(self, path):
+        if not self.client:
+            print(path)
+
+        params = copy.deepcopy(self.params)
+        configuration_fields = ['hyperparameter']
+        configuration = {field: params.pop(field) for field in configuration_fields}
+
+        with tempfile.TemporaryDirectory() as temp_path:
+            input_path = Path(path)
+            archive_path = Path(temp_path, 'archive.zip')
+            archive(input_path, archive_path)
+
+            return self.client.create_model({
+                'plugin': self.plugin_release.plugin,
+                'version': self.plugin_release.version,
+                'file': str(archive_path),
+                'configuration': configuration,
+                **params,
+            })

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/plugins/categories/neural_net/templates/plugin/train.py

@@ -9,6 +9,6 @@ def train(run, dataset, hyperparameter, checkpoint=None):
         loss = float(round((count_iterations - i) / count_iterations, 2))
         miou = 1 - loss
         run.log_metric('iteration', i, loss=loss, miou=miou)
-        run.set_progress(i, count_iterations, category='iteration')
+        run.set_progress(i, count_iterations, category='train')
 
     return {'weight': '/tmp/agent/test/a.txt', 'config': '/tmp/agent/test/b.txt'}

synapse_sdk-1.0.0a9/synapse_sdk/plugins/categories/smart_tool/actions/auto_label.py

@@ -0,0 +1,22 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+
+
+@register_action
+class AutoLabelAction(Action):
+    name = 'label'
+    category = PluginCategory.SMART_TOOL
+    method = RunMethod.TASK
+
+    def get_auto_label(self):
+        return self.entrypoint(**self.params)
+
+    def run_model(self, input_data):
+        return {}
+
+    def start(self):
+        auto_label = self.get_auto_label()
+        input_data = auto_label.handle_input(self.params['input_data'])
+        output_data = self.run_model(input_data)
+        return auto_label.handle_output(output_data)

synapse_sdk-1.0.0a9/synapse_sdk/plugins/categories/smart_tool/templates/config.yaml

@@ -0,0 +1,6 @@
+actions:
+  auto_label:
+    category: interactive_segmentation
+    entrypoint: plugin.auto_label.MyAutoLabel
+    model:
+      neural_nets: [sam2]

synapse_sdk-1.0.0a9/synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py

@@ -0,0 +1,11 @@
+class MyAutoLabel:
+    def __init__(self, **kwargs):
+        pass
+
+    def handle_input(self, input_data):
+        """smart tool의 input을 model의 input 형태로 변환"""
+        return
+
+    def handle_output(self, output_data):
+        """model의 output을 smart tool의 output 형태로 변환"""
+        return

synapse_sdk-1.0.0a9/synapse_sdk/plugins/exceptions.py

@@ -0,0 +1,6 @@
+class ActionValidationError(Exception):
+    errors = None
+
+    def __init__(self, errors, *args):
+        self.errors = errors
+        super().__init__(errors, *args)

{synapse_sdk-1.0.0a7 → synapse_sdk-1.0.0a9}/synapse_sdk/plugins/models.py

@@ -11,12 +11,14 @@ from synapse_sdk.utils.string import hash_text
 
 class PluginRelease:
     config: Dict[str, Any]
+    envs = None
 
-    def __init__(self, config=None, plugin_path=None):
+    def __init__(self, config=None, plugin_path=None, envs=None):
         if config:
             self.config = config
         else:
             self.config = read_plugin_config(plugin_path=plugin_path)
+        self.envs = envs
 
     @cached_property
     def plugin(self):
@@ -72,15 +74,20 @@ class Run:
         self.set_logger()
 
     def set_logger(self):
+        kwargs = {'progress_categories': self.context['progress_categories']}
         if self.job_id:
             client = BackendClient(
                 self.context['envs']['SYNAPSE_PLUGIN_RUN_HOST'],
                 self.context['envs']['SYNAPSE_PLUGIN_RUN_USER_TOKEN'],
                 self.context['envs']['SYNAPSE_PLUGIN_RUN_TENANT'],
             )
-            self.logger = BackendLogger(client, self.job_id)
+            self.logger = BackendLogger(client, self.job_id, **kwargs)
         else:
-            self.logger = ConsoleLogger()
+            self.logger = ConsoleLogger(**kwargs)
+
+    @property
+    def client(self):
+        return getattr(self.logger, 'client', None)
 
     def set_progress(self, current, total, category=''):
         self.logger.set_progress(current, total, category)