synapse-sdk 1.0.0a6__tar.gz → 1.0.0a8__tar.gz

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a6
+Version: 1.0.0a8
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/pyproject.toml

@@ -38,7 +38,7 @@ synapse = 'synapse_sdk.cli:cli'
 line-length = 120
 
 [tool.ruff.lint]
-select = ['E', 'F', 'Q']
+select = ['E', 'F', 'I', 'Q']
 ignore = ['W191', 'E111', 'E114', 'E117', 'D206', 'D300', 'Q000', 'Q001', 'Q002', 'Q003', 'COM812', 'COM819', 'ISC001', 'ISC002']
 
 [tool.ruff.lint.pydocstyle]

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/clients/backend/annotation.py

@@ -7,23 +7,23 @@ class AnnotationClientMixin(BaseClient):
         path = f'projects/{pk}/'
         return self._get(path)
 
-    def get_label_tag(self, pk):
-        path = f'label_tags/{pk}/'
+    def get_task_tag(self, pk):
+        path = f'task_tags/{pk}/'
         return self._get(path)
 
-    def list_label_tags(self, data):
-        path = 'label_tags/'
+    def list_task_tags(self, data):
+        path = 'task_tags/'
         return self._list(path, data=data)
 
-    def list_labels(self, data, url_conversion=None, list_all=False):
-        path = 'labels/'
+    def list_tasks(self, data, url_conversion=None, list_all=False):
+        path = 'tasks/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
 
-    def create_labels(self, data):
-        path = 'labels/'
+    def create_tasks(self, data):
+        path = 'tasks/'
         return self._post(path, data=data)
 
-    def set_tags_labels(self, data, params=None):
-        path = 'labels/set_tags/'
+    def set_tags_tasks(self, data, params=None):
+        path = 'tasks/set_tags/'
         return self._post(path, data=data, params=params)

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/clients/backend/dataset.py

@@ -1,4 +1,5 @@
 from multiprocessing import Pool
+
 from tqdm import tqdm
 
 from synapse_sdk.clients.base import BaseClient
@@ -32,15 +33,15 @@ class DatasetClientMixin(BaseClient):
             data_units = self.create_data_units(batch)
 
             if project_id:
-                labels_data = []
+                tasks_data = []
                 for data, data_unit in zip(batch, data_units):
-                    label_data = {'project': project_id, 'data_unit': data_unit['id']}
+                    task_data = {'project': project_id, 'data_unit': data_unit['id']}
                     if 'ground_truth' in data:
-                        label_data['ground_truth'] = data['ground_truth']
+                        task_data['ground_truth'] = data['ground_truth']
 
-                    labels_data.append(label_data)
+                    tasks_data.append(task_data)
 
-                self.create_labels(labels_data)
+                self.create_tasks(tasks_data)
 
     def import_data_file(self, data, dataset_id):
         for name, path in data['files'].items():

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/clients/backend/integration.py

@@ -27,6 +27,14 @@ class IntegrationClientMixin(BaseClient):
         files = {'file': data.pop('file')}
         return self._post(path, data=data, files=files)
 
+    def list_jobs(self, params=None):
+        path = 'jobs/'
+        return self._get(path, params=params)
+
+    def update_job(self, pk, data):
+        path = f'jobs/{pk}/'
+        return self._patch(path, data=data)
+
     def list_job_console_logs(self, pk):
         path = f'jobs/{pk}/console_logs/'
         return self._get(path)

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/clients/backend/ml.py

@@ -3,6 +3,10 @@ from synapse_sdk.clients.utils import get_default_url_conversion
 
 
 class MLClientMixin(BaseClient):
+    def list_models(self, params=None):
+        path = 'models/'
+        return self._get(path, params=params)
+
     def get_model(self, pk, params=None, url_conversion=None):
         path = f'models/{pk}/'
         url_conversion = get_default_url_conversion(
@@ -12,13 +16,10 @@ class MLClientMixin(BaseClient):
 
     def create_model(self, data):
         path = 'models/'
-        return self._post(path, data=data)
-
-    def update_model(self, pk, data, files=None):
-        path = f'models/{pk}/'
-        return self._patch(path, data=data, files=files)
+        files = {'file': data.pop('file')}
+        return self._post(path, data=data, files=files)
 
-    def list_train_dataset(self, params=None, url_conversion=None, list_all=False):
-        path = 'train_dataset/'
+    def list_ground_truth_events(self, params=None, url_conversion=None, list_all=False):
+        path = 'ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/clients/base.py

@@ -1,5 +1,5 @@
-import os
 import json
+import os
 from pathlib import Path
 
 import requests
@@ -33,6 +33,9 @@ class BaseClient:
             if kwargs.get('files') is not None:
                 for name, file in kwargs['files'].items():
                     kwargs['files'][name] = Path(str(file)).open(mode='rb')
+                for name, value in kwargs['data'].items():
+                    if isinstance(value, dict):
+                        kwargs['data'][name] = json.dumps(value)
             else:
                 headers['Content-Type'] = 'application/json'
                 if 'data' in kwargs:
@@ -79,3 +82,6 @@ class BaseClient:
         yield from response['results']
         if response['next']:
             yield from self._list_all(response['next'], url_conversion, **kwargs)
+
+    def exists(self, api, *args, **kwargs):
+        return getattr(self, api)(*args, **kwargs)['count'] > 0

synapse_sdk-1.0.0a8/synapse_sdk/loggers.py

@@ -0,0 +1,116 @@
+import datetime
+import time
+
+from synapse_sdk.clients.exceptions import ClientError
+
+
+class BaseLogger:
+    progress_record = {}
+    progress_categories = None
+    current_category = None
+    time_begin_per_category = {}
+
+    def __init__(self, progress_categories=None):
+        self.progress_categories = progress_categories
+        if progress_categories:
+            self.progress_record['categories'] = progress_categories
+
+    def set_progress(self, current, total, category=None):
+        assert 0 <= current <= total and total > 0
+        assert category is not None or 'categories' not in self.progress_record
+
+        percent = (current / total) * 100
+        percent = round(percent, 2)
+        # TODO current 0 으로 시작하지 않아도 작동되도록 수정
+        if current == 0:
+            self.time_begin_per_category[category] = time.time()
+            time_remaining = None
+        else:
+            seconds_per_item = (time.time() - self.time_begin_per_category[category]) / current
+            time_remaining = round(seconds_per_item * (total - current), 2)
+
+        current_progress = {'percent': percent, 'time_remaining': time_remaining}
+
+        if category:
+            self.current_category = category
+            self.progress_record['categories'][category].update(current_progress)
+        else:
+            self.progress_record.update(current_progress)
+
+    def get_current_progress(self):
+        categories = self.progress_record.get('categories')
+
+        if categories:
+            category_progress = None
+
+            overall = 0
+            for category, category_record in categories.items():
+                if category == self.current_category:
+                    break
+                overall += category_record['proportion']
+
+            category_record = categories[self.current_category]
+            category_percent = category_record.get('percent', 0)
+            if not category_progress and 'percent' in category_record:
+                category_progress = {
+                    'category': self.current_category,
+                    'percent': category_percent,
+                    'time_remaining': category_record.get('time_remaining'),
+                }
+            if category_percent > 0:
+                overall += round(category_record['proportion'] / 100 * category_percent, 2)
+            progress = {'overall': overall, **category_progress}
+        else:
+            progress = {
+                'overall': self.progress_record.get('percent'),
+                'time_remaining': self.progress_record.get('time_remaining'),
+            }
+
+        return progress
+
+
+class ConsoleLogger(BaseLogger):
+    def set_progress(self, current, total, category=None):
+        super().set_progress(current, total, category=category)
+        print(self.get_current_progress())
+
+    def log(self, action, data):
+        print(action, data)
+
+
+class BackendLogger(BaseLogger):
+    logs_queue = []
+    client = None
+    job_id = None
+
+    def __init__(self, client, job_id, **kwargs):
+        super().__init__(**kwargs)
+        self.client = client
+        self.job_id = job_id
+
+    def set_progress(self, current, total, category=None):
+        super().set_progress(current, total, category=category)
+        try:
+            progress_record = {
+                'record': self.progress_record,
+                'current_progress': self.get_current_progress(),
+            }
+            self.client.update_job(self.job_id, data={'progress_record': progress_record})
+        except ClientError:
+            pass
+
+    def log(self, action, data):
+        print(action, data)
+
+        log = {
+            'action': action,
+            'data': data,
+            'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
+            'job': self.job_id,
+        }
+        self.logs_queue.append(log)
+        try:
+            self.client.create_logs(self.logs_queue)
+            self.logs_queue.clear()
+        except ClientError as e:
+            print(e)

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/base.py

@@ -6,27 +6,36 @@ from pprint import pprint
 
 import ray
 import requests
+from pydantic import ValidationError
 from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
-from synapse_sdk.clients.backend import BackendClient
-from synapse_sdk.loggers import ConsoleLogger, BackendLogger
 from synapse_sdk.plugins.enums import RunMethod
-from synapse_sdk.plugins.models import PluginRelease
-from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload, download_and_upload
+from synapse_sdk.plugins.exceptions import ActionValidationError
+from synapse_sdk.plugins.models import PluginRelease, Run
+from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
 from synapse_sdk.utils.module_loading import import_string
+from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
 
 class Action:
+    # class 변수
     name = None
     category = None
     method = None
+    run_class = Run
+    params_model = None
+    progress_categories = None
+
+    # init 변수
     params = None
     plugin_config = None
     plugin_release = None
     config = None
-    client = None
-    logger = None
-    debug = False
+    job_id = None
+    direct = None
+    debug = None
+    envs = None
+    run = None
 
     default_envs = [
         'RAY_DASHBOARD_URL',
@@ -48,7 +57,7 @@ class Action:
         self.direct = direct
         self.debug = debug
         self.envs = {**envs, **self.get_default_envs()} if envs else self.get_default_envs()
-        self.set_logger()
+        self.run = self.get_run()
 
     @cached_property
     def entrypoint(self):
@@ -58,6 +67,10 @@ class Action:
     def plugin_storage_url(self):
         return self.envs['SYNAPSE_PLUGIN_STORAGE']
 
+    @property
+    def client(self):
+        return self.run.client
+
     @property
     def plugin_url(self):
         if self.debug:
@@ -84,16 +97,15 @@ class Action:
         self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
-    def set_logger(self):
-        if self.job_id:
-            client = BackendClient(
-                self.envs['SYNAPSE_PLUGIN_RUN_HOST'],
-                self.envs['SYNAPSE_PLUGIN_RUN_USER_TOKEN'],
-                self.envs['SYNAPSE_PLUGIN_RUN_TENANT'],
-            )
-            self.logger = BackendLogger(client, self.job_id)
-        else:
-            self.logger = ConsoleLogger()
+    def get_run(self):
+        context = {
+            'plugin_release': self.plugin_release,
+            'progress_categories': self.progress_categories,
+            'params': self.params,
+            'envs': self.envs,
+            'debug': self.debug,
+        }
+        return self.run_class(self.job_id, context)
 
     def get_default_envs(self):
         return {env: os.environ[env] for env in self.default_envs if env in os.environ}
@@ -105,7 +117,7 @@ class Action:
         }
 
         if self.debug:
-            runtime_env['pip'] = self.debug_modules + runtime_env['pip']
+            runtime_env['pip'] += self.debug_modules
 
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
@@ -114,18 +126,30 @@ class Action:
             pprint(runtime_env)
         return runtime_env
 
-    def run(self):
-        return self.entrypoint(self, **self.params)
+    def validate_params(self):
+        if self.params_model:
+            try:
+                self.params_model.model_validate(self.params, context={'action': self})
+            except ValidationError as e:
+                raise ActionValidationError({'params': pydantic_to_drf_error(e)})
 
     def run_action(self):
+        self.validate_params()
         if self.direct:
             if self.method == RunMethod.RESTAPI:
-                return self.run_by_restapi()
+                return self.start_by_restapi()
             else:
-                return self.run()
-        return getattr(self, f'run_by_{self.method.value}')()
+                result = self.start()
+                self.post_action_by_job(result)
+                return result
+        return getattr(self, f'start_by_{self.method.value}')()
 
-    def run_by_task(self):
+    def start(self):
+        if self.method == RunMethod.JOB:
+            return self.entrypoint(self.run, **self.params)
+        return self.entrypoint(**self.params)
+
+    def start_by_task(self):
         @ray.remote(runtime_env=self.get_runtime_env())
         def run_task(category, action, *args, **kwargs):
             from synapse_sdk.plugins.utils import get_action_class
@@ -149,7 +173,7 @@ class Action:
         kwargs['direct'] = True
         return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
 
-    def run_by_job(self):
+    def start_by_job(self):
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -169,7 +193,7 @@ class Action:
             runtime_env=self.get_runtime_env(),
         )
 
-    def run_by_restapi(self):
+    def start_by_restapi(self):
         path = self.params.pop('path', '')
         method = self.params.pop('method')
 
@@ -178,14 +202,6 @@ class Action:
         # TODO ok response가 아닌 경우 대응하기
         return response.json()
 
-    def set_progress(self, current, total, category=''):
-        self.logger.set_progress(current, total, category)
-
-    def log(self, action, data):
-        self.logger.log(action, data)
-
-    def log_event(self, message):
-        self.logger.log('event', {'content': message})
-
-    def end_log(self):
-        self.log_event('Plugin run is complete.')
+    def post_action_by_job(self, result):
+        if self.client:
+            self.client.update_job(self.job_id, data={'result': result})

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/data_validation/actions/validation.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

synapse_sdk-1.0.0a8/synapse_sdk/plugins/categories/data_validation/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  validation:
+    entrypoint: plugin.validation.validate

synapse_sdk-1.0.0a8/synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py

@@ -0,0 +1,5 @@
+def validate(data, assignment_id, **kwargs):
+    errors = []
+    if data.get('errors'):
+        errors += data['errors']
+    return errors

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/import/actions/import.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/neural_net/actions/deployment.py

@@ -17,7 +17,8 @@ class DeploymentAction(Action):
     def get_actor_options(self):
         return {'runtime_env': self.get_runtime_env()}
 
-    def run(self):
+    def start(self):
         deployment = self.get_deployment()
         serve.delete(self.plugin_release.code)
+        # TODO add run object
         serve.run(deployment.bind(), name=self.plugin_release.code, route_prefix=f'/{self.plugin_release.checksum}')

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/neural_net/actions/inference.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/neural_net/actions/test.py

@@ -1,10 +1,10 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action
 class TestAction(Action):
     name = 'test'
     category = PluginCategory.NEURAL_NET
-    method = RunMethod.TASK
+    method = RunMethod.JOB

synapse_sdk-1.0.0a8/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -0,0 +1,142 @@
+import copy
+import tempfile
+from decimal import Decimal
+from pathlib import Path
+from typing import Annotated
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.utils.file import archive
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+
+class TrainRun(Run):
+    def log_metric(self, x, i, **kwargs):
+        self.log(x, {x: i, **kwargs})
+
+
+class Hyperparameter(BaseModel):
+    batch_size: int
+    iterations: int
+    learning_rate: Decimal
+
+
+class TrainParams(BaseModel):
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str
+    checkpoint: int | None
+    dataset: int
+    hyperparameter: Hyperparameter
+
+    @field_validator('name')
+    @staticmethod
+    def unique_name(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            model_exists = client.exists('list_models', params={'name': value})
+            job_exists = client.exists(
+                'list_jobs',
+                params={
+                    'ids_ex': action.job_id,
+                    'category': 'neural_net',
+                    'action': 'train',
+                    'is_active': True,
+                    'params': f'name:{value}',
+                },
+            )
+            assert not model_exists and not job_exists, '존재하는 학습 이름입니다.'
+        except ClientError:
+            raise PydanticCustomError('client_error', '')
+        return value
+
+
+@register_action
+class TrainAction(Action):
+    name = 'train'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
+    run_class = TrainRun
+    params_model = TrainParams
+    progress_categories = {
+        'dataset': {
+            'proportion': 20,
+        },
+        'train': {
+            'proportion': 75,
+        },
+        'model_upload': {
+            'proportion': 5,
+        },
+    }
+
+    def start(self):
+        hyperparameter = self.params['hyperparameter']
+
+        # download dataset
+        self.run.log_event('Preparing dataset for training.')
+        input_dataset = self.get_dataset()
+
+        # train dataset
+        self.run.log_event('Starting model training.')
+
+        result = self.entrypoint(self.run, input_dataset, hyperparameter)
+
+        # upload model_data
+        self.run.log_event('Registering model data.')
+        self.run.set_progress(0, 1, category='model_upload')
+        model = self.create_model(result)
+        self.run.set_progress(1, 1, category='model_upload')
+
+        self.run.end_log()
+        return {'model_id': model['id']}
+
+    def get_dataset(self):
+        client = self.run.client
+        assert bool(client)
+
+        input_dataset = {}
+
+        ground_truths, count_dataset = client.list_ground_truth_events(
+            params={
+                'fields': ['category', 'files', 'data'],
+                'ground_truth_dataset_versions': self.params['dataset'],
+            },
+            list_all=True,
+        )
+        self.run.set_progress(0, count_dataset, category='dataset')
+        for i, ground_truth in enumerate(ground_truths, start=1):
+            self.run.set_progress(i, count_dataset, category='dataset')
+            try:
+                input_dataset[ground_truth['category']].append(ground_truth)
+            except KeyError:
+                input_dataset[ground_truth['category']] = [ground_truth]
+
+        return input_dataset
+
+    def create_model(self, path):
+        if not self.client:
+            print(path)
+
+        params = copy.deepcopy(self.params)
+        configuration_fields = ['hyperparameter']
+        configuration = {field: params.pop(field) for field in configuration_fields}
+
+        with tempfile.TemporaryDirectory() as temp_path:
+            input_path = Path(path)
+            archive_path = Path(temp_path, 'archive.zip')
+            archive(input_path, archive_path)
+
+            return self.client.create_model({
+                'plugin': self.plugin_release.plugin,
+                'version': self.plugin_release.version,
+                'file': str(archive_path),
+                'configuration': configuration,
+                **params,
+            })

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/neural_net/templates/plugin/train.py

@@ -9,6 +9,6 @@ def train(run, dataset, hyperparameter, checkpoint=None):
         loss = float(round((count_iterations - i) / count_iterations, 2))
         miou = 1 - loss
         run.log_metric('iteration', i, loss=loss, miou=miou)
-        run.set_progress(i, count_iterations, category='iteration')
+        run.set_progress(i, count_iterations, category='train')
 
     return {'weight': '/tmp/agent/test/a.txt', 'config': '/tmp/agent/test/b.txt'}

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a8}/synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action