synapse-sdk 1.0.0a6__tar.gz → 1.0.0a7__tar.gz

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a6
+Version: 1.0.0a7
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/pyproject.toml

@@ -38,7 +38,7 @@ synapse = 'synapse_sdk.cli:cli'
 line-length = 120
 
 [tool.ruff.lint]
-select = ['E', 'F', 'Q']
+select = ['E', 'F', 'I', 'Q']
 ignore = ['W191', 'E111', 'E114', 'E117', 'D206', 'D300', 'Q000', 'Q001', 'Q002', 'Q003', 'COM812', 'COM819', 'ISC001', 'ISC002']
 
 [tool.ruff.lint.pydocstyle]

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/clients/backend/dataset.py

@@ -1,4 +1,5 @@
 from multiprocessing import Pool
+
 from tqdm import tqdm
 
 from synapse_sdk.clients.base import BaseClient

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/base.py

@@ -8,11 +8,9 @@ import ray
 import requests
 from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
-from synapse_sdk.clients.backend import BackendClient
-from synapse_sdk.loggers import ConsoleLogger, BackendLogger
 from synapse_sdk.plugins.enums import RunMethod
-from synapse_sdk.plugins.models import PluginRelease
-from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload, download_and_upload
+from synapse_sdk.plugins.models import PluginRelease, Run
+from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
 from synapse_sdk.utils.module_loading import import_string
 
 
@@ -25,8 +23,9 @@ class Action:
     plugin_release = None
     config = None
     client = None
-    logger = None
     debug = False
+    run_class = Run
+    run = None
 
     default_envs = [
         'RAY_DASHBOARD_URL',
@@ -48,7 +47,7 @@ class Action:
         self.direct = direct
         self.debug = debug
         self.envs = {**envs, **self.get_default_envs()} if envs else self.get_default_envs()
-        self.set_logger()
+        self.run = self.get_run()
 
     @cached_property
     def entrypoint(self):
@@ -84,16 +83,13 @@ class Action:
         self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
-    def set_logger(self):
-        if self.job_id:
-            client = BackendClient(
-                self.envs['SYNAPSE_PLUGIN_RUN_HOST'],
-                self.envs['SYNAPSE_PLUGIN_RUN_USER_TOKEN'],
-                self.envs['SYNAPSE_PLUGIN_RUN_TENANT'],
-            )
-            self.logger = BackendLogger(client, self.job_id)
-        else:
-            self.logger = ConsoleLogger()
+    def get_run(self):
+        context = {
+            'params': self.params,
+            'envs': self.envs,
+            'debug': self.debug,
+        }
+        return self.run_class(self.job_id, context)
 
     def get_default_envs(self):
         return {env: os.environ[env] for env in self.default_envs if env in os.environ}
@@ -105,7 +101,7 @@ class Action:
         }
 
         if self.debug:
-            runtime_env['pip'] = self.debug_modules + runtime_env['pip']
+            runtime_env['pip'] += self.debug_modules
 
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
@@ -114,18 +110,20 @@ class Action:
             pprint(runtime_env)
         return runtime_env
 
-    def run(self):
-        return self.entrypoint(self, **self.params)
-
     def run_action(self):
         if self.direct:
             if self.method == RunMethod.RESTAPI:
-                return self.run_by_restapi()
+                return self.start_by_restapi()
             else:
-                return self.run()
-        return getattr(self, f'run_by_{self.method.value}')()
+                return self.start()
+        return getattr(self, f'start_by_{self.method.value}')()
+
+    def start(self):
+        if self.method == RunMethod.JOB:
+            return self.entrypoint(self.run, **self.params)
+        return self.entrypoint(**self.params)
 
-    def run_by_task(self):
+    def start_by_task(self):
         @ray.remote(runtime_env=self.get_runtime_env())
         def run_task(category, action, *args, **kwargs):
             from synapse_sdk.plugins.utils import get_action_class
@@ -149,7 +147,7 @@ class Action:
         kwargs['direct'] = True
         return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
 
-    def run_by_job(self):
+    def start_by_job(self):
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -169,7 +167,7 @@ class Action:
             runtime_env=self.get_runtime_env(),
         )
 
-    def run_by_restapi(self):
+    def start_by_restapi(self):
         path = self.params.pop('path', '')
         method = self.params.pop('method')
 
@@ -177,15 +175,3 @@ class Action:
         response = getattr(requests, method)(url, **self.params)
         # TODO ok response가 아닌 경우 대응하기
         return response.json()
-
-    def set_progress(self, current, total, category=''):
-        self.logger.set_progress(current, total, category)
-
-    def log(self, action, data):
-        self.logger.log(action, data)
-
-    def log_event(self, message):
-        self.logger.log('event', {'content': message})
-
-    def end_log(self):
-        self.log_event('Plugin run is complete.')

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/data_validation/actions/validation.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

synapse_sdk-1.0.0a7/synapse_sdk/plugins/categories/data_validation/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  validation:
+    entrypoint: plugin.validation.validate

synapse_sdk-1.0.0a7/synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py

@@ -0,0 +1,5 @@
+def validate(data, assignment_id, **kwargs):
+    errors = []
+    if data.get('errors'):
+        errors += data['errors']
+    return errors

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/import/actions/import.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/neural_net/actions/deployment.py

@@ -17,7 +17,8 @@ class DeploymentAction(Action):
     def get_actor_options(self):
         return {'runtime_env': self.get_runtime_env()}
 
-    def run(self):
+    def start(self):
         deployment = self.get_deployment()
         serve.delete(self.plugin_release.code)
+        # TODO add run object
         serve.run(deployment.bind(), name=self.plugin_release.code, route_prefix=f'/{self.plugin_release.checksum}')

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/neural_net/actions/inference.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/neural_net/actions/test.py

@@ -1,10 +1,10 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action
 class TestAction(Action):
     name = 'test'
     category = PluginCategory.NEURAL_NET
-    method = RunMethod.TASK
+    method = RunMethod.JOB

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -1,6 +1,15 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+
+
+class TrainRun(Run):
+    def log_metric(self, x, i, **kwargs):
+        self.log(x, {x: i, **kwargs})
+
+    def log_model(self, files, status=None):
+        pass
 
 
 @register_action
@@ -8,30 +17,25 @@ class TrainAction(Action):
     name = 'train'
     category = PluginCategory.NEURAL_NET
     method = RunMethod.JOB
+    run_class = TrainRun
 
     def get_dataset(self):
         return {}
 
-    def run(self):
+    def start(self):
         hyperparameter = self.params['hyperparameter']
 
         # download dataset
-        self.log_event('Preparing dataset for training.')
+        self.run.log_event('Preparing dataset for training.')
         input_dataset = self.get_dataset()
 
         # train dataset
-        self.log_event('Starting model training.')
+        self.run.log_event('Starting model training.')
 
-        model_files = self.entrypoint(self, input_dataset, hyperparameter)
+        model_files = self.entrypoint(self.run, input_dataset, hyperparameter)
 
         # upload model_data
-        self.log_event('Registering model data.')
+        self.run.log_event('Registering model data.')
 
-        self.end_log()
+        self.run.end_log()
         return model_files
-
-    def log_metric(self, x, i, **kwargs):
-        self.log(x, {x: i, **kwargs})
-
-    def log_model(self, files, status=None):
-        pass

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

synapse_sdk-1.0.0a7/synapse_sdk/plugins/categories/post_annotation/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  post_annotation:
+    entrypoint: plugin.post_annotation.post_annotate

synapse_sdk-1.0.0a7/synapse_sdk/plugins/categories/post_annotation/templates/plugin/post_annotation.py

@@ -0,0 +1,3 @@
+def post_annotate(data, assignment_id, **kwargs):
+    data['post_annotate'] = 'hello world'
+    return data

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py

@@ -1,6 +1,6 @@
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
 @register_action

synapse_sdk-1.0.0a7/synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  pre_annotation:
+    entrypoint: plugin.pre_annotation.pre_annotate

synapse_sdk-1.0.0a7/synapse_sdk/plugins/categories/pre_annotation/templates/plugin/pre_annotation.py

@@ -0,0 +1,3 @@
+def pre_annotate(data, assignment_id, **kwargs):
+    data['pre_annotate'] = 'hello world'
+    return data

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/enums.py

@@ -11,6 +11,11 @@ class PluginCategory(Enum):
     NEURAL_NET = 'neural_net'
     EXPORT = 'export'
     IMPORT = 'import'
+    SMART_TOOL = 'smart_tool'
     POST_ANNOTATION = 'post_annotation'
     PRE_ANNOTATION = 'pre_annotation'
     DATA_VALIDATION = 'data_validation'
+
+    @classmethod
+    def choices(cls):
+        return [(member.value, member.name.replace('_', ' ').title()) for member in cls]

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/models.py

@@ -1,7 +1,9 @@
 import os
 from functools import cached_property
-from typing import Dict, Any
+from typing import Any, Dict
 
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.loggers import BackendLogger, ConsoleLogger
 from synapse_sdk.plugins.utils import read_plugin_config
 from synapse_sdk.utils.storage import get_storage
 from synapse_sdk.utils.string import hash_text
@@ -59,8 +61,35 @@ class PluginRelease:
         return os.path.join(serve_address, self.checksum, path)
 
 
-class Job:
+class Run:
+    logger = None
     job_id = None
+    context = None
 
-    def __init__(self, job_id):
+    def __init__(self, job_id, context):
         self.job_id = job_id
+        self.context = context
+        self.set_logger()
+
+    def set_logger(self):
+        if self.job_id:
+            client = BackendClient(
+                self.context['envs']['SYNAPSE_PLUGIN_RUN_HOST'],
+                self.context['envs']['SYNAPSE_PLUGIN_RUN_USER_TOKEN'],
+                self.context['envs']['SYNAPSE_PLUGIN_RUN_TENANT'],
+            )
+            self.logger = BackendLogger(client, self.job_id)
+        else:
+            self.logger = ConsoleLogger()
+
+    def set_progress(self, current, total, category=''):
+        self.logger.set_progress(current, total, category)
+
+    def log(self, action, data):
+        self.logger.log(action, data)
+
+    def log_event(self, message):
+        self.logger.log('event', {'content': message})
+
+    def end_log(self):
+        self.log_event('Plugin run is complete.')

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/templates/hooks/pre_prompt.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+
 from synapse_sdk.plugins.utils import get_plugin_categories
 from synapse_sdk.utils.file import get_dict_from_file
 

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin/pyproject.toml

@@ -2,7 +2,7 @@
 line-length = 120
 
 [tool.ruff.lint]
-select = ['E', 'F', 'Q']
+select = ['E', 'F', 'I', 'Q']
 ignore = ['W191', 'E111', 'E114', 'E117', 'D206', 'D300', 'Q000', 'Q001', 'Q002', 'Q003', 'COM812', 'COM819', 'ISC001', 'ISC002']
 
 [tool.ruff.lint.pydocstyle]

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/utils/storage.py

@@ -1,5 +1,5 @@
 import os
-from urllib.parse import urlparse, parse_qs
+from urllib.parse import parse_qs, urlparse
 
 import boto3
 from botocore.exceptions import ClientError

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a6
+Version: 1.0.0a7
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk.egg-info/SOURCES.txt

@@ -40,6 +40,9 @@ synapse_sdk/plugins/categories/templates.py
 synapse_sdk/plugins/categories/data_validation/__init__.py
 synapse_sdk/plugins/categories/data_validation/actions/__init__.py
 synapse_sdk/plugins/categories/data_validation/actions/validation.py
+synapse_sdk/plugins/categories/data_validation/templates/config.yaml
+synapse_sdk/plugins/categories/data_validation/templates/plugin/__init__.py
+synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py
 synapse_sdk/plugins/categories/export/__init__.py
 synapse_sdk/plugins/categories/export/actions/__init__.py
 synapse_sdk/plugins/categories/export/actions/export.py
@@ -60,9 +63,15 @@ synapse_sdk/plugins/categories/neural_net/templates/plugin/train.py
 synapse_sdk/plugins/categories/post_annotation/__init__.py
 synapse_sdk/plugins/categories/post_annotation/actions/__init__.py
 synapse_sdk/plugins/categories/post_annotation/actions/post_annotation.py
+synapse_sdk/plugins/categories/post_annotation/templates/config.yaml
+synapse_sdk/plugins/categories/post_annotation/templates/plugin/__init__.py
+synapse_sdk/plugins/categories/post_annotation/templates/plugin/post_annotation.py
 synapse_sdk/plugins/categories/pre_annotation/__init__.py
 synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py
 synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py
+synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml
+synapse_sdk/plugins/categories/pre_annotation/templates/plugin/__init__.py
+synapse_sdk/plugins/categories/pre_annotation/templates/plugin/pre_annotation.py
 synapse_sdk/plugins/cli/__init__.py
 synapse_sdk/plugins/cli/publish.py
 synapse_sdk/plugins/cli/run.py

{synapse_sdk-1.0.0a6 → synapse_sdk-1.0.0a7}/synapse_sdk/clients/base.py

@@ -1,5 +1,5 @@
-import os
 import json
+import os
 from pathlib import Path
 
 import requests