synapse-sdk 1.0.0a2__tar.gz → 1.0.0a3__tar.gz

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a2
+Version: 1.0.0a3
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk/clients/agent/__init__.py

@@ -1,8 +1,8 @@
-from synapse_sdk.clients.agent.core import CoreClientMixin
 from synapse_sdk.clients.agent.service import ServiceClientMixin
 
 
-class AgentClient(CoreClientMixin, ServiceClientMixin):
+class AgentClient(ServiceClientMixin):
+    name = 'Agent'
     agent_token = None
     user_token = None
     tenant = None

synapse_sdk-1.0.0a3/synapse_sdk/clients/agent/service.py

@@ -0,0 +1,54 @@
+from synapse_sdk.clients.base import BaseClient
+
+
+class ServiceClientMixin(BaseClient):
+    def run_plugin_release(self, code, data):
+        path = f'plugin_releases/{code}/run/'
+        return self._post(path, data=data)
+
+    def run_debug_plugin_release(self, data):
+        path = 'plugin_releases/run_debug/'
+        return self._post(path, data=data)
+
+    def create_plugin_release(self, data):
+        path = 'plugin_releases/'
+        return self._post(path, data=data)
+
+    def get_job(self, pk):
+        path = f'jobs/{pk}/'
+        return self._get(path)
+
+    def list_jobs(self):
+        path = 'jobs/'
+        return self._get(path)
+
+    def list_job_logs(self, pk):
+        path = f'jobs/{pk}/logs/'
+        return self._get(path)
+
+    def tail_job_logs(self, pk):
+        path = f'jobs/{pk}/tail_logs/'
+
+        url = self._get_url(path)
+        headers = self._get_headers()
+
+        response = self.requests_session.get(url, headers=headers, stream=True)
+        for line in response.iter_lines():
+            if line:
+                yield line.decode('utf-8')
+
+    def get_node(self, pk):
+        path = f'nodes/{pk}/'
+        return self._get(path)
+
+    def list_nodes(self):
+        path = 'nodes/'
+        return self._get(path)
+
+    def get_task(self, pk):
+        path = f'tasks/{pk}/'
+        return self._get(path)
+
+    def list_tasks(self):
+        path = 'tasks/'
+        return self._get(path)

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk/clients/backend/__init__.py

@@ -5,6 +5,7 @@ from synapse_sdk.clients.backend.ml import MLClientMixin
 
 
 class BackendClient(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
+    name = 'Backend'
     token = None
     tenant = None
 

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk/clients/backend/integration.py

@@ -27,10 +27,21 @@ class IntegrationClientMixin(BaseClient):
         files = {'file': data.pop('file')}
         return self._post(path, data=data, files=files)
 
+    def list_job_console_logs(self, pk):
+        path = f'jobs/{pk}/console_logs/'
+        return self._get(path)
+
+    def tail_job_console_logs(self, pk):
+        path = f'jobs/{pk}/tail_console_logs/'
+
+        url = self._get_url(path)
+        headers = self._get_headers()
+
+        response = self.requests_session.get(url, headers=headers, stream=True)
+        for line in response.iter_lines():
+            if line:
+                yield line.decode('utf-8')
+
     def create_logs(self, data):
         path = 'logs/'
         return self._post(path, data=data)
-
-    def create_task(self, data):
-        path = 'agent_tasks/'
-        return self._post(path, data=data)

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk/clients/exceptions.py

@@ -6,11 +6,3 @@ class ClientError(Exception):
         self.status = status
         self.reason = reason
         super().__init__(status, reason, *args)
-
-    def as_validation_error(self):
-        if self.status == 400:
-            error = self.reason
-        else:
-            error = str(self)
-
-        return {'backend_errors': error}

synapse_sdk-1.0.0a3/synapse_sdk/loggers.py

@@ -0,0 +1,52 @@
+import datetime
+
+from synapse_sdk.clients.exceptions import ClientError
+
+
+class BaseLogger:
+    progress_records = {}
+
+    def set_progress(self, current, total, category=''):
+        percent = 0
+        if total > 0:
+            percent = (current / total) * 100
+            percent = float(round(percent, 2))
+
+        self.progress_records[category] = {'current': current, 'total': total, 'percent': percent}
+
+
+class ConsoleLogger(BaseLogger):
+    def set_progress(self, current, total, category=''):
+        super().set_progress(current, total, category=category)
+        print(self.progress_records)
+
+    def log(self, action, data):
+        print(action, data)
+
+
+class BackendLogger(BaseLogger):
+    logs_queue = []
+    client = None
+    job_id = None
+
+    def __init__(self, client, job_id):
+        self.client = client
+        self.job_id = job_id
+
+    def set_progress(self, current, total, category=''):
+        super().set_progress(current, total, category=category)
+        # TODO set_progress to the job
+
+    def log(self, action, data):
+        log = {
+            'action': action,
+            'data': data,
+            'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
+            'job': self.job_id,
+        }
+        self.logs_queue.append(log)
+        try:
+            self.client.create_logs(self.logs_queue)
+            self.logs_queue.clear()
+        except ClientError as e:
+            print(e)

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk/plugins/categories/base.py

@@ -8,7 +8,8 @@ import ray
 import requests
 from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
-from synapse_sdk.loggers import ConsoleLogger
+from synapse_sdk.clients.backend import BackendClient
+from synapse_sdk.loggers import ConsoleLogger, BackendLogger
 from synapse_sdk.plugins.enums import RunMethod
 from synapse_sdk.plugins.upload import build_and_upload, archive_and_upload, download_and_upload
 from synapse_sdk.plugins.utils import get_plugin_checksum
@@ -33,6 +34,9 @@ class Action:
         'SYNAPSE_PLUGIN_STORAGE',
         'SYNAPSE_DEBUG_PLUGIN_PATH',
         'SYNAPSE_DEBUG_MODULES',
+        'SYNAPSE_PLUGIN_RUN_HOST',
+        'SYNAPSE_PLUGIN_RUN_USER_TOKEN',
+        'SYNAPSE_PLUGIN_RUN_TENANT',
     ]
 
     def __init__(self, params, plugin_config, envs=None, job_id=None, direct=False, debug=False):
@@ -42,13 +46,8 @@ class Action:
         self.job_id = job_id
         self.direct = direct
         self.debug = debug
-        if envs:
-            self.envs = {**envs, **self.get_default_envs()}
-        else:
-            self.envs = self.get_default_envs()
-
-        # TODO logger 지정 방식 개선
-        self.logger = ConsoleLogger()
+        self.envs = {**envs, **self.get_default_envs()} if envs else self.get_default_envs()
+        self.set_logger()
 
     @cached_property
     def plugin_id(self):
@@ -95,6 +94,17 @@ class Action:
         self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
+    def set_logger(self):
+        if self.job_id:
+            client = BackendClient(
+                self.envs['SYNAPSE_PLUGIN_RUN_HOST'],
+                self.envs['SYNAPSE_PLUGIN_RUN_USER_TOKEN'],
+                self.envs['SYNAPSE_PLUGIN_RUN_TENANT'],
+            )
+            self.logger = BackendLogger(client, self.job_id)
+        else:
+            self.logger = ConsoleLogger()
+
     def get_default_envs(self):
         return {env: os.environ[env] for env in self.default_envs if env in os.environ}
 
@@ -150,14 +160,22 @@ class Action:
         return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
 
     def run_by_job(self):
-        entrypoint_args = ['run', self.name, f'"{json.dumps(self.params)}"', '--direct']
+        main_options = []
+        options = ['run', '--direct']
+        arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
+
         if self.debug:
-            entrypoint_args.insert(0, '--debug')
+            main_options.append('--debug')
+
+        if self.job_id:
+            options.append(f'--job-id={self.job_id}')
+
+        cmd = ' '.join(main_options + options + arguments)
 
         client = JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
         return client.submit_job(
             submission_id=self.job_id,
-            entrypoint=f'python main.py {" ".join(entrypoint_args)}',
+            entrypoint=f'python main.py {cmd}',
             runtime_env=self.get_runtime_env(),
         )
 

synapse_sdk-1.0.0a3/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -0,0 +1,37 @@
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import RunMethod, PluginCategory
+
+
+@register_action
+class TrainAction(Action):
+    name = 'train'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
+
+    def get_dataset(self):
+        return {}
+
+    def run(self):
+        hyperparameter = self.params['hyperparameter']
+
+        # download dataset
+        self.log_event('Preparing dataset for training.')
+        input_dataset = self.get_dataset()
+
+        # train dataset
+        self.log_event('Starting model training.')
+
+        model_files = self.entrypoint(self, input_dataset, hyperparameter)
+
+        # upload model_data
+        self.log_event('Registering model data.')
+
+        self.end_log()
+        return model_files
+
+    def log_metric(self, x, i, **kwargs):
+        self.log(x, {x: i, **kwargs})
+
+    def log_model(self, files, status=None):
+        pass

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk/plugins/cli/run.py

@@ -10,6 +10,7 @@ from synapse_sdk.plugins.utils import get_action, read_config
 @click.command()
 @click.argument('action')
 @click.argument('params')
+@click.option('--job-id')
 @click.option('--direct/--no-direct', default=False)
 @click.option('--run-by', type=click.Choice(['script', 'agent', 'backend']), default='script')
 @click.option('--agent-host')
@@ -19,28 +20,28 @@ from synapse_sdk.plugins.utils import get_action, read_config
 @click.option('--user-token')
 @click.option('--tenant')
 @click.pass_context
-def run(ctx, action, params, direct, run_by, agent_host, agent_token, host, agent, user_token, tenant):
+def run(ctx, action, params, job_id, direct, run_by, agent_host, agent_token, host, agent, user_token, tenant):
     debug = ctx.obj['DEBUG']
 
     if run_by == 'script':
-        run_by_script(action, params, direct, debug)
+        run_by_script(action, params, job_id, direct, debug)
     elif run_by == 'agent':
-        run_by_agent(action, params, agent_host, agent_token, user_token, tenant, debug)
+        run_by_agent(action, params, job_id, agent_host, agent_token, user_token, tenant, debug)
     elif run_by == 'backend':
         run_by_backend(action, params, agent, host, user_token, tenant)
 
 
-def run_by_script(action, params, direct, debug):
-    action = get_action(action, params, direct=direct, debug=debug)
+def run_by_script(action, params, job_id, direct, debug):
+    action = get_action(action, params, job_id=job_id, direct=direct, debug=debug)
     result = action.run_action()
 
     if debug:
         click.echo(result)
 
 
-def run_by_agent(action, params, agent_host, agent_token, user_token, tenant, debug):
+def run_by_agent(action, params, job_id, agent_host, agent_token, user_token, tenant, debug):
     client = AgentClient(agent_host, agent_token, user_token, tenant)
-    data = {'action': action, 'params': params}
+    data = {'action': action, 'params': params, 'job_id': job_id}
     if debug:
         data.update({
             'plugin_path': os.getcwd(),

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a2
+Version: 1.0.0a3
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a2 → synapse_sdk-1.0.0a3}/synapse_sdk.egg-info/SOURCES.txt

@@ -18,7 +18,6 @@ synapse_sdk/clients/base.py
 synapse_sdk/clients/exceptions.py
 synapse_sdk/clients/utils.py
 synapse_sdk/clients/agent/__init__.py
-synapse_sdk/clients/agent/core.py
 synapse_sdk/clients/agent/service.py
 synapse_sdk/clients/backend/__init__.py
 synapse_sdk/clients/backend/annotation.py

synapse_sdk-1.0.0a2/synapse_sdk/clients/agent/core.py

@@ -1,11 +0,0 @@
-from synapse_sdk.clients.base import BaseClient
-
-
-class CoreClientMixin(BaseClient):
-    def get_agent_system(self):
-        path = 'agents/system/'
-        return self._get(path)
-
-    def get_jobs_progress(self, data):
-        path = 'integration/jobs/progress/'
-        return self._post(path, data=data, timeout=1)

synapse_sdk-1.0.0a2/synapse_sdk/clients/agent/service.py

@@ -1,15 +0,0 @@
-from synapse_sdk.clients.base import BaseClient
-
-
-class ServiceClientMixin(BaseClient):
-    def run_plugin_release(self, code, data):
-        path = f'plugin_releases/{code}/run/'
-        return self._post(path, data=data)
-
-    def run_debug_plugin_release(self, data):
-        path = 'plugin_releases/run_debug/'
-        return self._post(path, data=data)
-
-    def create_plugin_release(self, data):
-        path = 'plugin_releases/'
-        return self._post(path, data=data)

synapse_sdk-1.0.0a2/synapse_sdk/loggers.py

@@ -1,48 +0,0 @@
-import datetime
-
-from synapse_sdk.clients.exceptions import ClientError
-
-
-class BaseLogger:
-    progress_records = {}
-    logs_queue = []
-    client = None
-
-    def __init__(self, client=None, task=None):
-        self.client = client
-        self.task = task
-
-    def set_progress(self, current, total, category=''):
-        percent = 0
-        if total > 0:
-            percent = (current / total) * 100
-            percent = float(round(percent, 2))
-
-        self.progress_records[category] = {'current': current, 'total': total, 'percent': percent}
-        if self.task:
-            self.task.update_state(state='PROGRESS', meta=self.progress_records)
-        else:
-            print(self.progress_records)
-
-    def log(self, action, data):
-        log = {'action': action, 'data': data, 'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}
-
-        if self.client and self.task:
-            log['task_id'] = self.task.request.id
-            self.logs_queue.append(log)
-            try:
-                self.client.create_logs(self.logs_queue)
-                self.logs_queue.clear()
-            except ClientError as e:
-                print(e)
-        else:
-            print(log)
-
-
-class ConsoleLogger(BaseLogger):
-    def log(self, action, data):
-        print(action, data)
-
-
-class SynapseLogger(BaseLogger):
-    pass

synapse_sdk-1.0.0a2/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -1,96 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import RunMethod, PluginCategory
-from synapse_sdk.utils.file import get_dict_from_file, files_url_to_path_from_objs
-from synapse_sdk.utils.module_loading import import_string
-
-
-@register_action
-class TrainAction(Action):
-    name = 'train'
-    category = PluginCategory.NEURAL_NET
-    method = RunMethod.JOB
-
-    def get_input_dataset_for_training(self, model_id=None):
-        """
-        :return:
-        {
-            "train": [
-                {
-                    "files": {
-                        "image": {
-                            "path": "/path/to/image.jpg",
-                            "meta": {
-                                "width": 265,
-                                "height": 190,
-                                "created": 1651563526.0277045,
-                                "file_size": 5191,
-                                "last_modified": 1651563526.0277045
-                            }
-                        }
-                    },
-                    "ground_truth": {
-                        ...label_data
-                    }
-                },
-                ...
-            ],
-            "validation": ...,
-            "test": ...
-        }
-        """
-
-        client = self.logger.client
-        input_dataset = {}
-        category_int_to_str = {1: 'train', 2: 'validation', 3: 'test'}
-
-        if client:
-            train_dataset, count_dataset = client.list_train_dataset(
-                payload={'fields': ['category', 'files', 'ground_truth'], 'model': model_id}, list_all=True
-            )
-
-            for i, train_data in enumerate(train_dataset, start=1):
-                self.set_progress(i, count_dataset, category='dataset_download')
-                category = category_int_to_str[train_data.pop('category')]
-                try:
-                    input_dataset[category].append(train_data)
-                except KeyError:
-                    input_dataset[category] = [train_data]
-
-        else:
-            for category in category_int_to_str.values():
-                dataset_path = self.task['dataset'].get(category)
-                if dataset_path:
-                    input_dataset[category] = get_dict_from_file(dataset_path)
-                    files_url_to_path_from_objs(input_dataset[category], ['files'], is_list=True)
-
-        return input_dataset
-
-    def run_train(self):
-        hyperparameter = self.task['hyperparameter']
-        train = import_string(self.plugin['train']['entrypoint'])
-
-        # download dataset
-        self.log_event('Preparing dataset for training.')
-        input_dataset = self.get_input_dataset_for_training()
-
-        # train dataset
-        self.log_event('Starting model training.')
-
-        model_files = train(self, input_dataset, hyperparameter)
-
-        # upload model_data
-        self.log_event('Registering model data.')
-
-        self.end_log()
-        return model_files
-
-    def start(self):
-        action = self.task['action']
-        getattr(self, f'run_{action}')()
-
-    def log_metric(self, x, i, **kwargs):
-        self.log(x, {x: i, **kwargs})
-
-    def log_model(self, files, status=None):
-        pass