synapse-sdk 1.0.0a20__py3-none-any.whl → 1.0.0a22__py3-none-any.whl

synapse_sdk/clients/backend/__init__.py

@@ -1,10 +1,17 @@
 from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
+from synapse_sdk.clients.backend.core import CoreClientMixin
 from synapse_sdk.clients.backend.dataset import DatasetClientMixin
 from synapse_sdk.clients.backend.integration import IntegrationClientMixin
 from synapse_sdk.clients.backend.ml import MLClientMixin
 
 
-class BackendClient(AnnotationClientMixin, DatasetClientMixin, IntegrationClientMixin, MLClientMixin):
+class BackendClient(
+    AnnotationClientMixin,
+    CoreClientMixin,
+    DatasetClientMixin,
+    IntegrationClientMixin,
+    MLClientMixin,
+):
     name = 'Backend'
     token = None
     tenant = None

synapse_sdk/clients/backend/core.py

@@ -0,0 +1,32 @@
+import hashlib
+import os
+from pathlib import Path
+
+from synapse_sdk.clients.base import BaseClient
+
+
+class CoreClientMixin(BaseClient):
+    def create_chunked_upload(self, file_path):
+        def read_file_in_chunks(file_path, chunk_size=1024 * 1024 * 50):
+            with open(file_path, 'rb') as file:
+                while chunk := file.read(chunk_size):
+                    yield chunk
+
+        file_path = Path(file_path)
+        size = os.path.getsize(file_path)
+        hash_md5 = hashlib.md5()
+
+        url = 'chunked_upload/'
+        offset = 0
+        for chunk in read_file_in_chunks(file_path):
+            hash_md5.update(chunk)
+            data = self._put(
+                url,
+                data={'filename': file_path.name},
+                files={'file': chunk},
+                headers={'Content-Range': f'bytes {offset}-{offset + len(chunk) - 1}/{size}'},
+            )
+            offset = data['offset']
+            url = data['url']
+
+        return self._post(url, data={'md5': hash_md5.hexdigest()})

synapse_sdk/clients/backend/ml.py

@@ -14,8 +14,9 @@ class MLClientMixin(BaseClient):
 
     def create_model(self, data):
         path = 'models/'
-        files = {'file': data.pop('file')}
-        return self._post(path, data=data, files=files)
+        file = data.pop('file')
+        data['chunked_upload'] = self.create_chunked_upload(file)['id']
+        return self._post(path, data=data)
 
     def list_ground_truth_events(self, params=None, url_conversion=None, list_all=False):
         path = 'ground_truth_events/'

synapse_sdk/clients/base.py

@@ -28,11 +28,13 @@ class BaseClient:
     def _request(self, method, path, **kwargs):
         url = self._get_url(path)
         headers = self._get_headers()
+        headers.update(kwargs.pop('headers', {}))
 
         if method in ['post', 'put', 'patch']:
             if kwargs.get('files') is not None:
                 for name, file in kwargs['files'].items():
-                    kwargs['files'][name] = Path(str(file)).open(mode='rb')
+                    if isinstance(file, (str, Path)):
+                        kwargs['files'][name] = Path(str(file)).open(mode='rb')
                 for name, value in kwargs['data'].items():
                     if isinstance(value, dict):
                         kwargs['data'][name] = json.dumps(value)

synapse_sdk/plugins/categories/base.py

@@ -30,6 +30,7 @@ class Action:
     plugin_config = None
     plugin_release = None
     config = None
+    requirements = None
     job_id = None
     direct = None
     debug = None
@@ -48,11 +49,12 @@ class Action:
         'SYNAPSE_PLUGIN_RUN_TENANT',
     ]
 
-    def __init__(self, params, plugin_config, envs=None, job_id=None, direct=False, debug=False):
+    def __init__(self, params, plugin_config, requirements=None, envs=None, job_id=None, direct=False, debug=False):
         self.params = params
         self.plugin_config = plugin_config
         self.plugin_release = PluginRelease(config=plugin_config)
         self.config = self.plugin_release.get_action_config(self.name)
+        self.requirements = requirements
         self.job_id = job_id
         self.direct = direct
         self.debug = debug
@@ -117,10 +119,10 @@ class Action:
         return {env: os.environ[env] for env in self.REQUIRED_ENVS if env in os.environ}
 
     def get_runtime_env(self):
-        runtime_env = {
-            'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
-            'working_dir': self.plugin_url,
-        }
+        runtime_env = {'pip': [], 'working_dir': self.plugin_url}
+
+        if self.requirements:
+            runtime_env['pip'] += self.requirements
 
         if self.debug:
             runtime_env['pip'] += self.debug_modules

synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -17,8 +17,8 @@ from synapse_sdk.utils.pydantic.validators import non_blank
 
 
 class TrainRun(Run):
-    def log_metric(self, key, value, **metrics):
-        self.log('metric', {'key': key, 'value': value, 'metrics': metrics})
+    def log_metric(self, category, key, value, **metrics):
+        self.log('metric', {'category': category, 'key': key, 'value': value, 'metrics': metrics})
 
 
 class Hyperparameter(BaseModel):

synapse_sdk/plugins/cli/publish.py

@@ -1,4 +1,3 @@
-import json
 from pathlib import Path
 
 import click
@@ -27,7 +26,7 @@ def publish(ctx, host, user_token, tenant, debug_modules):
     data = {'plugin': plugin_release.plugin, 'file': str(archive_path), 'debug': debug}
     if debug:
         modules = debug_modules.split(',') if debug_modules else []
-        data['debug_meta'] = json.dumps({'modules': modules})
+        data['meta'] = {'modules': modules}
 
     client = BackendClient(host, user_token, tenant=tenant)
     client.create_plugin_release(data)

synapse_sdk/plugins/cli/run.py

@@ -1,11 +1,12 @@
 import os
+from pathlib import Path
 
 import click
 
 from synapse_sdk.clients.agent import AgentClient
 from synapse_sdk.clients.backend import BackendClient
 from synapse_sdk.plugins.models import PluginRelease
-from synapse_sdk.plugins.utils import get_action
+from synapse_sdk.plugins.utils import read_requirements, run_plugin
 
 
 @click.command()
@@ -33,8 +34,8 @@ def run(ctx, action, params, job_id, direct, run_by, agent_host, agent_token, ho
 
 
 def run_by_script(action, params, job_id, direct, debug):
-    action = get_action(action, params, job_id=job_id, direct=direct, debug=debug)
-    result = action.run_action()
+    requirements = read_requirements('./requirements.txt')
+    result = run_plugin(action, params, requirements=requirements, debug=debug, job_id=job_id, direct=direct)
 
     if debug:
         click.echo(result)
@@ -43,17 +44,23 @@ def run_by_script(action, params, job_id, direct, debug):
 def run_by_agent(action, params, job_id, agent_host, agent_token, user_token, tenant, debug):
     client = AgentClient(agent_host, agent_token, user_token, tenant)
     data = {'action': action, 'params': params}
+    plugin_path = os.getcwd()
+    requirements = read_requirements(Path(plugin_path) / 'requirements.txt')
+    if requirements:
+        data = {'requirements': requirements}
+
     if job_id:
         data['job_id'] = job_id
+
     if debug:
         data.update({
-            'plugin_path': os.getcwd(),
+            'plugin_path': plugin_path,
             'modules': os.getenv('SYNAPSE_DEBUG_MODULES', '').split(','),
         })
         result = client.run_debug_plugin_release(data=data)
     else:
         plugin_release = PluginRelease()
-        result = client.run_plugin_release(code=plugin_release.code, data=data)
+        result = client.run_plugin_release(plugin_release.code, data=data)
 
     click.echo(result)
 

synapse_sdk/plugins/utils.py

@@ -52,17 +52,37 @@ def read_plugin_config(plugin_path=None):
     return get_dict_from_file(config_path)
 
 
-def run_plugin(action, params, plugin_config=None, plugin_path=None, modules=None, envs=None, debug=False, **kwargs):
+def read_requirements(file_path):
+    file_path = Path(file_path)
+    if not file_path.exists():
+        return None
+
+    requirements = []
+    for line in file_path.read_text().splitlines():
+        stripped_line = line.strip()
+        if stripped_line and not stripped_line.startswith('#'):
+            requirements.append(stripped_line)
+    return requirements
+
+
+def run_plugin(
+    action,
+    params,
+    plugin_config=None,
+    plugin_path=None,
+    modules=None,
+    requirements=None,
+    envs=None,
+    debug=False,
+    **kwargs,
+):
     from synapse_sdk.plugins.models import PluginRelease
 
     if not envs:
         envs = {}
 
     if debug:
-        if not plugin_path:
-            raise ActionError({'plugin_path': _('디버그 모드에서는 plugin_path는 필수입니다.')})
-
-        if plugin_path.startswith('http'):
+        if plugin_path and plugin_path.startswith('http'):
             if not plugin_config:
                 raise ActionError({'config': _('"plugin_path"가 url인 경우에는 "config"가 필수입니다.')})
             plugin_release = PluginRelease(config=plugin_config)
@@ -73,9 +93,12 @@ def run_plugin(action, params, plugin_config=None, plugin_path=None, modules=Non
         if action not in plugin_release.actions:
             raise ActionError({'action': _('해당 액션은 존재하지 않습니다.')})
 
-        envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_path
+        if plugin_path:
+            envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_path
+
         if modules:
             envs['SYNAPSE_DEBUG_MODULES'] = ','.join(modules)
+
     else:
         if plugin_config is None:
             raise ActionError({'config': _('플러그인 설정은 필수입니다.')})
@@ -84,9 +107,9 @@ def run_plugin(action, params, plugin_config=None, plugin_path=None, modules=Non
         action,
         params,
         config=plugin_config,
+        requirements=requirements,
         envs=envs,
         debug=debug,
         **kwargs,
     )
-
     return action.run_action()

{synapse_sdk-1.0.0a20.dist-info → synapse_sdk-1.0.0a22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a20
+Version: 1.0.0a22
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a20.dist-info → synapse_sdk-1.0.0a22.dist-info}/RECORD

@@ -8,18 +8,19 @@ synapse_sdk/loggers.py,sha256=RsDDOiOeUCih1XOkWQJseYdYCX_wt50AZJRe6aPf96Q,4004
 synapse_sdk/cli/__init__.py,sha256=WmYGW1qZEXXIGJe3SGr8QjOStY4svuZKK1Lp_aPvtPs,140
 synapse_sdk/cli/create_plugin.py,sha256=egbW_92WwxfHz50Gy4znX5Bf5fxDdQj3GFyd0l3Y3SY,228
 synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/clients/base.py,sha256=un2vseaQ7YY6rVQqd5fIyzclg7VYLAEx2TwUkZof3ik,3354
+synapse_sdk/clients/base.py,sha256=Yo_WUHNQ2NgNe3sBav0cM_armOdBz34o7fn3lbtrHO0,3462
 synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRTAjwk,220
 synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
 synapse_sdk/clients/agent/__init__.py,sha256=Pz8_iTbIbnb7ywGJ3feqoZVmO2I3mEbwpWsISIxh0BU,1968
 synapse_sdk/clients/agent/core.py,sha256=YicfkO0YvjDDOt1jNWoZ0mokrh8xxKibBL4qF5yOjKs,169
 synapse_sdk/clients/agent/ray.py,sha256=JrwLyVOUDG2yYsbPrxyUtWbM-FWp9B6Bl_GdDby0rt8,1559
 synapse_sdk/clients/agent/service.py,sha256=s7KuPK_DB1nr2VHrigttV1WyFonaGHNrPvU8loRxHcE,478
-synapse_sdk/clients/backend/__init__.py,sha256=50MW1CMWGaKnALSv2fOjhJZG55Xb3yrqlKtnSZldwcU,1004
+synapse_sdk/clients/backend/__init__.py,sha256=aozhPhvRTPHz1P90wxEay07B-Ct4vj_yTw5H9_PJEBE,1105
 synapse_sdk/clients/backend/annotation.py,sha256=eZc5EidgR_RfMGwvv1r1_mLkPdRd8e52c4zuuMjMX34,979
+synapse_sdk/clients/backend/core.py,sha256=5XAOdo6JZ0drfk-FMPJ96SeTd9oja-VnTwzGXdvK7Bg,1027
 synapse_sdk/clients/backend/dataset.py,sha256=a_svyCKgzF7N99l8V4u4wXD8JxiGuLW9z2EBinnz7b8,1738
 synapse_sdk/clients/backend/integration.py,sha256=Jg_8fEmbrgYXfZZcG8cDtLxR6ugPmnbNhPDyRu_Uib0,2160
-synapse_sdk/clients/backend/ml.py,sha256=vNo1FOG9welXGizcnylLSW_-l9iJSQrTI3DhyCCyFKw,1015
+synapse_sdk/clients/backend/ml.py,sha256=jlkqS9pI0S0Ubq4pWVeaaaPk-E0J-cZg5zkSX7GuQ_o,1063
 synapse_sdk/clients/ray/__init__.py,sha256=9ZSPXVVxlJ8Wp8ku7l021ENtPjVrGgQDgqifkkVAXgM,187
 synapse_sdk/clients/ray/core.py,sha256=a4wyCocAma2HAm-BHlbZnoVbpfdR-Aad2FM0z6vPFvw,731
 synapse_sdk/clients/ray/serve.py,sha256=rbCpXZYWf0oP8XJ9faa9QFNPYU7h8dltIG8xn9ZconY,907
@@ -28,9 +29,9 @@ synapse_sdk/plugins/enums.py,sha256=s59P6Oz2WAK9IX-kLVhNOvNKYJifKlWBhPpZbc9-ttE,
 synapse_sdk/plugins/exceptions.py,sha256=Qs7qODp_RRLO9y2otU2T4ryj5LFwIZODvSIXkAh91u0,691
 synapse_sdk/plugins/models.py,sha256=S0oC2DXTqO9qH1BTNKVYRflGKSMU_8LpKVmCIEpJxVA,3562
 synapse_sdk/plugins/upload.py,sha256=VJOotYMayylOH0lNoAGeGHRkLdhP7jnC_A0rFQMvQpQ,3228
-synapse_sdk/plugins/utils.py,sha256=UYkwxkmrs0-mRgQB63SkTGD1HpIQEIiHndzTcpdNaF4,2936
+synapse_sdk/plugins/utils.py,sha256=4_K6jIl0WrsXOEhFp94faMOriSsddOhIiaXcawYYUUA,3300
 synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/base.py,sha256=wsPPUcCFP2ZoM9Tly_b6NdCTkTls7_zsurA6ePC5_OA,8244
+synapse_sdk/plugins/categories/base.py,sha256=3EkcKGqLwpRzeeH8sH-gJ0AgB-Zhh1UrNR9CrxEOwLo,8317
 synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
 synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
 synapse_sdk/plugins/categories/templates.py,sha256=FF5FerhkZMeW1YcKLY5cylC0SkWSYdJODA_Qcm4OGYQ,887
@@ -51,7 +52,7 @@ synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBS
 synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=i3X2CCWADNAwoDhX-6XTZFEu8MekjJoLuqEojrx1vIc,1730
 synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=i18bDDjkuF9V8nxxW-T_umNIOD-Jnq_MMjIjZc6W8n8,645
 synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtMkGoo_qNqoaZkp3AQNEypJmeGzEog,320
-synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=ZjBE4uHOdVZ318BuepBt2l0KvSflLr1Fqt3dt4KYXJs,5113
+synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=4zq2ryWjqDa6MyI2BGe3sAofMh2NeY3XAo-1gEFXBs4,5145
 synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=dXKB1hO53hDZB73xnxLVCNQl8Sm7svMmVmuMrOCQmEU,343
 synapse_sdk/plugins/categories/neural_net/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py,sha256=InfqKWJYi6sqiUnfPKHC5KYGhxckDaWZNQ202u-uVP4,366
@@ -76,8 +77,8 @@ synapse_sdk/plugins/categories/smart_tool/templates/config.yaml,sha256=7bvb4M1PL
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/cli/__init__.py,sha256=LPtUO0jqkhKq6xR1grpse7da2R6OoT_BeDyCNyUY0T4,380
-synapse_sdk/plugins/cli/publish.py,sha256=Kg_MaOeTNCra9BIzWBASsIZxLfhnNkzbRPqdlf6aTyk,1251
-synapse_sdk/plugins/cli/run.py,sha256=lw1KbsL-xTGllF4NtD2cq-Rh6HMbhi-sO862_Ds-sUo,2330
+synapse_sdk/plugins/cli/publish.py,sha256=sIl1wiuSC3lAUpE3rOF4UDKDy2G5EVLlelMjk2aT05g,1221
+synapse_sdk/plugins/cli/run.py,sha256=xz5LRm3zh8Y9DMjw5FFRFVRWSCWtYfZJskfCmrPikaQ,2598
 synapse_sdk/plugins/templates/cookiecutter.json,sha256=NxOWk9A_v1pO0Ny4IYT9Cj5iiJ16--cIQrGC67QdR0I,396
 synapse_sdk/plugins/templates/hooks/post_gen_project.py,sha256=jqlYkY1O2TxIR-Vh3gnwILYy8k-D39Xx66d2KNQVMCs,147
 synapse_sdk/plugins/templates/hooks/pre_prompt.py,sha256=aOAMM623s0sKFGjTZaotAOYFvsNMxeii4tPyhOAFKVE,539
@@ -103,9 +104,9 @@ synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
-synapse_sdk-1.0.0a20.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a20.dist-info/METADATA,sha256=i8EQ5e9XfInYgkuud9Krg_jmj_MrhvDVA51UphXzeWY,1049
-synapse_sdk-1.0.0a20.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-synapse_sdk-1.0.0a20.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a20.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a20.dist-info/RECORD,,
+synapse_sdk-1.0.0a22.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a22.dist-info/METADATA,sha256=amlLIpkW9kqYRzaJueKGATMK0jY0WG4-2SfZP6cre4I,1049
+synapse_sdk-1.0.0a22.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+synapse_sdk-1.0.0a22.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a22.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a22.dist-info/RECORD,,