synapse-sdk 1.0.0a15__tar.gz → 1.0.0a17__tar.gz

{synapse_sdk-1.0.0a15/synapse_sdk.egg-info → synapse_sdk-1.0.0a17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a15
+Version: 1.0.0a17
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a15 → synapse_sdk-1.0.0a17}/synapse_sdk/clients/backend/ml.py

@@ -9,9 +9,7 @@ class MLClientMixin(BaseClient):
 
     def get_model(self, pk, params=None, url_conversion=None):
         path = f'models/{pk}/'
-        url_conversion = get_default_url_conversion(
-            url_conversion, files_fields=['files', 'parent.files'], is_list=False
-        )
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['file'], is_list=False)
         return self._get(path, params=params, url_conversion=url_conversion)
 
     def create_model(self, data):

{synapse_sdk-1.0.0a15 → synapse_sdk-1.0.0a17}/synapse_sdk/clients/base.py

@@ -62,7 +62,7 @@ class BaseClient:
         response = self._request('get', path, **kwargs)
         if url_conversion:
             if url_conversion['is_list']:
-                files_url_to_path_from_objs(response['results'], **url_conversion)
+                files_url_to_path_from_objs(response['results'], **url_conversion, is_async=True)
             else:
                 files_url_to_path_from_objs(response, **url_conversion)
         return response
@@ -80,14 +80,14 @@ class BaseClient:
         return self._request('delete', path, **kwargs)
 
     def _list(self, path, url_conversion=None, list_all=False, **kwargs):
-        response = self._get(path, url_conversion, **kwargs)
+        response = self._get(path, **kwargs)
         if list_all:
             return self._list_all(path, url_conversion, **kwargs), response['count']
         else:
             return response
 
-    def _list_all(self, path, url_conversion=None, **kwargs):
-        response = self._get(path, url_conversion, **kwargs)
+    def _list_all(self, path, url_conversion=None, params=None, **kwargs):
+        response = self._get(path, url_conversion, params=params, **kwargs)
         yield from response['results']
         if response['next']:
             yield from self._list_all(response['next'], url_conversion, **kwargs)

{synapse_sdk-1.0.0a15 → synapse_sdk-1.0.0a17}/synapse_sdk/plugins/categories/base.py

@@ -92,13 +92,15 @@ class Action:
     @property
     def debug_modules(self):
         debug_modules = []
-        for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
-            if module_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                module_url = module_path
-            else:
-                module_url = build_and_upload(module_path, self.plugin_storage_url)
-            debug_modules.append(module_url)
-        self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
+        if self.envs.get('SYNAPSE_DEBUG_MODULES'):
+            for module_path in self.envs['SYNAPSE_DEBUG_MODULES'].split(','):
+                # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
+                if module_path.startswith('https://'):
+                    module_url = module_path
+                else:
+                    module_url = build_and_upload(module_path, self.plugin_storage_url)
+                debug_modules.append(module_url)
+            self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
     def get_run(self):

{synapse_sdk-1.0.0a15 → synapse_sdk-1.0.0a17}/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -12,7 +12,7 @@ from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 from synapse_sdk.plugins.models import Run
-from synapse_sdk.utils.file import archive
+from synapse_sdk.utils.file import archive, get_temp_path, unarchive
 from synapse_sdk.utils.pydantic.validators import non_blank
 
 
@@ -83,10 +83,15 @@ class TrainAction(Action):
         self.run.log_message('Preparing dataset for training.')
         input_dataset = self.get_dataset()
 
+        # retrieve checkpoint
+        checkpoint = None
+        if self.params['checkpoint']:
+            self.run.log_message('Retrieving checkpoint.')
+            checkpoint = self.get_model(self.params['checkpoint'])
+
         # train dataset
         self.run.log_message('Starting model training.')
-
-        result = self.entrypoint(self.run, input_dataset, hyperparameter)
+        result = self.entrypoint(self.run, input_dataset, hyperparameter, checkpoint=checkpoint)
 
         # upload model_data
         self.run.log_message('Registering model data.')
@@ -106,6 +111,7 @@ class TrainAction(Action):
         ground_truths, count_dataset = client.list_ground_truth_events(
             params={
                 'fields': ['category', 'files', 'data'],
+                'expand': ['data'],
                 'ground_truth_dataset_versions': self.params['dataset'],
             },
             list_all=True,
@@ -120,11 +126,16 @@ class TrainAction(Action):
 
         return input_dataset
 
-    def create_model(self, path):
-        if not self.client:
-            print(path)
-            return None
+    def get_model(self, model_id):
+        model = self.client.get_model(model_id)
+        model_file = Path(model['file'])
+        output_path = get_temp_path(f'models/{model_file.stem}')
+        if not output_path.exists():
+            unarchive(model_file, output_path)
+        model['path'] = output_path
+        return model
 
+    def create_model(self, path):
         params = copy.deepcopy(self.params)
         configuration_fields = ['hyperparameter']
         configuration = {field: params.pop(field) for field in configuration_fields}

{synapse_sdk-1.0.0a15 → synapse_sdk-1.0.0a17}/synapse_sdk/plugins/cli/publish.py

@@ -26,8 +26,8 @@ def publish(ctx, host, user_token, tenant, debug_modules):
 
     data = {'plugin': plugin_release.plugin, 'file': str(archive_path), 'debug': debug}
     if debug:
-        if debug_modules:
-            data['debug_meta'] = json.dumps({'modules': debug_modules.split(',')})
+        modules = debug_modules.split(',') if debug_modules else []
+        data['debug_meta'] = json.dumps({'modules': modules})
 
     client = BackendClient(host, user_token, tenant=tenant)
     client.create_plugin_release(data)

synapse_sdk-1.0.0a17/synapse_sdk/utils/file.py

@@ -0,0 +1,168 @@
+import asyncio
+import hashlib
+import json
+import operator
+import zipfile
+from functools import reduce
+from pathlib import Path
+from urllib.parse import urlparse
+
+import aiohttp
+import requests
+import yaml
+
+
+def download_file(url, path_download, name=None, coerce=None):
+    if name:
+        name += Path(url).suffix
+    else:
+        name = Path(url).name
+
+    name = urlparse(name).path
+    path = Path(path_download) / name
+    if not path.is_file():
+        r = requests.get(url, allow_redirects=True)
+        open(str(path), 'wb').write(r.content)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+def files_url_to_path(files, coerce=None, file_field=None):
+    path_download = Path('/tmp/datamaker') / 'media'
+    path_download.mkdir(parents=True, exist_ok=True)
+    if file_field:
+        files[file_field] = download_file(files[file_field], path_download, coerce=coerce)
+    else:
+        for file_name in files:
+            if isinstance(files[file_name], str):
+                files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
+            else:
+                files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False, is_async=False):
+    if is_async:
+        asyncio.run(afiles_url_to_path_from_objs(objs, files_fields, coerce=coerce, is_list=is_list))
+    else:
+        if not is_list:
+            objs = [objs]
+
+        for obj in objs:
+            for files_field in files_fields:
+                try:
+                    files = reduce(operator.getitem, files_field.split('.'), obj)
+                    if isinstance(files, str):
+                        files_url_to_path(obj, coerce=coerce, file_field=files_field)
+                    else:
+                        files_url_to_path(files, coerce=coerce)
+                except KeyError:
+                    pass
+
+
+async def adownload_file(url, path_download, name=None, coerce=None):
+    if name:
+        name += Path(url).suffix
+    else:
+        name = Path(url).name
+
+    name = urlparse(name).path
+    path = Path(path_download) / name
+    if not path.is_file():
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                with open(str(path), 'wb') as file:
+                    while chunk := await response.content.read(1024 * 64):
+                        file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+async def afiles_url_to_path(files, coerce=None):
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+    for file_name in files:
+        if isinstance(files[file_name], str):
+            files[file_name] = await adownload_file(files[file_name], path_download, coerce=coerce)
+        else:
+            files[file_name]['path'] = await adownload_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+async def afiles_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False):
+    if not is_list:
+        objs = [objs]
+
+    tasks = []
+
+    for obj in objs:
+        for files_field in files_fields:
+            try:
+                files = reduce(operator.getitem, files_field.split('.'), obj)
+                tasks.append(afiles_url_to_path(files, coerce=coerce))
+            except KeyError:
+                pass
+
+    await asyncio.gather(*tasks)
+
+
+def get_dict_from_file(file_path):
+    if isinstance(file_path, str):
+        file_path = Path(file_path)
+
+    with open(file_path) as f:
+        if file_path.suffix == '.yaml':
+            return yaml.safe_load(f)
+        else:
+            return json.load(f)
+
+
+def calculate_checksum(file_path, prefix=''):
+    md5_hash = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b''):
+            md5_hash.update(byte_block)
+    checksum = md5_hash.hexdigest()
+    if prefix:
+        return f'dev-{checksum}'
+    return checksum
+
+
+def archive(input_path, output_path):
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+
+    with zipfile.ZipFile(output_path, mode='w', compression=zipfile.ZIP_DEFLATED) as zipf:
+        if input_path.is_file():
+            zipf.write(input_path, input_path.name)
+        else:
+            for file_path in input_path.rglob('*'):
+                if file_path.is_file():  # Only add files, skip directories
+                    arcname = file_path.relative_to(input_path.parent)
+                    zipf.write(file_path, arcname)
+
+
+def unarchive(file_path, output_path):
+    """
+    Unarchives a ZIP file to a given directory.
+
+    Parameters:
+        file_path (str | Path): The path to the ZIP file.
+        output_path (str): The directory where the files will be extracted.
+    """
+    output_path = Path(output_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(str(file_path), 'r') as zip_ref:
+        zip_ref.extractall(output_path)
+
+
+def get_temp_path(sub_path=None):
+    path = Path('/tmp/datamaker')
+    if sub_path:
+        path = path / sub_path
+    return path

{synapse_sdk-1.0.0a15 → synapse_sdk-1.0.0a17/synapse_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a15
+Version: 1.0.0a17
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

synapse_sdk-1.0.0a15/synapse_sdk/utils/file.py

@@ -1,87 +0,0 @@
-import hashlib
-import json
-import operator
-import zipfile
-from functools import reduce
-from pathlib import Path
-from urllib.parse import urlparse
-
-import requests
-import yaml
-
-
-def download_file(url, path_download, name=None, coerce=None):
-    if name:
-        name += Path(url).suffix
-    else:
-        name = Path(url).name
-
-    name = urlparse(name).path
-    path = Path(path_download) / name
-    if not path.is_file():
-        r = requests.get(url, allow_redirects=True)
-        open(str(path), 'wb').write(r.content)
-
-    if coerce:
-        path = coerce(path)
-
-    return path
-
-
-def files_url_to_path(files, coerce=None):
-    path_download = Path('/tmp/datamaker') / 'media'
-    path_download.mkdir(parents=True, exist_ok=True)
-    for file_name in files:
-        if isinstance(files[file_name], str):
-            files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
-        else:
-            files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
-
-
-def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False):
-    if not is_list:
-        objs = [objs]
-
-    for obj in objs:
-        for files_field in files_fields:
-            try:
-                files = reduce(operator.getitem, files_field.split('.'), obj)
-                files_url_to_path(files, coerce=coerce)
-            except KeyError:
-                pass
-
-
-def get_dict_from_file(file_path):
-    if isinstance(file_path, str):
-        file_path = Path(file_path)
-
-    with open(file_path) as f:
-        if file_path.suffix == '.yaml':
-            return yaml.safe_load(f)
-        else:
-            return json.load(f)
-
-
-def calculate_checksum(file_path, prefix=''):
-    md5_hash = hashlib.md5()
-    with open(file_path, 'rb') as f:
-        for byte_block in iter(lambda: f.read(4096), b''):
-            md5_hash.update(byte_block)
-    checksum = md5_hash.hexdigest()
-    if prefix:
-        return f'dev-{checksum}'
-    return checksum
-
-
-def archive(input_path, output_path):
-    input_path = Path(input_path)
-    output_path = Path(output_path)
-
-    with zipfile.ZipFile(output_path, mode='w', compression=zipfile.ZIP_DEFLATED) as zipf:
-        if input_path.is_file():
-            zipf.write(input_path, input_path.name)
-        else:
-            for file_path in input_path.rglob('*'):
-                if file_path.is_file():  # Only add files, skip directories
-                    arcname = file_path.relative_to(input_path.parent)
-                    zipf.write(file_path, arcname)