synapse-sdk 1.0.0a29__tar.gz → 1.0.0a31__tar.gz

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/.gitignore

@@ -7,6 +7,9 @@
 .idea
 *.iml
 
+# vscode
+.vscode
+
 # http
 /http/
 

{synapse_sdk-1.0.0a29/synapse_sdk.egg-info → synapse_sdk-1.0.0a31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a29
+Version: 1.0.0a31
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -17,6 +17,8 @@ Requires-Dist: python-dotenv
 Requires-Dist: pyyaml
 Requires-Dist: pydantic
 Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
 

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/pyproject.toml

@@ -21,7 +21,9 @@ dependencies = [
     "python-dotenv",
     "pyyaml",
     "pydantic",
-    "pyjwt"
+    "pyjwt",
+    "universal-pathlib",
+    "fsspec[gcs,s3,sftp]"
 ]
 dynamic = ["version"]
 

synapse_sdk-1.0.0a31/requirements.test.txt

@@ -0,0 +1 @@
+pytest

synapse_sdk-1.0.0a31/requirements.txt

@@ -0,0 +1,11 @@
+click
+cookiecutter
+requests
+tqdm
+pyyaml
+pydantic
+pyjwt
+python-dotenv
+ray[all]>=2.43.0
+fsspec[gcs,s3,sftp]==2025.2.0
+universal-pathlib==0.2.6

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk/cli/plugin/__init__.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import click
 from dotenv import load_dotenv
@@ -12,6 +13,7 @@ from .run import run
 
 load_dotenv_default_alias()
 load_dotenv(os.path.join(os.getcwd(), '.env'), override=True)
+sys.path.append(os.getcwd())
 
 
 @click.group(context_settings={'obj': {}, 'auto_envvar_prefix': 'SYNAPSE_PLUGIN'})

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk/clients/backend/integration.py

@@ -77,3 +77,7 @@ class IntegrationClientMixin(BaseClient):
     def list_serve_applications(self, params=None, list_all=False):
         path = 'serve_applications/'
         return self._list(path, params=params, list_all=list_all)
+
+    def get_storage(self, pk):
+        path = f'storages/{pk}/'
+        return self._get(path)

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk/clients/backend/ml.py

@@ -22,3 +22,7 @@ class MLClientMixin(BaseClient):
         path = 'ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def get_ground_truth_version(self, pk):
+        path = f'ground_truth_dataset_versions/{pk}/'
+        return self._get(path)

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk/clients/base.py

@@ -25,16 +25,38 @@ class BaseClient:
     def _get_headers(self):
         return {}
 
-    def _request(self, method, path, **kwargs):
+    def _request(self, method: str, path: str, **kwargs) -> dict | str:
+        """Request handler for all HTTP methods.
+
+        Args:
+            method (str): HTTP method to use.
+            path (str): URL path to request.
+            **kwargs: Additional keyword arguments to pass to the request.
+
+        Returns:
+            dict | str: JSON response or text response.
+        """
         url = self._get_url(path)
         headers = self._get_headers()
         headers.update(kwargs.pop('headers', {}))
 
+        # List to store opened files to close after request
+        opened_files = []
+
         if method in ['post', 'put', 'patch']:
+            # If files are included in the request, open them as binary files
             if kwargs.get('files') is not None:
                 for name, file in kwargs['files'].items():
-                    if isinstance(file, (str, Path)):
-                        kwargs['files'][name] = Path(str(file)).open(mode='rb')
+                    # If file is a path string, bind it as a Path object and open
+                    if isinstance(file, str):
+                        opened_file = Path(file).open(mode='rb')
+                        kwargs['files'][name] = opened_file
+                        opened_files.append(opened_file)
+                    # If file is a Path object, open it directly
+                    elif isinstance(file, Path):
+                        opened_file = file.open(mode='rb')
+                        kwargs['files'][name] = opened_file
+                        opened_files.append(opened_file)
                 if 'data' in kwargs:
                     for name, value in kwargs['data'].items():
                         if isinstance(value, dict):
@@ -53,6 +75,10 @@ class BaseClient:
         except requests.ConnectionError:
             raise ClientError(408, f'{self.name} is not responding')
 
+        # Close all opened files
+        for opened_file in opened_files:
+            opened_file.close()
+
         return self._post_response(response)
 
     def _post_response(self, response):

synapse_sdk-1.0.0a31/synapse_sdk/plugins/categories/export/actions/export.py

@@ -0,0 +1,82 @@
+from pydantic import BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class ExportParams(BaseModel):
+    storage: int
+    save_original_file: bool = True
+    path: str
+    ground_truth_dataset_version: int
+    filter: dict
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('ground_truth_dataset_version')
+    @staticmethod
+    def check_ground_truth_dataset_version_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_ground_truth_version(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
+        return value
+
+
+@register_action
+class ExportAction(Action):
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB
+    params_model = ExportParams
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+
+    def get_dataset(self, results):
+        """Get dataset for export."""
+        for result in results:
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'],
+                'id': result['ground_truth'],
+            }
+
+    def get_filtered_results(self):
+        """Get filtered ground truth events."""
+        self.params['filter']['ground_truth_dataset_versions'] = self.params['ground_truth_dataset_version']
+        filters = {'expand': 'data', **self.params['filter']}
+
+        try:
+            gt_dataset_events_list = self.client.list_ground_truth_events(params=filters, list_all=True)
+            results = gt_dataset_events_list[0]
+            count = gt_dataset_events_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+        return results, count
+
+    def start(self):
+        self.params['results'], self.params['count'] = self.get_filtered_results()
+        dataset = self.get_dataset(self.params['results'])
+
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        return self.entrypoint(self.run, dataset, pathlib_cwd, **self.params)

synapse_sdk-1.0.0a31/synapse_sdk/plugins/categories/export/actions/utils.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+
+def get_original_file_path(files):
+    return Path(next(iter(files.values()))['meta']['path_original'])

synapse_sdk-1.0.0a31/synapse_sdk/plugins/categories/export/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  export:
+    entrypoint: plugin.export.export

synapse_sdk-1.0.0a31/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -0,0 +1,126 @@
+import json
+
+import requests
+
+from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
+
+
+def export(run, input_dataset, path_root, **params):
+    """Executes the export task.
+
+    Args:
+        run : Execution object
+        input_dataset (generator):
+            - data (dict): dm_schema_data information.
+            - files (dict): File information. Includes file URL, original file path, metadata, etc.
+            - id (int): ground_truth ID
+        path_root : Save path
+        **params: Additional parameters
+
+    Returns:
+        dict: Result
+    """
+
+    path_root.mkdir(parents=True, exist_ok=True)
+    run.log_message('Starting export process.')
+
+    # results: Contains all information fetched through the list API.
+    results = params.get('results', [])
+
+    save_original_file = params.get('save_original_file')
+    errors_json_file_list = []
+    errors_original_file_list = []
+
+    # Path to save JSON files
+    json_output_path = path_root / 'json'
+    json_output_path.mkdir(parents=True, exist_ok=True)
+
+    # Path to save original files
+    origin_files_output_path = path_root / 'origin_files'
+    origin_files_output_path.mkdir(parents=True, exist_ok=True)
+
+    total = len(results)
+    for no, input_data in enumerate(input_dataset):
+        run.set_progress(no, total, category='dataset_conversion')
+        preprocessed_data = before_convert(input_data)
+        converted_data = convert_data(preprocessed_data)
+        final_data = after_convert(converted_data)
+
+        # Call if original file extraction is needed
+        if save_original_file:
+            save_original_file(final_data, origin_files_output_path, errors_original_file_list)
+
+        # Extract data as JSON files
+        save_as_json(final_data, json_output_path, errors_json_file_list)
+
+    run.log_message('Saving converted dataset.')
+    run.end_log()
+
+    # Save error list files
+    if len(errors_json_file_list) > 0 or len(errors_original_file_list) > 0:
+        export_error_file = {'json_file_name': errors_json_file_list, 'origin_file_name': errors_original_file_list}
+        with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
+            json.dump(export_error_file, f, indent=4, ensure_ascii=False)
+
+    return {'export_path': path_root}
+
+
+def convert_data(data):
+    """Converts the data."""
+    return data
+
+
+def before_convert(data):
+    """Preprocesses the data before conversion."""
+    return data
+
+
+def after_convert(data):
+    """Post-processes the data after conversion."""
+    return data
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    file_url = next(iter(result['files'].values()))['url']
+    file_name = get_original_file_path(result['files']).name
+    response = requests.get(file_url)
+    try:
+        with (base_path / file_name).open('wb') as file:
+            file.write(response.content)
+    except Exception as e:
+        error_file_list.append([file_name, str(e)])
+
+    return base_path
+
+
+def save_as_json(result, base_path, error_file_list):
+    """Saves the data as a JSON file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    # Default save file name: original file name
+    file_name = get_original_file_path(result['files']).stem
+    json_data = result['data']
+    try:
+        with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
+            json.dump(json_data, f, indent=4, ensure_ascii=False)
+    except Exception as e:
+        error_file_list.append([f'{file_name}.json', str(e)])
+
+    return base_path

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -51,7 +51,7 @@ class TrainParams(BaseModel):
                 params={
                     'ids_ex': action.job_id,
                     'category': 'neural_net',
-                    'action': 'train',
+                    'job__action': 'train',
                     'is_active': True,
                     'params': f'name:{value}',
                 },

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk/plugins/categories/neural_net/templates/config.yaml

@@ -8,6 +8,9 @@ actions:
       epoch:
         - loss
         - miou
+    hyperparameters:
+      ui_schema: |
+        Dumped FormKit Schema for hyperparameters
   deployment:
     entrypoint: plugin.inference.MockNetInference
   inference:

synapse_sdk-1.0.0a31/synapse_sdk/utils/storage/__init__.py

@@ -0,0 +1,20 @@
+from urllib.parse import urlparse
+
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.storage.registry import STORAGE_PROVIDERS
+
+
+def get_storage(connection_param: str | dict):
+    storage_scheme = None
+    if isinstance(connection_param, dict):
+        storage_scheme = connection_param['provider']
+    else:
+        storage_scheme = urlparse(connection_param).scheme
+
+    assert storage_scheme in STORAGE_PROVIDERS.keys(), _('지원하지 않는 저장소입니다.')
+    return STORAGE_PROVIDERS[storage_scheme](connection_param)
+
+
+def get_pathlib(storage_config, path_root):
+    storage_class = get_storage(storage_config)
+    return storage_class.get_pathlib(path_root)

synapse_sdk-1.0.0a31/synapse_sdk/utils/storage/providers/__init__.py

@@ -0,0 +1,42 @@
+from urllib.parse import parse_qs, urlparse
+
+
+class BaseStorage:
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, connection_params: str | dict):
+        self.url = None
+
+        if isinstance(connection_params, dict):
+            self.query_params = connection_params['configuration']
+        else:
+            self.url = urlparse(connection_params)
+            self.query_params = self.url_querystring_to_dict()
+
+    def url_querystring_to_dict(self):
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def upload(self, source, target):
+        raise NotImplementedError
+
+    def exists(self, target):
+        raise NotImplementedError
+
+    def get_url(self, target):
+        raise NotImplementedError
+
+    def get_pathlib(self, path):
+        raise NotImplementedError

synapse_sdk-1.0.0a31/synapse_sdk/utils/storage/providers/gcp.py

@@ -0,0 +1,13 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class GCPStorage(BaseStorage):
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = UPath(f'gs://{self.query_params["bucket_name"]}', token=self.query_params['credentials'])
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk-1.0.0a31/synapse_sdk/utils/storage/providers/s3.py

@@ -0,0 +1,43 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class S3Storage(BaseStorage):
+    ENDPOINT_URL = 'https://s3.amazonaws.com'
+    DEFAULT_REGION = 'us-east-1'
+
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = self._get_upath()
+
+    def _get_upath(self):
+        upath_kwargs = {
+            'key': self.query_params['access_key'],
+            'secret': self.query_params['secret_key'],
+            'client_kwargs': {'region_name': self.query_params.get('region_name')},
+        }
+
+        if self.query_params.get('endpoint_url'):
+            upath_kwargs['endpoint_url'] = self.query_params['endpoint_url']
+
+        return UPath(
+            f's3://{self.query_params["bucket_name"]}',
+            **upath_kwargs,
+        )
+
+    def upload(self, source, target):
+        with open(source, 'rb') as file:
+            self.upath.write_text(file.read(), target)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        return self.upath.exists(target)
+
+    def get_url(self, target):
+        return str(self.upath.joinuri(target))
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk-1.0.0a31/synapse_sdk/utils/storage/providers/sftp.py

@@ -0,0 +1,16 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class SFTPStorage(BaseStorage):
+    def get_pathlib(self, path):
+        credentials = self.query_params['params']
+        host = self.query_params['host']
+        root_path = self.query_params['root_path']
+
+        username = credentials['username']
+        password = credentials['password']
+        if path == '/':
+            path = ''
+        return UPath(f'sftp://{host}', username=username, password=password) / root_path / path

synapse_sdk-1.0.0a31/synapse_sdk/utils/storage/registry.py

@@ -0,0 +1,11 @@
+from synapse_sdk.utils.storage.providers.gcp import GCPStorage
+from synapse_sdk.utils.storage.providers.s3 import S3Storage
+from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
+
+STORAGE_PROVIDERS = {
+    's3': S3Storage,
+    'amazon_s3': S3Storage,
+    'minio': S3Storage,
+    'gcp': GCPStorage,
+    'sftp': SFTPStorage,
+}

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31/synapse_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a29
+Version: 1.0.0a31
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -17,6 +17,8 @@ Requires-Dist: python-dotenv
 Requires-Dist: pyyaml
 Requires-Dist: pydantic
 Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
 

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@
 LICENSE
 README.md
 pyproject.toml
+requirements.test.txt
 requirements.txt
 .github/workflows/lint.yml
 .github/workflows/pypi-publish.yml
@@ -73,6 +74,10 @@ synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py
 synapse_sdk/plugins/categories/export/__init__.py
 synapse_sdk/plugins/categories/export/actions/__init__.py
 synapse_sdk/plugins/categories/export/actions/export.py
+synapse_sdk/plugins/categories/export/actions/utils.py
+synapse_sdk/plugins/categories/export/templates/config.yaml
+synapse_sdk/plugins/categories/export/templates/plugin/__init__.py
+synapse_sdk/plugins/categories/export/templates/plugin/export.py
 synapse_sdk/plugins/categories/import/__init__.py
 synapse_sdk/plugins/categories/import/actions/__init__.py
 synapse_sdk/plugins/categories/import/actions/import.py
@@ -124,9 +129,17 @@ synapse_sdk/utils/debug.py
 synapse_sdk/utils/file.py
 synapse_sdk/utils/module_loading.py
 synapse_sdk/utils/network.py
-synapse_sdk/utils/storage.py
 synapse_sdk/utils/string.py
 synapse_sdk/utils/pydantic/__init__.py
 synapse_sdk/utils/pydantic/config.py
 synapse_sdk/utils/pydantic/errors.py
-synapse_sdk/utils/pydantic/validators.py
+synapse_sdk/utils/pydantic/validators.py
+synapse_sdk/utils/storage/__init__.py
+synapse_sdk/utils/storage/registry.py
+synapse_sdk/utils/storage/providers/__init__.py
+synapse_sdk/utils/storage/providers/gcp.py
+synapse_sdk/utils/storage/providers/s3.py
+synapse_sdk/utils/storage/providers/sftp.py
+tests/__init__.py
+tests/utils/__init__.py
+tests/utils/test_debug.py

{synapse_sdk-1.0.0a29 → synapse_sdk-1.0.0a31}/synapse_sdk.egg-info/requires.txt

@@ -7,6 +7,8 @@ python-dotenv
 pyyaml
 pydantic
 pyjwt
+universal-pathlib
+fsspec[gcs,s3,sftp]
 
 [all]
 ray[all]

synapse_sdk-1.0.0a31/tests/utils/test_debug.py

@@ -0,0 +1,6 @@
+from synapse_sdk.utils import debug
+
+
+def test_debug_get_message():
+    message = debug.get_message()
+    assert message == 'hello world from sdk'

synapse_sdk-1.0.0a29/requirements.txt

@@ -1,10 +0,0 @@
-boto3
-click
-cookiecutter
-requests
-tqdm
-pyyaml
-pydantic
-pyjwt
-python-dotenv
-ray[all]>=2.38.0

synapse_sdk-1.0.0a29/synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class ExportAction(Action):
-    name = 'export'
-    category = PluginCategory.EXPORT
-    method = RunMethod.JOB