synapse-sdk 1.0.0a28__py3-none-any.whl → 1.0.0a30__py3-none-any.whl

synapse_sdk/cli/plugin/__init__.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import click
 from dotenv import load_dotenv
@@ -12,6 +13,7 @@ from .run import run
 
 load_dotenv_default_alias()
 load_dotenv(os.path.join(os.getcwd(), '.env'), override=True)
+sys.path.append(os.getcwd())
 
 
 @click.group(context_settings={'obj': {}, 'auto_envvar_prefix': 'SYNAPSE_PLUGIN'})

synapse_sdk/clients/backend/integration.py

@@ -1,4 +1,5 @@
 from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.utils.file import convert_file_to_base64
 
 
 class IntegrationClientMixin(BaseClient):
@@ -60,6 +61,13 @@ class IntegrationClientMixin(BaseClient):
 
     def create_logs(self, data):
         path = 'logs/'
+        if not isinstance(data, list):
+            data = [data]
+
+        for item in data:
+            if 'file' in item:
+                item['file'] = convert_file_to_base64(item['file'])
+
         return self._post(path, data=data)
 
     def create_serve_application(self, data):
@@ -69,3 +77,7 @@ class IntegrationClientMixin(BaseClient):
     def list_serve_applications(self, params=None, list_all=False):
         path = 'serve_applications/'
         return self._list(path, params=params, list_all=list_all)
+
+    def get_storage(self, pk):
+        path = f'storages/{pk}/'
+        return self._get(path)

synapse_sdk/clients/backend/ml.py

@@ -22,3 +22,7 @@ class MLClientMixin(BaseClient):
         path = 'ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def get_ground_truth_version(self, pk):
+        path = f'ground_truth_dataset_versions/{pk}/'
+        return self._get(path)

synapse_sdk/clients/base.py

@@ -25,20 +25,46 @@ class BaseClient:
     def _get_headers(self):
         return {}
 
-    def _request(self, method, path, **kwargs):
+    def _request(self, method: str, path: str, **kwargs) -> dict | str:
+        """Request handler for all HTTP methods.
+
+        Args:
+            method (str): HTTP method to use.
+            path (str): URL path to request.
+            **kwargs: Additional keyword arguments to pass to the request.
+
+        Returns:
+            dict | str: JSON response or text response.
+        """
         url = self._get_url(path)
         headers = self._get_headers()
         headers.update(kwargs.pop('headers', {}))
 
         if method in ['post', 'put', 'patch']:
+            # If files are included in the request, open them as binary files
             if kwargs.get('files') is not None:
-                for name, file in kwargs['files'].items():
-                    if isinstance(file, (str, Path)):
-                        kwargs['files'][name] = Path(str(file)).open(mode='rb')
-                if 'data' in kwargs:
-                    for name, value in kwargs['data'].items():
-                        if isinstance(value, dict):
-                            kwargs['data'][name] = json.dumps(value)
+                opened_files = []  # List to store opened files
+                try:
+                    for name, file in kwargs['files'].items():
+                        # If file is a path string, bind it as a Path object and open
+                        if isinstance(file, str):
+                            opened_file = open(Path(file), mode='rb')
+                            kwargs['files'][name] = opened_file
+                            opened_files.append(opened_file)
+                        # If file is a Path object, open it directly
+                        elif isinstance(file, Path):
+                            opened_file = open(file, mode='rb')
+                            kwargs['files'][name] = opened_file
+                            opened_files.append(opened_file)
+                    if 'data' in kwargs:
+                        for name, value in kwargs['data'].items():
+                            if isinstance(value, dict):
+                                kwargs['data'][name] = json.dumps(value)
+                finally:
+                    # Close all opened files
+                    for opened_file in opened_files:
+                        opened_file.close()
+
             else:
                 headers['Content-Type'] = 'application/json'
                 if 'data' in kwargs:

synapse_sdk/loggers.py

@@ -68,13 +68,16 @@ class BaseLogger:
 
         return progress
 
+    def log(self, action, data, file=None):
+        raise NotImplementedError
+
 
 class ConsoleLogger(BaseLogger):
     def set_progress(self, current, total, category=None):
         super().set_progress(current, total, category=category)
         print(self.get_current_progress())
 
-    def log(self, action, data):
+    def log(self, action, data, file=None):
         print(action, data)
 
 
@@ -99,7 +102,7 @@ class BackendLogger(BaseLogger):
         except ClientError:
             pass
 
-    def log(self, event, data):
+    def log(self, event, data, file=None):
         print(event, data)
 
         log = {
@@ -108,7 +111,11 @@ class BackendLogger(BaseLogger):
             'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
             'job': self.job_id,
         }
+        if file:
+            log['file'] = file
+
         self.logs_queue.append(log)
+
         try:
             self.client.create_logs(self.logs_queue)
             self.logs_queue.clear()

synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,6 +1,42 @@
+from pydantic import BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class ExportParams(BaseModel):
+    storage: int
+    save_original_file: bool = True
+    path: str
+    ground_truth_dataset_version: int
+    filter: dict
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('ground_truth_dataset_version')
+    @staticmethod
+    def check_ground_truth_dataset_version_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_ground_truth_version(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
+        return value
 
 
 @register_action
@@ -8,3 +44,42 @@ class ExportAction(Action):
     name = 'export'
     category = PluginCategory.EXPORT
     method = RunMethod.JOB
+    params_model = ExportParams
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+
+    def get_dataset(self, results):
+        """Get dataset for export."""
+        for result in results:
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'],
+                'id': result['ground_truth'],
+            }
+
+    def get_filtered_results(self):
+        """Get filtered ground truth."""
+        ground_truth_dataset_version = self.params['ground_truth_dataset_version']
+        self.params['filter']['ground_truth_dataset'] = self.client.get_ground_truth_version(
+            ground_truth_dataset_version
+        )['ground_truth_dataset']
+        filters = {'expand': 'data', **self.params['filter']}
+
+        try:
+            gt_dataset_events_list = self.client.list_ground_truth_events(params=filters, list_all=True)
+            results = gt_dataset_events_list[0]
+            count = gt_dataset_events_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+        return results, count
+
+    def start(self):
+        self.params['results'], self.params['count'] = self.get_filtered_results()
+        dataset = self.get_dataset(self.params['results'])
+
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        return self.entrypoint(self.run, dataset, pathlib_cwd, **self.params)

synapse_sdk/plugins/categories/export/actions/utils.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+
+def get_original_file_path(files):
+    return Path(next(iter(files.values()))['meta']['path_original'])

synapse_sdk/plugins/categories/export/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  export:
+    entrypoint: plugin.export.export

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -0,0 +1,126 @@
+import json
+
+import requests
+
+from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
+
+
+def export(run, input_dataset, path_root, **params):
+    """Executes the export task.
+
+    Args:
+        run : Execution object
+        input_dataset (generator):
+            - data (dict): dm_schema_data information.
+            - files (dict): File information. Includes file URL, original file path, metadata, etc.
+            - id (int): ground_truth ID
+        path_root : Save path
+        **params: Additional parameters
+
+    Returns:
+        dict: Result
+    """
+
+    path_root.mkdir(parents=True, exist_ok=True)
+    run.log_message('Starting export process.')
+
+    # results: Contains all information fetched through the list API.
+    results = params.get('results', [])
+
+    save_original_file = params.get('save_original_file')
+    errors_json_file_list = []
+    errors_original_file_list = []
+
+    # Path to save JSON files
+    json_output_path = path_root / 'json'
+    json_output_path.mkdir(parents=True, exist_ok=True)
+
+    # Path to save original files
+    origin_files_output_path = path_root / 'origin_files'
+    origin_files_output_path.mkdir(parents=True, exist_ok=True)
+
+    total = len(results)
+    for no, input_data in enumerate(input_dataset):
+        run.set_progress(no, total, category='dataset_conversion')
+        preprocessed_data = before_convert(input_data)
+        converted_data = convert_data(preprocessed_data)
+        final_data = after_convert(converted_data)
+
+        # Call if original file extraction is needed
+        if save_original_file:
+            save_original_file(final_data, origin_files_output_path, errors_original_file_list)
+
+        # Extract data as JSON files
+        save_as_json(final_data, json_output_path, errors_json_file_list)
+
+    run.log_message('Saving converted dataset.')
+    run.end_log()
+
+    # Save error list files
+    if len(errors_json_file_list) > 0 or len(errors_original_file_list) > 0:
+        export_error_file = {'json_file_name': errors_json_file_list, 'origin_file_name': errors_original_file_list}
+        with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
+            json.dump(export_error_file, f, indent=4, ensure_ascii=False)
+
+    return {'export_path': path_root}
+
+
+def convert_data(data):
+    """Converts the data."""
+    return data
+
+
+def before_convert(data):
+    """Preprocesses the data before conversion."""
+    return data
+
+
+def after_convert(data):
+    """Post-processes the data after conversion."""
+    return data
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    file_url = next(iter(result['files'].values()))['url']
+    file_name = get_original_file_path(result['files']).name
+    response = requests.get(file_url)
+    try:
+        with (base_path / file_name).open('wb') as file:
+            file.write(response.content)
+    except Exception as e:
+        error_file_list.append([file_name, str(e)])
+
+    return base_path
+
+
+def save_as_json(result, base_path, error_file_list):
+    """Saves the data as a JSON file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    # Default save file name: original file name
+    file_name = get_original_file_path(result['files']).stem
+    json_data = result['data']
+    try:
+        with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
+            json.dump(json_data, f, indent=4, ensure_ascii=False)
+    except Exception as e:
+        error_file_list.append([f'{file_name}.json', str(e)])
+
+    return base_path

synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -18,8 +18,13 @@ from synapse_sdk.utils.pydantic.validators import non_blank
 
 class TrainRun(Run):
     def log_metric(self, category, key, value, **metrics):
+        # TODO validate input via plugin config
         self.log('metric', {'category': category, 'key': key, 'value': value, 'metrics': metrics})
 
+    def log_visualization(self, category, group, image, **meta):
+        # TODO validate input via plugin config
+        self.log('visualization', {'category': category, 'group': group, **meta}, file=image)
+
 
 class Hyperparameter(BaseModel):
     batch_size: int

synapse_sdk/plugins/categories/neural_net/templates/config.yaml

@@ -8,6 +8,9 @@ actions:
       epoch:
         - loss
         - miou
+    hyperparameters:
+      ui_schema: |
+        Dumped FormKit Schema for hyperparameters
   deployment:
     entrypoint: plugin.inference.MockNetInference
   inference:

synapse_sdk/plugins/models.py

@@ -113,8 +113,8 @@ class Run:
     def set_progress(self, current, total, category=''):
         self.logger.set_progress(current, total, category)
 
-    def log(self, event, data):
-        self.logger.log(event, data)
+    def log(self, event, data, file=None):
+        self.logger.log(event, data, file=file)
 
     def log_message(self, message, context=Context.INFO.value):
         self.logger.log('message', {'context': context, 'content': message})

synapse_sdk/utils/file.py

@@ -1,6 +1,8 @@
 import asyncio
+import base64
 import hashlib
 import json
+import mimetypes
 import operator
 import zipfile
 from functools import reduce
@@ -182,3 +184,36 @@ def get_temp_path(sub_path=None):
     if sub_path:
         path = path / sub_path
     return path
+
+
+def convert_file_to_base64(file_path):
+    """
+    Convert a file to base64 using pathlib.
+
+    Args:
+        file_path (str): Path to the file to convert
+
+    Returns:
+        str: Base64 encoded string of the file contents
+    """
+    # Convert string path to Path object
+    path = Path(file_path)
+
+    try:
+        # Read binary content of the file
+        binary_content = path.read_bytes()
+
+        # Convert to base64
+        base64_encoded = base64.b64encode(binary_content).decode('utf-8')
+
+        # Get the MIME type of the file
+        mime_type, _ = mimetypes.guess_type(path)
+        assert mime_type is not None, 'MIME type cannot be guessed'
+
+        # Convert bytes to string for readable output
+        return f'data:{mime_type};base64,{base64_encoded}'
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f'File not found: {file_path}')
+    except Exception as e:
+        raise Exception(f'Error converting file to base64: {str(e)}')

synapse_sdk/utils/storage/__init__.py

@@ -0,0 +1,20 @@
+from urllib.parse import urlparse
+
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.storage.registry import STORAGE_PROVIDERS
+
+
+def get_storage(connection_param: str | dict):
+    storage_scheme = None
+    if isinstance(connection_param, dict):
+        storage_scheme = connection_param['provider']
+    else:
+        storage_scheme = urlparse(connection_param).scheme
+
+    assert storage_scheme in STORAGE_PROVIDERS.keys(), _('지원하지 않는 저장소입니다.')
+    return STORAGE_PROVIDERS[storage_scheme](connection_param)
+
+
+def get_pathlib(storage_config, path_root):
+    storage_class = get_storage(storage_config)
+    return storage_class.get_pathlib(path_root)

synapse_sdk/utils/storage/providers/__init__.py

@@ -0,0 +1,42 @@
+from urllib.parse import parse_qs, urlparse
+
+
+class BaseStorage:
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, connection_params: str | dict):
+        self.url = None
+
+        if isinstance(connection_params, dict):
+            self.query_params = connection_params['configuration']
+        else:
+            self.url = urlparse(connection_params)
+            self.query_params = self.url_querystring_to_dict()
+
+    def url_querystring_to_dict(self):
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def upload(self, source, target):
+        raise NotImplementedError
+
+    def exists(self, target):
+        raise NotImplementedError
+
+    def get_url(self, target):
+        raise NotImplementedError
+
+    def get_pathlib(self, path):
+        raise NotImplementedError

synapse_sdk/utils/storage/providers/gcp.py

@@ -0,0 +1,13 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class GCPStorage(BaseStorage):
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = UPath(f'gs://{self.query_params["bucket_name"]}', token=self.query_params['credentials'])
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk/utils/storage/providers/s3.py

@@ -0,0 +1,43 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class S3Storage(BaseStorage):
+    ENDPOINT_URL = 'https://s3.amazonaws.com'
+    DEFAULT_REGION = 'us-east-1'
+
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = self._get_upath()
+
+    def _get_upath(self):
+        upath_kwargs = {
+            'key': self.query_params['access_key'],
+            'secret': self.query_params['secret_key'],
+            'client_kwargs': {'region_name': self.query_params.get('region_name')},
+        }
+
+        if self.query_params.get('endpoint_url'):
+            upath_kwargs['endpoint_url'] = self.query_params['endpoint_url']
+
+        return UPath(
+            f's3://{self.query_params["bucket_name"]}',
+            **upath_kwargs,
+        )
+
+    def upload(self, source, target):
+        with open(source, 'rb') as file:
+            self.upath.write_text(file.read(), target)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        return self.upath.exists(target)
+
+    def get_url(self, target):
+        return str(self.upath.joinuri(target))
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk/utils/storage/providers/sftp.py

@@ -0,0 +1,16 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class SFTPStorage(BaseStorage):
+    def get_pathlib(self, path):
+        credentials = self.query_params['params']
+        host = self.query_params['host']
+        root_path = self.query_params['root_path']
+
+        username = credentials['username']
+        password = credentials['password']
+        if path == '/':
+            path = ''
+        return UPath(f'sftp://{host}', username=username, password=password) / root_path / path

synapse_sdk/utils/storage/registry.py

@@ -0,0 +1,11 @@
+from synapse_sdk.utils.storage.providers.gcp import GCPStorage
+from synapse_sdk.utils.storage.providers.s3 import S3Storage
+from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
+
+STORAGE_PROVIDERS = {
+    's3': S3Storage,
+    'amazon_s3': S3Storage,
+    'minio': S3Storage,
+    'gcp': GCPStorage,
+    'sftp': SFTPStorage,
+}

{synapse_sdk-1.0.0a28.dist-info → synapse_sdk-1.0.0a30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a28
+Version: 1.0.0a30
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -17,6 +17,8 @@ Requires-Dist: python-dotenv
 Requires-Dist: pyyaml
 Requires-Dist: pydantic
 Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
 

{synapse_sdk-1.0.0a28.dist-info → synapse_sdk-1.0.0a30.dist-info}/RECORD

@@ -4,7 +4,7 @@ locale/ko/LC_MESSAGES/messages.mo,sha256=7HJEJA0wKlN14xQ5VF4FCNet54tjw6mfWYj3IaB
 locale/ko/LC_MESSAGES/messages.po,sha256=TFii_RbURDH-Du_9ZQf3wNh-2briGk1IqY33-9GKrMU,1126
 synapse_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/i18n.py,sha256=VXMR-Zm_1hTAg9iPk3YZNNq-T1Bhx1J2fEtRT6kyYbg,766
-synapse_sdk/loggers.py,sha256=RsDDOiOeUCih1XOkWQJseYdYCX_wt50AZJRe6aPf96Q,4004
+synapse_sdk/loggers.py,sha256=OSTDDhEAvj8fiAuYNZqsZ9bygGM20sMC5yJ_nOaLDDU,4155
 synapse_sdk/types.py,sha256=khzn8KpgxFdn1SrpbcuX84m_Md1Mz_HIoUoPq8uok40,698
 synapse_sdk/cli/__init__.py,sha256=P-_FXCqb_nTVdQznuHop6kDXF_JuncZpeAmgHiGoILQ,152
 synapse_sdk/cli/alias/__init__.py,sha256=jDy8N_KupVy7n_jKKWhjQOj76-mR-uoVvMoyzObUkuI,405
@@ -16,12 +16,12 @@ synapse_sdk/cli/alias/list.py,sha256=fsKe5MVyl9ki3bwEwDj1nWAWfnNKOmkLO3bbbHOfsL4
 synapse_sdk/cli/alias/read.py,sha256=gkcSO6zAqvqHMduKIajvzftEgYphp4pIVjsIShgPoHk,349
 synapse_sdk/cli/alias/update.py,sha256=LOpkpe6ZP_HcOOwC5RkJHVxQuNcQHGnyN98tABjz754,531
 synapse_sdk/cli/alias/utils.py,sha256=9kWrDfRCFqlkCtLXSleMJ-PH6Q49R1FfnbfyY34ck5s,1540
-synapse_sdk/cli/plugin/__init__.py,sha256=P5WMpTe_cVvaXdZngk8_ugJynRdel9rxVicwi05U3zo,744
+synapse_sdk/cli/plugin/__init__.py,sha256=fonDXnEc33Pz4ibFhp3WILLyElHDwMnXvaw0tZy6tak,784
 synapse_sdk/cli/plugin/create.py,sha256=HpYTpohV1NbSrULaVUlc4jWLWznPrx7glgydTM3sS5E,218
 synapse_sdk/cli/plugin/publish.py,sha256=sIl1wiuSC3lAUpE3rOF4UDKDy2G5EVLlelMjk2aT05g,1221
 synapse_sdk/cli/plugin/run.py,sha256=xz5LRm3zh8Y9DMjw5FFRFVRWSCWtYfZJskfCmrPikaQ,2598
 synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/clients/base.py,sha256=q5jIsL2ygRq8ivLWH42E3njvx4RexjsjALZqhtGxbxM,3511
+synapse_sdk/clients/base.py,sha256=l4_9BvqKykE9EiW9KygTRBf4tFUZLutG4CfwFIo2LRg,4710
 synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRTAjwk,220
 synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
 synapse_sdk/clients/agent/__init__.py,sha256=Pz8_iTbIbnb7ywGJ3feqoZVmO2I3mEbwpWsISIxh0BU,1968
@@ -32,15 +32,15 @@ synapse_sdk/clients/backend/__init__.py,sha256=aozhPhvRTPHz1P90wxEay07B-Ct4vj_yT
 synapse_sdk/clients/backend/annotation.py,sha256=eZc5EidgR_RfMGwvv1r1_mLkPdRd8e52c4zuuMjMX34,979
 synapse_sdk/clients/backend/core.py,sha256=5XAOdo6JZ0drfk-FMPJ96SeTd9oja-VnTwzGXdvK7Bg,1027
 synapse_sdk/clients/backend/dataset.py,sha256=w7izflbTjHKysiDl7ia7MAO391_dzN2ofK40A7QwtBQ,1721
-synapse_sdk/clients/backend/integration.py,sha256=Jg_8fEmbrgYXfZZcG8cDtLxR6ugPmnbNhPDyRu_Uib0,2160
-synapse_sdk/clients/backend/ml.py,sha256=jlkqS9pI0S0Ubq4pWVeaaaPk-E0J-cZg5zkSX7GuQ_o,1063
+synapse_sdk/clients/backend/integration.py,sha256=MCfeChpLySqlVRc1aZxCfDpQiRH--pfevkCdJDCNZEQ,2506
+synapse_sdk/clients/backend/ml.py,sha256=JoPH9Ly2E3HJ7S5mdGLtcGq7ruQVVrYfWArogwZLlms,1193
 synapse_sdk/clients/ray/__init__.py,sha256=9ZSPXVVxlJ8Wp8ku7l021ENtPjVrGgQDgqifkkVAXgM,187
 synapse_sdk/clients/ray/core.py,sha256=a4wyCocAma2HAm-BHlbZnoVbpfdR-Aad2FM0z6vPFvw,731
 synapse_sdk/clients/ray/serve.py,sha256=rbCpXZYWf0oP8XJ9faa9QFNPYU7h8dltIG8xn9ZconY,907
 synapse_sdk/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/enums.py,sha256=s59P6Oz2WAK9IX-kLVhNOvNKYJifKlWBhPpZbc9-ttE,486
 synapse_sdk/plugins/exceptions.py,sha256=Qs7qODp_RRLO9y2otU2T4ryj5LFwIZODvSIXkAh91u0,691
-synapse_sdk/plugins/models.py,sha256=JFaU1aqzminwWasNKuRBYd5hrBTRHDOZkl2nqZ1l0IM,3687
+synapse_sdk/plugins/models.py,sha256=7En1biVK_7kR8aI-3I-kJ-lXbveRRobppMGOeFd3ZpU,3709
 synapse_sdk/plugins/upload.py,sha256=VJOotYMayylOH0lNoAGeGHRkLdhP7jnC_A0rFQMvQpQ,3228
 synapse_sdk/plugins/utils.py,sha256=4_K6jIl0WrsXOEhFp94faMOriSsddOhIiaXcawYYUUA,3300
 synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -56,7 +56,11 @@ synapse_sdk/plugins/categories/data_validation/templates/plugin/__init__.py,sha2
 synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py,sha256=90I5boUpEXvO3mEuKKBs528ls2A4h8Iw4ReOID2h00Y,139
 synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/actions/export.py,sha256=GboGbFAUFPFqEedqO65L8IbX3AczrIFDoHbHBPaFCN0,320
+synapse_sdk/plugins/categories/export/actions/export.py,sha256=GNmsrJncf7WaxNfnN9I1sNiYYxMbqPhcbpnDcBQC1DM,3062
+synapse_sdk/plugins/categories/export/actions/utils.py,sha256=ixilT_VtgwtbX9kbBTrIybbAyW1kPLhNEkOHABKmZ88,131
+synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=N7YmnFROb3s3M35SA9nmabyzoSb5O2t2TRPicwFNN2o,56
+synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=422HPdxCTor0PlOwSw7J7-HyD3i0kq_1D_VjKegkr8c,3904
 synapse_sdk/plugins/categories/import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/import/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/import/actions/import.py,sha256=URn6TOp081odMT5D4NlZ2XEcyKelJx8fxzdoKSkXSAI,320
@@ -65,10 +69,10 @@ synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBS
 synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=Wmi7in_Mgizt1d5XcDR080h1CIMWKh2_mjub9N380qA,1917
 synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=0a655ELqNVjPFZTJDiw4EUdcMCPGveUEKyoYqpwMFBU,1019
 synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtMkGoo_qNqoaZkp3AQNEypJmeGzEog,320
-synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=4zq2ryWjqDa6MyI2BGe3sAofMh2NeY3XAo-1gEFXBs4,5145
+synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=CBg8ky2-bvS5pZlzuf49HcdpyronHkSugwq0UeHhiKg,5401
 synapse_sdk/plugins/categories/neural_net/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=R5DASI6-5vzsjDOYxqeGGMBjnav5qHF4hNJT8zNUR3I,1097
-synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=dXKB1hO53hDZB73xnxLVCNQl8Sm7svMmVmuMrOCQmEU,343
+synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=vSTHgKzi7sHzsuUvhOT001hzR6JBpPCue5nECmruEnI,433
 synapse_sdk/plugins/categories/neural_net/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py,sha256=InfqKWJYi6sqiUnfPKHC5KYGhxckDaWZNQ202u-uVP4,366
 synapse_sdk/plugins/categories/neural_net/templates/plugin/test.py,sha256=kYyk7l4UtcDUAH4nkdVUGrHHHjxI4p1U13HSLnmGPyE,53
@@ -105,18 +109,23 @@ synapse_sdk/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 synapse_sdk/shared/enums.py,sha256=WMZPag9deVF7VCXaQkLk7ly_uX1KwbNzRx9TdvgaeFE,138
 synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/debug.py,sha256=F7JlUwYjTFZAMRbBqKm6hxOIz-_IXYA8lBInOS4jbS4,100
-synapse_sdk/utils/file.py,sha256=eF1GDxjPaq2QOiKKCHSzgCkBRYSBSwP4rKOAApAQR3E,5661
+synapse_sdk/utils/file.py,sha256=zP8eOZifGiYP9PyC4ivQwxs-ljbtXRtbWN4yOjZF6tc,6658
 synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
 synapse_sdk/utils/network.py,sha256=wg-oFM0gKK5REqIUO8d-x9yXJfqbnkSbbF0_qyxpwz4,412
-synapse_sdk/utils/storage.py,sha256=a8OVbd38ATr0El4G4kuV07lr_tJZrpIJBSy4GHb0qZ8,2581
 synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
 synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
-synapse_sdk-1.0.0a28.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a28.dist-info/METADATA,sha256=--5rhzhDRDXJ5YbfkpzJcfW7tUHyPqrx44W2EAez2P8,1070
-synapse_sdk-1.0.0a28.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-synapse_sdk-1.0.0a28.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a28.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a28.dist-info/RECORD,,
+synapse_sdk/utils/storage/__init__.py,sha256=oDRyXwwp9xxdCkNtHzDF2-cUwv6EMhKt4rWYbyAKaEk,689
+synapse_sdk/utils/storage/registry.py,sha256=WaSN9SJR7s9sZgmTVl5k4mLFz-9R6X4ii82wefxs95A,335
+synapse_sdk/utils/storage/providers/__init__.py,sha256=NM9yRIWcPkH53DeNHgIhH9zaDFK8SJv0KptP1Afulyw,1125
+synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
+synapse_sdk/utils/storage/providers/s3.py,sha256=r94aUGVNf8yxihU0lN62yaXdxSS7P783_sfD-hCiK40,1191
+synapse_sdk/utils/storage/providers/sftp.py,sha256=TUQXkKJf0-fh8NhGC_1zzqI4autFNHZVCqngwkQ1aD4,523
+synapse_sdk-1.0.0a30.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a30.dist-info/METADATA,sha256=CZ9X66Gy-UkyIZic1g3FSh9kfCVJ-lWSkvON9qox20M,1138
+synapse_sdk-1.0.0a30.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+synapse_sdk-1.0.0a30.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a30.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a30.dist-info/RECORD,,

{synapse_sdk-1.0.0a28.dist-info → synapse_sdk-1.0.0a30.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

synapse_sdk/utils/storage.py

@@ -1,91 +0,0 @@
-import os
-from urllib.parse import parse_qs, urlparse
-
-import boto3
-from botocore.exceptions import ClientError
-
-
-class BaseStorage:
-    url = None
-    options = None
-    OPTION_CASTS = {}
-
-    def __init__(self, url):
-        self.url = urlparse(url)
-        self.query_params = self.url_querystring_to_dict()
-        self.options = self.get_options()
-
-    def url_querystring_to_dict(self):
-        query_string = self.url.query
-
-        query_dict = parse_qs(query_string)
-
-        for key, value in query_dict.items():
-            if len(value) == 1:
-                query_dict[key] = value[0]
-
-        return {
-            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
-            for key, value in query_dict.items()
-        }
-
-    def get_options(self):
-        return None
-
-    def upload(self, source, target):
-        raise NotImplementedError
-
-    def exists(self, target):
-        raise NotImplementedError
-
-    def get_url(self, target):
-        raise NotImplementedError
-
-
-class S3Storage(BaseStorage):
-    def __init__(self, url):
-        super().__init__(url)
-        self.client = boto3.client(
-            's3',
-            endpoint_url=self.options['endpoint_url'],
-            aws_access_key_id=self.options['access_key'],
-            aws_secret_access_key=self.options['secret_key'],
-        )
-
-    def get_options(self):
-        base_url = f'https://{self.url.hostname}'
-        local_endpoint = self.query_params.get('local_endpoint')
-        endpoint_url = f'http://{local_endpoint}' if local_endpoint else base_url
-        return {
-            'base_url': base_url,
-            'endpoint_url': endpoint_url,
-            'bucket_name': self.url.path[1:],
-            'access_key': self.url.username,
-            'secret_key': self.url.password,
-            **self.query_params,
-        }
-
-    def upload(self, source, target):
-        object_name = os.path.join(self.options['location'], target)
-        self.client.upload_file(source, self.options['bucket_name'], object_name)
-        return self.get_url(target)
-
-    def exists(self, target):
-        try:
-            self.client.head_object(Bucket=self.options['bucket_name'], Key=target)
-            return True
-        except ClientError:
-            return False
-
-    def get_url(self, target):
-        return os.path.join(self.options['base_url'], self.options['bucket_name'], self.options['location'], target)
-
-
-STORAGE_STORAGES = {
-    's3': S3Storage,
-}
-
-
-def get_storage(url):
-    storage_scheme = urlparse(url).scheme
-    return STORAGE_STORAGES[storage_scheme](url)