synapse-sdk 1.0.0a29__py3-none-any.whl → 1.0.0a30__py3-none-any.whl

synapse_sdk/cli/plugin/__init__.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import click
 from dotenv import load_dotenv
@@ -12,6 +13,7 @@ from .run import run
 
 load_dotenv_default_alias()
 load_dotenv(os.path.join(os.getcwd(), '.env'), override=True)
+sys.path.append(os.getcwd())
 
 
 @click.group(context_settings={'obj': {}, 'auto_envvar_prefix': 'SYNAPSE_PLUGIN'})

synapse_sdk/clients/backend/integration.py

@@ -77,3 +77,7 @@ class IntegrationClientMixin(BaseClient):
     def list_serve_applications(self, params=None, list_all=False):
         path = 'serve_applications/'
         return self._list(path, params=params, list_all=list_all)
+
+    def get_storage(self, pk):
+        path = f'storages/{pk}/'
+        return self._get(path)

synapse_sdk/clients/backend/ml.py

@@ -22,3 +22,7 @@ class MLClientMixin(BaseClient):
         path = 'ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def get_ground_truth_version(self, pk):
+        path = f'ground_truth_dataset_versions/{pk}/'
+        return self._get(path)

synapse_sdk/clients/base.py

@@ -25,20 +25,46 @@ class BaseClient:
     def _get_headers(self):
         return {}
 
-    def _request(self, method, path, **kwargs):
+    def _request(self, method: str, path: str, **kwargs) -> dict | str:
+        """Request handler for all HTTP methods.
+
+        Args:
+            method (str): HTTP method to use.
+            path (str): URL path to request.
+            **kwargs: Additional keyword arguments to pass to the request.
+
+        Returns:
+            dict | str: JSON response or text response.
+        """
         url = self._get_url(path)
         headers = self._get_headers()
         headers.update(kwargs.pop('headers', {}))
 
         if method in ['post', 'put', 'patch']:
+            # If files are included in the request, open them as binary files
             if kwargs.get('files') is not None:
-                for name, file in kwargs['files'].items():
-                    if isinstance(file, (str, Path)):
-                        kwargs['files'][name] = Path(str(file)).open(mode='rb')
-                if 'data' in kwargs:
-                    for name, value in kwargs['data'].items():
-                        if isinstance(value, dict):
-                            kwargs['data'][name] = json.dumps(value)
+                opened_files = []  # List to store opened files
+                try:
+                    for name, file in kwargs['files'].items():
+                        # If file is a path string, bind it as a Path object and open
+                        if isinstance(file, str):
+                            opened_file = open(Path(file), mode='rb')
+                            kwargs['files'][name] = opened_file
+                            opened_files.append(opened_file)
+                        # If file is a Path object, open it directly
+                        elif isinstance(file, Path):
+                            opened_file = open(file, mode='rb')
+                            kwargs['files'][name] = opened_file
+                            opened_files.append(opened_file)
+                    if 'data' in kwargs:
+                        for name, value in kwargs['data'].items():
+                            if isinstance(value, dict):
+                                kwargs['data'][name] = json.dumps(value)
+                finally:
+                    # Close all opened files
+                    for opened_file in opened_files:
+                        opened_file.close()
+
             else:
                 headers['Content-Type'] = 'application/json'
                 if 'data' in kwargs:

synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,6 +1,42 @@
+from pydantic import BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class ExportParams(BaseModel):
+    storage: int
+    save_original_file: bool = True
+    path: str
+    ground_truth_dataset_version: int
+    filter: dict
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('ground_truth_dataset_version')
+    @staticmethod
+    def check_ground_truth_dataset_version_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_ground_truth_version(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
+        return value
 
 
 @register_action
@@ -8,3 +44,42 @@ class ExportAction(Action):
     name = 'export'
     category = PluginCategory.EXPORT
     method = RunMethod.JOB
+    params_model = ExportParams
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+
+    def get_dataset(self, results):
+        """Get dataset for export."""
+        for result in results:
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'],
+                'id': result['ground_truth'],
+            }
+
+    def get_filtered_results(self):
+        """Get filtered ground truth."""
+        ground_truth_dataset_version = self.params['ground_truth_dataset_version']
+        self.params['filter']['ground_truth_dataset'] = self.client.get_ground_truth_version(
+            ground_truth_dataset_version
+        )['ground_truth_dataset']
+        filters = {'expand': 'data', **self.params['filter']}
+
+        try:
+            gt_dataset_events_list = self.client.list_ground_truth_events(params=filters, list_all=True)
+            results = gt_dataset_events_list[0]
+            count = gt_dataset_events_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+        return results, count
+
+    def start(self):
+        self.params['results'], self.params['count'] = self.get_filtered_results()
+        dataset = self.get_dataset(self.params['results'])
+
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        return self.entrypoint(self.run, dataset, pathlib_cwd, **self.params)

synapse_sdk/plugins/categories/export/actions/utils.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+
+def get_original_file_path(files):
+    return Path(next(iter(files.values()))['meta']['path_original'])

synapse_sdk/plugins/categories/export/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  export:
+    entrypoint: plugin.export.export

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -0,0 +1,126 @@
+import json
+
+import requests
+
+from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
+
+
+def export(run, input_dataset, path_root, **params):
+    """Executes the export task.
+
+    Args:
+        run : Execution object
+        input_dataset (generator):
+            - data (dict): dm_schema_data information.
+            - files (dict): File information. Includes file URL, original file path, metadata, etc.
+            - id (int): ground_truth ID
+        path_root : Save path
+        **params: Additional parameters
+
+    Returns:
+        dict: Result
+    """
+
+    path_root.mkdir(parents=True, exist_ok=True)
+    run.log_message('Starting export process.')
+
+    # results: Contains all information fetched through the list API.
+    results = params.get('results', [])
+
+    save_original_file = params.get('save_original_file')
+    errors_json_file_list = []
+    errors_original_file_list = []
+
+    # Path to save JSON files
+    json_output_path = path_root / 'json'
+    json_output_path.mkdir(parents=True, exist_ok=True)
+
+    # Path to save original files
+    origin_files_output_path = path_root / 'origin_files'
+    origin_files_output_path.mkdir(parents=True, exist_ok=True)
+
+    total = len(results)
+    for no, input_data in enumerate(input_dataset):
+        run.set_progress(no, total, category='dataset_conversion')
+        preprocessed_data = before_convert(input_data)
+        converted_data = convert_data(preprocessed_data)
+        final_data = after_convert(converted_data)
+
+        # Call if original file extraction is needed
+        if save_original_file:
+            save_original_file(final_data, origin_files_output_path, errors_original_file_list)
+
+        # Extract data as JSON files
+        save_as_json(final_data, json_output_path, errors_json_file_list)
+
+    run.log_message('Saving converted dataset.')
+    run.end_log()
+
+    # Save error list files
+    if len(errors_json_file_list) > 0 or len(errors_original_file_list) > 0:
+        export_error_file = {'json_file_name': errors_json_file_list, 'origin_file_name': errors_original_file_list}
+        with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
+            json.dump(export_error_file, f, indent=4, ensure_ascii=False)
+
+    return {'export_path': path_root}
+
+
+def convert_data(data):
+    """Converts the data."""
+    return data
+
+
+def before_convert(data):
+    """Preprocesses the data before conversion."""
+    return data
+
+
+def after_convert(data):
+    """Post-processes the data after conversion."""
+    return data
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    file_url = next(iter(result['files'].values()))['url']
+    file_name = get_original_file_path(result['files']).name
+    response = requests.get(file_url)
+    try:
+        with (base_path / file_name).open('wb') as file:
+            file.write(response.content)
+    except Exception as e:
+        error_file_list.append([file_name, str(e)])
+
+    return base_path
+
+
+def save_as_json(result, base_path, error_file_list):
+    """Saves the data as a JSON file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    # Default save file name: original file name
+    file_name = get_original_file_path(result['files']).stem
+    json_data = result['data']
+    try:
+        with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
+            json.dump(json_data, f, indent=4, ensure_ascii=False)
+    except Exception as e:
+        error_file_list.append([f'{file_name}.json', str(e)])
+
+    return base_path

synapse_sdk/plugins/categories/neural_net/templates/config.yaml

@@ -8,6 +8,9 @@ actions:
       epoch:
         - loss
         - miou
+    hyperparameters:
+      ui_schema: |
+        Dumped FormKit Schema for hyperparameters
   deployment:
     entrypoint: plugin.inference.MockNetInference
   inference:

synapse_sdk/utils/storage/__init__.py

@@ -0,0 +1,20 @@
+from urllib.parse import urlparse
+
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.storage.registry import STORAGE_PROVIDERS
+
+
+def get_storage(connection_param: str | dict):
+    storage_scheme = None
+    if isinstance(connection_param, dict):
+        storage_scheme = connection_param['provider']
+    else:
+        storage_scheme = urlparse(connection_param).scheme
+
+    assert storage_scheme in STORAGE_PROVIDERS.keys(), _('지원하지 않는 저장소입니다.')
+    return STORAGE_PROVIDERS[storage_scheme](connection_param)
+
+
+def get_pathlib(storage_config, path_root):
+    storage_class = get_storage(storage_config)
+    return storage_class.get_pathlib(path_root)

synapse_sdk/utils/storage/providers/__init__.py

@@ -0,0 +1,42 @@
+from urllib.parse import parse_qs, urlparse
+
+
+class BaseStorage:
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, connection_params: str | dict):
+        self.url = None
+
+        if isinstance(connection_params, dict):
+            self.query_params = connection_params['configuration']
+        else:
+            self.url = urlparse(connection_params)
+            self.query_params = self.url_querystring_to_dict()
+
+    def url_querystring_to_dict(self):
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def upload(self, source, target):
+        raise NotImplementedError
+
+    def exists(self, target):
+        raise NotImplementedError
+
+    def get_url(self, target):
+        raise NotImplementedError
+
+    def get_pathlib(self, path):
+        raise NotImplementedError

synapse_sdk/utils/storage/providers/gcp.py

@@ -0,0 +1,13 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class GCPStorage(BaseStorage):
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = UPath(f'gs://{self.query_params["bucket_name"]}', token=self.query_params['credentials'])
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk/utils/storage/providers/s3.py

@@ -0,0 +1,43 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class S3Storage(BaseStorage):
+    ENDPOINT_URL = 'https://s3.amazonaws.com'
+    DEFAULT_REGION = 'us-east-1'
+
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = self._get_upath()
+
+    def _get_upath(self):
+        upath_kwargs = {
+            'key': self.query_params['access_key'],
+            'secret': self.query_params['secret_key'],
+            'client_kwargs': {'region_name': self.query_params.get('region_name')},
+        }
+
+        if self.query_params.get('endpoint_url'):
+            upath_kwargs['endpoint_url'] = self.query_params['endpoint_url']
+
+        return UPath(
+            f's3://{self.query_params["bucket_name"]}',
+            **upath_kwargs,
+        )
+
+    def upload(self, source, target):
+        with open(source, 'rb') as file:
+            self.upath.write_text(file.read(), target)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        return self.upath.exists(target)
+
+    def get_url(self, target):
+        return str(self.upath.joinuri(target))
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk/utils/storage/providers/sftp.py

@@ -0,0 +1,16 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class SFTPStorage(BaseStorage):
+    def get_pathlib(self, path):
+        credentials = self.query_params['params']
+        host = self.query_params['host']
+        root_path = self.query_params['root_path']
+
+        username = credentials['username']
+        password = credentials['password']
+        if path == '/':
+            path = ''
+        return UPath(f'sftp://{host}', username=username, password=password) / root_path / path

synapse_sdk/utils/storage/registry.py

@@ -0,0 +1,11 @@
+from synapse_sdk.utils.storage.providers.gcp import GCPStorage
+from synapse_sdk.utils.storage.providers.s3 import S3Storage
+from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
+
+STORAGE_PROVIDERS = {
+    's3': S3Storage,
+    'amazon_s3': S3Storage,
+    'minio': S3Storage,
+    'gcp': GCPStorage,
+    'sftp': SFTPStorage,
+}

{synapse_sdk-1.0.0a29.dist-info → synapse_sdk-1.0.0a30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a29
+Version: 1.0.0a30
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -17,6 +17,8 @@ Requires-Dist: python-dotenv
 Requires-Dist: pyyaml
 Requires-Dist: pydantic
 Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
 

{synapse_sdk-1.0.0a29.dist-info → synapse_sdk-1.0.0a30.dist-info}/RECORD

@@ -16,12 +16,12 @@ synapse_sdk/cli/alias/list.py,sha256=fsKe5MVyl9ki3bwEwDj1nWAWfnNKOmkLO3bbbHOfsL4
 synapse_sdk/cli/alias/read.py,sha256=gkcSO6zAqvqHMduKIajvzftEgYphp4pIVjsIShgPoHk,349
 synapse_sdk/cli/alias/update.py,sha256=LOpkpe6ZP_HcOOwC5RkJHVxQuNcQHGnyN98tABjz754,531
 synapse_sdk/cli/alias/utils.py,sha256=9kWrDfRCFqlkCtLXSleMJ-PH6Q49R1FfnbfyY34ck5s,1540
-synapse_sdk/cli/plugin/__init__.py,sha256=P5WMpTe_cVvaXdZngk8_ugJynRdel9rxVicwi05U3zo,744
+synapse_sdk/cli/plugin/__init__.py,sha256=fonDXnEc33Pz4ibFhp3WILLyElHDwMnXvaw0tZy6tak,784
 synapse_sdk/cli/plugin/create.py,sha256=HpYTpohV1NbSrULaVUlc4jWLWznPrx7glgydTM3sS5E,218
 synapse_sdk/cli/plugin/publish.py,sha256=sIl1wiuSC3lAUpE3rOF4UDKDy2G5EVLlelMjk2aT05g,1221
 synapse_sdk/cli/plugin/run.py,sha256=xz5LRm3zh8Y9DMjw5FFRFVRWSCWtYfZJskfCmrPikaQ,2598
 synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/clients/base.py,sha256=q5jIsL2ygRq8ivLWH42E3njvx4RexjsjALZqhtGxbxM,3511
+synapse_sdk/clients/base.py,sha256=l4_9BvqKykE9EiW9KygTRBf4tFUZLutG4CfwFIo2LRg,4710
 synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRTAjwk,220
 synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
 synapse_sdk/clients/agent/__init__.py,sha256=Pz8_iTbIbnb7ywGJ3feqoZVmO2I3mEbwpWsISIxh0BU,1968
@@ -32,8 +32,8 @@ synapse_sdk/clients/backend/__init__.py,sha256=aozhPhvRTPHz1P90wxEay07B-Ct4vj_yT
 synapse_sdk/clients/backend/annotation.py,sha256=eZc5EidgR_RfMGwvv1r1_mLkPdRd8e52c4zuuMjMX34,979
 synapse_sdk/clients/backend/core.py,sha256=5XAOdo6JZ0drfk-FMPJ96SeTd9oja-VnTwzGXdvK7Bg,1027
 synapse_sdk/clients/backend/dataset.py,sha256=w7izflbTjHKysiDl7ia7MAO391_dzN2ofK40A7QwtBQ,1721
-synapse_sdk/clients/backend/integration.py,sha256=ekuMqpJ2OIt8ko3NISZCIvtoer7mhvDjXUgPXV4ju6E,2410
-synapse_sdk/clients/backend/ml.py,sha256=jlkqS9pI0S0Ubq4pWVeaaaPk-E0J-cZg5zkSX7GuQ_o,1063
+synapse_sdk/clients/backend/integration.py,sha256=MCfeChpLySqlVRc1aZxCfDpQiRH--pfevkCdJDCNZEQ,2506
+synapse_sdk/clients/backend/ml.py,sha256=JoPH9Ly2E3HJ7S5mdGLtcGq7ruQVVrYfWArogwZLlms,1193
 synapse_sdk/clients/ray/__init__.py,sha256=9ZSPXVVxlJ8Wp8ku7l021ENtPjVrGgQDgqifkkVAXgM,187
 synapse_sdk/clients/ray/core.py,sha256=a4wyCocAma2HAm-BHlbZnoVbpfdR-Aad2FM0z6vPFvw,731
 synapse_sdk/clients/ray/serve.py,sha256=rbCpXZYWf0oP8XJ9faa9QFNPYU7h8dltIG8xn9ZconY,907
@@ -56,7 +56,11 @@ synapse_sdk/plugins/categories/data_validation/templates/plugin/__init__.py,sha2
 synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py,sha256=90I5boUpEXvO3mEuKKBs528ls2A4h8Iw4ReOID2h00Y,139
 synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/actions/export.py,sha256=GboGbFAUFPFqEedqO65L8IbX3AczrIFDoHbHBPaFCN0,320
+synapse_sdk/plugins/categories/export/actions/export.py,sha256=GNmsrJncf7WaxNfnN9I1sNiYYxMbqPhcbpnDcBQC1DM,3062
+synapse_sdk/plugins/categories/export/actions/utils.py,sha256=ixilT_VtgwtbX9kbBTrIybbAyW1kPLhNEkOHABKmZ88,131
+synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=N7YmnFROb3s3M35SA9nmabyzoSb5O2t2TRPicwFNN2o,56
+synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=422HPdxCTor0PlOwSw7J7-HyD3i0kq_1D_VjKegkr8c,3904
 synapse_sdk/plugins/categories/import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/import/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/import/actions/import.py,sha256=URn6TOp081odMT5D4NlZ2XEcyKelJx8fxzdoKSkXSAI,320
@@ -68,7 +72,7 @@ synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtM
 synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=CBg8ky2-bvS5pZlzuf49HcdpyronHkSugwq0UeHhiKg,5401
 synapse_sdk/plugins/categories/neural_net/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=R5DASI6-5vzsjDOYxqeGGMBjnav5qHF4hNJT8zNUR3I,1097
-synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=dXKB1hO53hDZB73xnxLVCNQl8Sm7svMmVmuMrOCQmEU,343
+synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=vSTHgKzi7sHzsuUvhOT001hzR6JBpPCue5nECmruEnI,433
 synapse_sdk/plugins/categories/neural_net/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py,sha256=InfqKWJYi6sqiUnfPKHC5KYGhxckDaWZNQ202u-uVP4,366
 synapse_sdk/plugins/categories/neural_net/templates/plugin/test.py,sha256=kYyk7l4UtcDUAH4nkdVUGrHHHjxI4p1U13HSLnmGPyE,53
@@ -108,15 +112,20 @@ synapse_sdk/utils/debug.py,sha256=F7JlUwYjTFZAMRbBqKm6hxOIz-_IXYA8lBInOS4jbS4,10
 synapse_sdk/utils/file.py,sha256=zP8eOZifGiYP9PyC4ivQwxs-ljbtXRtbWN4yOjZF6tc,6658
 synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
 synapse_sdk/utils/network.py,sha256=wg-oFM0gKK5REqIUO8d-x9yXJfqbnkSbbF0_qyxpwz4,412
-synapse_sdk/utils/storage.py,sha256=a8OVbd38ATr0El4G4kuV07lr_tJZrpIJBSy4GHb0qZ8,2581
 synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
 synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
-synapse_sdk-1.0.0a29.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a29.dist-info/METADATA,sha256=1DF4WZPzyfQO1HZzHzzvP0isb2pCpCmQyx-8u6YhKlA,1070
-synapse_sdk-1.0.0a29.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-synapse_sdk-1.0.0a29.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a29.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a29.dist-info/RECORD,,
+synapse_sdk/utils/storage/__init__.py,sha256=oDRyXwwp9xxdCkNtHzDF2-cUwv6EMhKt4rWYbyAKaEk,689
+synapse_sdk/utils/storage/registry.py,sha256=WaSN9SJR7s9sZgmTVl5k4mLFz-9R6X4ii82wefxs95A,335
+synapse_sdk/utils/storage/providers/__init__.py,sha256=NM9yRIWcPkH53DeNHgIhH9zaDFK8SJv0KptP1Afulyw,1125
+synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
+synapse_sdk/utils/storage/providers/s3.py,sha256=r94aUGVNf8yxihU0lN62yaXdxSS7P783_sfD-hCiK40,1191
+synapse_sdk/utils/storage/providers/sftp.py,sha256=TUQXkKJf0-fh8NhGC_1zzqI4autFNHZVCqngwkQ1aD4,523
+synapse_sdk-1.0.0a30.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a30.dist-info/METADATA,sha256=CZ9X66Gy-UkyIZic1g3FSh9kfCVJ-lWSkvON9qox20M,1138
+synapse_sdk-1.0.0a30.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+synapse_sdk-1.0.0a30.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a30.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a30.dist-info/RECORD,,

{synapse_sdk-1.0.0a29.dist-info → synapse_sdk-1.0.0a30.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

synapse_sdk/utils/storage.py

@@ -1,91 +0,0 @@
-import os
-from urllib.parse import parse_qs, urlparse
-
-import boto3
-from botocore.exceptions import ClientError
-
-
-class BaseStorage:
-    url = None
-    options = None
-    OPTION_CASTS = {}
-
-    def __init__(self, url):
-        self.url = urlparse(url)
-        self.query_params = self.url_querystring_to_dict()
-        self.options = self.get_options()
-
-    def url_querystring_to_dict(self):
-        query_string = self.url.query
-
-        query_dict = parse_qs(query_string)
-
-        for key, value in query_dict.items():
-            if len(value) == 1:
-                query_dict[key] = value[0]
-
-        return {
-            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
-            for key, value in query_dict.items()
-        }
-
-    def get_options(self):
-        return None
-
-    def upload(self, source, target):
-        raise NotImplementedError
-
-    def exists(self, target):
-        raise NotImplementedError
-
-    def get_url(self, target):
-        raise NotImplementedError
-
-
-class S3Storage(BaseStorage):
-    def __init__(self, url):
-        super().__init__(url)
-        self.client = boto3.client(
-            's3',
-            endpoint_url=self.options['endpoint_url'],
-            aws_access_key_id=self.options['access_key'],
-            aws_secret_access_key=self.options['secret_key'],
-        )
-
-    def get_options(self):
-        base_url = f'https://{self.url.hostname}'
-        local_endpoint = self.query_params.get('local_endpoint')
-        endpoint_url = f'http://{local_endpoint}' if local_endpoint else base_url
-        return {
-            'base_url': base_url,
-            'endpoint_url': endpoint_url,
-            'bucket_name': self.url.path[1:],
-            'access_key': self.url.username,
-            'secret_key': self.url.password,
-            **self.query_params,
-        }
-
-    def upload(self, source, target):
-        object_name = os.path.join(self.options['location'], target)
-        self.client.upload_file(source, self.options['bucket_name'], object_name)
-        return self.get_url(target)
-
-    def exists(self, target):
-        try:
-            self.client.head_object(Bucket=self.options['bucket_name'], Key=target)
-            return True
-        except ClientError:
-            return False
-
-    def get_url(self, target):
-        return os.path.join(self.options['base_url'], self.options['bucket_name'], self.options['location'], target)
-
-
-STORAGE_STORAGES = {
-    's3': S3Storage,
-}
-
-
-def get_storage(url):
-    storage_scheme = urlparse(url).scheme
-    return STORAGE_STORAGES[storage_scheme](url)