synapse-sdk 1.0.0a83__tar.gz → 1.0.0a85__tar.gz

{synapse_sdk-1.0.0a83 → synapse_sdk-1.0.0a85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a83
+Version: 1.0.0a85
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a83 → synapse_sdk-1.0.0a85}/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -22,7 +22,14 @@ def export(run, export_items, path_root, **params):
         dict: Result
     """
 
-    path_root.mkdir(parents=True, exist_ok=True)
+    export_path = path_root / params['name']
+    unique_export_path = export_path
+    counter = 1
+    while unique_export_path.exists():
+        unique_export_path = export_path.with_name(f'{export_path.name}({counter})')
+        counter += 1
+    unique_export_path.mkdir(parents=True)
+
     run.log_message('Starting export process.')
 
     # results contains all information fetched through the list API.
@@ -34,12 +41,13 @@ def export(run, export_items, path_root, **params):
     errors_original_file_list = []
 
     # Path to save JSON files
-    json_output_path = path_root / 'json'
+    json_output_path = unique_export_path / 'json'
     json_output_path.mkdir(parents=True, exist_ok=True)
 
     # Path to save original files
-    origin_files_output_path = path_root / 'origin_files'
-    origin_files_output_path.mkdir(parents=True, exist_ok=True)
+    if save_original_file_flag:
+        origin_files_output_path = unique_export_path / 'origin_files'
+        origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
     total = params['count']
     original_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
@@ -88,7 +96,7 @@ def export(run, export_items, path_root, **params):
     # Save error list files
     if len(errors_json_file_list) > 0 or len(errors_original_file_list) > 0:
         export_error_file = {'json_file_name': errors_json_file_list, 'origin_file_name': errors_original_file_list}
-        with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
+        with (unique_export_path / 'error_file_list.json').open('w', encoding='utf-8') as f:
             json.dump(export_error_file, f, indent=4, ensure_ascii=False)
 
     return {'export_path': str(path_root)}

{synapse_sdk-1.0.0a83 → synapse_sdk-1.0.0a85}/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -53,7 +53,7 @@ class TrainParams(BaseModel):
                     'category': 'neural_net',
                     'job__action': 'train',
                     'is_active': True,
-                    'params': f'name:{value}',
+                    'params': f'name:{value.replace(":", "%3A")}',
                 },
             )
             assert not model_exists and not job_exists, '존재하는 학습 이름입니다.'

synapse_sdk-1.0.0a85/synapse_sdk/utils/storage/providers/__init__.py

@@ -0,0 +1,183 @@
+from urllib.parse import parse_qs, urlparse
+
+
+class BaseStorage:
+    """Base storage provider class for all storage implementations.
+
+    This is an abstract base class that defines the interface for storage providers.
+    All storage providers (S3, GCP, SFTP, HTTP, FileSystem) must inherit from this
+    class and implement all the abstract methods.
+
+    Args:
+        connection_params (str | dict): The connection parameters. Can be either:
+            - A URL string (e.g., 's3://bucket/path?access_key=key&secret_key=secret')
+            - A dict with 'provider' and 'configuration' keys
+
+    Attributes:
+        url: Parsed URL object if connection_params was a string
+        base_path: Base path for storage operations (provider-specific)
+        options: Storage-specific options
+        query_params: Parsed query parameters from URL or configuration dict
+        OPTION_CASTS: Dictionary mapping option names to type casting functions
+
+    Example:
+        >>> # URL-based initialization
+        >>> storage = SomeStorage('s3://bucket/path?access_key=key&secret_key=secret')
+        >>>
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 's3',
+        ...     'configuration': {
+        ...         'bucket_name': 'my-bucket',
+        ...         'access_key': 'key',
+        ...         'secret_key': 'secret'
+        ...     }
+        ... }
+        >>> storage = SomeStorage(config)
+    """
+
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, connection_params: str | dict):
+        """Initialize the storage provider with connection parameters.
+
+        Args:
+            connection_params (str | dict): Connection parameters for the storage provider.
+                If string, should be a valid URL with scheme and query parameters.
+                If dict, should contain 'provider' and 'configuration' keys.
+        """
+        self.url = None
+
+        if isinstance(connection_params, dict):
+            self.query_params = connection_params['configuration']
+        else:
+            self.url = urlparse(connection_params)
+            self.query_params = self.url_querystring_to_dict()
+
+    def url_querystring_to_dict(self):
+        """Parse URL query string into a dictionary with proper type casting.
+
+        Converts query parameters from the URL into a dictionary, applying
+        type casting based on OPTION_CASTS mapping. Single-value lists are
+        flattened to scalar values.
+
+        Returns:
+            dict: Parsed and type-cast query parameters.
+        """
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to the source file on local filesystem.
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL or identifier of the uploaded file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError
+
+    def exists(self, target):
+        """Check if a file exists at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            bool: True if the file exists, False otherwise.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError
+
+    def get_url(self, target):
+        """Get the URL for accessing a file at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL that can be used to access the file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError
+
+    def get_pathlib(self, path):
+        """Get the path as a pathlib-compatible object.
+
+        Args:
+            path (str): The path to convert, relative to the storage root.
+
+        Returns:
+            pathlib.Path or UPath: A pathlib-compatible object representing the path.
+            The exact type depends on the storage provider implementation.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            Different storage providers may return different path object types:
+            - FileSystemStorage returns pathlib.Path objects
+            - Cloud storage providers typically return UPath objects
+        """
+        raise NotImplementedError
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the total number of files in the given path.
+
+        Args:
+            pathlib_obj (Path | UPath): The path object to count files in.
+                Should be obtained from get_pathlib().
+
+        Returns:
+            int: The total number of files found recursively in the path.
+                Returns 1 for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method counts files recursively, including files in subdirectories.
+            Directories themselves are not counted, only regular files.
+        """
+        raise NotImplementedError
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of all files in the given path.
+
+        Args:
+            pathlib_obj (Path | UPath): The path object to calculate size for.
+                Should be obtained from get_pathlib().
+
+        Returns:
+            int: The total size in bytes of all files found recursively in the path.
+                Returns the file size for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method calculates the total size recursively, including files
+            in subdirectories. Only regular files contribute to the total size.
+        """
+        raise NotImplementedError

synapse_sdk-1.0.0a85/synapse_sdk/utils/storage/providers/file_system.py

@@ -0,0 +1,126 @@
+import shutil
+from pathlib import Path
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class FileSystemStorage(BaseStorage):
+    """Storage provider for file system.
+
+    * This storage do not support url based initialization.
+
+    Args:
+        url (str): The URL of the storage provider.
+
+    Examples:
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 'filesystem',
+        ...     'configuration': {
+        ...         'location': '/data'
+        ...     }
+        ... }
+        >>> storage = FileSystemStorage(config)
+    """
+
+    def __init__(self, url):
+        super().__init__(url)
+        self.base_path = Path(self.query_params['location'])
+
+    def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to source file
+            target (str): Target path relative to base path
+
+        Returns:
+            str: URL of uploaded file
+        """
+        source_path = Path(source)
+        target_path = self.base_path / target
+
+        # Create parent directories if they don't exist
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Copy the file
+        shutil.copy2(source_path, target_path)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        """Check if target file exists.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            bool: True if file exists, False otherwise
+        """
+        target_path = self.base_path / target
+        return target_path.exists()
+
+    def get_url(self, target):
+        """Get URL for target file.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            str: File URL
+        """
+        target_path = self.base_path / target
+        return f'file://{target_path.absolute()}'
+
+    def get_pathlib(self, path):
+        """Get the path as a pathlib object.
+
+        Args:
+            path (str): The path to convert.
+
+        Returns:
+            pathlib.Path: The converted path.
+        """
+        return self.base_path / path
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the file count in the path.
+
+        Args:
+            pathlib_obj (Path): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return 1
+
+        count = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of the path.
+
+        Args:
+            pathlib_obj (Path): The path to get total size.
+
+        Returns:
+            int: The total size of the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return pathlib_obj.stat().st_size
+
+        total_size = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                total_size += item.stat().st_size
+        return total_size

{synapse_sdk-1.0.0a83 → synapse_sdk-1.0.0a85}/synapse_sdk/utils/storage/registry.py

@@ -1,3 +1,4 @@
+from synapse_sdk.utils.storage.providers.file_system import FileSystemStorage
 from synapse_sdk.utils.storage.providers.gcp import GCPStorage
 from synapse_sdk.utils.storage.providers.http import HTTPStorage
 from synapse_sdk.utils.storage.providers.s3 import S3Storage
@@ -12,4 +13,5 @@ STORAGE_PROVIDERS = {
     'sftp': SFTPStorage,
     'http': HTTPStorage,
     'https': HTTPStorage,
+    'file_system': FileSystemStorage,
 }

{synapse_sdk-1.0.0a83 → synapse_sdk-1.0.0a85}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a83
+Version: 1.0.0a85
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a83 → synapse_sdk-1.0.0a85}/synapse_sdk.egg-info/SOURCES.txt

@@ -289,6 +289,7 @@ synapse_sdk/utils/pydantic/validators.py
 synapse_sdk/utils/storage/__init__.py
 synapse_sdk/utils/storage/registry.py
 synapse_sdk/utils/storage/providers/__init__.py
+synapse_sdk/utils/storage/providers/file_system.py
 synapse_sdk/utils/storage/providers/gcp.py
 synapse_sdk/utils/storage/providers/http.py
 synapse_sdk/utils/storage/providers/s3.py
@@ -421,4 +422,5 @@ tests/utils/converters/fixtures/data_types/text/dm/ver2/json/23811.json
 tests/utils/converters/fixtures/data_types/video/dm/ver1/json/23831.json
 tests/utils/converters/fixtures/data_types/video/dm/ver2/json/23831.json
 tests/utils/storage/conftest.py
+tests/utils/storage/test_file_system_storage.py
 tests/utils/storage/test_provider_selection.py