synapse-sdk 1.0.0a82__py3-none-any.whl → 1.0.0a84__py3-none-any.whl

synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py

@@ -1,7 +1,7 @@
 import json
 from datetime import datetime
 from enum import Enum
-from typing import Annotated, Any, Dict, List, Optional
+from typing import Annotated, Any, Dict, List, Optional, Tuple
 
 import requests
 from pydantic import AfterValidator, BaseModel, field_validator
@@ -382,7 +382,7 @@ class ToTaskAction(Action):
         data_file = files.get(target_specification_name)
 
         # Extract primary file URL from task data
-        primary_file_url = self._extract_primary_file_url(task)
+        primary_file_url, primary_file_original_name = self._extract_primary_file_url(task)
         if not primary_file_url:
             error_msg = 'Primary image URL not found in task data'
             self.run.log_message(f'{error_msg} for task {task_id}')
@@ -396,6 +396,13 @@ class ToTaskAction(Action):
             return {'success': False, 'error': error_msg}
 
         data_file_url = data_file.get('url')
+        data_file_original_name = data_file.get('file_name_original')
+        if not data_file_original_name:
+            error_msg = 'File original name not found'
+            self.run.log_message(f'{error_msg} for task {task_id}')
+            self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
+            return {'success': False, 'error': error_msg}
+
         if not data_file_url:
             error_msg = 'URL not found'
             self.run.log_message(f'{error_msg} for task {task_id}')
@@ -406,7 +413,9 @@ class ToTaskAction(Action):
         try:
             # Convert data to task object
             annotation_to_task = self.entrypoint(self.run)
-            converted_data = annotation_to_task.convert_data_from_file(primary_file_url, data_file_url)
+            converted_data = annotation_to_task.convert_data_from_file(
+                primary_file_url, primary_file_original_name, data_file_url, data_file_original_name
+            )
         except requests.RequestException as e:
             error_msg = f'Failed to fetch data from URL: {str(e)}'
             self.run.log_message(f'{error_msg} for task {task_id}')
@@ -467,7 +476,7 @@ class ToTaskAction(Action):
                 return pre_processor_status
 
             # Extract primary file URL from task data
-            primary_file_url = self._extract_primary_file_url(task)
+            primary_file_url, _ = self._extract_primary_file_url(task)
             if not primary_file_url:
                 error_msg = 'Primary image URL not found in task data'
                 self.run.log_message(f'{error_msg} for task {task_id}')
@@ -609,23 +618,23 @@ class ToTaskAction(Action):
         except Exception as e:
             return {'success': False, 'error': f'Failed to restart pre-processor: {str(e)}'}
 
-    def _extract_primary_file_url(self, task: Dict[str, Any]) -> Optional[str]:
+    def _extract_primary_file_url(self, task: Dict[str, Any]) -> Tuple[str, str]:
         """Extract the primary file URL from task data.
 
         Args:
             task (Dict[str, Any]): The task data.
 
         Returns:
-            Optional[str]: The primary file URL if found, None otherwise.
+            Tuple[str, str]: The primary file URL and original name.
         """
         data_unit = task.get('data_unit', {})
         files = data_unit.get('files', {})
 
         for file_info in files.values():
             if isinstance(file_info, dict) and file_info.get('is_primary') and file_info.get('url'):
-                return file_info['url']
+                return file_info['url'], file_info['file_name_original']
 
-        return None
+        return None, None
 
     def _run_inference(
         self, client: BackendClient, pre_processor_code: str, pre_processor_version: str, primary_file_url: str

synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py

@@ -7,12 +7,20 @@ class AnnotationToTask:
         """
         self.run = run
 
-    def convert_data_from_file(self, primary_file_url: str, data_file_url: str) -> dict:
+    def convert_data_from_file(
+        self,
+        primary_file_url: str,
+        primary_file_original_name: str,
+        data_file_url: str,
+        data_file_original_name: str,
+    ) -> dict:
         """Convert the data from a file to a task object.
 
         Args:
             primary_file_url (str): primary file url.
+            primary_file_original_name (str): primary file original name.
             data_file_url (str): data file url.
+            data_file_original_name (str): data file original name.
 
         Returns:
             dict: The converted data.

synapse_sdk/utils/converters/coco/to_dm.py

@@ -116,12 +116,13 @@ class COCOToDMConverter(ToDMConverter):
             result[img_filename] = (dm_json, img_path)
         return result
 
-    def convert_single_file(self, data: Dict[str, Any], original_file: IO) -> Dict[str, Any]:
+    def convert_single_file(self, data: Dict[str, Any], original_file: IO, original_image_name: str) -> Dict[str, Any]:
         """Convert a single COCO annotation data and corresponding image to DM format.
 
         Args:
             data: COCO format data dictionary (JSON content)
             original_file: File object for the corresponding original image
+            original_image_name: Original image name
 
         Returns:
             Dictionary containing DM format data for the single file
@@ -147,7 +148,7 @@ class COCOToDMConverter(ToDMConverter):
         matched_img = None
         for img in images:
             for key in ['file_name', 'filename', 'name']:
-                if key in img and os.path.basename(img[key]) == img_basename:
+                if key in img and os.path.basename(img[key]) == original_image_name:
                     matched_img = img
                     break
             if matched_img:
@@ -157,7 +158,6 @@ class COCOToDMConverter(ToDMConverter):
             raise ValueError(f'No matching image found in COCO data for file: {img_basename}')
 
         img_id = matched_img['id']
-        print('img_id : ', img_id)
         cat_map = {cat['id']: cat for cat in categories}
         anns = [ann for ann in annotations if ann['image_id'] == img_id]
 

synapse_sdk/utils/storage/providers/__init__.py

@@ -2,11 +2,52 @@ from urllib.parse import parse_qs, urlparse
 
 
 class BaseStorage:
+    """Base storage provider class for all storage implementations.
+
+    This is an abstract base class that defines the interface for storage providers.
+    All storage providers (S3, GCP, SFTP, HTTP, FileSystem) must inherit from this
+    class and implement all the abstract methods.
+
+    Args:
+        connection_params (str | dict): The connection parameters. Can be either:
+            - A URL string (e.g., 's3://bucket/path?access_key=key&secret_key=secret')
+            - A dict with 'provider' and 'configuration' keys
+
+    Attributes:
+        url: Parsed URL object if connection_params was a string
+        base_path: Base path for storage operations (provider-specific)
+        options: Storage-specific options
+        query_params: Parsed query parameters from URL or configuration dict
+        OPTION_CASTS: Dictionary mapping option names to type casting functions
+
+    Example:
+        >>> # URL-based initialization
+        >>> storage = SomeStorage('s3://bucket/path?access_key=key&secret_key=secret')
+        >>>
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 's3',
+        ...     'configuration': {
+        ...         'bucket_name': 'my-bucket',
+        ...         'access_key': 'key',
+        ...         'secret_key': 'secret'
+        ...     }
+        ... }
+        >>> storage = SomeStorage(config)
+    """
+
     url = None
     options = None
     OPTION_CASTS = {}
 
     def __init__(self, connection_params: str | dict):
+        """Initialize the storage provider with connection parameters.
+
+        Args:
+            connection_params (str | dict): Connection parameters for the storage provider.
+                If string, should be a valid URL with scheme and query parameters.
+                If dict, should contain 'provider' and 'configuration' keys.
+        """
         self.url = None
 
         if isinstance(connection_params, dict):
@@ -16,6 +57,15 @@ class BaseStorage:
             self.query_params = self.url_querystring_to_dict()
 
     def url_querystring_to_dict(self):
+        """Parse URL query string into a dictionary with proper type casting.
+
+        Converts query parameters from the URL into a dictionary, applying
+        type casting based on OPTION_CASTS mapping. Single-value lists are
+        flattened to scalar values.
+
+        Returns:
+            dict: Parsed and type-cast query parameters.
+        """
         query_string = self.url.query
 
         query_dict = parse_qs(query_string)
@@ -30,43 +80,104 @@ class BaseStorage:
         }
 
     def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to the source file on local filesystem.
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL or identifier of the uploaded file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
         raise NotImplementedError
 
     def exists(self, target):
+        """Check if a file exists at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            bool: True if the file exists, False otherwise.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
         raise NotImplementedError
 
     def get_url(self, target):
+        """Get the URL for accessing a file at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL that can be used to access the file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
         raise NotImplementedError
 
     def get_pathlib(self, path):
-        """Get the path as a pathlib object.
+        """Get the path as a pathlib-compatible object.
 
         Args:
-            path (str): The path to convert.
+            path (str): The path to convert, relative to the storage root.
 
         Returns:
-            pathlib.Path: The converted path.
+            pathlib.Path or UPath: A pathlib-compatible object representing the path.
+            The exact type depends on the storage provider implementation.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            Different storage providers may return different path object types:
+            - FileSystemStorage returns pathlib.Path objects
+            - Cloud storage providers typically return UPath objects
         """
         raise NotImplementedError
 
     def get_path_file_count(self, pathlib_obj):
-        """Get the file count in the path.
+        """Get the total number of files in the given path.
 
         Args:
-            pathlib_obj (UPath): The path to get file count.
+            pathlib_obj (Path | UPath): The path object to count files in.
+                Should be obtained from get_pathlib().
 
         Returns:
-            int: The file count in the path.
+            int: The total number of files found recursively in the path.
+                Returns 1 for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method counts files recursively, including files in subdirectories.
+            Directories themselves are not counted, only regular files.
         """
         raise NotImplementedError
 
     def get_path_total_size(self, pathlib_obj):
-        """Get the total size of the path.
+        """Get the total size of all files in the given path.
 
         Args:
-            pathlib_obj (UPath): The path to get total size.
+            pathlib_obj (Path | UPath): The path object to calculate size for.
+                Should be obtained from get_pathlib().
 
         Returns:
-            int: The total size of the path.
+            int: The total size in bytes of all files found recursively in the path.
+                Returns the file size for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method calculates the total size recursively, including files
+            in subdirectories. Only regular files contribute to the total size.
         """
         raise NotImplementedError

synapse_sdk/utils/storage/providers/file_system.py

@@ -0,0 +1,126 @@
+import shutil
+from pathlib import Path
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class FileSystemStorage(BaseStorage):
+    """Storage provider for file system.
+
+    * This storage do not support url based initialization.
+
+    Args:
+        url (str): The URL of the storage provider.
+
+    Examples:
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 'filesystem',
+        ...     'configuration': {
+        ...         'location': '/data'
+        ...     }
+        ... }
+        >>> storage = FileSystemStorage(config)
+    """
+
+    def __init__(self, url):
+        super().__init__(url)
+        self.base_path = Path(self.query_params['location'])
+
+    def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to source file
+            target (str): Target path relative to base path
+
+        Returns:
+            str: URL of uploaded file
+        """
+        source_path = Path(source)
+        target_path = self.base_path / target
+
+        # Create parent directories if they don't exist
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Copy the file
+        shutil.copy2(source_path, target_path)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        """Check if target file exists.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            bool: True if file exists, False otherwise
+        """
+        target_path = self.base_path / target
+        return target_path.exists()
+
+    def get_url(self, target):
+        """Get URL for target file.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            str: File URL
+        """
+        target_path = self.base_path / target
+        return f'file://{target_path.absolute()}'
+
+    def get_pathlib(self, path):
+        """Get the path as a pathlib object.
+
+        Args:
+            path (str): The path to convert.
+
+        Returns:
+            pathlib.Path: The converted path.
+        """
+        return self.base_path / path
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the file count in the path.
+
+        Args:
+            pathlib_obj (Path): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return 1
+
+        count = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of the path.
+
+        Args:
+            pathlib_obj (Path): The path to get total size.
+
+        Returns:
+            int: The total size of the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return pathlib_obj.stat().st_size
+
+        total_size = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                total_size += item.stat().st_size
+        return total_size

synapse_sdk/utils/storage/registry.py

@@ -1,3 +1,4 @@
+from synapse_sdk.utils.storage.providers.file_system import FileSystemStorage
 from synapse_sdk.utils.storage.providers.gcp import GCPStorage
 from synapse_sdk.utils.storage.providers.http import HTTPStorage
 from synapse_sdk.utils.storage.providers.s3 import S3Storage
@@ -12,4 +13,5 @@ STORAGE_PROVIDERS = {
     'sftp': SFTPStorage,
     'http': HTTPStorage,
     'https': HTTPStorage,
+    'file_system': FileSystemStorage,
 }

{synapse_sdk-1.0.0a82.dist-info → synapse_sdk-1.0.0a84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a82
+Version: 1.0.0a84
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a82.dist-info → synapse_sdk-1.0.0a84.dist-info}/RECORD

@@ -152,11 +152,11 @@ synapse_sdk/plugins/categories/post_annotation/templates/plugin/post_annotation.
 synapse_sdk/plugins/categories/pre_annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py,sha256=6ib3RmnGrjpsQ0e_G-mRH1lfFunQ3gh2M831vuDn7HU,344
-synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py,sha256=pmoOkDE5vidvlW_ZNabdOf7CuoRKZlQ0p_bvA7leCyw,31572
+synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py,sha256=otUpG2TfDPMyigFFe0GuqPc2myh9hZC1iBzx-L7QMio,32138
 synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml,sha256=VREoCp9wsvZ8T2E1d_MEKlR8TC_herDJGVQtu3ezAYU,589
 synapse_sdk/plugins/categories/pre_annotation/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/pre_annotation/templates/plugin/pre_annotation.py,sha256=HBHxHuv2gMBzDB2alFfrzI_SZ1Ztk6mo7eFbR5GqHKw,106
-synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py,sha256=2lGOIAns1V8GWt-lUU-i9lpdk-AJ6vNY8s7Ks7a72BY,883
+synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py,sha256=LfM3pWfBYZfTJjpCDq89eo--uAKi-sgs8PWkilj31tI,1135
 synapse_sdk/plugins/categories/smart_tool/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/smart_tool/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/smart_tool/actions/auto_label.py,sha256=fHiqA8ntmzjs2GMVMuByR7Clh2zhLie8OPF9B8OmwxM,1279
@@ -199,7 +199,7 @@ synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,2
 synapse_sdk/utils/converters/__init__.py,sha256=xQi_n7xS9BNyDiolsxH2jw1CtD6avxMPj2cHnwvidi8,11311
 synapse_sdk/utils/converters/coco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/converters/coco/from_dm.py,sha256=B9zvb8Kph9haYLVIZhzneWiHCImFbuWqAaE7g6Nk0lI,11365
-synapse_sdk/utils/converters/coco/to_dm.py,sha256=YmD_NHKSUL8RZbzWX52FgDaJG0uX4I8f8Omp7ilhSec,9054
+synapse_sdk/utils/converters/coco/to_dm.py,sha256=SOzzACYlK0vJE6SIpmFwMUV1ZbdmsQ_t5mceG0SQahE,9105
 synapse_sdk/utils/converters/dm/__init__.py,sha256=_B6w814bMPhisNCNlSPEiQOs9RH0EZHQqd89LnVhD1U,1983
 synapse_sdk/utils/converters/dm/from_v1.py,sha256=4BG_NA_7YdW5rI1F8LCFg39M-IJZVfRgi2b9FBxTAmw,26059
 synapse_sdk/utils/converters/dm/to_v1.py,sha256=A123zAR_dLqEW83BgAl5_J1ACstjZWTHivlW5qvOu_E,13432
@@ -214,15 +214,16 @@ synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
 synapse_sdk/utils/storage/__init__.py,sha256=HmZHqvoV-EogV2bE-Sw5XQRlrNuf3gfNL9irAJeRYsA,2195
-synapse_sdk/utils/storage/registry.py,sha256=MNhX2ohVfQsjhuHO2YMn0wiJIA0ELPp5NXH8LWoh2bQ,473
-synapse_sdk/utils/storage/providers/__init__.py,sha256=x7RGwZryT2FpVxS7fGWryRVpquHzAiIfTz-9uLgjleo,1860
+synapse_sdk/utils/storage/registry.py,sha256=-VNIM7wERubomcyXMuAlPPRKyy87kVpkRAsdlBRp2ig,589
+synapse_sdk/utils/storage/providers/__init__.py,sha256=crxrRzXXfBAmyR4nZS2RZvlLnr4IP1S-6eYeX_bSnLI,6412
+synapse_sdk/utils/storage/providers/file_system.py,sha256=MBBXxWRgVSDtjP4T0L0sbaZkXEI4B8ANtLweaMWtaaQ,3217
 synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
 synapse_sdk/utils/storage/providers/http.py,sha256=2DhIulND47JOnS5ZY7MZUex7Su3peAPksGo1Wwg07L4,5828
 synapse_sdk/utils/storage/providers/s3.py,sha256=ZmqekAvIgcQBdRU-QVJYv1Rlp6VHfXwtbtjTSphua94,2573
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-1.0.0a82.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a82.dist-info/METADATA,sha256=tCCYQhZDkDvBXPSQa50Dzur5VrBpp9Hm7fWI6jHxFBA,3805
-synapse_sdk-1.0.0a82.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-synapse_sdk-1.0.0a82.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a82.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a82.dist-info/RECORD,,
+synapse_sdk-1.0.0a84.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a84.dist-info/METADATA,sha256=TQr3AWnKuEIisWrMC0URwIn64lxJgnYVYimuUtoIp7E,3805
+synapse_sdk-1.0.0a84.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synapse_sdk-1.0.0a84.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a84.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a84.dist-info/RECORD,,