synapse-sdk 1.0.0a65__tar.gz → 1.0.0a66__tar.gz

{synapse_sdk-1.0.0a65 → synapse_sdk-1.0.0a66}/.gitignore

@@ -38,4 +38,6 @@ npm-debug.log*
 
 config.yaml*
 
-synapse-sample-plugin/
+synapse-sample-plugin/
+
+CLAUDE.md

{synapse_sdk-1.0.0a65 → synapse_sdk-1.0.0a66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a65
+Version: 1.0.0a66
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a65 → synapse_sdk-1.0.0a66}/synapse_sdk/utils/converters/__init__.py

@@ -1,7 +1,72 @@
+import json
 import os
+import shutil
+import uuid
 
 
-class FromDMConverter:
+class BaseConverter:
+    """Base class for shared logic between converters."""
+
+    def __init__(self, root_dir: str, is_categorized_dataset: bool = False) -> None:
+        self.root_dir: str = root_dir
+        self.is_categorized_dataset: bool = is_categorized_dataset
+        self.converted_data = None
+
+    @staticmethod
+    def ensure_dir(path: str) -> None:
+        """Ensure that the directory exists, creating it if necessary."""
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+    def _validate_required_dirs(self, dirs):
+        """Validate that all required directories exist."""
+        for name, path in dirs.items():
+            if not os.path.exists(path):
+                raise FileNotFoundError(f'[ERROR] Required directory "{name}" does not exist at {path}')
+
+    def _validate_optional_dirs(self, dirs):
+        """Validate optional directories and return those that exist."""
+        existing_dirs = {}
+        for name, path in dirs.items():
+            if os.path.exists(path):
+                existing_dirs[name] = path
+            else:
+                print(f'[WARNING] Optional directory "{name}" does not exist. Skipping.')
+        return existing_dirs
+
+    def _validate_splits(self, required_splits, optional_splits=[]):
+        """Validate required and optional splits in the dataset."""
+        splits = {}
+
+        if self.is_categorized_dataset:
+            required_dirs = {split: os.path.join(self.root_dir, split) for split in required_splits}
+            self._validate_required_dirs(required_dirs)
+            splits.update(required_dirs)
+
+            optional_dirs = {split: os.path.join(self.root_dir, split) for split in optional_splits}
+            splits.update(self._validate_optional_dirs(optional_dirs))
+        else:
+            required_dirs = {
+                'json': os.path.join(self.root_dir, 'json'),
+                'original_file': os.path.join(self.root_dir, 'original_file'),
+            }
+            self._validate_required_dirs(required_dirs)
+            splits['root'] = self.root_dir
+
+        return splits
+
+    def _set_directories(self, split=None):
+        """Set `self.json_dir` and `self.original_file_dir` based on the dataset split."""
+        if split:
+            split_dir = os.path.join(self.root_dir, split)
+            self.json_dir = os.path.join(split_dir, 'json')
+            self.original_file_dir = os.path.join(split_dir, 'original_file')
+        else:
+            self.json_dir = os.path.join(self.root_dir, 'json')
+            self.original_file_dir = os.path.join(self.root_dir, 'original_file')
+
+
+class FromDMConverter(BaseConverter):
     """Base class for converting data from DM format to a specific format.
 
     Attrs:
@@ -42,10 +107,8 @@ class FromDMConverter:
     """
 
     def __init__(self, root_dir: str, is_categorized_dataset: bool = False) -> None:
-        self.root_dir: str = root_dir
-        self.is_categorized_dataset: bool = is_categorized_dataset
+        super().__init__(root_dir, is_categorized_dataset)
         self.version: str = '1.0'
-        self.converted_data = None
 
     def convert(self):
         """Convert DM format to a specific format.
@@ -61,85 +124,85 @@ class FromDMConverter:
             # Automatically call convert() if converted_data is not set
             self.converted_data = self.convert()
 
-    @staticmethod
-    def ensure_dir(path: str) -> None:
-        """Ensure that the directory exists, creating it if necessary."""
-        if not os.path.exists(path):
-            os.makedirs(path)
 
-    def _validate_required_dirs(self, dirs):
-        """Validate that all required directories exist.
+class ToDMConverter(BaseConverter):
+    """Base class for converting data to DM format.
 
-        Args:
-            dirs (dict): A dictionary where keys are directory names and values are their paths.
+    Attrs:
+        root_dir (str): Root directory containing data.
+        is_categorized_dataset (bool): Whether to handle train, test, valid splits.
+        converted_data: Holds the converted data after calling `convert()`.
 
-        Raises:
-            FileNotFoundError: If any required directory does not exist.
-        """
-        for name, path in dirs.items():
-            if not os.path.exists(path):
-                raise FileNotFoundError(f'[ERROR] Required directory "{name}" does not exist at {path}')
+    Usage:
+        1. Subclass this base class and implement the `convert()` method.
+        2. Instantiate the converter with the required arguments.
+        3. Call `convert()` to perform the in-memory conversion and obtain the result as a dict or list of dicts.
+        4. Call `save_to_folder(output_dir)` to save the converted data and optionally copy original files.
 
-    def _validate_optional_dirs(self, dirs):
-        """Validate optional directories and return those that exist.
+    Args:
+        root_dir (str): Path to the root directory containing data.
+            - If `is_categorized_dataset=True`, the directory should contain subdirectories for
+            `train`, `valid`, and optionally `test`.
+            - Each subdirectory should contain `annotations.json` and the corresponding image files.
+            - `train` and `valid` are required, while `test` is optional.
+        is_categorized_dataset (bool): Whether to handle train, test, valid splits.
 
-        Args:
-            dirs (dict): A dictionary where keys are directory names and values are their paths.
+    Returns:
+        - convert(): Returns the converted data as a Python dict or a dictionary with keys for each split.
+        - save_to_folder(): Saves the converted data and optionally copies original files
+        to the specified output directory.
 
-        Returns:
-            dict: A dictionary of existing optional directories.
-        """
-        existing_dirs = {}
-        for name, path in dirs.items():
-            if os.path.exists(path):
-                existing_dirs[name] = path
-            else:
-                print(f'[WARNING] Optional directory "{name}" does not exist. Skipping.')
-        return existing_dirs
+    Example usage:
+        # Dataset with splits
+        converter = MyCustomToDMConverter(root_dir='/path/to/data', is_categorized_dataset=True)
+        converted = converter.convert()  # Returns a dict with keys for `train`, `valid`, and optionally `test`
+        converter.save_to_folder('/my/target/output')  # Writes files/folders to output location
 
-    def _validate_splits(self, required_splits, optional_splits=[]):
-        """Validate required and optional splits in the dataset.
+        # Dataset without splits
+        converter = MyCustomToDMConverter(root_dir='/path/to/data', is_categorized_dataset=False)
+        converted = converter.convert()  # Returns a dict or a list, depending on the implementation
+        converter.save_to_folder('/my/target/output')  # Writes files/folders to output location
+    """
 
-        Args:
-            required_splits (list): List of required split names (e.g., ['train', 'valid']).
-            optional_splits (list): List of optional split names (e.g., ['test']).
+    def convert(self):
+        """Convert data to DM format.
 
-        Returns:
-            dict: A dictionary with split names as keys and their corresponding directories as values.
+        This method should be implemented by subclasses to perform the actual conversion.
         """
-        splits = {}
-
-        if self.is_categorized_dataset:
-            # Validate required splits
-            required_dirs = {split: os.path.join(self.root_dir, split) for split in required_splits}
-            self._validate_required_dirs(required_dirs)
-            splits.update(required_dirs)
-
-            # Validate optional splits
-            optional_dirs = {split: os.path.join(self.root_dir, split) for split in optional_splits}
-            splits.update(self._validate_optional_dirs(optional_dirs))
-        else:
-            # Validate `json` and `original_file` folders for non-split datasets
-            required_dirs = {
-                'json': os.path.join(self.root_dir, 'json'),
-                'original_file': os.path.join(self.root_dir, 'original_file'),
-            }
-            self._validate_required_dirs(required_dirs)
-            splits['root'] = self.root_dir
+        raise NotImplementedError
 
-        return splits
+    def _generate_unique_id(self):
+        """Generate a unique 10-character UUID."""
+        return uuid.uuid4().hex[:10]
 
-    def _set_directories(self, split=None):
-        """Set `self.json_dir` and `self.original_file_dir` based on the dataset split.
+    def save_to_folder(self, output_dir: str) -> None:
+        """Save converted DM schema data to the specified folder."""
+        self.ensure_dir(output_dir)
+        if self.converted_data is None:
+            self.converted_data = self.convert()
 
-        Args:
-            split (str, optional): The name of the split (e.g., 'train', 'valid', 'test').
-                                   If None, assumes no dataset split.
-        """
-        if split:
-            split_dir = os.path.join(self.root_dir, split)
-            self.json_dir = os.path.join(split_dir, 'json')
-            self.original_file_dir = os.path.join(split_dir, 'original_file')
+        if self.is_categorized_dataset:
+            for split, img_dict in self.converted_data.items():
+                split_dir = os.path.join(output_dir, split)
+                json_dir = os.path.join(split_dir, 'json')
+                original_file_dir = os.path.join(split_dir, 'original_file')
+                self.ensure_dir(json_dir)
+                self.ensure_dir(original_file_dir)
+                for img_filename, (dm_json, img_src_path) in img_dict.items():
+                    json_filename = os.path.splitext(img_filename)[0] + '.json'
+                    with open(os.path.join(json_dir, json_filename), 'w', encoding='utf-8') as jf:
+                        json.dump(dm_json, jf, indent=2, ensure_ascii=False)
+                    if img_src_path and os.path.exists(img_src_path):
+                        shutil.copy(img_src_path, os.path.join(original_file_dir, img_filename))
         else:
-            self.json_dir = os.path.join(self.root_dir, 'json')
-            self.original_file_dir = os.path.join(self.root_dir, 'original_file')
+            json_dir = os.path.join(output_dir, 'json')
+            original_file_dir = os.path.join(output_dir, 'original_file')
+            self.ensure_dir(json_dir)
+            self.ensure_dir(original_file_dir)
+            for img_filename, (dm_json, img_src_path) in self.converted_data.items():
+                json_filename = os.path.splitext(img_filename)[0] + '.json'
+                with open(os.path.join(json_dir, json_filename), 'w', encoding='utf-8') as jf:
+                    json.dump(dm_json, jf, indent=2, ensure_ascii=False)
+                if img_src_path and os.path.exists(img_src_path):
+                    shutil.copy(img_src_path, os.path.join(original_file_dir, img_filename))
+        print(f'[DM] Data exported to {output_dir}')

synapse_sdk-1.0.0a66/synapse_sdk/utils/converters/coco/to_dm.py

@@ -0,0 +1,113 @@
+import json
+import os
+
+from synapse_sdk.utils.converters import ToDMConverter
+
+
+class COCOToDMConverter(ToDMConverter):
+    """Convert COCO format annotations to DM (Data Manager) format."""
+
+    def convert(self):
+        if self.is_categorized_dataset:
+            splits = self._validate_splits(['train', 'valid'], ['test'])
+            all_split_data = {}
+            for split, split_dir in splits.items():
+                annotation_path = os.path.join(split_dir, 'annotations.json')
+                if not os.path.exists(annotation_path):
+                    raise FileNotFoundError(f'annotations.json not found in {split_dir}')
+                with open(annotation_path, 'r', encoding='utf-8') as f:
+                    coco_data = json.load(f)
+                split_data = self._convert_coco_ann_to_dm(coco_data, split_dir)
+                all_split_data[split] = split_data
+            self.converted_data = all_split_data
+            return all_split_data
+        else:
+            annotation_path = os.path.join(self.root_dir, 'annotations.json')
+            if not os.path.exists(annotation_path):
+                raise FileNotFoundError(f'annotations.json not found in {self.root_dir}')
+            with open(annotation_path, 'r', encoding='utf-8') as f:
+                coco_data = json.load(f)
+            converted_data = self._convert_coco_ann_to_dm(coco_data, self.root_dir)
+            self.converted_data = converted_data
+            return converted_data
+
+    def _convert_coco_ann_to_dm(self, coco_data, base_dir):
+        """Convert COCO annotations to DM format."""
+        dataset_type = coco_data.get('type', 'image')  # Default to 'image' if type is not specified
+        if dataset_type == 'image':
+            return self._process_image_data(coco_data, base_dir)
+        else:
+            raise ValueError(f'Unsupported dataset type: {dataset_type}')
+
+    def _process_image_data(self, coco_data, img_base_dir):
+        """Process COCO image data and convert to DM format."""
+        images = coco_data.get('images', [])
+        annotations = coco_data.get('annotations', [])
+        categories = coco_data.get('categories', [])
+        cat_map = {cat['id']: cat for cat in categories}
+
+        # Build image_id -> annotation list
+        ann_by_img_id = {}
+        for ann in annotations:
+            img_id = ann['image_id']
+            ann_by_img_id.setdefault(img_id, []).append(ann)
+
+        result = {}
+        for img in images:
+            img_id = img['id']
+            img_filename = img['file_name']
+            img_path = os.path.join(img_base_dir, img_filename)
+            anns = ann_by_img_id.get(img_id, [])
+
+            # DM image structure
+            dm_img = {
+                'bounding_box': [],
+                'keypoint': [],
+                'relation': [],
+                'group': [],
+            }
+
+            # Handle bounding_box
+            bbox_ids = []
+            for ann in anns:
+                cat = cat_map.get(ann['category_id'], {})
+                if 'bbox' in ann and ann['bbox']:
+                    bbox_id = self._generate_unique_id()
+                    bbox_ids.append(bbox_id)
+                    dm_img['bounding_box'].append({
+                        'id': bbox_id,
+                        'classification': cat.get('name', str(ann['category_id'])),
+                        'attrs': ann.get('attrs', []),
+                        'data': list(ann['bbox']),
+                    })
+
+            # Handle keypoints
+            for ann in anns:
+                cat = cat_map.get(ann['category_id'], {})
+                attrs = ann.get('attrs', [])
+                if 'keypoints' in ann and ann['keypoints']:
+                    kp_names = cat.get('keypoints', [])
+                    kps = ann['keypoints']
+                    keypoint_ids = []
+                    for idx in range(min(len(kps) // 3, len(kp_names))):
+                        x, y, v = kps[idx * 3 : idx * 3 + 3]
+                        kp_id = self._generate_unique_id()
+                        keypoint_ids.append(kp_id)
+                        dm_img['keypoint'].append({
+                            'id': kp_id,
+                            'classification': kp_names[idx] if idx < len(kp_names) else f'keypoint_{idx}',
+                            'attrs': attrs,
+                            'data': [x, y],
+                        })
+                    group_ids = bbox_ids + keypoint_ids
+                    if group_ids:
+                        dm_img['group'].append({
+                            'id': self._generate_unique_id(),
+                            'classification': cat.get('name', str(ann['category_id'])),
+                            'attrs': attrs,
+                            'data': group_ids,
+                        })
+
+            dm_json = {'images': [dm_img]}
+            result[img_filename] = (dm_json, img_path)
+        return result

synapse_sdk-1.0.0a66/synapse_sdk/utils/storage/providers/http.py

@@ -0,0 +1,190 @@
+import os
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class HTTPStorage(BaseStorage):
+    """Storage provider for no-auth HTTP file servers (e.g., Django FileSystemStorage served over HTTP)."""
+
+    OPTION_CASTS = {
+        'timeout': int,
+    }
+
+    def __init__(self, connection_params: str | dict):
+        super().__init__(connection_params)
+
+        # Extract base URL
+        if isinstance(connection_params, dict):
+            self.base_url = self.query_params.get('base_url', '')
+            self.timeout = self.query_params.get('timeout', 30)
+        else:
+            # Parse URL like: http://example.com/media/
+            parsed = urlparse(connection_params)
+            self.base_url = f'{parsed.scheme}://{parsed.netloc}{parsed.path}'
+            self.timeout = self.query_params.get('timeout', 30)
+
+        # Ensure base_url ends with /
+        if not self.base_url.endswith('/'):
+            self.base_url += '/'
+
+        # Setup session for connection pooling
+        self.session = requests.Session()
+
+    def _get_full_url(self, path: str) -> str:
+        """Get the full URL for a given path."""
+        # Remove leading slash from path to avoid double slashes
+        if path.startswith('/'):
+            path = path[1:]
+        return urljoin(self.base_url, path)
+
+    def upload(self, source: str, target: str) -> str:
+        """Upload a file to the HTTP server.
+
+        Args:
+            source: Local file path to upload
+            target: Target path on the HTTP server
+
+        Returns:
+            URL of the uploaded file
+        """
+        url = self._get_full_url(target)
+
+        with open(source, 'rb') as f:
+            files = {'file': (os.path.basename(source), f)}
+
+            # Try PUT first (more RESTful), fallback to POST
+            response = self.session.put(url, files=files, timeout=self.timeout)
+
+            if response.status_code == 405:  # Method not allowed
+                # Reset file pointer and try POST
+                f.seek(0)
+                response = self.session.post(url, files=files, timeout=self.timeout)
+
+            response.raise_for_status()
+
+        return url
+
+    def exists(self, target: str) -> bool:
+        """Check if a file exists on the HTTP server.
+
+        Args:
+            target: Path to check
+
+        Returns:
+            True if file exists, False otherwise
+        """
+        url = self._get_full_url(target)
+
+        try:
+            response = self.session.head(url, timeout=self.timeout)
+            return response.status_code == 200
+        except requests.RequestException:
+            return False
+
+    def get_url(self, target: str) -> str:
+        """Get the URL for a file.
+
+        Args:
+            target: File path
+
+        Returns:
+            Full URL of the file
+        """
+        return self._get_full_url(target)
+
+    def get_pathlib(self, path: str) -> 'HTTPPath':
+        """Get a pathlib-like object for HTTP operations.
+
+        Args:
+            path: Path to wrap
+
+        Returns:
+            HTTPPath object
+        """
+        return HTTPPath(self, path)
+
+    def get_path_file_count(self, pathlib_obj) -> int:
+        """Get file count in a directory.
+
+        Note: This requires the HTTP server to provide directory listing functionality.
+
+        Args:
+            pathlib_obj: HTTPPath object
+
+        Returns:
+            File count
+        """
+        # Most HTTP servers don't provide directory listing
+        # This would need custom server-side support
+        raise NotImplementedError('File counting requires server-side directory listing support')
+
+    def get_path_total_size(self, pathlib_obj) -> int:
+        """Get total size of files in a directory.
+
+        Note: This requires the HTTP server to provide directory listing functionality.
+
+        Args:
+            pathlib_obj: HTTPPath object
+
+        Returns:
+            Total size in bytes
+        """
+        # Most HTTP servers don't provide directory listing
+        # This would need custom server-side support
+        raise NotImplementedError('Size calculation requires server-side directory listing support')
+
+
+class HTTPPath:
+    """A pathlib-like interface for HTTP paths."""
+
+    def __init__(self, storage: HTTPStorage, path: str):
+        self.storage = storage
+        self.path = path.strip('/')
+
+    def __str__(self):
+        return self.path
+
+    def __truediv__(self, other):
+        """Join paths using / operator."""
+        new_path = f'{self.path}/{other}' if self.path else str(other)
+        return HTTPPath(self.storage, new_path)
+
+    def joinuri(self, *parts):
+        """Join path parts."""
+        parts = [self.path] + [str(p) for p in parts]
+        new_path = '/'.join(p.strip('/') for p in parts if p)
+        return HTTPPath(self.storage, new_path)
+
+    @property
+    def name(self):
+        """Get the final component of the path."""
+        return os.path.basename(self.path)
+
+    @property
+    def parent(self):
+        """Get the parent directory."""
+        parent_path = os.path.dirname(self.path)
+        return HTTPPath(self.storage, parent_path)
+
+    def exists(self):
+        """Check if this path exists."""
+        return self.storage.exists(self.path)
+
+    def is_file(self):
+        """Check if this path is a file."""
+        # For HTTP, we assume it's a file if it exists
+        return self.exists()
+
+    def read_bytes(self):
+        """Read file contents as bytes."""
+        url = self.storage.get_url(self.path)
+        response = self.storage.session.get(url, timeout=self.storage.timeout)
+        response.raise_for_status()
+        return response.content
+
+    def read_text(self, encoding='utf-8'):
+        """Read file contents as text."""
+        return self.read_bytes().decode(encoding)

{synapse_sdk-1.0.0a65 → synapse_sdk-1.0.0a66}/synapse_sdk/utils/storage/registry.py

@@ -1,4 +1,5 @@
 from synapse_sdk.utils.storage.providers.gcp import GCPStorage
+from synapse_sdk.utils.storage.providers.http import HTTPStorage
 from synapse_sdk.utils.storage.providers.s3 import S3Storage
 from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
 
@@ -8,4 +9,6 @@ STORAGE_PROVIDERS = {
     'minio': S3Storage,
     'gcp': GCPStorage,
     'sftp': SFTPStorage,
+    'http': HTTPStorage,
+    'https': HTTPStorage,
 }

{synapse_sdk-1.0.0a65 → synapse_sdk-1.0.0a66}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a65
+Version: 1.0.0a66
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a65 → synapse_sdk-1.0.0a66}/synapse_sdk.egg-info/SOURCES.txt

@@ -252,6 +252,7 @@ synapse_sdk/utils/string.py
 synapse_sdk/utils/converters/__init__.py
 synapse_sdk/utils/converters/coco/__init__.py
 synapse_sdk/utils/converters/coco/from_dm.py
+synapse_sdk/utils/converters/coco/to_dm.py
 synapse_sdk/utils/pydantic/__init__.py
 synapse_sdk/utils/pydantic/config.py
 synapse_sdk/utils/pydantic/errors.py
@@ -260,6 +261,7 @@ synapse_sdk/utils/storage/__init__.py
 synapse_sdk/utils/storage/registry.py
 synapse_sdk/utils/storage/providers/__init__.py
 synapse_sdk/utils/storage/providers/gcp.py
+synapse_sdk/utils/storage/providers/http.py
 synapse_sdk/utils/storage/providers/s3.py
 synapse_sdk/utils/storage/providers/sftp.py
 tests/__init__.py