synapse-sdk 1.0.0a13__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/utils/storage/providers/file_system.py

@@ -0,0 +1,134 @@
+import shutil
+from pathlib import Path
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class FileSystemStorage(BaseStorage):
+    """Storage provider for file system.
+
+    * This storage do not support url based initialization.
+
+    Args:
+        url (str): The URL of the storage provider.
+
+    Examples:
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 'filesystem',
+        ...     'configuration': {
+        ...         'location': '/data'
+        ...     }
+        ... }
+        >>> storage = FileSystemStorage(config)
+    """
+
+    def __init__(self, connection_params: str | dict):
+        super().__init__(connection_params)
+        self.base_path = Path(self.query_params['location'])
+
+    def get_pathlib(self, path):
+        """Get the path as a pathlib object.
+
+        Args:
+            path (str): The path to convert.
+
+        Returns:
+            pathlib.Path: The converted path.
+        """
+        if path == '/' or path == '':
+            return self.base_path
+
+        # Strip leading slash to ensure path is relative to base_path.
+        # Path('/data') / '/subdir' would incorrectly resolve to '/subdir' instead of '/data/subdir'
+        if isinstance(path, str) and path.startswith('/'):
+            path = path.lstrip('/')
+
+        return self.base_path / path
+
+    def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to source file
+            target (str): Target path relative to base path
+
+        Returns:
+            str: URL of uploaded file
+        """
+        source_path = Path(source)
+        target_path = self.base_path / target
+
+        # Create parent directories if they don't exist
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Copy the file
+        shutil.copy2(source_path, target_path)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        """Check if target file exists.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            bool: True if file exists, False otherwise
+        """
+        target_path = self.base_path / target
+        return target_path.exists()
+
+    def get_url(self, target):
+        """Get URL for target file.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            str: File URL
+        """
+        target_path = self.base_path / target
+        return f'file://{target_path.absolute()}'
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the file count in the path.
+
+        Args:
+            pathlib_obj (Path): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return 1
+
+        count = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of the path.
+
+        Args:
+            pathlib_obj (Path): The path to get total size.
+
+        Returns:
+            int: The total size of the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return pathlib_obj.stat().st_size
+
+        total_size = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                total_size += item.stat().st_size
+        return total_size

synapse_sdk/utils/storage/providers/gcp.py

@@ -0,0 +1,13 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class GCPStorage(BaseStorage):
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = UPath(f'gs://{self.query_params["bucket_name"]}', token=self.query_params['credentials'])
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk/utils/storage/providers/http.py

@@ -0,0 +1,190 @@
+import os
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class HTTPStorage(BaseStorage):
+    """Storage provider for no-auth HTTP file servers (e.g., Django FileSystemStorage served over HTTP)."""
+
+    OPTION_CASTS = {
+        'timeout': int,
+    }
+
+    def __init__(self, connection_params: str | dict):
+        super().__init__(connection_params)
+
+        # Extract base URL
+        if isinstance(connection_params, dict):
+            self.base_url = self.query_params.get('base_url', '')
+            self.timeout = self.query_params.get('timeout', 30)
+        else:
+            # Parse URL like: http://example.com/media/
+            parsed = urlparse(connection_params)
+            self.base_url = f'{parsed.scheme}://{parsed.netloc}{parsed.path}'
+            self.timeout = self.query_params.get('timeout', 30)
+
+        # Ensure base_url ends with /
+        if not self.base_url.endswith('/'):
+            self.base_url += '/'
+
+        # Setup session for connection pooling
+        self.session = requests.Session()
+
+    def _get_full_url(self, path: str) -> str:
+        """Get the full URL for a given path."""
+        # Remove leading slash from path to avoid double slashes
+        if path.startswith('/'):
+            path = path[1:]
+        return urljoin(self.base_url, path)
+
+    def upload(self, source: str, target: str) -> str:
+        """Upload a file to the HTTP server.
+
+        Args:
+            source: Local file path to upload
+            target: Target path on the HTTP server
+
+        Returns:
+            URL of the uploaded file
+        """
+        url = self._get_full_url(target)
+
+        with open(source, 'rb') as f:
+            files = {'file': (os.path.basename(source), f)}
+
+            # Try PUT first (more RESTful), fallback to POST
+            response = self.session.put(url, files=files, timeout=self.timeout)
+
+            if response.status_code == 405:  # Method not allowed
+                # Reset file pointer and try POST
+                f.seek(0)
+                response = self.session.post(url, files=files, timeout=self.timeout)
+
+            response.raise_for_status()
+
+        return url
+
+    def exists(self, target: str) -> bool:
+        """Check if a file exists on the HTTP server.
+
+        Args:
+            target: Path to check
+
+        Returns:
+            True if file exists, False otherwise
+        """
+        url = self._get_full_url(target)
+
+        try:
+            response = self.session.head(url, timeout=self.timeout)
+            return response.status_code == 200
+        except requests.RequestException:
+            return False
+
+    def get_url(self, target: str) -> str:
+        """Get the URL for a file.
+
+        Args:
+            target: File path
+
+        Returns:
+            Full URL of the file
+        """
+        return self._get_full_url(target)
+
+    def get_pathlib(self, path: str) -> 'HTTPPath':
+        """Get a pathlib-like object for HTTP operations.
+
+        Args:
+            path: Path to wrap
+
+        Returns:
+            HTTPPath object
+        """
+        return HTTPPath(self, path)
+
+    def get_path_file_count(self, pathlib_obj) -> int:
+        """Get file count in a directory.
+
+        Note: This requires the HTTP server to provide directory listing functionality.
+
+        Args:
+            pathlib_obj: HTTPPath object
+
+        Returns:
+            File count
+        """
+        # Most HTTP servers don't provide directory listing
+        # This would need custom server-side support
+        raise NotImplementedError('File counting requires server-side directory listing support')
+
+    def get_path_total_size(self, pathlib_obj) -> int:
+        """Get total size of files in a directory.
+
+        Note: This requires the HTTP server to provide directory listing functionality.
+
+        Args:
+            pathlib_obj: HTTPPath object
+
+        Returns:
+            Total size in bytes
+        """
+        # Most HTTP servers don't provide directory listing
+        # This would need custom server-side support
+        raise NotImplementedError('Size calculation requires server-side directory listing support')
+
+
+class HTTPPath:
+    """A pathlib-like interface for HTTP paths."""
+
+    def __init__(self, storage: HTTPStorage, path: str):
+        self.storage = storage
+        self.path = path.strip('/')
+
+    def __str__(self):
+        return self.path
+
+    def __truediv__(self, other):
+        """Join paths using / operator."""
+        new_path = f'{self.path}/{other}' if self.path else str(other)
+        return HTTPPath(self.storage, new_path)
+
+    def joinuri(self, *parts):
+        """Join path parts."""
+        parts = [self.path] + [str(p) for p in parts]
+        new_path = '/'.join(p.strip('/') for p in parts if p)
+        return HTTPPath(self.storage, new_path)
+
+    @property
+    def name(self):
+        """Get the final component of the path."""
+        return os.path.basename(self.path)
+
+    @property
+    def parent(self):
+        """Get the parent directory."""
+        parent_path = os.path.dirname(self.path)
+        return HTTPPath(self.storage, parent_path)
+
+    def exists(self):
+        """Check if this path exists."""
+        return self.storage.exists(self.path)
+
+    def is_file(self):
+        """Check if this path is a file."""
+        # For HTTP, we assume it's a file if it exists
+        return self.exists()
+
+    def read_bytes(self):
+        """Read file contents as bytes."""
+        url = self.storage.get_url(self.path)
+        response = self.storage.session.get(url, timeout=self.storage.timeout)
+        response.raise_for_status()
+        return response.content
+
+    def read_text(self, encoding='utf-8'):
+        """Read file contents as text."""
+        return self.read_bytes().decode(encoding)

synapse_sdk/utils/storage/providers/s3.py

@@ -0,0 +1,91 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class S3Storage(BaseStorage):
+    ENDPOINT_URL = 'https://s3.amazonaws.com'
+    DEFAULT_REGION = 'us-east-1'
+
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = self._get_upath()
+
+    def _get_upath(self):
+        client_kwargs = {'region_name': self.query_params.get('region_name', self.DEFAULT_REGION)}
+
+        if self.query_params.get('endpoint_url'):
+            client_kwargs['endpoint_url'] = self.query_params['endpoint_url']
+
+        upath_kwargs = {
+            'key': self.query_params['access_key'],
+            'secret': self.query_params['secret_key'],
+            'client_kwargs': client_kwargs,
+        }
+
+        return UPath(
+            f's3://{self.query_params["bucket_name"]}',
+            **upath_kwargs,
+        )
+
+    def upload(self, source, target):
+        with open(source, 'rb') as file:
+            (self.upath / target).write_bytes(file.read())
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        return (self.upath / target).exists()
+
+    def get_url(self, target):
+        return str(self.upath.joinuri(target))
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)
+
+    def get_path_file_count(self, pathlib_obj: UPath):
+        """Get file count in the path from S3 provider.
+
+        TODO: Need to find a method to get file count using S3 API
+
+        Args:
+            pathlib_obj (UPath): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        count = 0
+        files = list(pathlib_obj.glob('**/*'))
+        for file in files:
+            if file.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj: UPath):
+        """Get total size of the files in the path from S3 provider.
+
+        TODO: Need to find a method to get total file size using S3 API
+
+        Args:
+            pathlib_obj (UPath): The path to get total size.
+
+        Returns:
+            int: The total size of the files in the path.
+        """
+        total_size = 0
+        for file in pathlib_obj.glob('**/*'):
+            if file.is_file():
+                total_size += file.stat().st_size
+        return total_size
+
+    def glob(self, pattern):
+        """Glob pattern matching in S3 storage.
+
+        Args:
+            pattern (str): The glob pattern to match.
+
+        Returns:
+            list: List of matching paths.
+        """
+        return list((self.upath / '').glob(pattern))

synapse_sdk/utils/storage/providers/sftp.py

@@ -0,0 +1,47 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class SFTPStorage(BaseStorage):
+    def get_pathlib(self, path):
+        credentials = self.query_params['params']
+        host = self.query_params['host']
+        root_path = self.query_params['root_path']
+
+        username = credentials['username']
+        password = credentials['password']
+        if path == '/':
+            path = ''
+        return UPath(f'sftp://{host}', username=username, password=password) / root_path / path
+
+    def get_path_file_count(self, pathlib_obj: UPath):
+        """Get file count in the path from SFTP provider.
+
+        Args:
+            pathlib_obj (UPath): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        count = 0
+        files = list(pathlib_obj.glob('**/*'))
+        for file in files:
+            if file.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj: UPath):
+        """Get total size of the files in the path from SFTP provider.
+
+        Args:
+            pathlib_obj (UPath): The path to get total size.
+
+        Returns:
+            int: The total size of the files in the path.
+        """
+        total_size = 0
+        for file in pathlib_obj.glob('**/*'):
+            if file.is_file():
+                total_size += file.stat().st_size
+        return total_size

synapse_sdk/utils/storage/registry.py

@@ -0,0 +1,17 @@
+from synapse_sdk.utils.storage.providers.file_system import FileSystemStorage
+from synapse_sdk.utils.storage.providers.gcp import GCPStorage
+from synapse_sdk.utils.storage.providers.http import HTTPStorage
+from synapse_sdk.utils.storage.providers.s3 import S3Storage
+from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
+
+STORAGE_PROVIDERS = {
+    's3': S3Storage,
+    'amazon_s3': S3Storage,
+    'minio': S3Storage,
+    'gs': GCPStorage,
+    'gcp': GCPStorage,
+    'sftp': SFTPStorage,
+    'http': HTTPStorage,
+    'https': HTTPStorage,
+    'file_system': FileSystemStorage,
+}

synapse_sdk-2025.11.7.dist-info/METADATA

@@ -0,0 +1,122 @@
+Metadata-Version: 2.4
+Name: synapse-sdk
+Version: 2025.11.7
+Summary: synapse sdk
+Author-email: datamaker <developer@datamaker.io>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: boto3
+Requires-Dist: click
+Requires-Dist: cookiecutter
+Requires-Dist: requests
+Requires-Dist: tqdm
+Requires-Dist: openpyxl
+Requires-Dist: python-dotenv
+Requires-Dist: pyyaml
+Requires-Dist: pydantic
+Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
+Requires-Dist: inquirer
+Requires-Dist: pillow
+Requires-Dist: websockets
+Requires-Dist: ffmpeg-python==0.2.0
+Requires-Dist: sentry-sdk==2.38.0
+Provides-Extra: all
+Requires-Dist: ray[all]==2.50.0; extra == "all"
+Requires-Dist: python-nmap; extra == "all"
+Requires-Dist: hyperopt; extra == "all"
+Requires-Dist: bayesian-optimization==1.4.3; extra == "all"
+Provides-Extra: devtools
+Requires-Dist: streamlit>=1.29.0; extra == "devtools"
+Requires-Dist: streamlit-ace>=0.1.1; extra == "devtools"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.10.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: pytest-xdist>=3.0.0; extra == "test"
+Requires-Dist: pytest-html>=3.1.0; extra == "test"
+Requires-Dist: pytest-json-report>=1.5.0; extra == "test"
+Requires-Dist: requests-mock>=1.10.0; extra == "test"
+Requires-Dist: responses>=0.25.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == "dev"
+Dynamic: license-file
+
+# 🧠 Synapse SDK
+
+![lint workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/lint.yml/badge.svg)
+![test workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/test.yml/badge.svg)
+
+A Python SDK for building and managing ML plugins, data annotation workflows, and AI agents.
+
+## ✨ Features
+
+- **🔌 Plugin System**: Create and manage ML plugins with categories like neural networks, data validation, and export tools
+- **🤖 Agent Management**: Backend and Ray-based agent clients for distributed AI workflows  
+- **🔄 Data Converters**: Convert between formats (COCO, Pascal VOC, YOLO) and annotation schemas
+- **🛠️ Development Tools**: Interactive web dashboard for monitoring and debugging
+- **⚡ CLI Interface**: Command-line tool for configuration, plugin management, and development
+
+## 📅 Versioning
+
+Synapse SDK uses **CalVer (Calendar Versioning)**.  
+The version format is `YYYY.MM.PATCH`, based on the release year and month.
+
+Examples:
+
+- `2025.9.5` → 5th release in September 2025  
+- `2025.10.1` → 1st release in October 2025  
+
+We recommend using the latest monthly release.  
+If needed, you can pin to a specific month, e.g. `2025.9.*`.
+
+## 🚀 Quick Start
+
+```bash
+pip install synapse-sdk
+synapse --help
+```
+
+## 🔍 Code Review
+
+This repository uses systematic code review with P1-P4 priority rules:
+
+### Using the Review-PR Command
+
+Review pull requests using the integrated review system:
+
+```bash
+# Review a PR with English comments  
+/review-pr 123
+
+# Review a PR with Korean comments
+/review-pr 123 ko
+```
+
+### Code Review Priority Levels
+
+- **[P1_rules.md](P1_rules.md)** - Security and Stability (Critical) 🔴
+- **[P2_rules.md](P2_rules.md)** - Core Functionality (High Priority) 🟡  
+- **[P3_rules.md](P3_rules.md)** - Best Practices (Medium Priority) 🟠
+- **[P4_rules.md](P4_rules.md)** - Code Style (Low Priority) 🔵
+
+### Review Process
+
+1. **Automated Analysis**: The review-pr command systematically applies P1-P4 rules
+2. **Priority-Based Feedback**: Issues are categorized by severity and impact
+3. **Actionable Comments**: Each issue includes specific recommendations and rule references
+4. **Language Support**: Comments can be generated in English or Korean
+5. **Decision Logic**:
+   - P1, P2, or P3 violations → Request Changes
+   - Only P4 violations or no issues → Approve
+
+See [AGENT.md](AGENT.md) for complete development guidelines and code review rules.
+
+## 📚 Documentation
+
+*Docs [https://docs.synapse.sh](https://docs.synapse.sh)*