synapse-sdk 1.0.0a35__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/utils/storage/providers/http.py

@@ -0,0 +1,190 @@
+import os
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class HTTPStorage(BaseStorage):
+    """Storage provider for no-auth HTTP file servers (e.g., Django FileSystemStorage served over HTTP)."""
+
+    OPTION_CASTS = {
+        'timeout': int,
+    }
+
+    def __init__(self, connection_params: str | dict):
+        super().__init__(connection_params)
+
+        # Extract base URL
+        if isinstance(connection_params, dict):
+            self.base_url = self.query_params.get('base_url', '')
+            self.timeout = self.query_params.get('timeout', 30)
+        else:
+            # Parse URL like: http://example.com/media/
+            parsed = urlparse(connection_params)
+            self.base_url = f'{parsed.scheme}://{parsed.netloc}{parsed.path}'
+            self.timeout = self.query_params.get('timeout', 30)
+
+        # Ensure base_url ends with /
+        if not self.base_url.endswith('/'):
+            self.base_url += '/'
+
+        # Setup session for connection pooling
+        self.session = requests.Session()
+
+    def _get_full_url(self, path: str) -> str:
+        """Get the full URL for a given path."""
+        # Remove leading slash from path to avoid double slashes
+        if path.startswith('/'):
+            path = path[1:]
+        return urljoin(self.base_url, path)
+
+    def upload(self, source: str, target: str) -> str:
+        """Upload a file to the HTTP server.
+
+        Args:
+            source: Local file path to upload
+            target: Target path on the HTTP server
+
+        Returns:
+            URL of the uploaded file
+        """
+        url = self._get_full_url(target)
+
+        with open(source, 'rb') as f:
+            files = {'file': (os.path.basename(source), f)}
+
+            # Try PUT first (more RESTful), fallback to POST
+            response = self.session.put(url, files=files, timeout=self.timeout)
+
+            if response.status_code == 405:  # Method not allowed
+                # Reset file pointer and try POST
+                f.seek(0)
+                response = self.session.post(url, files=files, timeout=self.timeout)
+
+            response.raise_for_status()
+
+        return url
+
+    def exists(self, target: str) -> bool:
+        """Check if a file exists on the HTTP server.
+
+        Args:
+            target: Path to check
+
+        Returns:
+            True if file exists, False otherwise
+        """
+        url = self._get_full_url(target)
+
+        try:
+            response = self.session.head(url, timeout=self.timeout)
+            return response.status_code == 200
+        except requests.RequestException:
+            return False
+
+    def get_url(self, target: str) -> str:
+        """Get the URL for a file.
+
+        Args:
+            target: File path
+
+        Returns:
+            Full URL of the file
+        """
+        return self._get_full_url(target)
+
+    def get_pathlib(self, path: str) -> 'HTTPPath':
+        """Get a pathlib-like object for HTTP operations.
+
+        Args:
+            path: Path to wrap
+
+        Returns:
+            HTTPPath object
+        """
+        return HTTPPath(self, path)
+
+    def get_path_file_count(self, pathlib_obj) -> int:
+        """Get file count in a directory.
+
+        Note: This requires the HTTP server to provide directory listing functionality.
+
+        Args:
+            pathlib_obj: HTTPPath object
+
+        Returns:
+            File count
+        """
+        # Most HTTP servers don't provide directory listing
+        # This would need custom server-side support
+        raise NotImplementedError('File counting requires server-side directory listing support')
+
+    def get_path_total_size(self, pathlib_obj) -> int:
+        """Get total size of files in a directory.
+
+        Note: This requires the HTTP server to provide directory listing functionality.
+
+        Args:
+            pathlib_obj: HTTPPath object
+
+        Returns:
+            Total size in bytes
+        """
+        # Most HTTP servers don't provide directory listing
+        # This would need custom server-side support
+        raise NotImplementedError('Size calculation requires server-side directory listing support')
+
+
+class HTTPPath:
+    """A pathlib-like interface for HTTP paths."""
+
+    def __init__(self, storage: HTTPStorage, path: str):
+        self.storage = storage
+        self.path = path.strip('/')
+
+    def __str__(self):
+        return self.path
+
+    def __truediv__(self, other):
+        """Join paths using / operator."""
+        new_path = f'{self.path}/{other}' if self.path else str(other)
+        return HTTPPath(self.storage, new_path)
+
+    def joinuri(self, *parts):
+        """Join path parts."""
+        parts = [self.path] + [str(p) for p in parts]
+        new_path = '/'.join(p.strip('/') for p in parts if p)
+        return HTTPPath(self.storage, new_path)
+
+    @property
+    def name(self):
+        """Get the final component of the path."""
+        return os.path.basename(self.path)
+
+    @property
+    def parent(self):
+        """Get the parent directory."""
+        parent_path = os.path.dirname(self.path)
+        return HTTPPath(self.storage, parent_path)
+
+    def exists(self):
+        """Check if this path exists."""
+        return self.storage.exists(self.path)
+
+    def is_file(self):
+        """Check if this path is a file."""
+        # For HTTP, we assume it's a file if it exists
+        return self.exists()
+
+    def read_bytes(self):
+        """Read file contents as bytes."""
+        url = self.storage.get_url(self.path)
+        response = self.storage.session.get(url, timeout=self.storage.timeout)
+        response.raise_for_status()
+        return response.content
+
+    def read_text(self, encoding='utf-8'):
+        """Read file contents as text."""
+        return self.read_bytes().decode(encoding)

synapse_sdk/utils/storage/providers/s3.py

@@ -13,15 +13,17 @@ class S3Storage(BaseStorage):
         self.upath = self._get_upath()
 
     def _get_upath(self):
+        client_kwargs = {'region_name': self.query_params.get('region_name', self.DEFAULT_REGION)}
+
+        if self.query_params.get('endpoint_url'):
+            client_kwargs['endpoint_url'] = self.query_params['endpoint_url']
+
         upath_kwargs = {
             'key': self.query_params['access_key'],
             'secret': self.query_params['secret_key'],
-            'client_kwargs': {'region_name': self.query_params.get('region_name')},
+            'client_kwargs': client_kwargs,
         }
 
-        if self.query_params.get('endpoint_url'):
-            upath_kwargs['endpoint_url'] = self.query_params['endpoint_url']
-
         return UPath(
             f's3://{self.query_params["bucket_name"]}',
             **upath_kwargs,
@@ -29,15 +31,61 @@ class S3Storage(BaseStorage):
 
     def upload(self, source, target):
         with open(source, 'rb') as file:
-            self.upath.write_text(file.read(), target)
+            (self.upath / target).write_bytes(file.read())
 
         return self.get_url(target)
 
     def exists(self, target):
-        return self.upath.exists(target)
+        return (self.upath / target).exists()
 
     def get_url(self, target):
         return str(self.upath.joinuri(target))
 
     def get_pathlib(self, path):
         return self.upath.joinuri(path)
+
+    def get_path_file_count(self, pathlib_obj: UPath):
+        """Get file count in the path from S3 provider.
+
+        TODO: Need to find a method to get file count using S3 API
+
+        Args:
+            pathlib_obj (UPath): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        count = 0
+        files = list(pathlib_obj.glob('**/*'))
+        for file in files:
+            if file.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj: UPath):
+        """Get total size of the files in the path from S3 provider.
+
+        TODO: Need to find a method to get total file size using S3 API
+
+        Args:
+            pathlib_obj (UPath): The path to get total size.
+
+        Returns:
+            int: The total size of the files in the path.
+        """
+        total_size = 0
+        for file in pathlib_obj.glob('**/*'):
+            if file.is_file():
+                total_size += file.stat().st_size
+        return total_size
+
+    def glob(self, pattern):
+        """Glob pattern matching in S3 storage.
+
+        Args:
+            pattern (str): The glob pattern to match.
+
+        Returns:
+            list: List of matching paths.
+        """
+        return list((self.upath / '').glob(pattern))

synapse_sdk/utils/storage/providers/sftp.py

@@ -14,3 +14,34 @@ class SFTPStorage(BaseStorage):
         if path == '/':
             path = ''
         return UPath(f'sftp://{host}', username=username, password=password) / root_path / path
+
+    def get_path_file_count(self, pathlib_obj: UPath):
+        """Get file count in the path from SFTP provider.
+
+        Args:
+            pathlib_obj (UPath): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        count = 0
+        files = list(pathlib_obj.glob('**/*'))
+        for file in files:
+            if file.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj: UPath):
+        """Get total size of the files in the path from SFTP provider.
+
+        Args:
+            pathlib_obj (UPath): The path to get total size.
+
+        Returns:
+            int: The total size of the files in the path.
+        """
+        total_size = 0
+        for file in pathlib_obj.glob('**/*'):
+            if file.is_file():
+                total_size += file.stat().st_size
+        return total_size

synapse_sdk/utils/storage/registry.py

@@ -1,4 +1,6 @@
+from synapse_sdk.utils.storage.providers.file_system import FileSystemStorage
 from synapse_sdk.utils.storage.providers.gcp import GCPStorage
+from synapse_sdk.utils.storage.providers.http import HTTPStorage
 from synapse_sdk.utils.storage.providers.s3 import S3Storage
 from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
 
@@ -6,6 +8,10 @@ STORAGE_PROVIDERS = {
     's3': S3Storage,
     'amazon_s3': S3Storage,
     'minio': S3Storage,
+    'gs': GCPStorage,
     'gcp': GCPStorage,
     'sftp': SFTPStorage,
+    'http': HTTPStorage,
+    'https': HTTPStorage,
+    'file_system': FileSystemStorage,
 }

synapse_sdk-2025.11.7.dist-info/METADATA

@@ -0,0 +1,122 @@
+Metadata-Version: 2.4
+Name: synapse-sdk
+Version: 2025.11.7
+Summary: synapse sdk
+Author-email: datamaker <developer@datamaker.io>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: boto3
+Requires-Dist: click
+Requires-Dist: cookiecutter
+Requires-Dist: requests
+Requires-Dist: tqdm
+Requires-Dist: openpyxl
+Requires-Dist: python-dotenv
+Requires-Dist: pyyaml
+Requires-Dist: pydantic
+Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
+Requires-Dist: inquirer
+Requires-Dist: pillow
+Requires-Dist: websockets
+Requires-Dist: ffmpeg-python==0.2.0
+Requires-Dist: sentry-sdk==2.38.0
+Provides-Extra: all
+Requires-Dist: ray[all]==2.50.0; extra == "all"
+Requires-Dist: python-nmap; extra == "all"
+Requires-Dist: hyperopt; extra == "all"
+Requires-Dist: bayesian-optimization==1.4.3; extra == "all"
+Provides-Extra: devtools
+Requires-Dist: streamlit>=1.29.0; extra == "devtools"
+Requires-Dist: streamlit-ace>=0.1.1; extra == "devtools"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.10.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: pytest-xdist>=3.0.0; extra == "test"
+Requires-Dist: pytest-html>=3.1.0; extra == "test"
+Requires-Dist: pytest-json-report>=1.5.0; extra == "test"
+Requires-Dist: requests-mock>=1.10.0; extra == "test"
+Requires-Dist: responses>=0.25.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == "dev"
+Dynamic: license-file
+
+# 🧠 Synapse SDK
+
+![lint workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/lint.yml/badge.svg)
+![test workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/test.yml/badge.svg)
+
+A Python SDK for building and managing ML plugins, data annotation workflows, and AI agents.
+
+## ✨ Features
+
+- **🔌 Plugin System**: Create and manage ML plugins with categories like neural networks, data validation, and export tools
+- **🤖 Agent Management**: Backend and Ray-based agent clients for distributed AI workflows  
+- **🔄 Data Converters**: Convert between formats (COCO, Pascal VOC, YOLO) and annotation schemas
+- **🛠️ Development Tools**: Interactive web dashboard for monitoring and debugging
+- **⚡ CLI Interface**: Command-line tool for configuration, plugin management, and development
+
+## 📅 Versioning
+
+Synapse SDK uses **CalVer (Calendar Versioning)**.  
+The version format is `YYYY.MM.PATCH`, based on the release year and month.
+
+Examples:
+
+- `2025.9.5` → 5th release in September 2025  
+- `2025.10.1` → 1st release in October 2025  
+
+We recommend using the latest monthly release.  
+If needed, you can pin to a specific month, e.g. `2025.9.*`.
+
+## 🚀 Quick Start
+
+```bash
+pip install synapse-sdk
+synapse --help
+```
+
+## 🔍 Code Review
+
+This repository uses systematic code review with P1-P4 priority rules:
+
+### Using the Review-PR Command
+
+Review pull requests using the integrated review system:
+
+```bash
+# Review a PR with English comments  
+/review-pr 123
+
+# Review a PR with Korean comments
+/review-pr 123 ko
+```
+
+### Code Review Priority Levels
+
+- **[P1_rules.md](P1_rules.md)** - Security and Stability (Critical) 🔴
+- **[P2_rules.md](P2_rules.md)** - Core Functionality (High Priority) 🟡  
+- **[P3_rules.md](P3_rules.md)** - Best Practices (Medium Priority) 🟠
+- **[P4_rules.md](P4_rules.md)** - Code Style (Low Priority) 🔵
+
+### Review Process
+
+1. **Automated Analysis**: The review-pr command systematically applies P1-P4 rules
+2. **Priority-Based Feedback**: Issues are categorized by severity and impact
+3. **Actionable Comments**: Each issue includes specific recommendations and rule references
+4. **Language Support**: Comments can be generated in English or Korean
+5. **Decision Logic**:
+   - P1, P2, or P3 violations → Request Changes
+   - Only P4 violations or no issues → Approve
+
+See [AGENT.md](AGENT.md) for complete development guidelines and code review rules.
+
+## 📚 Documentation
+
+*Docs [https://docs.synapse.sh](https://docs.synapse.sh)*