synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl

synapse_sdk/utils/storage/providers/s3.py

@@ -0,0 +1,177 @@
+"""S3-compatible storage provider (AWS S3, MinIO)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from synapse_sdk.utils.storage.config import S3StorageConfig
+from synapse_sdk.utils.storage.errors import (
+    StorageConnectionError,
+    StorageNotFoundError,
+    StorageUploadError,
+)
+from synapse_sdk.utils.storage.providers.base import _BaseStorageMixin
+
+if TYPE_CHECKING:
+    from upath import UPath
+
+
+class S3Storage(_BaseStorageMixin):
+    """Storage provider for S3-compatible services (AWS S3, MinIO).
+
+    Requires: universal-pathlib[s3] (pip install universal-pathlib[s3])
+
+    Args:
+        config: Configuration dict with S3 credentials.
+
+    Example:
+        >>> storage = S3Storage({
+        ...     'bucket_name': 'my-bucket',
+        ...     'access_key': 'AKIAIOSFODNN7EXAMPLE',
+        ...     'secret_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+        ...     'region_name': 'us-east-1',
+        ... })
+        >>> storage.upload(Path('/tmp/file.txt'), 'data/file.txt')
+        's3://my-bucket/data/file.txt'
+    """
+
+    DEFAULT_REGION = 'us-east-1'
+
+    def __init__(self, config: dict[str, Any]):
+        try:
+            from upath import UPath
+        except ImportError as e:
+            raise ImportError(
+                'S3Storage requires universal-pathlib[s3]. Install with: pip install universal-pathlib[s3]'
+            ) from e
+
+        validated = S3StorageConfig.model_validate(config)
+
+        self._bucket_name = validated.bucket_name
+        client_kwargs: dict[str, Any] = {
+            'region_name': validated.region_name or self.DEFAULT_REGION,
+        }
+
+        if validated.endpoint_url:
+            client_kwargs['endpoint_url'] = validated.endpoint_url
+
+        try:
+            self._upath = UPath(
+                f's3://{validated.bucket_name}',
+                key=validated.access_key,
+                secret=validated.secret_key,
+                client_kwargs=client_kwargs,
+            )
+        except Exception as e:
+            raise StorageConnectionError(
+                f'Failed to connect to S3: {e}',
+                details={'bucket': validated.bucket_name},
+            ) from e
+
+    def upload(self, source: Path, target: str) -> str:
+        """Upload a file to S3.
+
+        Args:
+            source: Local file path to upload.
+            target: Target path in S3 bucket.
+
+        Returns:
+            s3:// URL of uploaded file.
+        """
+        source_path = Path(source) if isinstance(source, str) else source
+
+        if not source_path.exists():
+            raise StorageNotFoundError(
+                f'Source file not found: {source_path}',
+                details={'source': str(source_path)},
+            )
+
+        target_path = self._normalize_path(target)
+
+        try:
+            with open(source_path, 'rb') as f:
+                (self._upath / target_path).write_bytes(f.read())
+        except Exception as e:
+            raise StorageUploadError(
+                f'Failed to upload to S3: {e}',
+                details={'source': str(source_path), 'target': target_path},
+            ) from e
+
+        return self.get_url(target)
+
+    def exists(self, target: str) -> bool:
+        """Check if file exists in S3.
+
+        Args:
+            target: Path to check.
+
+        Returns:
+            True if exists, False otherwise.
+        """
+        target_path = self._normalize_path(target)
+        return (self._upath / target_path).exists()
+
+    def get_url(self, target: str) -> str:
+        """Get s3:// URL for target.
+
+        Args:
+            target: Target path.
+
+        Returns:
+            s3:// URL string.
+        """
+        target_path = self._normalize_path(target)
+        if target_path:
+            return f's3://{self._bucket_name}/{target_path}'
+        return f's3://{self._bucket_name}'
+
+    def get_pathlib(self, path: str) -> UPath:
+        """Get UPath object for path.
+
+        Args:
+            path: Path relative to bucket root.
+
+        Returns:
+            UPath object.
+        """
+        normalized = self._normalize_path(path)
+        if not normalized:
+            return self._upath
+        return self._upath / normalized
+
+    def get_path_file_count(self, pathlib_obj: UPath) -> int:
+        """Count files in S3 path.
+
+        Args:
+            pathlib_obj: UPath object from get_pathlib().
+
+        Returns:
+            Number of files.
+        """
+        return self._count_files(pathlib_obj)
+
+    def get_path_total_size(self, pathlib_obj: UPath) -> int:
+        """Calculate total size of files in S3 path.
+
+        Args:
+            pathlib_obj: UPath object from get_pathlib().
+
+        Returns:
+            Total size in bytes.
+        """
+        return self._calculate_total_size(pathlib_obj)
+
+    def glob(self, pattern: str) -> list[UPath]:
+        """Glob pattern matching in S3.
+
+        Args:
+            pattern: Glob pattern.
+
+        Returns:
+            List of matching UPath objects.
+        """
+        return list(self._upath.glob(pattern))
+
+
+__all__ = ['S3Storage']

synapse_sdk/utils/storage/providers/sftp.py

@@ -0,0 +1,208 @@
+"""SFTP storage provider."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from synapse_sdk.utils.storage.config import SFTPStorageConfig
+from synapse_sdk.utils.storage.errors import (
+    StorageConnectionError,
+    StorageNotFoundError,
+    StorageUploadError,
+)
+from synapse_sdk.utils.storage.providers.base import _BaseStorageMixin
+
+if TYPE_CHECKING:
+    from upath import UPath
+
+
+class SFTPStorage(_BaseStorageMixin):
+    """Storage provider for SFTP servers.
+
+    Requires: universal-pathlib[sftp] (pip install universal-pathlib[sftp])
+
+    Supports both password and private key authentication.
+
+    Args:
+        config: Configuration dict with SFTP credentials.
+
+    Example:
+        >>> # Password authentication
+        >>> storage = SFTPStorage({
+        ...     'host': 'sftp.example.com',
+        ...     'username': 'user',
+        ...     'password': 'secret',
+        ...     'root_path': '/data',
+        ... })
+        >>>
+        >>> # Private key authentication
+        >>> storage = SFTPStorage({
+        ...     'host': 'sftp.example.com',
+        ...     'username': 'user',
+        ...     'private_key': '/path/to/id_rsa',
+        ...     'root_path': '/data',
+        ... })
+    """
+
+    def __init__(self, config: dict[str, Any]):
+        try:
+            from upath import UPath
+        except ImportError as e:
+            raise ImportError(
+                'SFTPStorage requires universal-pathlib[sftp]. Install with: pip install universal-pathlib[sftp]'
+            ) from e
+
+        validated = SFTPStorageConfig.model_validate(config)
+        self._config = validated
+
+        # Build UPath kwargs based on auth method
+        upath_kwargs: dict[str, Any] = {
+            'username': validated.username,
+        }
+
+        if validated.password:
+            upath_kwargs['password'] = validated.password
+
+        if validated.private_key:
+            upath_kwargs['key_filename'] = validated.private_key
+            if validated.private_key_passphrase:
+                upath_kwargs['passphrase'] = validated.private_key_passphrase
+
+        # Construct base URL with port if non-default
+        if validated.port != 22:
+            base_url = f'sftp://{validated.host}:{validated.port}'
+        else:
+            base_url = f'sftp://{validated.host}'
+
+        try:
+            self._base_upath = UPath(base_url, **upath_kwargs)
+            self._root_path = validated.root_path.rstrip('/')
+            self._host = validated.host
+            self._port = validated.port
+        except Exception as e:
+            raise StorageConnectionError(
+                f'Failed to connect to SFTP: {e}',
+                details={'host': validated.host, 'port': validated.port},
+            ) from e
+
+    def _get_full_path(self, path: str) -> UPath:
+        """Get full UPath including root_path."""
+        normalized = self._normalize_path(path)
+        if self._root_path:
+            full_path = f'{self._root_path}/{normalized}' if normalized else self._root_path
+        else:
+            full_path = normalized or '/'
+        return self._base_upath / full_path.lstrip('/')
+
+    def upload(self, source: Path, target: str) -> str:
+        """Upload a file via SFTP.
+
+        Args:
+            source: Local file path to upload.
+            target: Target path on SFTP server.
+
+        Returns:
+            sftp:// URL of uploaded file.
+        """
+        source_path = Path(source) if isinstance(source, str) else source
+
+        if not source_path.exists():
+            raise StorageNotFoundError(
+                f'Source file not found: {source_path}',
+                details={'source': str(source_path)},
+            )
+
+        target_upath = self._get_full_path(target)
+
+        try:
+            # Ensure parent directory exists
+            target_upath.parent.mkdir(parents=True, exist_ok=True)
+
+            with open(source_path, 'rb') as f:
+                target_upath.write_bytes(f.read())
+        except Exception as e:
+            raise StorageUploadError(
+                f'Failed to upload via SFTP: {e}',
+                details={'source': str(source_path), 'target': str(target_upath)},
+            ) from e
+
+        return self.get_url(target)
+
+    def exists(self, target: str) -> bool:
+        """Check if file exists on SFTP server.
+
+        Args:
+            target: Path to check.
+
+        Returns:
+            True if exists, False otherwise.
+        """
+        return self._get_full_path(target).exists()
+
+    def get_url(self, target: str) -> str:
+        """Get sftp:// URL for target.
+
+        Args:
+            target: Target path.
+
+        Returns:
+            sftp:// URL string.
+        """
+        normalized = self._normalize_path(target)
+        if self._root_path:
+            full_path = f'{self._root_path}/{normalized}' if normalized else self._root_path
+        else:
+            full_path = normalized or '/'
+
+        if self._port != 22:
+            return f'sftp://{self._host}:{self._port}{full_path}'
+        return f'sftp://{self._host}{full_path}'
+
+    def get_pathlib(self, path: str) -> UPath:
+        """Get UPath object for path.
+
+        Args:
+            path: Path relative to root_path.
+
+        Returns:
+            UPath object.
+        """
+        return self._get_full_path(path)
+
+    def get_path_file_count(self, pathlib_obj: UPath) -> int:
+        """Count files in SFTP path.
+
+        Args:
+            pathlib_obj: UPath object from get_pathlib().
+
+        Returns:
+            Number of files.
+        """
+        return self._count_files(pathlib_obj)
+
+    def get_path_total_size(self, pathlib_obj: UPath) -> int:
+        """Calculate total size of files in SFTP path.
+
+        Args:
+            pathlib_obj: UPath object from get_pathlib().
+
+        Returns:
+            Total size in bytes.
+        """
+        return self._calculate_total_size(pathlib_obj)
+
+    def glob(self, pattern: str) -> list[UPath]:
+        """Glob pattern matching on SFTP.
+
+        Args:
+            pattern: Glob pattern.
+
+        Returns:
+            List of matching UPath objects.
+        """
+        base = self._get_full_path('')
+        return list(base.glob(pattern))
+
+
+__all__ = ['SFTPStorage']

synapse_sdk/utils/storage/registry.py

@@ -0,0 +1,125 @@
+"""Storage provider registry with lazy loading."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable
+
+from synapse_sdk.utils.storage.errors import StorageProviderNotFoundError
+
+if TYPE_CHECKING:
+    from synapse_sdk.utils.storage import StorageProtocol
+
+# Registry stores factory functions for lazy loading
+_PROVIDER_REGISTRY: dict[str, Callable[[], type[StorageProtocol]]] = {}
+
+
+def _register_builtin_providers() -> None:
+    """Register built-in storage providers with lazy loading."""
+
+    def local_factory() -> type:
+        from synapse_sdk.utils.storage.providers.local import LocalStorage
+
+        return LocalStorage
+
+    def s3_factory() -> type:
+        from synapse_sdk.utils.storage.providers.s3 import S3Storage
+
+        return S3Storage
+
+    def gcs_factory() -> type:
+        from synapse_sdk.utils.storage.providers.gcs import GCSStorage
+
+        return GCSStorage
+
+    def sftp_factory() -> type:
+        from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
+
+        return SFTPStorage
+
+    def http_factory() -> type:
+        from synapse_sdk.utils.storage.providers.http import HTTPStorage
+
+        return HTTPStorage
+
+    # Local filesystem
+    _PROVIDER_REGISTRY['local'] = local_factory
+    _PROVIDER_REGISTRY['file_system'] = local_factory  # backward compatibility
+
+    # S3-compatible
+    _PROVIDER_REGISTRY['s3'] = s3_factory
+    _PROVIDER_REGISTRY['amazon_s3'] = s3_factory
+    _PROVIDER_REGISTRY['minio'] = s3_factory
+
+    # Google Cloud Storage
+    _PROVIDER_REGISTRY['gcs'] = gcs_factory
+    _PROVIDER_REGISTRY['gs'] = gcs_factory
+    _PROVIDER_REGISTRY['gcp'] = gcs_factory
+
+    # SFTP
+    _PROVIDER_REGISTRY['sftp'] = sftp_factory
+
+    # HTTP
+    _PROVIDER_REGISTRY['http'] = http_factory
+    _PROVIDER_REGISTRY['https'] = http_factory
+
+
+def get_provider_class(provider: str) -> type[StorageProtocol]:
+    """Get the storage provider class for the given provider name.
+
+    Args:
+        provider: Provider name (e.g., 's3', 'gcs', 'local').
+
+    Returns:
+        Storage provider class.
+
+    Raises:
+        StorageProviderNotFoundError: If provider is not registered.
+    """
+    if not _PROVIDER_REGISTRY:
+        _register_builtin_providers()
+
+    factory = _PROVIDER_REGISTRY.get(provider)
+    if not factory:
+        available = ', '.join(sorted(_PROVIDER_REGISTRY.keys()))
+        raise StorageProviderNotFoundError(
+            f"Provider '{provider}' not found. Available: {available}",
+            details={'provider': provider, 'available': list(_PROVIDER_REGISTRY.keys())},
+        )
+
+    return factory()
+
+
+def register_provider(name: str, factory: Callable[[], type[StorageProtocol]]) -> None:
+    """Register a custom storage provider.
+
+    Args:
+        name: Provider name to register.
+        factory: Factory function that returns the provider class.
+
+    Example:
+        >>> def custom_factory():
+        ...     from my_module import CustomStorage
+        ...     return CustomStorage
+        >>> register_provider('custom', custom_factory)
+    """
+    if not _PROVIDER_REGISTRY:
+        _register_builtin_providers()
+    _PROVIDER_REGISTRY[name] = factory
+
+
+def get_registered_providers() -> list[str]:
+    """Get list of registered provider names.
+
+    Returns:
+        List of registered provider names.
+    """
+    if not _PROVIDER_REGISTRY:
+        _register_builtin_providers()
+    return list(_PROVIDER_REGISTRY.keys())
+
+
+__all__ = [
+    'get_provider_class',
+    'register_provider',
+    'get_registered_providers',
+]

synapse_sdk/utils/websocket.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import json
+from typing import Generator
+
+from synapse_sdk.clients.exceptions import ClientError
+
+
+def stream_websocket(
+    url: str,
+    headers: dict[str, str] | None = None,
+    timeout: float = 30.0,
+) -> Generator[dict, None, None]:
+    """Stream raw events from a WebSocket connection.
+
+    Args:
+        url: WebSocket URL (ws:// or wss://).
+        headers: Optional headers dict.
+        timeout: Connection timeout in seconds.
+
+    Yields:
+        Parsed JSON events from the WebSocket.
+
+    Raises:
+        ClientError: On connection or protocol errors.
+    """
+    try:
+        import websocket
+    except ImportError:
+        raise ClientError(500, 'websocket-client package required for streaming')
+
+    header_list = [f'{k}: {v}' for k, v in (headers or {}).items()]
+
+    try:
+        ws = websocket.create_connection(url, header=header_list, timeout=timeout)
+    except websocket.WebSocketException as e:
+        raise ClientError(503, f'WebSocket connection failed: {e}')
+    except Exception as e:
+        raise ClientError(503, f'Connection error: {e}')
+
+    try:
+        while True:
+            try:
+                data = ws.recv()
+            except websocket.WebSocketTimeoutException:
+                break
+            except websocket.WebSocketConnectionClosedException:
+                break
+
+            if not data:
+                break
+
+            try:
+                event = json.loads(data)
+            except json.JSONDecodeError:
+                event = {'message': data}
+
+            yield event
+    finally:
+        ws.close()
+
+
+def stream_websocket_logs(
+    url: str,
+    headers: dict[str, str] | None = None,
+    timeout: float = 30.0,
+) -> Generator[str, None, None]:
+    """Stream log messages from a WebSocket connection.
+
+    Handles the standard log streaming protocol:
+    - 'log' events: yields the message
+    - 'error' events: raises ClientError
+    - 'complete' events: stops iteration
+
+    Args:
+        url: WebSocket URL (ws:// or wss://).
+        headers: Optional headers dict.
+        timeout: Connection timeout in seconds.
+
+    Yields:
+        Log message strings.
+
+    Raises:
+        ClientError: On connection, protocol, or server errors.
+    """
+    for event in stream_websocket(url, headers, timeout):
+        match event.get('type'):
+            case 'error':
+                raise ClientError(500, event.get('message', 'Unknown error'))
+            case 'complete':
+                return
+            case _:
+                if msg := event.get('message'):
+                    yield msg
+
+
+def http_to_ws_url(url: str) -> str:
+    """Convert HTTP URL to WebSocket URL."""
+    return url.replace('http://', 'ws://').replace('https://', 'wss://')