synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl

synapse_sdk/utils/storage/__init__.py

@@ -0,0 +1,238 @@
+"""Storage utilities module.
+
+This module provides a unified interface for working with different storage
+backends including local filesystem, S3, GCS, SFTP, and HTTP.
+
+Example:
+    >>> from synapse_sdk.utils.storage import get_storage, get_pathlib
+    >>>
+    >>> # Local filesystem
+    >>> storage = get_storage({
+    ...     'provider': 'local',
+    ...     'configuration': {'location': '/data'}
+    ... })
+    >>>
+    >>> # S3-compatible storage
+    >>> storage = get_storage({
+    ...     'provider': 's3',
+    ...     'configuration': {
+    ...         'bucket_name': 'my-bucket',
+    ...         'access_key': 'AKIAIOSFODNN7EXAMPLE',
+    ...         'secret_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+    ...     }
+    ... })
+    >>>
+    >>> # Get pathlib object for path operations
+    >>> path = get_pathlib(storage_config, '/uploads')
+    >>> for file in path.rglob('*'):
+    ...     print(file)
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from upath import UPath
+
+
+@runtime_checkable
+class StorageProtocol(Protocol):
+    """Protocol defining the storage provider interface.
+
+    All storage providers must implement these methods to be compatible
+    with the storage system. Uses structural typing (duck typing) rather
+    than inheritance, allowing third-party implementations.
+
+    Example:
+        >>> class CustomStorage:
+        ...     def upload(self, source: Path, target: str) -> str: ...
+        ...     def exists(self, target: str) -> bool: ...
+        ...     def get_url(self, target: str) -> str: ...
+        ...     def get_pathlib(self, path: str) -> Path: ...
+        ...     def get_path_file_count(self, pathlib_obj: Path) -> int: ...
+        ...     def get_path_total_size(self, pathlib_obj: Path) -> int: ...
+        >>>
+        >>> isinstance(CustomStorage(), StorageProtocol)  # True
+    """
+
+    def upload(self, source: Path, target: str) -> str:
+        """Upload a file from local source to target path.
+
+        Args:
+            source: Local file path to upload.
+            target: Target path in storage (relative to storage root).
+
+        Returns:
+            URL or identifier of the uploaded file.
+
+        Raises:
+            StorageNotFoundError: If source file doesn't exist.
+            StorageUploadError: If upload fails.
+        """
+        ...
+
+    def exists(self, target: str) -> bool:
+        """Check if a file or directory exists at target path.
+
+        Args:
+            target: Path to check (relative to storage root).
+
+        Returns:
+            True if path exists, False otherwise.
+        """
+        ...
+
+    def get_url(self, target: str) -> str:
+        """Get the URL for accessing a file.
+
+        Args:
+            target: Path to file (relative to storage root).
+
+        Returns:
+            URL string for accessing the file.
+        """
+        ...
+
+    def get_pathlib(self, path: str) -> Path | UPath:
+        """Get a pathlib-compatible object for the path.
+
+        Args:
+            path: Path relative to storage root.
+
+        Returns:
+            Path object (local) or UPath object (cloud/remote).
+        """
+        ...
+
+    def get_path_file_count(self, pathlib_obj: Path | UPath) -> int:
+        """Count files recursively in the given path.
+
+        Args:
+            pathlib_obj: Path object from get_pathlib().
+
+        Returns:
+            Number of files (excluding directories).
+        """
+        ...
+
+    def get_path_total_size(self, pathlib_obj: Path | UPath) -> int:
+        """Calculate total size of files recursively.
+
+        Args:
+            pathlib_obj: Path object from get_pathlib().
+
+        Returns:
+            Total size in bytes.
+        """
+        ...
+
+
+def get_storage(connection_param: dict[str, Any]) -> StorageProtocol:
+    """Get a storage provider instance from configuration.
+
+    Args:
+        connection_param: Dictionary with 'provider' and 'configuration' keys.
+            Example: {'provider': 's3', 'configuration': {'bucket_name': '...'}}
+
+    Returns:
+        Storage provider instance implementing StorageProtocol.
+
+    Raises:
+        StorageConfigError: If configuration is invalid.
+        StorageProviderNotFoundError: If provider is not registered.
+
+    Example:
+        >>> config = {
+        ...     'provider': 's3',
+        ...     'configuration': {
+        ...         'bucket_name': 'my-bucket',
+        ...         'access_key': 'AKIAIOSFODNN7EXAMPLE',
+        ...         'secret_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+        ...     }
+        ... }
+        >>> storage = get_storage(config)
+    """
+    from synapse_sdk.utils.storage.config import StorageConfig
+    from synapse_sdk.utils.storage.registry import get_provider_class
+
+    # Validate configuration with Pydantic
+    storage_config = StorageConfig.model_validate(connection_param)
+
+    # Get provider class and instantiate
+    provider_cls = get_provider_class(storage_config.provider)
+    return provider_cls(storage_config.configuration)
+
+
+def get_pathlib(storage_config: dict[str, Any], path_root: str) -> Path | UPath:
+    """Get pathlib object for a path in storage.
+
+    Convenience function that combines get_storage() and get_pathlib().
+
+    Args:
+        storage_config: Storage configuration dict.
+        path_root: Root path to get pathlib for.
+
+    Returns:
+        Path or UPath object.
+
+    Example:
+        >>> config = {'provider': 'local', 'configuration': {'location': '/data'}}
+        >>> path = get_pathlib(config, '/uploads')
+        >>> path.exists()
+        True
+    """
+    storage = get_storage(storage_config)
+    return storage.get_pathlib(path_root)
+
+
+def get_path_file_count(storage_config: dict[str, Any], path_root: str) -> int:
+    """Get file count in a storage path.
+
+    Args:
+        storage_config: Storage configuration dict.
+        path_root: Root path to count files in.
+
+    Returns:
+        Number of files.
+
+    Example:
+        >>> config = {'provider': 'local', 'configuration': {'location': '/data'}}
+        >>> count = get_path_file_count(config, '/uploads')
+        >>> print(f'Found {count} files')
+    """
+    storage = get_storage(storage_config)
+    pathlib_obj = storage.get_pathlib(path_root)
+    return storage.get_path_file_count(pathlib_obj)
+
+
+def get_path_total_size(storage_config: dict[str, Any], path_root: str) -> int:
+    """Get total size of files in a storage path.
+
+    Args:
+        storage_config: Storage configuration dict.
+        path_root: Root path to calculate size for.
+
+    Returns:
+        Total size in bytes.
+
+    Example:
+        >>> config = {'provider': 'local', 'configuration': {'location': '/data'}}
+        >>> size = get_path_total_size(config, '/uploads')
+        >>> print(f'Total size: {size / 1024 / 1024:.2f} MB')
+    """
+    storage = get_storage(storage_config)
+    pathlib_obj = storage.get_pathlib(path_root)
+    return storage.get_path_total_size(pathlib_obj)
+
+
+__all__ = [
+    # Protocol
+    'StorageProtocol',
+    # Public API functions
+    'get_storage',
+    'get_pathlib',
+    'get_path_file_count',
+    'get_path_total_size',
+]

synapse_sdk/utils/storage/config.py

@@ -0,0 +1,188 @@
+"""Storage configuration models using Pydantic v2."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+class LocalStorageConfig(BaseModel):
+    """Configuration for local filesystem storage.
+
+    Attributes:
+        location: Base directory path for storage operations.
+    """
+
+    location: str = Field(..., description='Base directory path')
+
+    @field_validator('location')
+    @classmethod
+    def validate_location(cls, v: str) -> str:
+        if not v:
+            raise ValueError('location cannot be empty')
+        return v
+
+
+class S3StorageConfig(BaseModel):
+    """Configuration for S3-compatible storage (AWS S3, MinIO).
+
+    Attributes:
+        bucket_name: S3 bucket name.
+        access_key: AWS access key ID.
+        secret_key: AWS secret access key.
+        region_name: AWS region (default: us-east-1).
+        endpoint_url: Custom endpoint for S3-compatible services (MinIO).
+    """
+
+    bucket_name: str
+    access_key: str
+    secret_key: str
+    region_name: str = 'us-east-1'
+    endpoint_url: str | None = None
+
+
+class GCSStorageConfig(BaseModel):
+    """Configuration for Google Cloud Storage.
+
+    Attributes:
+        bucket_name: GCS bucket name.
+        credentials: Path to service account JSON or credentials dict.
+        project: GCP project ID (optional, inferred from credentials).
+    """
+
+    bucket_name: str
+    credentials: str | dict[str, Any]
+    project: str | None = None
+
+
+class SFTPStorageConfig(BaseModel):
+    """Configuration for SFTP storage.
+
+    Attributes:
+        host: SFTP server hostname.
+        username: SSH username.
+        password: SSH password (for password auth).
+        private_key: Path to private key file (for key auth).
+        private_key_passphrase: Passphrase for encrypted private key.
+        port: SSH port (default: 22).
+        root_path: Base path on remote server.
+    """
+
+    host: str
+    username: str
+    password: str | None = None
+    private_key: str | None = None
+    private_key_passphrase: str | None = None
+    port: int = 22
+    root_path: str = '/'
+
+    @model_validator(mode='after')
+    def validate_auth(self) -> SFTPStorageConfig:
+        if not self.password and not self.private_key:
+            raise ValueError('Either password or private_key must be provided')
+        return self
+
+
+class HTTPStorageConfig(BaseModel):
+    """Configuration for HTTP storage.
+
+    Attributes:
+        base_url: Base URL of the HTTP file server.
+        timeout: Request timeout in seconds.
+        headers: Optional headers to include in requests.
+    """
+
+    base_url: str
+    timeout: int = 30
+    headers: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator('base_url')
+    @classmethod
+    def validate_base_url(cls, v: str) -> str:
+        if not v.startswith(('http://', 'https://')):
+            raise ValueError('base_url must start with http:// or https://')
+        # Ensure trailing slash
+        return v if v.endswith('/') else f'{v}/'
+
+
+# Type alias for all provider configurations
+ProviderConfig = LocalStorageConfig | S3StorageConfig | GCSStorageConfig | SFTPStorageConfig | HTTPStorageConfig
+
+# Provider type literals
+ProviderType = Literal[
+    'local',
+    'file_system',  # alias for local (backward compat)
+    's3',
+    'amazon_s3',  # alias for s3
+    'minio',  # alias for s3
+    'gcs',
+    'gs',  # alias for gcs
+    'gcp',  # alias for gcs
+    'sftp',
+    'http',
+    'https',  # alias for http
+]
+
+
+class StorageConfig(BaseModel):
+    """Top-level storage configuration model.
+
+    Attributes:
+        provider: Storage provider type.
+        configuration: Provider-specific configuration.
+
+    Example:
+        >>> config = StorageConfig(
+        ...     provider='s3',
+        ...     configuration={
+        ...         'bucket_name': 'my-bucket',
+        ...         'access_key': 'AKIAIOSFODNN7EXAMPLE',
+        ...         'secret_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+        ...     }
+        ... )
+    """
+
+    provider: ProviderType
+    configuration: dict[str, Any]
+
+    def get_typed_config(self) -> ProviderConfig:
+        """Get configuration as the appropriate typed model.
+
+        Returns:
+            Typed configuration model based on provider.
+
+        Raises:
+            ValueError: If provider is unknown.
+        """
+        config_map: dict[str, type[BaseModel]] = {
+            'local': LocalStorageConfig,
+            'file_system': LocalStorageConfig,
+            's3': S3StorageConfig,
+            'amazon_s3': S3StorageConfig,
+            'minio': S3StorageConfig,
+            'gcs': GCSStorageConfig,
+            'gs': GCSStorageConfig,
+            'gcp': GCSStorageConfig,
+            'sftp': SFTPStorageConfig,
+            'http': HTTPStorageConfig,
+            'https': HTTPStorageConfig,
+        }
+
+        config_cls = config_map.get(self.provider)
+        if not config_cls:
+            raise ValueError(f'Unknown provider: {self.provider}')
+
+        return config_cls.model_validate(self.configuration)
+
+
+__all__ = [
+    'StorageConfig',
+    'LocalStorageConfig',
+    'S3StorageConfig',
+    'GCSStorageConfig',
+    'SFTPStorageConfig',
+    'HTTPStorageConfig',
+    'ProviderConfig',
+    'ProviderType',
+]

synapse_sdk/utils/storage/errors.py

@@ -0,0 +1,52 @@
+"""Storage-specific exceptions."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class StorageError(Exception):
+    """Base exception for storage-related errors."""
+
+    def __init__(self, message: str, details: Any = None):
+        self.message = message
+        self.details = details
+        super().__init__(message)
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}(message={self.message!r}, details={self.details!r})'
+
+
+class StorageConfigError(StorageError):
+    """Raised when storage configuration is invalid."""
+
+
+class StorageProviderNotFoundError(StorageError):
+    """Raised when requested provider is not registered."""
+
+
+class StorageConnectionError(StorageError):
+    """Raised when connection to storage fails."""
+
+
+class StorageUploadError(StorageError):
+    """Raised when file upload fails."""
+
+
+class StorageNotFoundError(StorageError):
+    """Raised when requested path does not exist."""
+
+
+class StoragePermissionError(StorageError):
+    """Raised when access to storage is denied."""
+
+
+__all__ = [
+    'StorageError',
+    'StorageConfigError',
+    'StorageProviderNotFoundError',
+    'StorageConnectionError',
+    'StorageUploadError',
+    'StorageNotFoundError',
+    'StoragePermissionError',
+]

synapse_sdk/utils/storage/providers/__init__.py

@@ -0,0 +1,13 @@
+"""Storage providers module.
+
+This module exports all built-in storage providers. Import specific
+providers directly for better control over dependencies.
+
+Note: Providers are imported lazily via registry to avoid
+loading optional dependencies (boto3, gcsfs, paramiko) at import time.
+Direct imports are available for explicit usage.
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

synapse_sdk/utils/storage/providers/base.py

@@ -0,0 +1,76 @@
+"""Shared implementation for storage providers."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from upath import UPath
+
+
+class _BaseStorageMixin:
+    """Shared implementation for storage providers.
+
+    This mixin provides common implementations for file counting
+    and size calculation that work with both Path and UPath objects.
+    """
+
+    def _count_files(self, pathlib_obj: Path | UPath) -> int:
+        """Count files recursively in the given path.
+
+        Args:
+            pathlib_obj: Path object to count files in.
+
+        Returns:
+            Number of files.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return 1
+
+        count = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                count += 1
+        return count
+
+    def _calculate_total_size(self, pathlib_obj: Path | UPath) -> int:
+        """Calculate total size of files recursively.
+
+        Args:
+            pathlib_obj: Path object to calculate size for.
+
+        Returns:
+            Total size in bytes.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return pathlib_obj.stat().st_size
+
+        total_size = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                total_size += item.stat().st_size
+        return total_size
+
+    @staticmethod
+    def _normalize_path(path: str) -> str:
+        """Normalize path by removing leading slashes.
+
+        Args:
+            path: Path string to normalize.
+
+        Returns:
+            Normalized path without leading slashes.
+        """
+        if path in ('/', ''):
+            return ''
+        return path.lstrip('/')
+
+
+__all__ = ['_BaseStorageMixin']