svc-infra 0.1.595__py3-none-any.whl → 0.1.706__py3-none-any.whl

svc_infra/storage/backends/memory.py

@@ -0,0 +1,216 @@
+"""
+In-memory storage backend for testing and development.
+
+WARNING: Data is not persisted across restarts. Use only for testing or development.
+"""
+
+import asyncio
+from datetime import datetime, timezone
+from typing import Optional
+
+from ..base import FileNotFoundError, InvalidKeyError, QuotaExceededError
+
+
+class MemoryBackend:
+    """
+    In-memory storage backend.
+
+    Stores files in memory using dictionaries. Fast and simple for testing,
+    but data is lost when the process restarts.
+
+    Args:
+        max_size: Maximum total storage size in bytes (default: 100MB)
+        default_expires_in: Default URL expiration in seconds (default: 3600)
+
+    Example:
+        >>> backend = MemoryBackend(max_size=10_000_000)  # 10MB max
+        >>> url = await backend.put(
+        ...     key="test/file.txt",
+        ...     data=b"Hello, World!",
+        ...     content_type="text/plain"
+        ... )
+    """
+
+    def __init__(
+        self,
+        max_size: int = 100_000_000,  # 100MB
+        default_expires_in: int = 3600,
+    ):
+        self.max_size = max_size
+        self.default_expires_in = default_expires_in
+        self._storage: dict[str, bytes] = {}
+        self._metadata: dict[str, dict] = {}
+        self._lock = asyncio.Lock()
+
+    def _validate_key(self, key: str) -> None:
+        """Validate storage key format."""
+        if not key:
+            raise InvalidKeyError("Key cannot be empty")
+
+        if key.startswith("/"):
+            raise InvalidKeyError("Key cannot start with /")
+
+        if ".." in key:
+            raise InvalidKeyError("Key cannot contain .. (path traversal)")
+
+        if len(key) > 1024:
+            raise InvalidKeyError("Key cannot exceed 1024 characters")
+
+        # Check for safe characters
+        safe_chars = set(
+            "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/"
+        )
+        if not all(c in safe_chars for c in key):
+            raise InvalidKeyError(
+                "Key can only contain alphanumeric, dot, dash, underscore, and slash"
+            )
+
+    def _get_total_size(self) -> int:
+        """Calculate total storage size."""
+        return sum(len(data) for data in self._storage.values())
+
+    async def put(
+        self,
+        key: str,
+        data: bytes,
+        content_type: str,
+        metadata: Optional[dict] = None,
+    ) -> str:
+        """Store file in memory."""
+        self._validate_key(key)
+
+        async with self._lock:
+            # Check quota
+            current_size = self._get_total_size()
+            new_size = len(data)
+
+            # If replacing existing file, subtract its size
+            if key in self._storage:
+                current_size -= len(self._storage[key])
+
+            if current_size + new_size > self.max_size:
+                raise QuotaExceededError(
+                    f"Storage quota exceeded. "
+                    f"Current: {current_size}, New: {new_size}, Max: {self.max_size}"
+                )
+
+            # Store file
+            self._storage[key] = data
+
+            # Store metadata
+            self._metadata[key] = {
+                "size": len(data),
+                "content_type": content_type,
+                "created_at": datetime.now(timezone.utc).isoformat(),
+                **(metadata or {}),
+            }
+
+        # Return memory:// URL
+        return f"memory://{key}"
+
+    async def get(self, key: str) -> bytes:
+        """Retrieve file from memory."""
+        self._validate_key(key)
+
+        async with self._lock:
+            if key not in self._storage:
+                raise FileNotFoundError(f"File not found: {key}")
+
+            return self._storage[key]
+
+    async def delete(self, key: str) -> bool:
+        """Delete file from memory."""
+        self._validate_key(key)
+
+        async with self._lock:
+            if key not in self._storage:
+                return False
+
+            del self._storage[key]
+            del self._metadata[key]
+            return True
+
+    async def exists(self, key: str) -> bool:
+        """Check if file exists in memory."""
+        self._validate_key(key)
+
+        async with self._lock:
+            return key in self._storage
+
+    async def get_url(
+        self,
+        key: str,
+        expires_in: int = 3600,
+        download: bool = False,
+    ) -> str:
+        """
+        Generate memory:// URL.
+
+        Note: Memory backend doesn't support real URLs or expiration.
+        Returns memory:// scheme for testing purposes.
+        """
+        self._validate_key(key)
+
+        async with self._lock:
+            if key not in self._storage:
+                raise FileNotFoundError(f"File not found: {key}")
+
+        # Memory backend doesn't support real URLs
+        # Return memory:// scheme for testing
+        suffix = "?download=true" if download else ""
+        return f"memory://{key}{suffix}"
+
+    async def list_keys(
+        self,
+        prefix: str = "",
+        limit: int = 100,
+    ) -> list[str]:
+        """List stored keys with optional prefix filter."""
+        async with self._lock:
+            keys = [key for key in self._storage.keys() if key.startswith(prefix)]
+            return keys[:limit]
+
+    async def get_metadata(self, key: str) -> dict:
+        """Get file metadata."""
+        self._validate_key(key)
+
+        async with self._lock:
+            if key not in self._metadata:
+                raise FileNotFoundError(f"File not found: {key}")
+
+            return self._metadata[key].copy()
+
+    async def clear(self) -> None:
+        """
+        Clear all stored files (testing utility).
+
+        Example:
+            >>> backend = MemoryBackend()
+            >>> await backend.put("test.txt", b"data", "text/plain")
+            >>> await backend.clear()
+            >>> await backend.exists("test.txt")  # False
+        """
+        async with self._lock:
+            self._storage.clear()
+            self._metadata.clear()
+
+    def get_stats(self) -> dict:
+        """
+        Get storage statistics (testing utility).
+
+        Returns:
+            Dict with file_count, total_size, max_size
+
+        Example:
+            >>> backend = MemoryBackend(max_size=1000)
+            >>> stats = backend.get_stats()
+            >>> print(f"Files: {stats['file_count']}, Size: {stats['total_size']}")
+        """
+        return {
+            "file_count": len(self._storage),
+            "total_size": self._get_total_size(),
+            "max_size": self.max_size,
+        }
+
+
+__all__ = ["MemoryBackend"]

svc_infra/storage/backends/s3.py

@@ -0,0 +1,353 @@
+"""
+S3-compatible storage backend.
+
+Works with AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio, and
+any S3-compatible object storage service.
+"""
+
+from typing import Optional, cast
+
+try:
+    import aioboto3
+    from botocore.exceptions import ClientError, NoCredentialsError
+except ImportError:
+    aioboto3 = None
+    ClientError = Exception
+    NoCredentialsError = Exception
+
+from ..base import (
+    FileNotFoundError,
+    InvalidKeyError,
+    PermissionDeniedError,
+    StorageError,
+)
+
+
+class S3Backend:
+    """
+    S3-compatible storage backend.
+
+    Supports AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio,
+    and any S3-compatible object storage.
+
+    Args:
+        bucket: S3 bucket name
+        region: AWS region (default: "us-east-1")
+        endpoint: Custom endpoint URL for S3-compatible services
+        access_key: AWS access key (uses AWS_ACCESS_KEY_ID env var if not provided)
+        secret_key: AWS secret key (uses AWS_SECRET_ACCESS_KEY env var if not provided)
+
+    Example:
+        >>> # AWS S3
+        >>> backend = S3Backend(
+        ...     bucket="my-uploads",
+        ...     region="us-west-2"
+        ... )
+        >>>
+        >>> # DigitalOcean Spaces
+        >>> backend = S3Backend(
+        ...     bucket="my-uploads",
+        ...     region="nyc3",
+        ...     endpoint="https://nyc3.digitaloceanspaces.com",
+        ...     access_key="...",
+        ...     secret_key="..."
+        ... )
+        >>>
+        >>> # Wasabi
+        >>> backend = S3Backend(
+        ...     bucket="my-uploads",
+        ...     region="us-east-1",
+        ...     endpoint="https://s3.wasabisys.com"
+        ... )
+
+    Raises:
+        ImportError: If aioboto3 is not installed
+    """
+
+    def __init__(
+        self,
+        bucket: str,
+        region: str = "us-east-1",
+        endpoint: Optional[str] = None,
+        access_key: Optional[str] = None,
+        secret_key: Optional[str] = None,
+    ):
+        if aioboto3 is None:
+            raise ImportError(
+                "aioboto3 is required for S3Backend. "
+                "Install it with: pip install aioboto3"
+            )
+
+        self.bucket = bucket
+        self.region = region
+        self.endpoint = endpoint
+        self.access_key = access_key
+        self.secret_key = secret_key
+
+        # Session configuration
+        self._session_config = {
+            "region_name": region,
+        }
+        if endpoint:
+            self._session_config["endpoint_url"] = endpoint
+
+        # Client configuration
+        self._client_config = {}
+        if access_key and secret_key:
+            self._client_config = {
+                "aws_access_key_id": access_key,
+                "aws_secret_access_key": secret_key,
+            }
+
+    def _validate_key(self, key: str) -> None:
+        """Validate storage key format."""
+        if not key:
+            raise InvalidKeyError("Key cannot be empty")
+
+        if key.startswith("/"):
+            raise InvalidKeyError("Key cannot start with /")
+
+        if ".." in key:
+            raise InvalidKeyError("Key cannot contain .. (path traversal)")
+
+        if len(key) > 1024:
+            raise InvalidKeyError("Key cannot exceed 1024 characters")
+
+    async def put(
+        self,
+        key: str,
+        data: bytes,
+        content_type: str,
+        metadata: Optional[dict] = None,
+    ) -> str:
+        """Store file in S3."""
+        self._validate_key(key)
+
+        # Prepare S3 metadata (must be string key-value pairs)
+        s3_metadata = {}
+        if metadata:
+            for k, v in metadata.items():
+                s3_metadata[str(k)] = str(v)
+
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3", **self._session_config, **self._client_config
+            ) as s3:
+                # Upload file
+                await s3.put_object(
+                    Bucket=self.bucket,
+                    Key=key,
+                    Body=data,
+                    ContentType=content_type,
+                    Metadata=s3_metadata,
+                )
+
+        except NoCredentialsError as e:
+            raise PermissionDeniedError(f"S3 credentials not found: {e}")
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "AccessDenied":
+                raise PermissionDeniedError(f"S3 access denied: {e}")
+            elif error_code == "NoSuchBucket":
+                raise StorageError(f"S3 bucket does not exist: {self.bucket}")
+            else:
+                raise StorageError(f"S3 upload failed: {e}")
+        except Exception as e:
+            raise StorageError(f"Failed to upload to S3: {e}")
+
+        # Return presigned URL (1 hour expiration)
+        return await self.get_url(key, expires_in=3600)
+
+    async def get(self, key: str) -> bytes:
+        """Retrieve file from S3."""
+        self._validate_key(key)
+
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3", **self._session_config, **self._client_config
+            ) as s3:
+                response = await s3.get_object(Bucket=self.bucket, Key=key)
+                async with response["Body"] as stream:
+                    return cast(bytes, await stream.read())
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "NoSuchKey":
+                raise FileNotFoundError(f"File not found: {key}")
+            elif error_code == "AccessDenied":
+                raise PermissionDeniedError(f"S3 access denied: {e}")
+            else:
+                raise StorageError(f"S3 download failed: {e}")
+        except Exception as e:
+            raise StorageError(f"Failed to download from S3: {e}")
+
+    async def delete(self, key: str) -> bool:
+        """Delete file from S3."""
+        self._validate_key(key)
+
+        # Check if file exists first
+        if not await self.exists(key):
+            return False
+
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3", **self._session_config, **self._client_config
+            ) as s3:
+                await s3.delete_object(Bucket=self.bucket, Key=key)
+                return True
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "AccessDenied":
+                raise PermissionDeniedError(f"S3 access denied: {e}")
+            else:
+                raise StorageError(f"S3 delete failed: {e}")
+        except Exception as e:
+            raise StorageError(f"Failed to delete from S3: {e}")
+
+    async def exists(self, key: str) -> bool:
+        """Check if file exists in S3."""
+        self._validate_key(key)
+
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3", **self._session_config, **self._client_config
+            ) as s3:
+                await s3.head_object(Bucket=self.bucket, Key=key)
+                return True
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code in ("NoSuchKey", "404"):
+                return False
+            else:
+                raise StorageError(f"S3 head_object failed: {e}")
+        except Exception as e:
+            raise StorageError(f"Failed to check S3 file existence: {e}")
+
+    async def get_url(
+        self,
+        key: str,
+        expires_in: int = 3600,
+        download: bool = False,
+    ) -> str:
+        """
+        Generate presigned URL for file access.
+
+        Args:
+            key: Storage key
+            expires_in: URL expiration in seconds (default: 1 hour)
+            download: If True, force download instead of inline display
+
+        Returns:
+            Presigned S3 URL
+
+        Example:
+            >>> url = await backend.get_url("documents/invoice.pdf", expires_in=300)
+        """
+        self._validate_key(key)
+
+        # Check if file exists
+        if not await self.exists(key):
+            raise FileNotFoundError(f"File not found: {key}")
+
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3", **self._session_config, **self._client_config
+            ) as s3:
+                # Prepare parameters
+                params = {"Bucket": self.bucket, "Key": key}
+
+                # Add Content-Disposition for downloads
+                if download:
+                    # Extract filename from key
+                    filename = key.split("/")[-1]
+                    params["ResponseContentDisposition"] = (
+                        f'attachment; filename="{filename}"'
+                    )
+
+                # Generate presigned URL
+                url = await s3.generate_presigned_url(
+                    "get_object",
+                    Params=params,
+                    ExpiresIn=expires_in,
+                )
+                return cast(str, url)
+
+        except ClientError as e:
+            raise StorageError(f"Failed to generate presigned URL: {e}")
+        except Exception as e:
+            raise StorageError(f"Failed to generate presigned URL: {e}")
+
+    async def list_keys(
+        self,
+        prefix: str = "",
+        limit: int = 100,
+    ) -> list[str]:
+        """List stored keys with optional prefix filter."""
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3", **self._session_config, **self._client_config
+            ) as s3:
+                params = {
+                    "Bucket": self.bucket,
+                    "MaxKeys": limit,
+                }
+                if prefix:
+                    params["Prefix"] = prefix
+
+                response = await s3.list_objects_v2(**params)
+
+                # Extract keys from response
+                contents = response.get("Contents", [])
+                keys = [obj["Key"] for obj in contents]
+                return keys
+
+        except ClientError as e:
+            raise StorageError(f"S3 list failed: {e}")
+        except Exception as e:
+            raise StorageError(f"Failed to list S3 keys: {e}")
+
+    async def get_metadata(self, key: str) -> dict:
+        """Get file metadata from S3."""
+        self._validate_key(key)
+
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3", **self._session_config, **self._client_config
+            ) as s3:
+                response = await s3.head_object(Bucket=self.bucket, Key=key)
+
+                # Extract metadata
+                metadata = {
+                    "size": response["ContentLength"],
+                    "content_type": response.get(
+                        "ContentType", "application/octet-stream"
+                    ),
+                    "created_at": response["LastModified"].isoformat(),
+                }
+
+                # Add custom metadata
+                if "Metadata" in response:
+                    metadata.update(response["Metadata"])
+
+                return metadata
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "NoSuchKey":
+                raise FileNotFoundError(f"File not found: {key}")
+            else:
+                raise StorageError(f"S3 head_object failed: {e}")
+        except Exception as e:
+            raise StorageError(f"Failed to get S3 metadata: {e}")
+
+
+__all__ = ["S3Backend"]