msaas-resumable-upload 0.1.0__tar.gz

msaas_resumable_upload-0.1.0/.gitignore

@@ -0,0 +1,23 @@
+node_modules/
+dist/
+.next/
+.turbo/
+*.pyc
+__pycache__/
+.venv/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.env
+.env.*
+!.env.example
+!.env.*.example
+!.env.*.template
+.DS_Store
+coverage/
+
+# Runtime artifacts
+logs_llm/
+vectors.db
+vectors.db-shm
+vectors.db-wal

msaas_resumable_upload-0.1.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: msaas-resumable-upload
+Version: 0.1.0
+Summary: Chunked resumable upload sessions with S3 multipart and progress tracking
+Requires-Python: >=3.12
+Requires-Dist: msaas-api-core
+Requires-Dist: msaas-errors
+Requires-Dist: pydantic>=2.0
+Provides-Extra: all
+Requires-Dist: boto3>=1.34; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: s3
+Requires-Dist: boto3>=1.34; extra == 's3'

msaas_resumable_upload-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "msaas-resumable-upload"
+version = "0.1.0"
+description = "Chunked resumable upload sessions with S3 multipart and progress tracking"
+requires-python = ">=3.12"
+dependencies = [
+    "msaas-api-core",
+    "msaas-errors",
+    "pydantic>=2.0",
+]
+
+[project.optional-dependencies]
+s3 = ["boto3>=1.34"]
+all = ["boto3>=1.34"]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24",
+    "httpx>=0.27.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/resumable_upload"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+
+[tool.ruff]
+target-version = "py312"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP", "B", "SIM", "TCH"]
+
+[tool.uv.sources]
+msaas-api-core = { workspace = true }
+msaas-errors = { workspace = true }

msaas_resumable_upload-0.1.0/src/resumable_upload/__init__.py

@@ -0,0 +1,30 @@
+"""Willian Resumable Upload -- Chunked resumable upload sessions with S3 multipart support."""
+
+from resumable_upload.config import ResumableUploadConfig, get_config, init_resumable_upload
+from resumable_upload.models import (
+    ChunkInfo,
+    ChunkStatus,
+    UploadSession,
+    UploadSessionCreate,
+    UploadSessionStatus,
+    UploadSessionSummary,
+)
+from resumable_upload.router import router
+from resumable_upload.service import ResumableUploadService
+from resumable_upload.storage import S3MultipartStorage, StorageBackend
+
+__all__ = [
+    "ChunkInfo",
+    "ChunkStatus",
+    "ResumableUploadConfig",
+    "ResumableUploadService",
+    "S3MultipartStorage",
+    "StorageBackend",
+    "UploadSession",
+    "UploadSessionCreate",
+    "UploadSessionStatus",
+    "UploadSessionSummary",
+    "get_config",
+    "init_resumable_upload",
+    "router",
+]

msaas_resumable_upload-0.1.0/src/resumable_upload/config.py

@@ -0,0 +1,31 @@
+"""Resumable upload configuration."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+
+class ResumableUploadConfig(BaseModel):
+    min_chunk_size: int = Field(
+        default=5 * 1024 * 1024, description="Min chunk size in bytes (5MB)"
+    )
+    max_chunk_size: int = Field(default=100 * 1024 * 1024, description="Max chunk size (100MB)")
+    max_file_size: int = Field(default=5 * 1024 * 1024 * 1024, description="Max file size (5GB)")
+    session_ttl_hours: int = Field(default=24)
+    max_concurrent_sessions: int = Field(default=10)
+
+
+_config: ResumableUploadConfig | None = None
+
+
+def init_resumable_upload(config: ResumableUploadConfig | None = None) -> ResumableUploadConfig:
+    global _config
+    _config = config or ResumableUploadConfig()
+    return _config
+
+
+def get_config() -> ResumableUploadConfig:
+    global _config
+    if _config is None:
+        _config = ResumableUploadConfig()
+    return _config

msaas_resumable_upload-0.1.0/src/resumable_upload/models.py

@@ -0,0 +1,89 @@
+"""Resumable upload data models."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+
+def _utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+from enum import Enum
+from uuid import uuid4
+
+from pydantic import BaseModel, Field
+
+
+class UploadSessionStatus(str, Enum):
+    INITIALIZED = "initialized"
+    IN_PROGRESS = "in_progress"
+    COMPLETING = "completing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    EXPIRED = "expired"
+
+
+class ChunkStatus(str, Enum):
+    PENDING = "pending"
+    UPLOADING = "uploading"
+    UPLOADED = "uploaded"
+    FAILED = "failed"
+
+
+class ChunkInfo(BaseModel):
+    chunk_number: int = Field(ge=1)
+    size: int = Field(ge=0)
+    offset: int = Field(ge=0)
+    status: ChunkStatus = ChunkStatus.PENDING
+    etag: str | None = None
+    checksum: str | None = None
+    uploaded_at: datetime | None = None
+
+
+class UploadSessionCreate(BaseModel):
+    filename: str
+    file_size: int = Field(ge=1)
+    content_type: str = "application/octet-stream"
+    chunk_size: int | None = None
+    metadata: dict[str, str] = Field(default_factory=dict)
+    storage_path: str | None = None
+
+
+class UploadSession(BaseModel):
+    id: str = Field(default_factory=lambda: str(uuid4()))
+    filename: str
+    file_size: int
+    content_type: str = "application/octet-stream"
+    chunk_size: int = 5 * 1024 * 1024
+    total_chunks: int = 0
+    uploaded_chunks: int = 0
+    status: UploadSessionStatus = UploadSessionStatus.INITIALIZED
+    storage_path: str = ""
+    multipart_upload_id: str | None = None
+    chunks: list[ChunkInfo] = Field(default_factory=list)
+    metadata: dict[str, str] = Field(default_factory=dict)
+    created_at: datetime = Field(default_factory=_utcnow)
+    updated_at: datetime = Field(default_factory=_utcnow)
+    expires_at: datetime | None = None
+
+    @property
+    def progress(self) -> float:
+        if self.total_chunks == 0:
+            return 0.0
+        return round(self.uploaded_chunks / self.total_chunks * 100, 2)
+
+    @property
+    def is_complete(self) -> bool:
+        return self.uploaded_chunks == self.total_chunks and self.total_chunks > 0
+
+
+class UploadSessionSummary(BaseModel):
+    id: str
+    filename: str
+    file_size: int
+    status: UploadSessionStatus
+    progress: float
+    uploaded_chunks: int
+    total_chunks: int
+    created_at: datetime

msaas_resumable_upload-0.1.0/src/resumable_upload/router.py

@@ -0,0 +1,7 @@
+"""Resumable upload FastAPI router."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+router = APIRouter(prefix="/uploads", tags=["resumable-upload"])

msaas_resumable_upload-0.1.0/src/resumable_upload/service.py

@@ -0,0 +1,186 @@
+"""Resumable upload service."""
+
+from __future__ import annotations
+
+import math
+from datetime import UTC, datetime, timedelta
+
+from resumable_upload.config import get_config
+from resumable_upload.models import (
+    ChunkInfo,
+    ChunkStatus,
+    UploadSession,
+    UploadSessionCreate,
+    UploadSessionStatus,
+    UploadSessionSummary,
+)
+from resumable_upload.storage import StorageBackend
+
+
+class ResumableUploadService:
+    """Manages resumable upload sessions with chunked uploads."""
+
+    def __init__(self, storage: StorageBackend) -> None:
+        self._storage = storage
+        self._sessions: dict[str, UploadSession] = {}
+
+    async def create_session(self, params: UploadSessionCreate) -> UploadSession:
+        config = get_config()
+
+        if params.file_size > config.max_file_size:
+            from errors import ValidationError
+
+            raise ValidationError(
+                message=f"File size exceeds maximum ({config.max_file_size} bytes)",
+                field="file_size",
+            )
+
+        chunk_size = params.chunk_size or config.min_chunk_size
+        chunk_size = max(config.min_chunk_size, min(chunk_size, config.max_chunk_size))
+        total_chunks = math.ceil(params.file_size / chunk_size)
+
+        storage_path = params.storage_path or f"uploads/{params.filename}"
+        upload_id = await self._storage.initiate_multipart(storage_path, params.content_type)
+
+        chunks = [
+            ChunkInfo(
+                chunk_number=i + 1,
+                size=min(chunk_size, params.file_size - i * chunk_size),
+                offset=i * chunk_size,
+            )
+            for i in range(total_chunks)
+        ]
+
+        session = UploadSession(
+            filename=params.filename,
+            file_size=params.file_size,
+            content_type=params.content_type,
+            chunk_size=chunk_size,
+            total_chunks=total_chunks,
+            storage_path=storage_path,
+            multipart_upload_id=upload_id,
+            chunks=chunks,
+            metadata=params.metadata,
+            expires_at=datetime.now(UTC) + timedelta(hours=config.session_ttl_hours),
+        )
+
+        self._sessions[session.id] = session
+        return session
+
+    async def upload_chunk(self, session_id: str, chunk_number: int, data: bytes) -> ChunkInfo:
+        session = self._get_session(session_id)
+
+        if session.status in (UploadSessionStatus.COMPLETED, UploadSessionStatus.EXPIRED):
+            from errors import BusinessLogicError
+
+            raise BusinessLogicError(message=f"Session is {session.status.value}")
+
+        if chunk_number < 1 or chunk_number > session.total_chunks:
+            from errors import ValidationError
+
+            raise ValidationError(
+                message=f"Invalid chunk number {chunk_number} (1-{session.total_chunks})",
+                field="chunk_number",
+            )
+
+        chunk = session.chunks[chunk_number - 1]
+        chunk.status = ChunkStatus.UPLOADING
+        session.status = UploadSessionStatus.IN_PROGRESS
+
+        etag = await self._storage.upload_part(
+            session.storage_path,
+            session.multipart_upload_id,
+            chunk_number,
+            data,
+        )
+
+        chunk.etag = etag
+        chunk.status = ChunkStatus.UPLOADED
+        chunk.uploaded_at = datetime.now(UTC)
+        session.uploaded_chunks = sum(1 for c in session.chunks if c.status == ChunkStatus.UPLOADED)
+        session.updated_at = datetime.now(UTC)
+
+        return chunk
+
+    async def complete_session(self, session_id: str) -> UploadSession:
+        session = self._get_session(session_id)
+
+        if not session.is_complete:
+            from errors import BusinessLogicError
+
+            missing = session.total_chunks - session.uploaded_chunks
+            raise BusinessLogicError(message=f"Cannot complete: {missing} chunks still pending")
+
+        session.status = UploadSessionStatus.COMPLETING
+
+        parts = [
+            {"part_number": c.chunk_number, "etag": c.etag}
+            for c in session.chunks
+            if c.etag is not None
+        ]
+
+        await self._storage.complete_multipart(
+            session.storage_path, session.multipart_upload_id, parts
+        )
+
+        session.status = UploadSessionStatus.COMPLETED
+        session.updated_at = datetime.now(UTC)
+        return session
+
+    async def abort_session(self, session_id: str) -> None:
+        session = self._get_session(session_id)
+
+        if session.multipart_upload_id:
+            await self._storage.abort_multipart(session.storage_path, session.multipart_upload_id)
+
+        session.status = UploadSessionStatus.FAILED
+        session.updated_at = datetime.now(UTC)
+
+    def get_session(self, session_id: str) -> UploadSession:
+        return self._get_session(session_id)
+
+    def get_session_summary(self, session_id: str) -> UploadSessionSummary:
+        session = self._get_session(session_id)
+        return UploadSessionSummary(
+            id=session.id,
+            filename=session.filename,
+            file_size=session.file_size,
+            status=session.status,
+            progress=session.progress,
+            uploaded_chunks=session.uploaded_chunks,
+            total_chunks=session.total_chunks,
+            created_at=session.created_at,
+        )
+
+    def get_pending_chunks(self, session_id: str) -> list[ChunkInfo]:
+        session = self._get_session(session_id)
+        return [c for c in session.chunks if c.status == ChunkStatus.PENDING]
+
+    async def cleanup_expired(self) -> int:
+        now = datetime.now(UTC)
+        expired_ids = [
+            sid
+            for sid, s in self._sessions.items()
+            if s.expires_at
+            and s.expires_at < now
+            and s.status not in (UploadSessionStatus.COMPLETED, UploadSessionStatus.EXPIRED)
+        ]
+        for sid in expired_ids:
+            session = self._sessions[sid]
+            if session.multipart_upload_id:
+                try:
+                    await self._storage.abort_multipart(
+                        session.storage_path, session.multipart_upload_id
+                    )
+                except Exception:
+                    pass
+            session.status = UploadSessionStatus.EXPIRED
+        return len(expired_ids)
+
+    def _get_session(self, session_id: str) -> UploadSession:
+        session = self._sessions.get(session_id)
+        if session is None:
+            from errors import NotFoundError
+
+            raise NotFoundError(message=f"Upload session {session_id} not found")
+        return session

msaas_resumable_upload-0.1.0/src/resumable_upload/storage.py

@@ -0,0 +1,107 @@
+"""Storage backends for resumable uploads."""
+
+from __future__ import annotations
+
+import hashlib
+from abc import ABC, abstractmethod
+
+
+class StorageBackend(ABC):
+    """Abstract storage backend for multipart uploads."""
+
+    @abstractmethod
+    async def initiate_multipart(self, key: str, content_type: str) -> str:
+        """Start multipart upload, return upload ID."""
+
+    @abstractmethod
+    async def upload_part(self, key: str, upload_id: str, part_number: int, data: bytes) -> str:
+        """Upload a part, return ETag."""
+
+    @abstractmethod
+    async def complete_multipart(
+        self, key: str, upload_id: str, parts: list[dict[str, str | int]]
+    ) -> str:
+        """Complete multipart upload, return final key/URL."""
+
+    @abstractmethod
+    async def abort_multipart(self, key: str, upload_id: str) -> None:
+        """Abort multipart upload."""
+
+
+class S3MultipartStorage(StorageBackend):
+    """AWS S3 multipart upload backend."""
+
+    def __init__(self, bucket: str, region: str = "us-east-1", endpoint_url: str | None = None):
+        try:
+            import boto3
+        except ImportError:
+            raise ImportError(
+                "boto3 is required for S3 storage. "
+                "Install with: pip install msaas-resumable-upload[s3]"
+            )
+        self.bucket = bucket
+        self._client = boto3.client("s3", region_name=region, endpoint_url=endpoint_url)
+
+    async def initiate_multipart(self, key: str, content_type: str) -> str:
+        response = self._client.create_multipart_upload(
+            Bucket=self.bucket, Key=key, ContentType=content_type
+        )
+        return response["UploadId"]
+
+    async def upload_part(self, key: str, upload_id: str, part_number: int, data: bytes) -> str:
+        response = self._client.upload_part(
+            Bucket=self.bucket,
+            Key=key,
+            UploadId=upload_id,
+            PartNumber=part_number,
+            Body=data,
+        )
+        return response["ETag"]
+
+    async def complete_multipart(
+        self, key: str, upload_id: str, parts: list[dict[str, str | int]]
+    ) -> str:
+        self._client.complete_multipart_upload(
+            Bucket=self.bucket,
+            Key=key,
+            UploadId=upload_id,
+            MultipartUpload={
+                "Parts": [{"PartNumber": p["part_number"], "ETag": p["etag"]} for p in parts]
+            },
+        )
+        return f"s3://{self.bucket}/{key}"
+
+    async def abort_multipart(self, key: str, upload_id: str) -> None:
+        self._client.abort_multipart_upload(Bucket=self.bucket, Key=key, UploadId=upload_id)
+
+
+class InMemoryStorage(StorageBackend):
+    """In-memory storage backend for testing."""
+
+    def __init__(self) -> None:
+        self._uploads: dict[str, dict[int, bytes]] = {}
+        self._completed: dict[str, bytes] = {}
+        self._counter = 0
+
+    async def initiate_multipart(self, key: str, content_type: str) -> str:
+        self._counter += 1
+        upload_id = f"mem-upload-{self._counter}"
+        self._uploads[upload_id] = {}
+        return upload_id
+
+    async def upload_part(self, key: str, upload_id: str, part_number: int, data: bytes) -> str:
+        self._uploads[upload_id][part_number] = data
+        return hashlib.md5(data).hexdigest()
+
+    async def complete_multipart(
+        self, key: str, upload_id: str, parts: list[dict[str, str | int]]
+    ) -> str:
+        upload_parts = self._uploads.pop(upload_id, {})
+        combined = b"".join(
+            upload_parts[p["part_number"]] for p in sorted(parts, key=lambda x: x["part_number"])
+        )
+        self._completed[key] = combined
+        return f"mem://{key}"
+
+    async def abort_multipart(self, key: str, upload_id: str) -> None:
+        self._uploads.pop(upload_id, None)

msaas_resumable_upload-0.1.0/tests/test_service.py

@@ -0,0 +1,104 @@
+"""Tests for resumable upload service."""
+
+from __future__ import annotations
+
+import pytest
+
+from resumable_upload.models import (
+    ChunkStatus,
+    UploadSessionCreate,
+    UploadSessionStatus,
+)
+from resumable_upload.service import ResumableUploadService
+from resumable_upload.storage import InMemoryStorage
+
+
+@pytest.fixture
+def service() -> ResumableUploadService:
+    return ResumableUploadService(storage=InMemoryStorage())
+
+
+@pytest.fixture
+def create_params() -> UploadSessionCreate:
+    return UploadSessionCreate(
+        filename="test.pdf",
+        file_size=15 * 1024 * 1024,
+        content_type="application/pdf",
+        chunk_size=5 * 1024 * 1024,
+    )
+
+
+class TestResumableUploadService:
+    @pytest.mark.asyncio
+    async def test_create_session(
+        self, service: ResumableUploadService, create_params: UploadSessionCreate
+    ) -> None:
+        session = await service.create_session(create_params)
+        assert session.status == UploadSessionStatus.INITIALIZED
+        assert session.total_chunks == 3
+        assert session.uploaded_chunks == 0
+        assert session.progress == 0.0
+        assert len(session.chunks) == 3
+
+    @pytest.mark.asyncio
+    async def test_upload_chunk(
+        self, service: ResumableUploadService, create_params: UploadSessionCreate
+    ) -> None:
+        session = await service.create_session(create_params)
+        chunk_data = b"x" * (5 * 1024 * 1024)
+        chunk = await service.upload_chunk(session.id, 1, chunk_data)
+        assert chunk.status == ChunkStatus.UPLOADED
+        assert chunk.etag is not None
+
+        updated = service.get_session(session.id)
+        assert updated.uploaded_chunks == 1
+        assert updated.status == UploadSessionStatus.IN_PROGRESS
+
+    @pytest.mark.asyncio
+    async def test_complete_session(
+        self, service: ResumableUploadService, create_params: UploadSessionCreate
+    ) -> None:
+        session = await service.create_session(create_params)
+        for i in range(session.total_chunks):
+            size = session.chunks[i].size
+            await service.upload_chunk(session.id, i + 1, b"x" * size)
+
+        completed = await service.complete_session(session.id)
+        assert completed.status == UploadSessionStatus.COMPLETED
+        assert completed.progress == 100.0
+
+    @pytest.mark.asyncio
+    async def test_abort_session(
+        self, service: ResumableUploadService, create_params: UploadSessionCreate
+    ) -> None:
+        session = await service.create_session(create_params)
+        await service.abort_session(session.id)
+        aborted = service.get_session(session.id)
+        assert aborted.status == UploadSessionStatus.FAILED
+
+    @pytest.mark.asyncio
+    async def test_get_pending_chunks(
+        self, service: ResumableUploadService, create_params: UploadSessionCreate
+    ) -> None:
+        session = await service.create_session(create_params)
+        await service.upload_chunk(session.id, 1, b"x" * (5 * 1024 * 1024))
+        pending = service.get_pending_chunks(session.id)
+        assert len(pending) == 2
+
+    @pytest.mark.asyncio
+    async def test_session_summary(
+        self, service: ResumableUploadService, create_params: UploadSessionCreate
+    ) -> None:
+        session = await service.create_session(create_params)
+        summary = service.get_session_summary(session.id)
+        assert summary.filename == "test.pdf"
+        assert summary.progress == 0.0
+        assert summary.total_chunks == 3
+
+    @pytest.mark.asyncio
+    async def test_cannot_complete_incomplete(
+        self, service: ResumableUploadService, create_params: UploadSessionCreate
+    ) -> None:
+        session = await service.create_session(create_params)
+        with pytest.raises(Exception, match="chunks still pending"):
+            await service.complete_session(session.id)