resumable-upload 0.0.1__py3-none-any.whl

resumable_upload/__init__.py

@@ -0,0 +1,29 @@
+"""Resumable Upload Library
+
+A Python implementation of the TUS resumable upload protocol.
+Provides both server and client components with minimal dependencies.
+"""
+
+__version__ = "0.0.1"
+
+from resumable_upload.client import TusClient, TusClientWithRetry, UploadStats
+from resumable_upload.exceptions import TusCommunicationError, TusUploadFailed
+from resumable_upload.fingerprint import Fingerprint
+from resumable_upload.server import TusHTTPRequestHandler, TusServer
+from resumable_upload.storage import SQLiteStorage, Storage
+from resumable_upload.url_storage import FileURLStorage, URLStorage
+
+__all__ = [
+    "TusServer",
+    "TusHTTPRequestHandler",
+    "TusClient",
+    "TusClientWithRetry",
+    "UploadStats",
+    "Storage",
+    "SQLiteStorage",
+    "TusCommunicationError",
+    "TusUploadFailed",
+    "Fingerprint",
+    "URLStorage",
+    "FileURLStorage",
+]

resumable_upload/client/__init__.py

@@ -0,0 +1,7 @@
+"""TUS protocol client implementations."""
+
+from resumable_upload.client.base import TusClient
+from resumable_upload.client.retry import TusClientWithRetry
+from resumable_upload.client.stats import UploadStats
+
+__all__ = ["TusClient", "TusClientWithRetry", "UploadStats"]

resumable_upload/client/base.py

@@ -0,0 +1,379 @@
+"""TUS protocol client implementation."""
+
+import base64
+import hashlib
+import os
+import re
+from typing import IO, Callable, Optional, Union
+from urllib.error import HTTPError
+from urllib.parse import urljoin
+from urllib.request import Request, urlopen
+
+from resumable_upload.exceptions import TusCommunicationError, TusUploadFailed
+from resumable_upload.fingerprint import Fingerprint
+from resumable_upload.url_storage import URLStorage
+
+
+class TusClient:
+    """TUS protocol client for uploading files.
+
+    This client implements TUS protocol version 1.0.0 as specified at:
+    https://tus.io/protocols/resumable-upload.html
+
+    Version Handling:
+        - Uses TUS version 1.0.0
+        - Sends "Tus-Resumable: 1.0.0" header with all requests
+        - Compatible with TUS 1.0.0 compliant servers
+        - Server must support version 1.0.0 to accept uploads
+
+    Features:
+        - File upload with configurable chunk size
+        - Automatic resume of interrupted uploads
+        - Progress tracking via callbacks
+        - Optional SHA1 checksum verification
+        - Metadata support for file information
+
+    Example:
+        >>> client = TusClient("http://localhost:8080/files")
+        >>> url = client.upload_file(
+        ...     "large_file.bin",
+        ...     metadata={"filename": "large_file.bin"},
+        ...     progress_callback=lambda up, total: print(f"{up}/{total}"),
+        ... )
+        >>> # Resume interrupted upload
+        >>> client.resume_upload("large_file.bin", url)
+    """
+
+    TUS_VERSION = "1.0.0"
+
+    def __init__(
+        self,
+        url: str,
+        chunk_size: Union[int, float] = 1024 * 1024,
+        checksum: bool = True,
+        verify_tls_cert: bool = True,
+        metadata_encoding: str = "utf-8",
+        store_url: bool = False,
+        url_storage: Optional[URLStorage] = None,
+        fingerprinter: Optional[Fingerprint] = None,
+    ):
+        """Initialize TUS client.
+
+        Args:
+            url: Base URL of TUS server
+            chunk_size: Size of upload chunks in bytes (default: 1MB). Can be int or float.
+            checksum: Enable checksum verification (default: True)
+            verify_tls_cert: Verify TLS certificates (default: True)
+            metadata_encoding: Encoding for metadata values (default: utf-8)
+            store_url: Store upload URLs for resumability (default: False)
+            url_storage: Custom URL storage implementation
+            fingerprinter: Custom fingerprint implementation
+
+        Raises:
+            ValueError: If chunk_size is less than 1
+        """
+        if chunk_size < 1:
+            raise ValueError(f"chunk_size must be at least 1 byte, got {chunk_size}")
+        self.url = url.rstrip("/")
+        self.chunk_size = int(chunk_size)
+        self.checksum = checksum
+        self.verify_tls_cert = verify_tls_cert
+        self.metadata_encoding = metadata_encoding
+        self.store_url = store_url
+        self.url_storage = url_storage
+        self.fingerprinter = fingerprinter or Fingerprint()
+
+    def upload_file(
+        self,
+        file_path: Optional[str] = None,
+        file_stream: Optional[IO] = None,
+        metadata: Optional[dict[str, str]] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+        stop_at: Optional[int] = None,
+    ) -> str:
+        """Upload a file to the server.
+
+        Args:
+            file_path: Path to file to upload (required if file_stream not provided)
+            file_stream: File stream to upload (alternative to file_path)
+            metadata: Optional metadata dictionary
+            progress_callback: Optional callback function(uploaded_bytes, total_bytes)
+            stop_at: Stop upload at this byte offset (for partial uploads)
+
+        Returns:
+            URL of the uploaded file
+
+        Raises:
+            ValueError: If neither file_path nor file_stream provided
+            FileNotFoundError: If file doesn't exist
+            TusCommunicationError: If upload fails
+        """
+        if not file_path and not file_stream:
+            raise ValueError("Either file_path or file_stream must be provided")
+
+        if file_path and not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        # Get file size
+        if file_stream:
+            file_stream.seek(0, os.SEEK_END)
+            file_size = file_stream.tell()
+            file_stream.seek(0)
+        else:
+            file_size = os.path.getsize(file_path)
+
+        metadata = metadata or {}
+
+        # Add filename to metadata if not present and we have a file_path
+        if "filename" not in metadata and file_path:
+            metadata["filename"] = os.path.basename(file_path)
+
+        # Check for stored URL if enabled
+        upload_url = None
+        if self.store_url and self.url_storage:
+            fingerprint = self.fingerprinter.get_fingerprint(file_path or file_stream)
+            upload_url = self.url_storage.get_url(fingerprint)
+
+        # Create upload if no stored URL
+        if not upload_url:
+            upload_url = self._create_upload(file_size, metadata)
+            if self.store_url and self.url_storage:
+                fingerprint = self.fingerprinter.get_fingerprint(file_path or file_stream)
+                self.url_storage.set_url(fingerprint, upload_url)
+
+        # Upload file in chunks
+        if file_stream:
+            fs = file_stream
+            fs.seek(0)
+        else:
+            fs = open(file_path, "rb")  # noqa: SIM115
+
+        try:
+            offset = self._get_offset(upload_url)
+            fs.seek(offset)
+
+            max_offset = stop_at if stop_at is not None else file_size
+
+            while offset < max_offset:
+                chunk_size = min(self.chunk_size, max_offset - offset)
+                chunk = fs.read(chunk_size)
+                if not chunk:
+                    break
+
+                self._upload_chunk(upload_url, offset, chunk)
+                offset += len(chunk)
+
+                if progress_callback:
+                    progress_callback(offset, file_size)
+        finally:
+            if not file_stream and file_path:
+                fs.close()
+
+        return upload_url
+
+    def resume_upload(
+        self,
+        file_path: str,
+        upload_url: str,
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+    ) -> str:
+        """Resume an interrupted upload.
+
+        Args:
+            file_path: Path to file to upload
+            upload_url: URL of the existing upload
+            progress_callback: Optional callback function(uploaded_bytes, total_bytes)
+
+        Returns:
+            URL of the uploaded file
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            HTTPError: If upload fails
+        """
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        file_size = os.path.getsize(file_path)
+
+        # Get current offset
+        offset = self._get_offset(upload_url)
+
+        # Resume upload from current offset
+        with open(file_path, "rb") as f:
+            f.seek(offset)
+            while offset < file_size:
+                chunk = f.read(self.chunk_size)
+                if not chunk:
+                    break
+
+                self._upload_chunk(upload_url, offset, chunk)
+                offset += len(chunk)
+
+                if progress_callback:
+                    progress_callback(offset, file_size)
+
+        return upload_url
+
+    def delete_upload(self, upload_url: str) -> None:
+        """Delete an upload from the server.
+
+        Args:
+            upload_url: URL of the upload to delete
+
+        Raises:
+            HTTPError: If deletion fails
+        """
+        headers = {
+            "Tus-Resumable": self.TUS_VERSION,
+        }
+
+        req = Request(upload_url, headers=headers, method="DELETE")
+        try:
+            with urlopen(req):
+                pass
+        except HTTPError as e:
+            if e.code != 404:
+                raise
+
+    def _create_upload(self, file_size: int, metadata: dict[str, str]) -> str:
+        """Create a new upload on the server."""
+        # Encode metadata
+        encoded_metadata = self.encode_metadata(metadata)
+
+        headers = {
+            "Tus-Resumable": self.TUS_VERSION,
+            "Upload-Length": str(file_size),
+        }
+
+        if encoded_metadata:
+            headers["Upload-Metadata"] = ",".join(encoded_metadata)
+
+        try:
+            req = Request(self.url, headers=headers, method="POST")
+            with urlopen(req) as response:
+                location = response.headers.get("Location")
+                if not location:
+                    raise TusCommunicationError("Server did not return Location header")
+
+                # Handle relative URLs
+                if not location.startswith("http"):
+                    location = urljoin(self.url, location)
+
+                return location
+        except HTTPError as e:
+            raise TusCommunicationError(
+                f"Failed to create upload: {e.reason}",
+                status_code=e.code,
+                response_content=e.read(),
+            ) from e
+
+    def encode_metadata(self, metadata: dict[str, str]) -> list:
+        """
+        Encode metadata according to TUS protocol specification.
+
+        Args:
+            metadata: Dictionary of metadata key-value pairs
+
+        Returns:
+            List of encoded metadata strings
+
+        Raises:
+            ValueError: If metadata keys contain invalid characters
+        """
+        encoded_list = []
+        for key, value in metadata.items():
+            key_str = str(key)
+
+            # Validate key does not contain spaces or commas
+            if re.search(r"^$|[\s,]+", key_str):
+                raise ValueError(
+                    f'Upload-metadata key "{key_str}" cannot be empty nor contain spaces or commas.'
+                )
+
+            value_bytes = value.encode(self.metadata_encoding)
+            encoded_value = base64.b64encode(value_bytes).decode("ascii")
+            encoded_list.append(f"{key_str} {encoded_value}")
+
+        return encoded_list
+
+    def _get_offset(self, upload_url: str) -> int:
+        """Get the current upload offset."""
+        headers = {
+            "Tus-Resumable": self.TUS_VERSION,
+        }
+
+        try:
+            req = Request(upload_url, headers=headers, method="HEAD")
+            with urlopen(req) as response:
+                offset = response.headers.get("Upload-Offset")
+                if offset is None:
+                    raise TusCommunicationError("Server did not return Upload-Offset header")
+                return int(offset)
+        except HTTPError as e:
+            raise TusCommunicationError(
+                f"Failed to get offset: {e.reason}",
+                status_code=e.code,
+                response_content=e.read(),
+            ) from e
+
+    def _upload_chunk(self, upload_url: str, offset: int, data: bytes) -> None:
+        """Upload a chunk of data."""
+        headers = {
+            "Tus-Resumable": self.TUS_VERSION,
+            "Upload-Offset": str(offset),
+            "Content-Type": "application/offset+octet-stream",
+            "Content-Length": str(len(data)),
+        }
+
+        # Add checksum if enabled
+        if self.checksum:
+            checksum_bytes = hashlib.sha1(data).digest()
+            checksum_b64 = base64.b64encode(checksum_bytes).decode("ascii")
+            headers["Upload-Checksum"] = f"sha1 {checksum_b64}"
+
+        try:
+            req = Request(upload_url, data=data, headers=headers, method="PATCH")
+            with urlopen(req):
+                pass
+        except HTTPError as e:
+            raise TusUploadFailed(
+                f"Failed to upload chunk at offset {offset}: {e.reason}",
+                status_code=e.code,
+                response_content=e.read(),
+            ) from e
+
+    def get_file_size(self, file_source: Union[str, IO]) -> int:
+        """
+        Get the size of a file.
+
+        Args:
+            file_source: Either a file path (str) or file stream (IO)
+
+        Returns:
+            File size in bytes
+        """
+        if isinstance(file_source, str):
+            return os.path.getsize(file_source)
+        else:
+            current_pos = file_source.tell()
+            file_source.seek(0, os.SEEK_END)
+            size = file_source.tell()
+            file_source.seek(current_pos)
+            return size
+
+    def get_file_stream(self, file_source: Union[str, IO]) -> IO:
+        """
+        Get a file stream from a file path or stream.
+
+        Args:
+            file_source: Either a file path (str) or file stream (IO)
+
+        Returns:
+            File stream object
+        """
+        if isinstance(file_source, str):
+            return open(file_source, "rb")
+        else:
+            file_source.seek(0)
+            return file_source

resumable_upload/client/retry.py

@@ -0,0 +1,203 @@
+"""Advanced TUS client with retry logic and robust error handling."""
+
+import logging
+import time
+from threading import Lock
+from typing import Callable, Optional, Union
+from urllib.error import HTTPError, URLError
+
+from resumable_upload.client.base import TusClient
+from resumable_upload.client.stats import UploadStats
+
+
+class TusClientWithRetry(TusClient):
+    """TUS client with automatic retry logic and detailed progress tracking.
+
+    Extends the base TusClient with:
+    - Automatic retry with exponential backoff
+    - Detailed progress statistics via UploadStats
+    - Comprehensive error handling and logging
+    - Configurable retry parameters
+
+    The TUS protocol requires sequential chunk uploads (not parallel) because
+    each chunk must be uploaded at the correct offset. The server validates that
+    the Upload-Offset header matches the current file position.
+
+    Example:
+        >>> client = TusClientWithRetry(
+        ...     "http://localhost:8080/files", max_retries=3, retry_delay=1.0
+        ... )
+        >>> def progress(stats):
+        ...     print(f"Progress: {stats.progress_percent:.1f}%")
+        >>> url = client.upload_file("file.bin", progress_callback=progress)
+    """
+
+    def __init__(
+        self,
+        url: str,
+        chunk_size: Union[int, float] = 1024 * 1024,  # 1MB chunks
+        checksum: bool = True,  # Enable checksum verification
+        max_retries: int = 3,  # Retry attempts per chunk
+        retry_delay: float = 1.0,  # Initial delay between retries (seconds)
+    ):
+        """Initialize TUS client with retry capability.
+
+        Args:
+            url: Base URL of TUS server
+            chunk_size: Size of each chunk in bytes (default: 1MB). Can be int or float.
+            checksum: Enable SHA1 checksum verification (default: True)
+            max_retries: Maximum retry attempts for failed chunks (default: 3)
+            retry_delay: Base delay between retry attempts in seconds (default: 1.0)
+        """
+        super().__init__(url=url, chunk_size=chunk_size, checksum=checksum)
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+        self.stats_lock = Lock()
+        self.logger = logging.getLogger(__name__)
+
+    def upload_file(
+        self,
+        file_path: str,
+        metadata: Optional[dict[str, str]] = None,
+        chunk_size: Optional[Union[int, float]] = None,
+        progress_callback: Optional[Callable[[UploadStats], None]] = None,
+    ) -> str:
+        """Upload file with retry logic and progress tracking.
+
+        Uploads chunks sequentially as required by the TUS protocol. Each chunk
+        must be uploaded at the correct offset for data integrity.
+
+        Args:
+            file_path: Path to file to upload
+            metadata: Optional metadata dictionary
+            chunk_size: Optional override for chunk size. Can be int or float.
+            progress_callback: Optional callback for progress updates with UploadStats
+
+        Returns:
+            URL of uploaded file
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            ValueError: If chunk_size is less than 1
+            Exception: On upload failure after all retries
+        """
+        import os
+
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        file_size = os.path.getsize(file_path)
+        stats = UploadStats(total_bytes=file_size)
+
+        # Use provided chunk_size or instance default, convert to int
+        actual_chunk_size_raw = chunk_size if chunk_size is not None else self.chunk_size
+        if actual_chunk_size_raw < 1:
+            raise ValueError(f"chunk_size must be at least 1 byte, got {actual_chunk_size_raw}")
+        actual_chunk_size = int(actual_chunk_size_raw)
+
+        self.logger.info(f"Starting upload of {file_path} ({file_size} bytes)")
+        self.logger.info(f"Chunk size: {actual_chunk_size}, Max retries: {self.max_retries}")
+
+        # Create upload
+        upload_url = self._create_upload(file_size, metadata or {})
+        self.logger.info(f"Upload created: {upload_url}")
+
+        # Calculate chunks
+        num_chunks = (file_size + actual_chunk_size - 1) // actual_chunk_size
+        self.logger.info(f"Split into {num_chunks} chunks")
+
+        # Upload chunks sequentially (required by TUS protocol)
+        for i in range(num_chunks):
+            offset = i * actual_chunk_size
+            size = min(actual_chunk_size, file_size - offset)
+
+            try:
+                self._upload_chunk_with_retry(file_path, upload_url, offset, size, stats)
+
+                with self.stats_lock:
+                    stats.uploaded_bytes += size
+                    stats.chunks_completed += 1
+
+                if progress_callback:
+                    progress_callback(stats)
+
+                self.logger.debug(f"Chunk {i + 1}/{num_chunks} completed (offset {offset})")
+
+            except Exception as e:
+                self.logger.error(f"Chunk at offset {offset} failed after all retries: {e}")
+                with self.stats_lock:
+                    stats.chunks_failed += 1
+                raise
+
+        self.logger.info(
+            f"Upload completed in {stats.elapsed_time:.2f}s ({stats.upload_speed_mbps:.2f} MB/s)"
+        )
+        return upload_url
+
+    def _upload_chunk_with_retry(
+        self,
+        file_path: str,
+        upload_url: str,
+        offset: int,
+        size: int,
+        stats: UploadStats,
+    ) -> None:
+        """Upload a single chunk with retry logic.
+
+        Args:
+            file_path: Path to source file
+            upload_url: Upload URL
+            offset: Byte offset in file
+            size: Chunk size in bytes
+            stats: Upload statistics object
+
+        Raises:
+            Exception: If all retry attempts fail
+        """
+        last_error = None
+
+        for attempt in range(self.max_retries):
+            try:
+                # Read chunk from file
+                with open(file_path, "rb") as f:
+                    f.seek(offset)
+                    data = f.read(size)
+
+                # Verify chunk was read correctly
+                if len(data) != size:
+                    raise ValueError(f"Read {len(data)} bytes, expected {size} at offset {offset}")
+
+                # Upload chunk using base class method
+                self._upload_chunk(upload_url, offset, data)
+
+                # Success!
+                if attempt > 0:
+                    with self.stats_lock:
+                        stats.chunks_retried += 1
+                    self.logger.info(
+                        f"Chunk at offset {offset} succeeded after {attempt + 1} attempts"
+                    )
+
+                return
+
+            except (HTTPError, URLError, OSError) as e:
+                last_error = e
+                if attempt < self.max_retries - 1:
+                    # Exponential backoff
+                    delay = self.retry_delay * (2**attempt)
+                    self.logger.warning(
+                        f"Chunk at offset {offset} failed "
+                        f"(attempt {attempt + 1}/{self.max_retries}): {e}. "
+                        f"Retrying in {delay:.1f}s..."
+                    )
+                    time.sleep(delay)
+                else:
+                    self.logger.error(
+                        f"Chunk at offset {offset} failed after {self.max_retries} attempts"
+                    )
+
+        # All retries failed
+        raise Exception(
+            f"Failed to upload chunk at offset {offset} after {self.max_retries} "
+            f"attempts: {last_error}"
+        )

resumable_upload/client/stats.py

@@ -0,0 +1,66 @@
+"""Upload statistics tracking for TUS client."""
+
+import time
+from dataclasses import dataclass
+
+
+@dataclass
+class UploadStats:
+    """Statistics for upload progress.
+
+    Attributes:
+        total_bytes: Total number of bytes to upload
+        uploaded_bytes: Number of bytes uploaded so far
+        chunks_completed: Number of chunks successfully uploaded
+        chunks_failed: Number of chunks that failed
+        chunks_retried: Number of chunks that required retries
+        start_time: Timestamp when upload started
+    """
+
+    total_bytes: int
+    uploaded_bytes: int = 0
+    chunks_completed: int = 0
+    chunks_failed: int = 0
+    chunks_retried: int = 0
+    start_time: float = 0.0
+
+    def __post_init__(self):
+        if self.start_time == 0.0:
+            self.start_time = time.time()
+
+    @property
+    def elapsed_time(self) -> float:
+        """Get elapsed time in seconds."""
+        return time.time() - self.start_time
+
+    @property
+    def upload_speed(self) -> float:
+        """Get upload speed in bytes/second."""
+        if self.elapsed_time > 0:
+            return self.uploaded_bytes / self.elapsed_time
+        return 0.0
+
+    @property
+    def upload_speed_mbps(self) -> float:
+        """Get upload speed in MB/second."""
+        return self.upload_speed / (1024 * 1024)
+
+    @property
+    def progress_percent(self) -> float:
+        """Get progress as percentage (0-100)."""
+        if self.total_bytes > 0:
+            return (self.uploaded_bytes / self.total_bytes) * 100
+        return 0.0
+
+    @property
+    def eta_seconds(self) -> float:
+        """Get estimated time to completion in seconds."""
+        if self.upload_speed > 0:
+            remaining_bytes = self.total_bytes - self.uploaded_bytes
+            return remaining_bytes / self.upload_speed
+        return 0.0
+
+    @property
+    def total_chunks(self) -> int:
+        """Get total number of chunks (completed + failed)."""
+        return self.chunks_completed + self.chunks_failed