s3ui 1.0.0__py3-none-any.whl

s3ui/core/download_worker.py

@@ -0,0 +1,260 @@
+"""Download worker — handles single and ranged-GET downloads as a QRunnable."""
+
+from __future__ import annotations
+
+import logging
+import random
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from PyQt6.QtCore import QObject, QRunnable, pyqtSignal
+
+from s3ui.constants import DEFAULT_PART_SIZE, MAX_RETRY_ATTEMPTS, MULTIPART_THRESHOLD
+
+if TYPE_CHECKING:
+    from s3ui.core.s3_client import S3Client
+    from s3ui.db.database import Database
+
+logger = logging.getLogger("s3ui.download_worker")
+
+
+class DownloadWorkerSignals(QObject):
+    progress = pyqtSignal(int, int, int)  # transfer_id, bytes_done, total
+    speed = pyqtSignal(int, float)  # transfer_id, bytes_per_sec
+    finished = pyqtSignal(int)  # transfer_id
+    failed = pyqtSignal(int, str, str)  # transfer_id, user_msg, detail
+
+
+class DownloadWorker(QRunnable):
+    """Downloads a file from S3, with ranged GET and resume support."""
+
+    def __init__(
+        self,
+        transfer_id: int,
+        s3_client: S3Client,
+        db: Database,
+        bucket: str,
+        pause_event: threading.Event,
+        cancel_event: threading.Event,
+    ) -> None:
+        super().__init__()
+        self.setAutoDelete(True)
+        self.signals = DownloadWorkerSignals()
+        self.transfer_id = transfer_id
+        self._s3 = s3_client
+        self._db = db
+        self._bucket = bucket
+        self._pause = pause_event
+        self._cancel = cancel_event
+
+        self._speed_window: list[tuple[float, int]] = []
+        self._last_speed_emit = 0.0
+
+    def run(self) -> None:
+        try:
+            self._do_download()
+        except Exception as e:
+            import traceback
+
+            logger.error("Download %d failed: %s", self.transfer_id, e)
+            try:
+                self._mark_failed(str(e))
+            except Exception:
+                logger.exception("Failed to mark download %d as failed", self.transfer_id)
+            self.signals.failed.emit(self.transfer_id, str(e), traceback.format_exc())
+
+    def _do_download(self) -> None:
+        row = self._db.fetchone("SELECT * FROM transfers WHERE id = ?", (self.transfer_id,))
+        if not row:
+            self.signals.failed.emit(self.transfer_id, "Transfer record not found.", "")
+            return
+
+        local_path = Path(row["local_path"])
+        object_key = row["object_key"]
+
+        # Validate destination directory
+        if not local_path.parent.exists():
+            self._mark_failed("Destination directory does not exist.")
+            self.signals.failed.emit(
+                self.transfer_id,
+                "Destination directory does not exist.",
+                str(local_path.parent),
+            )
+            return
+
+        self._db.execute(
+            "UPDATE transfers SET status = 'in_progress', updated_at = datetime('now') "
+            "WHERE id = ?",
+            (self.transfer_id,),
+        )
+
+        # Get object metadata
+        item = self._s3.head_object(self._bucket, object_key)
+        total_size = item.size or 0
+
+        self._db.execute(
+            "UPDATE transfers SET total_bytes = ? WHERE id = ?",
+            (total_size, self.transfer_id),
+        )
+
+        temp_path = local_path.parent / f".s3ui-download-{self.transfer_id}.tmp"
+
+        if total_size < MULTIPART_THRESHOLD:
+            self._single_download(object_key, local_path, temp_path, total_size)
+        else:
+            self._ranged_download(object_key, local_path, temp_path, total_size)
+
+    def _single_download(self, key: str, final_path: Path, temp_path: Path, total: int) -> None:
+        body = self._s3.get_object(self._bucket, key)
+        data = body.read()
+        temp_path.write_bytes(data)
+        temp_path.rename(final_path)
+        self._complete(total)
+
+    def _ranged_download(self, key: str, final_path: Path, temp_path: Path, total: int) -> None:
+        chunk_size = DEFAULT_PART_SIZE
+
+        # Resume from existing temp file
+        offset = 0
+        if temp_path.exists():
+            offset = temp_path.stat().st_size
+            self._db.execute(
+                "UPDATE transfers SET transferred = ? WHERE id = ?",
+                (offset, self.transfer_id),
+            )
+
+        cancelled = False
+        paused = False
+        with open(temp_path, "ab") as f:
+            while offset < total:
+                if self._cancel.is_set():
+                    cancelled = True
+                    break
+                if self._pause.is_set():
+                    paused = True
+                    break
+
+                end = min(offset + chunk_size - 1, total - 1)
+                range_header = f"bytes={offset}-{end}"
+
+                data = self._download_chunk_with_retry(key, range_header)
+                if data is None:
+                    return  # failed signal already emitted
+
+                f.write(data)
+                offset += len(data)
+
+                self._db.execute(
+                    "UPDATE transfers SET transferred = ?, "
+                    "updated_at = datetime('now') WHERE id = ?",
+                    (offset, self.transfer_id),
+                )
+                self.signals.progress.emit(self.transfer_id, offset, total)
+                self._update_speed(len(data))
+
+        if cancelled:
+            self._do_cancel(temp_path)
+            return
+        if paused:
+            self._do_pause(offset)
+            return
+
+        # Verify size
+        actual_size = temp_path.stat().st_size
+        if actual_size != total:
+            msg = f"Size mismatch: expected {total}, got {actual_size}"
+            self._mark_failed(msg)
+            self.signals.failed.emit(self.transfer_id, msg, "")
+            return
+
+        # Atomic rename
+        temp_path.rename(final_path)
+        self._complete(total)
+
+    def _download_chunk_with_retry(self, key: str, range_header: str) -> bytes | None:
+        for attempt in range(MAX_RETRY_ATTEMPTS):
+            try:
+                body = self._s3.get_object(self._bucket, key, range_header)
+                return body.read()
+            except Exception as e:
+                if attempt < MAX_RETRY_ATTEMPTS - 1:
+                    delay = _backoff_delay(attempt)
+                    logger.warning(
+                        "Download chunk attempt %d failed, retrying in %.1fs: %s",
+                        attempt + 1,
+                        delay,
+                        e,
+                    )
+                    time.sleep(delay)
+                else:
+                    self._mark_failed(str(e))
+                    self.signals.failed.emit(
+                        self.transfer_id,
+                        f"Download failed after {MAX_RETRY_ATTEMPTS} attempts.",
+                        str(e),
+                    )
+                    return None
+
+    def _complete(self, total: int) -> None:
+        self._db.execute(
+            "UPDATE transfers SET status = 'completed', transferred = ?, "
+            "updated_at = datetime('now') WHERE id = ?",
+            (total, self.transfer_id),
+        )
+        self.signals.progress.emit(self.transfer_id, total, total)
+        self.signals.finished.emit(self.transfer_id)
+        logger.info("Download %d completed", self.transfer_id)
+
+    def _mark_failed(self, msg: str) -> None:
+        self._db.execute(
+            "UPDATE transfers SET status = 'failed', error_message = ?, "
+            "updated_at = datetime('now') WHERE id = ?",
+            (msg, self.transfer_id),
+        )
+
+    def _do_cancel(self, temp_path: Path) -> None:
+        if temp_path.exists():
+            try:
+                temp_path.unlink()
+            except Exception:
+                logger.exception(
+                    "Failed to remove temp file for transfer %d: %s",
+                    self.transfer_id,
+                    temp_path,
+                )
+        self._db.execute(
+            "UPDATE transfers SET status = 'cancelled', updated_at = datetime('now') WHERE id = ?",
+            (self.transfer_id,),
+        )
+        logger.info("Download %d cancelled", self.transfer_id)
+
+    def _do_pause(self, offset: int) -> None:
+        self._db.execute(
+            "UPDATE transfers SET status = 'paused', transferred = ?, "
+            "updated_at = datetime('now') WHERE id = ?",
+            (offset, self.transfer_id),
+        )
+        logger.info("Download %d paused at offset %d", self.transfer_id, offset)
+
+    def _update_speed(self, chunk_bytes: int) -> None:
+        now = time.monotonic()
+        self._speed_window.append((now, chunk_bytes))
+        self._speed_window = [(t, b) for t, b in self._speed_window if now - t <= 3.0]
+        if now - self._last_speed_emit >= 0.5 and self._speed_window:
+            window_time = now - self._speed_window[0][0]
+            if window_time > 0:
+                total_bytes = sum(b for _, b in self._speed_window)
+                bps = total_bytes / window_time
+                self.signals.speed.emit(self.transfer_id, bps)
+            self._last_speed_emit = now
+
+
+def _backoff_delay(attempt: int) -> float:
+    """Exponential backoff with jitter."""
+    if attempt == 0:
+        return 0.0
+    base = 4 ** (attempt - 1)
+    jitter_max = base * 0.5
+    return base + random.uniform(0, jitter_max)

s3ui/core/errors.py

@@ -0,0 +1,104 @@
+"""Maps boto3/botocore exceptions to plain-language error messages."""
+
+import logging
+
+logger = logging.getLogger("s3ui.errors")
+
+# Maps AWS error codes to (user-facing message, suggestion)
+ERROR_MESSAGES: dict[str, tuple[str, str]] = {
+    "InvalidAccessKeyId": (
+        "Invalid access key.",
+        "Check that your Access Key ID is correct in Settings.",
+    ),
+    "SignatureDoesNotMatch": (
+        "Invalid secret key.",
+        "Check that your Secret Access Key is correct in Settings.",
+    ),
+    "AccessDenied": (
+        "Access denied.",
+        "Your AWS credentials don't have permission for this action. Check your IAM policy.",
+    ),
+    "NoSuchBucket": (
+        "Bucket not found.",
+        "The bucket may have been deleted or you may have a typo in the name.",
+    ),
+    "NoSuchKey": (
+        "File not found.",
+        "The file may have been deleted or moved by someone else.",
+    ),
+    "BucketAlreadyOwnedByYou": (
+        "You already own this bucket.",
+        "",
+    ),
+    "BucketNotEmpty": (
+        "Bucket is not empty.",
+        "Delete all files in the bucket before deleting it.",
+    ),
+    "EntityTooLarge": (
+        "File is too large for a single upload.",
+        "This shouldn't happen — the app should use multipart upload. Please report this bug.",
+    ),
+    "SlowDown": (
+        "S3 is asking us to slow down.",
+        "Too many requests. The app will retry automatically.",
+    ),
+    "ServiceUnavailable": (
+        "S3 is temporarily unavailable.",
+        "Try again in a few moments.",
+    ),
+    "InternalError": (
+        "S3 encountered an internal error.",
+        "Try again in a few moments.",
+    ),
+    "RequestTimeout": (
+        "The request timed out.",
+        "Check your network connection and try again.",
+    ),
+    "ExpiredToken": (
+        "Your credentials have expired.",
+        "Update your credentials in Settings.",
+    ),
+    "InvalidBucketName": (
+        "Invalid bucket name.",
+        "Bucket names must be 3-63 characters, lowercase letters, numbers, and hyphens.",
+    ),
+    "KeyTooLongError": (
+        "File name is too long.",
+        "S3 keys can be at most 1024 bytes.",
+    ),
+}
+
+
+def translate_error(exc: Exception) -> tuple[str, str]:
+    """Translate a boto3 exception to (user_message, raw_detail).
+
+    Returns a tuple of (plain-language message for the user, raw error string for
+    the "Show Details" expander).
+    """
+    raw_detail = str(exc)
+
+    # botocore ClientError
+    if hasattr(exc, "response"):
+        code = exc.response.get("Error", {}).get("Code", "")
+        if code in ERROR_MESSAGES:
+            user_msg, suggestion = ERROR_MESSAGES[code]
+            if suggestion:
+                user_msg = f"{user_msg} {suggestion}"
+            return user_msg, raw_detail
+        # Unknown AWS error code
+        message = exc.response.get("Error", {}).get("Message", "")
+        return f"AWS error: {message}" if message else "An AWS error occurred.", raw_detail
+
+    # Connection errors
+    err_type = type(exc).__name__
+    if "ConnectionError" in err_type or "EndpointConnectionError" in err_type:
+        return (
+            "Could not connect to S3. Check your network connection and try again.",
+            raw_detail,
+        )
+
+    if "ReadTimeoutError" in err_type or "ConnectTimeoutError" in err_type:
+        return "The connection timed out. Check your network connection.", raw_detail
+
+    # Fallback
+    return "An unexpected error occurred.", raw_detail

s3ui/core/listing_cache.py

@@ -0,0 +1,178 @@
+"""Stale-while-revalidate listing cache for S3 prefix listings."""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections import OrderedDict
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from s3ui.constants import LISTING_CACHE_MAX_ENTRIES, LISTING_CACHE_STALE_SECONDS
+from s3ui.models.s3_objects import S3Item
+
+logger = logging.getLogger("s3ui.listing_cache")
+
+
+@dataclass
+class CachedListing:
+    """A single cached listing result."""
+
+    prefix: str
+    items: list[S3Item]
+    fetched_at: float  # monotonic time
+    dirty: bool = False
+    mutation_counter: int = 0
+
+
+class ListingCache:
+    """LRU cache for S3 prefix listings with mutation tracking.
+
+    Thread-safe: all access is protected by a lock.
+    """
+
+    def __init__(
+        self,
+        max_entries: int = LISTING_CACHE_MAX_ENTRIES,
+        stale_seconds: float = LISTING_CACHE_STALE_SECONDS,
+    ) -> None:
+        self._max_entries = max_entries
+        self._stale_seconds = stale_seconds
+        self._cache: OrderedDict[str, CachedListing] = OrderedDict()
+        self._lock = threading.Lock()
+
+    def get(self, prefix: str) -> CachedListing | None:
+        """Get a cached listing, promoting it to MRU. Returns None on miss."""
+        with self._lock:
+            entry = self._cache.get(prefix)
+            if entry is not None:
+                self._cache.move_to_end(prefix)
+                return entry
+            return None
+
+    def put(self, prefix: str, items: list[S3Item]) -> None:
+        """Store a listing, evicting LRU if over capacity."""
+        with self._lock:
+            if prefix in self._cache:
+                existing = self._cache[prefix]
+                existing.items = list(items)
+                existing.fetched_at = time.monotonic()
+                existing.dirty = False
+                # Don't reset mutation_counter — revalidation handles that
+                self._cache.move_to_end(prefix)
+            else:
+                self._cache[prefix] = CachedListing(
+                    prefix=prefix,
+                    items=list(items),
+                    fetched_at=time.monotonic(),
+                )
+            self._evict_if_needed()
+
+    def invalidate(self, prefix: str) -> bool:
+        """Remove one entry. Returns True if it existed."""
+        with self._lock:
+            if prefix in self._cache:
+                del self._cache[prefix]
+                return True
+            return False
+
+    def invalidate_all(self) -> None:
+        """Clear the entire cache."""
+        with self._lock:
+            self._cache.clear()
+
+    def is_stale(self, prefix: str) -> bool:
+        """True if entry is missing or older than stale threshold."""
+        with self._lock:
+            entry = self._cache.get(prefix)
+            if entry is None:
+                return True
+            age = time.monotonic() - entry.fetched_at
+            return age > self._stale_seconds
+
+    def apply_mutation(self, prefix: str, fn: Callable[[list[S3Item]], None]) -> bool:
+        """Apply a mutation function to a cached listing's items.
+
+        Sets dirty=True and increments mutation_counter.
+        Returns False if prefix is not cached.
+        """
+        with self._lock:
+            entry = self._cache.get(prefix)
+            if entry is None:
+                return False
+            fn(entry.items)
+            entry.dirty = True
+            entry.mutation_counter += 1
+            return True
+
+    def get_mutation_counter(self, prefix: str) -> int:
+        """Returns current mutation counter for a prefix. 0 if not cached."""
+        with self._lock:
+            entry = self._cache.get(prefix)
+            if entry is None:
+                return 0
+            return entry.mutation_counter
+
+    def safe_revalidate(
+        self,
+        prefix: str,
+        new_items: list[S3Item],
+        counter_at_fetch_start: int,
+    ) -> bool:
+        """Apply background revalidation results safely.
+
+        If mutation_counter matches counter_at_fetch_start, does a standard replace.
+        If mutations happened since fetch started, merges: preserves optimistic
+        additions while incorporating external changes.
+
+        Returns True if the cache was updated.
+        """
+        with self._lock:
+            entry = self._cache.get(prefix)
+            if entry is None:
+                # Cache was cleared while we were fetching — just store the result
+                self._cache[prefix] = CachedListing(
+                    prefix=prefix,
+                    items=list(new_items),
+                    fetched_at=time.monotonic(),
+                )
+                self._evict_if_needed()
+                return True
+
+            if entry.mutation_counter == counter_at_fetch_start:
+                # No mutations since fetch started — safe to replace
+                entry.items = list(new_items)
+                entry.fetched_at = time.monotonic()
+                entry.dirty = False
+                return True
+
+            # Mutations happened — merge strategy:
+            # Keep items that exist in new_items (server truth)
+            # Also keep items that were added by optimistic mutations
+            # (items in current cache but NOT in the old server state)
+            new_keys = {item.key for item in new_items}
+
+            # Optimistic items: in current cache but not from server
+            optimistic = [item for item in entry.items if item.key not in new_keys]
+
+            # Build merged list: server items + optimistic items
+            merged = list(new_items) + optimistic
+
+            entry.items = merged
+            entry.fetched_at = time.monotonic()
+            entry.dirty = bool(optimistic)  # still dirty if we have optimistic items
+            # Don't reset mutation_counter — it tracks total mutations
+            logger.debug(
+                "Merged revalidation for '%s': %d server + %d optimistic items",
+                prefix,
+                len(new_items),
+                len(optimistic),
+            )
+            return True
+
+    def _evict_if_needed(self) -> None:
+        """Evict LRU entries if over capacity. Must be called with lock held."""
+        while len(self._cache) > self._max_entries:
+            evicted_key, _ = self._cache.popitem(last=False)
+            logger.debug("Evicted cache entry: '%s'", evicted_key)