dexverse-cli 0.0.2__py3-none-any.whl

dexverse/downloader/http_backend.py

@@ -0,0 +1,508 @@
+"""HTTP / presigned-URL backend with retry and single-file resume support."""
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+from urllib.parse import urlparse
+
+import httpx
+
+from ..manifest import FileSpec, Manifest, resolve_dest
+from .base import FileResult, ProgressReporter
+
+_HTTP_BACKEND_NAMES = ("http", "https", "presigned")
+_ALLOWED_SCHEMES = ("http", "https")
+_RETRYABLE_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504}
+_CONTENT_RANGE_RE = re.compile(r"^bytes (\d+)-(\d+)/(\d+|\*)$")
+_UNSATISFIED_RANGE_RE = re.compile(r"^bytes \*/(\d+)$")
+
+
+class _DownloadError(Exception):
+    """A permanent response/protocol error for one file."""
+
+
+class _RetryableDownloadError(Exception):
+    """A response/protocol error worth retrying."""
+
+    def __init__(self, message: str, retry_after: Optional[float] = None):
+        super().__init__(message)
+        self.retry_after = retry_after
+
+
+class _RestartDownload(Exception):
+    """The partial file cannot be appended safely and must be restarted."""
+
+
+def _validate_presigned_url(url: str) -> None:
+    """Reject non-http(s) schemes and optionally enforce a host allowlist."""
+    parsed = urlparse(url)
+    if parsed.scheme not in _ALLOWED_SCHEMES:
+        raise ValueError(
+            f"unsupported presignedUrl scheme {parsed.scheme!r}; "
+            f"expected one of {_ALLOWED_SCHEMES}"
+        )
+    allow = os.getenv("DEXVERSE_ALLOWED_HOSTS")
+    if allow:
+        allowed = {h.strip().lower() for h in allow.split(",") if h.strip()}
+        host = (parsed.hostname or "").lower()
+        if host and host not in allowed:
+            raise ValueError(
+                f"presignedUrl host {host!r} not in DEXVERSE_ALLOWED_HOSTS"
+            )
+
+
+def _part_paths(dest: Path) -> Tuple[Path, Path]:
+    part = dest.with_name(dest.name + ".part")
+    return part, part.with_name(part.name + ".meta")
+
+
+def _load_part_meta(path: Path) -> Dict[str, object]:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError, TypeError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _save_part_meta(
+    path: Path,
+    *,
+    etag: Optional[str],
+    expected_size: Optional[int],
+) -> None:
+    data = {"etag": etag, "expectedSize": expected_size}
+    staging = path.with_name(path.name + ".tmp")
+    staging.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8")
+    staging.replace(path)
+
+
+def _parse_int_header(value: Optional[str]) -> Optional[int]:
+    if value is None or not value.isdigit():
+        return None
+    return int(value)
+
+
+def _parse_content_range(value: Optional[str]) -> Optional[Tuple[int, int, Optional[int]]]:
+    if not value:
+        return None
+    match = _CONTENT_RANGE_RE.match(value.strip())
+    if not match:
+        return None
+    start, end, total = match.groups()
+    return int(start), int(end), None if total == "*" else int(total)
+
+
+def _retry_after_seconds(value: Optional[str]) -> Optional[float]:
+    if value is None:
+        return None
+    try:
+        return max(0.0, min(float(value), 60.0))
+    except ValueError:
+        return None
+
+
+def _file_size(path: Path) -> int:
+    try:
+        return path.stat().st_size
+    except OSError:
+        return 0
+
+
+class HttpBackend:
+    name = "http"
+
+    def __init__(
+        self,
+        *,
+        verify: bool = True,
+        timeout: float = 120.0,
+        connect_timeout: float = 30.0,
+        max_retries: int = 5,
+        retry_backoff: float = 1.0,
+        client_factory=None,
+    ):
+        if timeout <= 0 or connect_timeout <= 0:
+            raise ValueError("HTTP timeouts must be greater than zero")
+        if max_retries < 0:
+            raise ValueError("max_retries must be zero or greater")
+        if retry_backoff < 0:
+            raise ValueError("retry_backoff must be zero or greater")
+        self.verify = verify
+        self.timeout = timeout
+        self.connect_timeout = connect_timeout
+        self.max_retries = max_retries
+        self.retry_backoff = retry_backoff
+        self._client_factory = client_factory
+        self._tls = threading.local()
+        self._clients: List[httpx.Client] = []
+        self._clients_lock = threading.Lock()
+        self._log_lock = threading.Lock()
+
+    def _client(self) -> httpx.Client:
+        cached = getattr(self._tls, "client", None)
+        if cached is None:
+            if self._client_factory:
+                cached = self._client_factory()
+            else:
+                cached = httpx.Client(
+                    timeout=httpx.Timeout(
+                        self.timeout,
+                        connect=self.connect_timeout,
+                    ),
+                    verify=self.verify,
+                    follow_redirects=True,
+                )
+            self._tls.client = cached
+            with self._clients_lock:
+                self._clients.append(cached)
+        return cached
+
+    def download(
+        self,
+        manifest: Manifest,
+        files: List[FileSpec],
+        out: Path,
+        *,
+        parallel: int,
+        log_path: Optional[Path] = None,
+        progress: Optional[ProgressReporter] = None,
+    ) -> Dict[str, FileResult]:
+        if not files:
+            return {}
+        workers = max(1, min(parallel or 4, 32))
+        results: Dict[str, FileResult] = {}
+
+        def _one(file_spec: FileSpec) -> FileResult:
+            url = file_spec.presignedUrl
+            key = file_spec.effective_key()
+            if not url:
+                return FileResult(key, False, error="no presignedUrl")
+            try:
+                _validate_presigned_url(url)
+            except ValueError as exc:
+                return FileResult(key, False, error=str(exc))
+
+            dest = resolve_dest(out, file_spec.relpath())
+            part, meta_path = _part_paths(dest)
+            part.parent.mkdir(parents=True, exist_ok=True)
+            if not part.exists():
+                meta_path.unlink(missing_ok=True)
+            elif file_spec.size is not None and _file_size(part) > file_spec.size:
+                part.unlink(missing_ok=True)
+                meta_path.unlink(missing_ok=True)
+
+            last_error = ""
+            for attempt in range(self.max_retries + 1):
+                try:
+                    expected_size = self._download_once(
+                        file_spec,
+                        url,
+                        part,
+                        meta_path,
+                        progress,
+                    )
+                    local_size = _file_size(part)
+                    if expected_size is not None and local_size < expected_size:
+                        raise _RetryableDownloadError(
+                            f"incomplete response: got {local_size}, "
+                            f"expected {expected_size}"
+                        )
+                    if expected_size is not None and local_size > expected_size:
+                        part.unlink(missing_ok=True)
+                        meta_path.unlink(missing_ok=True)
+                        raise _DownloadError(
+                            f"size mismatch: got {local_size}, "
+                            f"expected {expected_size}"
+                        )
+                    part.replace(dest)
+                    meta_path.unlink(missing_ok=True)
+                    return FileResult(
+                        key,
+                        True,
+                        localSize=local_size,
+                        expectedSize=expected_size,
+                    )
+                except _RestartDownload as exc:
+                    last_error = str(exc)
+                    part.unlink(missing_ok=True)
+                    meta_path.unlink(missing_ok=True)
+                    retryable = True
+                    retry_after = 0.0
+                except _RetryableDownloadError as exc:
+                    last_error = str(exc)
+                    retryable = True
+                    retry_after = exc.retry_after
+                except (httpx.TransportError, httpx.StreamError) as exc:
+                    last_error = str(exc) or type(exc).__name__
+                    retryable = True
+                    retry_after = None
+                except _DownloadError as exc:
+                    return self._failed_result(
+                        key, str(exc), file_spec, part, meta_path
+                    )
+                except Exception as exc:  # noqa: BLE001 - per-file failure
+                    return self._failed_result(
+                        key,
+                        str(exc) or type(exc).__name__,
+                        file_spec,
+                        part,
+                        meta_path,
+                    )
+
+                if not retryable or attempt >= self.max_retries:
+                    break
+                delay = (
+                    retry_after
+                    if retry_after is not None
+                    else min(self.retry_backoff * (2 ** attempt), 30.0)
+                )
+                self._append_log(
+                    log_path,
+                    f"  RETRY {key}: attempt={attempt + 2}/"
+                    f"{self.max_retries + 1} offset={_file_size(part)} "
+                    f"delay={delay:g}s reason={last_error}\n",
+                )
+                if delay:
+                    time.sleep(delay)
+
+            return self._failed_result(
+                key,
+                f"{last_error}; exhausted {self.max_retries + 1} attempts",
+                file_spec,
+                part,
+                meta_path,
+            )
+
+        try:
+            with ThreadPoolExecutor(max_workers=workers) as pool:
+                futures = [pool.submit(_one, f) for f in files]
+                for future in as_completed(futures):
+                    result = future.result()
+                    results[result.objectKey] = result
+                    if progress is not None:
+                        progress.complete_file()
+        finally:
+            self._close_thread_clients()
+
+        self._log(log_path, files, results)
+        return results
+
+    def _download_once(
+        self,
+        file_spec: FileSpec,
+        url: str,
+        part: Path,
+        meta_path: Path,
+        progress: Optional[ProgressReporter],
+    ) -> Optional[int]:
+        offset = _file_size(part)
+        saved_meta = _load_part_meta(meta_path) if offset else {}
+        saved_etag = saved_meta.get("etag")
+        saved_expected = saved_meta.get("expectedSize")
+        if not isinstance(saved_etag, str):
+            saved_etag = None
+        if not isinstance(saved_expected, int):
+            saved_expected = None
+
+        headers = {"Accept-Encoding": "identity"}
+        if offset:
+            headers["Range"] = f"bytes={offset}-"
+            if saved_etag:
+                headers["If-Range"] = saved_etag
+
+        with self._client().stream("GET", url, headers=headers) as response:
+            status = response.status_code
+            if status == 416 and offset:
+                value = response.headers.get("content-range", "").strip()
+                match = _UNSATISFIED_RANGE_RE.match(value)
+                total = int(match.group(1)) if match else None
+                if file_spec.size is not None:
+                    expected = file_spec.size
+                elif saved_expected is not None:
+                    expected = saved_expected
+                else:
+                    expected = total
+                if total == offset and (expected is None or expected == total):
+                    return expected
+                if total is not None and offset > total:
+                    raise _RestartDownload(
+                        f"partial file is larger than remote object ({offset} > {total})"
+                    )
+                raise _DownloadError(
+                    f"HTTP 416 for {file_spec.effective_key()} at offset {offset}"
+                )
+            if status in _RETRYABLE_STATUS_CODES:
+                raise _RetryableDownloadError(
+                    f"HTTP {status} for {file_spec.effective_key()}",
+                    _retry_after_seconds(response.headers.get("retry-after")),
+                )
+            if status >= 400:
+                raise _DownloadError(
+                    f"HTTP {status} for {file_spec.effective_key()}"
+                )
+
+            content_length = _parse_int_header(
+                response.headers.get("content-length")
+            )
+            response_etag = response.headers.get("etag")
+            content_range = _parse_content_range(
+                response.headers.get("content-range")
+            )
+
+            if status == 206:
+                if content_range is None:
+                    raise _DownloadError(
+                        "HTTP 206 response is missing a valid Content-Range"
+                    )
+                range_start, range_end, range_total = content_range
+                if range_start != offset:
+                    raise _RestartDownload(
+                        f"server resumed at offset {range_start}, requested {offset}"
+                    )
+                if range_end < range_start:
+                    raise _DownloadError("invalid Content-Range end offset")
+                if range_total is not None and range_end >= range_total:
+                    raise _DownloadError("Content-Range exceeds remote object size")
+                if (
+                    content_length is not None
+                    and range_end - range_start + 1 != content_length
+                ):
+                    raise _DownloadError(
+                        "Content-Length does not match Content-Range"
+                    )
+                if saved_etag and response_etag and saved_etag != response_etag:
+                    raise _RestartDownload("remote object changed while resuming")
+                response_expected = range_total
+                if response_expected is None and content_length is not None:
+                    response_expected = offset + content_length
+                mode = "ab"
+                base_size = offset
+            else:
+                # A server may ignore Range or reject If-Range by returning 200.
+                # Overwrite instead of appending so the final file cannot contain
+                # a duplicated prefix.
+                response_expected = content_length
+                mode = "wb"
+                base_size = 0
+
+            if (
+                file_spec.size is not None
+                and response_expected is not None
+                and file_spec.size != response_expected
+            ):
+                raise _DownloadError(
+                    f"remote size {response_expected} does not match manifest "
+                    f"size {file_spec.size}"
+                )
+            if (
+                status == 206
+                and saved_expected is not None
+                and response_expected is not None
+                and saved_expected != response_expected
+            ):
+                raise _RestartDownload("remote object size changed while resuming")
+
+            if file_spec.size is not None:
+                expected_size = file_spec.size
+            elif response_expected is not None:
+                expected_size = response_expected
+            else:
+                expected_size = saved_expected
+            _save_part_meta(
+                meta_path,
+                etag=response_etag or saved_etag,
+                expected_size=expected_size,
+            )
+
+            size = base_size
+            with part.open(mode) as file_handle:
+                # Use transport-native chunks. Supplying a large chunk_size makes
+                # httpx buffer a short tail in memory, which is then lost if the
+                # connection times out before that buffer fills.
+                for chunk in response.iter_bytes():
+                    if not chunk:
+                        continue
+                    file_handle.write(chunk)
+                    size += len(chunk)
+                    if progress is not None:
+                        progress.advance(len(chunk))
+
+            if expected_size is not None and size < expected_size:
+                raise _RetryableDownloadError(
+                    f"incomplete response: got {size}, expected {expected_size}"
+                )
+            return expected_size
+
+    def _failed_result(
+        self,
+        key: str,
+        error: str,
+        file_spec: FileSpec,
+        part: Path,
+        meta_path: Path,
+    ) -> FileResult:
+        local_size = _file_size(part) if part.exists() else None
+        meta = _load_part_meta(meta_path)
+        meta_expected = meta.get("expectedSize")
+        expected_size = (
+            file_spec.size
+            if file_spec.size is not None
+            else meta_expected if isinstance(meta_expected, int) else None
+        )
+        if local_size:
+            error = (
+                f"{error}; partial download retained at {part.name} "
+                f"({local_size} bytes)"
+            )
+        return FileResult(
+            key,
+            False,
+            error=error,
+            localSize=local_size,
+            expectedSize=expected_size,
+        )
+
+    def _close_thread_clients(self) -> None:
+        with self._clients_lock:
+            clients, self._clients = self._clients, []
+        for client in clients:
+            try:
+                client.close()
+            except Exception:
+                pass
+
+    def _append_log(self, log_path: Optional[Path], message: str) -> None:
+        if not log_path:
+            return
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        with self._log_lock:
+            with log_path.open("a", encoding="utf-8") as log_file:
+                log_file.write(message)
+
+    def _log(
+        self,
+        log_path: Optional[Path],
+        files: List[FileSpec],
+        results: Dict[str, FileResult],
+    ) -> None:
+        if not log_path:
+            return
+        ok = sum(1 for result in results.values() if result.ok)
+        self._append_log(
+            log_path,
+            f"[http backend] files={len(files)} ok={ok} "
+            f"verify_ssl={self.verify} read_timeout={self.timeout:g}s "
+            f"max_retries={self.max_retries}\n",
+        )
+        for result in results.values():
+            if not result.ok:
+                self._append_log(
+                    log_path,
+                    f"  FAIL {result.objectKey}: {result.error}\n",
+                )