flux-bootstrap 0.1.0__tar.gz

flux_bootstrap-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.3
+Name: flux-bootstrap
+Version: 0.1.0
+Requires-Dist: aiohttp>=3.13.3
+Requires-Dist: aiofiles>=25.1.0
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: setproctitle>=1.3.7
+Requires-Dist: click>=8.3.1
+Requires-Python: >=3.13

flux_bootstrap-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "flux-bootstrap"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = [
+    "aiohttp>=3.13.3",
+    "aiofiles>=25.1.0",
+    "pyyaml>=6.0.3",
+    "setproctitle>=1.3.7",
+    "click>=8.3.1",
+]
+
+[project.scripts]
+flux-bootstrap = "flux_bootstrap.main:run"
+
+[dependency-groups]
+dev = [
+    "ruff",
+    "ty",
+]
+
+[tool.uv]
+package = true
+
+[build-system]
+requires = ["uv_build>=0.9.22,<0.10.0"]
+build-backend = "uv_build"
+
+[tool.ruff]
+target-version = "py313"
+line-length = 88
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+    "ARG", # flake8-unused-arguments
+    "SIM", # flake8-simplify
+    "TCH", # flake8-type-checking
+]

flux_bootstrap-0.1.0/src/flux_bootstrap/__init__.py

@@ -0,0 +1,34 @@
+"""flux-bootstrap - Flux blockchain bootstrap downloader.
+
+This library provides an easy way to download and extract Flux blockchain
+bootstrap files from CDN with progress tracking, resume support, and automatic
+CDN failover.
+
+Example:
+    >>> from flux_bootstrap import download_bootstrap_async
+    >>> import asyncio
+    >>>
+    >>> async def main():
+    ...     def on_progress(progress):
+    ...         print(f"Progress: {progress['percent']:.1f}%")
+    ...     success = await download_bootstrap_async(
+    ...         "/path/to/destination",
+    ...         progress_callback=on_progress
+    ...     )
+    ...     return success
+    >>> asyncio.run(main())
+"""
+
+from flux_bootstrap.api import (
+    ProgressInfo,
+    download_bootstrap,
+    download_bootstrap_async,
+)
+
+__all__ = [
+    "download_bootstrap",
+    "download_bootstrap_async",
+    "ProgressInfo",
+]
+
+__version__ = "0.1.0"

flux_bootstrap-0.1.0/src/flux_bootstrap/api.py

@@ -0,0 +1,183 @@
+"""Public library API for flux-downloader."""
+
+import asyncio
+import multiprocessing
+import typing
+from dataclasses import dataclass
+from pathlib import Path
+
+from flux_bootstrap.data_structures import DEFAULT_API_URL, DEFAULT_CDN_URL
+from flux_bootstrap.main import _download_bootstrap_impl
+
+
+@dataclass
+class ProgressInfo:
+    """Progress information for download callbacks.
+
+    Attributes:
+        bytes_downloaded: Total bytes downloaded across all parts
+        total_bytes: Total expected bytes
+        percent: Download percentage (0-100)
+        speed_mbps: Current download speed in Mbps
+        cdn_server: Current CDN hostname (e.g., "cdn-1.runonflux.io")
+        current_part: Current part being downloaded (1-indexed)
+        total_parts: Total number of parts
+        source: Source type ("cdn")
+    """
+
+    bytes_downloaded: int
+    total_bytes: int
+    percent: float
+    speed_mbps: float
+    cdn_server: str | None
+    current_part: int
+    total_parts: int
+    source: str
+
+
+async def download_bootstrap_async(
+    destination: str | Path,
+    *,
+    api_url: str | None = None,
+    cdn_url: str | None = None,
+    parts_dir: str | Path | None = None,
+    progress_callback: typing.Callable[[dict[str, typing.Any]], None] | None = None,
+    cancellation_event: asyncio.Event | None = None,
+) -> bool:
+    """Download and extract Flux bootstrap files (async API).
+
+    This is the primary async API for library use. Downloads blockchain bootstrap
+    files from CDN in parts, verifies with SHA256, and extracts to destination.
+
+    Args:
+        destination: Directory where bootstrap will be extracted
+        api_url: Optional API endpoint (default: https://cdn.runonflux.io/fluxd/api/latest_bootstrap)
+        cdn_url: Optional CDN base URL (default: https://cdn.runonflux.io)
+        parts_dir: Optional directory for part files (default: <destination>/bootstrap_parts)
+        progress_callback: Optional callback for progress updates. Called with dict containing:
+            - bytes_downloaded: int
+            - total_bytes: int
+            - percent: float (0-100)
+            - speed_mbps: float
+            - cdn_server: str | None
+            - current_part: int (1-indexed)
+            - total_parts: int
+            - source: str ("cdn")
+        cancellation_event: Optional asyncio.Event to signal cancellation
+
+    Returns:
+        True if download and extraction succeeded, False otherwise
+
+    Raises:
+        ValueError: If destination is invalid
+        RuntimeError: If download or extraction fails critically
+
+    Example:
+        >>> async def main():
+        ...     def on_progress(progress):
+        ...         print(f"Progress: {progress['percent']:.1f}%")
+        ...     success = await download_bootstrap_async(
+        ...         "/path/to/destination",
+        ...         progress_callback=on_progress
+        ...     )
+        ...     return success
+        >>> asyncio.run(main())
+    """
+    # Validate and normalize destination
+    if not destination:
+        raise ValueError("destination cannot be empty")
+
+    dest_path = Path(destination).resolve()
+
+    # Set defaults
+    if api_url is None:
+        api_url = DEFAULT_API_URL
+
+    if cdn_url is None:
+        cdn_url = DEFAULT_CDN_URL
+
+    if parts_dir is None:
+        parts_dir_path = dest_path / "bootstrap_parts"
+    else:
+        parts_dir_path = Path(parts_dir).resolve()
+
+    # Create multiprocessing.Event for internal use (workers need it)
+    mp_shutdown_event = multiprocessing.Event()
+
+    # If user provided asyncio.Event, monitor it and bridge to multiprocessing.Event
+    monitor_task: asyncio.Task | None = None
+    if cancellation_event:
+
+        async def monitor_cancellation():
+            """Monitor asyncio.Event and forward to multiprocessing.Event."""
+            await cancellation_event.wait()
+            mp_shutdown_event.set()
+
+        monitor_task = asyncio.create_task(monitor_cancellation())
+
+    try:
+        # Call internal implementation with library mode flags
+        result = await _download_bootstrap_impl(
+            destination=dest_path,
+            parts_dir=parts_dir_path,
+            api_url=api_url,
+            cdn_url=cdn_url,
+            shutdown_event=mp_shutdown_event,
+            setup_logging_flag=False,  # Library mode: caller controls logging
+            set_process_title=False,  # Library mode: don't change process title
+            progress_callback=progress_callback,
+        )
+        return result
+
+    finally:
+        # Cancel monitor task if it's still running
+        if monitor_task and not monitor_task.done():
+            monitor_task.cancel()
+            try:
+                await monitor_task
+            except asyncio.CancelledError:
+                pass
+
+
+def download_bootstrap(
+    destination: str | Path,
+    **kwargs,
+) -> bool:
+    """Download and extract Flux bootstrap files (sync API).
+
+    Synchronous wrapper for download_bootstrap_async(). Uses asyncio.run() to
+    execute the async version.
+
+    Args:
+        destination: Directory where bootstrap will be extracted
+        **kwargs: Additional arguments passed to download_bootstrap_async()
+
+    Returns:
+        True if download and extraction succeeded, False otherwise
+
+    Raises:
+        ValueError: If destination is invalid
+        RuntimeError: If download or extraction fails critically, or if called
+            from within an existing event loop
+
+    Example:
+        >>> def on_progress(progress):
+        ...     print(f"Progress: {progress['percent']:.1f}%")
+        >>> success = download_bootstrap(
+        ...     "/path/to/destination",
+        ...     progress_callback=on_progress
+        ... )
+    """
+    # Check if we're already in an event loop
+    try:
+        asyncio.get_running_loop()
+        raise RuntimeError(
+            "download_bootstrap() cannot be called from within an async "
+            "context. Use download_bootstrap_async() instead."
+        )
+    except RuntimeError as e:
+        if "no running event loop" not in str(e).lower():
+            raise
+
+    # Run async version
+    return asyncio.run(download_bootstrap_async(destination, **kwargs))

flux_bootstrap-0.1.0/src/flux_bootstrap/cdn_manager.py

@@ -0,0 +1,250 @@
+"""
+CDN failover management for flux-downloader.
+
+Provides intelligent CDN failover when the main proxy CDN fails or is too slow.
+Uses x-served-by header to identify which backend CDN served the request and
+tries alternative backends.
+"""
+
+import logging
+import re
+
+
+def extract_backend_from_header(header_value: str) -> str | None:
+    """
+    Parse x-served-by header to identify backend CDN.
+
+    Examples:
+        - "cdn-1.runonflux.io" → "cdn-1"
+        - "cache-cdn-2-xyz" → "cdn-2"
+        - "cdn-3" → "cdn-3"
+
+    Args:
+        header_value: Value of x-served-by or similar header
+
+    Returns:
+        Backend identifier ("cdn-1", "cdn-2", "cdn-3"), or None if not found
+    """
+    if not header_value:
+        return None
+
+    match = re.search(r"cdn-([123])", header_value.lower())
+    return f"cdn-{match.group(1)}" if match else None
+
+
+class SlowDownloadError(Exception):
+    """
+    Raised when download speed is below threshold for too long.
+
+    This exception triggers CDN failover to try an alternative CDN.
+    """
+
+    def __init__(self, speed_mbps: float, served_by: str | None):
+        """
+        Initialize slow download error.
+
+        Args:
+            speed_mbps: Actual download speed in megabits per second
+            served_by: Backend CDN that served the request (if known)
+        """
+        self.speed_mbps = speed_mbps
+        self.served_by = served_by
+        super().__init__(
+            f"Download too slow: {speed_mbps:.2f} Mbps (served by: {served_by})"
+        )
+
+
+class CDNFailoverStrategy:
+    """
+    Manages CDN failover strategy for a single part download.
+
+    Implements 2+1+1+1 attempt strategy:
+    - Phase 1 (proxy): 2 attempts on main proxy CDN
+    - Phase 2 (direct): 1 attempt each on 3 direct backend CDNs
+
+    Smart ordering: If proxy identified which backend served the request,
+    try that backend LAST in direct phase (other backends first).
+    """
+
+    def __init__(self, proxy_url: str, direct_urls: list[str]):
+        """
+        Initialize failover strategy.
+
+        Args:
+            proxy_url: Main proxy CDN URL (e.g., https://cdn.runonflux.io)
+            direct_urls: List of direct backend CDN URLs
+        """
+        self.proxy_url = proxy_url
+        self.direct_urls = direct_urls.copy()  # Don't mutate original list
+
+        # State tracking
+        self.phase = "proxy"  # "proxy" or "direct"
+        self.proxy_attempts = 0
+        self.direct_index = 0
+        self.last_served_by: str | None = None
+        self.direct_cdn_order: list[str] = []
+
+        # Attempt limits
+        self.max_proxy_attempts = 2
+        self.max_direct_attempts = len(direct_urls)  # 1 per backend
+
+    def get_next_cdn(self) -> tuple[str | None, int, int]:
+        """
+        Get next CDN to try.
+
+        Returns:
+            Tuple of (cdn_url, attempt_number, max_attempts_for_this_cdn)
+            Returns (None, 0, 0) if all attempts exhausted
+        """
+        if self.phase == "proxy":
+            if self.proxy_attempts < self.max_proxy_attempts:
+                self.proxy_attempts += 1
+                return (self.proxy_url, self.proxy_attempts, self.max_proxy_attempts)
+            else:
+                # Transition to direct phase
+                self._initialize_direct_phase()
+                self.phase = "direct"
+                # Fall through to direct phase logic
+
+        if self.phase == "direct":
+            if self.direct_index < len(self.direct_cdn_order):
+                cdn_url = self.direct_cdn_order[self.direct_index]
+                self.direct_index += 1
+                # Each direct CDN gets 1 attempt
+                return (cdn_url, 1, 1)
+
+        # All attempts exhausted
+        return (None, 0, 0)
+
+    def _initialize_direct_phase(self) -> None:
+        """
+        Initialize direct CDN phase with smart ordering.
+
+        If last_served_by is known, put that backend LAST in the order.
+        Rationale: If proxy via cdn-2 failed, cdn-2 might be blocked for this user.
+        """
+        if not self.last_served_by:
+            # No backend identified, use default order
+            self.direct_cdn_order = self.direct_urls.copy()
+            return
+
+        # Find the backend URL that matches last_served_by
+        identified_backend_url = None
+        other_backends = []
+
+        for url in self.direct_urls:
+            backend_id = extract_backend_from_header(url)
+            if backend_id == self.last_served_by:
+                identified_backend_url = url
+            else:
+                other_backends.append(url)
+
+        # Smart ordering: try other backends first, identified backend last
+        if identified_backend_url:
+            self.direct_cdn_order = other_backends + [identified_backend_url]
+            logging.info(
+                f"CDN ordering: Trying {len(other_backends)} other backends before {self.last_served_by}"
+            )
+        else:
+            # Backend not found in direct URLs (shouldn't happen)
+            self.direct_cdn_order = self.direct_urls.copy()
+
+    def record_failure(
+        self, served_by_header: str | None, reason: str
+    ) -> None:
+        """
+        Record a failure for the current CDN attempt.
+
+        Args:
+            served_by_header: Value of x-served-by header (if available)
+            reason: Human-readable failure reason (for logging)
+        """
+        # Extract backend identifier from header
+        if served_by_header and self.phase == "proxy":
+            backend_id = extract_backend_from_header(served_by_header)
+            if backend_id:
+                self.last_served_by = backend_id
+                logging.debug(f"Identified backend from proxy: {backend_id}")
+
+        logging.debug(
+            f"CDN failure in {self.phase} phase: {reason} "
+            f"(served_by: {served_by_header or 'unknown'})"
+        )
+
+    def get_phase_summary(self) -> str:
+        """
+        Get human-readable summary of current phase.
+
+        Returns:
+            Summary string for logging
+        """
+        if self.phase == "proxy":
+            return f"proxy phase ({self.proxy_attempts}/{self.max_proxy_attempts})"
+        else:
+            return f"direct phase ({self.direct_index}/{len(self.direct_cdn_order)})"
+
+
+class CDNManager:
+    """
+    Manages CDN configuration and provides failover strategies.
+
+    Coordinates between main proxy CDN and direct backend CDNs.
+    """
+
+    def __init__(
+        self,
+        proxy_url: str = "https://cdn.runonflux.io",
+        direct_urls: list[str] | None = None,
+    ):
+        """
+        Initialize CDN manager.
+
+        Args:
+            proxy_url: Main proxy CDN URL
+            direct_urls: List of direct backend CDN URLs (defaults to cdn-1/2/3)
+        """
+        self.proxy_url = proxy_url
+
+        if direct_urls is None:
+            # Default to standard Flux CDN backends
+            self.direct_urls = [
+                "https://cdn-1.runonflux.io",
+                "https://cdn-2.runonflux.io",
+                "https://cdn-3.runonflux.io",
+            ]
+        else:
+            self.direct_urls = direct_urls
+
+        logging.debug(
+            f"CDN Manager initialized: proxy={proxy_url}, "
+            f"direct={len(self.direct_urls)} backends"
+        )
+
+    def create_failover_strategy(self) -> CDNFailoverStrategy:
+        """
+        Create a new failover strategy for a part download.
+
+        Each part download gets its own independent strategy instance.
+
+        Returns:
+            New CDNFailoverStrategy instance
+        """
+        return CDNFailoverStrategy(self.proxy_url, self.direct_urls)
+
+    def get_proxy_url(self) -> str:
+        """
+        Get main proxy CDN URL.
+
+        Returns:
+            Proxy CDN URL
+        """
+        return self.proxy_url
+
+    def get_direct_urls(self) -> list[str]:
+        """
+        Get list of direct backend CDN URLs.
+
+        Returns:
+            List of backend CDN URLs
+        """
+        return self.direct_urls.copy()

flux_bootstrap-0.1.0/src/flux_bootstrap/data_structures.py

@@ -0,0 +1,50 @@
+"""Data structures and constants for flux-downloader."""
+
+from dataclasses import dataclass
+
+# Download and streaming constants
+DOWNLOAD_CHUNK_SIZE = 16 * 1024 * 1024  # 16MB chunks for HTTP streaming to disk
+FIFO_CHUNK_SIZE = 64 * 1024  # 64KB chunks for FIFO writing (tar needs smaller chunks)
+MAX_CONCURRENT_DOWNLOADS = 2  # Max concurrent part downloads
+MAX_DOWNLOAD_ATTEMPTS = 3  # Max attempts per download (1 initial + 2 retries)
+RETRY_DELAY_SECONDS = 5  # Delay between retry attempts
+MAX_UNVERIFIED_PARTS = 4  # Max parts sent to worker but not yet verified
+
+# Timeout configuration
+CONNECT_TIMEOUT_SECONDS = 10  # Connection timeout (reduced from 30 for faster CDN failover)
+SOCK_READ_TIMEOUT_SECONDS = 300  # Socket read timeout (5 minutes)
+
+# Speed monitoring for CDN failover
+MIN_SPEED_MBPS = 1.0  # Minimum acceptable speed in megabits per second
+SPEED_CHECK_WINDOW_SECONDS = 120  # Speed monitoring window (2 minutes)
+
+# File paths
+DEFAULT_API_URL = "https://cdn.runonflux.io/fluxd/api/latest_bootstrap"
+DEFAULT_CDN_URL = "https://cdn.runonflux.io"
+
+# CDN failover configuration
+DIRECT_CDN_URLS = [
+    "https://cdn-1.runonflux.io",
+    "https://cdn-2.runonflux.io",
+    "https://cdn-3.runonflux.io",
+]
+
+
+@dataclass
+class PartNotification:
+    """Notification sent from main process to worker when a part is complete."""
+
+    part_id: int  # Part number (0-indexed)
+    filepath: str  # Path to the downloaded part file
+    expected_sha256: str  # Expected SHA256 checksum
+    size: int  # Size in bytes
+    already_verified: bool = False  # True if part already verified (resume scenario)
+
+
+@dataclass
+class VerificationResult:
+    """Result sent from worker back to main process after SHA256 verification."""
+
+    part_id: int  # Part number (0-indexed)
+    verified: bool  # True if SHA256 matched, False otherwise
+    actual_sha256: str  # Actual SHA256 computed (for logging)