mapillary-downloader 0.4.2__tar.gz → 0.5.1__tar.gz

{mapillary_downloader-0.4.2 → mapillary_downloader-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.4.2
+Version: 0.5.1
 Summary: Download your Mapillary data before it's gone
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10

{mapillary_downloader-0.4.2 → mapillary_downloader-0.5.1}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mapillary_downloader"
 description = "Download your Mapillary data before it's gone"
-version = "0.4.2"
+version = "0.5.1"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

mapillary_downloader-0.5.1/src/mapillary_downloader/downloader.py

@@ -0,0 +1,471 @@
+"""Main downloader logic."""
+
+import gzip
+import json
+import logging
+import os
+import shutil
+import time
+from pathlib import Path
+from mapillary_downloader.utils import format_size, format_time
+from mapillary_downloader.ia_meta import generate_ia_metadata
+from mapillary_downloader.ia_check import check_ia_exists
+from mapillary_downloader.worker import worker_process
+from mapillary_downloader.worker_pool import AdaptiveWorkerPool
+from mapillary_downloader.metadata_reader import MetadataReader
+from mapillary_downloader.tar_sequences import tar_sequence_directories
+from mapillary_downloader.logging_config import add_file_handler
+
+logger = logging.getLogger("mapillary_downloader")
+
+
+def get_cache_dir():
+    """Get XDG cache directory for staging downloads.
+
+    Returns:
+        Path to cache directory for mapillary_downloader
+    """
+    xdg_cache = os.environ.get("XDG_CACHE_HOME")
+    if xdg_cache:
+        cache_dir = Path(xdg_cache)
+    else:
+        cache_dir = Path.home() / ".cache"
+
+    mapillary_cache = cache_dir / "mapillary_downloader"
+    mapillary_cache.mkdir(parents=True, exist_ok=True)
+    return mapillary_cache
+
+
+class MapillaryDownloader:
+    """Handles downloading Mapillary data for a user."""
+
+    def __init__(
+        self,
+        client,
+        output_dir,
+        username=None,
+        quality=None,
+        workers=None,
+        tar_sequences=True,
+        convert_webp=False,
+        check_ia=True,
+    ):
+        """Initialize the downloader.
+
+        Args:
+            client: MapillaryClient instance
+            output_dir: Base directory to save downloads (final destination)
+            username: Mapillary username (for collection directory)
+            quality: Image quality (for collection directory)
+            workers: Number of parallel workers (default: half of cpu_count)
+            tar_sequences: Whether to tar sequence directories after download (default: True)
+            convert_webp: Whether to convert images to WebP (affects collection name)
+            check_ia: Whether to check if collection exists on Internet Archive (default: True)
+        """
+        self.client = client
+        self.base_output_dir = Path(output_dir)
+        self.username = username
+        self.quality = quality
+        self.workers = workers if workers is not None else max(1, os.cpu_count() // 2)
+        self.tar_sequences = tar_sequences
+        self.convert_webp = convert_webp
+        self.check_ia = check_ia
+
+        # Determine collection name
+        if username and quality:
+            collection_name = f"mapillary-{username}-{quality}"
+            if convert_webp:
+                collection_name += "-webp"
+            self.collection_name = collection_name
+        else:
+            self.collection_name = None
+
+        # Set up staging directory in cache
+        cache_dir = get_cache_dir()
+        if self.collection_name:
+            self.staging_dir = cache_dir / self.collection_name
+            self.final_dir = self.base_output_dir / self.collection_name
+        else:
+            self.staging_dir = cache_dir / "download"
+            self.final_dir = self.base_output_dir
+
+        # Work in staging directory during download
+        self.output_dir = self.staging_dir
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        logger.info(f"Staging directory: {self.staging_dir}")
+        logger.info(f"Final destination: {self.final_dir}")
+
+        # Set up file logging for archival with timestamp for incremental runs
+        timestamp = time.strftime("%Y%m%d-%H%M%S")
+        log_file = self.output_dir / f"download.log.{timestamp}"
+        add_file_handler(log_file)
+        logger.info(f"Logging to: {log_file}")
+
+        self.metadata_file = self.output_dir / "metadata.jsonl"
+        self.progress_file = self.output_dir / "progress.json"
+        self.downloaded = self._load_progress()
+
+    def _load_progress(self):
+        """Load previously downloaded image IDs for this quality."""
+        if self.progress_file.exists():
+            with open(self.progress_file) as f:
+                data = json.load(f)
+                # Support both old format (single list) and new format (per-quality dict)
+                if isinstance(data, dict):
+                    if "downloaded" in data:
+                        # Old format: {"downloaded": [...]}
+                        return set(data["downloaded"])
+                    else:
+                        # New format: {"256": [...], "1024": [...], ...}
+                        return set(data.get(str(self.quality), []))
+                else:
+                    # Very old format: just a list
+                    return set(data)
+        return set()
+
+    def _save_progress(self):
+        """Save progress to disk atomically, per-quality."""
+        # Load existing progress for all qualities
+        if self.progress_file.exists():
+            with open(self.progress_file) as f:
+                data = json.load(f)
+                # Convert old format to new format if needed
+                if isinstance(data, dict) and "downloaded" in data:
+                    # Old format: {"downloaded": [...]} - migrate to per-quality
+                    progress = {}
+                else:
+                    progress = data if isinstance(data, dict) else {}
+        else:
+            progress = {}
+
+        # Update this quality's progress
+        progress[str(self.quality)] = list(self.downloaded)
+
+        # Write atomically
+        temp_file = self.progress_file.with_suffix(".json.tmp")
+        with open(temp_file, "w") as f:
+            json.dump(progress, f)
+            f.flush()
+            os.fsync(f.fileno())
+        temp_file.replace(self.progress_file)
+
+    def download_user_data(self, bbox=None, convert_webp=False):
+        """Download all images for a user using streaming queue-based architecture.
+
+        Args:
+            bbox: Optional bounding box [west, south, east, north]
+            convert_webp: Convert images to WebP format after download
+        """
+        if not self.username or not self.quality:
+            raise ValueError("Username and quality must be provided during initialization")
+
+        # Check if collection already exists on Internet Archive
+        if self.check_ia and self.collection_name:
+            logger.info(f"Checking if {self.collection_name} exists on Internet Archive...")
+            if check_ia_exists(self.collection_name):
+                logger.info("Collection already exists on archive.org, skipping download")
+                return
+
+        # Check if collection already exists in final destination
+        if self.final_dir.exists():
+            logger.info(f"Collection already exists at {self.final_dir}, skipping download")
+            return
+
+        quality_field = f"thumb_{self.quality}_url"
+
+        logger.info(f"Downloading images for user: {self.username}")
+        logger.info(f"Output directory: {self.output_dir}")
+        logger.info(f"Quality: {self.quality}")
+        logger.info(f"Using {self.workers} parallel workers")
+
+        start_time = time.time()
+
+        # Step 1: Check if API fetch is already complete
+        reader = MetadataReader(self.metadata_file)
+        api_complete = reader.is_complete
+        if api_complete:
+            logger.info("API fetch already complete, will only download")
+        else:
+            logger.info("API fetch incomplete, will fetch and download in parallel")
+
+        # Step 2: Start worker pool
+        # Since workers do both I/O (download) and CPU (WebP), need many more workers
+        # Cap at 128 for now - will build proper dynamic scaling on a new branch later
+        pool = AdaptiveWorkerPool(worker_process, min_workers=self.workers, max_workers=128, monitoring_interval=10)
+        pool.start()
+
+        # Step 3: Download images from metadata file while fetching new from API
+        downloaded_count = 0
+        total_bytes = 0
+        failed_count = 0
+        submitted = 0
+        batch_start = time.time()
+
+        logger.info("Starting parallel download and API fetch...")
+
+        try:
+            # Step 3a: Fetch metadata from API in parallel (write-only, don't block on queue)
+            if not api_complete:
+                import threading
+
+                api_fetch_complete = threading.Event()
+                new_images_count = [0]  # Mutable so thread can update it
+
+                def fetch_api_metadata():
+                    """Fetch metadata from API and write to file (runs in thread)."""
+                    try:
+                        logger.info("API fetch thread: Starting...")
+                        with open(self.metadata_file, "a") as meta_f:
+                            for image in self.client.get_user_images(self.username, bbox=bbox):
+                                new_images_count[0] += 1
+
+                                # Save metadata (don't dedupe here, let the tailer handle it)
+                                meta_f.write(json.dumps(image) + "\n")
+                                meta_f.flush()
+
+                                if new_images_count[0] % 1000 == 0:
+                                    logger.info(f"API: Fetched {new_images_count[0]} images from API")
+
+                            # Mark as complete
+                            MetadataReader.mark_complete(self.metadata_file)
+                            logger.info(f"API fetch complete: {new_images_count[0]} images")
+                    finally:
+                        api_fetch_complete.set()
+
+                # Start API fetch in background thread
+                api_thread = threading.Thread(target=fetch_api_metadata, daemon=True)
+                api_thread.start()
+            else:
+                logger.info("API fetch already complete, skipping API thread")
+                api_fetch_complete = None
+
+            # Step 3b: Tail metadata file and submit to workers
+            logger.info("Starting metadata tail and download queue feeder...")
+            last_position = 0
+
+            # Helper to process results from queue
+            def process_results():
+                nonlocal downloaded_count, total_bytes, failed_count
+                while True:
+                    result = pool.get_result(timeout=0.001)
+                    if result is None:
+                        break
+
+                    image_id, bytes_dl, success, error_msg = result
+
+                    if success:
+                        self.downloaded.add(image_id)
+                        downloaded_count += 1
+                        total_bytes += bytes_dl
+
+                        # Log every download for first 10, then every 100
+                        should_log = downloaded_count <= 10 or downloaded_count % 100 == 0
+                        if should_log:
+                            elapsed = time.time() - batch_start
+                            rate = downloaded_count / elapsed if elapsed > 0 else 0
+                            logger.info(
+                                f"Downloaded: {downloaded_count} ({format_size(total_bytes)}) "
+                                f"- Rate: {rate:.1f} images/sec"
+                            )
+
+                        if downloaded_count % 100 == 0:
+                            self._save_progress()
+                            pool.check_throughput(downloaded_count)
+                    else:
+                        failed_count += 1
+                        logger.warning(f"Failed to download {image_id}: {error_msg}")
+
+            # Tail the metadata file and submit to workers
+            while True:
+                # Check if API fetch is done and we've processed everything
+                if api_fetch_complete and api_fetch_complete.is_set():
+                    # Read any remaining lines
+                    if self.metadata_file.exists():
+                        with open(self.metadata_file) as f:
+                            f.seek(last_position)
+                            for line in f:
+                                line = line.strip()
+                                if not line:
+                                    continue
+
+                                try:
+                                    image = json.loads(line)
+                                except json.JSONDecodeError:
+                                    # Incomplete line, will retry
+                                    continue
+
+                                # Skip completion marker
+                                if image.get("__complete__"):
+                                    continue
+
+                                image_id = image.get("id")
+                                if not image_id:
+                                    continue
+
+                                # Skip if already downloaded or no quality URL
+                                if image_id in self.downloaded:
+                                    continue
+                                if not image.get(quality_field):
+                                    continue
+
+                                # Submit to workers
+                                work_item = (
+                                    image,
+                                    str(self.output_dir),
+                                    self.quality,
+                                    convert_webp,
+                                    self.client.access_token,
+                                )
+                                pool.submit(work_item)
+                                submitted += 1
+
+                                if submitted % 1000 == 0:
+                                    logger.info(f"Queue: Submitted {submitted} images")
+
+                                # Process results while submitting
+                                process_results()
+
+                            last_position = f.tell()
+
+                    # API done and all lines processed, break
+                    break
+
+                # API still running or API was already complete, tail the file
+                if self.metadata_file.exists():
+                    with open(self.metadata_file) as f:
+                        f.seek(last_position)
+                        for line in f:
+                            line = line.strip()
+                            if not line:
+                                continue
+
+                            try:
+                                image = json.loads(line)
+                            except json.JSONDecodeError:
+                                # Incomplete line, will retry next iteration
+                                continue
+
+                            # Skip completion marker
+                            if image.get("__complete__"):
+                                continue
+
+                            image_id = image.get("id")
+                            if not image_id:
+                                continue
+
+                            # Skip if already downloaded or no quality URL
+                            if image_id in self.downloaded:
+                                continue
+                            if not image.get(quality_field):
+                                continue
+
+                            # Submit to workers
+                            work_item = (
+                                image,
+                                str(self.output_dir),
+                                self.quality,
+                                convert_webp,
+                                self.client.access_token,
+                            )
+                            pool.submit(work_item)
+                            submitted += 1
+
+                            if submitted % 1000 == 0:
+                                logger.info(f"Queue: Submitted {submitted} images")
+
+                            # Process results while submitting
+                            process_results()
+
+                        last_position = f.tell()
+
+                # Sleep briefly before next tail iteration
+                time.sleep(0.1)
+
+                # Process any results that came in
+                process_results()
+
+            # Send shutdown signals
+            logger.info(f"Submitted {submitted} images, waiting for workers to finish...")
+            for _ in range(pool.current_workers):
+                pool.submit(None)
+
+            # Collect remaining results
+            completed = downloaded_count + failed_count
+
+            while completed < submitted:
+                result = pool.get_result(timeout=5)
+                if result is None:
+                    # Check throughput periodically
+                    pool.check_throughput(downloaded_count)
+                    continue
+
+                image_id, bytes_dl, success, error_msg = result
+                completed += 1
+
+                if success:
+                    self.downloaded.add(image_id)
+                    downloaded_count += 1
+                    total_bytes += bytes_dl
+
+                    if downloaded_count % 10 == 0:
+                        elapsed = time.time() - batch_start
+                        rate = downloaded_count / elapsed if elapsed > 0 else 0
+                        remaining = submitted - completed
+                        eta_seconds = remaining / rate if rate > 0 else 0
+
+                        logger.info(
+                            f"Downloaded: {downloaded_count}/{submitted} ({format_size(total_bytes)}) "
+                            f"- ETA: {format_time(eta_seconds)}"
+                        )
+                        self._save_progress()
+                        pool.check_throughput(downloaded_count)
+                else:
+                    failed_count += 1
+                    logger.warning(f"Failed to download {image_id}: {error_msg}")
+
+        finally:
+            # Shutdown worker pool
+            pool.shutdown()
+
+        self._save_progress()
+        elapsed = time.time() - start_time
+
+        logger.info(f"Complete! Downloaded {downloaded_count} ({format_size(total_bytes)}), " f"failed {failed_count}")
+        logger.info(f"Total time: {format_time(elapsed)}")
+
+        # Tar sequence directories for efficient IA uploads
+        if self.tar_sequences:
+            tar_sequence_directories(self.output_dir)
+
+        # Gzip metadata.jsonl to save space
+        if self.metadata_file.exists():
+            logger.info("Compressing metadata.jsonl...")
+            original_size = self.metadata_file.stat().st_size
+            gzipped_file = self.metadata_file.with_suffix(".jsonl.gz")
+
+            with open(self.metadata_file, "rb") as f_in:
+                with gzip.open(gzipped_file, "wb", compresslevel=9) as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+
+            compressed_size = gzipped_file.stat().st_size
+            self.metadata_file.unlink()
+
+            savings = 100 * (1 - compressed_size / original_size)
+            logger.info(
+                f"Compressed metadata: {format_size(original_size)} → {format_size(compressed_size)} "
+                f"({savings:.1f}% savings)"
+            )
+
+        # Generate IA metadata
+        generate_ia_metadata(self.output_dir)
+
+        # Move from staging to final destination
+        logger.info("Moving collection from staging to final destination...")
+        if self.final_dir.exists():
+            logger.warning(f"Destination already exists, removing: {self.final_dir}")
+            shutil.rmtree(self.final_dir)
+
+        self.final_dir.parent.mkdir(parents=True, exist_ok=True)
+        shutil.move(str(self.staging_dir), str(self.final_dir))
+        logger.info(f"Collection moved to: {self.final_dir}")

mapillary_downloader-0.5.1/src/mapillary_downloader/metadata_reader.py

@@ -0,0 +1,163 @@
+"""Streaming metadata reader with filtering."""
+
+import gzip
+import json
+import logging
+from pathlib import Path
+
+logger = logging.getLogger("mapillary_downloader")
+
+
+class MetadataReader:
+    """Streams metadata.jsonl line-by-line with filtering.
+
+    This avoids loading millions of image dicts into memory.
+    """
+
+    COMPLETION_MARKER = {"__complete__": True}
+
+    def __init__(self, metadata_file):
+        """Initialize metadata reader.
+
+        Args:
+            metadata_file: Path to metadata.jsonl or metadata.jsonl.gz
+        """
+        self.metadata_file = Path(metadata_file)
+        self.is_complete = self._check_complete()
+
+    def _check_complete(self):
+        """Check if metadata file has completion marker.
+
+        Returns:
+            True if completion marker found, False otherwise
+        """
+        if not self.metadata_file.exists():
+            return False
+
+        # Check last few lines for completion marker (it should be at the end)
+        try:
+            if self.metadata_file.suffix == ".gz":
+                file_handle = gzip.open(self.metadata_file, "rt")
+            else:
+                file_handle = open(self.metadata_file)
+
+            with file_handle as f:
+                # Read last 10 lines to find completion marker
+                lines = []
+                for line in f:
+                    lines.append(line)
+                    if len(lines) > 10:
+                        lines.pop(0)
+
+                # Check if any of the last lines is the completion marker
+                for line in reversed(lines):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        data = json.loads(line)
+                        if data.get("__complete__"):
+                            return True
+                    except json.JSONDecodeError:
+                        continue
+
+            return False
+        except Exception:
+            return False
+
+    def iter_images(self, quality_field=None, downloaded_ids=None):
+        """Stream images from metadata file with filtering.
+
+        Args:
+            quality_field: Optional field to check exists (e.g., 'thumb_1024_url')
+            downloaded_ids: Optional set of already downloaded IDs to skip
+
+        Yields:
+            Image metadata dicts that pass filters
+        """
+        if not self.metadata_file.exists():
+            return
+
+        # Handle gzipped files
+        if self.metadata_file.suffix == ".gz":
+            file_handle = gzip.open(self.metadata_file, "rt")
+        else:
+            file_handle = open(self.metadata_file)
+
+        with file_handle as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+
+                image = json.loads(line)
+
+                # Check for completion marker
+                if image.get("__complete__"):
+                    self.is_complete = True
+                    logger.debug("Found API fetch completion marker")
+                    continue
+
+                image_id = image.get("id")
+                if not image_id:
+                    continue
+
+                # Filter by downloaded status
+                if downloaded_ids and image_id in downloaded_ids:
+                    continue
+
+                # Filter by quality field availability
+                if quality_field and not image.get(quality_field):
+                    continue
+
+                yield image
+
+    def get_all_ids(self):
+        """Get set of all image IDs in metadata file.
+
+        Returns:
+            Set of image IDs (for building seen_ids)
+        """
+        ids = set()
+
+        if not self.metadata_file.exists():
+            return ids
+
+        # Handle gzipped files
+        if self.metadata_file.suffix == ".gz":
+            file_handle = gzip.open(self.metadata_file, "rt")
+        else:
+            file_handle = open(self.metadata_file)
+
+        with file_handle as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+
+                image = json.loads(line)
+
+                # Skip completion marker
+                if image.get("__complete__"):
+                    self.is_complete = True
+                    continue
+
+                image_id = image.get("id")
+                if image_id:
+                    ids.add(image_id)
+
+        return ids
+
+    @staticmethod
+    def mark_complete(metadata_file):
+        """Append completion marker to metadata file.
+
+        Args:
+            metadata_file: Path to metadata.jsonl
+        """
+        metadata_file = Path(metadata_file)
+        if metadata_file.exists():
+            with open(metadata_file, "a") as f:
+                f.write(json.dumps(MetadataReader.COMPLETION_MARKER) + "\n")
+                f.flush()
+            logger.info("Marked metadata file as complete")

{mapillary_downloader-0.4.2 → mapillary_downloader-0.5.1}/src/mapillary_downloader/worker.py

@@ -9,7 +9,38 @@ from mapillary_downloader.exif_writer import write_exif_to_image
 from mapillary_downloader.webp_converter import convert_to_webp
 
 
-def download_and_convert_image(image_data, output_dir, quality, convert_webp, access_token):
+def worker_process(work_queue, result_queue, worker_id):
+    """Worker process that pulls from queue and processes images.
+
+    Args:
+        work_queue: Queue to pull work items from
+        result_queue: Queue to push results to
+        worker_id: Unique worker identifier
+    """
+    # Create session once per worker (reuse HTTP connections)
+    session = requests.Session()
+
+    while True:
+        work_item = work_queue.get()
+
+        # None is the shutdown signal
+        if work_item is None:
+            break
+
+        # Unpack work item
+        image_data, output_dir, quality, convert_webp, access_token = work_item
+
+        # Update session auth for this request
+        session.headers.update({"Authorization": f"OAuth {access_token}"})
+
+        # Process the image
+        result = download_and_convert_image(image_data, output_dir, quality, convert_webp, session)
+
+        # Push result back
+        result_queue.put(result)
+
+
+def download_and_convert_image(image_data, output_dir, quality, convert_webp, session):
     """Download and optionally convert a single image.
 
     This function is designed to run in a worker process.
@@ -19,7 +50,7 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, ac
         output_dir: Base output directory path
         quality: Quality level (256, 1024, 2048, original)
         convert_webp: Whether to convert to WebP
-        access_token: Mapillary API access token
+        session: requests.Session with auth already configured
 
     Returns:
         Tuple of (image_id, bytes_downloaded, success, error_msg)
@@ -53,11 +84,7 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, ac
             jpg_path = img_dir / f"{image_id}.jpg"
             final_path = jpg_path
 
-        # Download image
-        # No retries for CDN images - they're cheap, just skip failures and move on
-        session = requests.Session()
-        session.headers.update({"Authorization": f"OAuth {access_token}"})
-
+        # Download image (using session passed from worker)
         bytes_downloaded = 0
 
         try:

mapillary_downloader-0.5.1/src/mapillary_downloader/worker_pool.py

@@ -0,0 +1,204 @@
+"""Adaptive worker pool for parallel processing."""
+
+import logging
+import multiprocessing as mp
+import queue
+import time
+from collections import deque
+
+logger = logging.getLogger("mapillary_downloader")
+
+
+class AdaptiveWorkerPool:
+    """Worker pool that scales based on throughput.
+
+    Monitors throughput every 30 seconds and adjusts worker count:
+    - If throughput increasing: add workers (up to max)
+    - If throughput plateauing/decreasing: reduce workers
+    """
+
+    def __init__(self, worker_func, min_workers=4, max_workers=16, monitoring_interval=10):
+        """Initialize adaptive worker pool.
+
+        Args:
+            worker_func: Function to run in each worker (must accept work_queue, result_queue)
+            min_workers: Minimum number of workers
+            max_workers: Maximum number of workers
+            monitoring_interval: Seconds between throughput checks
+        """
+        self.worker_func = worker_func
+        self.min_workers = min_workers
+        self.max_workers = max_workers
+        self.monitoring_interval = monitoring_interval
+
+        # Queues
+        self.work_queue = mp.Queue(maxsize=max_workers)
+        self.result_queue = mp.Queue()
+
+        # Worker management
+        self.workers = []
+        self.current_workers = min_workers  # Start small and ramp up
+
+        # Throughput monitoring
+        self.throughput_history = deque(maxlen=5)  # Last 5 measurements
+        self.worker_count_history = deque(maxlen=5)  # Track worker counts at each measurement
+        self.last_processed = 0
+        self.last_check_time = time.time()
+
+        self.running = False
+
+    def start(self):
+        """Start the worker pool."""
+        self.running = True
+        logger.info(f"Starting worker pool with {self.current_workers} workers")
+
+        for i in range(self.current_workers):
+            self._add_worker(i)
+
+    def _add_worker(self, worker_id):
+        """Add a new worker to the pool."""
+        p = mp.Process(target=self.worker_func, args=(self.work_queue, self.result_queue, worker_id))
+        p.start()
+        self.workers.append(p)
+        logger.debug(f"Started worker {worker_id}")
+
+    def submit(self, work_item):
+        """Submit work to the pool (blocks if queue is full)."""
+        self.work_queue.put(work_item)
+
+    def get_result(self, timeout=None):
+        """Get a result from the workers.
+
+        Returns:
+            Result from worker, or None if timeout
+        """
+        try:
+            return self.result_queue.get(timeout=timeout)
+        except queue.Empty:
+            return None
+
+    def check_throughput(self, total_processed):
+        """Check throughput and adjust workers if needed.
+
+        Args:
+            total_processed: Total number of items processed so far
+        """
+        now = time.time()
+        elapsed = now - self.last_check_time
+
+        if elapsed < self.monitoring_interval:
+            logger.debug(f"Throughput check skipped (elapsed {elapsed:.1f}s < {self.monitoring_interval}s)")
+            return
+
+        # Calculate current throughput (items/sec)
+        items_since_check = total_processed - self.last_processed
+        throughput = items_since_check / elapsed
+
+        current_workers = len(self.workers)
+        self.throughput_history.append(throughput)
+        self.worker_count_history.append(current_workers)
+        self.last_processed = total_processed
+        self.last_check_time = now
+
+        logger.info(
+            f"Throughput: {throughput:.1f} items/s (workers: {current_workers}/{self.max_workers}, "
+            f"history: {len(self.throughput_history)} measurements)"
+        )
+
+        # Need at least 2 measurements to calculate gain per worker
+        if len(self.throughput_history) < 2:
+            # First measurement - add 20% more workers
+            if current_workers < self.max_workers:
+                workers_to_add = max(1, int(current_workers * 0.2))
+                for i in range(workers_to_add):
+                    if len(self.workers) < self.max_workers:
+                        new_worker_id = len(self.workers)
+                        self._add_worker(new_worker_id)
+                        self.current_workers += 1
+                logger.info(
+                    f"Ramping up: added {workers_to_add} workers (now {self.current_workers}/{self.max_workers})"
+                )
+            return
+
+        # Calculate throughput gain per worker added
+        current_throughput = self.throughput_history[-1]
+        previous_throughput = self.throughput_history[-2]
+        previous_workers = self.worker_count_history[-2]
+
+        throughput_gain = current_throughput - previous_throughput
+        workers_added = current_workers - previous_workers
+
+        logger.debug(
+            f"Trend: {previous_throughput:.1f} items/s @ {previous_workers} workers → "
+            f"{current_throughput:.1f} items/s @ {current_workers} workers "
+            f"(gain: {throughput_gain:.1f}, added: {workers_added})"
+        )
+
+        # If throughput decreased significantly, stop adding workers
+        if current_throughput < previous_throughput * 0.95:
+            logger.info(
+                f"Throughput decreasing ({current_throughput:.1f} vs {previous_throughput:.1f} items/s), "
+                f"stopping at {current_workers} workers"
+            )
+        # If throughput is still increasing or stable, add more workers
+        elif current_throughput >= previous_throughput * 0.95 and current_workers < self.max_workers:
+            if workers_added > 0 and throughput_gain > 0:
+                # Calculate gain per worker
+                gain_per_worker = throughput_gain / workers_added
+                logger.debug(f"Gain per worker: {gain_per_worker:.2f} items/s")
+
+                # Estimate how many more workers we could benefit from
+                # Assume diminishing returns, so be conservative
+                if gain_per_worker > 0.5:
+                    # Good gain per worker - add more aggressively
+                    workers_to_add = max(1, int(current_workers * 0.3))
+                elif gain_per_worker > 0.2:
+                    # Moderate gain - add moderately
+                    workers_to_add = max(1, int(current_workers * 0.2))
+                else:
+                    # Small gain - add conservatively
+                    workers_to_add = max(1, int(current_workers * 0.1))
+
+                added = 0
+                for i in range(workers_to_add):
+                    if len(self.workers) < self.max_workers:
+                        new_worker_id = len(self.workers)
+                        self._add_worker(new_worker_id)
+                        self.current_workers += 1
+                        added += 1
+
+                logger.info(
+                    f"Throughput increasing (gain: {gain_per_worker:.2f} items/s per worker), "
+                    f"added {added} workers (now {self.current_workers}/{self.max_workers})"
+                )
+            else:
+                # Fallback to 20% if we can't calculate gain per worker
+                workers_to_add = max(1, int(current_workers * 0.2))
+                added = 0
+                for i in range(workers_to_add):
+                    if len(self.workers) < self.max_workers:
+                        new_worker_id = len(self.workers)
+                        self._add_worker(new_worker_id)
+                        self.current_workers += 1
+                        added += 1
+                logger.info(f"Ramping up: added {added} workers (now {self.current_workers}/{self.max_workers})")
+        else:
+            logger.info(f"At optimal worker count: {current_workers} workers, {current_throughput:.1f} items/s")
+
+    def shutdown(self, timeout=30):
+        """Shutdown the worker pool gracefully."""
+        logger.info("Shutting down worker pool...")
+        self.running = False
+
+        # Send stop signals
+        for _ in self.workers:
+            self.work_queue.put(None)
+
+        # Wait for workers to finish
+        for p in self.workers:
+            p.join(timeout=timeout)
+            if p.is_alive():
+                logger.warning(f"Worker {p.pid} did not exit cleanly, terminating")
+                p.terminate()
+
+        logger.info("Worker pool shutdown complete")

mapillary_downloader-0.4.2/src/mapillary_downloader/downloader.py

@@ -1,326 +0,0 @@
-"""Main downloader logic."""
-
-import gzip
-import json
-import logging
-import os
-import shutil
-import time
-from pathlib import Path
-from concurrent.futures import ProcessPoolExecutor, as_completed
-from mapillary_downloader.utils import format_size, format_time
-from mapillary_downloader.ia_meta import generate_ia_metadata
-from mapillary_downloader.ia_check import check_ia_exists
-from mapillary_downloader.worker import download_and_convert_image
-from mapillary_downloader.tar_sequences import tar_sequence_directories
-from mapillary_downloader.logging_config import add_file_handler
-
-logger = logging.getLogger("mapillary_downloader")
-
-
-def get_cache_dir():
-    """Get XDG cache directory for staging downloads.
-
-    Returns:
-        Path to cache directory for mapillary_downloader
-    """
-    xdg_cache = os.environ.get("XDG_CACHE_HOME")
-    if xdg_cache:
-        cache_dir = Path(xdg_cache)
-    else:
-        cache_dir = Path.home() / ".cache"
-
-    mapillary_cache = cache_dir / "mapillary_downloader"
-    mapillary_cache.mkdir(parents=True, exist_ok=True)
-    return mapillary_cache
-
-
-class MapillaryDownloader:
-    """Handles downloading Mapillary data for a user."""
-
-    def __init__(
-        self,
-        client,
-        output_dir,
-        username=None,
-        quality=None,
-        workers=None,
-        tar_sequences=True,
-        convert_webp=False,
-        check_ia=True,
-    ):
-        """Initialize the downloader.
-
-        Args:
-            client: MapillaryClient instance
-            output_dir: Base directory to save downloads (final destination)
-            username: Mapillary username (for collection directory)
-            quality: Image quality (for collection directory)
-            workers: Number of parallel workers (default: half of cpu_count)
-            tar_sequences: Whether to tar sequence directories after download (default: True)
-            convert_webp: Whether to convert images to WebP (affects collection name)
-            check_ia: Whether to check if collection exists on Internet Archive (default: True)
-        """
-        self.client = client
-        self.base_output_dir = Path(output_dir)
-        self.username = username
-        self.quality = quality
-        self.workers = workers if workers is not None else max(1, os.cpu_count() // 2)
-        self.tar_sequences = tar_sequences
-        self.convert_webp = convert_webp
-        self.check_ia = check_ia
-
-        # Determine collection name
-        if username and quality:
-            collection_name = f"mapillary-{username}-{quality}"
-            if convert_webp:
-                collection_name += "-webp"
-            self.collection_name = collection_name
-        else:
-            self.collection_name = None
-
-        # Set up staging directory in cache
-        cache_dir = get_cache_dir()
-        if self.collection_name:
-            self.staging_dir = cache_dir / self.collection_name
-            self.final_dir = self.base_output_dir / self.collection_name
-        else:
-            self.staging_dir = cache_dir / "download"
-            self.final_dir = self.base_output_dir
-
-        # Work in staging directory during download
-        self.output_dir = self.staging_dir
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-
-        logger.info(f"Staging directory: {self.staging_dir}")
-        logger.info(f"Final destination: {self.final_dir}")
-
-        # Set up file logging for archival with timestamp for incremental runs
-        timestamp = time.strftime("%Y%m%d-%H%M%S")
-        log_file = self.output_dir / f"download.log.{timestamp}"
-        add_file_handler(log_file)
-        logger.info(f"Logging to: {log_file}")
-
-        self.metadata_file = self.output_dir / "metadata.jsonl"
-        self.progress_file = self.output_dir / "progress.json"
-        self.downloaded = self._load_progress()
-
-    def _load_progress(self):
-        """Load previously downloaded image IDs."""
-        if self.progress_file.exists():
-            with open(self.progress_file) as f:
-                return set(json.load(f).get("downloaded", []))
-        return set()
-
-    def _save_progress(self):
-        """Save progress to disk atomically."""
-        temp_file = self.progress_file.with_suffix(".json.tmp")
-        with open(temp_file, "w") as f:
-            json.dump({"downloaded": list(self.downloaded)}, f)
-            f.flush()
-            os.fsync(f.fileno())
-        temp_file.replace(self.progress_file)
-
-    def download_user_data(self, bbox=None, convert_webp=False):
-        """Download all images for a user.
-
-        Args:
-            bbox: Optional bounding box [west, south, east, north]
-            convert_webp: Convert images to WebP format after download
-        """
-        if not self.username or not self.quality:
-            raise ValueError("Username and quality must be provided during initialization")
-
-        # Check if collection already exists on Internet Archive
-        if self.check_ia and self.collection_name:
-            logger.info(f"Checking if {self.collection_name} exists on Internet Archive...")
-            if check_ia_exists(self.collection_name):
-                logger.info("Collection already exists on archive.org, skipping download")
-                return
-
-        # Check if collection already exists in final destination
-        if self.final_dir.exists():
-            logger.info(f"Collection already exists at {self.final_dir}, skipping download")
-            return
-
-        quality_field = f"thumb_{self.quality}_url"
-
-        logger.info(f"Downloading images for user: {self.username}")
-        logger.info(f"Output directory: {self.output_dir}")
-        logger.info(f"Quality: {self.quality}")
-        logger.info(f"Using {self.workers} parallel workers")
-
-        processed = 0
-        downloaded_count = 0
-        skipped = 0
-        total_bytes = 0
-        failed_count = 0
-
-        start_time = time.time()
-
-        # Track which image IDs we've seen in metadata to avoid re-fetching
-        seen_ids = set()
-
-        # Collect images to download from existing metadata
-        images_to_download = []
-
-        if self.metadata_file.exists():
-            logger.info("Processing existing metadata file...")
-            with open(self.metadata_file) as f:
-                for line in f:
-                    if line.strip():
-                        image = json.loads(line)
-                        image_id = image["id"]
-                        seen_ids.add(image_id)
-                        processed += 1
-
-                        if image_id in self.downloaded:
-                            skipped += 1
-                            continue
-
-                        # Queue for download
-                        if image.get(quality_field):
-                            images_to_download.append(image)
-
-        # Download images from existing metadata in parallel
-        if images_to_download:
-            logger.info(f"Downloading {len(images_to_download)} images from existing metadata...")
-            downloaded_count, total_bytes, failed_count = self._download_images_parallel(
-                images_to_download, convert_webp
-            )
-
-        # Always check API for new images (will skip duplicates via seen_ids)
-        logger.info("Checking for new images from API...")
-        new_images = []
-
-        with open(self.metadata_file, "a") as meta_f:
-            for image in self.client.get_user_images(self.username, bbox=bbox):
-                image_id = image["id"]
-
-                # Skip if we already have this in our metadata file
-                if image_id in seen_ids:
-                    continue
-
-                seen_ids.add(image_id)
-                processed += 1
-
-                # Save new metadata
-                meta_f.write(json.dumps(image) + "\n")
-                meta_f.flush()
-
-                # Skip if already downloaded
-                if image_id in self.downloaded:
-                    skipped += 1
-                    continue
-
-                # Queue for download
-                if image.get(quality_field):
-                    new_images.append(image)
-
-        # Download new images in parallel
-        if new_images:
-            logger.info(f"Downloading {len(new_images)} new images...")
-            new_downloaded, new_bytes, new_failed = self._download_images_parallel(new_images, convert_webp)
-            downloaded_count += new_downloaded
-            total_bytes += new_bytes
-            failed_count += new_failed
-
-        self._save_progress()
-        elapsed = time.time() - start_time
-        logger.info(
-            f"Complete! Processed {processed} images, downloaded {downloaded_count} ({format_size(total_bytes)}), "
-            f"skipped {skipped}, failed {failed_count}"
-        )
-        logger.info(f"Total time: {format_time(elapsed)}")
-
-        # Tar sequence directories for efficient IA uploads
-        if self.tar_sequences:
-            tar_sequence_directories(self.output_dir)
-
-        # Gzip metadata.jsonl to save space
-        if self.metadata_file.exists():
-            logger.info("Compressing metadata.jsonl...")
-            original_size = self.metadata_file.stat().st_size
-            gzipped_file = self.metadata_file.with_suffix(".jsonl.gz")
-
-            with open(self.metadata_file, "rb") as f_in:
-                with gzip.open(gzipped_file, "wb", compresslevel=9) as f_out:
-                    shutil.copyfileobj(f_in, f_out)
-
-            compressed_size = gzipped_file.stat().st_size
-            self.metadata_file.unlink()
-
-            savings = 100 * (1 - compressed_size / original_size)
-            logger.info(
-                f"Compressed metadata: {format_size(original_size)} → {format_size(compressed_size)} "
-                f"({savings:.1f}% savings)"
-            )
-
-        # Generate IA metadata
-        generate_ia_metadata(self.output_dir)
-
-        # Move from staging to final destination
-        logger.info("Moving collection from staging to final destination...")
-        if self.final_dir.exists():
-            logger.warning(f"Destination already exists, removing: {self.final_dir}")
-            shutil.rmtree(self.final_dir)
-
-        self.final_dir.parent.mkdir(parents=True, exist_ok=True)
-        shutil.move(str(self.staging_dir), str(self.final_dir))
-        logger.info(f"Collection moved to: {self.final_dir}")
-
-    def _download_images_parallel(self, images, convert_webp):
-        """Download images in parallel using worker pool.
-
-        Args:
-            images: List of image metadata dicts
-            convert_webp: Whether to convert to WebP
-
-        Returns:
-            Tuple of (downloaded_count, total_bytes, failed_count)
-        """
-        downloaded_count = 0
-        total_bytes = 0
-        failed_count = 0
-        batch_start_time = time.time()
-
-        with ProcessPoolExecutor(max_workers=self.workers) as executor:
-            # Submit all tasks
-            future_to_image = {}
-            for image in images:
-                future = executor.submit(
-                    download_and_convert_image,
-                    image,
-                    str(self.output_dir),
-                    self.quality,
-                    convert_webp,
-                    self.client.access_token,
-                )
-                future_to_image[future] = image["id"]
-
-            # Process results as they complete
-            for future in as_completed(future_to_image):
-                image_id, bytes_dl, success, error_msg = future.result()
-
-                if success:
-                    self.downloaded.add(image_id)
-                    downloaded_count += 1
-                    total_bytes += bytes_dl
-
-                    if downloaded_count % 10 == 0:
-                        # Calculate ETA
-                        elapsed = time.time() - batch_start_time
-                        rate = downloaded_count / elapsed if elapsed > 0 else 0
-                        remaining = len(images) - downloaded_count
-                        eta_seconds = remaining / rate if rate > 0 else 0
-
-                        logger.info(
-                            f"Downloaded: {downloaded_count}/{len(images)} ({format_size(total_bytes)}) "
-                            f"- ETA: {format_time(eta_seconds)}"
-                        )
-                        self._save_progress()
-                else:
-                    failed_count += 1
-                    logger.warning(f"Failed to download {image_id}: {error_msg}")
-
-        return downloaded_count, total_bytes, failed_count