mapillary-downloader 0.6.0__tar.gz → 0.7.0__tar.gz

{mapillary_downloader-0.6.0 → mapillary_downloader-0.7.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.6.0
-Summary: Download your Mapillary data before it's gone
+Version: 0.7.0
+Summary: Archive user data from Mapillary
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -100,21 +100,28 @@ mapillary-downloader --no-webp USERNAME
 
 ## Tarballs
 
-Images are organized by sequence ID, bucketed by the first character of the
-sequence to reduce directory count:
+Images are organized by capture date (YYYY-MM-DD) for incremental archiving:
 
 ```
 mapillary-username-quality/
-  a/
+  2024-01-15/
     abc123/
       image1.webp
       image2.webp
+    bcd456/
+      image3.webp
+  2024-01-16/
+    def789/
+      image4.webp
 ```
 
-By default, these bucket directories are automatically tarred after download
-(resulting in `a.tar`, `b.tar`, etc. - about 62 tar files total). This is done
-because large collections with millions of images would otherwise create hundreds
-of thousands of tiny tars, and anger the archive gods.
+By default, these date directories are automatically tarred after download
+(resulting in `2024-01-15.tar`, `2024-01-16.tar`, etc.). This date-based
+organization enables:
+
+- **Incremental uploads** - Upload each day's tar as soon as it's ready
+- **Manageable file counts** - ~365 days/year × 10 years = 3,650 tars max
+- **Chronological organization** - Natural sorting and progress tracking
 
 To keep individual files instead of creating tars, use the `--no-tar` flag.
 
@@ -128,8 +135,15 @@ See inlay for details:
 
 * [📀 rip](https://bitplane.net/dev/sh/rip)
 
+## 📊 Stats
+
+To see overall project progress, or an estimate, use `--stats`
+
+```bash
+mapillary-downloader --stats
+```
 
-## Development
+## 🚧 Development
 
 ```bash
 make dev      # Setup dev environment
@@ -138,7 +152,7 @@ make dist     # Build the distribution
 make help     # See other make options
 ```
 
-## Links
+## 🔗 Links
 
 * [🏠 home](https://bitplane.net/dev/python/mapillary_downloader)
   * [📖 pydoc](https://bitplane.net/dev/python/mapillary_downloader/pydoc)

{mapillary_downloader-0.6.0 → mapillary_downloader-0.7.0}/README.md

@@ -70,21 +70,28 @@ mapillary-downloader --no-webp USERNAME
 
 ## Tarballs
 
-Images are organized by sequence ID, bucketed by the first character of the
-sequence to reduce directory count:
+Images are organized by capture date (YYYY-MM-DD) for incremental archiving:
 
 ```
 mapillary-username-quality/
-  a/
+  2024-01-15/
     abc123/
       image1.webp
       image2.webp
+    bcd456/
+      image3.webp
+  2024-01-16/
+    def789/
+      image4.webp
 ```
 
-By default, these bucket directories are automatically tarred after download
-(resulting in `a.tar`, `b.tar`, etc. - about 62 tar files total). This is done
-because large collections with millions of images would otherwise create hundreds
-of thousands of tiny tars, and anger the archive gods.
+By default, these date directories are automatically tarred after download
+(resulting in `2024-01-15.tar`, `2024-01-16.tar`, etc.). This date-based
+organization enables:
+
+- **Incremental uploads** - Upload each day's tar as soon as it's ready
+- **Manageable file counts** - ~365 days/year × 10 years = 3,650 tars max
+- **Chronological organization** - Natural sorting and progress tracking
 
 To keep individual files instead of creating tars, use the `--no-tar` flag.
 
@@ -98,8 +105,15 @@ See inlay for details:
 
 * [📀 rip](https://bitplane.net/dev/sh/rip)
 
+## 📊 Stats
+
+To see overall project progress, or an estimate, use `--stats`
+
+```bash
+mapillary-downloader --stats
+```
 
-## Development
+## 🚧 Development
 
 ```bash
 make dev      # Setup dev environment
@@ -108,7 +122,7 @@ make dist     # Build the distribution
 make help     # See other make options
 ```
 
-## Links
+## 🔗 Links
 
 * [🏠 home](https://bitplane.net/dev/python/mapillary_downloader)
   * [📖 pydoc](https://bitplane.net/dev/python/mapillary_downloader/pydoc)

{mapillary_downloader-0.6.0 → mapillary_downloader-0.7.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mapillary_downloader"
-description = "Download your Mapillary data before it's gone"
-version = "0.6.0"
+description = "Archive user data from Mapillary"
+version = "0.7.0"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

{mapillary_downloader-0.6.0 → mapillary_downloader-0.7.0}/src/mapillary_downloader/__main__.py

@@ -26,7 +26,7 @@ def main():
         default=os.environ.get("MAPILLARY_TOKEN"),
         help="Mapillary API access token (or set MAPILLARY_TOKEN env var)",
     )
-    parser.add_argument("usernames", nargs="+", help="Mapillary username(s) to download")
+    parser.add_argument("usernames", nargs="*", help="Mapillary username(s) to download")
     parser.add_argument("--output", default="./mapillary_data", help="Output directory (default: ./mapillary_data)")
     parser.add_argument(
         "--quality",
@@ -61,9 +61,21 @@ def main():
         action="store_true",
         help="Enable debug logging (EXIF data, API responses, etc.)",
     )
+    parser.add_argument(
+        "--stats",
+        action="store_true",
+        help="Show statistics of collections on archive.org and exit",
+    )
 
     args = parser.parse_args()
 
+    # Handle --stats early (before token check)
+    if args.stats:
+        from mapillary_downloader.ia_stats import show_stats
+
+        show_stats()
+        sys.exit(0)
+
     # Set debug logging level if requested
     if args.debug:
         import logging
@@ -71,6 +83,11 @@ def main():
         logging.getLogger("mapillary_downloader").setLevel(logging.DEBUG)
         logger.debug("Debug logging enabled")
 
+    # Check for usernames (required unless using --stats)
+    if not args.usernames:
+        logger.error("Error: At least one username is required")
+        sys.exit(1)
+
     # Check for token
     if not args.token:
         logger.error("Error: Mapillary API token required. Use --token or set MAPILLARY_TOKEN environment variable")

{mapillary_downloader-0.6.0 → mapillary_downloader-0.7.0}/src/mapillary_downloader/downloader.py

@@ -7,7 +7,7 @@ import os
 import shutil
 import time
 from pathlib import Path
-from mapillary_downloader.utils import format_size, format_time
+from mapillary_downloader.utils import format_size, format_time, safe_json_save
 from mapillary_downloader.ia_meta import generate_ia_metadata
 from mapillary_downloader.ia_check import check_ia_exists
 from mapillary_downloader.worker import worker_process
@@ -143,13 +143,8 @@ class MapillaryDownloader:
         # Update this quality's progress
         progress[str(self.quality)] = list(self.downloaded)
 
-        # Write atomically
-        temp_file = self.progress_file.with_suffix(".json.tmp")
-        with open(temp_file, "w") as f:
-            json.dump(progress, f)
-            f.flush()
-            os.fsync(f.fileno())
-        temp_file.replace(self.progress_file)
+        # Write atomically using utility function
+        safe_json_save(self.progress_file, progress)
 
     def download_user_data(self, bbox=None, convert_webp=False):
         """Download all images for a user using streaming queue-based architecture.

mapillary_downloader-0.7.0/src/mapillary_downloader/ia_stats.py

@@ -0,0 +1,242 @@
+"""Internet Archive statistics for mapillary_downloader collections."""
+
+import json
+import logging
+import re
+from mapillary_downloader.utils import safe_json_save, http_get_with_retry, format_size
+from mapillary_downloader.downloader import get_cache_dir
+
+logger = logging.getLogger("mapillary_downloader")
+
+CACHE_FILE = get_cache_dir() / ".stats.json"
+
+
+def search_ia_collections():
+    """Search IA for all mapillary_downloader collections.
+
+    Returns:
+        List of dicts with: identifier, description, item_size, uploader
+    """
+    logger.info("Searching archive.org for mapillary_downloader collections...")
+
+    url = "https://archive.org/advancedsearch.php"
+    params = {
+        "q": "mapillary_downloader:*",
+        "fl[]": ["identifier", "description", "item_size", "uploader"],
+        "rows": 10000,
+        "output": "json",
+    }
+
+    response = http_get_with_retry(url, params=params, max_retries=3)
+    data = response.json()
+
+    collections = data["response"]["docs"]
+    logger.info(f"Found {len(collections)} collections on archive.org")
+
+    return collections
+
+
+def parse_collection_info(identifier):
+    """Parse username, quality, webp from collection identifier.
+
+    Returns:
+        dict with username, quality, is_webp or None if invalid
+    """
+    match = re.match(r"mapillary-(.+)-(256|1024|2048|original)(?:-webp)?$", identifier)
+    if match:
+        return {"username": match.group(1), "quality": match.group(2), "is_webp": "-webp" in identifier}
+    return None
+
+
+def extract_image_count(description):
+    """Extract image count from IA description field.
+
+    Description format: "Contains 12,345 images in..."
+    """
+    if not description:
+        return None
+
+    match = re.search(r"Contains ([\d,]+) images", description)
+    if match:
+        return int(match.group(1).replace(",", ""))
+    return None
+
+
+def load_cache():
+    """Load cached collection data.
+
+    Returns:
+        dict of {collection_id: {size, uploader, images, quality, username}}
+    """
+    if CACHE_FILE.exists():
+        try:
+            with open(CACHE_FILE) as f:
+                return json.load(f)
+        except Exception as e:
+            logger.warning(f"Failed to load cache: {e}")
+    return {}
+
+
+def update_cache(ia_collections):
+    """Update cache with new IA search results.
+
+    Merges new collections into existing cache.
+
+    Returns:
+        Updated cache dict
+    """
+    cache = load_cache()
+
+    for item in ia_collections:
+        identifier = item.get("identifier")
+        if not identifier:
+            continue
+
+        info = parse_collection_info(identifier)
+        if not info:
+            logger.debug(f"Skipping non-mapillary collection: {identifier}")
+            continue
+
+        # Parse item data
+        size_bytes = item.get("item_size", 0)
+        if isinstance(size_bytes, str):
+            size_bytes = int(size_bytes)
+
+        image_count = extract_image_count(item.get("description"))
+
+        # Update cache entry
+        cache[identifier] = {
+            "size": size_bytes,
+            "uploader": item.get("uploader"),
+            "images": image_count,
+            "quality": info["quality"],
+            "username": info["username"],
+            "is_webp": info["is_webp"],
+        }
+
+    # Save updated cache
+    safe_json_save(CACHE_FILE, cache)
+    logger.info(f"Updated cache with {len(cache)} collections")
+
+    return cache
+
+
+def aggregate_stats(cache):
+    """Aggregate statistics from cached collection data.
+
+    Returns:
+        dict with total and per-quality stats
+    """
+    stats = {
+        "total": {"collections": 0, "total_images": 0, "unique_images": 0, "bytes": 0},
+        "by_quality": {},
+        "users": set(),
+    }
+
+    # Track images per user for deduplication
+    user_images = {}  # {username: max_images_across_qualities}
+
+    for collection_id, data in cache.items():
+        images = data.get("images") or 0
+        size = data.get("size") or 0
+        quality = data.get("quality", "unknown")
+        username = data.get("username")
+
+        # Track user coverage
+        if username:
+            stats["users"].add(username)
+            # Keep maximum image count across all qualities for this user
+            if username not in user_images or images > user_images[username]:
+                user_images[username] = images
+
+        # Total stats (collections, total images, and bytes)
+        stats["total"]["collections"] += 1
+        stats["total"]["total_images"] += images
+        stats["total"]["bytes"] += size
+
+        # Per-quality stats
+        if quality not in stats["by_quality"]:
+            stats["by_quality"][quality] = {"collections": 0, "images": 0, "bytes": 0}
+
+        stats["by_quality"][quality]["collections"] += 1
+        stats["by_quality"][quality]["images"] += images
+        stats["by_quality"][quality]["bytes"] += size
+
+    # Unique images is sum of max images per user
+    stats["total"]["unique_images"] = sum(user_images.values())
+
+    return stats
+
+
+def format_stats(stats):
+    """Format statistics as human-readable text.
+
+    Args:
+        stats: Dict from aggregate_stats()
+
+    Returns:
+        Formatted string
+    """
+    TOTAL_MAPILLARY_IMAGES = 2_000_000_000  # 2 billion
+
+    output = []
+    output.append("=" * 70)
+    output.append("Mapillary Downloader - Archive.org Statistics")
+    output.append("=" * 70)
+    output.append("")
+
+    # Total stats
+    total = stats["total"]
+    unique_pct = (total["unique_images"] / TOTAL_MAPILLARY_IMAGES * 100) if total["unique_images"] else 0
+
+    output.append(f"Total Collections: {total['collections']:,}")
+    output.append(f"Total Users:       {len(stats['users']):,}")
+    output.append(f"Total Images:      {total['total_images']:,}")
+    output.append(f"Unique Images:     {total['unique_images']:,} ({unique_pct:.3f}% of 2B)")
+    output.append(f"Total Size:        {format_size(total['bytes'])}")
+    output.append("")
+
+    # Per-quality breakdown
+    output.append("By Quality:")
+    output.append("-" * 70)
+
+    # Sort by quality (original first, then numeric)
+    qualities = sorted(stats["by_quality"].items(), key=lambda x: (x[0] != "original", x[0]))
+
+    for quality, data in qualities:
+        pct = (data["images"] / TOTAL_MAPILLARY_IMAGES * 100) if data["images"] else 0
+        output.append(
+            f"  {quality:8s}  {data['collections']:3d} collections  "
+            f"{data['images']:12,d} images ({pct:.3f}%)  "
+            f"{format_size(data['bytes']):>8s}"
+        )
+
+    output.append("")
+    output.append(f"Cache: {CACHE_FILE}")
+
+    return "\n".join(output)
+
+
+def show_stats(refresh=True):
+    """Show archive.org statistics for mapillary_downloader collections.
+
+    Args:
+        refresh: If True, fetch fresh data from IA. If False, use cache only.
+    """
+    if refresh:
+        try:
+            ia_collections = search_ia_collections()
+            cache = update_cache(ia_collections)
+        except Exception as e:
+            logger.error(f"Failed to fetch IA data: {e}")
+            logger.info("Using cached data...")
+            cache = load_cache()
+    else:
+        cache = load_cache()
+
+    if not cache:
+        logger.error("No cached data and failed to fetch from IA")
+        return
+
+    stats = aggregate_stats(cache)
+    print(format_stats(stats))

{mapillary_downloader-0.6.0 → mapillary_downloader-0.7.0}/src/mapillary_downloader/tar_sequences.py

@@ -1,6 +1,7 @@
 """Tar sequence directories for efficient Internet Archive uploads."""
 
 import logging
+import re
 import tarfile
 from pathlib import Path
 from mapillary_downloader.utils import format_size
@@ -9,7 +10,9 @@ logger = logging.getLogger("mapillary_downloader")
 
 
 def tar_sequence_directories(collection_dir):
-    """Tar all sequence directories in a collection for faster IA uploads.
+    """Tar all date directories in a collection for faster IA uploads.
+
+    Organizes by capture date (YYYY-MM-DD) for incremental archive.org uploads.
 
     Args:
         collection_dir: Path to collection directory (e.g., mapillary-user-quality/)
@@ -23,41 +26,44 @@ def tar_sequence_directories(collection_dir):
         logger.error(f"Collection directory not found: {collection_dir}")
         return 0, 0
 
-    # Find all bucket directories (skip special dirs)
-    # Now we tar entire bucket dirs (e.g., a/, b/, etc) to get ~62 tar files
+    # Find all date directories (skip special dirs)
+    # Date format: YYYY-MM-DD or unknown-date
     skip_dirs = {".meta", "__pycache__"}
-    bucket_dirs = []
+    date_dirs = []
 
     for item in collection_dir.iterdir():
         if item.is_dir() and item.name not in skip_dirs:
-            # Check if this is a bucket dir (single char)
-            if len(item.name) == 1:
-                bucket_dirs.append(item)
+            # Check if this is a date dir (YYYY-MM-DD) or unknown-date
+            if re.match(r"\d{4}-\d{2}-\d{2}$", item.name) or item.name == "unknown-date":
+                date_dirs.append(item)
 
-    if not bucket_dirs:
-        logger.info("No bucket directories to tar")
+    if not date_dirs:
+        logger.info("No date directories to tar")
         return 0, 0
 
-    logger.info(f"Tarring {len(bucket_dirs)} bucket directories...")
+    # Sort date directories chronologically (YYYY-MM-DD sorts naturally)
+    date_dirs = sorted(date_dirs, key=lambda x: x.name)
+
+    logger.info(f"Tarring {len(date_dirs)} date directories...")
 
     tarred_count = 0
     total_files = 0
     total_tar_bytes = 0
 
-    for bucket_dir in bucket_dirs:
-        bucket_name = bucket_dir.name
-        tar_path = collection_dir / f"{bucket_name}.tar"
+    for date_dir in date_dirs:
+        date_name = date_dir.name
+        tar_path = collection_dir / f"{date_name}.tar"
 
-        # Count files in bucket
-        files_to_tar = sorted([f for f in bucket_dir.rglob("*") if f.is_file()], key=lambda x: str(x))
+        # Count files in date directory
+        files_to_tar = sorted([f for f in date_dir.rglob("*") if f.is_file()], key=lambda x: str(x))
         file_count = len(files_to_tar)
 
         if file_count == 0:
-            logger.warning(f"Skipping empty bucket directory: {bucket_name}")
+            logger.warning(f"Skipping empty date directory: {date_name}")
             continue
 
         try:
-            logger.info(f"Tarring bucket '{bucket_name}' ({file_count} files)...")
+            logger.info(f"Tarring date '{date_name}' ({file_count} files)...")
 
             # Create reproducible uncompressed tar (WebP already compressed)
             with tarfile.open(tar_path, "w") as tar:
@@ -84,36 +90,34 @@ def tar_sequence_directories(collection_dir):
                 tar_size = tar_path.stat().st_size
                 total_tar_bytes += tar_size
 
-                # Remove original bucket directory
-                for file in bucket_dir.rglob("*"):
+                # Remove original date directory
+                for file in date_dir.rglob("*"):
                     if file.is_file():
                         file.unlink()
 
                 # Remove empty subdirs and main dir
-                for subdir in list(bucket_dir.rglob("*")):
+                for subdir in list(date_dir.rglob("*")):
                     if subdir.is_dir():
                         try:
                             subdir.rmdir()
                         except OSError:
                             pass  # Not empty yet
 
-                bucket_dir.rmdir()
+                date_dir.rmdir()
 
                 tarred_count += 1
                 total_files += file_count
 
-                logger.info(f"Tarred bucket '{bucket_name}': {file_count:,} files, {format_size(tar_size)}")
+                logger.info(f"Tarred date '{date_name}': {file_count:,} files, {format_size(tar_size)}")
             else:
                 logger.error(f"Tar file empty or not created: {tar_path}")
                 if tar_path.exists():
                     tar_path.unlink()
 
         except Exception as e:
-            logger.error(f"Error tarring bucket {bucket_name}: {e}")
+            logger.error(f"Error tarring date {date_name}: {e}")
             if tar_path.exists():
                 tar_path.unlink()
 
-    logger.info(
-        f"Tarred {tarred_count} sequences ({total_files:,} files, {format_size(total_tar_bytes)} total tar size)"
-    )
+    logger.info(f"Tarred {tarred_count} dates ({total_files:,} files, {format_size(total_tar_bytes)} total tar size)")
     return tarred_count, total_files

mapillary_downloader-0.7.0/src/mapillary_downloader/utils.py

@@ -0,0 +1,108 @@
+"""Utility functions for formatting and display."""
+
+import json
+import logging
+import os
+import time
+from pathlib import Path
+import requests
+from requests.exceptions import RequestException
+
+logger = logging.getLogger("mapillary_downloader")
+
+
+def format_size(bytes_count):
+    """Format bytes as human-readable size.
+
+    Args:
+        bytes_count: Number of bytes
+
+    Returns:
+        Formatted string (e.g. "1.23 GB", "456.78 MB")
+    """
+    if bytes_count >= 1_000_000_000:
+        return f"{bytes_count / 1_000_000_000:.2f} GB"
+    if bytes_count >= 1_000_000:
+        return f"{bytes_count / 1_000_000:.2f} MB"
+    if bytes_count >= 1_000:
+        return f"{bytes_count / 1000:.2f} KB"
+    return f"{bytes_count} B"
+
+
+def format_time(seconds):
+    """Format seconds as human-readable time.
+
+    Args:
+        seconds: Number of seconds
+
+    Returns:
+        Formatted string (e.g. "2h 15m", "45m 30s", "30s")
+    """
+    if seconds < 60:
+        return f"{int(seconds)}s"
+
+    minutes = int(seconds / 60)
+    remaining_seconds = int(seconds % 60)
+
+    if minutes < 60:
+        if remaining_seconds > 0:
+            return f"{minutes}m {remaining_seconds}s"
+        return f"{minutes}m"
+
+    hours = int(minutes / 60)
+    remaining_minutes = minutes % 60
+
+    if remaining_minutes > 0:
+        return f"{hours}h {remaining_minutes}m"
+    return f"{hours}h"
+
+
+def safe_json_save(file_path, data):
+    """Atomically save JSON data to file.
+
+    Writes to temp file, then atomic rename to prevent corruption.
+
+    Args:
+        file_path: Path to JSON file
+        data: Data to serialize to JSON
+    """
+    file_path = Path(file_path)
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    temp_file = file_path.with_suffix(".json.tmp")
+    with open(temp_file, "w") as f:
+        json.dump(data, f, indent=2)
+        f.flush()
+        os.fsync(f.fileno())
+    temp_file.replace(file_path)
+
+
+def http_get_with_retry(url, params=None, max_retries=5, base_delay=1.0, timeout=60):
+    """HTTP GET with exponential backoff retry.
+
+    Args:
+        url: URL to fetch
+        params: Optional query parameters
+        max_retries: Maximum retry attempts (default: 5)
+        base_delay: Initial delay in seconds (default: 1.0)
+        timeout: Request timeout in seconds (default: 60)
+
+    Returns:
+        requests.Response object
+
+    Raises:
+        requests.RequestException: If all retries exhausted
+    """
+    for attempt in range(max_retries):
+        try:
+            response = requests.get(url, params=params, timeout=timeout)
+            response.raise_for_status()
+            return response
+        except RequestException as e:
+            if attempt == max_retries - 1:
+                raise
+
+            delay = base_delay * (2**attempt)
+            logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
+            logger.info(f"Retrying in {delay:.1f} seconds...")
+            time.sleep(delay)

{mapillary_downloader-0.6.0 → mapillary_downloader-0.7.0}/src/mapillary_downloader/worker.py

@@ -3,11 +3,12 @@
 import os
 import signal
 import tempfile
+from datetime import datetime
 from pathlib import Path
 import requests
-from requests.exceptions import RequestException
 from mapillary_downloader.exif_writer import write_exif_to_image
 from mapillary_downloader.webp_converter import convert_to_webp
+from mapillary_downloader.utils import http_get_with_retry
 
 
 def worker_process(work_queue, result_queue, worker_id):
@@ -69,16 +70,25 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, se
         if not image_url:
             return (image_id, 0, False, f"No {quality} URL")
 
-        # Determine final output directory - organize by first char of sequence ID
+        # Determine final output directory - organize by capture date
         output_dir = Path(output_dir)
         sequence_id = image_data.get("sequence")
+
+        # Extract date from captured_at timestamp (milliseconds since epoch)
+        captured_at = image_data.get("captured_at")
+        if captured_at:
+            # Convert to UTC date string (YYYY-MM-DD)
+            date_str = datetime.utcfromtimestamp(captured_at / 1000).strftime("%Y-%m-%d")
+        else:
+            # Fallback for missing timestamp (should be rare per API docs)
+            date_str = "unknown-date"
+
         if sequence_id:
-            # Use first character as bucket (gives us ~62 dirs instead of millions)
-            first_char = sequence_id[0]
-            img_dir = output_dir / first_char / sequence_id
+            img_dir = output_dir / date_str / sequence_id
             img_dir.mkdir(parents=True, exist_ok=True)
         else:
-            img_dir = output_dir
+            img_dir = output_dir / date_str
+            img_dir.mkdir(parents=True, exist_ok=True)
 
         # If converting to WebP, use /tmp for intermediate JPEG
         # Otherwise write JPEG directly to final location
@@ -90,19 +100,18 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, se
             jpg_path = img_dir / f"{image_id}.jpg"
             final_path = jpg_path
 
-        # Download image (using session passed from worker)
+        # Download image with retry logic
         bytes_downloaded = 0
 
         try:
-            # 60 second timeout for entire download (connection + read)
-            response = session.get(image_url, stream=True, timeout=60)
-            response.raise_for_status()
+            # Use retry logic with 3 attempts for image downloads
+            response = http_get_with_retry(image_url, max_retries=3, base_delay=1.0, timeout=60)
 
             with open(jpg_path, "wb") as f:
                 for chunk in response.iter_content(chunk_size=8192):
                     f.write(chunk)
                     bytes_downloaded += len(chunk)
-        except RequestException as e:
+        except Exception as e:
             return (image_id, 0, False, f"Download failed: {e}")
 
         # Write EXIF metadata

mapillary_downloader-0.6.0/src/mapillary_downloader/utils.py

@@ -1,47 +0,0 @@
-"""Utility functions for formatting and display."""
-
-
-def format_size(bytes_count):
-    """Format bytes as human-readable size.
-
-    Args:
-        bytes_count: Number of bytes
-
-    Returns:
-        Formatted string (e.g. "1.23 GB", "456.78 MB")
-    """
-    if bytes_count >= 1_000_000_000:
-        return f"{bytes_count / 1_000_000_000:.2f} GB"
-    if bytes_count >= 1_000_000:
-        return f"{bytes_count / 1_000_000:.2f} MB"
-    if bytes_count >= 1_000:
-        return f"{bytes_count / 1000:.2f} KB"
-    return f"{bytes_count} B"
-
-
-def format_time(seconds):
-    """Format seconds as human-readable time.
-
-    Args:
-        seconds: Number of seconds
-
-    Returns:
-        Formatted string (e.g. "2h 15m", "45m 30s", "30s")
-    """
-    if seconds < 60:
-        return f"{int(seconds)}s"
-
-    minutes = int(seconds / 60)
-    remaining_seconds = int(seconds % 60)
-
-    if minutes < 60:
-        if remaining_seconds > 0:
-            return f"{minutes}m {remaining_seconds}s"
-        return f"{minutes}m"
-
-    hours = int(minutes / 60)
-    remaining_minutes = minutes % 60
-
-    if remaining_minutes > 0:
-        return f"{hours}h {remaining_minutes}m"
-    return f"{hours}h"