mapillary-downloader 0.6.0__tar.gz → 0.6.1__tar.gz

{mapillary_downloader-0.6.0 → mapillary_downloader-0.6.1}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.6.0
-Summary: Download your Mapillary data before it's gone
+Version: 0.6.1
+Summary: Archive user data from Mapillary
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{mapillary_downloader-0.6.0 → mapillary_downloader-0.6.1}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mapillary_downloader"
-description = "Download your Mapillary data before it's gone"
-version = "0.6.0"
+description = "Archive user data from Mapillary"
+version = "0.6.1"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

{mapillary_downloader-0.6.0 → mapillary_downloader-0.6.1}/src/mapillary_downloader/__main__.py

@@ -26,7 +26,7 @@ def main():
         default=os.environ.get("MAPILLARY_TOKEN"),
         help="Mapillary API access token (or set MAPILLARY_TOKEN env var)",
     )
-    parser.add_argument("usernames", nargs="+", help="Mapillary username(s) to download")
+    parser.add_argument("usernames", nargs="*", help="Mapillary username(s) to download")
     parser.add_argument("--output", default="./mapillary_data", help="Output directory (default: ./mapillary_data)")
     parser.add_argument(
         "--quality",
@@ -61,9 +61,21 @@ def main():
         action="store_true",
         help="Enable debug logging (EXIF data, API responses, etc.)",
     )
+    parser.add_argument(
+        "--stats",
+        action="store_true",
+        help="Show statistics of collections on archive.org and exit",
+    )
 
     args = parser.parse_args()
 
+    # Handle --stats early (before token check)
+    if args.stats:
+        from mapillary_downloader.ia_stats import show_stats
+
+        show_stats()
+        sys.exit(0)
+
     # Set debug logging level if requested
     if args.debug:
         import logging
@@ -71,6 +83,11 @@ def main():
         logging.getLogger("mapillary_downloader").setLevel(logging.DEBUG)
         logger.debug("Debug logging enabled")
 
+    # Check for usernames (required unless using --stats)
+    if not args.usernames:
+        logger.error("Error: At least one username is required")
+        sys.exit(1)
+
     # Check for token
     if not args.token:
         logger.error("Error: Mapillary API token required. Use --token or set MAPILLARY_TOKEN environment variable")

{mapillary_downloader-0.6.0 → mapillary_downloader-0.6.1}/src/mapillary_downloader/downloader.py

@@ -7,7 +7,7 @@ import os
 import shutil
 import time
 from pathlib import Path
-from mapillary_downloader.utils import format_size, format_time
+from mapillary_downloader.utils import format_size, format_time, safe_json_save
 from mapillary_downloader.ia_meta import generate_ia_metadata
 from mapillary_downloader.ia_check import check_ia_exists
 from mapillary_downloader.worker import worker_process
@@ -143,13 +143,8 @@ class MapillaryDownloader:
         # Update this quality's progress
         progress[str(self.quality)] = list(self.downloaded)
 
-        # Write atomically
-        temp_file = self.progress_file.with_suffix(".json.tmp")
-        with open(temp_file, "w") as f:
-            json.dump(progress, f)
-            f.flush()
-            os.fsync(f.fileno())
-        temp_file.replace(self.progress_file)
+        # Write atomically using utility function
+        safe_json_save(self.progress_file, progress)
 
     def download_user_data(self, bbox=None, convert_webp=False):
         """Download all images for a user using streaming queue-based architecture.

mapillary_downloader-0.6.1/src/mapillary_downloader/ia_stats.py

@@ -0,0 +1,242 @@
+"""Internet Archive statistics for mapillary_downloader collections."""
+
+import json
+import logging
+import re
+from mapillary_downloader.utils import safe_json_save, http_get_with_retry, format_size
+from mapillary_downloader.downloader import get_cache_dir
+
+logger = logging.getLogger("mapillary_downloader")
+
+CACHE_FILE = get_cache_dir() / ".stats.json"
+
+
+def search_ia_collections():
+    """Search IA for all mapillary_downloader collections.
+
+    Returns:
+        List of dicts with: identifier, description, item_size, uploader
+    """
+    logger.info("Searching archive.org for mapillary_downloader collections...")
+
+    url = "https://archive.org/advancedsearch.php"
+    params = {
+        "q": "mapillary_downloader:*",
+        "fl[]": ["identifier", "description", "item_size", "uploader"],
+        "rows": 10000,
+        "output": "json",
+    }
+
+    response = http_get_with_retry(url, params=params, max_retries=3)
+    data = response.json()
+
+    collections = data["response"]["docs"]
+    logger.info(f"Found {len(collections)} collections on archive.org")
+
+    return collections
+
+
+def parse_collection_info(identifier):
+    """Parse username, quality, webp from collection identifier.
+
+    Returns:
+        dict with username, quality, is_webp or None if invalid
+    """
+    match = re.match(r"mapillary-(.+)-(256|1024|2048|original)(?:-webp)?$", identifier)
+    if match:
+        return {"username": match.group(1), "quality": match.group(2), "is_webp": "-webp" in identifier}
+    return None
+
+
+def extract_image_count(description):
+    """Extract image count from IA description field.
+
+    Description format: "Contains 12,345 images in..."
+    """
+    if not description:
+        return None
+
+    match = re.search(r"Contains ([\d,]+) images", description)
+    if match:
+        return int(match.group(1).replace(",", ""))
+    return None
+
+
+def load_cache():
+    """Load cached collection data.
+
+    Returns:
+        dict of {collection_id: {size, uploader, images, quality, username}}
+    """
+    if CACHE_FILE.exists():
+        try:
+            with open(CACHE_FILE) as f:
+                return json.load(f)
+        except Exception as e:
+            logger.warning(f"Failed to load cache: {e}")
+    return {}
+
+
+def update_cache(ia_collections):
+    """Update cache with new IA search results.
+
+    Merges new collections into existing cache.
+
+    Returns:
+        Updated cache dict
+    """
+    cache = load_cache()
+
+    for item in ia_collections:
+        identifier = item.get("identifier")
+        if not identifier:
+            continue
+
+        info = parse_collection_info(identifier)
+        if not info:
+            logger.debug(f"Skipping non-mapillary collection: {identifier}")
+            continue
+
+        # Parse item data
+        size_bytes = item.get("item_size", 0)
+        if isinstance(size_bytes, str):
+            size_bytes = int(size_bytes)
+
+        image_count = extract_image_count(item.get("description"))
+
+        # Update cache entry
+        cache[identifier] = {
+            "size": size_bytes,
+            "uploader": item.get("uploader"),
+            "images": image_count,
+            "quality": info["quality"],
+            "username": info["username"],
+            "is_webp": info["is_webp"],
+        }
+
+    # Save updated cache
+    safe_json_save(CACHE_FILE, cache)
+    logger.info(f"Updated cache with {len(cache)} collections")
+
+    return cache
+
+
+def aggregate_stats(cache):
+    """Aggregate statistics from cached collection data.
+
+    Returns:
+        dict with total and per-quality stats
+    """
+    stats = {
+        "total": {"collections": 0, "total_images": 0, "unique_images": 0, "bytes": 0},
+        "by_quality": {},
+        "users": set(),
+    }
+
+    # Track images per user for deduplication
+    user_images = {}  # {username: max_images_across_qualities}
+
+    for collection_id, data in cache.items():
+        images = data.get("images") or 0
+        size = data.get("size") or 0
+        quality = data.get("quality", "unknown")
+        username = data.get("username")
+
+        # Track user coverage
+        if username:
+            stats["users"].add(username)
+            # Keep maximum image count across all qualities for this user
+            if username not in user_images or images > user_images[username]:
+                user_images[username] = images
+
+        # Total stats (collections, total images, and bytes)
+        stats["total"]["collections"] += 1
+        stats["total"]["total_images"] += images
+        stats["total"]["bytes"] += size
+
+        # Per-quality stats
+        if quality not in stats["by_quality"]:
+            stats["by_quality"][quality] = {"collections": 0, "images": 0, "bytes": 0}
+
+        stats["by_quality"][quality]["collections"] += 1
+        stats["by_quality"][quality]["images"] += images
+        stats["by_quality"][quality]["bytes"] += size
+
+    # Unique images is sum of max images per user
+    stats["total"]["unique_images"] = sum(user_images.values())
+
+    return stats
+
+
+def format_stats(stats):
+    """Format statistics as human-readable text.
+
+    Args:
+        stats: Dict from aggregate_stats()
+
+    Returns:
+        Formatted string
+    """
+    TOTAL_MAPILLARY_IMAGES = 2_000_000_000  # 2 billion
+
+    output = []
+    output.append("=" * 70)
+    output.append("Mapillary Downloader - Archive.org Statistics")
+    output.append("=" * 70)
+    output.append("")
+
+    # Total stats
+    total = stats["total"]
+    unique_pct = (total["unique_images"] / TOTAL_MAPILLARY_IMAGES * 100) if total["unique_images"] else 0
+
+    output.append(f"Total Collections: {total['collections']:,}")
+    output.append(f"Total Users:       {len(stats['users']):,}")
+    output.append(f"Total Images:      {total['total_images']:,}")
+    output.append(f"Unique Images:     {total['unique_images']:,} ({unique_pct:.3f}% of 2B)")
+    output.append(f"Total Size:        {format_size(total['bytes'])}")
+    output.append("")
+
+    # Per-quality breakdown
+    output.append("By Quality:")
+    output.append("-" * 70)
+
+    # Sort by quality (original first, then numeric)
+    qualities = sorted(stats["by_quality"].items(), key=lambda x: (x[0] != "original", x[0]))
+
+    for quality, data in qualities:
+        pct = (data["images"] / TOTAL_MAPILLARY_IMAGES * 100) if data["images"] else 0
+        output.append(
+            f"  {quality:8s}  {data['collections']:3d} collections  "
+            f"{data['images']:12,d} images ({pct:.3f}%)  "
+            f"{format_size(data['bytes']):>8s}"
+        )
+
+    output.append("")
+    output.append(f"Cache: {CACHE_FILE}")
+
+    return "\n".join(output)
+
+
+def show_stats(refresh=True):
+    """Show archive.org statistics for mapillary_downloader collections.
+
+    Args:
+        refresh: If True, fetch fresh data from IA. If False, use cache only.
+    """
+    if refresh:
+        try:
+            ia_collections = search_ia_collections()
+            cache = update_cache(ia_collections)
+        except Exception as e:
+            logger.error(f"Failed to fetch IA data: {e}")
+            logger.info("Using cached data...")
+            cache = load_cache()
+    else:
+        cache = load_cache()
+
+    if not cache:
+        logger.error("No cached data and failed to fetch from IA")
+        return
+
+    stats = aggregate_stats(cache)
+    print(format_stats(stats))

{mapillary_downloader-0.6.0 → mapillary_downloader-0.6.1}/src/mapillary_downloader/tar_sequences.py

@@ -38,6 +38,9 @@ def tar_sequence_directories(collection_dir):
         logger.info("No bucket directories to tar")
         return 0, 0
 
+    # Sort bucket directories alphabetically for consistent progress tracking
+    bucket_dirs = sorted(bucket_dirs, key=lambda x: x.name)
+
     logger.info(f"Tarring {len(bucket_dirs)} bucket directories...")
 
     tarred_count = 0

mapillary_downloader-0.6.1/src/mapillary_downloader/utils.py

@@ -0,0 +1,108 @@
+"""Utility functions for formatting and display."""
+
+import json
+import logging
+import os
+import time
+from pathlib import Path
+import requests
+from requests.exceptions import RequestException
+
+logger = logging.getLogger("mapillary_downloader")
+
+
+def format_size(bytes_count):
+    """Format bytes as human-readable size.
+
+    Args:
+        bytes_count: Number of bytes
+
+    Returns:
+        Formatted string (e.g. "1.23 GB", "456.78 MB")
+    """
+    if bytes_count >= 1_000_000_000:
+        return f"{bytes_count / 1_000_000_000:.2f} GB"
+    if bytes_count >= 1_000_000:
+        return f"{bytes_count / 1_000_000:.2f} MB"
+    if bytes_count >= 1_000:
+        return f"{bytes_count / 1000:.2f} KB"
+    return f"{bytes_count} B"
+
+
+def format_time(seconds):
+    """Format seconds as human-readable time.
+
+    Args:
+        seconds: Number of seconds
+
+    Returns:
+        Formatted string (e.g. "2h 15m", "45m 30s", "30s")
+    """
+    if seconds < 60:
+        return f"{int(seconds)}s"
+
+    minutes = int(seconds / 60)
+    remaining_seconds = int(seconds % 60)
+
+    if minutes < 60:
+        if remaining_seconds > 0:
+            return f"{minutes}m {remaining_seconds}s"
+        return f"{minutes}m"
+
+    hours = int(minutes / 60)
+    remaining_minutes = minutes % 60
+
+    if remaining_minutes > 0:
+        return f"{hours}h {remaining_minutes}m"
+    return f"{hours}h"
+
+
+def safe_json_save(file_path, data):
+    """Atomically save JSON data to file.
+
+    Writes to temp file, then atomic rename to prevent corruption.
+
+    Args:
+        file_path: Path to JSON file
+        data: Data to serialize to JSON
+    """
+    file_path = Path(file_path)
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    temp_file = file_path.with_suffix(".json.tmp")
+    with open(temp_file, "w") as f:
+        json.dump(data, f, indent=2)
+        f.flush()
+        os.fsync(f.fileno())
+    temp_file.replace(file_path)
+
+
+def http_get_with_retry(url, params=None, max_retries=5, base_delay=1.0, timeout=60):
+    """HTTP GET with exponential backoff retry.
+
+    Args:
+        url: URL to fetch
+        params: Optional query parameters
+        max_retries: Maximum retry attempts (default: 5)
+        base_delay: Initial delay in seconds (default: 1.0)
+        timeout: Request timeout in seconds (default: 60)
+
+    Returns:
+        requests.Response object
+
+    Raises:
+        requests.RequestException: If all retries exhausted
+    """
+    for attempt in range(max_retries):
+        try:
+            response = requests.get(url, params=params, timeout=timeout)
+            response.raise_for_status()
+            return response
+        except RequestException as e:
+            if attempt == max_retries - 1:
+                raise
+
+            delay = base_delay * (2**attempt)
+            logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
+            logger.info(f"Retrying in {delay:.1f} seconds...")
+            time.sleep(delay)

{mapillary_downloader-0.6.0 → mapillary_downloader-0.6.1}/src/mapillary_downloader/worker.py

@@ -5,9 +5,9 @@ import signal
 import tempfile
 from pathlib import Path
 import requests
-from requests.exceptions import RequestException
 from mapillary_downloader.exif_writer import write_exif_to_image
 from mapillary_downloader.webp_converter import convert_to_webp
+from mapillary_downloader.utils import http_get_with_retry
 
 
 def worker_process(work_queue, result_queue, worker_id):
@@ -90,19 +90,18 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, se
             jpg_path = img_dir / f"{image_id}.jpg"
             final_path = jpg_path
 
-        # Download image (using session passed from worker)
+        # Download image with retry logic
         bytes_downloaded = 0
 
         try:
-            # 60 second timeout for entire download (connection + read)
-            response = session.get(image_url, stream=True, timeout=60)
-            response.raise_for_status()
+            # Use retry logic with 3 attempts for image downloads
+            response = http_get_with_retry(image_url, max_retries=3, base_delay=1.0, timeout=60)
 
             with open(jpg_path, "wb") as f:
                 for chunk in response.iter_content(chunk_size=8192):
                     f.write(chunk)
                     bytes_downloaded += len(chunk)
-        except RequestException as e:
+        except Exception as e:
             return (image_id, 0, False, f"Download failed: {e}")
 
         # Write EXIF metadata

mapillary_downloader-0.6.0/src/mapillary_downloader/utils.py

@@ -1,47 +0,0 @@
-"""Utility functions for formatting and display."""
-
-
-def format_size(bytes_count):
-    """Format bytes as human-readable size.
-
-    Args:
-        bytes_count: Number of bytes
-
-    Returns:
-        Formatted string (e.g. "1.23 GB", "456.78 MB")
-    """
-    if bytes_count >= 1_000_000_000:
-        return f"{bytes_count / 1_000_000_000:.2f} GB"
-    if bytes_count >= 1_000_000:
-        return f"{bytes_count / 1_000_000:.2f} MB"
-    if bytes_count >= 1_000:
-        return f"{bytes_count / 1000:.2f} KB"
-    return f"{bytes_count} B"
-
-
-def format_time(seconds):
-    """Format seconds as human-readable time.
-
-    Args:
-        seconds: Number of seconds
-
-    Returns:
-        Formatted string (e.g. "2h 15m", "45m 30s", "30s")
-    """
-    if seconds < 60:
-        return f"{int(seconds)}s"
-
-    minutes = int(seconds / 60)
-    remaining_seconds = int(seconds % 60)
-
-    if minutes < 60:
-        if remaining_seconds > 0:
-            return f"{minutes}m {remaining_seconds}s"
-        return f"{minutes}m"
-
-    hours = int(minutes / 60)
-    remaining_minutes = minutes % 60
-
-    if remaining_minutes > 0:
-        return f"{hours}h {remaining_minutes}m"
-    return f"{hours}h"