mapillary-downloader 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

mapillary_downloader/__main__.py

@@ -4,37 +4,30 @@ import argparse
 import sys
 from mapillary_downloader.client import MapillaryClient
 from mapillary_downloader.downloader import MapillaryDownloader
+from mapillary_downloader.logging_config import setup_logging
+from mapillary_downloader.webp_converter import check_cwebp_available
 
 
 def main():
     """Main CLI entry point."""
-    parser = argparse.ArgumentParser(
-        description="Download your Mapillary data before it's gone"
-    )
-    parser.add_argument(
-        "--token",
-        required=True,
-        help="Mapillary API access token"
-    )
-    parser.add_argument(
-        "--username",
-        required=True,
-        help="Your Mapillary username"
-    )
-    parser.add_argument(
-        "--output",
-        default="./mapillary_data",
-        help="Output directory (default: ./mapillary_data)"
-    )
+    # Set up logging
+    logger = setup_logging()
+
+    parser = argparse.ArgumentParser(description="Download your Mapillary data before it's gone")
+    parser.add_argument("--token", required=True, help="Mapillary API access token")
+    parser.add_argument("--username", required=True, help="Your Mapillary username")
+    parser.add_argument("--output", default="./mapillary_data", help="Output directory (default: ./mapillary_data)")
     parser.add_argument(
         "--quality",
         choices=["256", "1024", "2048", "original"],
         default="original",
-        help="Image quality to download (default: original)"
+        help="Image quality to download (default: original)",
     )
+    parser.add_argument("--bbox", help="Bounding box: west,south,east,north")
     parser.add_argument(
-        "--bbox",
-        help="Bounding box: west,south,east,north"
+        "--webp",
+        action="store_true",
+        help="Convert images to WebP format (saves ~70%% disk space, requires cwebp binary)",
     )
 
     args = parser.parse_args()
@@ -46,18 +39,25 @@ def main():
             if len(bbox) != 4:
                 raise ValueError
         except ValueError:
-            print("Error: bbox must be four comma-separated numbers")
+            logger.error("Error: bbox must be four comma-separated numbers")
+            sys.exit(1)
+
+    # Check for cwebp binary if WebP conversion is requested
+    if args.webp:
+        if not check_cwebp_available():
+            logger.error("Error: cwebp binary not found. Install webp package (e.g., apt install webp)")
             sys.exit(1)
+        logger.info("WebP conversion enabled - images will be converted after download")
 
     try:
         client = MapillaryClient(args.token)
         downloader = MapillaryDownloader(client, args.output)
-        downloader.download_user_data(args.username, args.quality, bbox)
+        downloader.download_user_data(args.username, args.quality, bbox, convert_webp=args.webp)
     except KeyboardInterrupt:
-        print("\nInterrupted by user")
+        logger.info("\nInterrupted by user")
         sys.exit(1)
     except Exception as e:
-        print(f"Error: {e}")
+        logger.error(f"Error: {e}")
         sys.exit(1)
 
 

mapillary_downloader/client.py

@@ -1,9 +1,12 @@
 """Mapillary API client."""
 
+import logging
 import time
 import requests
 from requests.exceptions import RequestException
 
+logger = logging.getLogger("mapillary_downloader")
+
 
 class MapillaryClient:
     """Client for interacting with Mapillary API v4."""
@@ -65,6 +68,7 @@ class MapillaryClient:
             params["bbox"] = ",".join(map(str, bbox))
 
         url = f"{self.base_url}/images"
+        total_fetched = 0
 
         while url:
             max_retries = 10
@@ -72,21 +76,24 @@ class MapillaryClient:
 
             for attempt in range(max_retries):
                 try:
-                    response = self.session.get(url, params=params)
+                    response = self.session.get(url, params=params, timeout=60)
                     response.raise_for_status()
                     break
                 except RequestException as e:
                     if attempt == max_retries - 1:
                         raise
 
-                    delay = base_delay * (2 ** attempt)
-                    print(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
-                    print(f"Retrying in {delay:.1f} seconds...")
+                    delay = base_delay * (2**attempt)
+                    logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
+                    logger.info(f"Retrying in {delay:.1f} seconds...")
                     time.sleep(delay)
 
             data = response.json()
+            images = data.get("data", [])
+            total_fetched += len(images)
+            logger.info(f"Fetched metadata for {total_fetched:,} images...")
 
-            for image in data.get("data", []):
+            for image in images:
                 yield image
 
             # Get next page URL
@@ -111,7 +118,7 @@ class MapillaryClient:
 
         for attempt in range(max_retries):
             try:
-                response = self.session.get(image_url, stream=True)
+                response = self.session.get(image_url, stream=True, timeout=60)
                 response.raise_for_status()
 
                 total_bytes = 0
@@ -123,10 +130,10 @@ class MapillaryClient:
                 return total_bytes
             except RequestException as e:
                 if attempt == max_retries - 1:
-                    print(f"Error downloading {image_url} after {max_retries} attempts: {e}")
+                    logger.error(f"Error downloading {image_url} after {max_retries} attempts: {e}")
                     return 0
 
-                delay = base_delay * (2 ** attempt)
-                print(f"Download failed (attempt {attempt + 1}/{max_retries}): {e}")
-                print(f"Retrying in {delay:.1f} seconds...")
+                delay = base_delay * (2**attempt)
+                logger.warning(f"Download failed (attempt {attempt + 1}/{max_retries}): {e}")
+                logger.info(f"Retrying in {delay:.1f} seconds...")
                 time.sleep(delay)

mapillary_downloader/downloader.py

@@ -1,20 +1,16 @@
 """Main downloader logic."""
 
 import json
+import logging
 import os
+import time
 from pathlib import Path
+from collections import deque
 from mapillary_downloader.exif_writer import write_exif_to_image
+from mapillary_downloader.utils import format_size, format_time
+from mapillary_downloader.webp_converter import convert_to_webp
 
-
-def format_bytes(bytes_count):
-    """Format bytes as human-readable string."""
-    if bytes_count < 1024:
-        return f"{bytes_count} B"
-    if bytes_count < 1024 * 1024:
-        return f"{bytes_count / 1024:.3f} KB"
-    if bytes_count < 1024 * 1024 * 1024:
-        return f"{bytes_count / (1024 * 1024):.3f} MB"
-    return f"{bytes_count / (1024 * 1024 * 1024):.3f} GB"
+logger = logging.getLogger("mapillary_downloader")
 
 
 class MapillaryDownloader:
@@ -51,42 +47,115 @@ class MapillaryDownloader:
             os.fsync(f.fileno())
         temp_file.replace(self.progress_file)
 
-    def download_user_data(self, username, quality="original", bbox=None):
+    def download_user_data(self, username, quality="original", bbox=None, convert_webp=False):
         """Download all images for a user.
 
         Args:
             username: Mapillary username
             quality: Image quality to download (256, 1024, 2048, original)
             bbox: Optional bounding box [west, south, east, north]
+            convert_webp: Convert images to WebP format after download
         """
         quality_field = f"thumb_{quality}_url"
 
-        print(f"Downloading images for user: {username}")
-        print(f"Output directory: {self.output_dir}")
-        print(f"Quality: {quality}")
+        logger.info(f"Downloading images for user: {username}")
+        logger.info(f"Output directory: {self.output_dir}")
+        logger.info(f"Quality: {quality}")
 
         processed = 0
         downloaded_count = 0
         skipped = 0
         total_bytes = 0
 
+        # Track download times for adaptive ETA (last 50 downloads)
+        download_times = deque(maxlen=50)
+        start_time = time.time()
+
+        # Track which image IDs we've seen in metadata to avoid re-fetching
+        seen_ids = set()
+
+        # First, process any existing metadata without re-fetching from API
+        if self.metadata_file.exists():
+            logger.info("Processing existing metadata file...")
+            with open(self.metadata_file) as f:
+                for line in f:
+                    if line.strip():
+                        image = json.loads(line)
+                        image_id = image["id"]
+                        seen_ids.add(image_id)
+                        processed += 1
+
+                        if image_id in self.downloaded:
+                            skipped += 1
+                            continue
+
+                        # Download this un-downloaded image
+                        image_url = image.get(quality_field)
+                        if not image_url:
+                            logger.warning(f"No {quality} URL for image {image_id}")
+                            continue
+
+                        sequence_id = image.get("sequence")
+                        if sequence_id:
+                            img_dir = self.output_dir / sequence_id
+                            img_dir.mkdir(exist_ok=True)
+                        else:
+                            img_dir = self.output_dir
+
+                        output_path = img_dir / f"{image_id}.jpg"
+
+                        download_start = time.time()
+                        bytes_downloaded = self.client.download_image(image_url, output_path)
+                        if bytes_downloaded:
+                            download_time = time.time() - download_start
+                            download_times.append(download_time)
+
+                            write_exif_to_image(output_path, image)
+
+                            # Convert to WebP if requested
+                            if convert_webp:
+                                webp_path = convert_to_webp(output_path)
+                                if webp_path:
+                                    output_path = webp_path
+
+                            self.downloaded.add(image_id)
+                            downloaded_count += 1
+                            total_bytes += bytes_downloaded
+
+                            progress_str = (
+                                f"Processed: {processed}, Downloaded: {downloaded_count} ({format_size(total_bytes)})"
+                            )
+                            logger.info(progress_str)
+
+                            if downloaded_count % 10 == 0:
+                                self._save_progress()
+
+        # Always check API for new images (will skip duplicates via seen_ids)
+        logger.info("Checking for new images from API...")
         with open(self.metadata_file, "a") as meta_f:
             for image in self.client.get_user_images(username, bbox=bbox):
                 image_id = image["id"]
-                processed += 1
 
-                if image_id in self.downloaded:
-                    skipped += 1
+                # Skip if we already have this in our metadata file
+                if image_id in seen_ids:
                     continue
 
-                # Save metadata
+                seen_ids.add(image_id)
+                processed += 1
+
+                # Save new metadata
                 meta_f.write(json.dumps(image) + "\n")
                 meta_f.flush()
 
+                # Skip if already downloaded
+                if image_id in self.downloaded:
+                    skipped += 1
+                    continue
+
                 # Download image
                 image_url = image.get(quality_field)
                 if not image_url:
-                    print(f"No {quality} URL for image {image_id}")
+                    logger.warning(f"No {quality} URL for image {image_id}")
                     continue
 
                 # Use sequence ID for organization
@@ -99,21 +168,39 @@ class MapillaryDownloader:
 
                 output_path = img_dir / f"{image_id}.jpg"
 
+                download_start = time.time()
                 bytes_downloaded = self.client.download_image(image_url, output_path)
                 if bytes_downloaded:
+                    download_time = time.time() - download_start
+                    download_times.append(download_time)
+
                     # Write EXIF metadata to the downloaded image
                     write_exif_to_image(output_path, image)
 
+                    # Convert to WebP if requested
+                    if convert_webp:
+                        webp_path = convert_to_webp(output_path)
+                        if webp_path:
+                            output_path = webp_path
+
                     self.downloaded.add(image_id)
                     downloaded_count += 1
                     total_bytes += bytes_downloaded
-                    print(f"Processed: {processed}, Downloaded: {downloaded_count} ({format_bytes(total_bytes)})")
+
+                    # Calculate progress
+                    progress_str = (
+                        f"Processed: {processed}, Downloaded: {downloaded_count} ({format_size(total_bytes)})"
+                    )
+
+                    logger.info(progress_str)
 
                     # Save progress every 10 images
                     if downloaded_count % 10 == 0:
                         self._save_progress()
 
         self._save_progress()
-        print(
-            f"\nComplete! Processed {processed} images, downloaded {downloaded_count} ({format_bytes(total_bytes)}), skipped {skipped}"
+        elapsed = time.time() - start_time
+        logger.info(
+            f"Complete! Processed {processed} images, downloaded {downloaded_count} ({format_size(total_bytes)}), skipped {skipped}"
         )
+        logger.info(f"Total time: {format_time(elapsed)}")

mapillary_downloader/logging_config.py

@@ -0,0 +1,62 @@
+"""Logging configuration with colored output for TTY."""
+
+import logging
+import sys
+
+
+class ColoredFormatter(logging.Formatter):
+    """Formatter that adds color to log levels when output is a TTY."""
+
+    # ANSI color codes
+    COLORS = {
+        "ERROR": "\033[91m",  # Red
+        "WARNING": "\033[93m",  # Yellow
+        "INFO": "\033[92m",  # Green
+        "DEBUG": "\033[94m",  # Blue
+        "RESET": "\033[0m",
+    }
+
+    def __init__(self, fmt=None, datefmt=None, use_color=True):
+        """Initialize the formatter.
+
+        Args:
+            fmt: Log format string
+            datefmt: Date format string
+            use_color: Whether to use colored output
+        """
+        super().__init__(fmt, datefmt)
+        self.use_color = use_color and sys.stdout.isatty()
+
+    def format(self, record):
+        """Format the log record with colors if appropriate.
+
+        Args:
+            record: LogRecord to format
+
+        Returns:
+            Formatted log string
+        """
+        if self.use_color:
+            levelname = record.levelname
+            if levelname in self.COLORS:
+                record.levelname = f"{self.COLORS[levelname]}{levelname}{self.COLORS['RESET']}"
+
+        return super().format(record)
+
+
+def setup_logging(level=logging.INFO):
+    """Set up logging with timestamps and colored output.
+
+    Args:
+        level: Logging level to use
+    """
+    formatter = ColoredFormatter(fmt="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger("mapillary_downloader")
+    logger.setLevel(level)
+    logger.addHandler(handler)
+
+    return logger

mapillary_downloader/utils.py

@@ -0,0 +1,47 @@
+"""Utility functions for formatting and display."""
+
+
+def format_size(bytes_count):
+    """Format bytes as human-readable size.
+
+    Args:
+        bytes_count: Number of bytes
+
+    Returns:
+        Formatted string (e.g. "1.23 GB", "456.78 MB")
+    """
+    if bytes_count >= 1_000_000_000:
+        return f"{bytes_count / 1_000_000_000:.2f} GB"
+    if bytes_count >= 1_000_000:
+        return f"{bytes_count / 1_000_000:.2f} MB"
+    if bytes_count >= 1_000:
+        return f"{bytes_count / 1000:.2f} KB"
+    return f"{bytes_count} B"
+
+
+def format_time(seconds):
+    """Format seconds as human-readable time.
+
+    Args:
+        seconds: Number of seconds
+
+    Returns:
+        Formatted string (e.g. "2h 15m", "45m 30s", "30s")
+    """
+    if seconds < 60:
+        return f"{int(seconds)}s"
+
+    minutes = int(seconds / 60)
+    remaining_seconds = int(seconds % 60)
+
+    if minutes < 60:
+        if remaining_seconds > 0:
+            return f"{minutes}m {remaining_seconds}s"
+        return f"{minutes}m"
+
+    hours = int(minutes / 60)
+    remaining_minutes = minutes % 60
+
+    if remaining_minutes > 0:
+        return f"{hours}h {remaining_minutes}m"
+    return f"{hours}h"

mapillary_downloader/webp_converter.py

@@ -0,0 +1,54 @@
+"""WebP image conversion utilities."""
+
+import logging
+import shutil
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger("mapillary_downloader")
+
+
+def check_cwebp_available():
+    """Check if cwebp binary is available.
+
+    Returns:
+        bool: True if cwebp is found, False otherwise
+    """
+    return shutil.which("cwebp") is not None
+
+
+def convert_to_webp(jpg_path):
+    """Convert a JPG image to WebP format, preserving EXIF metadata.
+
+    Args:
+        jpg_path: Path to the JPG file
+
+    Returns:
+        Path object to the new WebP file, or None if conversion failed
+    """
+    jpg_path = Path(jpg_path)
+    webp_path = jpg_path.with_suffix(".webp")
+
+    try:
+        # Convert with cwebp, preserving all metadata
+        result = subprocess.run(
+            ["cwebp", "-metadata", "all", str(jpg_path), "-o", str(webp_path)],
+            capture_output=True,
+            text=True,
+            timeout=60,
+        )
+
+        if result.returncode != 0:
+            logger.error(f"cwebp conversion failed for {jpg_path}: {result.stderr}")
+            return None
+
+        # Delete original JPG after successful conversion
+        jpg_path.unlink()
+        return webp_path
+
+    except subprocess.TimeoutExpired:
+        logger.error(f"cwebp conversion timed out for {jpg_path}")
+        return None
+    except Exception as e:
+        logger.error(f"Error converting {jpg_path} to WebP: {e}")
+        return None

{mapillary_downloader-0.1.2.dist-info → mapillary_downloader-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.1.2
+Version: 0.2.0
 Summary: Download your Mapillary data before it's gone
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
@@ -28,7 +28,7 @@ Project-URL: Issues, https://github.com/bitplane/mapillary_downloader/issues
 Project-URL: Repository, https://github.com/bitplane/mapillary_downloader
 Provides-Extra: dev
 
-# Mapillary Downloader
+# 🗺️ Mapillary Downloader
 
 Download your Mapillary data before it's gone.
 
@@ -59,13 +59,35 @@ mapillary-downloader --token YOUR_TOKEN --username YOUR_USERNAME --output ./down
 | `--output`    | Output directory                      | `./mapillary_data` |
 | `--quality`   | 256, 1024, 2048 or original           | `original`         |
 | `--bbox`      | `west,south,east,north`               | `None`             |
+| `--webp`      | Convert to WebP (saves ~70% space)    | `False`            |
 
 The downloader will:
 
 * 💾 Fetch all your uploaded images from Mapillary
-- 📷 Download full-resolution images organized by sequence
-- 📜 Inject EXIF metadata (GPS coordinates, camera info, timestamps, compass direction)
-- 🛟 Save progress so you can safely resume if interrupted
+* 📷 Download full-resolution images organized by sequence
+* 📜 Inject EXIF metadata (GPS coordinates, camera info, timestamps,
+  compass direction)
+* 🛟 Save progress so you can safely resume if interrupted
+* 🗜️ Optionally convert to WebP format for massive space savings
+
+## WebP Conversion
+
+Use the `--webp` flag to convert images to WebP format after download:
+
+```bash
+mapillary-downloader --token YOUR_TOKEN --username YOUR_USERNAME --webp
+```
+
+This reduces storage by approximately 70% while preserving all EXIF metadata
+including GPS coordinates. Requires the `cwebp` binary to be installed:
+
+```bash
+# Debian/Ubuntu
+sudo apt install webp
+
+# macOS
+brew install webp
+```
 
 ## Development
 
@@ -88,5 +110,6 @@ WTFPL with one additional clause
 
 1. Don't blame me
 
-Do wtf you want, but don't blame me when it breaks.
+Do wtf you want, but don't blame me if it makes jokes about the size of your
+disk drive.
 

mapillary_downloader-0.2.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+mapillary_downloader/__init__.py,sha256=KEjiBRghXDeA7E15RJeLBfQm-yNJkowZarL59QOh_1w,120
+mapillary_downloader/__main__.py,sha256=ANouPtrcIZ5oZgeEw2JwgTcFPe2wlT9zqfA1_ViZBNs,2332
+mapillary_downloader/client.py,sha256=O7JgshaM3QKUv0xXuBbe_uPqsTr4lgyuVUHYndvXTfA,4611
+mapillary_downloader/downloader.py,sha256=L6Rd9KyFl-S3AXvsGiN22fw95o-6bklja7P5uj1ZoRE,7959
+mapillary_downloader/exif_writer.py,sha256=Bn1u3QULfHtae86FnUGcqN450NccJwtwW9wVaSRyx9E,4615
+mapillary_downloader/logging_config.py,sha256=WgjXthS2Lpl0RPO5sb-Aj60W2maog1bbzJzhblMLs1c,1715
+mapillary_downloader/utils.py,sha256=yzVgS1mwsklDAqrimaFafgTTXtRYQUbKP98Xgh9d2KA,1174
+mapillary_downloader/webp_converter.py,sha256=oNodgKI1SLvTgTRGZFvcbeP0PzMcxcKrsPbuZMWXWeU,1441
+mapillary_downloader-0.2.0.dist-info/entry_points.txt,sha256=PdYtxOXHMJrUhmiPO4G-F98VuhUI4MN9D_T4KPrVZ5w,75
+mapillary_downloader-0.2.0.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
+mapillary_downloader-0.2.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+mapillary_downloader-0.2.0.dist-info/METADATA,sha256=cH9f301fb0KzjtXibHNToexqjt_vi8wy7BsuChTUODg,3614
+mapillary_downloader-0.2.0.dist-info/RECORD,,

mapillary_downloader-0.1.2.dist-info/RECORD

@@ -1,10 +0,0 @@
-mapillary_downloader/__init__.py,sha256=KEjiBRghXDeA7E15RJeLBfQm-yNJkowZarL59QOh_1w,120
-mapillary_downloader/__main__.py,sha256=xKYhamK0HYXqx98fGb5CVOEw0syURWgX7jnFIdsK5Ao,1720
-mapillary_downloader/client.py,sha256=w936htOireWnKgfXJDYls-umYxOzr0FQ4yzklQ6fPM0,4315
-mapillary_downloader/downloader.py,sha256=n5Y7aAoin3vBa_H3et9hpTNoPrEarbU_LdnHT619c5Y,4216
-mapillary_downloader/exif_writer.py,sha256=Bn1u3QULfHtae86FnUGcqN450NccJwtwW9wVaSRyx9E,4615
-mapillary_downloader-0.1.2.dist-info/entry_points.txt,sha256=PdYtxOXHMJrUhmiPO4G-F98VuhUI4MN9D_T4KPrVZ5w,75
-mapillary_downloader-0.1.2.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
-mapillary_downloader-0.1.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-mapillary_downloader-0.1.2.dist-info/METADATA,sha256=07TO0LnPoOTq7gI6G1ZmCe2gbuA3Gzy0yV5rXAEeXlI,3013
-mapillary_downloader-0.1.2.dist-info/RECORD,,