mapillary-downloader 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

mapillary_downloader/__main__.py

@@ -4,38 +4,25 @@ import argparse
 import sys
 from mapillary_downloader.client import MapillaryClient
 from mapillary_downloader.downloader import MapillaryDownloader
+from mapillary_downloader.logging_config import setup_logging
 
 
 def main():
     """Main CLI entry point."""
-    parser = argparse.ArgumentParser(
-        description="Download your Mapillary data before it's gone"
-    )
-    parser.add_argument(
-        "--token",
-        required=True,
-        help="Mapillary API access token"
-    )
-    parser.add_argument(
-        "--username",
-        required=True,
-        help="Your Mapillary username"
-    )
-    parser.add_argument(
-        "--output",
-        default="./mapillary_data",
-        help="Output directory (default: ./mapillary_data)"
-    )
+    # Set up logging
+    logger = setup_logging()
+
+    parser = argparse.ArgumentParser(description="Download your Mapillary data before it's gone")
+    parser.add_argument("--token", required=True, help="Mapillary API access token")
+    parser.add_argument("--username", required=True, help="Your Mapillary username")
+    parser.add_argument("--output", default="./mapillary_data", help="Output directory (default: ./mapillary_data)")
     parser.add_argument(
         "--quality",
         choices=["256", "1024", "2048", "original"],
         default="original",
-        help="Image quality to download (default: original)"
-    )
-    parser.add_argument(
-        "--bbox",
-        help="Bounding box: west,south,east,north"
+        help="Image quality to download (default: original)",
     )
+    parser.add_argument("--bbox", help="Bounding box: west,south,east,north")
 
     args = parser.parse_args()
 
@@ -46,7 +33,7 @@ def main():
             if len(bbox) != 4:
                 raise ValueError
         except ValueError:
-            print("Error: bbox must be four comma-separated numbers")
+            logger.error("Error: bbox must be four comma-separated numbers")
             sys.exit(1)
 
     try:
@@ -54,10 +41,10 @@ def main():
         downloader = MapillaryDownloader(client, args.output)
         downloader.download_user_data(args.username, args.quality, bbox)
     except KeyboardInterrupt:
-        print("\nInterrupted by user")
+        logger.info("\nInterrupted by user")
         sys.exit(1)
     except Exception as e:
-        print(f"Error: {e}")
+        logger.error(f"Error: {e}")
         sys.exit(1)
 
 

mapillary_downloader/client.py

@@ -1,9 +1,12 @@
 """Mapillary API client."""
 
+import logging
 import time
 import requests
 from requests.exceptions import RequestException
 
+logger = logging.getLogger("mapillary_downloader")
+
 
 class MapillaryClient:
     """Client for interacting with Mapillary API v4."""
@@ -65,6 +68,7 @@ class MapillaryClient:
             params["bbox"] = ",".join(map(str, bbox))
 
         url = f"{self.base_url}/images"
+        total_fetched = 0
 
         while url:
             max_retries = 10
@@ -72,21 +76,24 @@ class MapillaryClient:
 
             for attempt in range(max_retries):
                 try:
-                    response = self.session.get(url, params=params)
+                    response = self.session.get(url, params=params, timeout=60)
                     response.raise_for_status()
                     break
                 except RequestException as e:
                     if attempt == max_retries - 1:
                         raise
 
-                    delay = base_delay * (2 ** attempt)
-                    print(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
-                    print(f"Retrying in {delay:.1f} seconds...")
+                    delay = base_delay * (2**attempt)
+                    logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
+                    logger.info(f"Retrying in {delay:.1f} seconds...")
                     time.sleep(delay)
 
             data = response.json()
+            images = data.get("data", [])
+            total_fetched += len(images)
+            logger.info(f"Fetched metadata for {total_fetched:,} images...")
 
-            for image in data.get("data", []):
+            for image in images:
                 yield image
 
             # Get next page URL
@@ -111,7 +118,7 @@ class MapillaryClient:
 
         for attempt in range(max_retries):
             try:
-                response = self.session.get(image_url, stream=True)
+                response = self.session.get(image_url, stream=True, timeout=60)
                 response.raise_for_status()
 
                 total_bytes = 0
@@ -123,10 +130,10 @@ class MapillaryClient:
                 return total_bytes
             except RequestException as e:
                 if attempt == max_retries - 1:
-                    print(f"Error downloading {image_url} after {max_retries} attempts: {e}")
+                    logger.error(f"Error downloading {image_url} after {max_retries} attempts: {e}")
                     return 0
 
-                delay = base_delay * (2 ** attempt)
-                print(f"Download failed (attempt {attempt + 1}/{max_retries}): {e}")
-                print(f"Retrying in {delay:.1f} seconds...")
+                delay = base_delay * (2**attempt)
+                logger.warning(f"Download failed (attempt {attempt + 1}/{max_retries}): {e}")
+                logger.info(f"Retrying in {delay:.1f} seconds...")
                 time.sleep(delay)

mapillary_downloader/downloader.py

@@ -1,20 +1,15 @@
 """Main downloader logic."""
 
 import json
+import logging
 import os
+import time
 from pathlib import Path
+from collections import deque
 from mapillary_downloader.exif_writer import write_exif_to_image
+from mapillary_downloader.utils import format_size, format_time
 
-
-def format_bytes(bytes_count):
-    """Format bytes as human-readable string."""
-    if bytes_count < 1024:
-        return f"{bytes_count} B"
-    if bytes_count < 1024 * 1024:
-        return f"{bytes_count / 1024:.3f} KB"
-    if bytes_count < 1024 * 1024 * 1024:
-        return f"{bytes_count / (1024 * 1024):.3f} MB"
-    return f"{bytes_count / (1024 * 1024 * 1024):.3f} GB"
+logger = logging.getLogger("mapillary_downloader")
 
 
 class MapillaryDownloader:
@@ -61,32 +56,98 @@ class MapillaryDownloader:
         """
         quality_field = f"thumb_{quality}_url"
 
-        print(f"Downloading images for user: {username}")
-        print(f"Output directory: {self.output_dir}")
-        print(f"Quality: {quality}")
+        logger.info(f"Downloading images for user: {username}")
+        logger.info(f"Output directory: {self.output_dir}")
+        logger.info(f"Quality: {quality}")
 
         processed = 0
         downloaded_count = 0
         skipped = 0
         total_bytes = 0
 
+        # Track download times for adaptive ETA (last 50 downloads)
+        download_times = deque(maxlen=50)
+        start_time = time.time()
+
+        # Track which image IDs we've seen in metadata to avoid re-fetching
+        seen_ids = set()
+
+        # First, process any existing metadata without re-fetching from API
+        if self.metadata_file.exists():
+            logger.info("Processing existing metadata file...")
+            with open(self.metadata_file) as f:
+                for line in f:
+                    if line.strip():
+                        image = json.loads(line)
+                        image_id = image["id"]
+                        seen_ids.add(image_id)
+                        processed += 1
+
+                        if image_id in self.downloaded:
+                            skipped += 1
+                            continue
+
+                        # Download this un-downloaded image
+                        image_url = image.get(quality_field)
+                        if not image_url:
+                            logger.warning(f"No {quality} URL for image {image_id}")
+                            continue
+
+                        sequence_id = image.get("sequence")
+                        if sequence_id:
+                            img_dir = self.output_dir / sequence_id
+                            img_dir.mkdir(exist_ok=True)
+                        else:
+                            img_dir = self.output_dir
+
+                        output_path = img_dir / f"{image_id}.jpg"
+
+                        download_start = time.time()
+                        bytes_downloaded = self.client.download_image(image_url, output_path)
+                        if bytes_downloaded:
+                            download_time = time.time() - download_start
+                            download_times.append(download_time)
+
+                            write_exif_to_image(output_path, image)
+
+                            self.downloaded.add(image_id)
+                            downloaded_count += 1
+                            total_bytes += bytes_downloaded
+
+                            progress_str = (
+                                f"Processed: {processed}, Downloaded: {downloaded_count} ({format_size(total_bytes)})"
+                            )
+                            logger.info(progress_str)
+
+                            if downloaded_count % 10 == 0:
+                                self._save_progress()
+
+        # Always check API for new images (will skip duplicates via seen_ids)
+        logger.info("Checking for new images from API...")
         with open(self.metadata_file, "a") as meta_f:
             for image in self.client.get_user_images(username, bbox=bbox):
                 image_id = image["id"]
-                processed += 1
 
-                if image_id in self.downloaded:
-                    skipped += 1
+                # Skip if we already have this in our metadata file
+                if image_id in seen_ids:
                     continue
 
-                # Save metadata
+                seen_ids.add(image_id)
+                processed += 1
+
+                # Save new metadata
                 meta_f.write(json.dumps(image) + "\n")
                 meta_f.flush()
 
+                # Skip if already downloaded
+                if image_id in self.downloaded:
+                    skipped += 1
+                    continue
+
                 # Download image
                 image_url = image.get(quality_field)
                 if not image_url:
-                    print(f"No {quality} URL for image {image_id}")
+                    logger.warning(f"No {quality} URL for image {image_id}")
                     continue
 
                 # Use sequence ID for organization
@@ -99,21 +160,33 @@ class MapillaryDownloader:
 
                 output_path = img_dir / f"{image_id}.jpg"
 
+                download_start = time.time()
                 bytes_downloaded = self.client.download_image(image_url, output_path)
                 if bytes_downloaded:
+                    download_time = time.time() - download_start
+                    download_times.append(download_time)
+
                     # Write EXIF metadata to the downloaded image
                     write_exif_to_image(output_path, image)
 
                     self.downloaded.add(image_id)
                     downloaded_count += 1
                     total_bytes += bytes_downloaded
-                    print(f"Processed: {processed}, Downloaded: {downloaded_count} ({format_bytes(total_bytes)})")
+
+                    # Calculate progress
+                    progress_str = (
+                        f"Processed: {processed}, Downloaded: {downloaded_count} ({format_size(total_bytes)})"
+                    )
+
+                    logger.info(progress_str)
 
                     # Save progress every 10 images
                     if downloaded_count % 10 == 0:
                         self._save_progress()
 
         self._save_progress()
-        print(
-            f"\nComplete! Processed {processed} images, downloaded {downloaded_count} ({format_bytes(total_bytes)}), skipped {skipped}"
+        elapsed = time.time() - start_time
+        logger.info(
+            f"Complete! Processed {processed} images, downloaded {downloaded_count} ({format_size(total_bytes)}), skipped {skipped}"
         )
+        logger.info(f"Total time: {format_time(elapsed)}")

mapillary_downloader/logging_config.py

@@ -0,0 +1,62 @@
+"""Logging configuration with colored output for TTY."""
+
+import logging
+import sys
+
+
+class ColoredFormatter(logging.Formatter):
+    """Formatter that adds color to log levels when output is a TTY."""
+
+    # ANSI color codes
+    COLORS = {
+        "ERROR": "\033[91m",  # Red
+        "WARNING": "\033[93m",  # Yellow
+        "INFO": "\033[92m",  # Green
+        "DEBUG": "\033[94m",  # Blue
+        "RESET": "\033[0m",
+    }
+
+    def __init__(self, fmt=None, datefmt=None, use_color=True):
+        """Initialize the formatter.
+
+        Args:
+            fmt: Log format string
+            datefmt: Date format string
+            use_color: Whether to use colored output
+        """
+        super().__init__(fmt, datefmt)
+        self.use_color = use_color and sys.stdout.isatty()
+
+    def format(self, record):
+        """Format the log record with colors if appropriate.
+
+        Args:
+            record: LogRecord to format
+
+        Returns:
+            Formatted log string
+        """
+        if self.use_color:
+            levelname = record.levelname
+            if levelname in self.COLORS:
+                record.levelname = f"{self.COLORS[levelname]}{levelname}{self.COLORS['RESET']}"
+
+        return super().format(record)
+
+
+def setup_logging(level=logging.INFO):
+    """Set up logging with timestamps and colored output.
+
+    Args:
+        level: Logging level to use
+    """
+    formatter = ColoredFormatter(fmt="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger("mapillary_downloader")
+    logger.setLevel(level)
+    logger.addHandler(handler)
+
+    return logger

mapillary_downloader/utils.py

@@ -0,0 +1,47 @@
+"""Utility functions for formatting and display."""
+
+
+def format_size(bytes_count):
+    """Format bytes as human-readable size.
+
+    Args:
+        bytes_count: Number of bytes
+
+    Returns:
+        Formatted string (e.g. "1.23 GB", "456.78 MB")
+    """
+    if bytes_count >= 1_000_000_000:
+        return f"{bytes_count / 1_000_000_000:.2f} GB"
+    if bytes_count >= 1_000_000:
+        return f"{bytes_count / 1_000_000:.2f} MB"
+    if bytes_count >= 1_000:
+        return f"{bytes_count / 1000:.2f} KB"
+    return f"{bytes_count} B"
+
+
+def format_time(seconds):
+    """Format seconds as human-readable time.
+
+    Args:
+        seconds: Number of seconds
+
+    Returns:
+        Formatted string (e.g. "2h 15m", "45m 30s", "30s")
+    """
+    if seconds < 60:
+        return f"{int(seconds)}s"
+
+    minutes = int(seconds / 60)
+    remaining_seconds = int(seconds % 60)
+
+    if minutes < 60:
+        if remaining_seconds > 0:
+            return f"{minutes}m {remaining_seconds}s"
+        return f"{minutes}m"
+
+    hours = int(minutes / 60)
+    remaining_minutes = minutes % 60
+
+    if remaining_minutes > 0:
+        return f"{hours}h {remaining_minutes}m"
+    return f"{hours}h"

{mapillary_downloader-0.1.1.dist-info → mapillary_downloader-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.1.1
+Version: 0.1.3
 Summary: Download your Mapillary data before it's gone
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
@@ -28,7 +28,7 @@ Project-URL: Issues, https://github.com/bitplane/mapillary_downloader/issues
 Project-URL: Repository, https://github.com/bitplane/mapillary_downloader
 Provides-Extra: dev
 
-# Mapillary Downloader
+# 🗺️ Mapillary Downloader
 
 Download your Mapillary data before it's gone.
 
@@ -49,28 +49,24 @@ make install
 First, get your Mapillary API access token from https://www.mapillary.com/dashboard/developers
 
 ```bash
-mapillary-download --token YOUR_TOKEN --username YOUR_USERNAME --output ./downloads
+mapillary-downloader --token YOUR_TOKEN --username YOUR_USERNAME --output ./downloads
 ```
 
-Options:
-- `--token`: Your Mapillary API access token (required)
-- `--username`: Your Mapillary username (required)
-- `--output`: Output directory (default: ./mapillary_data)
-- `--quality`: Image quality - 256, 1024, 2048, or original (default: original)
-- `--bbox`: Bounding box filter: west,south,east,north
+| option        | because                               | default            |
+| ------------- | ------------------------------------- | ------------------ |
+| `--token`     | Your Mapillary API access token       | None (required)    |
+| `--username`  | Your Mapillary username               | None (required)    |
+| `--output`    | Output directory                      | `./mapillary_data` |
+| `--quality`   | 256, 1024, 2048 or original           | `original`         |
+| `--bbox`      | `west,south,east,north`               | `None`             |
 
 The downloader will:
-- Fetch all your uploaded images from Mapillary
-- Download full-resolution images organized by sequence
-- Inject EXIF metadata (GPS coordinates, camera info, timestamps, compass direction)
-- Save progress so you can safely resume if interrupted
 
-## Features
-
-- **Resume capability**: Interrupt and restart anytime - it tracks what's downloaded
-- **EXIF restoration**: Restores GPS, camera, and timestamp metadata that Mapillary stripped
-- **Atomic writes**: Progress tracking uses atomic file operations to prevent corruption
-- **Organized output**: Images organized by sequence ID with metadata in JSONL format
+* 💾 Fetch all your uploaded images from Mapillary
+* 📷 Download full-resolution images organized by sequence
+* 📜 Inject EXIF metadata (GPS coordinates, camera info, timestamps,
+  compass direction)
+* 🛟 Save progress so you can safely resume if interrupted
 
 ## Development
 
@@ -93,5 +89,6 @@ WTFPL with one additional clause
 
 1. Don't blame me
 
-Do wtf you want, but don't blame me when it breaks.
+Do wtf you want, but don't blame me if it makes jokes about the size of your
+disk drive.
 

mapillary_downloader-0.1.3.dist-info/RECORD

@@ -0,0 +1,12 @@
+mapillary_downloader/__init__.py,sha256=KEjiBRghXDeA7E15RJeLBfQm-yNJkowZarL59QOh_1w,120
+mapillary_downloader/__main__.py,sha256=i6nqTnH-K5LYbjUo_tgFvO7JkepchQyyChfqi9tyAE4,1728
+mapillary_downloader/client.py,sha256=O7JgshaM3QKUv0xXuBbe_uPqsTr4lgyuVUHYndvXTfA,4611
+mapillary_downloader/downloader.py,sha256=_AT1U2Y3FmGYDP0aOTQYVO3U_JDEl2nfvZyrgwCMZv0,7276
+mapillary_downloader/exif_writer.py,sha256=Bn1u3QULfHtae86FnUGcqN450NccJwtwW9wVaSRyx9E,4615
+mapillary_downloader/logging_config.py,sha256=WgjXthS2Lpl0RPO5sb-Aj60W2maog1bbzJzhblMLs1c,1715
+mapillary_downloader/utils.py,sha256=yzVgS1mwsklDAqrimaFafgTTXtRYQUbKP98Xgh9d2KA,1174
+mapillary_downloader-0.1.3.dist-info/entry_points.txt,sha256=PdYtxOXHMJrUhmiPO4G-F98VuhUI4MN9D_T4KPrVZ5w,75
+mapillary_downloader-0.1.3.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
+mapillary_downloader-0.1.3.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+mapillary_downloader-0.1.3.dist-info/METADATA,sha256=I81NFr74xyJPKbZfC91sGOHzyIOalVmL6KNkq3YLL3o,3060
+mapillary_downloader-0.1.3.dist-info/RECORD,,

mapillary_downloader-0.1.3.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+mapillary-downloader=mapillary_downloader.__main__:main
+

mapillary_downloader-0.1.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-mapillary_downloader/__init__.py,sha256=KEjiBRghXDeA7E15RJeLBfQm-yNJkowZarL59QOh_1w,120
-mapillary_downloader/__main__.py,sha256=xKYhamK0HYXqx98fGb5CVOEw0syURWgX7jnFIdsK5Ao,1720
-mapillary_downloader/client.py,sha256=w936htOireWnKgfXJDYls-umYxOzr0FQ4yzklQ6fPM0,4315
-mapillary_downloader/downloader.py,sha256=n5Y7aAoin3vBa_H3et9hpTNoPrEarbU_LdnHT619c5Y,4216
-mapillary_downloader/exif_writer.py,sha256=Bn1u3QULfHtae86FnUGcqN450NccJwtwW9wVaSRyx9E,4615
-mapillary_downloader-0.1.1.dist-info/entry_points.txt,sha256=qJK_zi0at7Mi6uzrcQ2U8Q31bG9NhRfhKphvIytjiIY,73
-mapillary_downloader-0.1.1.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
-mapillary_downloader-0.1.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-mapillary_downloader-0.1.1.dist-info/METADATA,sha256=ZoxfQ9-RBDxfyl1ws9eGIprLwZMqX3kumH20ttzf8GU,3114
-mapillary_downloader-0.1.1.dist-info/RECORD,,

mapillary_downloader-0.1.1.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-mapillary-download=mapillary_downloader.__main__:main
-