mapillary-downloader 0.1.2__tar.gz → 0.1.3__tar.gz

{mapillary_downloader-0.1.2 → mapillary_downloader-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.1.2
+Version: 0.1.3
 Summary: Download your Mapillary data before it's gone
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
@@ -28,7 +28,7 @@ Project-URL: Issues, https://github.com/bitplane/mapillary_downloader/issues
 Project-URL: Repository, https://github.com/bitplane/mapillary_downloader
 Provides-Extra: dev
 
-# Mapillary Downloader
+# 🗺️ Mapillary Downloader
 
 Download your Mapillary data before it's gone.
 
@@ -63,9 +63,10 @@ mapillary-downloader --token YOUR_TOKEN --username YOUR_USERNAME --output ./down
 The downloader will:
 
 * 💾 Fetch all your uploaded images from Mapillary
-- 📷 Download full-resolution images organized by sequence
-- 📜 Inject EXIF metadata (GPS coordinates, camera info, timestamps, compass direction)
-- 🛟 Save progress so you can safely resume if interrupted
+* 📷 Download full-resolution images organized by sequence
+* 📜 Inject EXIF metadata (GPS coordinates, camera info, timestamps,
+  compass direction)
+* 🛟 Save progress so you can safely resume if interrupted
 
 ## Development
 
@@ -88,5 +89,6 @@ WTFPL with one additional clause
 
 1. Don't blame me
 
-Do wtf you want, but don't blame me when it breaks.
+Do wtf you want, but don't blame me if it makes jokes about the size of your
+disk drive.
 

{mapillary_downloader-0.1.2 → mapillary_downloader-0.1.3}/README.md

@@ -1,4 +1,4 @@
-# Mapillary Downloader
+# 🗺️ Mapillary Downloader
 
 Download your Mapillary data before it's gone.
 
@@ -33,9 +33,10 @@ mapillary-downloader --token YOUR_TOKEN --username YOUR_USERNAME --output ./down
 The downloader will:
 
 * 💾 Fetch all your uploaded images from Mapillary
-- 📷 Download full-resolution images organized by sequence
-- 📜 Inject EXIF metadata (GPS coordinates, camera info, timestamps, compass direction)
-- 🛟 Save progress so you can safely resume if interrupted
+* 📷 Download full-resolution images organized by sequence
+* 📜 Inject EXIF metadata (GPS coordinates, camera info, timestamps,
+  compass direction)
+* 🛟 Save progress so you can safely resume if interrupted
 
 ## Development
 
@@ -58,4 +59,5 @@ WTFPL with one additional clause
 
 1. Don't blame me
 
-Do wtf you want, but don't blame me when it breaks.
+Do wtf you want, but don't blame me if it makes jokes about the size of your
+disk drive.

{mapillary_downloader-0.1.2 → mapillary_downloader-0.1.3}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mapillary_downloader"
 description = "Download your Mapillary data before it's gone"
-version = "0.1.2"
+version = "0.1.3"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

{mapillary_downloader-0.1.2 → mapillary_downloader-0.1.3}/src/mapillary_downloader/__main__.py

@@ -4,38 +4,25 @@ import argparse
 import sys
 from mapillary_downloader.client import MapillaryClient
 from mapillary_downloader.downloader import MapillaryDownloader
+from mapillary_downloader.logging_config import setup_logging
 
 
 def main():
     """Main CLI entry point."""
-    parser = argparse.ArgumentParser(
-        description="Download your Mapillary data before it's gone"
-    )
-    parser.add_argument(
-        "--token",
-        required=True,
-        help="Mapillary API access token"
-    )
-    parser.add_argument(
-        "--username",
-        required=True,
-        help="Your Mapillary username"
-    )
-    parser.add_argument(
-        "--output",
-        default="./mapillary_data",
-        help="Output directory (default: ./mapillary_data)"
-    )
+    # Set up logging
+    logger = setup_logging()
+
+    parser = argparse.ArgumentParser(description="Download your Mapillary data before it's gone")
+    parser.add_argument("--token", required=True, help="Mapillary API access token")
+    parser.add_argument("--username", required=True, help="Your Mapillary username")
+    parser.add_argument("--output", default="./mapillary_data", help="Output directory (default: ./mapillary_data)")
     parser.add_argument(
         "--quality",
         choices=["256", "1024", "2048", "original"],
         default="original",
-        help="Image quality to download (default: original)"
-    )
-    parser.add_argument(
-        "--bbox",
-        help="Bounding box: west,south,east,north"
+        help="Image quality to download (default: original)",
     )
+    parser.add_argument("--bbox", help="Bounding box: west,south,east,north")
 
     args = parser.parse_args()
 
@@ -46,7 +33,7 @@ def main():
             if len(bbox) != 4:
                 raise ValueError
         except ValueError:
-            print("Error: bbox must be four comma-separated numbers")
+            logger.error("Error: bbox must be four comma-separated numbers")
             sys.exit(1)
 
     try:
@@ -54,10 +41,10 @@ def main():
         downloader = MapillaryDownloader(client, args.output)
         downloader.download_user_data(args.username, args.quality, bbox)
     except KeyboardInterrupt:
-        print("\nInterrupted by user")
+        logger.info("\nInterrupted by user")
         sys.exit(1)
     except Exception as e:
-        print(f"Error: {e}")
+        logger.error(f"Error: {e}")
         sys.exit(1)
 
 

{mapillary_downloader-0.1.2 → mapillary_downloader-0.1.3}/src/mapillary_downloader/client.py

@@ -1,9 +1,12 @@
 """Mapillary API client."""
 
+import logging
 import time
 import requests
 from requests.exceptions import RequestException
 
+logger = logging.getLogger("mapillary_downloader")
+
 
 class MapillaryClient:
     """Client for interacting with Mapillary API v4."""
@@ -65,6 +68,7 @@ class MapillaryClient:
             params["bbox"] = ",".join(map(str, bbox))
 
         url = f"{self.base_url}/images"
+        total_fetched = 0
 
         while url:
             max_retries = 10
@@ -72,21 +76,24 @@ class MapillaryClient:
 
             for attempt in range(max_retries):
                 try:
-                    response = self.session.get(url, params=params)
+                    response = self.session.get(url, params=params, timeout=60)
                     response.raise_for_status()
                     break
                 except RequestException as e:
                     if attempt == max_retries - 1:
                         raise
 
-                    delay = base_delay * (2 ** attempt)
-                    print(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
-                    print(f"Retrying in {delay:.1f} seconds...")
+                    delay = base_delay * (2**attempt)
+                    logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
+                    logger.info(f"Retrying in {delay:.1f} seconds...")
                     time.sleep(delay)
 
             data = response.json()
+            images = data.get("data", [])
+            total_fetched += len(images)
+            logger.info(f"Fetched metadata for {total_fetched:,} images...")
 
-            for image in data.get("data", []):
+            for image in images:
                 yield image
 
             # Get next page URL
@@ -111,7 +118,7 @@ class MapillaryClient:
 
         for attempt in range(max_retries):
             try:
-                response = self.session.get(image_url, stream=True)
+                response = self.session.get(image_url, stream=True, timeout=60)
                 response.raise_for_status()
 
                 total_bytes = 0
@@ -123,10 +130,10 @@ class MapillaryClient:
                 return total_bytes
             except RequestException as e:
                 if attempt == max_retries - 1:
-                    print(f"Error downloading {image_url} after {max_retries} attempts: {e}")
+                    logger.error(f"Error downloading {image_url} after {max_retries} attempts: {e}")
                     return 0
 
-                delay = base_delay * (2 ** attempt)
-                print(f"Download failed (attempt {attempt + 1}/{max_retries}): {e}")
-                print(f"Retrying in {delay:.1f} seconds...")
+                delay = base_delay * (2**attempt)
+                logger.warning(f"Download failed (attempt {attempt + 1}/{max_retries}): {e}")
+                logger.info(f"Retrying in {delay:.1f} seconds...")
                 time.sleep(delay)

mapillary_downloader-0.1.3/src/mapillary_downloader/downloader.py

@@ -0,0 +1,192 @@
+"""Main downloader logic."""
+
+import json
+import logging
+import os
+import time
+from pathlib import Path
+from collections import deque
+from mapillary_downloader.exif_writer import write_exif_to_image
+from mapillary_downloader.utils import format_size, format_time
+
+logger = logging.getLogger("mapillary_downloader")
+
+
+class MapillaryDownloader:
+    """Handles downloading Mapillary data for a user."""
+
+    def __init__(self, client, output_dir):
+        """Initialize the downloader.
+
+        Args:
+            client: MapillaryClient instance
+            output_dir: Directory to save downloads
+        """
+        self.client = client
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        self.metadata_file = self.output_dir / "metadata.jsonl"
+        self.progress_file = self.output_dir / "progress.json"
+        self.downloaded = self._load_progress()
+
+    def _load_progress(self):
+        """Load previously downloaded image IDs."""
+        if self.progress_file.exists():
+            with open(self.progress_file) as f:
+                return set(json.load(f).get("downloaded", []))
+        return set()
+
+    def _save_progress(self):
+        """Save progress to disk atomically."""
+        temp_file = self.progress_file.with_suffix(".json.tmp")
+        with open(temp_file, "w") as f:
+            json.dump({"downloaded": list(self.downloaded)}, f)
+            f.flush()
+            os.fsync(f.fileno())
+        temp_file.replace(self.progress_file)
+
+    def download_user_data(self, username, quality="original", bbox=None):
+        """Download all images for a user.
+
+        Args:
+            username: Mapillary username
+            quality: Image quality to download (256, 1024, 2048, original)
+            bbox: Optional bounding box [west, south, east, north]
+        """
+        quality_field = f"thumb_{quality}_url"
+
+        logger.info(f"Downloading images for user: {username}")
+        logger.info(f"Output directory: {self.output_dir}")
+        logger.info(f"Quality: {quality}")
+
+        processed = 0
+        downloaded_count = 0
+        skipped = 0
+        total_bytes = 0
+
+        # Track download times for adaptive ETA (last 50 downloads)
+        download_times = deque(maxlen=50)
+        start_time = time.time()
+
+        # Track which image IDs we've seen in metadata to avoid re-fetching
+        seen_ids = set()
+
+        # First, process any existing metadata without re-fetching from API
+        if self.metadata_file.exists():
+            logger.info("Processing existing metadata file...")
+            with open(self.metadata_file) as f:
+                for line in f:
+                    if line.strip():
+                        image = json.loads(line)
+                        image_id = image["id"]
+                        seen_ids.add(image_id)
+                        processed += 1
+
+                        if image_id in self.downloaded:
+                            skipped += 1
+                            continue
+
+                        # Download this un-downloaded image
+                        image_url = image.get(quality_field)
+                        if not image_url:
+                            logger.warning(f"No {quality} URL for image {image_id}")
+                            continue
+
+                        sequence_id = image.get("sequence")
+                        if sequence_id:
+                            img_dir = self.output_dir / sequence_id
+                            img_dir.mkdir(exist_ok=True)
+                        else:
+                            img_dir = self.output_dir
+
+                        output_path = img_dir / f"{image_id}.jpg"
+
+                        download_start = time.time()
+                        bytes_downloaded = self.client.download_image(image_url, output_path)
+                        if bytes_downloaded:
+                            download_time = time.time() - download_start
+                            download_times.append(download_time)
+
+                            write_exif_to_image(output_path, image)
+
+                            self.downloaded.add(image_id)
+                            downloaded_count += 1
+                            total_bytes += bytes_downloaded
+
+                            progress_str = (
+                                f"Processed: {processed}, Downloaded: {downloaded_count} ({format_size(total_bytes)})"
+                            )
+                            logger.info(progress_str)
+
+                            if downloaded_count % 10 == 0:
+                                self._save_progress()
+
+        # Always check API for new images (will skip duplicates via seen_ids)
+        logger.info("Checking for new images from API...")
+        with open(self.metadata_file, "a") as meta_f:
+            for image in self.client.get_user_images(username, bbox=bbox):
+                image_id = image["id"]
+
+                # Skip if we already have this in our metadata file
+                if image_id in seen_ids:
+                    continue
+
+                seen_ids.add(image_id)
+                processed += 1
+
+                # Save new metadata
+                meta_f.write(json.dumps(image) + "\n")
+                meta_f.flush()
+
+                # Skip if already downloaded
+                if image_id in self.downloaded:
+                    skipped += 1
+                    continue
+
+                # Download image
+                image_url = image.get(quality_field)
+                if not image_url:
+                    logger.warning(f"No {quality} URL for image {image_id}")
+                    continue
+
+                # Use sequence ID for organization
+                sequence_id = image.get("sequence")
+                if sequence_id:
+                    img_dir = self.output_dir / sequence_id
+                    img_dir.mkdir(exist_ok=True)
+                else:
+                    img_dir = self.output_dir
+
+                output_path = img_dir / f"{image_id}.jpg"
+
+                download_start = time.time()
+                bytes_downloaded = self.client.download_image(image_url, output_path)
+                if bytes_downloaded:
+                    download_time = time.time() - download_start
+                    download_times.append(download_time)
+
+                    # Write EXIF metadata to the downloaded image
+                    write_exif_to_image(output_path, image)
+
+                    self.downloaded.add(image_id)
+                    downloaded_count += 1
+                    total_bytes += bytes_downloaded
+
+                    # Calculate progress
+                    progress_str = (
+                        f"Processed: {processed}, Downloaded: {downloaded_count} ({format_size(total_bytes)})"
+                    )
+
+                    logger.info(progress_str)
+
+                    # Save progress every 10 images
+                    if downloaded_count % 10 == 0:
+                        self._save_progress()
+
+        self._save_progress()
+        elapsed = time.time() - start_time
+        logger.info(
+            f"Complete! Processed {processed} images, downloaded {downloaded_count} ({format_size(total_bytes)}), skipped {skipped}"
+        )
+        logger.info(f"Total time: {format_time(elapsed)}")

mapillary_downloader-0.1.3/src/mapillary_downloader/logging_config.py

@@ -0,0 +1,62 @@
+"""Logging configuration with colored output for TTY."""
+
+import logging
+import sys
+
+
+class ColoredFormatter(logging.Formatter):
+    """Formatter that adds color to log levels when output is a TTY."""
+
+    # ANSI color codes
+    COLORS = {
+        "ERROR": "\033[91m",  # Red
+        "WARNING": "\033[93m",  # Yellow
+        "INFO": "\033[92m",  # Green
+        "DEBUG": "\033[94m",  # Blue
+        "RESET": "\033[0m",
+    }
+
+    def __init__(self, fmt=None, datefmt=None, use_color=True):
+        """Initialize the formatter.
+
+        Args:
+            fmt: Log format string
+            datefmt: Date format string
+            use_color: Whether to use colored output
+        """
+        super().__init__(fmt, datefmt)
+        self.use_color = use_color and sys.stdout.isatty()
+
+    def format(self, record):
+        """Format the log record with colors if appropriate.
+
+        Args:
+            record: LogRecord to format
+
+        Returns:
+            Formatted log string
+        """
+        if self.use_color:
+            levelname = record.levelname
+            if levelname in self.COLORS:
+                record.levelname = f"{self.COLORS[levelname]}{levelname}{self.COLORS['RESET']}"
+
+        return super().format(record)
+
+
+def setup_logging(level=logging.INFO):
+    """Set up logging with timestamps and colored output.
+
+    Args:
+        level: Logging level to use
+    """
+    formatter = ColoredFormatter(fmt="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger("mapillary_downloader")
+    logger.setLevel(level)
+    logger.addHandler(handler)
+
+    return logger

mapillary_downloader-0.1.3/src/mapillary_downloader/utils.py

@@ -0,0 +1,47 @@
+"""Utility functions for formatting and display."""
+
+
+def format_size(bytes_count):
+    """Format bytes as human-readable size.
+
+    Args:
+        bytes_count: Number of bytes
+
+    Returns:
+        Formatted string (e.g. "1.23 GB", "456.78 MB")
+    """
+    if bytes_count >= 1_000_000_000:
+        return f"{bytes_count / 1_000_000_000:.2f} GB"
+    if bytes_count >= 1_000_000:
+        return f"{bytes_count / 1_000_000:.2f} MB"
+    if bytes_count >= 1_000:
+        return f"{bytes_count / 1000:.2f} KB"
+    return f"{bytes_count} B"
+
+
+def format_time(seconds):
+    """Format seconds as human-readable time.
+
+    Args:
+        seconds: Number of seconds
+
+    Returns:
+        Formatted string (e.g. "2h 15m", "45m 30s", "30s")
+    """
+    if seconds < 60:
+        return f"{int(seconds)}s"
+
+    minutes = int(seconds / 60)
+    remaining_seconds = int(seconds % 60)
+
+    if minutes < 60:
+        if remaining_seconds > 0:
+            return f"{minutes}m {remaining_seconds}s"
+        return f"{minutes}m"
+
+    hours = int(minutes / 60)
+    remaining_minutes = minutes % 60
+
+    if remaining_minutes > 0:
+        return f"{hours}h {remaining_minutes}m"
+    return f"{hours}h"

mapillary_downloader-0.1.2/src/mapillary_downloader/downloader.py

@@ -1,119 +0,0 @@
-"""Main downloader logic."""
-
-import json
-import os
-from pathlib import Path
-from mapillary_downloader.exif_writer import write_exif_to_image
-
-
-def format_bytes(bytes_count):
-    """Format bytes as human-readable string."""
-    if bytes_count < 1024:
-        return f"{bytes_count} B"
-    if bytes_count < 1024 * 1024:
-        return f"{bytes_count / 1024:.3f} KB"
-    if bytes_count < 1024 * 1024 * 1024:
-        return f"{bytes_count / (1024 * 1024):.3f} MB"
-    return f"{bytes_count / (1024 * 1024 * 1024):.3f} GB"
-
-
-class MapillaryDownloader:
-    """Handles downloading Mapillary data for a user."""
-
-    def __init__(self, client, output_dir):
-        """Initialize the downloader.
-
-        Args:
-            client: MapillaryClient instance
-            output_dir: Directory to save downloads
-        """
-        self.client = client
-        self.output_dir = Path(output_dir)
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-
-        self.metadata_file = self.output_dir / "metadata.jsonl"
-        self.progress_file = self.output_dir / "progress.json"
-        self.downloaded = self._load_progress()
-
-    def _load_progress(self):
-        """Load previously downloaded image IDs."""
-        if self.progress_file.exists():
-            with open(self.progress_file) as f:
-                return set(json.load(f).get("downloaded", []))
-        return set()
-
-    def _save_progress(self):
-        """Save progress to disk atomically."""
-        temp_file = self.progress_file.with_suffix(".json.tmp")
-        with open(temp_file, "w") as f:
-            json.dump({"downloaded": list(self.downloaded)}, f)
-            f.flush()
-            os.fsync(f.fileno())
-        temp_file.replace(self.progress_file)
-
-    def download_user_data(self, username, quality="original", bbox=None):
-        """Download all images for a user.
-
-        Args:
-            username: Mapillary username
-            quality: Image quality to download (256, 1024, 2048, original)
-            bbox: Optional bounding box [west, south, east, north]
-        """
-        quality_field = f"thumb_{quality}_url"
-
-        print(f"Downloading images for user: {username}")
-        print(f"Output directory: {self.output_dir}")
-        print(f"Quality: {quality}")
-
-        processed = 0
-        downloaded_count = 0
-        skipped = 0
-        total_bytes = 0
-
-        with open(self.metadata_file, "a") as meta_f:
-            for image in self.client.get_user_images(username, bbox=bbox):
-                image_id = image["id"]
-                processed += 1
-
-                if image_id in self.downloaded:
-                    skipped += 1
-                    continue
-
-                # Save metadata
-                meta_f.write(json.dumps(image) + "\n")
-                meta_f.flush()
-
-                # Download image
-                image_url = image.get(quality_field)
-                if not image_url:
-                    print(f"No {quality} URL for image {image_id}")
-                    continue
-
-                # Use sequence ID for organization
-                sequence_id = image.get("sequence")
-                if sequence_id:
-                    img_dir = self.output_dir / sequence_id
-                    img_dir.mkdir(exist_ok=True)
-                else:
-                    img_dir = self.output_dir
-
-                output_path = img_dir / f"{image_id}.jpg"
-
-                bytes_downloaded = self.client.download_image(image_url, output_path)
-                if bytes_downloaded:
-                    # Write EXIF metadata to the downloaded image
-                    write_exif_to_image(output_path, image)
-
-                    self.downloaded.add(image_id)
-                    downloaded_count += 1
-                    total_bytes += bytes_downloaded
-                    print(f"Processed: {processed}, Downloaded: {downloaded_count} ({format_bytes(total_bytes)})")
-
-                    # Save progress every 10 images
-                    if downloaded_count % 10 == 0:
-                        self._save_progress()
-
-        self._save_progress()
-        print(
-            f"\nComplete! Processed {processed} images, downloaded {downloaded_count} ({format_bytes(total_bytes)}), skipped {skipped}"
-        )