mapillary-downloader 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

mapillary_downloader/__main__.py

@@ -3,6 +3,7 @@
 import argparse
 import os
 import sys
+from importlib.metadata import version
 from mapillary_downloader.client import MapillaryClient
 from mapillary_downloader.downloader import MapillaryDownloader
 from mapillary_downloader.logging_config import setup_logging
@@ -15,6 +16,11 @@ def main():
     logger = setup_logging()
 
     parser = argparse.ArgumentParser(description="Download your Mapillary data before it's gone")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {version('mapillary-downloader')}",
+    )
     parser.add_argument(
         "--token",
         default=os.environ.get("MAPILLARY_TOKEN"),
@@ -38,7 +44,7 @@ def main():
         "--workers",
         type=int,
         default=None,
-        help="Number of parallel workers (default: number of CPU cores)",
+        help="Number of parallel workers (default: half of CPU cores)",
     )
     parser.add_argument(
         "--no-tar",

mapillary_downloader/downloader.py

@@ -10,6 +10,7 @@ from mapillary_downloader.utils import format_size, format_time
 from mapillary_downloader.ia_meta import generate_ia_metadata
 from mapillary_downloader.worker import download_and_convert_image
 from mapillary_downloader.tar_sequences import tar_sequence_directories
+from mapillary_downloader.logging_config import add_file_handler
 
 logger = logging.getLogger("mapillary_downloader")
 
@@ -25,14 +26,14 @@ class MapillaryDownloader:
             output_dir: Base directory to save downloads
             username: Mapillary username (for collection directory)
             quality: Image quality (for collection directory)
-            workers: Number of parallel workers (default: cpu_count)
+            workers: Number of parallel workers (default: half of cpu_count)
             tar_sequences: Whether to tar sequence directories after download (default: True)
         """
         self.client = client
         self.base_output_dir = Path(output_dir)
         self.username = username
         self.quality = quality
-        self.workers = workers if workers is not None else os.cpu_count()
+        self.workers = workers if workers is not None else max(1, os.cpu_count() // 2)
         self.tar_sequences = tar_sequences
 
         # If username and quality provided, create collection directory
@@ -44,6 +45,11 @@ class MapillaryDownloader:
 
         self.output_dir.mkdir(parents=True, exist_ok=True)
 
+        # Set up file logging for archival
+        log_file = self.output_dir / "download.log"
+        add_file_handler(log_file)
+        logger.info(f"Logging to: {log_file}")
+
         self.metadata_file = self.output_dir / "metadata.jsonl"
         self.progress_file = self.output_dir / "progress.json"
         self.downloaded = self._load_progress()

mapillary_downloader/ia_meta.py

@@ -135,7 +135,7 @@ def generate_ia_metadata(collection_dir):
     write_meta_tag(
         meta_dir,
         "title",
-        f"Mapillary images by {username} ({quality} quality)",
+        f"Mapillary images by {username}",
     )
 
     description = (

mapillary_downloader/logging_config.py

@@ -60,3 +60,23 @@ def setup_logging(level=logging.INFO):
     logger.addHandler(handler)
 
     return logger
+
+
+def add_file_handler(log_file, level=logging.INFO):
+    """Add a file handler to the logger for archival.
+
+    Args:
+        log_file: Path to log file
+        level: Logging level for file handler
+    """
+    # Use plain formatter for file (no colors)
+    formatter = logging.Formatter(fmt="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+
+    handler = logging.FileHandler(log_file, mode="a", encoding="utf-8")
+    handler.setFormatter(formatter)
+    handler.setLevel(level)
+
+    logger = logging.getLogger("mapillary_downloader")
+    logger.addHandler(handler)
+
+    return handler

mapillary_downloader/worker.py

@@ -1,10 +1,10 @@
 """Worker process for parallel image download and conversion."""
 
+import os
 import tempfile
 from pathlib import Path
 import requests
 from requests.exceptions import RequestException
-import time
 from mapillary_downloader.exif_writer import write_exif_to_image
 from mapillary_downloader.webp_converter import convert_to_webp
 
@@ -54,29 +54,23 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, ac
             final_path = jpg_path
 
         # Download image
+        # No retries for CDN images - they're cheap, just skip failures and move on
         session = requests.Session()
         session.headers.update({"Authorization": f"OAuth {access_token}"})
 
-        max_retries = 10
-        base_delay = 1.0
         bytes_downloaded = 0
 
-        for attempt in range(max_retries):
-            try:
-                response = session.get(image_url, stream=True, timeout=60)
-                response.raise_for_status()
-
-                with open(jpg_path, "wb") as f:
-                    for chunk in response.iter_content(chunk_size=8192):
-                        f.write(chunk)
-                        bytes_downloaded += len(chunk)
-                break
-            except RequestException as e:
-                if attempt == max_retries - 1:
-                    return (image_id, 0, False, f"Download failed: {e}")
+        try:
+            # 60 second timeout for entire download (connection + read)
+            response = session.get(image_url, stream=True, timeout=60)
+            response.raise_for_status()
 
-                delay = base_delay * (2**attempt)
-                time.sleep(delay)
+            with open(jpg_path, "wb") as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+                    bytes_downloaded += len(chunk)
+        except RequestException as e:
+            return (image_id, 0, False, f"Download failed: {e}")
 
         # Write EXIF metadata
         write_exif_to_image(jpg_path, image_data)
@@ -87,6 +81,12 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, ac
             if not webp_path:
                 return (image_id, bytes_downloaded, False, "WebP conversion failed")
 
+        # Set file mtime to captured_at timestamp for reproducibility
+        if "captured_at" in image_data:
+            # captured_at is in milliseconds, convert to seconds
+            mtime = image_data["captured_at"] / 1000
+            os.utime(final_path, (mtime, mtime))
+
         return (image_id, bytes_downloaded, True, None)
 
     except Exception as e:

{mapillary_downloader-0.3.0.dist-info → mapillary_downloader-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.3.0
+Version: 0.3.2
 Summary: Download your Mapillary data before it's gone
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
@@ -63,7 +63,7 @@ mapillary-downloader --token YOUR_TOKEN --username SOME_USERNAME --output ./down
 | `--quality`   | 256, 1024, 2048 or original           | `original`         |
 | `--bbox`      | `west,south,east,north`               | `None`             |
 | `--webp`      | Convert to WebP (saves ~70% space)    | `False`            |
-| `--workers`   | Number of parallel download workers   | CPU count          |
+| `--workers`   | Number of parallel download workers   | Half of CPU count  |
 | `--no-tar`    | Don't tar sequence directories        | `False`            |
 
 The downloader will:

mapillary_downloader-0.3.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+mapillary_downloader/__init__.py,sha256=KEjiBRghXDeA7E15RJeLBfQm-yNJkowZarL59QOh_1w,120
+mapillary_downloader/__main__.py,sha256=Zvvib_XFhErPLY6AWviQ83iT0TiXFHWV4CDJ5vtzNYQ,3187
+mapillary_downloader/client.py,sha256=O7JgshaM3QKUv0xXuBbe_uPqsTr4lgyuVUHYndvXTfA,4611
+mapillary_downloader/downloader.py,sha256=JmR-0HbCBXGoRylgVzVbEdvlRey3pklZffhf2BRCw9s,8476
+mapillary_downloader/exif_writer.py,sha256=Bn1u3QULfHtae86FnUGcqN450NccJwtwW9wVaSRyx9E,4615
+mapillary_downloader/ia_meta.py,sha256=zUG23Vv2X8hFa_oNL_wfvFVt7yqkQkvXiZ4feoKt7S0,5596
+mapillary_downloader/logging_config.py,sha256=Z-wNq34nt7aIhJWdeKc1feTY46P9-Or7HtiX7eUFjEI,2324
+mapillary_downloader/tar_sequences.py,sha256=X5Vru0vp4pm_unmFjdEIZ7SBIk0gwi49NwOLPhUKeYk,3809
+mapillary_downloader/utils.py,sha256=yzVgS1mwsklDAqrimaFafgTTXtRYQUbKP98Xgh9d2KA,1174
+mapillary_downloader/webp_converter.py,sha256=vYLLQxDmdnqRz0nm7wXwRUd4x9mQZNah-DrncpA8sNs,1901
+mapillary_downloader/worker.py,sha256=eqaBhP5NE_VoJSTZfFb4OAqGyVX65xyoVUp2vosYBM8,3722
+mapillary_downloader-0.3.2.dist-info/entry_points.txt,sha256=PdYtxOXHMJrUhmiPO4G-F98VuhUI4MN9D_T4KPrVZ5w,75
+mapillary_downloader-0.3.2.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
+mapillary_downloader-0.3.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+mapillary_downloader-0.3.2.dist-info/METADATA,sha256=ckXRXc0rHTQdPleWCAo8oRwbu-FKJwOSdkc7g5DrKug,4607
+mapillary_downloader-0.3.2.dist-info/RECORD,,

mapillary_downloader-0.3.0.dist-info/RECORD

@@ -1,16 +0,0 @@
-mapillary_downloader/__init__.py,sha256=KEjiBRghXDeA7E15RJeLBfQm-yNJkowZarL59QOh_1w,120
-mapillary_downloader/__main__.py,sha256=TaJIsUQkM4vkAAujmInJu6Nqjrf9-yI9u5teh6d5U-E,3009
-mapillary_downloader/client.py,sha256=O7JgshaM3QKUv0xXuBbe_uPqsTr4lgyuVUHYndvXTfA,4611
-mapillary_downloader/downloader.py,sha256=U3a5oM1QC22DqfUFyBUFEbfZ5E2wqVTkHdlWMVsUlXA,8212
-mapillary_downloader/exif_writer.py,sha256=Bn1u3QULfHtae86FnUGcqN450NccJwtwW9wVaSRyx9E,4615
-mapillary_downloader/ia_meta.py,sha256=QX1jGCdKvUMFsr6xBZYwS78a0qd0eQI8ynJhIfLIJ5w,5616
-mapillary_downloader/logging_config.py,sha256=WgjXthS2Lpl0RPO5sb-Aj60W2maog1bbzJzhblMLs1c,1715
-mapillary_downloader/tar_sequences.py,sha256=X5Vru0vp4pm_unmFjdEIZ7SBIk0gwi49NwOLPhUKeYk,3809
-mapillary_downloader/utils.py,sha256=yzVgS1mwsklDAqrimaFafgTTXtRYQUbKP98Xgh9d2KA,1174
-mapillary_downloader/webp_converter.py,sha256=vYLLQxDmdnqRz0nm7wXwRUd4x9mQZNah-DrncpA8sNs,1901
-mapillary_downloader/worker.py,sha256=2_U4EaCWjq9WxtsvOYiu41KsTWTHxKb1m43-Qa-HC9M,3577
-mapillary_downloader-0.3.0.dist-info/entry_points.txt,sha256=PdYtxOXHMJrUhmiPO4G-F98VuhUI4MN9D_T4KPrVZ5w,75
-mapillary_downloader-0.3.0.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
-mapillary_downloader-0.3.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-mapillary_downloader-0.3.0.dist-info/METADATA,sha256=c6naco_sXoVlWkHwhrFTm2s5Gi4mt4l4JDQKkJXxCTg,4607
-mapillary_downloader-0.3.0.dist-info/RECORD,,