mapillary-downloader 0.6.1__tar.gz → 0.7.2__tar.gz

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.6.1
+Version: 0.7.2
 Summary: Archive user data from Mapillary
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
@@ -66,7 +66,7 @@ mapillary-downloader --output ./downloads USERNAME1
 | `--quality`     | 256, 1024, 2048 or original                  | `original`         |
 | `--bbox`        | `west,south,east,north`                      | `None`             |
 | `--no-webp`     | Don't convert to WebP                        | `False`            |
-| `--max-workers` | Maximum number of parallel download workers  | `128`              |
+| `--max-workers` | Maximum number of parallel download workers  | CPU count          |
 | `--no-tar`      | Don't tar bucket directories                 | `False`            |
 | `--no-check-ia` | Don't check if exists on Internet Archive    | `False`            |
 
@@ -100,21 +100,28 @@ mapillary-downloader --no-webp USERNAME
 
 ## Tarballs
 
-Images are organized by sequence ID, bucketed by the first character of the
-sequence to reduce directory count:
+Images are organized by capture date (YYYY-MM-DD) for incremental archiving:
 
 ```
 mapillary-username-quality/
-  a/
+  2024-01-15/
     abc123/
       image1.webp
       image2.webp
+    bcd456/
+      image3.webp
+  2024-01-16/
+    def789/
+      image4.webp
 ```
 
-By default, these bucket directories are automatically tarred after download
-(resulting in `a.tar`, `b.tar`, etc. - about 62 tar files total). This is done
-because large collections with millions of images would otherwise create hundreds
-of thousands of tiny tars, and anger the archive gods.
+By default, these date directories are automatically tarred after download
+(resulting in `2024-01-15.tar`, `2024-01-16.tar`, etc.). This date-based
+organization enables:
+
+- **Incremental uploads** - Upload each day's tar as soon as it's ready
+- **Manageable file counts** - ~365 days/year × 10 years = 3,650 tars max
+- **Chronological organization** - Natural sorting and progress tracking
 
 To keep individual files instead of creating tars, use the `--no-tar` flag.
 
@@ -128,8 +135,15 @@ See inlay for details:
 
 * [📀 rip](https://bitplane.net/dev/sh/rip)
 
+## 📊 Stats
+
+To see overall project progress, or an estimate, use `--stats`
+
+```bash
+mapillary-downloader --stats
+```
 
-## Development
+## 🚧 Development
 
 ```bash
 make dev      # Setup dev environment
@@ -138,7 +152,7 @@ make dist     # Build the distribution
 make help     # See other make options
 ```
 
-## Links
+## 🔗 Links
 
 * [🏠 home](https://bitplane.net/dev/python/mapillary_downloader)
   * [📖 pydoc](https://bitplane.net/dev/python/mapillary_downloader/pydoc)

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/README.md

@@ -36,7 +36,7 @@ mapillary-downloader --output ./downloads USERNAME1
 | `--quality`     | 256, 1024, 2048 or original                  | `original`         |
 | `--bbox`        | `west,south,east,north`                      | `None`             |
 | `--no-webp`     | Don't convert to WebP                        | `False`            |
-| `--max-workers` | Maximum number of parallel download workers  | `128`              |
+| `--max-workers` | Maximum number of parallel download workers  | CPU count          |
 | `--no-tar`      | Don't tar bucket directories                 | `False`            |
 | `--no-check-ia` | Don't check if exists on Internet Archive    | `False`            |
 
@@ -70,21 +70,28 @@ mapillary-downloader --no-webp USERNAME
 
 ## Tarballs
 
-Images are organized by sequence ID, bucketed by the first character of the
-sequence to reduce directory count:
+Images are organized by capture date (YYYY-MM-DD) for incremental archiving:
 
 ```
 mapillary-username-quality/
-  a/
+  2024-01-15/
     abc123/
       image1.webp
       image2.webp
+    bcd456/
+      image3.webp
+  2024-01-16/
+    def789/
+      image4.webp
 ```
 
-By default, these bucket directories are automatically tarred after download
-(resulting in `a.tar`, `b.tar`, etc. - about 62 tar files total). This is done
-because large collections with millions of images would otherwise create hundreds
-of thousands of tiny tars, and anger the archive gods.
+By default, these date directories are automatically tarred after download
+(resulting in `2024-01-15.tar`, `2024-01-16.tar`, etc.). This date-based
+organization enables:
+
+- **Incremental uploads** - Upload each day's tar as soon as it's ready
+- **Manageable file counts** - ~365 days/year × 10 years = 3,650 tars max
+- **Chronological organization** - Natural sorting and progress tracking
 
 To keep individual files instead of creating tars, use the `--no-tar` flag.
 
@@ -98,8 +105,15 @@ See inlay for details:
 
 * [📀 rip](https://bitplane.net/dev/sh/rip)
 
+## 📊 Stats
+
+To see overall project progress, or an estimate, use `--stats`
+
+```bash
+mapillary-downloader --stats
+```
 
-## Development
+## 🚧 Development
 
 ```bash
 make dev      # Setup dev environment
@@ -108,7 +122,7 @@ make dist     # Build the distribution
 make help     # See other make options
 ```
 
-## Links
+## 🔗 Links
 
 * [🏠 home](https://bitplane.net/dev/python/mapillary_downloader)
   * [📖 pydoc](https://bitplane.net/dev/python/mapillary_downloader/pydoc)

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mapillary_downloader"
 description = "Archive user data from Mapillary"
-version = "0.6.1"
+version = "0.7.2"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/src/mapillary_downloader/__main__.py

@@ -43,8 +43,8 @@ def main():
     parser.add_argument(
         "--max-workers",
         type=int,
-        default=128,
-        help="Maximum number of parallel workers (default: 128)",
+        default=os.cpu_count() or 8,
+        help=f"Maximum number of parallel workers (default: CPU count = {os.cpu_count() or 8})",
     )
     parser.add_argument(
         "--no-tar",

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/src/mapillary_downloader/downloader.py

@@ -67,7 +67,6 @@ class MapillaryDownloader:
         self.username = username
         self.quality = quality
         self.max_workers = max_workers
-        self.initial_workers = os.cpu_count() or 1  # Start with CPU count
         self.tar_sequences = tar_sequences
         self.convert_webp = convert_webp
         self.check_ia = check_ia
@@ -173,7 +172,7 @@ class MapillaryDownloader:
         logger.info(f"Downloading images for user: {self.username}")
         logger.info(f"Output directory: {self.output_dir}")
         logger.info(f"Quality: {self.quality}")
-        logger.info(f"Worker pool: {self.initial_workers} initial, {self.max_workers} max")
+        logger.info(f"Worker pool: max {self.max_workers} workers")
 
         start_time = time.time()
 
@@ -188,9 +187,7 @@ class MapillaryDownloader:
         # Step 2: Start worker pool
         # Since workers do both I/O (download) and CPU (WebP), need many more workers
         # Start with CPU count and scale up based on throughput
-        pool = AdaptiveWorkerPool(
-            worker_process, min_workers=self.initial_workers, max_workers=self.max_workers, monitoring_interval=10
-        )
+        pool = AdaptiveWorkerPool(worker_process, max_workers=self.max_workers, monitoring_interval=10)
         pool.start()
 
         # Step 3: Download images from metadata file while fetching new from API

mapillary_downloader-0.7.2/src/mapillary_downloader/graphql_web.py

@@ -0,0 +1,193 @@
+"""GraphQL web API utilities (unofficial, experimental).
+
+This module provides access to Mapillary's GraphQL endpoint used by the web interface.
+Unlike the official v4 REST API, this requires a public web token extracted from the
+JavaScript bundle.
+
+Use cases:
+- Get user image counts without pagination
+- Access leaderboard data
+- Check for updates to existing downloads
+
+WARNING: This is not officially documented and may break at any time.
+"""
+
+import json
+import logging
+import re
+from datetime import datetime
+from urllib.parse import urlencode, quote
+import requests
+
+logger = logging.getLogger("mapillary_downloader")
+
+# Fallback token (extracted from main JS bundle as of 2025-01-09)
+FALLBACK_TOKEN = "MLY|4223665974375089|d62822dd792b6a823d0794ef26450398"
+
+
+def extract_token_from_js():
+    """Extract public web token from Mapillary's JavaScript bundle.
+
+    This fetches the main page, finds the main JS bundle, and extracts
+    the hardcoded MLY token used for GraphQL queries.
+
+    Returns:
+        Token string (e.g., "MLY|123|abc...") or None if extraction failed
+    """
+    try:
+        # Fetch main page to find JS bundle URL
+        # Need consent cookie to get actual page (not GDPR banner)
+        logger.debug("Fetching Mapillary main page...")
+        # Generate today's date in the format YYYY_MM_DD for cookie
+        today = datetime.now().strftime("%Y_%m_%d")
+        cookies = {
+            "mly_cb": f'{{"version":"1","date":"{today}","third_party_consent":"withdrawn","categories":{{"content_and_media":"withdrawn"}},"integration_controls":{{"YOUTUBE":"withdrawn"}}}}'
+        }
+        headers = {
+            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.5",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+        }
+        response = requests.get("https://www.mapillary.com/app/", cookies=cookies, headers=headers, timeout=30)
+        response.raise_for_status()
+
+        # Find main JS file URL
+        # Pattern: <script src="main.{hash}.js" type="module"></script>
+        js_match = re.search(r'src="(main\.[a-f0-9]+\.js)"', response.text)
+        if not js_match:
+            logger.warning("Could not find main JS bundle URL in page")
+            return None
+
+        # URL is relative to /app/ base path
+        js_url = f"https://www.mapillary.com/app/{js_match.group(1)}"
+        logger.debug(f"Found JS bundle: {js_url}")
+
+        # Fetch JS bundle
+        logger.debug("Fetching JS bundle...")
+        js_response = requests.get(js_url, timeout=30)
+        js_response.raise_for_status()
+
+        # Extract token
+        # Pattern: "MLY|{client_id}|{secret}"
+        token_match = re.search(r'"(MLY\|[^"]+)"', js_response.text)
+        if not token_match:
+            logger.warning("Could not find MLY token in JS bundle")
+            return None
+
+        token = token_match.group(1)
+        logger.info(f"Extracted web token: {token[:20]}...")
+        return token
+
+    except requests.RequestException as e:
+        logger.error(f"Failed to extract web token: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Unexpected error extracting web token: {e}")
+        return None
+
+
+def get_leaderboard(key="global", token=None):
+    """Get leaderboard data from Mapillary GraphQL API.
+
+    Args:
+        key: Leaderboard key (e.g., "global", country name, etc.)
+        token: MLY token (if None, will extract from JS bundle or use fallback)
+
+    Returns:
+        Dict with leaderboard data, or None on error
+    """
+    if token is None:
+        token = extract_token_from_js()
+        if token is None:
+            logger.warning("Failed to extract token, using fallback")
+            token = FALLBACK_TOKEN
+
+    # GraphQL query for leaderboard (lifetime stats only)
+    query = """query getUserLeaderboard($key: String!) {
+  user_leaderboards(key: $key) {
+    lifetime {
+      count
+      user {
+        id
+        username
+        profile_photo_url
+        __typename
+      }
+      __typename
+    }
+    __typename
+  }
+}"""
+
+    try:
+        headers = {
+            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0",
+            "Accept": "*/*",
+            "Accept-Language": "en-GB,en;q=0.5",
+            "Referer": "https://www.mapillary.com/",
+            "content-type": "application/json",
+            "authorization": f"OAuth {token}",
+            "Origin": "https://www.mapillary.com",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-site",
+        }
+
+        # Build query params - use quote_via=quote to get %20 instead of +
+        # Note: both 'doc' and 'query' params seem to be required (from observed curl)
+        params = {
+            "doc": query,
+            "query": query,
+            "operationName": "getUserLeaderboard",
+            "variables": json.dumps({"key": key}, separators=(',', ':')),
+        }
+
+        # Build URL with proper percent encoding (not + for spaces)
+        # Don't encode parentheses to match curl behavior
+        query_string = urlencode(params, quote_via=lambda s, safe='', encoding=None, errors=None: quote(s, safe='()!'))
+        url = f"https://graph.mapillary.com/graphql?{query_string}"
+
+        logger.debug(f"Querying leaderboard for key: {key}")
+
+        response = requests.get(
+            url,
+            headers=headers,
+            timeout=30
+        )
+        response.raise_for_status()
+
+        return response.json()
+
+    except requests.RequestException as e:
+        logger.error(f"Failed to query leaderboard: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Unexpected error querying leaderboard: {e}")
+        return None
+
+
+if __name__ == "__main__":
+    # Test the extraction and leaderboard query
+    logging.basicConfig(level=logging.DEBUG)
+
+    print("=== Extracting token ===")
+    token = extract_token_from_js()
+    if token:
+        print(f"Success! Token: {token}")
+    else:
+        print("Failed to extract token")
+        print(f"Fallback: {FALLBACK_TOKEN}")
+        token = FALLBACK_TOKEN
+
+    print("\n=== Querying global leaderboard ===")
+    data = get_leaderboard("global", token=token)
+    if data:
+        print(json.dumps(data, indent=2))
+    else:
+        print("Failed to get leaderboard data")

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/src/mapillary_downloader/ia_meta.py

@@ -182,7 +182,7 @@ def generate_ia_metadata(collection_dir):
     write_meta_tag(meta_dir, "coverage", f"{first_date} - {last_date}")
     write_meta_tag(meta_dir, "licenseurl", "https://creativecommons.org/licenses/by-sa/4.0/")
     write_meta_tag(meta_dir, "mediatype", "data")
-    write_meta_tag(meta_dir, "collection", "opensource_media")
+    write_meta_tag(meta_dir, "collection", "mapillary-images")
 
     # Source and scanner metadata
     write_meta_tag(meta_dir, "source", f"https://www.mapillary.com/app/user/{username}")

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/src/mapillary_downloader/tar_sequences.py

@@ -1,6 +1,7 @@
 """Tar sequence directories for efficient Internet Archive uploads."""
 
 import logging
+import re
 import tarfile
 from pathlib import Path
 from mapillary_downloader.utils import format_size
@@ -9,7 +10,9 @@ logger = logging.getLogger("mapillary_downloader")
 
 
 def tar_sequence_directories(collection_dir):
-    """Tar all sequence directories in a collection for faster IA uploads.
+    """Tar all date directories in a collection for faster IA uploads.
+
+    Organizes by capture date (YYYY-MM-DD) for incremental archive.org uploads.
 
     Args:
         collection_dir: Path to collection directory (e.g., mapillary-user-quality/)
@@ -23,44 +26,44 @@ def tar_sequence_directories(collection_dir):
         logger.error(f"Collection directory not found: {collection_dir}")
         return 0, 0
 
-    # Find all bucket directories (skip special dirs)
-    # Now we tar entire bucket dirs (e.g., a/, b/, etc) to get ~62 tar files
+    # Find all date directories (skip special dirs)
+    # Date format: YYYY-MM-DD or unknown-date
     skip_dirs = {".meta", "__pycache__"}
-    bucket_dirs = []
+    date_dirs = []
 
     for item in collection_dir.iterdir():
         if item.is_dir() and item.name not in skip_dirs:
-            # Check if this is a bucket dir (single char)
-            if len(item.name) == 1:
-                bucket_dirs.append(item)
+            # Check if this is a date dir (YYYY-MM-DD) or unknown-date
+            if re.match(r"\d{4}-\d{2}-\d{2}$", item.name) or item.name == "unknown-date":
+                date_dirs.append(item)
 
-    if not bucket_dirs:
-        logger.info("No bucket directories to tar")
+    if not date_dirs:
+        logger.info("No date directories to tar")
         return 0, 0
 
-    # Sort bucket directories alphabetically for consistent progress tracking
-    bucket_dirs = sorted(bucket_dirs, key=lambda x: x.name)
+    # Sort date directories chronologically (YYYY-MM-DD sorts naturally)
+    date_dirs = sorted(date_dirs, key=lambda x: x.name)
 
-    logger.info(f"Tarring {len(bucket_dirs)} bucket directories...")
+    logger.info(f"Tarring {len(date_dirs)} date directories...")
 
     tarred_count = 0
     total_files = 0
     total_tar_bytes = 0
 
-    for bucket_dir in bucket_dirs:
-        bucket_name = bucket_dir.name
-        tar_path = collection_dir / f"{bucket_name}.tar"
+    for date_dir in date_dirs:
+        date_name = date_dir.name
+        tar_path = collection_dir / f"{date_name}.tar"
 
-        # Count files in bucket
-        files_to_tar = sorted([f for f in bucket_dir.rglob("*") if f.is_file()], key=lambda x: str(x))
+        # Count files in date directory
+        files_to_tar = sorted([f for f in date_dir.rglob("*") if f.is_file()], key=lambda x: str(x))
         file_count = len(files_to_tar)
 
         if file_count == 0:
-            logger.warning(f"Skipping empty bucket directory: {bucket_name}")
+            logger.warning(f"Skipping empty date directory: {date_name}")
             continue
 
         try:
-            logger.info(f"Tarring bucket '{bucket_name}' ({file_count} files)...")
+            logger.info(f"Tarring date '{date_name}' ({file_count} files)...")
 
             # Create reproducible uncompressed tar (WebP already compressed)
             with tarfile.open(tar_path, "w") as tar:
@@ -87,36 +90,34 @@ def tar_sequence_directories(collection_dir):
                 tar_size = tar_path.stat().st_size
                 total_tar_bytes += tar_size
 
-                # Remove original bucket directory
-                for file in bucket_dir.rglob("*"):
+                # Remove original date directory
+                for file in date_dir.rglob("*"):
                     if file.is_file():
                         file.unlink()
 
                 # Remove empty subdirs and main dir
-                for subdir in list(bucket_dir.rglob("*")):
+                for subdir in list(date_dir.rglob("*")):
                     if subdir.is_dir():
                         try:
                             subdir.rmdir()
                         except OSError:
                             pass  # Not empty yet
 
-                bucket_dir.rmdir()
+                date_dir.rmdir()
 
                 tarred_count += 1
                 total_files += file_count
 
-                logger.info(f"Tarred bucket '{bucket_name}': {file_count:,} files, {format_size(tar_size)}")
+                logger.info(f"Tarred date '{date_name}': {file_count:,} files, {format_size(tar_size)}")
             else:
                 logger.error(f"Tar file empty or not created: {tar_path}")
                 if tar_path.exists():
                     tar_path.unlink()
 
         except Exception as e:
-            logger.error(f"Error tarring bucket {bucket_name}: {e}")
+            logger.error(f"Error tarring date {date_name}: {e}")
             if tar_path.exists():
                 tar_path.unlink()
 
-    logger.info(
-        f"Tarred {tarred_count} sequences ({total_files:,} files, {format_size(total_tar_bytes)} total tar size)"
-    )
+    logger.info(f"Tarred {tarred_count} dates ({total_files:,} files, {format_size(total_tar_bytes)} total tar size)")
     return tarred_count, total_files

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/src/mapillary_downloader/worker.py

@@ -3,6 +3,7 @@
 import os
 import signal
 import tempfile
+from datetime import datetime
 from pathlib import Path
 import requests
 from mapillary_downloader.exif_writer import write_exif_to_image
@@ -69,16 +70,25 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, se
         if not image_url:
             return (image_id, 0, False, f"No {quality} URL")
 
-        # Determine final output directory - organize by first char of sequence ID
+        # Determine final output directory - organize by capture date
         output_dir = Path(output_dir)
         sequence_id = image_data.get("sequence")
+
+        # Extract date from captured_at timestamp (milliseconds since epoch)
+        captured_at = image_data.get("captured_at")
+        if captured_at:
+            # Convert to UTC date string (YYYY-MM-DD)
+            date_str = datetime.utcfromtimestamp(captured_at / 1000).strftime("%Y-%m-%d")
+        else:
+            # Fallback for missing timestamp (should be rare per API docs)
+            date_str = "unknown-date"
+
         if sequence_id:
-            # Use first character as bucket (gives us ~62 dirs instead of millions)
-            first_char = sequence_id[0]
-            img_dir = output_dir / first_char / sequence_id
+            img_dir = output_dir / date_str / sequence_id
             img_dir.mkdir(parents=True, exist_ok=True)
         else:
-            img_dir = output_dir
+            img_dir = output_dir / date_str
+            img_dir.mkdir(parents=True, exist_ok=True)
 
         # If converting to WebP, use /tmp for intermediate JPEG
         # Otherwise write JPEG directly to final location

{mapillary_downloader-0.6.1 → mapillary_downloader-0.7.2}/src/mapillary_downloader/worker_pool.py

@@ -17,17 +17,15 @@ class AdaptiveWorkerPool:
     - If throughput plateauing/decreasing: reduce workers
     """
 
-    def __init__(self, worker_func, min_workers=4, max_workers=16, monitoring_interval=10):
+    def __init__(self, worker_func, max_workers=16, monitoring_interval=10):
         """Initialize adaptive worker pool.
 
         Args:
             worker_func: Function to run in each worker (must accept work_queue, result_queue)
-            min_workers: Minimum number of workers
             max_workers: Maximum number of workers
             monitoring_interval: Seconds between throughput checks
         """
         self.worker_func = worker_func
-        self.min_workers = min_workers
         self.max_workers = max_workers
         self.monitoring_interval = monitoring_interval
 
@@ -37,7 +35,8 @@ class AdaptiveWorkerPool:
 
         # Worker management
         self.workers = []
-        self.current_workers = min_workers  # Start small and ramp up
+        # Start at 25% of max_workers (at least 1)
+        self.current_workers = max(1, int(max_workers * 0.25))
 
         # Throughput monitoring
         self.throughput_history = deque(maxlen=5)  # Last 5 measurements