mapillary-downloader 0.7.0__tar.gz → 0.7.2__tar.gz

{mapillary_downloader-0.7.0 → mapillary_downloader-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.7.0
+Version: 0.7.2
 Summary: Archive user data from Mapillary
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10
@@ -66,7 +66,7 @@ mapillary-downloader --output ./downloads USERNAME1
 | `--quality`     | 256, 1024, 2048 or original                  | `original`         |
 | `--bbox`        | `west,south,east,north`                      | `None`             |
 | `--no-webp`     | Don't convert to WebP                        | `False`            |
-| `--max-workers` | Maximum number of parallel download workers  | `128`              |
+| `--max-workers` | Maximum number of parallel download workers  | CPU count          |
 | `--no-tar`      | Don't tar bucket directories                 | `False`            |
 | `--no-check-ia` | Don't check if exists on Internet Archive    | `False`            |
 

{mapillary_downloader-0.7.0 → mapillary_downloader-0.7.2}/README.md

@@ -36,7 +36,7 @@ mapillary-downloader --output ./downloads USERNAME1
 | `--quality`     | 256, 1024, 2048 or original                  | `original`         |
 | `--bbox`        | `west,south,east,north`                      | `None`             |
 | `--no-webp`     | Don't convert to WebP                        | `False`            |
-| `--max-workers` | Maximum number of parallel download workers  | `128`              |
+| `--max-workers` | Maximum number of parallel download workers  | CPU count          |
 | `--no-tar`      | Don't tar bucket directories                 | `False`            |
 | `--no-check-ia` | Don't check if exists on Internet Archive    | `False`            |
 

{mapillary_downloader-0.7.0 → mapillary_downloader-0.7.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mapillary_downloader"
 description = "Archive user data from Mapillary"
-version = "0.7.0"
+version = "0.7.2"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

{mapillary_downloader-0.7.0 → mapillary_downloader-0.7.2}/src/mapillary_downloader/__main__.py

@@ -43,8 +43,8 @@ def main():
     parser.add_argument(
         "--max-workers",
         type=int,
-        default=128,
-        help="Maximum number of parallel workers (default: 128)",
+        default=os.cpu_count() or 8,
+        help=f"Maximum number of parallel workers (default: CPU count = {os.cpu_count() or 8})",
     )
     parser.add_argument(
         "--no-tar",

{mapillary_downloader-0.7.0 → mapillary_downloader-0.7.2}/src/mapillary_downloader/downloader.py

@@ -67,7 +67,6 @@ class MapillaryDownloader:
         self.username = username
         self.quality = quality
         self.max_workers = max_workers
-        self.initial_workers = os.cpu_count() or 1  # Start with CPU count
         self.tar_sequences = tar_sequences
         self.convert_webp = convert_webp
         self.check_ia = check_ia
@@ -173,7 +172,7 @@ class MapillaryDownloader:
         logger.info(f"Downloading images for user: {self.username}")
         logger.info(f"Output directory: {self.output_dir}")
         logger.info(f"Quality: {self.quality}")
-        logger.info(f"Worker pool: {self.initial_workers} initial, {self.max_workers} max")
+        logger.info(f"Worker pool: max {self.max_workers} workers")
 
         start_time = time.time()
 
@@ -188,9 +187,7 @@ class MapillaryDownloader:
         # Step 2: Start worker pool
         # Since workers do both I/O (download) and CPU (WebP), need many more workers
         # Start with CPU count and scale up based on throughput
-        pool = AdaptiveWorkerPool(
-            worker_process, min_workers=self.initial_workers, max_workers=self.max_workers, monitoring_interval=10
-        )
+        pool = AdaptiveWorkerPool(worker_process, max_workers=self.max_workers, monitoring_interval=10)
         pool.start()
 
         # Step 3: Download images from metadata file while fetching new from API

mapillary_downloader-0.7.2/src/mapillary_downloader/graphql_web.py

@@ -0,0 +1,193 @@
+"""GraphQL web API utilities (unofficial, experimental).
+
+This module provides access to Mapillary's GraphQL endpoint used by the web interface.
+Unlike the official v4 REST API, this requires a public web token extracted from the
+JavaScript bundle.
+
+Use cases:
+- Get user image counts without pagination
+- Access leaderboard data
+- Check for updates to existing downloads
+
+WARNING: This is not officially documented and may break at any time.
+"""
+
+import json
+import logging
+import re
+from datetime import datetime
+from urllib.parse import urlencode, quote
+import requests
+
+logger = logging.getLogger("mapillary_downloader")
+
+# Fallback token (extracted from main JS bundle as of 2025-01-09)
+FALLBACK_TOKEN = "MLY|4223665974375089|d62822dd792b6a823d0794ef26450398"
+
+
+def extract_token_from_js():
+    """Extract public web token from Mapillary's JavaScript bundle.
+
+    This fetches the main page, finds the main JS bundle, and extracts
+    the hardcoded MLY token used for GraphQL queries.
+
+    Returns:
+        Token string (e.g., "MLY|123|abc...") or None if extraction failed
+    """
+    try:
+        # Fetch main page to find JS bundle URL
+        # Need consent cookie to get actual page (not GDPR banner)
+        logger.debug("Fetching Mapillary main page...")
+        # Generate today's date in the format YYYY_MM_DD for cookie
+        today = datetime.now().strftime("%Y_%m_%d")
+        cookies = {
+            "mly_cb": f'{{"version":"1","date":"{today}","third_party_consent":"withdrawn","categories":{{"content_and_media":"withdrawn"}},"integration_controls":{{"YOUTUBE":"withdrawn"}}}}'
+        }
+        headers = {
+            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-GB,en;q=0.5",
+            "Sec-GPC": "1",
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+        }
+        response = requests.get("https://www.mapillary.com/app/", cookies=cookies, headers=headers, timeout=30)
+        response.raise_for_status()
+
+        # Find main JS file URL
+        # Pattern: <script src="main.{hash}.js" type="module"></script>
+        js_match = re.search(r'src="(main\.[a-f0-9]+\.js)"', response.text)
+        if not js_match:
+            logger.warning("Could not find main JS bundle URL in page")
+            return None
+
+        # URL is relative to /app/ base path
+        js_url = f"https://www.mapillary.com/app/{js_match.group(1)}"
+        logger.debug(f"Found JS bundle: {js_url}")
+
+        # Fetch JS bundle
+        logger.debug("Fetching JS bundle...")
+        js_response = requests.get(js_url, timeout=30)
+        js_response.raise_for_status()
+
+        # Extract token
+        # Pattern: "MLY|{client_id}|{secret}"
+        token_match = re.search(r'"(MLY\|[^"]+)"', js_response.text)
+        if not token_match:
+            logger.warning("Could not find MLY token in JS bundle")
+            return None
+
+        token = token_match.group(1)
+        logger.info(f"Extracted web token: {token[:20]}...")
+        return token
+
+    except requests.RequestException as e:
+        logger.error(f"Failed to extract web token: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Unexpected error extracting web token: {e}")
+        return None
+
+
+def get_leaderboard(key="global", token=None):
+    """Get leaderboard data from Mapillary GraphQL API.
+
+    Args:
+        key: Leaderboard key (e.g., "global", country name, etc.)
+        token: MLY token (if None, will extract from JS bundle or use fallback)
+
+    Returns:
+        Dict with leaderboard data, or None on error
+    """
+    if token is None:
+        token = extract_token_from_js()
+        if token is None:
+            logger.warning("Failed to extract token, using fallback")
+            token = FALLBACK_TOKEN
+
+    # GraphQL query for leaderboard (lifetime stats only)
+    query = """query getUserLeaderboard($key: String!) {
+  user_leaderboards(key: $key) {
+    lifetime {
+      count
+      user {
+        id
+        username
+        profile_photo_url
+        __typename
+      }
+      __typename
+    }
+    __typename
+  }
+}"""
+
+    try:
+        headers = {
+            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0",
+            "Accept": "*/*",
+            "Accept-Language": "en-GB,en;q=0.5",
+            "Referer": "https://www.mapillary.com/",
+            "content-type": "application/json",
+            "authorization": f"OAuth {token}",
+            "Origin": "https://www.mapillary.com",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-site",
+        }
+
+        # Build query params - use quote_via=quote to get %20 instead of +
+        # Note: both 'doc' and 'query' params seem to be required (from observed curl)
+        params = {
+            "doc": query,
+            "query": query,
+            "operationName": "getUserLeaderboard",
+            "variables": json.dumps({"key": key}, separators=(',', ':')),
+        }
+
+        # Build URL with proper percent encoding (not + for spaces)
+        # Don't encode parentheses to match curl behavior
+        query_string = urlencode(params, quote_via=lambda s, safe='', encoding=None, errors=None: quote(s, safe='()!'))
+        url = f"https://graph.mapillary.com/graphql?{query_string}"
+
+        logger.debug(f"Querying leaderboard for key: {key}")
+
+        response = requests.get(
+            url,
+            headers=headers,
+            timeout=30
+        )
+        response.raise_for_status()
+
+        return response.json()
+
+    except requests.RequestException as e:
+        logger.error(f"Failed to query leaderboard: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Unexpected error querying leaderboard: {e}")
+        return None
+
+
+if __name__ == "__main__":
+    # Test the extraction and leaderboard query
+    logging.basicConfig(level=logging.DEBUG)
+
+    print("=== Extracting token ===")
+    token = extract_token_from_js()
+    if token:
+        print(f"Success! Token: {token}")
+    else:
+        print("Failed to extract token")
+        print(f"Fallback: {FALLBACK_TOKEN}")
+        token = FALLBACK_TOKEN
+
+    print("\n=== Querying global leaderboard ===")
+    data = get_leaderboard("global", token=token)
+    if data:
+        print(json.dumps(data, indent=2))
+    else:
+        print("Failed to get leaderboard data")

{mapillary_downloader-0.7.0 → mapillary_downloader-0.7.2}/src/mapillary_downloader/ia_meta.py

@@ -182,7 +182,7 @@ def generate_ia_metadata(collection_dir):
     write_meta_tag(meta_dir, "coverage", f"{first_date} - {last_date}")
     write_meta_tag(meta_dir, "licenseurl", "https://creativecommons.org/licenses/by-sa/4.0/")
     write_meta_tag(meta_dir, "mediatype", "data")
-    write_meta_tag(meta_dir, "collection", "opensource_media")
+    write_meta_tag(meta_dir, "collection", "mapillary-images")
 
     # Source and scanner metadata
     write_meta_tag(meta_dir, "source", f"https://www.mapillary.com/app/user/{username}")

{mapillary_downloader-0.7.0 → mapillary_downloader-0.7.2}/src/mapillary_downloader/worker_pool.py

@@ -17,17 +17,15 @@ class AdaptiveWorkerPool:
     - If throughput plateauing/decreasing: reduce workers
     """
 
-    def __init__(self, worker_func, min_workers=4, max_workers=16, monitoring_interval=10):
+    def __init__(self, worker_func, max_workers=16, monitoring_interval=10):
         """Initialize adaptive worker pool.
 
         Args:
             worker_func: Function to run in each worker (must accept work_queue, result_queue)
-            min_workers: Minimum number of workers
             max_workers: Maximum number of workers
             monitoring_interval: Seconds between throughput checks
         """
         self.worker_func = worker_func
-        self.min_workers = min_workers
         self.max_workers = max_workers
         self.monitoring_interval = monitoring_interval
 
@@ -37,7 +35,8 @@ class AdaptiveWorkerPool:
 
         # Worker management
         self.workers = []
-        self.current_workers = min_workers  # Start small and ramp up
+        # Start at 25% of max_workers (at least 1)
+        self.current_workers = max(1, int(max_workers * 0.25))
 
         # Throughput monitoring
         self.throughput_history = deque(maxlen=5)  # Last 5 measurements