mapillary-downloader 0.7.2__tar.gz → 0.7.3__tar.gz

{mapillary_downloader-0.7.2 → mapillary_downloader-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.7.2
+Version: 0.7.3
 Summary: Archive user data from Mapillary
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10

{mapillary_downloader-0.7.2 → mapillary_downloader-0.7.3}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mapillary_downloader"
 description = "Archive user data from Mapillary"
-version = "0.7.2"
+version = "0.7.3"
 authors = [
     { name = "Gareth Davidson", email = "gaz@bitplane.net" }
 ]

{mapillary_downloader-0.7.2 → mapillary_downloader-0.7.3}/src/mapillary_downloader/downloader.py

@@ -99,7 +99,7 @@ class MapillaryDownloader:
         # Set up file logging for archival with timestamp for incremental runs
         timestamp = time.strftime("%Y%m%d-%H%M%S")
         log_file = self.output_dir / f"download.log.{timestamp}"
-        add_file_handler(log_file)
+        self.file_handler = add_file_handler(log_file)
         logger.info(f"Logging to: {log_file}")
 
         self.metadata_file = self.output_dir / "metadata.jsonl"
@@ -169,24 +169,15 @@ class MapillaryDownloader:
 
         quality_field = f"thumb_{self.quality}_url"
 
-        logger.info(f"Downloading images for user: {self.username}")
-        logger.info(f"Output directory: {self.output_dir}")
-        logger.info(f"Quality: {self.quality}")
-        logger.info(f"Worker pool: max {self.max_workers} workers")
+        logger.info(f"Downloading {self.username} @ {self.quality} (max {self.max_workers} workers)")
 
         start_time = time.time()
 
         # Step 1: Check if API fetch is already complete
         reader = MetadataReader(self.metadata_file)
         api_complete = reader.is_complete
-        if api_complete:
-            logger.info("API fetch already complete, will only download")
-        else:
-            logger.info("API fetch incomplete, will fetch and download in parallel")
 
         # Step 2: Start worker pool
-        # Since workers do both I/O (download) and CPU (WebP), need many more workers
-        # Start with CPU count and scale up based on throughput
         pool = AdaptiveWorkerPool(worker_process, max_workers=self.max_workers, monitoring_interval=10)
         pool.start()
 
@@ -195,9 +186,6 @@ class MapillaryDownloader:
         total_bytes = 0
         failed_count = 0
         submitted = 0
-        batch_start = time.time()
-
-        logger.info("Starting parallel download and API fetch...")
 
         try:
             # Step 3a: Fetch metadata from API in parallel (write-only, don't block on queue)
@@ -210,7 +198,7 @@ class MapillaryDownloader:
                 def fetch_api_metadata():
                     """Fetch metadata from API and write to file (runs in thread)."""
                     try:
-                        logger.info("API fetch thread: Starting...")
+                        logger.debug("API fetch thread starting")
                         with open(self.metadata_file, "a") as meta_f:
                             for image in self.client.get_user_images(self.username, bbox=bbox):
                                 new_images_count[0] += 1
@@ -220,11 +208,11 @@ class MapillaryDownloader:
                                 meta_f.flush()
 
                                 if new_images_count[0] % 1000 == 0:
-                                    logger.info(f"API: Fetched {new_images_count[0]} images from API")
+                                    logger.info(f"API: fetched {new_images_count[0]:,} image URLs")
 
                             # Mark as complete
                             MetadataReader.mark_complete(self.metadata_file)
-                            logger.info(f"API fetch complete: {new_images_count[0]} images")
+                            logger.info(f"API fetch complete: {new_images_count[0]:,} images")
                     finally:
                         api_fetch_complete.set()
 
@@ -232,11 +220,10 @@ class MapillaryDownloader:
                 api_thread = threading.Thread(target=fetch_api_metadata, daemon=True)
                 api_thread.start()
             else:
-                logger.info("API fetch already complete, skipping API thread")
                 api_fetch_complete = None
 
             # Step 3b: Tail metadata file and submit to workers
-            logger.info("Starting metadata tail and download queue feeder...")
+            logger.debug("Starting metadata tail and download queue feeder")
             last_position = 0
 
             # Helper to process results from queue
@@ -258,12 +245,7 @@ class MapillaryDownloader:
                         # Log every download for first 10, then every 100
                         should_log = downloaded_count <= 10 or downloaded_count % 100 == 0
                         if should_log:
-                            elapsed = time.time() - batch_start
-                            rate = downloaded_count / elapsed if elapsed > 0 else 0
-                            logger.info(
-                                f"Downloaded: {downloaded_count} ({format_size(total_bytes)}) "
-                                f"- Rate: {rate:.1f} images/sec"
-                            )
+                            logger.info(f"Downloaded: {downloaded_count:,} ({format_size(total_bytes)})")
 
                         if downloaded_count % 100 == 0:
                             self._save_progress()
@@ -301,6 +283,7 @@ class MapillaryDownloader:
 
                                 # Skip if already downloaded or no quality URL
                                 if image_id in self.downloaded:
+                                    downloaded_count += 1
                                     continue
                                 if not image.get(quality_field):
                                     continue
@@ -317,7 +300,7 @@ class MapillaryDownloader:
                                 submitted += 1
 
                                 if submitted % 1000 == 0:
-                                    logger.info(f"Queue: Submitted {submitted} images")
+                                    logger.info(f"Queue: submitted {submitted:,} images")
 
                                 # Process results while submitting
                                 process_results()
@@ -352,6 +335,7 @@ class MapillaryDownloader:
 
                             # Skip if already downloaded or no quality URL
                             if image_id in self.downloaded:
+                                downloaded_count += 1
                                 continue
                             if not image.get(quality_field):
                                 continue
@@ -368,7 +352,7 @@ class MapillaryDownloader:
                             submitted += 1
 
                             if submitted % 1000 == 0:
-                                logger.info(f"Queue: Submitted {submitted} images")
+                                logger.info(f"Queue: submitted {submitted:,} images")
 
                             # Process results while submitting
                             process_results()
@@ -386,7 +370,7 @@ class MapillaryDownloader:
                 process_results()
 
             # Send shutdown signals
-            logger.info(f"Submitted {submitted} images, waiting for workers to finish...")
+            logger.debug(f"Submitted {submitted:,} images, waiting for workers")
             for _ in range(pool.current_workers):
                 pool.submit(None)
 
@@ -408,16 +392,8 @@ class MapillaryDownloader:
                     downloaded_count += 1
                     total_bytes += bytes_dl
 
-                    if downloaded_count % 10 == 0:
-                        elapsed = time.time() - batch_start
-                        rate = downloaded_count / elapsed if elapsed > 0 else 0
-                        remaining = submitted - completed
-                        eta_seconds = remaining / rate if rate > 0 else 0
-
-                        logger.info(
-                            f"Downloaded: {downloaded_count}/{submitted} ({format_size(total_bytes)}) "
-                            f"- ETA: {format_time(eta_seconds)}"
-                        )
+                    if downloaded_count % 100 == 0:
+                        logger.info(f"Downloaded: {downloaded_count:,} ({format_size(total_bytes)})")
                         self._save_progress()
                         pool.check_throughput(downloaded_count)
                 else:
@@ -431,7 +407,7 @@ class MapillaryDownloader:
         self._save_progress()
         elapsed = time.time() - start_time
 
-        logger.info(f"Complete! Downloaded {downloaded_count} ({format_size(total_bytes)}), " f"failed {failed_count}")
+        logger.info(f"Complete! Downloaded {downloaded_count:,} ({format_size(total_bytes)}), failed {failed_count:,}")
         logger.info(f"Total time: {format_time(elapsed)}")
 
         # Tar sequence directories for efficient IA uploads
@@ -460,12 +436,16 @@ class MapillaryDownloader:
         # Generate IA metadata
         generate_ia_metadata(self.output_dir)
 
+        # Close log file handler before moving directory
+        self.file_handler.close()
+        logger.removeHandler(self.file_handler)
+
         # Move from staging to final destination
-        logger.info("Moving collection from staging to final destination...")
+        logger.info("Moving to final destination...")
         if self.final_dir.exists():
             logger.warning(f"Destination already exists, removing: {self.final_dir}")
             shutil.rmtree(self.final_dir)
 
         self.final_dir.parent.mkdir(parents=True, exist_ok=True)
         shutil.move(str(self.staging_dir), str(self.final_dir))
-        logger.info(f"Collection moved to: {self.final_dir}")
+        logger.info(f"Done: {self.final_dir}")

{mapillary_downloader-0.7.2 → mapillary_downloader-0.7.3}/src/mapillary_downloader/ia_stats.py

@@ -15,14 +15,14 @@ def search_ia_collections():
     """Search IA for all mapillary_downloader collections.
 
     Returns:
-        List of dicts with: identifier, description, item_size, uploader
+        List of dicts with: identifier, description, item_size, collection
     """
     logger.info("Searching archive.org for mapillary_downloader collections...")
 
     url = "https://archive.org/advancedsearch.php"
     params = {
         "q": "mapillary_downloader:*",
-        "fl[]": ["identifier", "description", "item_size", "uploader"],
+        "fl[]": ["identifier", "description", "item_size", "collection"],
         "rows": 10000,
         "output": "json",
     }
@@ -31,11 +31,29 @@ def search_ia_collections():
     data = response.json()
 
     collections = data["response"]["docs"]
-    logger.info(f"Found {len(collections)} collections on archive.org")
+    logger.info(f"Found {len(collections):,} collections on archive.org")
 
     return collections
 
 
+def fetch_uploader(identifier):
+    """Fetch uploader email from item metadata.
+
+    Args:
+        identifier: IA item identifier
+
+    Returns:
+        Uploader email or None
+    """
+    url = f"https://archive.org/metadata/{identifier}/metadata/uploader"
+    try:
+        response = http_get_with_retry(url, max_retries=2)
+        data = response.json()
+        return data.get("result")
+    except Exception:
+        return None
+
+
 def parse_collection_info(identifier):
     """Parse username, quality, webp from collection identifier.
 
@@ -104,19 +122,28 @@ def update_cache(ia_collections):
 
         image_count = extract_image_count(item.get("description"))
 
+        # Get IA collection(s) - can be a string or list
+        ia_collection = item.get("collection", [])
+        if isinstance(ia_collection, str):
+            ia_collection = [ia_collection]
+
+        # Preserve existing uploader if we have it cached
+        existing = cache.get(identifier, {})
+
         # Update cache entry
         cache[identifier] = {
             "size": size_bytes,
-            "uploader": item.get("uploader"),
+            "uploader": existing.get("uploader"),  # Preserve cached uploader
             "images": image_count,
             "quality": info["quality"],
             "username": info["username"],
             "is_webp": info["is_webp"],
+            "ia_collection": ia_collection,
         }
 
     # Save updated cache
     safe_json_save(CACHE_FILE, cache)
-    logger.info(f"Updated cache with {len(cache)} collections")
+    logger.info(f"Updated cache with {len(cache):,} collections")
 
     return cache
 
@@ -168,11 +195,12 @@ def aggregate_stats(cache):
     return stats
 
 
-def format_stats(stats):
+def format_stats(stats, cache):
     """Format statistics as human-readable text.
 
     Args:
         stats: Dict from aggregate_stats()
+        cache: Dict of collection data
 
     Returns:
         Formatted string
@@ -212,6 +240,62 @@ def format_stats(stats):
         )
 
     output.append("")
+
+    # Find items not in mapillary-images and fetch uploaders
+    not_in_mapillary_images = []
+    need_uploader_fetch = []
+
+    for identifier, data in cache.items():
+        ia_collections = data.get("ia_collection", [])
+        if "mapillary-images" not in ia_collections:
+            not_in_mapillary_images.append(identifier)
+            if not data.get("uploader"):
+                need_uploader_fetch.append(identifier)
+
+    # Fetch missing uploaders
+    if need_uploader_fetch:
+        logger.info(f"Fetching uploader info for {len(need_uploader_fetch)} items...")
+        for i, identifier in enumerate(need_uploader_fetch, 1):
+            logger.info(f"  [{i}/{len(need_uploader_fetch)}] {identifier}")
+            uploader = fetch_uploader(identifier)
+            if uploader:
+                cache[identifier]["uploader"] = uploader
+        # Save updated cache with uploaders
+        safe_json_save(CACHE_FILE, cache)
+
+    # Group by uploader (only for items not in mapillary-images)
+    by_uploader = {}
+    for identifier in not_in_mapillary_images:
+        uploader = cache[identifier].get("uploader") or "unknown"
+        if uploader not in by_uploader:
+            by_uploader[uploader] = {"items": [], "images": 0, "size": 0}
+        by_uploader[uploader]["items"].append(identifier)
+        by_uploader[uploader]["images"] += cache[identifier].get("images") or 0
+        by_uploader[uploader]["size"] += cache[identifier].get("size") or 0
+
+    # By uploader (only those with items outside mapillary-images)
+    if by_uploader:
+        output.append("Uploaders with items outside mapillary-images:")
+        output.append("-" * 70)
+        for uploader, data in sorted(by_uploader.items(), key=lambda x: -len(x[1]["items"])):
+            output.append(
+                f"  {uploader}: {len(data['items'])} items, " f"{data['images']:,} images, {format_size(data['size'])}"
+            )
+        output.append("")
+
+    # Items not in mapillary-images, grouped by uploader
+    if not_in_mapillary_images:
+        output.append(f"Items NOT in mapillary-images ({len(not_in_mapillary_images)}):")
+        output.append("-" * 70)
+        for uploader, data in sorted(by_uploader.items(), key=lambda x: x[0].lower()):
+            output.append(f"{uploader}:")
+            for identifier in sorted(data["items"]):
+                output.append(identifier)
+            output.append("")
+    else:
+        output.append("All items are in mapillary-images collection!")
+        output.append("")
+
     output.append(f"Cache: {CACHE_FILE}")
 
     return "\n".join(output)
@@ -239,4 +323,4 @@ def show_stats(refresh=True):
         return
 
     stats = aggregate_stats(cache)
-    print(format_stats(stats))
+    print(format_stats(stats, cache))

{mapillary_downloader-0.7.2 → mapillary_downloader-0.7.3}/src/mapillary_downloader/worker_pool.py

@@ -49,7 +49,7 @@ class AdaptiveWorkerPool:
     def start(self):
         """Start the worker pool."""
         self.running = True
-        logger.info(f"Starting worker pool with {self.current_workers} workers")
+        logger.debug(f"Starting worker pool with {self.current_workers} workers")
 
         for i in range(self.current_workers):
             self._add_worker(i)
@@ -99,10 +99,7 @@ class AdaptiveWorkerPool:
         self.last_processed = total_processed
         self.last_check_time = now
 
-        logger.info(
-            f"Throughput: {throughput:.1f} items/s (workers: {current_workers}/{self.max_workers}, "
-            f"history: {len(self.throughput_history)} measurements)"
-        )
+        logger.info(f"Throughput: {throughput:.1f} items/s (workers: {current_workers}/{self.max_workers})")
 
         # Need at least 2 measurements to calculate gain per worker
         if len(self.throughput_history) < 2:
@@ -181,12 +178,10 @@ class AdaptiveWorkerPool:
                         self.current_workers += 1
                         added += 1
                 logger.info(f"Ramping up: added {added} workers (now {self.current_workers}/{self.max_workers})")
-        else:
-            logger.info(f"At optimal worker count: {current_workers} workers, {current_throughput:.1f} items/s")
 
     def shutdown(self, timeout=2):
         """Shutdown the worker pool gracefully."""
-        logger.info("Shutting down worker pool...")
+        logger.debug("Shutting down worker pool")
         self.running = False
 
         # Terminate all workers immediately (they ignore SIGINT so we need to be forceful)
@@ -197,5 +192,3 @@ class AdaptiveWorkerPool:
         # Give them a brief moment to exit
         for p in self.workers:
             p.join(timeout=timeout)
-
-        logger.info("Worker pool shutdown complete")