mapillary-downloader 0.7.5__py3-none-any.whl → 0.7.7__py3-none-any.whl

mapillary_downloader/downloader.py

@@ -105,6 +105,7 @@ class MapillaryDownloader:
         self.metadata_file = self.output_dir / "metadata.jsonl"
         self.progress_file = self.output_dir / "progress.json"
         self.downloaded = self._load_progress()
+        self._last_save_time = time.time()
 
     def _load_progress(self):
         """Load previously downloaded image IDs for this quality."""
@@ -248,8 +249,11 @@ class MapillaryDownloader:
                             logger.info(f"Downloaded: {downloaded_count:,} ({format_size(total_bytes)})")
 
                         if downloaded_count % 100 == 0:
-                            self._save_progress()
                             pool.check_throughput(downloaded_count)
+                            # Save progress every 5 minutes
+                            if time.time() - self._last_save_time >= 300:
+                                self._save_progress()
+                                self._last_save_time = time.time()
                     else:
                         failed_count += 1
                         logger.warning(f"Failed to download {image_id}: {error_msg}")
@@ -394,8 +398,11 @@ class MapillaryDownloader:
 
                     if downloaded_count % 100 == 0:
                         logger.info(f"Downloaded: {downloaded_count:,} ({format_size(total_bytes)})")
-                        self._save_progress()
                         pool.check_throughput(downloaded_count)
+                        # Save progress every 5 minutes
+                        if time.time() - self._last_save_time >= 300:
+                            self._save_progress()
+                            self._last_save_time = time.time()
                 else:
                     failed_count += 1
                     logger.warning(f"Failed to download {image_id}: {error_msg}")

mapillary_downloader/utils.py

@@ -77,7 +77,7 @@ def safe_json_save(file_path, data):
     temp_file.replace(file_path)
 
 
-def http_get_with_retry(url, params=None, max_retries=5, base_delay=1.0, timeout=60):
+def http_get_with_retry(url, params=None, max_retries=5, base_delay=1.0, timeout=60, session=None):
     """HTTP GET with exponential backoff retry.
 
     Args:
@@ -86,6 +86,7 @@ def http_get_with_retry(url, params=None, max_retries=5, base_delay=1.0, timeout
         max_retries: Maximum retry attempts (default: 5)
         base_delay: Initial delay in seconds (default: 1.0)
         timeout: Request timeout in seconds (default: 60)
+        session: Optional requests.Session for connection pooling
 
     Returns:
         requests.Response object
@@ -93,9 +94,10 @@ def http_get_with_retry(url, params=None, max_retries=5, base_delay=1.0, timeout
     Raises:
         requests.RequestException: If all retries exhausted
     """
+    getter = session or requests
     for attempt in range(max_retries):
         try:
-            response = requests.get(url, params=params, timeout=timeout)
+            response = getter.get(url, params=params, timeout=timeout)
             response.raise_for_status()
             return response
         except RequestException as e:

mapillary_downloader/worker.py

@@ -105,7 +105,7 @@ def download_and_convert_image(image_data, output_dir, quality, convert_webp, se
 
         try:
             # Use retry logic with 3 attempts for image downloads
-            response = http_get_with_retry(image_url, max_retries=3, base_delay=1.0, timeout=60)
+            response = http_get_with_retry(image_url, max_retries=3, base_delay=1.0, timeout=60, session=session)
 
             with open(jpg_path, "wb") as f:
                 for chunk in response.iter_content(chunk_size=8192):

{mapillary_downloader-0.7.5.dist-info → mapillary_downloader-0.7.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_downloader
-Version: 0.7.5
+Version: 0.7.7
 Summary: Archive user data from Mapillary
 Author-email: Gareth Davidson <gaz@bitplane.net>
 Requires-Python: >=3.10

{mapillary_downloader-0.7.5.dist-info → mapillary_downloader-0.7.7.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mapillary_downloader/__init__.py,sha256=KEjiBRghXDeA7E15RJeLBfQm-yNJkowZarL59QOh_1w,120
 mapillary_downloader/__main__.py,sha256=iuDGZoFVu8q_dTvJuExSpj4Jx1x9xASSjUITRGwd0RA,4864
 mapillary_downloader/client.py,sha256=a5n43FLHP45EHodEjl0ieziBK-b6Ey-rZJwYB6EFhNI,4743
-mapillary_downloader/downloader.py,sha256=TWk3CiPbuEWgFZoyA146xdvH-heZnmBWFJKKaBbwa40,18456
+mapillary_downloader/downloader.py,sha256=l6MT3dFOB-lZfoLEVVGIkioKSXcDu30Q9xc2MZ17iGI,18897
 mapillary_downloader/exif_writer.py,sha256=K_441EG1siWyNMmFGZSfnORUCjBThkeg4JFtbg9AOsA,5120
 mapillary_downloader/ia_check.py,sha256=L2MEbG_KmlAd5NLmo2HQkO8HWvRN0brE5wXXoyNMbq8,1100
 mapillary_downloader/ia_meta.py,sha256=DTmFwIKN03aNgBaerQWF5x_hveDpjvrMBTdRAgHoFRk,6365
@@ -9,12 +9,12 @@ mapillary_downloader/ia_stats.py,sha256=kjbNUVXtZziWxTx1yi2TLTZt_F0BWjrv1WWyy6Ze
 mapillary_downloader/logging_config.py,sha256=Z-wNq34nt7aIhJWdeKc1feTY46P9-Or7HtiX7eUFjEI,2324
 mapillary_downloader/metadata_reader.py,sha256=Re-HN0Vfc7Hs1eOut7uOoW7jWJ2PIbKoNzC7Ak3ah5o,4933
 mapillary_downloader/tar_sequences.py,sha256=hh77hfj0DFSPrPRfbGrOhvnZMctKESgO0gSpJXUxCCs,4886
-mapillary_downloader/utils.py,sha256=VgcwbC8yb2XlTGerTNwHBU42K2IN14VU7P-I52Vb01c,2947
+mapillary_downloader/utils.py,sha256=qQ_ewhN0b0r4KLfBtf9tjwewF9PHVF1swLt71t8x9F0,3058
 mapillary_downloader/webp_converter.py,sha256=vYLLQxDmdnqRz0nm7wXwRUd4x9mQZNah-DrncpA8sNs,1901
-mapillary_downloader/worker.py,sha256=K2DkQgFzALKs20TsG1KibNUdFiWN_v8MtVnBX_0xVyc,5162
+mapillary_downloader/worker.py,sha256=HOfbOtJG4fgYp6wvBjZnAKZDf59tWbWF0ivcD2z4SYk,5179
 mapillary_downloader/worker_pool.py,sha256=QnqYcPCi3GNu2e8GNG_qQ8v680PWzCZcGE5KeskqZxU,7868
-mapillary_downloader-0.7.5.dist-info/entry_points.txt,sha256=PdYtxOXHMJrUhmiPO4G-F98VuhUI4MN9D_T4KPrVZ5w,75
-mapillary_downloader-0.7.5.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
-mapillary_downloader-0.7.5.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-mapillary_downloader-0.7.5.dist-info/METADATA,sha256=UMqlEfuzAOLO30QSQdqCveUoIrhiq4WE_cgZvuWkxnw,5540
-mapillary_downloader-0.7.5.dist-info/RECORD,,
+mapillary_downloader-0.7.7.dist-info/entry_points.txt,sha256=PdYtxOXHMJrUhmiPO4G-F98VuhUI4MN9D_T4KPrVZ5w,75
+mapillary_downloader-0.7.7.dist-info/licenses/LICENSE.md,sha256=7_BIuQ-veOrsF-WarH8kTkm0-xrCLvJ1PFE1C4Ebs64,146
+mapillary_downloader-0.7.7.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+mapillary_downloader-0.7.7.dist-info/METADATA,sha256=vLcis_SSkhfsgixLzPAiQMay3Ug0bHcG7iPdCRzXeAI,5540
+mapillary_downloader-0.7.7.dist-info/RECORD,,