mapillary-tools 0.14.2__py3-none-any.whl → 0.14.4__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.14.2"
+VERSION = "0.14.4"

mapillary_tools/history.py

@@ -1,24 +1,17 @@
 from __future__ import annotations
 
-import contextlib
-import dbm
 import json
 import logging
+import os
+import sqlite3
 import string
 import threading
 import time
 import typing as T
+from functools import wraps
 from pathlib import Path
 
-# dbm modules are dynamically imported, so here we explicitly import dbm.sqlite3 to make sure pyinstaller include it
-# Otherwise you will see: ImportError: no dbm clone found; tried ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
-try:
-    import dbm.sqlite3  # type: ignore
-except ImportError:
-    pass
-
-
-from . import constants, types
+from . import constants, store, types
 from .serializer.description import DescriptionJSONSerializer
 
 JSONDict = T.Dict[str, T.Union[str, int, float, None]]
@@ -85,98 +78,140 @@ def write_history(
         fp.write(json.dumps(history))
 
 
+def _retry_on_database_lock_error(fn):
+    """
+    Decorator to retry a function if it raises a sqlite3.OperationalError with
+    "database is locked" in the message.
+    """
+
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        while True:
+            try:
+                return fn(*args, **kwargs)
+            except sqlite3.OperationalError as ex:
+                if "database is locked" in str(ex).lower():
+                    LOG.warning(f"{str(ex)}")
+                    LOG.info("Retrying in 1 second...")
+                    time.sleep(1)
+                else:
+                    raise ex
+
+    return wrapper
+
+
 class PersistentCache:
-    _lock: contextlib.nullcontext | threading.Lock
+    _lock: threading.Lock
 
     def __init__(self, file: str):
-        # SQLite3 backend supports concurrent access without a lock
-        if dbm.whichdb(file) == "dbm.sqlite3":
-            self._lock = contextlib.nullcontext()
-        else:
-            self._lock = threading.Lock()
         self._file = file
+        self._lock = threading.Lock()
 
     def get(self, key: str) -> str | None:
+        if not self._db_existed():
+            return None
+
         s = time.perf_counter()
 
-        with self._lock:
-            with dbm.open(self._file, flag="c") as db:
-                value: bytes | None = db.get(key)
+        with store.KeyValueStore(self._file, flag="r") as db:
+            try:
+                raw_payload: bytes | None = db.get(key)  # data retrieved from db[key]
+            except Exception as ex:
+                if self._table_not_found(ex):
+                    return None
+                raise ex
 
-        if value is None:
+        if raw_payload is None:
             return None
 
-        payload = self._decode(value)
+        data: JSONDict = self._decode(raw_payload)  # JSON dict decoded from db[key]
 
-        if self._is_expired(payload):
+        if self._is_expired(data):
             return None
 
-        file_handle = payload.get("file_handle")
+        cached_value = data.get("value")  # value in the JSON dict decoded from db[key]
 
         LOG.debug(
             f"Found file handle for {key} in cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
         )
 
-        return T.cast(str, file_handle)
+        return T.cast(str, cached_value)
 
-    def set(self, key: str, file_handle: str, expires_in: int = 3600 * 24 * 2) -> None:
+    @_retry_on_database_lock_error
+    def set(self, key: str, value: str, expires_in: int = 3600 * 24 * 2) -> None:
         s = time.perf_counter()
 
-        payload = {
+        data = {
             "expires_at": time.time() + expires_in,
-            "file_handle": file_handle,
+            "value": value,
         }
 
-        value: bytes = json.dumps(payload).encode("utf-8")
+        payload: bytes = json.dumps(data).encode("utf-8")
 
         with self._lock:
-            with dbm.open(self._file, flag="c") as db:
-                db[key] = value
+            with store.KeyValueStore(self._file, flag="c") as db:
+                db[key] = payload
 
         LOG.debug(
             f"Cached file handle for {key} ({(time.perf_counter() - s) * 1000:.0f} ms)"
         )
 
+    @_retry_on_database_lock_error
     def clear_expired(self) -> list[str]:
-        s = time.perf_counter()
-
         expired_keys: list[str] = []
 
-        with self._lock:
-            with dbm.open(self._file, flag="c") as db:
-                if hasattr(db, "items"):
-                    items: T.Iterable[tuple[str | bytes, bytes]] = db.items()
-                else:
-                    items = ((key, db[key]) for key in db.keys())
+        s = time.perf_counter()
 
-                for key, value in items:
-                    payload = self._decode(value)
-                    if self._is_expired(payload):
+        with self._lock:
+            with store.KeyValueStore(self._file, flag="c") as db:
+                for key, raw_payload in db.items():
+                    data = self._decode(raw_payload)
+                    if self._is_expired(data):
                         del db[key]
                         expired_keys.append(T.cast(str, key))
 
-        if expired_keys:
-            LOG.debug(
-                f"Cleared {len(expired_keys)} expired entries from the cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
-            )
+        LOG.debug(
+            f"Cleared {len(expired_keys)} expired entries from the cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
+        )
 
         return expired_keys
 
-    def _is_expired(self, payload: JSONDict) -> bool:
-        expires_at = payload.get("expires_at")
+    def keys(self) -> list[str]:
+        if not self._db_existed():
+            return []
+
+        try:
+            with store.KeyValueStore(self._file, flag="r") as db:
+                return [key.decode("utf-8") for key in db.keys()]
+        except Exception as ex:
+            if self._table_not_found(ex):
+                return []
+            raise ex
+
+    def _is_expired(self, data: JSONDict) -> bool:
+        expires_at = data.get("expires_at")
         if isinstance(expires_at, (int, float)):
             return expires_at is None or expires_at <= time.time()
         return False
 
-    def _decode(self, value: bytes) -> JSONDict:
+    def _decode(self, raw_payload: bytes) -> JSONDict:
         try:
-            payload = json.loads(value.decode("utf-8"))
+            data = json.loads(raw_payload.decode("utf-8"))
         except json.JSONDecodeError as ex:
             LOG.warning(f"Failed to decode cache value: {ex}")
             return {}
 
-        if not isinstance(payload, dict):
-            LOG.warning(f"Invalid cache value format: {payload}")
+        if not isinstance(data, dict):
+            LOG.warning(f"Invalid cache value format: {raw_payload!r}")
             return {}
 
-        return payload
+        return data
+
+    def _db_existed(self) -> bool:
+        return os.path.exists(self._file)
+
+    def _table_not_found(self, ex: Exception) -> bool:
+        if isinstance(ex, sqlite3.OperationalError):
+            if "no such table" in str(ex):
+                return True
+        return False

mapillary_tools/store.py

@@ -0,0 +1,128 @@
+"""
+This module provides a persistent key-value store based on SQLite.
+
+This implementation is mostly copied from dbm.sqlite3 in the Python standard library,
+but works for Python >= 3.9, whereas dbm.sqlite3 is only available for Python 3.13.
+
+Source: https://github.com/python/cpython/blob/3.13/Lib/dbm/sqlite3.py
+"""
+
+import os
+import sqlite3
+import sys
+from collections.abc import MutableMapping
+from contextlib import closing, suppress
+from pathlib import Path
+
+BUILD_TABLE = """
+  CREATE TABLE IF NOT EXISTS Dict (
+    key BLOB UNIQUE NOT NULL,
+    value BLOB NOT NULL
+  )
+"""
+GET_SIZE = "SELECT COUNT (key) FROM Dict"
+LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)"
+STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))"
+DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)"
+ITER_KEYS = "SELECT key FROM Dict"
+
+
+def _normalize_uri(path):
+    path = Path(path)
+    uri = path.absolute().as_uri()
+    while "//" in uri:
+        uri = uri.replace("//", "/")
+    return uri
+
+
+class KeyValueStore(MutableMapping):
+    def __init__(self, path, /, *, flag="r", mode=0o666):
+        """Open a key-value database and return the object.
+
+        The 'path' parameter is the name of the database file.
+
+        The optional 'flag' parameter can be one of ...:
+            'r' (default): open an existing database for read only access
+            'w': open an existing database for read/write access
+            'c': create a database if it does not exist; open for read/write access
+            'n': always create a new, empty database; open for read/write access
+
+        The optional 'mode' parameter is the Unix file access mode of the database;
+        only used when creating a new database. Default: 0o666.
+        """
+        path = os.fsdecode(path)
+        if flag == "r":
+            flag = "ro"
+        elif flag == "w":
+            flag = "rw"
+        elif flag == "c":
+            flag = "rwc"
+            Path(path).touch(mode=mode, exist_ok=True)
+        elif flag == "n":
+            flag = "rwc"
+            Path(path).unlink(missing_ok=True)
+            Path(path).touch(mode=mode)
+        else:
+            raise ValueError(f"Flag must be one of 'r', 'w', 'c', or 'n', not {flag!r}")
+
+        # We use the URI format when opening the database.
+        uri = _normalize_uri(path)
+        uri = f"{uri}?mode={flag}"
+
+        if sys.version_info >= (3, 12):
+            # This is the preferred way, but only available in Python 3.10 and newer.
+            self._cx = sqlite3.connect(uri, autocommit=True, uri=True)
+        else:
+            self._cx = sqlite3.connect(uri, uri=True)
+
+        # This is an optimization only; it's ok if it fails.
+        with suppress(sqlite3.OperationalError):
+            self._cx.execute("PRAGMA journal_mode = wal")
+
+        if flag == "rwc":
+            self._execute(BUILD_TABLE)
+
+    def _execute(self, *args, **kwargs):
+        if sys.version_info >= (3, 12):
+            return closing(self._cx.execute(*args, **kwargs))
+        else:
+            # Use a context manager to commit the changes
+            with self._cx:
+                return closing(self._cx.execute(*args, **kwargs))
+
+    def __len__(self):
+        with self._execute(GET_SIZE) as cu:
+            row = cu.fetchone()
+        return row[0]
+
+    def __getitem__(self, key):
+        with self._execute(LOOKUP_KEY, (key,)) as cu:
+            row = cu.fetchone()
+        if not row:
+            raise KeyError(key)
+        return row[0]
+
+    def __setitem__(self, key, value):
+        self._execute(STORE_KV, (key, value))
+
+    def __delitem__(self, key):
+        with self._execute(DELETE_KEY, (key,)) as cu:
+            if not cu.rowcount:
+                raise KeyError(key)
+
+    def __iter__(self):
+        with self._execute(ITER_KEYS) as cu:
+            for row in cu:
+                yield row[0]
+
+    def close(self):
+        self._cx.close()
+
+    def keys(self):
+        return list(super().keys())
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()

mapillary_tools/uploader.py

@@ -2,6 +2,9 @@ from __future__ import annotations
 
 import concurrent.futures
 import dataclasses
+import datetime
+import email.utils
+import hashlib
 import io
 import json
 import logging
@@ -56,6 +59,9 @@ class UploadOptions:
     user_items: config.UserItem
     chunk_size: int = int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024)
     num_upload_workers: int = constants.MAX_IMAGE_UPLOAD_WORKERS
+    # When set, upload cache will be read/write there
+    # This option is exposed for testing purpose. In PROD, the path is calculated based on envvar and user_items
+    upload_cache_path: Path | None = None
     dry_run: bool = False
     nofinish: bool = False
     noresume: bool = False
@@ -471,7 +477,7 @@ class ZipUploader:
                 # Arcname should be unique, the name does not matter
                 arcname = f"{idx}.jpg"
                 zipinfo = zipfile.ZipInfo(arcname, date_time=(1980, 1, 1, 0, 0, 0))
-                zipf.writestr(zipinfo, SingleImageUploader.dump_image_bytes(metadata))
+                zipf.writestr(zipinfo, CachedImageUploader.dump_image_bytes(metadata))
             assert len(sequence) == len(set(zipf.namelist()))
             zipf.comment = json.dumps(
                 {"sequence_md5sum": sequence_md5sum},
@@ -537,6 +543,13 @@ class ImageSequenceUploader:
     def __init__(self, upload_options: UploadOptions, emitter: EventEmitter):
         self.upload_options = upload_options
         self.emitter = emitter
+        # Create a single shared SingleImageUploader instance that will be used across all uploads
+        cache = _maybe_create_persistent_cache_instance(self.upload_options)
+        if cache:
+            cache.clear_expired()
+        self.cached_image_uploader = CachedImageUploader(
+            self.upload_options, cache=cache
+        )
 
     def upload_images(
         self, image_metadatas: T.Sequence[types.ImageMetadata]
@@ -688,10 +701,6 @@ class ImageSequenceUploader:
         with api_v4.create_user_session(
             self.upload_options.user_items["user_upload_token"]
         ) as user_session:
-            single_image_uploader = SingleImageUploader(
-                self.upload_options, user_session=user_session
-            )
-
             while True:
                 # Assert that all images are already pushed into the queue
                 try:
@@ -710,8 +719,8 @@ class ImageSequenceUploader:
                 }
 
                 # image_progress will be updated during uploading
-                file_handle = single_image_uploader.upload(
-                    image_metadata, image_progress
+                file_handle = self.cached_image_uploader.upload(
+                    user_session, image_metadata, image_progress
                 )
 
                 # Update chunk_size (it was constant if set)
@@ -731,24 +740,27 @@ class ImageSequenceUploader:
         return indexed_file_handles
 
 
-class SingleImageUploader:
+class CachedImageUploader:
     def __init__(
         self,
         upload_options: UploadOptions,
-        user_session: requests.Session | None = None,
+        cache: history.PersistentCache | None = None,
     ):
         self.upload_options = upload_options
-        self.user_session = user_session
-        self.cache = self._maybe_create_persistent_cache_instance(
-            self.upload_options.user_items, upload_options
-        )
+        self.cache = cache
+        if self.cache:
+            self.cache.clear_expired()
 
+    # Thread-safe
     def upload(
-        self, image_metadata: types.ImageMetadata, image_progress: dict[str, T.Any]
+        self,
+        user_session: requests.Session,
+        image_metadata: types.ImageMetadata,
+        image_progress: dict[str, T.Any],
     ) -> str:
         image_bytes = self.dump_image_bytes(image_metadata)
 
-        uploader = Uploader(self.upload_options, user_session=self.user_session)
+        uploader = Uploader(self.upload_options, user_session=user_session)
 
         session_key = uploader._gen_session_key(io.BytesIO(image_bytes), image_progress)
 
@@ -786,51 +798,7 @@ class SingleImageUploader:
                 f"Failed to dump EXIF bytes: {ex}", metadata.filename
             ) from ex
 
-    @classmethod
-    def _maybe_create_persistent_cache_instance(
-        cls, user_items: config.UserItem, upload_options: UploadOptions
-    ) -> history.PersistentCache | None:
-        if not constants.UPLOAD_CACHE_DIR:
-            LOG.debug(
-                "Upload cache directory is set empty, skipping caching upload file handles"
-            )
-            return None
-
-        if upload_options.dry_run:
-            LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
-            return None
-
-        # Different python/CLI versions use different cache (dbm) formats.
-        # Separate them to avoid conflicts
-        py_version_parts = [str(part) for part in sys.version_info[:3]]
-        version = f"py_{'_'.join(py_version_parts)}_{VERSION}"
-
-        cache_path_dir = (
-            Path(constants.UPLOAD_CACHE_DIR)
-            .joinpath(version)
-            .joinpath(api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"))
-            .joinpath(
-                user_items.get("MAPSettingsUserKey", user_items["user_upload_token"])
-            )
-        )
-        cache_path_dir.mkdir(parents=True, exist_ok=True)
-        cache_path = cache_path_dir.joinpath("cached_file_handles")
-
-        # Sanitize sensitive segments for logging
-        sanitized_cache_path = (
-            Path(constants.UPLOAD_CACHE_DIR)
-            .joinpath(version)
-            .joinpath("***")
-            .joinpath("***")
-            .joinpath("cached_file_handles")
-        )
-        LOG.debug(f"File handle cache path: {sanitized_cache_path}")
-
-        cache = history.PersistentCache(str(cache_path.resolve()))
-        cache.clear_expired()
-
-        return cache
-
+    # Thread-safe
     def _get_cached_file_handle(self, key: str) -> str | None:
         if self.cache is None:
             return None
@@ -840,6 +808,7 @@ class SingleImageUploader:
 
         return self.cache.get(key)
 
+    # Thread-safe
     def _set_file_handle_cache(self, key: str, value: str) -> None:
         if self.cache is None:
             return
@@ -979,27 +948,33 @@ class Uploader:
         begin_offset = progress.get("begin_offset")
         offset = progress.get("offset")
 
-        if retries <= constants.MAX_UPLOAD_RETRIES and _is_retriable_exception(ex):
-            self.emitter.emit("upload_retrying", progress)
+        LOG.warning(
+            f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
+        )
 
-            LOG.warning(
-                f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
-            )
+        if retries <= constants.MAX_UPLOAD_RETRIES:
+            retriable, retry_after_sec = _is_retriable_exception(ex)
+            if retriable:
+                self.emitter.emit("upload_retrying", progress)
 
-            # Keep things immutable here. Will increment retries in the caller
-            retries += 1
-            if _is_immediate_retriable_exception(ex):
-                sleep_for = 0
-            else:
-                sleep_for = min(2**retries, 16)
-            LOG.info(
-                f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
-            )
-            if sleep_for:
-                time.sleep(sleep_for)
-        else:
-            self.emitter.emit("upload_failed", progress)
-            raise ex
+                # Keep things immutable here. Will increment retries in the caller
+                retries += 1
+                if _is_immediate_retriable_exception(ex):
+                    sleep_for = 0
+                else:
+                    sleep_for = min(2**retries, 16)
+                sleep_for += retry_after_sec
+
+                LOG.info(
+                    f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
+                )
+                if sleep_for:
+                    time.sleep(sleep_for)
+
+                return
+
+        self.emitter.emit("upload_failed", progress)
+        raise ex
 
     @classmethod
     def _upload_name(cls, progress: UploaderProgress):
@@ -1116,23 +1091,188 @@ def _is_immediate_retriable_exception(ex: BaseException) -> bool:
     return False
 
 
-def _is_retriable_exception(ex: BaseException) -> bool:
+def _is_retriable_exception(ex: BaseException) -> tuple[bool, int]:
+    """
+    Determine if an exception should be retried and how long to wait.
+
+    Args:
+        ex: Exception to check for retryability
+
+    Returns:
+        Tuple of (retriable, retry_after_sec) where:
+        - retriable: True if the exception should be retried
+        - retry_after_sec: Seconds to wait before retry (>= 0)
+
+    Examples:
+    >>> resp = requests.Response()
+    >>> resp._content = b"foo"
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (False, 0)
+    >>> resp._content = b'{"backoff": 13000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 13)
+    >>> resp._content = b'{"backoff": "foo", "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b'{"debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b"foo"
+    >>> resp.status_code = 429
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b"foo"
+    >>> resp.status_code = 429
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b'{"backoff": 12000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 429
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 12)
+    >>> resp._content = b'{"backoff": 12000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.headers = {"Retry-After": "1"}
+    >>> resp.status_code = 503
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 1)
+    """
+
+    DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC = 10
+
     if isinstance(ex, (requests.ConnectionError, requests.Timeout)):
-        return True
+        return True, 0
 
     if isinstance(ex, requests.HTTPError) and isinstance(
         ex.response, requests.Response
     ):
-        if 400 <= ex.response.status_code < 500:
+        status_code = ex.response.status_code
+
+        # Always retry with some delay
+        if status_code == 429:
+            retry_after_sec = (
+                _parse_retry_after_from_header(ex.response)
+                or DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC
+            )
+
             try:
-                resp = ex.response.json()
-            except json.JSONDecodeError:
-                return False
-            return resp.get("debug_info", {}).get("retriable", False)
-        else:
-            return True
+                data = ex.response.json()
+            except requests.JSONDecodeError:
+                return True, retry_after_sec
 
-    return False
+            backoff_ms = _parse_backoff(data.get("backoff"))
+            if backoff_ms is None:
+                return True, retry_after_sec
+            else:
+                return True, max(0, int(int(backoff_ms) / 1000))
+
+        if 400 <= status_code < 500:
+            try:
+                data = ex.response.json()
+            except requests.JSONDecodeError:
+                return False, (_parse_retry_after_from_header(ex.response) or 0)
+
+            debug_info = data.get("debug_info", {})
+
+            if isinstance(debug_info, dict):
+                error_type = debug_info.get("type")
+            else:
+                error_type = None
+
+            # The server may respond 429 RequestRateLimitedError but with retryable=False
+            # We should retry for this case regardless
+            # e.g. HTTP 429 {"backoff": 10000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}
+            if error_type == "RequestRateLimitedError":
+                backoff_ms = _parse_backoff(data.get("backoff"))
+                if backoff_ms is None:
+                    return True, (
+                        _parse_retry_after_from_header(ex.response)
+                        or DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC
+                    )
+                else:
+                    return True, max(0, int(int(backoff_ms) / 1000))
+
+            return debug_info.get("retriable", False), 0
+
+        if 500 <= status_code < 600:
+            return True, (_parse_retry_after_from_header(ex.response) or 0)
+
+    return False, 0
+
+
+def _parse_backoff(backoff: T.Any) -> int | None:
+    if backoff is not None:
+        try:
+            backoff_ms = int(backoff)
+        except (ValueError, TypeError):
+            backoff_ms = None
+    else:
+        backoff_ms = None
+    return backoff_ms
+
+
+def _parse_retry_after_from_header(resp: requests.Response) -> int | None:
+    """
+    Parse Retry-After header from HTTP response.
+    See See https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After
+
+    Args:
+        resp: HTTP response object with headers
+
+    Returns:
+        Number of seconds to wait (>= 0) or None if header missing/invalid.
+
+    Examples:
+    >>> resp = requests.Response()
+    >>> resp.headers = {"Retry-After": "1"}
+    >>> _parse_retry_after_from_header(resp)
+    1
+    >>> resp.headers = {"Retry-After": "-1"}
+    >>> _parse_retry_after_from_header(resp)
+    0
+    >>> resp.headers = {"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}
+    >>> _parse_retry_after_from_header(resp)
+    0
+    >>> resp.headers = {"Retry-After": "Wed, 21 Oct 2315 07:28:00"}
+    >>> _parse_retry_after_from_header(resp)
+    """
+
+    value = resp.headers.get("Retry-After")
+    if value is None:
+        return None
+
+    try:
+        return max(0, int(value))
+    except (ValueError, TypeError):
+        pass
+
+    # e.g. "Wed, 21 Oct 2015 07:28:00 GMT"
+    try:
+        dt = email.utils.parsedate_to_datetime(value)
+    except (ValueError, TypeError):
+        dt = None
+
+    if dt is None:
+        LOG.warning(f"Error parsing Retry-After: {value}")
+        return None
+
+    try:
+        delta = dt - datetime.datetime.now(datetime.timezone.utc)
+    except (TypeError, ValueError):
+        # e.g. TypeError: can't subtract offset-naive and offset-aware datetimes
+        return None
+
+    return max(0, int(delta.total_seconds()))
 
 
 _SUFFIX_MAP: dict[api_v4.ClusterFileType | types.FileType, str] = {
@@ -1168,3 +1308,57 @@ def _prefixed_uuid4():
 
 def _is_uuid(key: str) -> bool:
     return key.startswith("uuid_") or key.startswith("mly_tools_uuid_")
+
+
+def _build_upload_cache_path(upload_options: UploadOptions) -> Path:
+    # Different python/CLI versions use different cache formats.
+    # Separate them to avoid conflicts
+    py_version_parts = [str(part) for part in sys.version_info[:3]]
+    version = f"py_{'_'.join(py_version_parts)}_{VERSION}"
+    # File handles are not sharable between different users
+    user_id = str(
+        upload_options.user_items.get(
+            "MAPSettingsUserKey", upload_options.user_items["user_upload_token"]
+        )
+    )
+    # Use hash to avoid log sensitive data
+    user_fingerprint = utils.md5sum_fp(
+        io.BytesIO((api_v4.MAPILLARY_CLIENT_TOKEN + user_id).encode("utf-8")),
+        md5=hashlib.sha256(),
+    ).hexdigest()[:24]
+
+    cache_path = (
+        Path(constants.UPLOAD_CACHE_DIR)
+        .joinpath(version)
+        .joinpath(user_fingerprint)
+        .joinpath("cached_file_handles")
+    )
+
+    return cache_path
+
+
+def _maybe_create_persistent_cache_instance(
+    upload_options: UploadOptions,
+) -> history.PersistentCache | None:
+    """Create a persistent cache instance if caching is enabled."""
+
+    if upload_options.dry_run:
+        LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
+        return None
+
+    if upload_options.upload_cache_path is None:
+        if not constants.UPLOAD_CACHE_DIR:
+            LOG.debug(
+                "Upload cache directory is set empty, skipping caching upload file handles"
+            )
+            return None
+
+        cache_path = _build_upload_cache_path(upload_options)
+    else:
+        cache_path = upload_options.upload_cache_path
+
+    LOG.debug(f"File handle cache path: {cache_path}")
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
+    return history.PersistentCache(str(cache_path.resolve()))

mapillary_tools/utils.py

@@ -247,8 +247,8 @@ def configure_logger(
     try:
         # Disable globally for now. TODO Disable it in non-interactive mode only
         raise ImportError
-        from rich.console import Console  # type: ignore
-        from rich.logging import RichHandler  # type: ignore
+        from rich.console import Console  # type: ignore[import]
+        from rich.logging import RichHandler  # type: ignore[import]
     except ImportError:
         formatter = logging.Formatter(
             "%(asctime)s.%(msecs)03d - %(levelname)-7s - %(message)s",

{mapillary_tools-0.14.2.dist-info → mapillary_tools-0.14.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_tools
-Version: 0.14.2
+Version: 0.14.4
 Summary: Mapillary Image/Video Import Pipeline
 Author-email: Mapillary <support@mapillary.com>
 License: BSD

{mapillary_tools-0.14.2.dist-info → mapillary_tools-0.14.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-mapillary_tools/__init__.py,sha256=qdi1NvyXrEYAkFGfNSU2jMM-Ua-ehuWizcwvW341obw,19
+mapillary_tools/__init__.py,sha256=SlLq3LO5ZN4Ap8_XR8x6nXyckJ4qcIktiWVQg34oqMY,19
 mapillary_tools/api_v4.py,sha256=bckAU_atUs0pSuqySeY4W0Rs011a21ClJHo_mbbcXXw,4864
 mapillary_tools/authenticate.py,sha256=mmaOwjQ444DcX4lRw2ms3naBg5Y_xwIJAIWeVdsQfqM,11742
 mapillary_tools/blackvue_parser.py,sha256=ea2JtU9MWU6yB0bQlF970_Of0bJVofSTRq1P30WKW-0,5623
@@ -12,18 +12,19 @@ mapillary_tools/exiftool_read_video.py,sha256=23O_bjUOVq6j7i3xMz6fY-XIEsjinsCejK
 mapillary_tools/exiftool_runner.py,sha256=g4gSyqeh3D6EnMJ-c3s-RnO2EP_jD354Qkaz0Y-4D04,1658
 mapillary_tools/ffmpeg.py,sha256=akpvvsjAR-Iiv-hOrUoJvPM9vUU3JqMQ5HJL1_NgwB8,22908
 mapillary_tools/geo.py,sha256=mWaESfDf_zHmyvnt5aVFro4FGrjiULNsuZ6HfGUWvSA,11009
-mapillary_tools/history.py,sha256=LP6e0zEYVBwRGUbFaGoE_AaBIEdpB4XrZsg9qwJVvRI,5344
+mapillary_tools/history.py,sha256=MoJVp2D-JUPoORDvNhGt-2dgBstPLZ4nyPToIuqIAg4,6287
 mapillary_tools/http.py,sha256=-df_oGyImO2AOmPnXcKMcztlL4LOZLArE6ki81NMGUA,6411
 mapillary_tools/ipc.py,sha256=DwWQb9hNshx0bg0Fo5NjY0mXjs-FkbR6tIQmjMgMtmg,1089
 mapillary_tools/process_geotag_properties.py,sha256=3EaVvjfKB-O38OjopBcxeEdP6qI5IPIxqmO6isjcXKM,14205
 mapillary_tools/process_sequence_properties.py,sha256=n4VjQHrgVjksIr3WoBviRhrQIBBDHGXMClolfyz6tu4,24057
 mapillary_tools/sample_video.py,sha256=pKSj1Vc8e5p1XGjykBuKY9XieTOskc-9L3F4L407jDM,13935
+mapillary_tools/store.py,sha256=dA1D0afDvhVm0MYEI1yA5FfqjQM5etYK4pcfMc4nKAU,4077
 mapillary_tools/telemetry.py,sha256=lL6qQbtOZft4DZZrCNK3njlwHT_30zLyYS_YRN5pgHY,1568
 mapillary_tools/types.py,sha256=pIU2wcxiOUWT5Pd05pgNzY9EVEDlwoldtlF2IIYYvE0,5909
 mapillary_tools/upload.py,sha256=XejAgmVW4Y33MiQ2g-shvHZA_zXTekEsOUHUHNx2AE4,24047
 mapillary_tools/upload_api_v4.py,sha256=VgOf7RhfUuzmlSBUp5CpekKIJ0xQrC0r-r0Ds9-wU4I,7344
-mapillary_tools/uploader.py,sha256=Rw-1AkxE4TnddJNU6EW--9wmKYRqHbcTeheujdaluiM,39813
-mapillary_tools/utils.py,sha256=cP9idKt4EJqfC0qqOGneSoPNpPiYhaW8VjQ9CLYjESc,8092
+mapillary_tools/uploader.py,sha256=T2BNlncuFibg5RJ7c2qweUXDAg6-zYc-rdgbV_JEKDU,46636
+mapillary_tools/utils.py,sha256=HjTZ01GQv_UNGySaTZ_Mc1Gn_Y0x3knQf7Vh17whDFw,8108
 mapillary_tools/camm/camm_builder.py,sha256=ub6Z9ijep8zAo1NOlU51Gxk95kQ2vfN58YgVCLmNMRk,9211
 mapillary_tools/camm/camm_parser.py,sha256=aNHP65hNXYQBWBTfhaj_S5XYzmAHhjwcAfGhbm83__o,18043
 mapillary_tools/commands/__init__.py,sha256=41CFrPLGlG3566uhxssEF3TGAtSpADFPPcDMHbViU0E,171
@@ -68,9 +69,9 @@ mapillary_tools/mp4/simple_mp4_builder.py,sha256=9TUGk1hzI6mQFN1P30jwHL3dCYz3Zz7
 mapillary_tools/mp4/simple_mp4_parser.py,sha256=g3vvPhBoNu7anhVzC5_XQCV7IwfRWro1vJ6d6GyDkHE,6315
 mapillary_tools/serializer/description.py,sha256=ECnQxC-1LOgkAKE5qFi9Y2KuCeH8KPUjjNFDiwebjvo,18647
 mapillary_tools/serializer/gpx.py,sha256=_xx6gHjaWHrlXaUpB5GGBrbRKzbExFyIzWWAH-CvksI,4383
-mapillary_tools-0.14.2.dist-info/licenses/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
-mapillary_tools-0.14.2.dist-info/METADATA,sha256=rEmF5Twbh9m-zYNF1klQjGLOIrmiv5mCAQnbOb1Z8GE,22200
-mapillary_tools-0.14.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mapillary_tools-0.14.2.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
-mapillary_tools-0.14.2.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
-mapillary_tools-0.14.2.dist-info/RECORD,,
+mapillary_tools-0.14.4.dist-info/licenses/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
+mapillary_tools-0.14.4.dist-info/METADATA,sha256=DmtexHTWyoTovIswRtvkT638DbdPNXcfVfPcaCflu4w,22200
+mapillary_tools-0.14.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mapillary_tools-0.14.4.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
+mapillary_tools-0.14.4.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
+mapillary_tools-0.14.4.dist-info/RECORD,,