mapillary-tools 0.14.1__tar.gz → 0.14.4__tar.gz

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_tools
-Version: 0.14.1
+Version: 0.14.4
 Summary: Mapillary Image/Video Import Pipeline
 Author-email: Mapillary <support@mapillary.com>
 License: BSD

mapillary_tools-0.14.4/mapillary_tools/__init__.py

@@ -0,0 +1 @@
+VERSION = "0.14.4"

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/authenticate.py

@@ -165,7 +165,7 @@ def _prompt(message: str) -> str:
 
 def _validate_profile(user_items: config.UserItem) -> config.UserItem:
     try:
-        jsonschema.validate(user_items, config.UserItemSchema)
+        config.UserItemSchemaValidator.validate(user_items)
     except jsonschema.ValidationError as ex:
         raise exceptions.MapillaryBadParameterError(
             f"Invalid profile format: {ex.message}"

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/config.py

@@ -6,6 +6,8 @@ import sys
 import typing as T
 from typing import TypedDict
 
+import jsonschema
+
 if sys.version_info >= (3, 11):
     from typing import Required
 else:
@@ -50,6 +52,9 @@ UserItemSchema = {
 }
 
 
+UserItemSchemaValidator = jsonschema.Draft202012Validator(UserItemSchema)
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/geotag/options.py

@@ -173,8 +173,11 @@ SourceOptionSchema = {
 }
 
 
+SourceOptionSchemaValidator = jsonschema.Draft202012Validator(SourceOptionSchema)
+
+
 def validate_option(instance):
-    jsonschema.validate(instance=instance, schema=SourceOptionSchema)
+    SourceOptionSchemaValidator.validate(instance=instance)
 
 
 if __name__ == "__main__":

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/history.py

@@ -1,24 +1,17 @@
 from __future__ import annotations
 
-import contextlib
-import dbm
 import json
 import logging
+import os
+import sqlite3
 import string
 import threading
 import time
 import typing as T
+from functools import wraps
 from pathlib import Path
 
-# dbm modules are dynamically imported, so here we explicitly import dbm.sqlite3 to make sure pyinstaller include it
-# Otherwise you will see: ImportError: no dbm clone found; tried ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
-try:
-    import dbm.sqlite3  # type: ignore
-except ImportError:
-    pass
-
-
-from . import constants, types
+from . import constants, store, types
 from .serializer.description import DescriptionJSONSerializer
 
 JSONDict = T.Dict[str, T.Union[str, int, float, None]]
@@ -85,98 +78,140 @@ def write_history(
         fp.write(json.dumps(history))
 
 
+def _retry_on_database_lock_error(fn):
+    """
+    Decorator to retry a function if it raises a sqlite3.OperationalError with
+    "database is locked" in the message.
+    """
+
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        while True:
+            try:
+                return fn(*args, **kwargs)
+            except sqlite3.OperationalError as ex:
+                if "database is locked" in str(ex).lower():
+                    LOG.warning(f"{str(ex)}")
+                    LOG.info("Retrying in 1 second...")
+                    time.sleep(1)
+                else:
+                    raise ex
+
+    return wrapper
+
+
 class PersistentCache:
-    _lock: contextlib.nullcontext | threading.Lock
+    _lock: threading.Lock
 
     def __init__(self, file: str):
-        # SQLite3 backend supports concurrent access without a lock
-        if dbm.whichdb(file) == "dbm.sqlite3":
-            self._lock = contextlib.nullcontext()
-        else:
-            self._lock = threading.Lock()
         self._file = file
+        self._lock = threading.Lock()
 
     def get(self, key: str) -> str | None:
+        if not self._db_existed():
+            return None
+
         s = time.perf_counter()
 
-        with self._lock:
-            with dbm.open(self._file, flag="c") as db:
-                value: bytes | None = db.get(key)
+        with store.KeyValueStore(self._file, flag="r") as db:
+            try:
+                raw_payload: bytes | None = db.get(key)  # data retrieved from db[key]
+            except Exception as ex:
+                if self._table_not_found(ex):
+                    return None
+                raise ex
 
-        if value is None:
+        if raw_payload is None:
             return None
 
-        payload = self._decode(value)
+        data: JSONDict = self._decode(raw_payload)  # JSON dict decoded from db[key]
 
-        if self._is_expired(payload):
+        if self._is_expired(data):
             return None
 
-        file_handle = payload.get("file_handle")
+        cached_value = data.get("value")  # value in the JSON dict decoded from db[key]
 
         LOG.debug(
             f"Found file handle for {key} in cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
         )
 
-        return T.cast(str, file_handle)
+        return T.cast(str, cached_value)
 
-    def set(self, key: str, file_handle: str, expires_in: int = 3600 * 24 * 2) -> None:
+    @_retry_on_database_lock_error
+    def set(self, key: str, value: str, expires_in: int = 3600 * 24 * 2) -> None:
         s = time.perf_counter()
 
-        payload = {
+        data = {
             "expires_at": time.time() + expires_in,
-            "file_handle": file_handle,
+            "value": value,
         }
 
-        value: bytes = json.dumps(payload).encode("utf-8")
+        payload: bytes = json.dumps(data).encode("utf-8")
 
         with self._lock:
-            with dbm.open(self._file, flag="c") as db:
-                db[key] = value
+            with store.KeyValueStore(self._file, flag="c") as db:
+                db[key] = payload
 
         LOG.debug(
             f"Cached file handle for {key} ({(time.perf_counter() - s) * 1000:.0f} ms)"
         )
 
+    @_retry_on_database_lock_error
     def clear_expired(self) -> list[str]:
-        s = time.perf_counter()
-
         expired_keys: list[str] = []
 
-        with self._lock:
-            with dbm.open(self._file, flag="c") as db:
-                if hasattr(db, "items"):
-                    items: T.Iterable[tuple[str | bytes, bytes]] = db.items()
-                else:
-                    items = ((key, db[key]) for key in db.keys())
+        s = time.perf_counter()
 
-                for key, value in items:
-                    payload = self._decode(value)
-                    if self._is_expired(payload):
+        with self._lock:
+            with store.KeyValueStore(self._file, flag="c") as db:
+                for key, raw_payload in db.items():
+                    data = self._decode(raw_payload)
+                    if self._is_expired(data):
                         del db[key]
                         expired_keys.append(T.cast(str, key))
 
-        if expired_keys:
-            LOG.debug(
-                f"Cleared {len(expired_keys)} expired entries from the cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
-            )
+        LOG.debug(
+            f"Cleared {len(expired_keys)} expired entries from the cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
+        )
 
         return expired_keys
 
-    def _is_expired(self, payload: JSONDict) -> bool:
-        expires_at = payload.get("expires_at")
+    def keys(self) -> list[str]:
+        if not self._db_existed():
+            return []
+
+        try:
+            with store.KeyValueStore(self._file, flag="r") as db:
+                return [key.decode("utf-8") for key in db.keys()]
+        except Exception as ex:
+            if self._table_not_found(ex):
+                return []
+            raise ex
+
+    def _is_expired(self, data: JSONDict) -> bool:
+        expires_at = data.get("expires_at")
         if isinstance(expires_at, (int, float)):
             return expires_at is None or expires_at <= time.time()
         return False
 
-    def _decode(self, value: bytes) -> JSONDict:
+    def _decode(self, raw_payload: bytes) -> JSONDict:
         try:
-            payload = json.loads(value.decode("utf-8"))
+            data = json.loads(raw_payload.decode("utf-8"))
         except json.JSONDecodeError as ex:
             LOG.warning(f"Failed to decode cache value: {ex}")
             return {}
 
-        if not isinstance(payload, dict):
-            LOG.warning(f"Invalid cache value format: {payload}")
+        if not isinstance(data, dict):
+            LOG.warning(f"Invalid cache value format: {raw_payload!r}")
             return {}
 
-        return payload
+        return data
+
+    def _db_existed(self) -> bool:
+        return os.path.exists(self._file)
+
+    def _table_not_found(self, ex: Exception) -> bool:
+        if isinstance(ex, sqlite3.OperationalError):
+            if "no such table" in str(ex):
+                return True
+        return False

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/process_geotag_properties.py

@@ -304,19 +304,12 @@ def _validate_metadatas(
     # TypeError: __init__() missing 3 required positional arguments: 'image_time', 'gpx_start_time', and 'gpx_end_time'
     # See https://stackoverflow.com/a/61432070
     good_metadatas, error_metadatas = types.separate_errors(metadatas)
-    map_results = utils.mp_map_maybe(
-        validate_and_fail_metadata,
-        T.cast(T.Iterable[types.Metadata], good_metadatas),
-        num_processes=num_processes,
-    )
 
     validated_metadatas = list(
-        tqdm(
-            map_results,
-            desc="Validating metadatas",
-            unit="metadata",
-            disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-            total=len(good_metadatas),
+        utils.mp_map_maybe(
+            validate_and_fail_metadata,
+            T.cast(T.Iterable[types.Metadata], good_metadatas),
+            num_processes=num_processes,
         )
     )
 

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/serializer/description.py

@@ -259,6 +259,11 @@ ImageDescriptionFileSchema = _merge_schema(
 )
 
 
+ImageDescriptionFileSchemaValidator = jsonschema.Draft202012Validator(
+    ImageDescriptionFileSchema
+)
+
+
 VideoDescriptionFileSchema = _merge_schema(
     VideoDescriptionSchema,
     {
@@ -295,6 +300,11 @@ VideoDescriptionFileSchema = _merge_schema(
 )
 
 
+VideoDescriptionFileSchemaValidator = jsonschema.Draft202012Validator(
+    VideoDescriptionFileSchema
+)
+
+
 ImageVideoDescriptionFileSchema = {
     "oneOf": [VideoDescriptionFileSchema, ImageDescriptionFileSchema]
 }
@@ -520,7 +530,7 @@ def parse_capture_time(time: str) -> datetime.datetime:
 
 def validate_image_desc(desc: T.Any) -> None:
     try:
-        jsonschema.validate(instance=desc, schema=ImageDescriptionFileSchema)
+        ImageDescriptionFileSchemaValidator.validate(desc)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
         raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
@@ -533,7 +543,7 @@ def validate_image_desc(desc: T.Any) -> None:
 
 def validate_video_desc(desc: T.Any) -> None:
     try:
-        jsonschema.validate(instance=desc, schema=VideoDescriptionFileSchema)
+        VideoDescriptionFileSchemaValidator.validate(desc)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
         raise exceptions.MapillaryMetadataValidationError(ex.message) from ex

mapillary_tools-0.14.4/mapillary_tools/store.py

@@ -0,0 +1,128 @@
+"""
+This module provides a persistent key-value store based on SQLite.
+
+This implementation is mostly copied from dbm.sqlite3 in the Python standard library,
+but works for Python >= 3.9, whereas dbm.sqlite3 is only available for Python 3.13.
+
+Source: https://github.com/python/cpython/blob/3.13/Lib/dbm/sqlite3.py
+"""
+
+import os
+import sqlite3
+import sys
+from collections.abc import MutableMapping
+from contextlib import closing, suppress
+from pathlib import Path
+
+BUILD_TABLE = """
+  CREATE TABLE IF NOT EXISTS Dict (
+    key BLOB UNIQUE NOT NULL,
+    value BLOB NOT NULL
+  )
+"""
+GET_SIZE = "SELECT COUNT (key) FROM Dict"
+LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)"
+STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))"
+DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)"
+ITER_KEYS = "SELECT key FROM Dict"
+
+
+def _normalize_uri(path):
+    path = Path(path)
+    uri = path.absolute().as_uri()
+    while "//" in uri:
+        uri = uri.replace("//", "/")
+    return uri
+
+
+class KeyValueStore(MutableMapping):
+    def __init__(self, path, /, *, flag="r", mode=0o666):
+        """Open a key-value database and return the object.
+
+        The 'path' parameter is the name of the database file.
+
+        The optional 'flag' parameter can be one of ...:
+            'r' (default): open an existing database for read only access
+            'w': open an existing database for read/write access
+            'c': create a database if it does not exist; open for read/write access
+            'n': always create a new, empty database; open for read/write access
+
+        The optional 'mode' parameter is the Unix file access mode of the database;
+        only used when creating a new database. Default: 0o666.
+        """
+        path = os.fsdecode(path)
+        if flag == "r":
+            flag = "ro"
+        elif flag == "w":
+            flag = "rw"
+        elif flag == "c":
+            flag = "rwc"
+            Path(path).touch(mode=mode, exist_ok=True)
+        elif flag == "n":
+            flag = "rwc"
+            Path(path).unlink(missing_ok=True)
+            Path(path).touch(mode=mode)
+        else:
+            raise ValueError(f"Flag must be one of 'r', 'w', 'c', or 'n', not {flag!r}")
+
+        # We use the URI format when opening the database.
+        uri = _normalize_uri(path)
+        uri = f"{uri}?mode={flag}"
+
+        if sys.version_info >= (3, 12):
+            # This is the preferred way, but only available in Python 3.10 and newer.
+            self._cx = sqlite3.connect(uri, autocommit=True, uri=True)
+        else:
+            self._cx = sqlite3.connect(uri, uri=True)
+
+        # This is an optimization only; it's ok if it fails.
+        with suppress(sqlite3.OperationalError):
+            self._cx.execute("PRAGMA journal_mode = wal")
+
+        if flag == "rwc":
+            self._execute(BUILD_TABLE)
+
+    def _execute(self, *args, **kwargs):
+        if sys.version_info >= (3, 12):
+            return closing(self._cx.execute(*args, **kwargs))
+        else:
+            # Use a context manager to commit the changes
+            with self._cx:
+                return closing(self._cx.execute(*args, **kwargs))
+
+    def __len__(self):
+        with self._execute(GET_SIZE) as cu:
+            row = cu.fetchone()
+        return row[0]
+
+    def __getitem__(self, key):
+        with self._execute(LOOKUP_KEY, (key,)) as cu:
+            row = cu.fetchone()
+        if not row:
+            raise KeyError(key)
+        return row[0]
+
+    def __setitem__(self, key, value):
+        self._execute(STORE_KV, (key, value))
+
+    def __delitem__(self, key):
+        with self._execute(DELETE_KEY, (key,)) as cu:
+            if not cu.rowcount:
+                raise KeyError(key)
+
+    def __iter__(self):
+        with self._execute(ITER_KEYS) as cu:
+            for row in cu:
+                yield row[0]
+
+    def close(self):
+        self._cx.close()
+
+    def keys(self):
+        return list(super().keys())
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/upload.py

@@ -10,7 +10,6 @@ import uuid
 from pathlib import Path
 
 import humanize
-import jsonschema
 import requests
 from tqdm import tqdm
 
@@ -57,7 +56,7 @@ def upload(
 
     metadatas = _load_descs(_metadatas_from_process, import_paths, desc_path)
 
-    jsonschema.validate(instance=user_items, schema=config.UserItemSchema)
+    config.UserItemSchemaValidator.validate(user_items)
 
     # Setup the emitter -- the order matters here
 

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/uploader.py

@@ -2,6 +2,9 @@ from __future__ import annotations
 
 import concurrent.futures
 import dataclasses
+import datetime
+import email.utils
+import hashlib
 import io
 import json
 import logging
@@ -36,6 +39,7 @@ from . import (
     types,
     upload_api_v4,
     utils,
+    VERSION,
 )
 from .camm import camm_builder, camm_parser
 from .gpmf import gpmf_parser
@@ -55,6 +59,9 @@ class UploadOptions:
     user_items: config.UserItem
     chunk_size: int = int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024)
     num_upload_workers: int = constants.MAX_IMAGE_UPLOAD_WORKERS
+    # When set, upload cache will be read/write there
+    # This option is exposed for testing purpose. In PROD, the path is calculated based on envvar and user_items
+    upload_cache_path: Path | None = None
     dry_run: bool = False
     nofinish: bool = False
     noresume: bool = False
@@ -470,7 +477,7 @@ class ZipUploader:
                 # Arcname should be unique, the name does not matter
                 arcname = f"{idx}.jpg"
                 zipinfo = zipfile.ZipInfo(arcname, date_time=(1980, 1, 1, 0, 0, 0))
-                zipf.writestr(zipinfo, SingleImageUploader.dump_image_bytes(metadata))
+                zipf.writestr(zipinfo, CachedImageUploader.dump_image_bytes(metadata))
             assert len(sequence) == len(set(zipf.namelist()))
             zipf.comment = json.dumps(
                 {"sequence_md5sum": sequence_md5sum},
@@ -536,6 +543,13 @@ class ImageSequenceUploader:
     def __init__(self, upload_options: UploadOptions, emitter: EventEmitter):
         self.upload_options = upload_options
         self.emitter = emitter
+        # Create a single shared SingleImageUploader instance that will be used across all uploads
+        cache = _maybe_create_persistent_cache_instance(self.upload_options)
+        if cache:
+            cache.clear_expired()
+        self.cached_image_uploader = CachedImageUploader(
+            self.upload_options, cache=cache
+        )
 
     def upload_images(
         self, image_metadatas: T.Sequence[types.ImageMetadata]
@@ -687,10 +701,6 @@ class ImageSequenceUploader:
         with api_v4.create_user_session(
             self.upload_options.user_items["user_upload_token"]
         ) as user_session:
-            single_image_uploader = SingleImageUploader(
-                self.upload_options, user_session=user_session
-            )
-
             while True:
                 # Assert that all images are already pushed into the queue
                 try:
@@ -709,8 +719,8 @@ class ImageSequenceUploader:
                 }
 
                 # image_progress will be updated during uploading
-                file_handle = single_image_uploader.upload(
-                    image_metadata, image_progress
+                file_handle = self.cached_image_uploader.upload(
+                    user_session, image_metadata, image_progress
                 )
 
                 # Update chunk_size (it was constant if set)
@@ -730,24 +740,27 @@ class ImageSequenceUploader:
         return indexed_file_handles
 
 
-class SingleImageUploader:
+class CachedImageUploader:
     def __init__(
         self,
         upload_options: UploadOptions,
-        user_session: requests.Session | None = None,
+        cache: history.PersistentCache | None = None,
     ):
         self.upload_options = upload_options
-        self.user_session = user_session
-        self.cache = self._maybe_create_persistent_cache_instance(
-            self.upload_options.user_items, upload_options
-        )
+        self.cache = cache
+        if self.cache:
+            self.cache.clear_expired()
 
+    # Thread-safe
     def upload(
-        self, image_metadata: types.ImageMetadata, image_progress: dict[str, T.Any]
+        self,
+        user_session: requests.Session,
+        image_metadata: types.ImageMetadata,
+        image_progress: dict[str, T.Any],
     ) -> str:
         image_bytes = self.dump_image_bytes(image_metadata)
 
-        uploader = Uploader(self.upload_options, user_session=self.user_session)
+        uploader = Uploader(self.upload_options, user_session=user_session)
 
         session_key = uploader._gen_session_key(io.BytesIO(image_bytes), image_progress)
 
@@ -785,44 +798,7 @@ class SingleImageUploader:
                 f"Failed to dump EXIF bytes: {ex}", metadata.filename
             ) from ex
 
-    @classmethod
-    def _maybe_create_persistent_cache_instance(
-        cls, user_items: config.UserItem, upload_options: UploadOptions
-    ) -> history.PersistentCache | None:
-        if not constants.UPLOAD_CACHE_DIR:
-            LOG.debug(
-                "Upload cache directory is set empty, skipping caching upload file handles"
-            )
-            return None
-
-        if upload_options.dry_run:
-            LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
-            return None
-
-        cache_path_dir = (
-            Path(constants.UPLOAD_CACHE_DIR)
-            .joinpath(api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"))
-            .joinpath(
-                user_items.get("MAPSettingsUserKey", user_items["user_upload_token"])
-            )
-        )
-        cache_path_dir.mkdir(parents=True, exist_ok=True)
-        cache_path = cache_path_dir.joinpath("cached_file_handles")
-
-        # Sanitize sensitive segments for logging
-        sanitized_cache_path = (
-            Path(constants.UPLOAD_CACHE_DIR)
-            .joinpath("***")
-            .joinpath("***")
-            .joinpath("cached_file_handles")
-        )
-        LOG.debug(f"File handle cache path: {sanitized_cache_path}")
-
-        cache = history.PersistentCache(str(cache_path.resolve()))
-        cache.clear_expired()
-
-        return cache
-
+    # Thread-safe
     def _get_cached_file_handle(self, key: str) -> str | None:
         if self.cache is None:
             return None
@@ -832,6 +808,7 @@ class SingleImageUploader:
 
         return self.cache.get(key)
 
+    # Thread-safe
     def _set_file_handle_cache(self, key: str, value: str) -> None:
         if self.cache is None:
             return
@@ -971,27 +948,33 @@ class Uploader:
         begin_offset = progress.get("begin_offset")
         offset = progress.get("offset")
 
-        if retries <= constants.MAX_UPLOAD_RETRIES and _is_retriable_exception(ex):
-            self.emitter.emit("upload_retrying", progress)
+        LOG.warning(
+            f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
+        )
 
-            LOG.warning(
-                f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
-            )
+        if retries <= constants.MAX_UPLOAD_RETRIES:
+            retriable, retry_after_sec = _is_retriable_exception(ex)
+            if retriable:
+                self.emitter.emit("upload_retrying", progress)
 
-            # Keep things immutable here. Will increment retries in the caller
-            retries += 1
-            if _is_immediate_retriable_exception(ex):
-                sleep_for = 0
-            else:
-                sleep_for = min(2**retries, 16)
-            LOG.info(
-                f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
-            )
-            if sleep_for:
-                time.sleep(sleep_for)
-        else:
-            self.emitter.emit("upload_failed", progress)
-            raise ex
+                # Keep things immutable here. Will increment retries in the caller
+                retries += 1
+                if _is_immediate_retriable_exception(ex):
+                    sleep_for = 0
+                else:
+                    sleep_for = min(2**retries, 16)
+                sleep_for += retry_after_sec
+
+                LOG.info(
+                    f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
+                )
+                if sleep_for:
+                    time.sleep(sleep_for)
+
+                return
+
+        self.emitter.emit("upload_failed", progress)
+        raise ex
 
     @classmethod
     def _upload_name(cls, progress: UploaderProgress):
@@ -1108,23 +1091,188 @@ def _is_immediate_retriable_exception(ex: BaseException) -> bool:
     return False
 
 
-def _is_retriable_exception(ex: BaseException) -> bool:
+def _is_retriable_exception(ex: BaseException) -> tuple[bool, int]:
+    """
+    Determine if an exception should be retried and how long to wait.
+
+    Args:
+        ex: Exception to check for retryability
+
+    Returns:
+        Tuple of (retriable, retry_after_sec) where:
+        - retriable: True if the exception should be retried
+        - retry_after_sec: Seconds to wait before retry (>= 0)
+
+    Examples:
+    >>> resp = requests.Response()
+    >>> resp._content = b"foo"
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (False, 0)
+    >>> resp._content = b'{"backoff": 13000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 13)
+    >>> resp._content = b'{"backoff": "foo", "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b'{"debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 400
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b"foo"
+    >>> resp.status_code = 429
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b"foo"
+    >>> resp.status_code = 429
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 10)
+    >>> resp._content = b'{"backoff": 12000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.status_code = 429
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 12)
+    >>> resp._content = b'{"backoff": 12000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
+    >>> resp.headers = {"Retry-After": "1"}
+    >>> resp.status_code = 503
+    >>> ex = requests.HTTPError("error", response=resp)
+    >>> _is_retriable_exception(ex)
+    (True, 1)
+    """
+
+    DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC = 10
+
     if isinstance(ex, (requests.ConnectionError, requests.Timeout)):
-        return True
+        return True, 0
 
     if isinstance(ex, requests.HTTPError) and isinstance(
         ex.response, requests.Response
     ):
-        if 400 <= ex.response.status_code < 500:
+        status_code = ex.response.status_code
+
+        # Always retry with some delay
+        if status_code == 429:
+            retry_after_sec = (
+                _parse_retry_after_from_header(ex.response)
+                or DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC
+            )
+
             try:
-                resp = ex.response.json()
-            except json.JSONDecodeError:
-                return False
-            return resp.get("debug_info", {}).get("retriable", False)
-        else:
-            return True
+                data = ex.response.json()
+            except requests.JSONDecodeError:
+                return True, retry_after_sec
 
-    return False
+            backoff_ms = _parse_backoff(data.get("backoff"))
+            if backoff_ms is None:
+                return True, retry_after_sec
+            else:
+                return True, max(0, int(int(backoff_ms) / 1000))
+
+        if 400 <= status_code < 500:
+            try:
+                data = ex.response.json()
+            except requests.JSONDecodeError:
+                return False, (_parse_retry_after_from_header(ex.response) or 0)
+
+            debug_info = data.get("debug_info", {})
+
+            if isinstance(debug_info, dict):
+                error_type = debug_info.get("type")
+            else:
+                error_type = None
+
+            # The server may respond 429 RequestRateLimitedError but with retryable=False
+            # We should retry for this case regardless
+            # e.g. HTTP 429 {"backoff": 10000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}
+            if error_type == "RequestRateLimitedError":
+                backoff_ms = _parse_backoff(data.get("backoff"))
+                if backoff_ms is None:
+                    return True, (
+                        _parse_retry_after_from_header(ex.response)
+                        or DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC
+                    )
+                else:
+                    return True, max(0, int(int(backoff_ms) / 1000))
+
+            return debug_info.get("retriable", False), 0
+
+        if 500 <= status_code < 600:
+            return True, (_parse_retry_after_from_header(ex.response) or 0)
+
+    return False, 0
+
+
+def _parse_backoff(backoff: T.Any) -> int | None:
+    if backoff is not None:
+        try:
+            backoff_ms = int(backoff)
+        except (ValueError, TypeError):
+            backoff_ms = None
+    else:
+        backoff_ms = None
+    return backoff_ms
+
+
+def _parse_retry_after_from_header(resp: requests.Response) -> int | None:
+    """
+    Parse Retry-After header from HTTP response.
+    See See https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After
+
+    Args:
+        resp: HTTP response object with headers
+
+    Returns:
+        Number of seconds to wait (>= 0) or None if header missing/invalid.
+
+    Examples:
+    >>> resp = requests.Response()
+    >>> resp.headers = {"Retry-After": "1"}
+    >>> _parse_retry_after_from_header(resp)
+    1
+    >>> resp.headers = {"Retry-After": "-1"}
+    >>> _parse_retry_after_from_header(resp)
+    0
+    >>> resp.headers = {"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}
+    >>> _parse_retry_after_from_header(resp)
+    0
+    >>> resp.headers = {"Retry-After": "Wed, 21 Oct 2315 07:28:00"}
+    >>> _parse_retry_after_from_header(resp)
+    """
+
+    value = resp.headers.get("Retry-After")
+    if value is None:
+        return None
+
+    try:
+        return max(0, int(value))
+    except (ValueError, TypeError):
+        pass
+
+    # e.g. "Wed, 21 Oct 2015 07:28:00 GMT"
+    try:
+        dt = email.utils.parsedate_to_datetime(value)
+    except (ValueError, TypeError):
+        dt = None
+
+    if dt is None:
+        LOG.warning(f"Error parsing Retry-After: {value}")
+        return None
+
+    try:
+        delta = dt - datetime.datetime.now(datetime.timezone.utc)
+    except (TypeError, ValueError):
+        # e.g. TypeError: can't subtract offset-naive and offset-aware datetimes
+        return None
+
+    return max(0, int(delta.total_seconds()))
 
 
 _SUFFIX_MAP: dict[api_v4.ClusterFileType | types.FileType, str] = {
@@ -1160,3 +1308,57 @@ def _prefixed_uuid4():
 
 def _is_uuid(key: str) -> bool:
     return key.startswith("uuid_") or key.startswith("mly_tools_uuid_")
+
+
+def _build_upload_cache_path(upload_options: UploadOptions) -> Path:
+    # Different python/CLI versions use different cache formats.
+    # Separate them to avoid conflicts
+    py_version_parts = [str(part) for part in sys.version_info[:3]]
+    version = f"py_{'_'.join(py_version_parts)}_{VERSION}"
+    # File handles are not sharable between different users
+    user_id = str(
+        upload_options.user_items.get(
+            "MAPSettingsUserKey", upload_options.user_items["user_upload_token"]
+        )
+    )
+    # Use hash to avoid log sensitive data
+    user_fingerprint = utils.md5sum_fp(
+        io.BytesIO((api_v4.MAPILLARY_CLIENT_TOKEN + user_id).encode("utf-8")),
+        md5=hashlib.sha256(),
+    ).hexdigest()[:24]
+
+    cache_path = (
+        Path(constants.UPLOAD_CACHE_DIR)
+        .joinpath(version)
+        .joinpath(user_fingerprint)
+        .joinpath("cached_file_handles")
+    )
+
+    return cache_path
+
+
+def _maybe_create_persistent_cache_instance(
+    upload_options: UploadOptions,
+) -> history.PersistentCache | None:
+    """Create a persistent cache instance if caching is enabled."""
+
+    if upload_options.dry_run:
+        LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
+        return None
+
+    if upload_options.upload_cache_path is None:
+        if not constants.UPLOAD_CACHE_DIR:
+            LOG.debug(
+                "Upload cache directory is set empty, skipping caching upload file handles"
+            )
+            return None
+
+        cache_path = _build_upload_cache_path(upload_options)
+    else:
+        cache_path = upload_options.upload_cache_path
+
+    LOG.debug(f"File handle cache path: {cache_path}")
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
+    return history.PersistentCache(str(cache_path.resolve()))

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools/utils.py

@@ -247,8 +247,8 @@ def configure_logger(
     try:
         # Disable globally for now. TODO Disable it in non-interactive mode only
         raise ImportError
-        from rich.console import Console  # type: ignore
-        from rich.logging import RichHandler  # type: ignore
+        from rich.console import Console  # type: ignore[import]
+        from rich.logging import RichHandler  # type: ignore[import]
     except ImportError:
         formatter = logging.Formatter(
             "%(asctime)s.%(msecs)03d - %(levelname)-7s - %(message)s",

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_tools
-Version: 0.14.1
+Version: 0.14.4
 Summary: Mapillary Image/Video Import Pipeline
 Author-email: Mapillary <support@mapillary.com>
 License: BSD

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ mapillary_tools/ipc.py
 mapillary_tools/process_geotag_properties.py
 mapillary_tools/process_sequence_properties.py
 mapillary_tools/sample_video.py
+mapillary_tools/store.py
 mapillary_tools/telemetry.py
 mapillary_tools/types.py
 mapillary_tools/upload.py

{mapillary_tools-0.14.1 → mapillary_tools-0.14.4}/pyproject.toml

@@ -81,5 +81,6 @@ module = [
     "exifread.*",
     "construct.*",
     "py.*",
+    "humanize.*",
 ]
 ignore_missing_imports = true

mapillary_tools-0.14.1/mapillary_tools/__init__.py

@@ -1 +0,0 @@
-VERSION = "0.14.1"