mapillary-tools 0.14.0a2__py3-none-any.whl → 0.14.1__py3-none-any.whl

mapillary_tools/blackvue_parser.py

@@ -14,15 +14,14 @@ from .mp4 import simple_mp4_parser as sparser
 
 
 LOG = logging.getLogger(__name__)
-# An example: [1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]
 NMEA_LINE_REGEX = re.compile(
     rb"""
     ^\s*
-    \[(\d+)\] # timestamp
+    \[(\d+)\] # Timestamp
     \s*
-    (\$\w{5}.*) # nmea line
+    (\$\w{5}.*) # NMEA message
     \s*
-    (\[\d+\])? # strange timestamp
+    (\[\d+\])? # Strange timestamp
     \s*$
     """,
     re.X,
@@ -47,7 +46,7 @@ def extract_blackvue_info(fp: T.BinaryIO) -> BlackVueInfo | None:
     if gps_data is None:
         return None
 
-    points = list(_parse_gps_box(gps_data))
+    points = _parse_gps_box(gps_data)
     points.sort(key=lambda p: p.time)
 
     if points:
@@ -83,8 +82,12 @@ def extract_camera_model(fp: T.BinaryIO) -> str:
 
 
 def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
-    # examples: b' {"model":"DR900X Plus","ver":0.918,"lang":"English","direct":1,"psn":"","temp":34,"GPS":1}\x00'
-    #           b' Pittasoft Co., Ltd.;DR900S-1CH;1.008;English;1;D90SS1HAE00661;T69;\x00'
+    """
+    >>> _extract_camera_model_from_cprt(b' {"model":"DR900X Plus","ver":0.918,"lang":"English","direct":1,"psn":"","temp":34,"GPS":1}')
+    'DR900X Plus'
+    >>> _extract_camera_model_from_cprt(b' Pittasoft Co., Ltd.;DR900S-1CH;1.008;English;1;D90SS1HAE00661;T69;')
+    'DR900S-1CH'
+    """
     cprt_bytes = cprt_bytes.strip().strip(b"\x00")
 
     try:
@@ -111,28 +114,82 @@ def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
         return ""
 
 
-def _parse_gps_box(gps_data: bytes) -> T.Generator[geo.Point, None, None]:
+def _parse_gps_box(gps_data: bytes) -> list[geo.Point]:
+    """
+    >>> list(_parse_gps_box(b"[1623057074211]$GPGGA,202530.00,5109.0262,N,11401.8407,W,5,40,0.5,1097.36,M,-17.00,M,18,TSTR*61"))
+    [Point(time=1623057074211, lat=51.150436666666664, lon=-114.03067833333333, alt=1097.36, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGGA,175322.00,3244.53126,N,11710.97811,W,1,12,0.84,17.4,M,-34.0,M,,*45"))
+    [Point(time=1629874404069, lat=32.742187666666666, lon=-117.1829685, alt=17.4, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGLL,4404.14012,N,12118.85993,W,001037.00,A,A*67"))
+    [Point(time=1629874404069, lat=44.069002, lon=-121.31433216666667, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNRMC,001031.00,A,4404.13993,N,12118.86023,W,0.146,,100117,,,A*7B"))
+    [Point(time=1629874404069, lat=44.06899883333333, lon=-121.31433716666666, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]"))
+    []
+    """
+    points_by_sentence_type: dict[str, list[geo.Point]] = {}
+
     for line_bytes in gps_data.splitlines():
         match = NMEA_LINE_REGEX.match(line_bytes)
         if match is None:
             continue
         nmea_line_bytes = match.group(2)
-        if nmea_line_bytes.startswith(b"$GPGGA"):
-            try:
-                nmea_line = nmea_line_bytes.decode("utf8")
-            except UnicodeDecodeError:
-                continue
-            try:
-                nmea = pynmea2.parse(nmea_line)
-            except pynmea2.nmea.ParseError:
+
+        if not nmea_line_bytes:
+            continue
+
+        try:
+            nmea_line = nmea_line_bytes.decode("utf8")
+        except UnicodeDecodeError:
+            continue
+
+        if not nmea_line:
+            continue
+
+        try:
+            message = pynmea2.parse(nmea_line)
+        except pynmea2.nmea.ParseError:
+            continue
+
+        epoch_ms = int(match.group(1))
+
+        # https://tavotech.com/gps-nmea-sentence-structure/
+        if message.sentence_type in ["GGA"]:
+            if not message.is_valid:
                 continue
-            if not nmea.is_valid:
+            point = geo.Point(
+                time=epoch_ms,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=message.altitude,
+                angle=None,
+            )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+        elif message.sentence_type in ["RMC", "GLL"]:
+            if not message.is_valid:
                 continue
-            epoch_ms = int(match.group(1))
-            yield geo.Point(
+            point = geo.Point(
                 time=epoch_ms,
-                lat=nmea.latitude,
-                lon=nmea.longitude,
-                alt=nmea.altitude,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=None,
                 angle=None,
             )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+    # This is the extraction order in exiftool
+    if "RMC" in points_by_sentence_type:
+        return points_by_sentence_type["RMC"]
+
+    if "GGA" in points_by_sentence_type:
+        return points_by_sentence_type["GGA"]
+
+    if "GLL" in points_by_sentence_type:
+        return points_by_sentence_type["GLL"]
+
+    return []

mapillary_tools/commands/__main__.py

@@ -7,6 +7,8 @@ from pathlib import Path
 import requests
 
 from .. import api_v4, constants, exceptions, VERSION
+from ..upload import log_exception
+from ..utils import configure_logger, get_app_name
 from . import (
     authenticate,
     process,
@@ -30,8 +32,8 @@ mapillary_tools_commands = [
 ]
 
 
-# do not use __name__ here is because if you run tools as a module, __name__ will be "__main__"
-LOG = logging.getLogger("mapillary_tools")
+# Root logger of mapillary_tools (not including third-party libraries)
+LOG = logging.getLogger(get_app_name())
 
 
 # Handle shared arguments/options here
@@ -78,13 +80,6 @@ def add_general_arguments(parser, command):
         )
 
 
-def configure_logger(logger: logging.Logger, stream=None) -> None:
-    formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s")
-    handler = logging.StreamHandler(stream)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-
-
 def _log_params(argvars: dict) -> None:
     MAX_ENTRIES = 5
 
@@ -151,9 +146,7 @@ def main():
 
     args = parser.parse_args()
 
-    log_level = logging.DEBUG if args.verbose else logging.INFO
-    configure_logger(LOG, sys.stderr)
-    LOG.setLevel(log_level)
+    configure_logger(LOG, level=logging.DEBUG if args.verbose else logging.INFO)
 
     LOG.debug("%s", version_text)
     argvars = vars(args)
@@ -162,16 +155,22 @@ def main():
     try:
         args.func(argvars)
     except requests.HTTPError as ex:
-        LOG.error("%s: %s", ex.__class__.__name__, api_v4.readable_http_error(ex))
+        log_exception(ex)
         # TODO: standardize exit codes as exceptions.MapillaryUserError
         sys.exit(16)
 
+    except api_v4.HTTPContentError as ex:
+        log_exception(ex)
+        sys.exit(17)
+
     except exceptions.MapillaryUserError as ex:
-        LOG.error(
-            "%s: %s", ex.__class__.__name__, ex, exc_info=log_level == logging.DEBUG
-        )
+        log_exception(ex)
         sys.exit(ex.exit_code)
 
+    except KeyboardInterrupt:
+        LOG.info("Interrupted by user...")
+        sys.exit(130)
+
 
 if __name__ == "__main__":
     main()

mapillary_tools/commands/upload.py

@@ -8,13 +8,13 @@ from .process import bold_text
 
 class Command:
     name = "upload"
-    help = "upload images to Mapillary"
+    help = "Upload processed data to Mapillary"
 
     @staticmethod
     def add_common_upload_options(group):
         group.add_argument(
             "--user_name",
-            help="The Mapillary user account to upload to. If you only have one account authorized, it will upload to that account by default.",
+            help="The Mapillary user account to upload to.",
             required=False,
         )
         group.add_argument(
@@ -23,9 +23,38 @@ class Command:
             default=None,
             required=False,
         )
+        group.add_argument(
+            "--num_upload_workers",
+            help="Number of concurrent upload workers for uploading images. [default: %(default)s]",
+            default=constants.MAX_IMAGE_UPLOAD_WORKERS,
+            type=int,
+            required=False,
+        )
+        group.add_argument(
+            "--reupload",
+            help="Re-upload data that has already been uploaded.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
         group.add_argument(
             "--dry_run",
-            help='Instead of uploading to the Mapillary server, simulate uploading to the local directory "mapillary_public_uploads" for debugging purposes.',
+            "--dryrun",
+            help="[DEVELOPMENT] Simulate upload by sending data to a local directory instead of Mapillary servers. Uses a temporary directory by default unless specified by MAPILLARY_UPLOAD_ENDPOINT environment variable.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--nofinish",
+            help="[DEVELOPMENT] Upload data without finalizing. The data will NOT be stored permanently or appear on the Mapillary website.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--noresume",
+            help="[DEVELOPMENT] Start upload from the beginning, ignoring any previously interrupted upload sessions.",
             action="store_true",
             default=False,
             required=False,
@@ -35,7 +64,7 @@ class Command:
         group = parser.add_argument_group(bold_text("UPLOAD OPTIONS"))
         group.add_argument(
             "--desc_path",
-            help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
+            help=f'Path to the description file with processed image and video metadata (from process command). Use "-" for STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
             default=None,
             required=False,
         )

mapillary_tools/config.py

@@ -2,31 +2,54 @@ from __future__ import annotations
 
 import configparser
 import os
+import sys
 import typing as T
+from typing import TypedDict
 
-from . import api_v4, types
+if sys.version_info >= (3, 11):
+    from typing import Required
+else:
+    from typing_extensions import Required
 
+from . import api_v4
 
-_CLIENT_ID = api_v4.MAPILLARY_CLIENT_TOKEN
-# Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
-_CLIENT_ID = _CLIENT_ID.replace("|", "_", 2)
-
-DEFAULT_MAPILLARY_FOLDER = os.path.join(
-    os.path.expanduser("~"),
-    ".config",
-    "mapillary",
-)
 
+DEFAULT_MAPILLARY_FOLDER = os.path.join(os.path.expanduser("~"), ".config", "mapillary")
 MAPILLARY_CONFIG_PATH = os.getenv(
     "MAPILLARY_CONFIG_PATH",
     os.path.join(
         DEFAULT_MAPILLARY_FOLDER,
         "configs",
-        _CLIENT_ID,
+        # Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
+        api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"),
     ),
 )
 
 
+class UserItem(TypedDict, total=False):
+    MAPOrganizationKey: int | str
+    # Username
+    MAPSettingsUsername: str
+    # User ID
+    MAPSettingsUserKey: str
+    # User access token
+    user_upload_token: Required[str]
+
+
+UserItemSchema = {
+    "type": "object",
+    "properties": {
+        "MAPOrganizationKey": {"type": ["integer", "string"]},
+        # Not in use. Keep here for back-compatibility
+        "MAPSettingsUsername": {"type": "string"},
+        "MAPSettingsUserKey": {"type": "string"},
+        "user_upload_token": {"type": "string"},
+    },
+    "required": ["user_upload_token"],
+    "additionalProperties": True,
+}
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)
@@ -36,19 +59,17 @@ def _load_config(config_path: str) -> configparser.ConfigParser:
     return config
 
 
-def load_user(
-    profile_name: str, config_path: str | None = None
-) -> types.UserItem | None:
+def load_user(profile_name: str, config_path: str | None = None) -> UserItem | None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
     if not config.has_section(profile_name):
         return None
     user_items = dict(config.items(profile_name))
-    return T.cast(types.UserItem, user_items)
+    return T.cast(UserItem, user_items)
 
 
-def list_all_users(config_path: str | None = None) -> dict[str, types.UserItem]:
+def list_all_users(config_path: str | None = None) -> dict[str, UserItem]:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     cp = _load_config(config_path)
@@ -60,7 +81,7 @@ def list_all_users(config_path: str | None = None) -> dict[str, types.UserItem]:
 
 
 def update_config(
-    profile_name: str, user_items: types.UserItem, config_path: str | None = None
+    profile_name: str, user_items: UserItem, config_path: str | None = None
 ) -> None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH

mapillary_tools/constants.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import functools
 import os
+import tempfile
 
 import appdirs
 
@@ -8,44 +10,92 @@ _ENV_PREFIX = "MAPILLARY_TOOLS_"
 
 
 def _yes_or_no(val: str) -> bool:
-    return val.strip().upper() in [
-        "1",
-        "TRUE",
-        "YES",
-    ]
+    return val.strip().upper() in ["1", "TRUE", "YES"]
 
 
-# In meters
-CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
+def _parse_scaled_integers(
+    value: str, scale: dict[str, int] | None = None
+) -> int | None:
+    """
+    >>> scale = {"": 1, "b": 1, "K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024}
+    >>> _parse_scaled_integers("0", scale=scale)
+    0
+    >>> _parse_scaled_integers("10", scale=scale)
+    10
+    >>> _parse_scaled_integers("100B", scale=scale)
+    100
+    >>> _parse_scaled_integers("100k", scale=scale)
+    102400
+    >>> _parse_scaled_integers("100t", scale=scale)
+    Traceback (most recent call last):
+    ValueError: Expect valid integer ends with , b, K, M, G, but got 100T
+    """
+
+    if scale is None:
+        scale = {"": 1}
+
+    value = value.strip().upper()
+
+    if value in ["INF", "INFINITY"]:
+        return None
+
+    try:
+        for k, v in scale.items():
+            k = k.upper()
+            if k and value.endswith(k):
+                return int(value[: -len(k)]) * v
+
+        if "" in scale:
+            return int(value) * scale[""]
+    except ValueError:
+        pass
+
+    raise ValueError(
+        f"Expect valid integer ends with {', '.join(scale.keys())}, but got {value}"
+    )
+
+
+_parse_pixels = functools.partial(
+    _parse_scaled_integers,
+    scale={
+        "": 1,
+        "K": 1000,
+        "M": 1000 * 1000,
+        "MP": 1000 * 1000,
+        "G": 1000 * 1000 * 1000,
+        "GP": 1000 * 1000 * 1000,
+    },
+)
+
+_parse_filesize = functools.partial(
+    _parse_scaled_integers,
+    scale={"B": 1, "K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024},
+)
+
+###################
+##### GENERAL #####
+###################
+USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary")
+PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
+
+
+############################
+##### VIDEO PROCESSING #####
+############################
 # In seconds
-CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
-DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
-DUPLICATE_ANGLE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_ANGLE", 5))
-MAX_AVG_SPEED = float(
-    os.getenv(_ENV_PREFIX + "MAX_AVG_SPEED", 400_000 / 3600)
-)  # 400 KM/h
-# in seconds
 VIDEO_SAMPLE_INTERVAL = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_INTERVAL", -1))
-# in meters
+# In meters
 VIDEO_SAMPLE_DISTANCE = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_DISTANCE", 3))
 VIDEO_DURATION_RATIO = float(os.getenv(_ENV_PREFIX + "VIDEO_DURATION_RATIO", 1))
 FFPROBE_PATH: str = os.getenv(_ENV_PREFIX + "FFPROBE_PATH", "ffprobe")
 FFMPEG_PATH: str = os.getenv(_ENV_PREFIX + "FFMPEG_PATH", "ffmpeg")
-# When not set, MT will try to check both "exiftool" and "exiftool.exe" from $PATH
-EXIFTOOL_PATH: str | None = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH")
+EXIFTOOL_PATH: str = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH", "exiftool")
 IMAGE_DESCRIPTION_FILENAME = os.getenv(
     _ENV_PREFIX + "IMAGE_DESCRIPTION_FILENAME", "mapillary_image_description.json"
 )
 SAMPLED_VIDEO_FRAMES_FILENAME = os.getenv(
     _ENV_PREFIX + "SAMPLED_VIDEO_FRAMES_FILENAME", "mapillary_sampled_video_frames"
 )
-USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary")
-# The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
-# for uploading data to MLY upload service.
-# Changing this size does not change the number of requests nor affect upload performance,
-# but it affects the responsiveness of the upload progress bar
-UPLOAD_CHUNK_SIZE_MB = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 1))
-
 # DoP value, the lower the better
 # See https://github.com/gopro/gpmf-parser#hero5-black-with-gps-enabled-adds
 # It is used to filter out noisy points
@@ -54,38 +104,72 @@ GOPRO_MAX_DOP100 = int(os.getenv(_ENV_PREFIX + "GOPRO_MAX_DOP100", 1000))
 GOPRO_GPS_FIXES: set[int] = set(
     int(fix) for fix in os.getenv(_ENV_PREFIX + "GOPRO_GPS_FIXES", "2,3").split(",")
 )
-MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
-
 # GPS precision, in meters, is used to filter outliers
 GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
+MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
+    os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU", "NO")
+)
+
 
+#################################
+###### SEQUENCE PROCESSING ######
+#################################
+# In meters
+CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
+# In seconds
+CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
+DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
+DUPLICATE_ANGLE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_ANGLE", 5))
+MAX_CAPTURE_SPEED_KMH = float(
+    os.getenv(_ENV_PREFIX + "MAX_CAPTURE_SPEED_KMH", 400)
+)  # 400 KM/h
 # WARNING: Changing the following envvars might result in failed uploads
 # Max number of images per sequence
-MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
+MAX_SEQUENCE_LENGTH: int | None = _parse_scaled_integers(
+    os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", "1000")
+)
 # Max file size per sequence (sum of image filesizes in the sequence)
-MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
+MAX_SEQUENCE_FILESIZE: int | None = _parse_filesize(
+    os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
+)
 # Max number of pixels per sequence (sum of image pixels in the sequence)
-MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
-
-PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
-
-_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
-    os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
+MAX_SEQUENCE_PIXELS: int | None = _parse_pixels(
+    os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
 )
 
+
+##################
+##### UPLOAD #####
+##################
 MAPILLARY_DISABLE_API_LOGGING: bool = _yes_or_no(
     os.getenv("MAPILLARY_DISABLE_API_LOGGING", "NO")
 )
+MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
+    "MAPILLARY_UPLOAD_HISTORY_PATH", os.path.join(USER_DATA_DIR, "upload_history")
+)
+UPLOAD_CACHE_DIR: str = os.getenv(
+    _ENV_PREFIX + "UPLOAD_CACHE_DIR",
+    os.path.join(tempfile.gettempdir(), "mapillary_tools", "upload_cache"),
+)
+# The minimal upload speed is used to calculate the read timeout to avoid upload hanging:
+# timeout = upload_size / MIN_UPLOAD_SPEED
+MIN_UPLOAD_SPEED: int | None = _parse_filesize(
+    os.getenv(_ENV_PREFIX + "MIN_UPLOAD_SPEED", "50K")  # 50 Kb/s
+)
+# Maximum number of parallel workers for uploading images within a single sequence.
+# NOTE: Sequences themselves are uploaded sequentially, not in parallel.
+MAX_IMAGE_UPLOAD_WORKERS: int = int(
+    os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 4)
+)
+# The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
+# for uploading data to MLY upload service.
+# Changing this size does not change the number of requests nor affect upload performance,
+# but it affects the responsiveness of the upload progress bar
+UPLOAD_CHUNK_SIZE_MB: float = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 2))
+MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
 MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN: bool = _yes_or_no(
     os.getenv("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", "NO")
 )
-MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
-    os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU", "NO")
-)
-MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
-    "MAPILLARY_UPLOAD_HISTORY_PATH",
-    os.path.join(
-        USER_DATA_DIR,
-        "upload_history",
-    ),
+_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
+    os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
 )

mapillary_tools/exceptions.py

@@ -51,6 +51,10 @@ class MapillaryVideoGPSNotFoundError(MapillaryDescriptionError):
     pass
 
 
+class MapillaryInvalidVideoError(MapillaryDescriptionError):
+    pass
+
+
 class MapillaryGPXEmptyError(MapillaryDescriptionError):
     pass
 

mapillary_tools/exif_read.py

@@ -871,7 +871,8 @@ class ExifRead(ExifReadFromEXIF):
 
     def _xmp_with_reason(self, reason: str) -> ExifReadFromXMP | None:
         if not self._xml_extracted:
-            LOG.debug('Extracting XMP for "%s"', reason)
+            # TODO Disabled because too verbose but still useful to know
+            # LOG.debug('Extracting XMP for "%s"', reason)
             self._cached_xml = self._extract_xmp()
             self._xml_extracted = True
 

mapillary_tools/exif_write.py

@@ -42,7 +42,9 @@ class ExifEdit:
 
     def add_image_description(self, data: dict) -> None:
         """Add a dict to image description."""
-        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(data)
+        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(
+            data, sort_keys=True, separators=(",", ":")
+        )
 
     def add_orientation(self, orientation: int) -> None:
         """Add image orientation to image."""