mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0b1__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.14.0a1"
+VERSION = "0.14.0b1"

mapillary_tools/api_v4.py

@@ -25,6 +25,7 @@ class ClusterFileType(enum.Enum):
     ZIP = "zip"
     BLACKVUE = "mly_blackvue_video"
     CAMM = "mly_camm_video"
+    MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
 
 
 class HTTPSystemCertsAdapter(HTTPAdapter):
@@ -135,7 +136,7 @@ def _log_debug_response(resp: requests.Response):
     if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
         return
 
-    data: T.Union[str, bytes]
+    data: str | bytes
     try:
         data = _truncate(dumps(_sanitize(resp.json())))
     except Exception:
@@ -148,7 +149,7 @@ def readable_http_error(ex: requests.HTTPError) -> str:
     req = ex.request
     resp = ex.response
 
-    data: T.Union[str, bytes]
+    data: str | bytes
     try:
         data = _truncate(dumps(_sanitize(resp.json())))
     except Exception:
@@ -284,7 +285,7 @@ def get_upload_token(email: str, password: str) -> requests.Response:
 
 
 def fetch_organization(
-    user_access_token: str, organization_id: T.Union[int, str]
+    user_access_token: str, organization_id: int | str
 ) -> requests.Response:
     resp = request_get(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}",
@@ -329,7 +330,7 @@ ActionType = T.Literal[
 ]
 
 
-def log_event(action_type: ActionType, properties: T.Dict) -> requests.Response:
+def log_event(action_type: ActionType, properties: dict) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging",
         json={

mapillary_tools/authenticate.py

@@ -11,7 +11,7 @@ import jsonschema
 
 import requests
 
-from . import api_v4, config, constants, exceptions, types
+from . import api_v4, config, constants, exceptions
 
 
 LOG = logging.getLogger(__name__)
@@ -64,7 +64,7 @@ def authenticate(
             LOG.info('Creating new profile: "%s"', profile_name)
 
         if jwt:
-            user_items: types.UserItem = {"user_upload_token": jwt}
+            user_items: config.UserItem = {"user_upload_token": jwt}
             user_items = _verify_user_auth(_validate_profile(user_items))
         else:
             user_items = _prompt_login(
@@ -89,7 +89,7 @@ def authenticate(
 def fetch_user_items(
     user_name: str | None = None,
     organization_key: str | None = None,
-) -> types.UserItem:
+) -> config.UserItem:
     """
     Read user information from the config file,
     or prompt the user to authenticate if the specified profile does not exist
@@ -155,9 +155,9 @@ def _prompt(message: str) -> str:
     return input()
 
 
-def _validate_profile(user_items: types.UserItem) -> types.UserItem:
+def _validate_profile(user_items: config.UserItem) -> config.UserItem:
     try:
-        jsonschema.validate(user_items, types.UserItemSchema)
+        jsonschema.validate(user_items, config.UserItemSchema)
     except jsonschema.ValidationError as ex:
         raise exceptions.MapillaryBadParameterError(
             f"Invalid profile format: {ex.message}"
@@ -165,7 +165,7 @@ def _validate_profile(user_items: types.UserItem) -> types.UserItem:
     return user_items
 
 
-def _verify_user_auth(user_items: types.UserItem) -> types.UserItem:
+def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
     """
     Verify that the user access token is valid
     """
@@ -205,7 +205,7 @@ def _validate_profile_name(profile_name: str):
         )
 
 
-def _list_all_profiles(profiles: dict[str, types.UserItem]) -> None:
+def _list_all_profiles(profiles: dict[str, config.UserItem]) -> None:
     _echo("Existing Mapillary profiles:")
 
     # Header
@@ -256,7 +256,7 @@ def _is_login_retryable(ex: requests.HTTPError) -> bool:
 def _prompt_login(
     user_email: str | None = None,
     user_password: str | None = None,
-) -> types.UserItem:
+) -> config.UserItem:
     _enabled = _prompt_enabled()
 
     if user_email is None:
@@ -288,7 +288,7 @@ def _prompt_login(
 
     data = resp.json()
 
-    user_items: types.UserItem = {
+    user_items: config.UserItem = {
         "user_upload_token": str(data["access_token"]),
         "MAPSettingsUserKey": str(data["user_id"]),
     }

mapillary_tools/blackvue_parser.py

@@ -14,15 +14,14 @@ from .mp4 import simple_mp4_parser as sparser
 
 
 LOG = logging.getLogger(__name__)
-# An example: [1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]
 NMEA_LINE_REGEX = re.compile(
     rb"""
     ^\s*
-    \[(\d+)\] # timestamp
+    \[(\d+)\] # Timestamp
     \s*
-    (\$\w{5}.*) # nmea line
+    (\$\w{5}.*) # NMEA message
     \s*
-    (\[\d+\])? # strange timestamp
+    (\[\d+\])? # Strange timestamp
     \s*$
     """,
     re.X,
@@ -47,7 +46,7 @@ def extract_blackvue_info(fp: T.BinaryIO) -> BlackVueInfo | None:
     if gps_data is None:
         return None
 
-    points = list(_parse_gps_box(gps_data))
+    points = _parse_gps_box(gps_data)
     points.sort(key=lambda p: p.time)
 
     if points:
@@ -83,8 +82,12 @@ def extract_camera_model(fp: T.BinaryIO) -> str:
 
 
 def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
-    # examples: b' {"model":"DR900X Plus","ver":0.918,"lang":"English","direct":1,"psn":"","temp":34,"GPS":1}\x00'
-    #           b' Pittasoft Co., Ltd.;DR900S-1CH;1.008;English;1;D90SS1HAE00661;T69;\x00'
+    """
+    >>> _extract_camera_model_from_cprt(b' {"model":"DR900X Plus","ver":0.918,"lang":"English","direct":1,"psn":"","temp":34,"GPS":1}')
+    'DR900X Plus'
+    >>> _extract_camera_model_from_cprt(b' Pittasoft Co., Ltd.;DR900S-1CH;1.008;English;1;D90SS1HAE00661;T69;')
+    'DR900S-1CH'
+    """
     cprt_bytes = cprt_bytes.strip().strip(b"\x00")
 
     try:
@@ -111,28 +114,82 @@ def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
         return ""
 
 
-def _parse_gps_box(gps_data: bytes) -> T.Generator[geo.Point, None, None]:
+def _parse_gps_box(gps_data: bytes) -> list[geo.Point]:
+    """
+    >>> list(_parse_gps_box(b"[1623057074211]$GPGGA,202530.00,5109.0262,N,11401.8407,W,5,40,0.5,1097.36,M,-17.00,M,18,TSTR*61"))
+    [Point(time=1623057074211, lat=51.150436666666664, lon=-114.03067833333333, alt=1097.36, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGGA,175322.00,3244.53126,N,11710.97811,W,1,12,0.84,17.4,M,-34.0,M,,*45"))
+    [Point(time=1629874404069, lat=32.742187666666666, lon=-117.1829685, alt=17.4, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGLL,4404.14012,N,12118.85993,W,001037.00,A,A*67"))
+    [Point(time=1629874404069, lat=44.069002, lon=-121.31433216666667, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNRMC,001031.00,A,4404.13993,N,12118.86023,W,0.146,,100117,,,A*7B"))
+    [Point(time=1629874404069, lat=44.06899883333333, lon=-121.31433716666666, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]"))
+    []
+    """
+    points_by_sentence_type: dict[str, list[geo.Point]] = {}
+
     for line_bytes in gps_data.splitlines():
         match = NMEA_LINE_REGEX.match(line_bytes)
         if match is None:
             continue
         nmea_line_bytes = match.group(2)
-        if nmea_line_bytes.startswith(b"$GPGGA"):
-            try:
-                nmea_line = nmea_line_bytes.decode("utf8")
-            except UnicodeDecodeError:
-                continue
-            try:
-                nmea = pynmea2.parse(nmea_line)
-            except pynmea2.nmea.ParseError:
+
+        if not nmea_line_bytes:
+            continue
+
+        try:
+            nmea_line = nmea_line_bytes.decode("utf8")
+        except UnicodeDecodeError:
+            continue
+
+        if not nmea_line:
+            continue
+
+        try:
+            message = pynmea2.parse(nmea_line)
+        except pynmea2.nmea.ParseError:
+            continue
+
+        epoch_ms = int(match.group(1))
+
+        # https://tavotech.com/gps-nmea-sentence-structure/
+        if message.sentence_type in ["GGA"]:
+            if not message.is_valid:
                 continue
-            if not nmea.is_valid:
+            point = geo.Point(
+                time=epoch_ms,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=message.altitude,
+                angle=None,
+            )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+        elif message.sentence_type in ["RMC", "GLL"]:
+            if not message.is_valid:
                 continue
-            epoch_ms = int(match.group(1))
-            yield geo.Point(
+            point = geo.Point(
                 time=epoch_ms,
-                lat=nmea.latitude,
-                lon=nmea.longitude,
-                alt=nmea.altitude,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=None,
                 angle=None,
             )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+    # This is the extraction order in exiftool
+    if "RMC" in points_by_sentence_type:
+        return points_by_sentence_type["RMC"]
+
+    if "GGA" in points_by_sentence_type:
+        return points_by_sentence_type["GGA"]
+
+    if "GLL" in points_by_sentence_type:
+        return points_by_sentence_type["GLL"]
+
+    return []

mapillary_tools/camm/camm_parser.py

@@ -373,7 +373,7 @@ SAMPLE_ENTRY_CLS_BY_CAMM_TYPE = {
 assert len(SAMPLE_ENTRY_CLS_BY_CAMM_TYPE) == 5, SAMPLE_ENTRY_CLS_BY_CAMM_TYPE.keys()
 
 
-_SWITCH: T.Dict[int, C.Struct] = {
+_SWITCH: dict[int, C.Struct] = {
     # Angle_axis
     CAMMType.ANGLE_AXIS.value: _Float[3],  # type: ignore
     # Exposure time
@@ -436,7 +436,7 @@ def _parse_telemetry_from_sample(
 
 def _filter_telemetry_by_elst_segments(
     measurements: T.Iterable[TelemetryMeasurement],
-    elst: T.Sequence[T.Tuple[float, float]],
+    elst: T.Sequence[tuple[float, float]],
 ) -> T.Generator[TelemetryMeasurement, None, None]:
     empty_elst = [entry for entry in elst if entry[0] == -1]
     if empty_elst:
@@ -466,8 +466,8 @@ def _filter_telemetry_by_elst_segments(
 
 
 def elst_entry_to_seconds(
-    entry: T.Dict, movie_timescale: int, media_timescale: int
-) -> T.Tuple[float, float]:
+    entry: dict, movie_timescale: int, media_timescale: int
+) -> tuple[float, float]:
     assert movie_timescale > 0, "expected positive movie_timescale"
     assert media_timescale > 0, "expected positive media_timescale"
     media_time, duration = entry["media_time"], entry["segment_duration"]
@@ -477,7 +477,7 @@ def elst_entry_to_seconds(
     return (media_time, duration)
 
 
-def _is_camm_description(description: T.Dict) -> bool:
+def _is_camm_description(description: dict) -> bool:
     return description["format"] == b"camm"
 
 

mapillary_tools/commands/__main__.py

@@ -2,7 +2,6 @@ import argparse
 import enum
 import logging
 import sys
-import typing as T
 from pathlib import Path
 
 import requests
@@ -86,7 +85,7 @@ def configure_logger(logger: logging.Logger, stream=None) -> None:
     logger.addHandler(handler)
 
 
-def _log_params(argvars: T.Dict) -> None:
+def _log_params(argvars: dict) -> None:
     MAX_ENTRIES = 5
 
     def _stringify(x) -> str:

mapillary_tools/config.py

@@ -1,30 +1,55 @@
+from __future__ import annotations
+
 import configparser
 import os
+import sys
 import typing as T
+from typing import TypedDict
 
-from . import api_v4, types
-
+if sys.version_info >= (3, 11):
+    from typing import Required
+else:
+    from typing_extensions import Required
 
-_CLIENT_ID = api_v4.MAPILLARY_CLIENT_TOKEN
-# Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
-_CLIENT_ID = _CLIENT_ID.replace("|", "_", 2)
+from . import api_v4
 
-DEFAULT_MAPILLARY_FOLDER = os.path.join(
-    os.path.expanduser("~"),
-    ".config",
-    "mapillary",
-)
 
+DEFAULT_MAPILLARY_FOLDER = os.path.join(os.path.expanduser("~"), ".config", "mapillary")
 MAPILLARY_CONFIG_PATH = os.getenv(
     "MAPILLARY_CONFIG_PATH",
     os.path.join(
         DEFAULT_MAPILLARY_FOLDER,
         "configs",
-        _CLIENT_ID,
+        # Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
+        api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"),
     ),
 )
 
 
+class UserItem(TypedDict, total=False):
+    MAPOrganizationKey: int | str
+    # Username
+    MAPSettingsUsername: str
+    # User ID
+    MAPSettingsUserKey: str
+    # User access token
+    user_upload_token: Required[str]
+
+
+UserItemSchema = {
+    "type": "object",
+    "properties": {
+        "MAPOrganizationKey": {"type": ["integer", "string"]},
+        # Not in use. Keep here for back-compatibility
+        "MAPSettingsUsername": {"type": "string"},
+        "MAPSettingsUserKey": {"type": "string"},
+        "user_upload_token": {"type": "string"},
+    },
+    "required": ["user_upload_token"],
+    "additionalProperties": True,
+}
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)
@@ -34,19 +59,17 @@ def _load_config(config_path: str) -> configparser.ConfigParser:
     return config
 
 
-def load_user(
-    profile_name: str, config_path: T.Optional[str] = None
-) -> T.Optional[types.UserItem]:
+def load_user(profile_name: str, config_path: str | None = None) -> UserItem | None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
     if not config.has_section(profile_name):
         return None
     user_items = dict(config.items(profile_name))
-    return T.cast(types.UserItem, user_items)
+    return T.cast(UserItem, user_items)
 
 
-def list_all_users(config_path: T.Optional[str] = None) -> T.Dict[str, types.UserItem]:
+def list_all_users(config_path: str | None = None) -> dict[str, UserItem]:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     cp = _load_config(config_path)
@@ -58,7 +81,7 @@ def list_all_users(config_path: T.Optional[str] = None) -> T.Dict[str, types.Use
 
 
 def update_config(
-    profile_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None
+    profile_name: str, user_items: UserItem, config_path: str | None = None
 ) -> None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
@@ -72,7 +95,7 @@ def update_config(
         config.write(fp)
 
 
-def remove_config(profile_name: str, config_path: T.Optional[str] = None) -> None:
+def remove_config(profile_name: str, config_path: str | None = None) -> None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
 

mapillary_tools/constants.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-import typing as T
 
 import appdirs
 
@@ -52,7 +51,7 @@ UPLOAD_CHUNK_SIZE_MB = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 1))
 # It is used to filter out noisy points
 GOPRO_MAX_DOP100 = int(os.getenv(_ENV_PREFIX + "GOPRO_MAX_DOP100", 1000))
 # Within the GPS stream: 0 - no lock, 2 or 3 - 2D or 3D Lock
-GOPRO_GPS_FIXES: T.Set[int] = set(
+GOPRO_GPS_FIXES: set[int] = set(
     int(fix) for fix in os.getenv(_ENV_PREFIX + "GOPRO_GPS_FIXES", "2,3").split(",")
 )
 MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
@@ -90,3 +89,5 @@ MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
         "upload_history",
     ),
 )
+
+MAX_IMAGE_UPLOAD_WORKERS = int(os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 64))

mapillary_tools/exceptions.py

@@ -87,7 +87,7 @@ class MapillaryDuplicationError(MapillaryDescriptionError):
         self.angle_diff = angle_diff
 
 
-class MapillaryEXIFNotFoundError(MapillaryDescriptionError):
+class MapillaryExifToolXMLNotFoundError(MapillaryDescriptionError):
     pass