mapillary-tools 0.14.0a2__py3-none-any.whl → 0.14.0b1__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.14.0a2"
+VERSION = "0.14.0b1"

mapillary_tools/api_v4.py

@@ -25,6 +25,7 @@ class ClusterFileType(enum.Enum):
     ZIP = "zip"
     BLACKVUE = "mly_blackvue_video"
     CAMM = "mly_camm_video"
+    MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
 
 
 class HTTPSystemCertsAdapter(HTTPAdapter):

mapillary_tools/authenticate.py

@@ -11,7 +11,7 @@ import jsonschema
 
 import requests
 
-from . import api_v4, config, constants, exceptions, types
+from . import api_v4, config, constants, exceptions
 
 
 LOG = logging.getLogger(__name__)
@@ -64,7 +64,7 @@ def authenticate(
             LOG.info('Creating new profile: "%s"', profile_name)
 
         if jwt:
-            user_items: types.UserItem = {"user_upload_token": jwt}
+            user_items: config.UserItem = {"user_upload_token": jwt}
             user_items = _verify_user_auth(_validate_profile(user_items))
         else:
             user_items = _prompt_login(
@@ -89,7 +89,7 @@ def authenticate(
 def fetch_user_items(
     user_name: str | None = None,
     organization_key: str | None = None,
-) -> types.UserItem:
+) -> config.UserItem:
     """
     Read user information from the config file,
     or prompt the user to authenticate if the specified profile does not exist
@@ -155,9 +155,9 @@ def _prompt(message: str) -> str:
     return input()
 
 
-def _validate_profile(user_items: types.UserItem) -> types.UserItem:
+def _validate_profile(user_items: config.UserItem) -> config.UserItem:
     try:
-        jsonschema.validate(user_items, types.UserItemSchema)
+        jsonschema.validate(user_items, config.UserItemSchema)
     except jsonschema.ValidationError as ex:
         raise exceptions.MapillaryBadParameterError(
             f"Invalid profile format: {ex.message}"
@@ -165,7 +165,7 @@ def _validate_profile(user_items: types.UserItem) -> types.UserItem:
     return user_items
 
 
-def _verify_user_auth(user_items: types.UserItem) -> types.UserItem:
+def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
     """
     Verify that the user access token is valid
     """
@@ -205,7 +205,7 @@ def _validate_profile_name(profile_name: str):
         )
 
 
-def _list_all_profiles(profiles: dict[str, types.UserItem]) -> None:
+def _list_all_profiles(profiles: dict[str, config.UserItem]) -> None:
     _echo("Existing Mapillary profiles:")
 
     # Header
@@ -256,7 +256,7 @@ def _is_login_retryable(ex: requests.HTTPError) -> bool:
 def _prompt_login(
     user_email: str | None = None,
     user_password: str | None = None,
-) -> types.UserItem:
+) -> config.UserItem:
     _enabled = _prompt_enabled()
 
     if user_email is None:
@@ -288,7 +288,7 @@ def _prompt_login(
 
     data = resp.json()
 
-    user_items: types.UserItem = {
+    user_items: config.UserItem = {
         "user_upload_token": str(data["access_token"]),
         "MAPSettingsUserKey": str(data["user_id"]),
     }

mapillary_tools/blackvue_parser.py

@@ -14,15 +14,14 @@ from .mp4 import simple_mp4_parser as sparser
 
 
 LOG = logging.getLogger(__name__)
-# An example: [1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]
 NMEA_LINE_REGEX = re.compile(
     rb"""
     ^\s*
-    \[(\d+)\] # timestamp
+    \[(\d+)\] # Timestamp
     \s*
-    (\$\w{5}.*) # nmea line
+    (\$\w{5}.*) # NMEA message
     \s*
-    (\[\d+\])? # strange timestamp
+    (\[\d+\])? # Strange timestamp
     \s*$
     """,
     re.X,
@@ -47,7 +46,7 @@ def extract_blackvue_info(fp: T.BinaryIO) -> BlackVueInfo | None:
     if gps_data is None:
         return None
 
-    points = list(_parse_gps_box(gps_data))
+    points = _parse_gps_box(gps_data)
     points.sort(key=lambda p: p.time)
 
     if points:
@@ -83,8 +82,12 @@ def extract_camera_model(fp: T.BinaryIO) -> str:
 
 
 def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
-    # examples: b' {"model":"DR900X Plus","ver":0.918,"lang":"English","direct":1,"psn":"","temp":34,"GPS":1}\x00'
-    #           b' Pittasoft Co., Ltd.;DR900S-1CH;1.008;English;1;D90SS1HAE00661;T69;\x00'
+    """
+    >>> _extract_camera_model_from_cprt(b' {"model":"DR900X Plus","ver":0.918,"lang":"English","direct":1,"psn":"","temp":34,"GPS":1}')
+    'DR900X Plus'
+    >>> _extract_camera_model_from_cprt(b' Pittasoft Co., Ltd.;DR900S-1CH;1.008;English;1;D90SS1HAE00661;T69;')
+    'DR900S-1CH'
+    """
     cprt_bytes = cprt_bytes.strip().strip(b"\x00")
 
     try:
@@ -111,28 +114,82 @@ def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
         return ""
 
 
-def _parse_gps_box(gps_data: bytes) -> T.Generator[geo.Point, None, None]:
+def _parse_gps_box(gps_data: bytes) -> list[geo.Point]:
+    """
+    >>> list(_parse_gps_box(b"[1623057074211]$GPGGA,202530.00,5109.0262,N,11401.8407,W,5,40,0.5,1097.36,M,-17.00,M,18,TSTR*61"))
+    [Point(time=1623057074211, lat=51.150436666666664, lon=-114.03067833333333, alt=1097.36, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGGA,175322.00,3244.53126,N,11710.97811,W,1,12,0.84,17.4,M,-34.0,M,,*45"))
+    [Point(time=1629874404069, lat=32.742187666666666, lon=-117.1829685, alt=17.4, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGLL,4404.14012,N,12118.85993,W,001037.00,A,A*67"))
+    [Point(time=1629874404069, lat=44.069002, lon=-121.31433216666667, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNRMC,001031.00,A,4404.13993,N,12118.86023,W,0.146,,100117,,,A*7B"))
+    [Point(time=1629874404069, lat=44.06899883333333, lon=-121.31433716666666, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]"))
+    []
+    """
+    points_by_sentence_type: dict[str, list[geo.Point]] = {}
+
     for line_bytes in gps_data.splitlines():
         match = NMEA_LINE_REGEX.match(line_bytes)
         if match is None:
             continue
         nmea_line_bytes = match.group(2)
-        if nmea_line_bytes.startswith(b"$GPGGA"):
-            try:
-                nmea_line = nmea_line_bytes.decode("utf8")
-            except UnicodeDecodeError:
-                continue
-            try:
-                nmea = pynmea2.parse(nmea_line)
-            except pynmea2.nmea.ParseError:
+
+        if not nmea_line_bytes:
+            continue
+
+        try:
+            nmea_line = nmea_line_bytes.decode("utf8")
+        except UnicodeDecodeError:
+            continue
+
+        if not nmea_line:
+            continue
+
+        try:
+            message = pynmea2.parse(nmea_line)
+        except pynmea2.nmea.ParseError:
+            continue
+
+        epoch_ms = int(match.group(1))
+
+        # https://tavotech.com/gps-nmea-sentence-structure/
+        if message.sentence_type in ["GGA"]:
+            if not message.is_valid:
                 continue
-            if not nmea.is_valid:
+            point = geo.Point(
+                time=epoch_ms,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=message.altitude,
+                angle=None,
+            )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+        elif message.sentence_type in ["RMC", "GLL"]:
+            if not message.is_valid:
                 continue
-            epoch_ms = int(match.group(1))
-            yield geo.Point(
+            point = geo.Point(
                 time=epoch_ms,
-                lat=nmea.latitude,
-                lon=nmea.longitude,
-                alt=nmea.altitude,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=None,
                 angle=None,
             )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+    # This is the extraction order in exiftool
+    if "RMC" in points_by_sentence_type:
+        return points_by_sentence_type["RMC"]
+
+    if "GGA" in points_by_sentence_type:
+        return points_by_sentence_type["GGA"]
+
+    if "GLL" in points_by_sentence_type:
+        return points_by_sentence_type["GLL"]
+
+    return []

mapillary_tools/config.py

@@ -2,31 +2,54 @@ from __future__ import annotations
 
 import configparser
 import os
+import sys
 import typing as T
+from typing import TypedDict
 
-from . import api_v4, types
+if sys.version_info >= (3, 11):
+    from typing import Required
+else:
+    from typing_extensions import Required
 
+from . import api_v4
 
-_CLIENT_ID = api_v4.MAPILLARY_CLIENT_TOKEN
-# Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
-_CLIENT_ID = _CLIENT_ID.replace("|", "_", 2)
-
-DEFAULT_MAPILLARY_FOLDER = os.path.join(
-    os.path.expanduser("~"),
-    ".config",
-    "mapillary",
-)
 
+DEFAULT_MAPILLARY_FOLDER = os.path.join(os.path.expanduser("~"), ".config", "mapillary")
 MAPILLARY_CONFIG_PATH = os.getenv(
     "MAPILLARY_CONFIG_PATH",
     os.path.join(
         DEFAULT_MAPILLARY_FOLDER,
         "configs",
-        _CLIENT_ID,
+        # Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
+        api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"),
     ),
 )
 
 
+class UserItem(TypedDict, total=False):
+    MAPOrganizationKey: int | str
+    # Username
+    MAPSettingsUsername: str
+    # User ID
+    MAPSettingsUserKey: str
+    # User access token
+    user_upload_token: Required[str]
+
+
+UserItemSchema = {
+    "type": "object",
+    "properties": {
+        "MAPOrganizationKey": {"type": ["integer", "string"]},
+        # Not in use. Keep here for back-compatibility
+        "MAPSettingsUsername": {"type": "string"},
+        "MAPSettingsUserKey": {"type": "string"},
+        "user_upload_token": {"type": "string"},
+    },
+    "required": ["user_upload_token"],
+    "additionalProperties": True,
+}
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)
@@ -36,19 +59,17 @@ def _load_config(config_path: str) -> configparser.ConfigParser:
     return config
 
 
-def load_user(
-    profile_name: str, config_path: str | None = None
-) -> types.UserItem | None:
+def load_user(profile_name: str, config_path: str | None = None) -> UserItem | None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
     if not config.has_section(profile_name):
         return None
     user_items = dict(config.items(profile_name))
-    return T.cast(types.UserItem, user_items)
+    return T.cast(UserItem, user_items)
 
 
-def list_all_users(config_path: str | None = None) -> dict[str, types.UserItem]:
+def list_all_users(config_path: str | None = None) -> dict[str, UserItem]:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     cp = _load_config(config_path)
@@ -60,7 +81,7 @@ def list_all_users(config_path: str | None = None) -> dict[str, types.UserItem]:
 
 
 def update_config(
-    profile_name: str, user_items: types.UserItem, config_path: str | None = None
+    profile_name: str, user_items: UserItem, config_path: str | None = None
 ) -> None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH

mapillary_tools/constants.py

@@ -89,3 +89,5 @@ MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
         "upload_history",
     ),
 )
+
+MAX_IMAGE_UPLOAD_WORKERS = int(os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 64))

mapillary_tools/exiftool_read_video.py

@@ -82,6 +82,32 @@ def _extract_alternative_fields(
     return None
 
 
+def _same_gps_point(left: GPSPoint, right: GPSPoint) -> bool:
+    """
+    >>> left =  GPSPoint(time=56.0, lat=36.741385, lon=29.021274, alt=141.6, angle=1.54, epoch_time=None, fix=None, precision=None, ground_speed=None)
+    >>> right = GPSPoint(time=56.0, lat=36.741385, lon=29.021274, alt=142.4, angle=1.54, epoch_time=None, fix=None, precision=None, ground_speed=None)
+    >>> _same_gps_point(left, right)
+    True
+    """
+    return (
+        left.time == right.time
+        and left.lon == right.lon
+        and left.lat == right.lat
+        and left.epoch_time == right.epoch_time
+        and left.angle == right.angle
+    )
+
+
+def _deduplicate_gps_points(
+    track: list[GPSPoint], same_gps_point: T.Callable[[GPSPoint, GPSPoint], bool]
+) -> list[GPSPoint]:
+    deduplicated_track: list[GPSPoint] = []
+    for point in track:
+        if not deduplicated_track or not same_gps_point(deduplicated_track[-1], point):
+            deduplicated_track.append(point)
+    return deduplicated_track
+
+
 def _aggregate_gps_track(
     texts_by_tag: dict[str, list[str]],
     time_tag: str | None,
@@ -174,7 +200,7 @@ def _aggregate_gps_track(
                 epoch_time = geo.as_unix_time(dt)
 
     # build track
-    track = []
+    track: list[GPSPoint] = []
     for timestamp, lon, lat, alt, direction, ground_speed in zip(
         timestamps,
         lons,
@@ -185,22 +211,26 @@ def _aggregate_gps_track(
     ):
         if timestamp is None or lon is None or lat is None:
             continue
-        track.append(
-            GPSPoint(
-                time=timestamp,
-                lon=lon,
-                lat=lat,
-                alt=alt,
-                angle=direction,
-                epoch_time=epoch_time,
-                fix=None,
-                precision=None,
-                ground_speed=ground_speed,
-            )
+
+        point = GPSPoint(
+            time=timestamp,
+            lon=lon,
+            lat=lat,
+            alt=alt,
+            angle=direction,
+            epoch_time=epoch_time,
+            fix=None,
+            precision=None,
+            ground_speed=ground_speed,
         )
 
+        if not track or not _same_gps_point(track[-1], point):
+            track.append(point)
+
     track.sort(key=lambda point: point.time)
 
+    track = _deduplicate_gps_points(track, same_gps_point=_same_gps_point)
+
     if time_tag is not None:
         if track:
             first_time = track[0].time
@@ -310,7 +340,10 @@ class ExifToolReadVideo:
         etree: ET.ElementTree,
     ) -> None:
         self.etree = etree
-        self._texts_by_tag = _index_text_by_tag(self.etree.getroot())
+        root = self.etree.getroot()
+        if root is None:
+            raise ValueError("ElementTree root is None")
+        self._texts_by_tag = _index_text_by_tag(root)
         self._all_tags = set(self._texts_by_tag.keys())
 
     def extract_gps_track(self) -> list[geo.Point]:
@@ -371,6 +404,10 @@ class ExifToolReadVideo:
         return model
 
     def _extract_gps_track_from_track(self) -> list[GPSPoint]:
+        root = self.etree.getroot()
+        if root is None:
+            raise ValueError("ElementTree root is None")
+
         for track_id in range(1, MAX_TRACK_ID + 1):
             track_ns = f"Track{track_id}"
             if self._all_tags_exists(
@@ -382,7 +419,7 @@ class ExifToolReadVideo:
                 }
             ):
                 sample_iterator = _aggregate_samples(
-                    self.etree.getroot(),
+                    root,
                     f"{track_ns}:SampleTime",
                     f"{track_ns}:SampleDuration",
                 )

mapillary_tools/exiftool_runner.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-import platform
-import shutil
 import subprocess
 import typing as T
 from pathlib import Path
@@ -12,32 +10,14 @@ class ExiftoolRunner:
     Wrapper around ExifTool to run it in a subprocess
     """
 
-    def __init__(self, exiftool_path: str | None = None, recursive: bool = False):
-        if exiftool_path is None:
-            exiftool_path = self._search_preferred_exiftool_path()
-        self.exiftool_path = exiftool_path
+    def __init__(self, exiftool_executable: str = "exiftool", recursive: bool = False):
+        self.exiftool_executable = exiftool_executable
         self.recursive = recursive
 
-    def _search_preferred_exiftool_path(self) -> str:
-        system = platform.system()
-
-        if system and system.lower() == "windows":
-            exiftool_paths = ["exiftool.exe", "exiftool"]
-        else:
-            exiftool_paths = ["exiftool", "exiftool.exe"]
-
-        for path in exiftool_paths:
-            full_path = shutil.which(path)
-            if full_path:
-                return path
-
-        # Always return the prefered one, even if it is not found,
-        # and let the subprocess.run figure out the error later
-        return exiftool_paths[0]
-
     def _build_args_read_stdin(self) -> list[str]:
         args: list[str] = [
-            self.exiftool_path,
+            self.exiftool_executable,
+            "-fast",
             "-q",
             "-n",  # Disable print conversion
             "-X",  # XML output