mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

mapillary_tools/commands/__main__.py

@@ -2,10 +2,11 @@ import argparse
 import enum
 import logging
 import sys
-import typing as T
 from pathlib import Path
 
-from .. import constants, exceptions, VERSION
+import requests
+
+from .. import api_v4, constants, exceptions, VERSION
 from . import (
     authenticate,
     process,
@@ -84,7 +85,7 @@ def configure_logger(logger: logging.Logger, stream=None) -> None:
     logger.addHandler(handler)
 
 
-def _log_params(argvars: T.Dict) -> None:
+def _log_params(argvars: dict) -> None:
     MAX_ENTRIES = 5
 
     def _stringify(x) -> str:
@@ -160,11 +161,16 @@ def main():
 
     try:
         args.func(argvars)
-    except exceptions.MapillaryUserError as exc:
+    except requests.HTTPError as ex:
+        LOG.error("%s: %s", ex.__class__.__name__, api_v4.readable_http_error(ex))
+        # TODO: standardize exit codes as exceptions.MapillaryUserError
+        sys.exit(16)
+
+    except exceptions.MapillaryUserError as ex:
         LOG.error(
-            "%s: %s", exc.__class__.__name__, exc, exc_info=log_level == logging.DEBUG
+            "%s: %s", ex.__class__.__name__, ex, exc_info=log_level == logging.DEBUG
         )
-        sys.exit(exc.exit_code)
+        sys.exit(ex.exit_code)
 
 
 if __name__ == "__main__":

mapillary_tools/commands/authenticate.py

@@ -10,7 +10,7 @@ class Command:
 
     def add_basic_arguments(self, parser: argparse.ArgumentParser):
         parser.add_argument(
-            "--user_name", help="Mapillary user name", default=None, required=False
+            "--user_name", help="Mapillary user profile", default=None, required=False
         )
         parser.add_argument(
             "--user_email",
@@ -27,6 +27,13 @@ class Command:
         parser.add_argument(
             "--jwt", help="Mapillary user access token", default=None, required=False
         )
+        parser.add_argument(
+            "--delete",
+            help="Delete the specified user profile",
+            default=False,
+            required=False,
+            action="store_true",
+        )
 
     def run(self, vars_args: dict):
         authenticate(

mapillary_tools/commands/process.py

@@ -1,41 +1,37 @@
+from __future__ import annotations
+
 import argparse
 import inspect
-import typing as T
 from pathlib import Path
 
-from .. import constants
+from .. import constants, types
 from ..process_geotag_properties import (
-    FileType,
-    GeotagSource,
+    DEFAULT_GEOTAG_SOURCE_OPTIONS,
     process_finalize,
     process_geotag_properties,
+    SourceType,
 )
 from ..process_sequence_properties import process_sequence_properties
 
 
+def bold_text(text: str) -> str:
+    ANSI_BOLD = "\033[1m"
+    ANSI_RESET_ALL = "\033[0m"
+    return f"{ANSI_BOLD}{text}{ANSI_RESET_ALL}"
+
+
 class Command:
     name = "process"
     help = "process images and videos"
 
     def add_basic_arguments(self, parser: argparse.ArgumentParser):
-        geotag_sources: T.List[GeotagSource] = [
-            "blackvue_videos",
-            "camm",
-            "exif",
-            "exiftool",
-            "gopro_videos",
-            "gpx",
-            "nmea",
+        geotag_gpx_based_sources: list[str] = [
+            SourceType.GPX.value,
+            SourceType.NMEA.value,
+            SourceType.GOPRO.value,
+            SourceType.BLACKVUE.value,
+            SourceType.CAMM.value,
         ]
-        geotag_gpx_based_sources: T.List[GeotagSource] = [
-            "gpx",
-            "gopro_videos",
-            "nmea",
-            "blackvue_videos",
-            "camm",
-        ]
-        for source in geotag_gpx_based_sources:
-            assert source in geotag_sources
 
         parser.add_argument(
             "--skip_process_errors",
@@ -47,14 +43,12 @@ class Command:
         parser.add_argument(
             "--filetypes",
             "--file_types",
-            help=f"Process files of the specified types only. Supported file types: {','.join(sorted(t.value for t in FileType))} [default: %(default)s]",
-            type=lambda option: set(FileType(t) for t in option.split(",")),
-            default=",".join(sorted(t.value for t in FileType)),
+            help=f"Process files of the specified types only. Supported file types: {','.join(sorted(t.value for t in types.FileType))} [default: %(default)s]",
+            type=lambda option: set(types.FileType(t) for t in option.split(",")),
+            default=None,
             required=False,
         )
-        group = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS EXIF OPTIONS{constants.ANSI_RESET_ALL}"
-        )
+        group = parser.add_argument_group(bold_text("PROCESS EXIF OPTIONS"))
         group.add_argument(
             "--overwrite_all_EXIF_tags",
             help="Overwrite all of the relevant EXIF tags with the values obtained in process. It is equivalent to supplying all the --overwrite_EXIF_*_tag flags.",
@@ -92,7 +86,7 @@ class Command:
         )
 
         group_metadata = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS METADATA OPTIONS{constants.ANSI_RESET_ALL}"
+            bold_text("PROCESS METADATA OPTIONS")
         )
         group_metadata.add_argument(
             "--device_make",
@@ -108,7 +102,7 @@ class Command:
         )
 
         group_geotagging = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS GEOTAGGING OPTIONS{constants.ANSI_RESET_ALL}"
+            bold_text("PROCESS GEOTAGGING OPTIONS")
         )
         group_geotagging.add_argument(
             "--desc_path",
@@ -118,10 +112,9 @@ class Command:
         )
         group_geotagging.add_argument(
             "--geotag_source",
-            help="Provide the source of date/time and GPS information needed for geotagging. [default: %(default)s]",
-            action="store",
-            choices=geotag_sources,
-            default="exif",
+            help=f"Provide the source of date/time and GPS information needed for geotagging. Supported source types: {', '.join(g.value for g in SourceType)} [default: {','.join(DEFAULT_GEOTAG_SOURCE_OPTIONS)}]",
+            action="append",
+            default=[],
             required=False,
         )
         group_geotagging.add_argument(
@@ -174,7 +167,7 @@ class Command:
         )
 
         group_sequence = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS SEQUENCE OPTIONS{constants.ANSI_RESET_ALL}"
+            bold_text("PROCESS SEQUENCE OPTIONS")
         )
         group_sequence.add_argument(
             "--cutoff_distance",
@@ -212,24 +205,7 @@ class Command:
         )
 
     def run(self, vars_args: dict):
-        if (
-            "geotag_source" in vars_args
-            and vars_args["geotag_source"] == "blackvue_videos"
-            and (
-                "device_make" not in vars_args
-                or ("device_make" in vars_args and not vars_args["device_make"])
-            )
-        ):
-            vars_args["device_make"] = "Blackvue"
-        if (
-            "device_make" in vars_args
-            and vars_args["device_make"]
-            and vars_args["device_make"].lower() == "blackvue"
-        ):
-            vars_args["duplicate_angle"] = 360
-
         metadatas = process_geotag_properties(
-            vars_args=vars_args,
             **(
                 {
                     k: v

mapillary_tools/commands/process_and_upload.py

@@ -1,3 +1,7 @@
+import inspect
+
+from ..authenticate import fetch_user_items
+
 from .process import Command as ProcessCommand
 from .upload import Command as UploadCommand
 
@@ -10,10 +14,20 @@ class Command:
         ProcessCommand().add_basic_arguments(parser)
         UploadCommand().add_basic_arguments(parser)
 
-    def run(self, args: dict):
-        if args.get("desc_path") is None:
+    def run(self, vars_args: dict):
+        if vars_args.get("desc_path") is None:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
-            args["desc_path"] = "\x00"
-        ProcessCommand().run(args)
-        UploadCommand().run(args)
+            vars_args["desc_path"] = "\x00"
+
+        if "user_items" not in vars_args:
+            vars_args["user_items"] = fetch_user_items(
+                **{
+                    k: v
+                    for k, v in vars_args.items()
+                    if k in inspect.getfullargspec(fetch_user_items).args
+                }
+            )
+
+        ProcessCommand().run(vars_args)
+        UploadCommand().run(vars_args)

mapillary_tools/commands/sample_video.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 from .. import constants
 from ..sample_video import sample_video
+from .process import bold_text
 
 
 class Command:
@@ -11,9 +12,7 @@ class Command:
     help = "sample video into images"
 
     def add_basic_arguments(self, parser: argparse.ArgumentParser):
-        group = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}VIDEO PROCESS OPTIONS{constants.ANSI_RESET_ALL}"
-        )
+        group = parser.add_argument_group(bold_text("VIDEO PROCESS OPTIONS"))
         group.add_argument(
             "--video_sample_distance",
             help="The minimal distance interval, in meters, for sampling video frames. [default: %(default)s]",

mapillary_tools/commands/upload.py

@@ -1,7 +1,9 @@
 import inspect
 
 from .. import constants
+from ..authenticate import fetch_user_items
 from ..upload import upload
+from .process import bold_text
 
 
 class Command:
@@ -30,9 +32,7 @@ class Command:
         )
 
     def add_basic_arguments(self, parser):
-        group = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}"
-        )
+        group = parser.add_argument_group(bold_text("UPLOAD OPTIONS"))
         group.add_argument(
             "--desc_path",
             help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
@@ -42,9 +42,18 @@ class Command:
         Command.add_common_upload_options(group)
 
     def run(self, vars_args: dict):
-        args = {
-            k: v
-            for k, v in vars_args.items()
-            if k in inspect.getfullargspec(upload).args
-        }
-        upload(**args)
+        if "user_items" not in vars_args:
+            user_items_args = {
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(fetch_user_items).args
+            }
+            vars_args["user_items"] = fetch_user_items(**user_items_args)
+
+        upload(
+            **{
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(upload).args
+            }
+        )

mapillary_tools/commands/video_process_and_upload.py

@@ -1,3 +1,7 @@
+import inspect
+
+from ..authenticate import fetch_user_items
+
 from .upload import Command as UploadCommand
 from .video_process import Command as VideoProcessCommand
 
@@ -10,10 +14,20 @@ class Command:
         VideoProcessCommand().add_basic_arguments(parser)
         UploadCommand().add_basic_arguments(parser)
 
-    def run(self, args: dict):
-        if args.get("desc_path") is None:
+    def run(self, vars_args: dict):
+        if vars_args.get("desc_path") is None:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
-            args["desc_path"] = "\x00"
-        VideoProcessCommand().run(args)
-        UploadCommand().run(args)
+            vars_args["desc_path"] = "\x00"
+
+        if "user_items" not in vars_args:
+            vars_args["user_items"] = fetch_user_items(
+                **{
+                    k: v
+                    for k, v in vars_args.items()
+                    if k in inspect.getfullargspec(fetch_user_items).args
+                }
+            )
+
+        VideoProcessCommand().run(vars_args)
+        UploadCommand().run(vars_args)

mapillary_tools/config.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import configparser
 import os
 import typing as T
@@ -35,37 +37,53 @@ def _load_config(config_path: str) -> configparser.ConfigParser:
 
 
 def load_user(
-    user_name: str, config_path: T.Optional[str] = None
-) -> T.Optional[types.UserItem]:
+    profile_name: str, config_path: str | None = None
+) -> types.UserItem | None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
-    if not config.has_section(user_name):
+    if not config.has_section(profile_name):
         return None
-    user_items = dict(config.items(user_name))
+    user_items = dict(config.items(profile_name))
     return T.cast(types.UserItem, user_items)
 
 
-def list_all_users(config_path: T.Optional[str] = None) -> T.List[types.UserItem]:
+def list_all_users(config_path: str | None = None) -> dict[str, types.UserItem]:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     cp = _load_config(config_path)
-    users = [
-        load_user(user_name, config_path=config_path) for user_name in cp.sections()
-    ]
-    return [item for item in users if item is not None]
+    users = {
+        profile_name: load_user(profile_name, config_path=config_path)
+        for profile_name in cp.sections()
+    }
+    return {profile: item for profile, item in users.items() if item is not None}
 
 
 def update_config(
-    user_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None
+    profile_name: str, user_items: types.UserItem, config_path: str | None = None
 ) -> None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
-    if not config.has_section(user_name):
-        config.add_section(user_name)
+    if not config.has_section(profile_name):
+        config.add_section(profile_name)
     for key, val in user_items.items():
-        config.set(user_name, key, T.cast(str, val))
+        config.set(profile_name, key, T.cast(str, val))
+    os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True)
+    with open(config_path, "w") as fp:
+        config.write(fp)
+
+
+def remove_config(profile_name: str, config_path: str | None = None) -> None:
+    if config_path is None:
+        config_path = MAPILLARY_CONFIG_PATH
+
+    config = _load_config(config_path)
+    if not config.has_section(profile_name):
+        return
+
+    config.remove_section(profile_name)
+
     os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True)
     with open(config_path, "w") as fp:
         config.write(fp)

mapillary_tools/constants.py

@@ -1,16 +1,29 @@
+from __future__ import annotations
+
 import os
-import typing as T
 
 import appdirs
 
 _ENV_PREFIX = "MAPILLARY_TOOLS_"
 
-ANSI_BOLD = "\033[1m"
-ANSI_RESET_ALL = "\033[0m"
+
+def _yes_or_no(val: str) -> bool:
+    return val.strip().upper() in [
+        "1",
+        "TRUE",
+        "YES",
+    ]
+
+
+# In meters
 CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
+# In seconds
 CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
 DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
 DUPLICATE_ANGLE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_ANGLE", 5))
+MAX_AVG_SPEED = float(
+    os.getenv(_ENV_PREFIX + "MAX_AVG_SPEED", 400_000 / 3600)
+)  # 400 KM/h
 # in seconds
 VIDEO_SAMPLE_INTERVAL = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_INTERVAL", -1))
 # in meters
@@ -18,7 +31,8 @@ VIDEO_SAMPLE_DISTANCE = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_DISTANCE", 3
 VIDEO_DURATION_RATIO = float(os.getenv(_ENV_PREFIX + "VIDEO_DURATION_RATIO", 1))
 FFPROBE_PATH: str = os.getenv(_ENV_PREFIX + "FFPROBE_PATH", "ffprobe")
 FFMPEG_PATH: str = os.getenv(_ENV_PREFIX + "FFMPEG_PATH", "ffmpeg")
-EXIFTOOL_PATH: str = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH", "exiftool")
+# When not set, MT will try to check both "exiftool" and "exiftool.exe" from $PATH
+EXIFTOOL_PATH: str | None = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH")
 IMAGE_DESCRIPTION_FILENAME = os.getenv(
     _ENV_PREFIX + "IMAGE_DESCRIPTION_FILENAME", "mapillary_image_description.json"
 )
@@ -26,14 +40,18 @@ SAMPLED_VIDEO_FRAMES_FILENAME = os.getenv(
     _ENV_PREFIX + "SAMPLED_VIDEO_FRAMES_FILENAME", "mapillary_sampled_video_frames"
 )
 USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary")
-UPLOAD_CHUNK_SIZE_MB = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 16))
+# The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
+# for uploading data to MLY upload service.
+# Changing this size does not change the number of requests nor affect upload performance,
+# but it affects the responsiveness of the upload progress bar
+UPLOAD_CHUNK_SIZE_MB = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 1))
 
 # DoP value, the lower the better
 # See https://github.com/gopro/gpmf-parser#hero5-black-with-gps-enabled-adds
 # It is used to filter out noisy points
 GOPRO_MAX_DOP100 = int(os.getenv(_ENV_PREFIX + "GOPRO_MAX_DOP100", 1000))
 # Within the GPS stream: 0 - no lock, 2 or 3 - 2D or 3D Lock
-GOPRO_GPS_FIXES: T.Set[int] = set(
+GOPRO_GPS_FIXES: set[int] = set(
     int(fix) for fix in os.getenv(_ENV_PREFIX + "GOPRO_GPS_FIXES", "2,3").split(",")
 )
 MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
@@ -48,3 +66,26 @@ MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
 MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
 # Max number of pixels per sequence (sum of image pixels in the sequence)
 MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
+
+PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
+
+_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
+    os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
+)
+
+MAPILLARY_DISABLE_API_LOGGING: bool = _yes_or_no(
+    os.getenv("MAPILLARY_DISABLE_API_LOGGING", "NO")
+)
+MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN: bool = _yes_or_no(
+    os.getenv("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", "NO")
+)
+MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
+    os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU", "NO")
+)
+MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
+    "MAPILLARY_UPLOAD_HISTORY_PATH",
+    os.path.join(
+        USER_DATA_DIR,
+        "upload_history",
+    ),
+)

mapillary_tools/exceptions.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import typing as T
 
 
@@ -5,6 +7,18 @@ class MapillaryUserError(Exception):
     exit_code: int
 
 
+class MapillaryProcessError(MapillaryUserError):
+    """
+    Base exception for process specific errors
+    """
+
+    exit_code = 6
+
+
+class MapillaryDescriptionError(Exception):
+    pass
+
+
 class MapillaryBadParameterError(MapillaryUserError):
     exit_code = 2
 
@@ -17,44 +31,35 @@ class MapillaryInvalidDescriptionFile(MapillaryUserError):
     exit_code = 4
 
 
-class MapillaryUnknownFileTypeError(MapillaryUserError):
-    exit_code = 5
-
-
-class MapillaryProcessError(MapillaryUserError):
-    exit_code = 6
-
-
 class MapillaryVideoError(MapillaryUserError):
     exit_code = 7
 
 
 class MapillaryFFmpegNotFoundError(MapillaryUserError):
     exit_code = 8
-    help = "https://github.com/mapillary/mapillary_tools#video-support"
 
 
 class MapillaryExiftoolNotFoundError(MapillaryUserError):
     exit_code = 8
 
 
-class MapillaryDescriptionError(Exception):
+class MapillaryGeoTaggingError(MapillaryDescriptionError):
     pass
 
 
-class MapillaryGeoTaggingError(MapillaryDescriptionError):
+class MapillaryVideoGPSNotFoundError(MapillaryDescriptionError):
     pass
 
 
-class MapillaryGPXEmptyError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 9
+class MapillaryGPXEmptyError(MapillaryDescriptionError):
+    pass
 
 
-class MapillaryVideoGPSNotFoundError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 9
+class MapillaryGPSNoiseError(MapillaryDescriptionError):
+    pass
 
 
-class MapillaryGPSNoiseError(MapillaryDescriptionError):
+class MapillaryStationaryVideoError(MapillaryDescriptionError):
     pass
 
 
@@ -68,21 +73,13 @@ class MapillaryOutsideGPXTrackError(MapillaryDescriptionError):
         self.gpx_end_time = gpx_end_time
 
 
-class MapillaryStationaryVideoError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 10
-
-
-class MapillaryInvalidBlackVueVideoError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 11
-
-
 class MapillaryDuplicationError(MapillaryDescriptionError):
     def __init__(
         self,
         message: str,
         desc: T.Mapping[str, T.Any],
         distance: float,
-        angle_diff: T.Optional[float],
+        angle_diff: float | None,
     ) -> None:
         super().__init__(message)
         self.desc = desc
@@ -90,17 +87,19 @@ class MapillaryDuplicationError(MapillaryDescriptionError):
         self.angle_diff = angle_diff
 
 
-class MapillaryUploadedAlreadyError(MapillaryDescriptionError):
-    def __init__(
-        self,
-        message: str,
-        desc: T.Mapping[str, T.Any],
-    ) -> None:
-        super().__init__(message)
-        self.desc = desc
+class MapillaryExifToolXMLNotFoundError(MapillaryDescriptionError):
+    pass
+
+
+class MapillaryFileTooLargeError(MapillaryDescriptionError):
+    pass
+
+
+class MapillaryCaptureSpeedTooFastError(MapillaryDescriptionError):
+    pass
 
 
-class MapillaryEXIFNotFoundError(MapillaryDescriptionError):
+class MapillaryNullIslandError(MapillaryDescriptionError):
     pass
 
 
@@ -116,5 +115,5 @@ class MapillaryUploadUnauthorizedError(MapillaryUserError):
     exit_code = 14
 
 
-class MapillaryMetadataValidationError(MapillaryUserError, MapillaryDescriptionError):
+class MapillaryMetadataValidationError(MapillaryUserError):
     exit_code = 15