mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0__py3-none-any.whl

mapillary_tools/commands/__main__.py

@@ -2,10 +2,12 @@ import argparse
 import enum
 import logging
 import sys
-import typing as T
 from pathlib import Path
 
-from .. import constants, exceptions, VERSION
+import requests
+
+from .. import api_v4, constants, exceptions, VERSION
+from ..upload import log_exception
 from . import (
     authenticate,
     process,
@@ -84,7 +86,7 @@ def configure_logger(logger: logging.Logger, stream=None) -> None:
     logger.addHandler(handler)
 
 
-def _log_params(argvars: T.Dict) -> None:
+def _log_params(argvars: dict) -> None:
     MAX_ENTRIES = 5
 
     def _stringify(x) -> str:
@@ -160,11 +162,18 @@ def main():
 
     try:
         args.func(argvars)
-    except exceptions.MapillaryUserError as exc:
-        LOG.error(
-            "%s: %s", exc.__class__.__name__, exc, exc_info=log_level == logging.DEBUG
-        )
-        sys.exit(exc.exit_code)
+    except requests.HTTPError as ex:
+        log_exception(ex)
+        # TODO: standardize exit codes as exceptions.MapillaryUserError
+        sys.exit(16)
+
+    except api_v4.HTTPContentError as ex:
+        log_exception(ex)
+        sys.exit(17)
+
+    except exceptions.MapillaryUserError as ex:
+        log_exception(ex)
+        sys.exit(ex.exit_code)
 
 
 if __name__ == "__main__":

mapillary_tools/commands/authenticate.py

@@ -10,7 +10,7 @@ class Command:
 
     def add_basic_arguments(self, parser: argparse.ArgumentParser):
         parser.add_argument(
-            "--user_name", help="Mapillary user name", default=None, required=False
+            "--user_name", help="Mapillary user profile", default=None, required=False
         )
         parser.add_argument(
             "--user_email",
@@ -27,6 +27,13 @@ class Command:
         parser.add_argument(
             "--jwt", help="Mapillary user access token", default=None, required=False
         )
+        parser.add_argument(
+            "--delete",
+            help="Delete the specified user profile",
+            default=False,
+            required=False,
+            action="store_true",
+        )
 
     def run(self, vars_args: dict):
         authenticate(

mapillary_tools/commands/process.py

@@ -1,41 +1,37 @@
+from __future__ import annotations
+
 import argparse
 import inspect
-import typing as T
 from pathlib import Path
 
-from .. import constants
+from .. import constants, types
 from ..process_geotag_properties import (
-    FileType,
-    GeotagSource,
+    DEFAULT_GEOTAG_SOURCE_OPTIONS,
     process_finalize,
     process_geotag_properties,
+    SourceType,
 )
 from ..process_sequence_properties import process_sequence_properties
 
 
+def bold_text(text: str) -> str:
+    ANSI_BOLD = "\033[1m"
+    ANSI_RESET_ALL = "\033[0m"
+    return f"{ANSI_BOLD}{text}{ANSI_RESET_ALL}"
+
+
 class Command:
     name = "process"
     help = "process images and videos"
 
     def add_basic_arguments(self, parser: argparse.ArgumentParser):
-        geotag_sources: T.List[GeotagSource] = [
-            "blackvue_videos",
-            "camm",
-            "exif",
-            "exiftool",
-            "gopro_videos",
-            "gpx",
-            "nmea",
+        geotag_gpx_based_sources: list[str] = [
+            SourceType.GPX.value,
+            SourceType.NMEA.value,
+            SourceType.GOPRO.value,
+            SourceType.BLACKVUE.value,
+            SourceType.CAMM.value,
         ]
-        geotag_gpx_based_sources: T.List[GeotagSource] = [
-            "gpx",
-            "gopro_videos",
-            "nmea",
-            "blackvue_videos",
-            "camm",
-        ]
-        for source in geotag_gpx_based_sources:
-            assert source in geotag_sources
 
         parser.add_argument(
             "--skip_process_errors",
@@ -47,14 +43,12 @@ class Command:
         parser.add_argument(
             "--filetypes",
             "--file_types",
-            help=f"Process files of the specified types only. Supported file types: {','.join(sorted(t.value for t in FileType))} [default: %(default)s]",
-            type=lambda option: set(FileType(t) for t in option.split(",")),
-            default=",".join(sorted(t.value for t in FileType)),
+            help=f"Process files of the specified types only. Supported file types: {','.join(sorted(t.value for t in types.FileType))} [default: %(default)s]",
+            type=lambda option: set(types.FileType(t) for t in option.split(",")),
+            default=None,
             required=False,
         )
-        group = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS EXIF OPTIONS{constants.ANSI_RESET_ALL}"
-        )
+        group = parser.add_argument_group(bold_text("PROCESS EXIF OPTIONS"))
         group.add_argument(
             "--overwrite_all_EXIF_tags",
             help="Overwrite all of the relevant EXIF tags with the values obtained in process. It is equivalent to supplying all the --overwrite_EXIF_*_tag flags.",
@@ -92,7 +86,7 @@ class Command:
         )
 
         group_metadata = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS METADATA OPTIONS{constants.ANSI_RESET_ALL}"
+            bold_text("PROCESS METADATA OPTIONS")
         )
         group_metadata.add_argument(
             "--device_make",
@@ -108,7 +102,7 @@ class Command:
         )
 
         group_geotagging = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS GEOTAGGING OPTIONS{constants.ANSI_RESET_ALL}"
+            bold_text("PROCESS GEOTAGGING OPTIONS")
         )
         group_geotagging.add_argument(
             "--desc_path",
@@ -118,10 +112,9 @@ class Command:
         )
         group_geotagging.add_argument(
             "--geotag_source",
-            help="Provide the source of date/time and GPS information needed for geotagging. [default: %(default)s]",
-            action="store",
-            choices=geotag_sources,
-            default="exif",
+            help=f"Provide the source of date/time and GPS information needed for geotagging. Supported source types: {', '.join(g.value for g in SourceType)} [default: {','.join(DEFAULT_GEOTAG_SOURCE_OPTIONS)}]",
+            action="append",
+            default=[],
             required=False,
         )
         group_geotagging.add_argument(
@@ -174,7 +167,7 @@ class Command:
         )
 
         group_sequence = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}PROCESS SEQUENCE OPTIONS{constants.ANSI_RESET_ALL}"
+            bold_text("PROCESS SEQUENCE OPTIONS")
         )
         group_sequence.add_argument(
             "--cutoff_distance",
@@ -212,24 +205,7 @@ class Command:
         )
 
     def run(self, vars_args: dict):
-        if (
-            "geotag_source" in vars_args
-            and vars_args["geotag_source"] == "blackvue_videos"
-            and (
-                "device_make" not in vars_args
-                or ("device_make" in vars_args and not vars_args["device_make"])
-            )
-        ):
-            vars_args["device_make"] = "Blackvue"
-        if (
-            "device_make" in vars_args
-            and vars_args["device_make"]
-            and vars_args["device_make"].lower() == "blackvue"
-        ):
-            vars_args["duplicate_angle"] = 360
-
         metadatas = process_geotag_properties(
-            vars_args=vars_args,
             **(
                 {
                     k: v

mapillary_tools/commands/process_and_upload.py

@@ -1,3 +1,7 @@
+import inspect
+
+from ..authenticate import fetch_user_items
+
 from .process import Command as ProcessCommand
 from .upload import Command as UploadCommand
 
@@ -10,10 +14,20 @@ class Command:
         ProcessCommand().add_basic_arguments(parser)
         UploadCommand().add_basic_arguments(parser)
 
-    def run(self, args: dict):
-        if args.get("desc_path") is None:
+    def run(self, vars_args: dict):
+        if vars_args.get("desc_path") is None:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
-            args["desc_path"] = "\x00"
-        ProcessCommand().run(args)
-        UploadCommand().run(args)
+            vars_args["desc_path"] = "\x00"
+
+        if "user_items" not in vars_args:
+            vars_args["user_items"] = fetch_user_items(
+                **{
+                    k: v
+                    for k, v in vars_args.items()
+                    if k in inspect.getfullargspec(fetch_user_items).args
+                }
+            )
+
+        ProcessCommand().run(vars_args)
+        UploadCommand().run(vars_args)

mapillary_tools/commands/sample_video.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 from .. import constants
 from ..sample_video import sample_video
+from .process import bold_text
 
 
 class Command:
@@ -11,9 +12,7 @@ class Command:
     help = "sample video into images"
 
     def add_basic_arguments(self, parser: argparse.ArgumentParser):
-        group = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}VIDEO PROCESS OPTIONS{constants.ANSI_RESET_ALL}"
-        )
+        group = parser.add_argument_group(bold_text("VIDEO PROCESS OPTIONS"))
         group.add_argument(
             "--video_sample_distance",
             help="The minimal distance interval, in meters, for sampling video frames. [default: %(default)s]",

mapillary_tools/commands/upload.py

@@ -1,18 +1,20 @@
 import inspect
 
 from .. import constants
+from ..authenticate import fetch_user_items
 from ..upload import upload
+from .process import bold_text
 
 
 class Command:
     name = "upload"
-    help = "upload images to Mapillary"
+    help = "Upload processed data to Mapillary"
 
     @staticmethod
     def add_common_upload_options(group):
         group.add_argument(
             "--user_name",
-            help="The Mapillary user account to upload to. If you only have one account authorized, it will upload to that account by default.",
+            help="The Mapillary user account to upload to.",
             required=False,
         )
         group.add_argument(
@@ -21,30 +23,59 @@ class Command:
             default=None,
             required=False,
         )
+        group.add_argument(
+            "--reupload",
+            help="Re-upload data that has already been uploaded.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
         group.add_argument(
             "--dry_run",
-            help='Instead of uploading to the Mapillary server, simulate uploading to the local directory "mapillary_public_uploads" for debugging purposes.',
+            "--dryrun",
+            help="[DEVELOPMENT] Simulate upload by sending data to a local directory instead of Mapillary servers. Uses a temporary directory by default unless specified by MAPILLARY_UPLOAD_ENDPOINT environment variable.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--nofinish",
+            help="[DEVELOPMENT] Upload data without finalizing. The data will NOT be stored permanently or appear on the Mapillary website.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--noresume",
+            help="[DEVELOPMENT] Start upload from the beginning, ignoring any previously interrupted upload sessions.",
             action="store_true",
             default=False,
             required=False,
         )
 
     def add_basic_arguments(self, parser):
-        group = parser.add_argument_group(
-            f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}"
-        )
+        group = parser.add_argument_group(bold_text("UPLOAD OPTIONS"))
         group.add_argument(
             "--desc_path",
-            help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
+            help=f'Path to the description file with processed image and video metadata (from process command). Use "-" for STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
             default=None,
             required=False,
         )
         Command.add_common_upload_options(group)
 
     def run(self, vars_args: dict):
-        args = {
-            k: v
-            for k, v in vars_args.items()
-            if k in inspect.getfullargspec(upload).args
-        }
-        upload(**args)
+        if "user_items" not in vars_args:
+            user_items_args = {
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(fetch_user_items).args
+            }
+            vars_args["user_items"] = fetch_user_items(**user_items_args)
+
+        upload(
+            **{
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(upload).args
+            }
+        )

mapillary_tools/commands/video_process_and_upload.py

@@ -1,3 +1,7 @@
+import inspect
+
+from ..authenticate import fetch_user_items
+
 from .upload import Command as UploadCommand
 from .video_process import Command as VideoProcessCommand
 
@@ -10,10 +14,20 @@ class Command:
         VideoProcessCommand().add_basic_arguments(parser)
         UploadCommand().add_basic_arguments(parser)
 
-    def run(self, args: dict):
-        if args.get("desc_path") is None:
+    def run(self, vars_args: dict):
+        if vars_args.get("desc_path") is None:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
-            args["desc_path"] = "\x00"
-        VideoProcessCommand().run(args)
-        UploadCommand().run(args)
+            vars_args["desc_path"] = "\x00"
+
+        if "user_items" not in vars_args:
+            vars_args["user_items"] = fetch_user_items(
+                **{
+                    k: v
+                    for k, v in vars_args.items()
+                    if k in inspect.getfullargspec(fetch_user_items).args
+                }
+            )
+
+        VideoProcessCommand().run(vars_args)
+        UploadCommand().run(vars_args)

mapillary_tools/config.py

@@ -1,30 +1,55 @@
+from __future__ import annotations
+
 import configparser
 import os
+import sys
 import typing as T
+from typing import TypedDict
 
-from . import api_v4, types
-
+if sys.version_info >= (3, 11):
+    from typing import Required
+else:
+    from typing_extensions import Required
 
-_CLIENT_ID = api_v4.MAPILLARY_CLIENT_TOKEN
-# Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
-_CLIENT_ID = _CLIENT_ID.replace("|", "_", 2)
+from . import api_v4
 
-DEFAULT_MAPILLARY_FOLDER = os.path.join(
-    os.path.expanduser("~"),
-    ".config",
-    "mapillary",
-)
 
+DEFAULT_MAPILLARY_FOLDER = os.path.join(os.path.expanduser("~"), ".config", "mapillary")
 MAPILLARY_CONFIG_PATH = os.getenv(
     "MAPILLARY_CONFIG_PATH",
     os.path.join(
         DEFAULT_MAPILLARY_FOLDER,
         "configs",
-        _CLIENT_ID,
+        # Windows is not happy with | so we convert MLY|ID|TOKEN to MLY_ID_TOKEN
+        api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"),
     ),
 )
 
 
+class UserItem(TypedDict, total=False):
+    MAPOrganizationKey: int | str
+    # Username
+    MAPSettingsUsername: str
+    # User ID
+    MAPSettingsUserKey: str
+    # User access token
+    user_upload_token: Required[str]
+
+
+UserItemSchema = {
+    "type": "object",
+    "properties": {
+        "MAPOrganizationKey": {"type": ["integer", "string"]},
+        # Not in use. Keep here for back-compatibility
+        "MAPSettingsUsername": {"type": "string"},
+        "MAPSettingsUserKey": {"type": "string"},
+        "user_upload_token": {"type": "string"},
+    },
+    "required": ["user_upload_token"],
+    "additionalProperties": True,
+}
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)
@@ -34,38 +59,52 @@ def _load_config(config_path: str) -> configparser.ConfigParser:
     return config
 
 
-def load_user(
-    user_name: str, config_path: T.Optional[str] = None
-) -> T.Optional[types.UserItem]:
+def load_user(profile_name: str, config_path: str | None = None) -> UserItem | None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
-    if not config.has_section(user_name):
+    if not config.has_section(profile_name):
         return None
-    user_items = dict(config.items(user_name))
-    return T.cast(types.UserItem, user_items)
+    user_items = dict(config.items(profile_name))
+    return T.cast(UserItem, user_items)
 
 
-def list_all_users(config_path: T.Optional[str] = None) -> T.List[types.UserItem]:
+def list_all_users(config_path: str | None = None) -> dict[str, UserItem]:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     cp = _load_config(config_path)
-    users = [
-        load_user(user_name, config_path=config_path) for user_name in cp.sections()
-    ]
-    return [item for item in users if item is not None]
+    users = {
+        profile_name: load_user(profile_name, config_path=config_path)
+        for profile_name in cp.sections()
+    }
+    return {profile: item for profile, item in users.items() if item is not None}
 
 
 def update_config(
-    user_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None
+    profile_name: str, user_items: UserItem, config_path: str | None = None
 ) -> None:
     if config_path is None:
         config_path = MAPILLARY_CONFIG_PATH
     config = _load_config(config_path)
-    if not config.has_section(user_name):
-        config.add_section(user_name)
+    if not config.has_section(profile_name):
+        config.add_section(profile_name)
     for key, val in user_items.items():
-        config.set(user_name, key, T.cast(str, val))
+        config.set(profile_name, key, T.cast(str, val))
+    os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True)
+    with open(config_path, "w") as fp:
+        config.write(fp)
+
+
+def remove_config(profile_name: str, config_path: str | None = None) -> None:
+    if config_path is None:
+        config_path = MAPILLARY_CONFIG_PATH
+
+    config = _load_config(config_path)
+    if not config.has_section(profile_name):
+        return
+
+    config.remove_section(profile_name)
+
     os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True)
     with open(config_path, "w") as fp:
         config.write(fp)