mapillary-tools 0.14.0b1__tar.gz → 0.14.2__tar.gz

{mapillary_tools-0.14.0b1 → mapillary_tools-0.14.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mapillary_tools
-Version: 0.14.0b1
+Version: 0.14.2
 Summary: Mapillary Image/Video Import Pipeline
 Author-email: Mapillary <support@mapillary.com>
 License: BSD
@@ -22,11 +22,12 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: appdirs<2.0.0,>=1.4.4
-Requires-Dist: construct<3.0.0,>=2.10.0
-Requires-Dist: exifread==2.3.2
-Requires-Dist: gpxpy<1.6.0,>=1.5.0
-Requires-Dist: jsonschema~=4.17.3
-Requires-Dist: piexif==1.1.3
+Requires-Dist: construct~=2.10.0
+Requires-Dist: exifread~=3.0
+Requires-Dist: gpxpy~=1.6.0
+Requires-Dist: humanize>=4.12.3
+Requires-Dist: jsonschema~=4.17.0
+Requires-Dist: piexif~=1.1
 Requires-Dist: pynmea2<2.0.0,>=1.12.0
 Requires-Dist: requests[socks]<3.0.0,>=2.20.0
 Requires-Dist: tqdm<5.0,>=4.0

mapillary_tools-0.14.2/mapillary_tools/__init__.py

@@ -0,0 +1 @@
+VERSION = "0.14.2"

mapillary_tools-0.14.2/mapillary_tools/api_v4.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import enum
+import logging
+import os
+import typing as T
+
+import requests
+
+from . import http
+
+LOG = logging.getLogger(__name__)
+MAPILLARY_CLIENT_TOKEN = os.getenv(
+    "MAPILLARY_CLIENT_TOKEN", "MLY|5675152195860640|6b02c72e6e3c801e5603ab0495623282"
+)
+MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
+    "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
+)
+REQUESTS_TIMEOUT: float = 60  # 1 minutes
+
+
+class HTTPContentError(Exception):
+    """
+    Raised when the HTTP response is ok (200) but the content is not as expected
+    e.g. not JSON or not a valid response.
+    """
+
+    def __init__(self, message: str, response: requests.Response):
+        self.response = response
+        super().__init__(message)
+
+
+class ClusterFileType(enum.Enum):
+    ZIP = "zip"
+    BLACKVUE = "mly_blackvue_video"
+    CAMM = "mly_camm_video"
+    MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
+
+
+def create_user_session(user_access_token: str) -> requests.Session:
+    session = http.Session()
+    session.headers["Authorization"] = f"OAuth {user_access_token}"
+    return session
+
+
+def create_client_session(disable_logging: bool = False) -> requests.Session:
+    session = http.Session()
+    session.headers["Authorization"] = f"OAuth {MAPILLARY_CLIENT_TOKEN}"
+    if disable_logging:
+        session.disable_logging_request = True
+        session.disable_logging_response = True
+    return session
+
+
+def is_auth_error(resp: requests.Response) -> bool:
+    if resp.status_code in [401, 403]:
+        return True
+
+    if resp.status_code in [400]:
+        try:
+            error_body = resp.json()
+        except Exception:
+            error_body = {}
+
+        type = error_body.get("debug_info", {}).get("type")
+        if type in ["NotAuthorizedError"]:
+            return True
+
+    return False
+
+
+def extract_auth_error_message(resp: requests.Response) -> str:
+    assert is_auth_error(resp), "has to be an auth error"
+
+    try:
+        error_body = resp.json()
+    except Exception:
+        error_body = {}
+
+    # from Graph APIs
+    message = error_body.get("error", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    # from upload service
+    message = error_body.get("debug_info", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    return resp.text
+
+
+def get_upload_token(
+    client_session: requests.Session, email: str, password: str
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/login"
+    json_data = {"email": email, "password": password, "locale": "en_US"}
+
+    resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
+    resp.raise_for_status()
+
+    return resp
+
+
+def fetch_organization(
+    user_session: requests.Session, organization_id: int | str
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}"
+    params = {"fields": ",".join(["slug", "description", "name"])}
+
+    resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
+    resp.raise_for_status()
+
+    return resp
+
+
+def fetch_user_or_me(
+    user_session: requests.Session, user_id: int | str | None = None
+) -> requests.Response:
+    if user_id is None:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
+    else:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
+    params = {"fields": ",".join(["id", "username"])}
+
+    resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
+    resp.raise_for_status()
+
+    return resp
+
+
+ActionType = T.Literal[
+    "upload_started_upload", "upload_finished_upload", "upload_failed_upload"
+]
+
+
+def log_event(
+    client_session: requests.Session, action_type: ActionType, properties: dict
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging"
+    json_data = {"action_type": action_type, "properties": properties}
+
+    resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
+    resp.raise_for_status()
+
+    return resp
+
+
+def finish_upload(
+    user_session: requests.Session,
+    file_handle: str,
+    cluster_filetype: ClusterFileType,
+    organization_id: int | str | None = None,
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
+    json_data: dict[str, str | int] = {
+        "file_handle": file_handle,
+        "file_type": cluster_filetype.value,
+    }
+    if organization_id is not None:
+        json_data["organization_id"] = organization_id
+
+    resp = user_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
+    resp.raise_for_status()
+
+    return resp
+
+
+def jsonify_response(resp: requests.Response) -> T.Any:
+    """
+    Convert the response to JSON, raising HTTPContentError if the response is not JSON.
+    """
+    try:
+        return resp.json()
+    except requests.JSONDecodeError as ex:
+        raise HTTPContentError("Invalid JSON response", resp) from ex

{mapillary_tools-0.14.0b1 → mapillary_tools-0.14.2}/mapillary_tools/authenticate.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import getpass
-import json
 import logging
 import re
 import sys
@@ -11,7 +10,7 @@ import jsonschema
 
 import requests
 
-from . import api_v4, config, constants, exceptions
+from . import api_v4, config, constants, exceptions, http
 
 
 LOG = logging.getLogger(__name__)
@@ -78,17 +77,16 @@ def authenticate(
         # TODO: print more user information
         if profile_name in all_user_items:
             LOG.info(
-                'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items)
+                'Profile "%s" updated: %s', profile_name, http._sanitize(user_items)
             )
         else:
             LOG.info(
-                'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items)
+                'Profile "%s" created: %s', profile_name, http._sanitize(user_items)
             )
 
 
 def fetch_user_items(
-    user_name: str | None = None,
-    organization_key: str | None = None,
+    user_name: str | None = None, organization_key: str | None = None
 ) -> config.UserItem:
     """
     Read user information from the config file,
@@ -129,18 +127,28 @@ def fetch_user_items(
 
     assert profile_name is not None, "profile_name should be set"
 
-    user_items = _verify_user_auth(_validate_profile(user_items))
-
-    LOG.info(
-        'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items)
-    )
+    try:
+        LOG.info(f'Verifying profile "{profile_name}"...')
+        user_items = _verify_user_auth(_validate_profile(user_items))
 
-    if organization_key is not None:
-        resp = api_v4.fetch_organization(
-            user_items["user_upload_token"], organization_key
+        LOG.info(
+            f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
         )
-        LOG.info("Uploading to Mapillary organization: %s", json.dumps(resp.json()))
-        user_items["MAPOrganizationKey"] = organization_key
+
+        if organization_key is not None:
+            with api_v4.create_user_session(user_items["user_upload_token"]) as session:
+                resp = api_v4.fetch_organization(session, organization_key)
+            data = api_v4.jsonify_response(resp)
+            LOG.info(
+                f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})"
+            )
+            user_items["MAPOrganizationKey"] = data.get("id")
+
+    except requests.Timeout as ex:
+        raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex
+
+    except requests.ConnectionError as ex:
+        raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex
 
     return user_items
 
@@ -157,7 +165,7 @@ def _prompt(message: str) -> str:
 
 def _validate_profile(user_items: config.UserItem) -> config.UserItem:
     try:
-        jsonschema.validate(user_items, config.UserItemSchema)
+        config.UserItemSchemaValidator.validate(user_items)
     except jsonschema.ValidationError as ex:
         raise exceptions.MapillaryBadParameterError(
             f"Invalid profile format: {ex.message}"
@@ -172,23 +180,22 @@ def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
     if constants._AUTH_VERIFICATION_DISABLED:
         return user_items
 
-    try:
-        resp = api_v4.fetch_user_or_me(
-            user_access_token=user_items["user_upload_token"]
-        )
-    except requests.HTTPError as ex:
-        if api_v4.is_auth_error(ex.response):
-            message = api_v4.extract_auth_error_message(ex.response)
-            raise exceptions.MapillaryUploadUnauthorizedError(message)
-        else:
-            raise ex
+    with api_v4.create_user_session(user_items["user_upload_token"]) as session:
+        try:
+            resp = api_v4.fetch_user_or_me(session)
+        except requests.HTTPError as ex:
+            if api_v4.is_auth_error(ex.response):
+                message = api_v4.extract_auth_error_message(ex.response)
+                raise exceptions.MapillaryUploadUnauthorizedError(message)
+            else:
+                raise ex
 
-    user_json = resp.json()
+    data = api_v4.jsonify_response(resp)
 
     return {
         **user_items,
-        "MAPSettingsUsername": user_json.get("username"),
-        "MAPSettingsUserKey": user_json.get("id"),
+        "MAPSettingsUsername": data.get("username"),
+        "MAPSettingsUserKey": data.get("id"),
     }
 
 
@@ -275,18 +282,19 @@ def _prompt_login(
             if user_password:
                 break
 
-    try:
-        resp = api_v4.get_upload_token(user_email, user_password)
-    except requests.HTTPError as ex:
-        if not _enabled:
-            raise ex
+    with api_v4.create_client_session() as session:
+        try:
+            resp = api_v4.get_upload_token(session, user_email, user_password)
+        except requests.HTTPError as ex:
+            if not _enabled:
+                raise ex
 
-        if _is_login_retryable(ex):
-            return _prompt_login()
+            if _is_login_retryable(ex):
+                return _prompt_login()
 
-        raise ex
+            raise ex
 
-    data = resp.json()
+    data = api_v4.jsonify_response(resp)
 
     user_items: config.UserItem = {
         "user_upload_token": str(data["access_token"]),

{mapillary_tools-0.14.0b1 → mapillary_tools-0.14.2}/mapillary_tools/commands/__main__.py

@@ -7,6 +7,8 @@ from pathlib import Path
 import requests
 
 from .. import api_v4, constants, exceptions, VERSION
+from ..upload import log_exception
+from ..utils import configure_logger, get_app_name
 from . import (
     authenticate,
     process,
@@ -30,8 +32,8 @@ mapillary_tools_commands = [
 ]
 
 
-# do not use __name__ here is because if you run tools as a module, __name__ will be "__main__"
-LOG = logging.getLogger("mapillary_tools")
+# Root logger of mapillary_tools (not including third-party libraries)
+LOG = logging.getLogger(get_app_name())
 
 
 # Handle shared arguments/options here
@@ -78,13 +80,6 @@ def add_general_arguments(parser, command):
         )
 
 
-def configure_logger(logger: logging.Logger, stream=None) -> None:
-    formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s")
-    handler = logging.StreamHandler(stream)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-
-
 def _log_params(argvars: dict) -> None:
     MAX_ENTRIES = 5
 
@@ -151,9 +146,7 @@ def main():
 
     args = parser.parse_args()
 
-    log_level = logging.DEBUG if args.verbose else logging.INFO
-    configure_logger(LOG, sys.stderr)
-    LOG.setLevel(log_level)
+    configure_logger(LOG, level=logging.DEBUG if args.verbose else logging.INFO)
 
     LOG.debug("%s", version_text)
     argvars = vars(args)
@@ -162,16 +155,22 @@ def main():
     try:
         args.func(argvars)
     except requests.HTTPError as ex:
-        LOG.error("%s: %s", ex.__class__.__name__, api_v4.readable_http_error(ex))
+        log_exception(ex)
         # TODO: standardize exit codes as exceptions.MapillaryUserError
         sys.exit(16)
 
+    except api_v4.HTTPContentError as ex:
+        log_exception(ex)
+        sys.exit(17)
+
     except exceptions.MapillaryUserError as ex:
-        LOG.error(
-            "%s: %s", ex.__class__.__name__, ex, exc_info=log_level == logging.DEBUG
-        )
+        log_exception(ex)
         sys.exit(ex.exit_code)
 
+    except KeyboardInterrupt:
+        LOG.info("Interrupted by user...")
+        sys.exit(130)
+
 
 if __name__ == "__main__":
     main()

mapillary_tools-0.14.2/mapillary_tools/commands/upload.py

@@ -0,0 +1,88 @@
+import inspect
+
+from .. import constants
+from ..authenticate import fetch_user_items
+from ..upload import upload
+from .process import bold_text
+
+
+class Command:
+    name = "upload"
+    help = "Upload processed data to Mapillary"
+
+    @staticmethod
+    def add_common_upload_options(group):
+        group.add_argument(
+            "--user_name",
+            help="The Mapillary user account to upload to.",
+            required=False,
+        )
+        group.add_argument(
+            "--organization_key",
+            help="The Mapillary organization ID to upload to.",
+            default=None,
+            required=False,
+        )
+        group.add_argument(
+            "--num_upload_workers",
+            help="Number of concurrent upload workers for uploading images. [default: %(default)s]",
+            default=constants.MAX_IMAGE_UPLOAD_WORKERS,
+            type=int,
+            required=False,
+        )
+        group.add_argument(
+            "--reupload",
+            help="Re-upload data that has already been uploaded.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--dry_run",
+            "--dryrun",
+            help="[DEVELOPMENT] Simulate upload by sending data to a local directory instead of Mapillary servers. Uses a temporary directory by default unless specified by MAPILLARY_UPLOAD_ENDPOINT environment variable.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--nofinish",
+            help="[DEVELOPMENT] Upload data without finalizing. The data will NOT be stored permanently or appear on the Mapillary website.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--noresume",
+            help="[DEVELOPMENT] Start upload from the beginning, ignoring any previously interrupted upload sessions.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+
+    def add_basic_arguments(self, parser):
+        group = parser.add_argument_group(bold_text("UPLOAD OPTIONS"))
+        group.add_argument(
+            "--desc_path",
+            help=f'Path to the description file with processed image and video metadata (from process command). Use "-" for STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
+            default=None,
+            required=False,
+        )
+        Command.add_common_upload_options(group)
+
+    def run(self, vars_args: dict):
+        if "user_items" not in vars_args:
+            user_items_args = {
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(fetch_user_items).args
+            }
+            vars_args["user_items"] = fetch_user_items(**user_items_args)
+
+        upload(
+            **{
+                k: v
+                for k, v in vars_args.items()
+                if k in inspect.getfullargspec(upload).args
+            }
+        )

{mapillary_tools-0.14.0b1 → mapillary_tools-0.14.2}/mapillary_tools/config.py

@@ -6,6 +6,8 @@ import sys
 import typing as T
 from typing import TypedDict
 
+import jsonschema
+
 if sys.version_info >= (3, 11):
     from typing import Required
 else:
@@ -50,6 +52,9 @@ UserItemSchema = {
 }
 
 
+UserItemSchemaValidator = jsonschema.Draft202012Validator(UserItemSchema)
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)