mapillary-tools 0.14.0b1__py3-none-any.whl → 0.14.1__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.14.0b1"
+VERSION = "0.14.1"

mapillary_tools/api_v4.py

@@ -3,12 +3,11 @@ from __future__ import annotations
 import enum
 import logging
 import os
-import ssl
 import typing as T
-from json import dumps
 
 import requests
-from requests.adapters import HTTPAdapter
+
+from . import http
 
 LOG = logging.getLogger(__name__)
 MAPILLARY_CLIENT_TOKEN = os.getenv(
@@ -17,222 +16,40 @@ MAPILLARY_CLIENT_TOKEN = os.getenv(
 MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
     "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
 )
-REQUESTS_TIMEOUT = 60  # 1 minutes
-USE_SYSTEM_CERTS: bool = False
-
-
-class ClusterFileType(enum.Enum):
-    ZIP = "zip"
-    BLACKVUE = "mly_blackvue_video"
-    CAMM = "mly_camm_video"
-    MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
+REQUESTS_TIMEOUT: float = 60  # 1 minutes
 
 
-class HTTPSystemCertsAdapter(HTTPAdapter):
+class HTTPContentError(Exception):
     """
-    This adapter uses the system's certificate store instead of the certifi module.
-
-    The implementation is based on the project https://pypi.org/project/pip-system-certs/,
-    which has a system-wide effect.
+    Raised when the HTTP response is ok (200) but the content is not as expected
+    e.g. not JSON or not a valid response.
     """
 
-    def init_poolmanager(self, *args, **kwargs):
-        ssl_context = ssl.create_default_context()
-        ssl_context.load_default_certs()
-        kwargs["ssl_context"] = ssl_context
-
-        super().init_poolmanager(*args, **kwargs)
-
-    def cert_verify(self, *args, **kwargs):
-        super().cert_verify(*args, **kwargs)
-
-        # By default Python requests uses the ca_certs from the certifi module
-        # But we want to use the certificate store instead.
-        # By clearing the ca_certs variable we force it to fall back on that behaviour (handled in urllib3)
-        if "conn" in kwargs:
-            conn = kwargs["conn"]
-        else:
-            conn = args[0]
-
-        conn.ca_certs = None
-
-
-@T.overload
-def _truncate(s: bytes, limit: int = 512) -> bytes: ...
-
-
-@T.overload
-def _truncate(s: str, limit: int = 512) -> str: ...
-
-
-def _truncate(s, limit=512):
-    if limit < len(s):
-        remaining = len(s) - limit
-        if isinstance(s, bytes):
-            return (
-                s[:limit]
-                + b"..."
-                + f"({remaining} more bytes truncated)".encode("utf-8")
-            )
-        else:
-            return str(s[:limit]) + f"...({remaining} more chars truncated)"
-    else:
-        return s
-
-
-def _sanitize(headers: T.Mapping[T.Any, T.Any]) -> T.Mapping[T.Any, T.Any]:
-    new_headers = {}
-
-    for k, v in headers.items():
-        if k.lower() in [
-            "authorization",
-            "cookie",
-            "x-fb-access-token",
-            "access-token",
-            "access_token",
-            "password",
-            "user_upload_token",
-        ]:
-            new_headers[k] = "[REDACTED]"
-        else:
-            new_headers[k] = _truncate(v)
-
-    return new_headers
-
-
-def _log_debug_request(
-    method: str,
-    url: str,
-    json: dict | None = None,
-    params: dict | None = None,
-    headers: dict | None = None,
-    timeout: T.Any = None,
-):
-    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
-        return
-
-    msg = f"HTTP {method} {url}"
-
-    if USE_SYSTEM_CERTS:
-        msg += " (w/sys_certs)"
-
-    if json:
-        t = _truncate(dumps(_sanitize(json)))
-        msg += f" JSON={t}"
-
-    if params:
-        msg += f" PARAMS={_sanitize(params)}"
-
-    if headers:
-        msg += f" HEADERS={_sanitize(headers)}"
-
-    if timeout is not None:
-        msg += f" TIMEOUT={timeout}"
-
-    LOG.debug(msg)
-
-
-def _log_debug_response(resp: requests.Response):
-    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
-        return
-
-    data: str | bytes
-    try:
-        data = _truncate(dumps(_sanitize(resp.json())))
-    except Exception:
-        data = _truncate(resp.content)
+    def __init__(self, message: str, response: requests.Response):
+        self.response = response
+        super().__init__(message)
 
-    LOG.debug(f"HTTP {resp.status_code} ({resp.reason}): %s", data)
-
-
-def readable_http_error(ex: requests.HTTPError) -> str:
-    req = ex.request
-    resp = ex.response
-
-    data: str | bytes
-    try:
-        data = _truncate(dumps(_sanitize(resp.json())))
-    except Exception:
-        data = _truncate(resp.content)
-
-    return f"{req.method} {resp.url} => {resp.status_code} ({resp.reason}): {str(data)}"
 
+class ClusterFileType(enum.Enum):
+    ZIP = "zip"
+    BLACKVUE = "mly_blackvue_video"
+    CAMM = "mly_camm_video"
+    MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
 
-def request_post(
-    url: str,
-    data: T.Any | None = None,
-    json: dict | None = None,
-    **kwargs,
-) -> requests.Response:
-    global USE_SYSTEM_CERTS
-
-    _log_debug_request(
-        "POST",
-        url,
-        json=json,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
-
-    if USE_SYSTEM_CERTS:
-        with requests.Session() as session:
-            session.mount("https://", HTTPSystemCertsAdapter())
-            resp = session.post(url, data=data, json=json, **kwargs)
 
-    else:
-        try:
-            resp = requests.post(url, data=data, json=json, **kwargs)
-        except requests.exceptions.SSLError as ex:
-            if "SSLCertVerificationError" not in str(ex):
-                raise ex
-            USE_SYSTEM_CERTS = True
-            # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
-            LOG.warning(
-                "SSL error occurred, falling back to system SSL certificates: %s", ex
-            )
-            return request_post(url, data=data, json=json, **kwargs)
-
-    _log_debug_response(resp)
+def create_user_session(user_access_token: str) -> requests.Session:
+    session = http.Session()
+    session.headers["Authorization"] = f"OAuth {user_access_token}"
+    return session
 
-    return resp
 
-
-def request_get(
-    url: str,
-    params: dict | None = None,
-    **kwargs,
-) -> requests.Response:
-    global USE_SYSTEM_CERTS
-
-    _log_debug_request(
-        "GET",
-        url,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
-
-    if USE_SYSTEM_CERTS:
-        with requests.Session() as session:
-            session.mount("https://", HTTPSystemCertsAdapter())
-            resp = session.get(url, params=params, **kwargs)
-    else:
-        try:
-            resp = requests.get(url, params=params, **kwargs)
-        except requests.exceptions.SSLError as ex:
-            if "SSLCertVerificationError" not in str(ex):
-                raise ex
-            USE_SYSTEM_CERTS = True
-            # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
-            LOG.warning(
-                "SSL error occurred, falling back to system SSL certificates: %s", ex
-            )
-            resp = request_get(url, params=params, **kwargs)
-
-    _log_debug_response(resp)
-
-    return resp
+def create_client_session(disable_logging: bool = False) -> requests.Session:
+    session = http.Session()
+    session.headers["Authorization"] = f"OAuth {MAPILLARY_CLIENT_TOKEN}"
+    if disable_logging:
+        session.disable_logging_request = True
+        session.disable_logging_response = True
+    return session
 
 
 def is_auth_error(resp: requests.Response) -> bool:
@@ -273,55 +90,42 @@ def extract_auth_error_message(resp: requests.Response) -> str:
     return resp.text
 
 
-def get_upload_token(email: str, password: str) -> requests.Response:
-    resp = request_post(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
-        headers={"Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}"},
-        json={"email": email, "password": password, "locale": "en_US"},
-        timeout=REQUESTS_TIMEOUT,
-    )
+def get_upload_token(
+    client_session: requests.Session, email: str, password: str
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/login"
+    json_data = {"email": email, "password": password, "locale": "en_US"}
+
+    resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
 def fetch_organization(
-    user_access_token: str, organization_id: int | str
+    user_session: requests.Session, organization_id: int | str
 ) -> requests.Response:
-    resp = request_get(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}",
-        params={
-            "fields": ",".join(["slug", "description", "name"]),
-        },
-        headers={
-            "Authorization": f"OAuth {user_access_token}",
-        },
-        timeout=REQUESTS_TIMEOUT,
-    )
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}"
+    params = {"fields": ",".join(["slug", "description", "name"])}
+
+    resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
 def fetch_user_or_me(
-    user_access_token: str,
-    user_id: int | str | None = None,
+    user_session: requests.Session, user_id: int | str | None = None
 ) -> requests.Response:
     if user_id is None:
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
     else:
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
+    params = {"fields": ",".join(["id", "username"])}
 
-    resp = request_get(
-        url,
-        params={
-            "fields": ",".join(["id", "username"]),
-        },
-        headers={
-            "Authorization": f"OAuth {user_access_token}",
-        },
-        timeout=REQUESTS_TIMEOUT,
-    )
-
+    resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
@@ -330,44 +134,43 @@ ActionType = T.Literal[
 ]
 
 
-def log_event(action_type: ActionType, properties: dict) -> requests.Response:
-    resp = request_post(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging",
-        json={
-            "action_type": action_type,
-            "properties": properties,
-        },
-        headers={
-            "Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}",
-        },
-        timeout=REQUESTS_TIMEOUT,
-    )
+def log_event(
+    client_session: requests.Session, action_type: ActionType, properties: dict
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging"
+    json_data = {"action_type": action_type, "properties": properties}
+
+    resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
 def finish_upload(
-    user_access_token: str,
+    user_session: requests.Session,
     file_handle: str,
     cluster_filetype: ClusterFileType,
     organization_id: int | str | None = None,
 ) -> requests.Response:
-    data: dict[str, str | int] = {
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
+    json_data: dict[str, str | int] = {
         "file_handle": file_handle,
         "file_type": cluster_filetype.value,
     }
     if organization_id is not None:
-        data["organization_id"] = organization_id
-
-    resp = request_post(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload",
-        headers={
-            "Authorization": f"OAuth {user_access_token}",
-        },
-        json=data,
-        timeout=REQUESTS_TIMEOUT,
-    )
+        json_data["organization_id"] = organization_id
 
+    resp = user_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
 
     return resp
+
+
+def jsonify_response(resp: requests.Response) -> T.Any:
+    """
+    Convert the response to JSON, raising HTTPContentError if the response is not JSON.
+    """
+    try:
+        return resp.json()
+    except requests.JSONDecodeError as ex:
+        raise HTTPContentError("Invalid JSON response", resp) from ex

mapillary_tools/authenticate.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import getpass
-import json
 import logging
 import re
 import sys
@@ -11,7 +10,7 @@ import jsonschema
 
 import requests
 
-from . import api_v4, config, constants, exceptions
+from . import api_v4, config, constants, exceptions, http
 
 
 LOG = logging.getLogger(__name__)
@@ -78,17 +77,16 @@ def authenticate(
         # TODO: print more user information
         if profile_name in all_user_items:
             LOG.info(
-                'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items)
+                'Profile "%s" updated: %s', profile_name, http._sanitize(user_items)
             )
         else:
             LOG.info(
-                'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items)
+                'Profile "%s" created: %s', profile_name, http._sanitize(user_items)
             )
 
 
 def fetch_user_items(
-    user_name: str | None = None,
-    organization_key: str | None = None,
+    user_name: str | None = None, organization_key: str | None = None
 ) -> config.UserItem:
     """
     Read user information from the config file,
@@ -129,18 +127,28 @@ def fetch_user_items(
 
     assert profile_name is not None, "profile_name should be set"
 
-    user_items = _verify_user_auth(_validate_profile(user_items))
-
-    LOG.info(
-        'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items)
-    )
+    try:
+        LOG.info(f'Verifying profile "{profile_name}"...')
+        user_items = _verify_user_auth(_validate_profile(user_items))
 
-    if organization_key is not None:
-        resp = api_v4.fetch_organization(
-            user_items["user_upload_token"], organization_key
+        LOG.info(
+            f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
         )
-        LOG.info("Uploading to Mapillary organization: %s", json.dumps(resp.json()))
-        user_items["MAPOrganizationKey"] = organization_key
+
+        if organization_key is not None:
+            with api_v4.create_user_session(user_items["user_upload_token"]) as session:
+                resp = api_v4.fetch_organization(session, organization_key)
+            data = api_v4.jsonify_response(resp)
+            LOG.info(
+                f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})"
+            )
+            user_items["MAPOrganizationKey"] = data.get("id")
+
+    except requests.Timeout as ex:
+        raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex
+
+    except requests.ConnectionError as ex:
+        raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex
 
     return user_items
 
@@ -172,23 +180,22 @@ def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
     if constants._AUTH_VERIFICATION_DISABLED:
         return user_items
 
-    try:
-        resp = api_v4.fetch_user_or_me(
-            user_access_token=user_items["user_upload_token"]
-        )
-    except requests.HTTPError as ex:
-        if api_v4.is_auth_error(ex.response):
-            message = api_v4.extract_auth_error_message(ex.response)
-            raise exceptions.MapillaryUploadUnauthorizedError(message)
-        else:
-            raise ex
+    with api_v4.create_user_session(user_items["user_upload_token"]) as session:
+        try:
+            resp = api_v4.fetch_user_or_me(session)
+        except requests.HTTPError as ex:
+            if api_v4.is_auth_error(ex.response):
+                message = api_v4.extract_auth_error_message(ex.response)
+                raise exceptions.MapillaryUploadUnauthorizedError(message)
+            else:
+                raise ex
 
-    user_json = resp.json()
+    data = api_v4.jsonify_response(resp)
 
     return {
         **user_items,
-        "MAPSettingsUsername": user_json.get("username"),
-        "MAPSettingsUserKey": user_json.get("id"),
+        "MAPSettingsUsername": data.get("username"),
+        "MAPSettingsUserKey": data.get("id"),
     }
 
 
@@ -275,18 +282,19 @@ def _prompt_login(
             if user_password:
                 break
 
-    try:
-        resp = api_v4.get_upload_token(user_email, user_password)
-    except requests.HTTPError as ex:
-        if not _enabled:
-            raise ex
+    with api_v4.create_client_session() as session:
+        try:
+            resp = api_v4.get_upload_token(session, user_email, user_password)
+        except requests.HTTPError as ex:
+            if not _enabled:
+                raise ex
 
-        if _is_login_retryable(ex):
-            return _prompt_login()
+            if _is_login_retryable(ex):
+                return _prompt_login()
 
-        raise ex
+            raise ex
 
-    data = resp.json()
+    data = api_v4.jsonify_response(resp)
 
     user_items: config.UserItem = {
         "user_upload_token": str(data["access_token"]),

mapillary_tools/commands/__main__.py

@@ -7,6 +7,8 @@ from pathlib import Path
 import requests
 
 from .. import api_v4, constants, exceptions, VERSION
+from ..upload import log_exception
+from ..utils import configure_logger, get_app_name
 from . import (
     authenticate,
     process,
@@ -30,8 +32,8 @@ mapillary_tools_commands = [
 ]
 
 
-# do not use __name__ here is because if you run tools as a module, __name__ will be "__main__"
-LOG = logging.getLogger("mapillary_tools")
+# Root logger of mapillary_tools (not including third-party libraries)
+LOG = logging.getLogger(get_app_name())
 
 
 # Handle shared arguments/options here
@@ -78,13 +80,6 @@ def add_general_arguments(parser, command):
         )
 
 
-def configure_logger(logger: logging.Logger, stream=None) -> None:
-    formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s")
-    handler = logging.StreamHandler(stream)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-
-
 def _log_params(argvars: dict) -> None:
     MAX_ENTRIES = 5
 
@@ -151,9 +146,7 @@ def main():
 
     args = parser.parse_args()
 
-    log_level = logging.DEBUG if args.verbose else logging.INFO
-    configure_logger(LOG, sys.stderr)
-    LOG.setLevel(log_level)
+    configure_logger(LOG, level=logging.DEBUG if args.verbose else logging.INFO)
 
     LOG.debug("%s", version_text)
     argvars = vars(args)
@@ -162,16 +155,22 @@ def main():
     try:
         args.func(argvars)
     except requests.HTTPError as ex:
-        LOG.error("%s: %s", ex.__class__.__name__, api_v4.readable_http_error(ex))
+        log_exception(ex)
         # TODO: standardize exit codes as exceptions.MapillaryUserError
         sys.exit(16)
 
+    except api_v4.HTTPContentError as ex:
+        log_exception(ex)
+        sys.exit(17)
+
     except exceptions.MapillaryUserError as ex:
-        LOG.error(
-            "%s: %s", ex.__class__.__name__, ex, exc_info=log_level == logging.DEBUG
-        )
+        log_exception(ex)
         sys.exit(ex.exit_code)
 
+    except KeyboardInterrupt:
+        LOG.info("Interrupted by user...")
+        sys.exit(130)
+
 
 if __name__ == "__main__":
     main()

mapillary_tools/commands/upload.py

@@ -8,13 +8,13 @@ from .process import bold_text
 
 class Command:
     name = "upload"
-    help = "upload images to Mapillary"
+    help = "Upload processed data to Mapillary"
 
     @staticmethod
     def add_common_upload_options(group):
         group.add_argument(
             "--user_name",
-            help="The Mapillary user account to upload to. If you only have one account authorized, it will upload to that account by default.",
+            help="The Mapillary user account to upload to.",
             required=False,
         )
         group.add_argument(
@@ -23,9 +23,38 @@ class Command:
             default=None,
             required=False,
         )
+        group.add_argument(
+            "--num_upload_workers",
+            help="Number of concurrent upload workers for uploading images. [default: %(default)s]",
+            default=constants.MAX_IMAGE_UPLOAD_WORKERS,
+            type=int,
+            required=False,
+        )
+        group.add_argument(
+            "--reupload",
+            help="Re-upload data that has already been uploaded.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
         group.add_argument(
             "--dry_run",
-            help='Instead of uploading to the Mapillary server, simulate uploading to the local directory "mapillary_public_uploads" for debugging purposes.',
+            "--dryrun",
+            help="[DEVELOPMENT] Simulate upload by sending data to a local directory instead of Mapillary servers. Uses a temporary directory by default unless specified by MAPILLARY_UPLOAD_ENDPOINT environment variable.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--nofinish",
+            help="[DEVELOPMENT] Upload data without finalizing. The data will NOT be stored permanently or appear on the Mapillary website.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--noresume",
+            help="[DEVELOPMENT] Start upload from the beginning, ignoring any previously interrupted upload sessions.",
             action="store_true",
             default=False,
             required=False,
@@ -35,7 +64,7 @@ class Command:
         group = parser.add_argument_group(bold_text("UPLOAD OPTIONS"))
         group.add_argument(
             "--desc_path",
-            help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
+            help=f'Path to the description file with processed image and video metadata (from process command). Use "-" for STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
             default=None,
             required=False,
         )