mapillary-tools 0.14.0a2__py3-none-any.whl → 0.14.1__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.14.0a2"
+VERSION = "0.14.1"

mapillary_tools/api_v4.py

@@ -3,12 +3,11 @@ from __future__ import annotations
 import enum
 import logging
 import os
-import ssl
 import typing as T
-from json import dumps
 
 import requests
-from requests.adapters import HTTPAdapter
+
+from . import http
 
 LOG = logging.getLogger(__name__)
 MAPILLARY_CLIENT_TOKEN = os.getenv(
@@ -17,221 +16,40 @@ MAPILLARY_CLIENT_TOKEN = os.getenv(
 MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
     "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
 )
-REQUESTS_TIMEOUT = 60  # 1 minutes
-USE_SYSTEM_CERTS: bool = False
-
-
-class ClusterFileType(enum.Enum):
-    ZIP = "zip"
-    BLACKVUE = "mly_blackvue_video"
-    CAMM = "mly_camm_video"
+REQUESTS_TIMEOUT: float = 60  # 1 minutes
 
 
-class HTTPSystemCertsAdapter(HTTPAdapter):
+class HTTPContentError(Exception):
     """
-    This adapter uses the system's certificate store instead of the certifi module.
-
-    The implementation is based on the project https://pypi.org/project/pip-system-certs/,
-    which has a system-wide effect.
+    Raised when the HTTP response is ok (200) but the content is not as expected
+    e.g. not JSON or not a valid response.
     """
 
-    def init_poolmanager(self, *args, **kwargs):
-        ssl_context = ssl.create_default_context()
-        ssl_context.load_default_certs()
-        kwargs["ssl_context"] = ssl_context
-
-        super().init_poolmanager(*args, **kwargs)
-
-    def cert_verify(self, *args, **kwargs):
-        super().cert_verify(*args, **kwargs)
-
-        # By default Python requests uses the ca_certs from the certifi module
-        # But we want to use the certificate store instead.
-        # By clearing the ca_certs variable we force it to fall back on that behaviour (handled in urllib3)
-        if "conn" in kwargs:
-            conn = kwargs["conn"]
-        else:
-            conn = args[0]
-
-        conn.ca_certs = None
-
-
-@T.overload
-def _truncate(s: bytes, limit: int = 512) -> bytes: ...
-
-
-@T.overload
-def _truncate(s: str, limit: int = 512) -> str: ...
-
-
-def _truncate(s, limit=512):
-    if limit < len(s):
-        remaining = len(s) - limit
-        if isinstance(s, bytes):
-            return (
-                s[:limit]
-                + b"..."
-                + f"({remaining} more bytes truncated)".encode("utf-8")
-            )
-        else:
-            return str(s[:limit]) + f"...({remaining} more chars truncated)"
-    else:
-        return s
-
-
-def _sanitize(headers: T.Mapping[T.Any, T.Any]) -> T.Mapping[T.Any, T.Any]:
-    new_headers = {}
-
-    for k, v in headers.items():
-        if k.lower() in [
-            "authorization",
-            "cookie",
-            "x-fb-access-token",
-            "access-token",
-            "access_token",
-            "password",
-            "user_upload_token",
-        ]:
-            new_headers[k] = "[REDACTED]"
-        else:
-            new_headers[k] = _truncate(v)
-
-    return new_headers
-
-
-def _log_debug_request(
-    method: str,
-    url: str,
-    json: dict | None = None,
-    params: dict | None = None,
-    headers: dict | None = None,
-    timeout: T.Any = None,
-):
-    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
-        return
-
-    msg = f"HTTP {method} {url}"
-
-    if USE_SYSTEM_CERTS:
-        msg += " (w/sys_certs)"
-
-    if json:
-        t = _truncate(dumps(_sanitize(json)))
-        msg += f" JSON={t}"
-
-    if params:
-        msg += f" PARAMS={_sanitize(params)}"
-
-    if headers:
-        msg += f" HEADERS={_sanitize(headers)}"
-
-    if timeout is not None:
-        msg += f" TIMEOUT={timeout}"
-
-    LOG.debug(msg)
-
-
-def _log_debug_response(resp: requests.Response):
-    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
-        return
-
-    data: str | bytes
-    try:
-        data = _truncate(dumps(_sanitize(resp.json())))
-    except Exception:
-        data = _truncate(resp.content)
+    def __init__(self, message: str, response: requests.Response):
+        self.response = response
+        super().__init__(message)
 
-    LOG.debug(f"HTTP {resp.status_code} ({resp.reason}): %s", data)
 
+class ClusterFileType(enum.Enum):
+    ZIP = "zip"
+    BLACKVUE = "mly_blackvue_video"
+    CAMM = "mly_camm_video"
+    MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
 
-def readable_http_error(ex: requests.HTTPError) -> str:
-    req = ex.request
-    resp = ex.response
-
-    data: str | bytes
-    try:
-        data = _truncate(dumps(_sanitize(resp.json())))
-    except Exception:
-        data = _truncate(resp.content)
-
-    return f"{req.method} {resp.url} => {resp.status_code} ({resp.reason}): {str(data)}"
-
-
-def request_post(
-    url: str,
-    data: T.Any | None = None,
-    json: dict | None = None,
-    **kwargs,
-) -> requests.Response:
-    global USE_SYSTEM_CERTS
-
-    _log_debug_request(
-        "POST",
-        url,
-        json=json,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
-
-    if USE_SYSTEM_CERTS:
-        with requests.Session() as session:
-            session.mount("https://", HTTPSystemCertsAdapter())
-            resp = session.post(url, data=data, json=json, **kwargs)
-
-    else:
-        try:
-            resp = requests.post(url, data=data, json=json, **kwargs)
-        except requests.exceptions.SSLError as ex:
-            if "SSLCertVerificationError" not in str(ex):
-                raise ex
-            USE_SYSTEM_CERTS = True
-            # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
-            LOG.warning(
-                "SSL error occurred, falling back to system SSL certificates: %s", ex
-            )
-            return request_post(url, data=data, json=json, **kwargs)
-
-    _log_debug_response(resp)
-
-    return resp
 
+def create_user_session(user_access_token: str) -> requests.Session:
+    session = http.Session()
+    session.headers["Authorization"] = f"OAuth {user_access_token}"
+    return session
 
-def request_get(
-    url: str,
-    params: dict | None = None,
-    **kwargs,
-) -> requests.Response:
-    global USE_SYSTEM_CERTS
-
-    _log_debug_request(
-        "GET",
-        url,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
-
-    if USE_SYSTEM_CERTS:
-        with requests.Session() as session:
-            session.mount("https://", HTTPSystemCertsAdapter())
-            resp = session.get(url, params=params, **kwargs)
-    else:
-        try:
-            resp = requests.get(url, params=params, **kwargs)
-        except requests.exceptions.SSLError as ex:
-            if "SSLCertVerificationError" not in str(ex):
-                raise ex
-            USE_SYSTEM_CERTS = True
-            # HTTPSConnectionPool(host='graph.mapillary.com', port=443): Max retries exceeded with url: /login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1018)')))
-            LOG.warning(
-                "SSL error occurred, falling back to system SSL certificates: %s", ex
-            )
-            resp = request_get(url, params=params, **kwargs)
-
-    _log_debug_response(resp)
 
-    return resp
+def create_client_session(disable_logging: bool = False) -> requests.Session:
+    session = http.Session()
+    session.headers["Authorization"] = f"OAuth {MAPILLARY_CLIENT_TOKEN}"
+    if disable_logging:
+        session.disable_logging_request = True
+        session.disable_logging_response = True
+    return session
 
 
 def is_auth_error(resp: requests.Response) -> bool:
@@ -272,55 +90,42 @@ def extract_auth_error_message(resp: requests.Response) -> str:
     return resp.text
 
 
-def get_upload_token(email: str, password: str) -> requests.Response:
-    resp = request_post(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
-        headers={"Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}"},
-        json={"email": email, "password": password, "locale": "en_US"},
-        timeout=REQUESTS_TIMEOUT,
-    )
+def get_upload_token(
+    client_session: requests.Session, email: str, password: str
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/login"
+    json_data = {"email": email, "password": password, "locale": "en_US"}
+
+    resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
 def fetch_organization(
-    user_access_token: str, organization_id: int | str
+    user_session: requests.Session, organization_id: int | str
 ) -> requests.Response:
-    resp = request_get(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}",
-        params={
-            "fields": ",".join(["slug", "description", "name"]),
-        },
-        headers={
-            "Authorization": f"OAuth {user_access_token}",
-        },
-        timeout=REQUESTS_TIMEOUT,
-    )
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}"
+    params = {"fields": ",".join(["slug", "description", "name"])}
+
+    resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
 def fetch_user_or_me(
-    user_access_token: str,
-    user_id: int | str | None = None,
+    user_session: requests.Session, user_id: int | str | None = None
 ) -> requests.Response:
     if user_id is None:
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
     else:
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
+    params = {"fields": ",".join(["id", "username"])}
 
-    resp = request_get(
-        url,
-        params={
-            "fields": ",".join(["id", "username"]),
-        },
-        headers={
-            "Authorization": f"OAuth {user_access_token}",
-        },
-        timeout=REQUESTS_TIMEOUT,
-    )
-
+    resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
@@ -329,44 +134,43 @@ ActionType = T.Literal[
 ]
 
 
-def log_event(action_type: ActionType, properties: dict) -> requests.Response:
-    resp = request_post(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging",
-        json={
-            "action_type": action_type,
-            "properties": properties,
-        },
-        headers={
-            "Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}",
-        },
-        timeout=REQUESTS_TIMEOUT,
-    )
+def log_event(
+    client_session: requests.Session, action_type: ActionType, properties: dict
+) -> requests.Response:
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging"
+    json_data = {"action_type": action_type, "properties": properties}
+
+    resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
+
     return resp
 
 
 def finish_upload(
-    user_access_token: str,
+    user_session: requests.Session,
     file_handle: str,
     cluster_filetype: ClusterFileType,
     organization_id: int | str | None = None,
 ) -> requests.Response:
-    data: dict[str, str | int] = {
+    url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
+    json_data: dict[str, str | int] = {
         "file_handle": file_handle,
         "file_type": cluster_filetype.value,
     }
     if organization_id is not None:
-        data["organization_id"] = organization_id
-
-    resp = request_post(
-        f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload",
-        headers={
-            "Authorization": f"OAuth {user_access_token}",
-        },
-        json=data,
-        timeout=REQUESTS_TIMEOUT,
-    )
+        json_data["organization_id"] = organization_id
 
+    resp = user_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
     resp.raise_for_status()
 
     return resp
+
+
+def jsonify_response(resp: requests.Response) -> T.Any:
+    """
+    Convert the response to JSON, raising HTTPContentError if the response is not JSON.
+    """
+    try:
+        return resp.json()
+    except requests.JSONDecodeError as ex:
+        raise HTTPContentError("Invalid JSON response", resp) from ex

mapillary_tools/authenticate.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import getpass
-import json
 import logging
 import re
 import sys
@@ -11,7 +10,7 @@ import jsonschema
 
 import requests
 
-from . import api_v4, config, constants, exceptions, types
+from . import api_v4, config, constants, exceptions, http
 
 
 LOG = logging.getLogger(__name__)
@@ -64,7 +63,7 @@ def authenticate(
             LOG.info('Creating new profile: "%s"', profile_name)
 
         if jwt:
-            user_items: types.UserItem = {"user_upload_token": jwt}
+            user_items: config.UserItem = {"user_upload_token": jwt}
             user_items = _verify_user_auth(_validate_profile(user_items))
         else:
             user_items = _prompt_login(
@@ -78,18 +77,17 @@ def authenticate(
         # TODO: print more user information
         if profile_name in all_user_items:
             LOG.info(
-                'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items)
+                'Profile "%s" updated: %s', profile_name, http._sanitize(user_items)
             )
         else:
             LOG.info(
-                'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items)
+                'Profile "%s" created: %s', profile_name, http._sanitize(user_items)
             )
 
 
 def fetch_user_items(
-    user_name: str | None = None,
-    organization_key: str | None = None,
-) -> types.UserItem:
+    user_name: str | None = None, organization_key: str | None = None
+) -> config.UserItem:
     """
     Read user information from the config file,
     or prompt the user to authenticate if the specified profile does not exist
@@ -129,18 +127,28 @@ def fetch_user_items(
 
     assert profile_name is not None, "profile_name should be set"
 
-    user_items = _verify_user_auth(_validate_profile(user_items))
-
-    LOG.info(
-        'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items)
-    )
+    try:
+        LOG.info(f'Verifying profile "{profile_name}"...')
+        user_items = _verify_user_auth(_validate_profile(user_items))
 
-    if organization_key is not None:
-        resp = api_v4.fetch_organization(
-            user_items["user_upload_token"], organization_key
+        LOG.info(
+            f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
         )
-        LOG.info("Uploading to Mapillary organization: %s", json.dumps(resp.json()))
-        user_items["MAPOrganizationKey"] = organization_key
+
+        if organization_key is not None:
+            with api_v4.create_user_session(user_items["user_upload_token"]) as session:
+                resp = api_v4.fetch_organization(session, organization_key)
+            data = api_v4.jsonify_response(resp)
+            LOG.info(
+                f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})"
+            )
+            user_items["MAPOrganizationKey"] = data.get("id")
+
+    except requests.Timeout as ex:
+        raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex
+
+    except requests.ConnectionError as ex:
+        raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex
 
     return user_items
 
@@ -155,9 +163,9 @@ def _prompt(message: str) -> str:
     return input()
 
 
-def _validate_profile(user_items: types.UserItem) -> types.UserItem:
+def _validate_profile(user_items: config.UserItem) -> config.UserItem:
     try:
-        jsonschema.validate(user_items, types.UserItemSchema)
+        jsonschema.validate(user_items, config.UserItemSchema)
     except jsonschema.ValidationError as ex:
         raise exceptions.MapillaryBadParameterError(
             f"Invalid profile format: {ex.message}"
@@ -165,30 +173,29 @@ def _validate_profile(user_items: types.UserItem) -> types.UserItem:
     return user_items
 
 
-def _verify_user_auth(user_items: types.UserItem) -> types.UserItem:
+def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
     """
     Verify that the user access token is valid
     """
     if constants._AUTH_VERIFICATION_DISABLED:
         return user_items
 
-    try:
-        resp = api_v4.fetch_user_or_me(
-            user_access_token=user_items["user_upload_token"]
-        )
-    except requests.HTTPError as ex:
-        if api_v4.is_auth_error(ex.response):
-            message = api_v4.extract_auth_error_message(ex.response)
-            raise exceptions.MapillaryUploadUnauthorizedError(message)
-        else:
-            raise ex
+    with api_v4.create_user_session(user_items["user_upload_token"]) as session:
+        try:
+            resp = api_v4.fetch_user_or_me(session)
+        except requests.HTTPError as ex:
+            if api_v4.is_auth_error(ex.response):
+                message = api_v4.extract_auth_error_message(ex.response)
+                raise exceptions.MapillaryUploadUnauthorizedError(message)
+            else:
+                raise ex
 
-    user_json = resp.json()
+    data = api_v4.jsonify_response(resp)
 
     return {
         **user_items,
-        "MAPSettingsUsername": user_json.get("username"),
-        "MAPSettingsUserKey": user_json.get("id"),
+        "MAPSettingsUsername": data.get("username"),
+        "MAPSettingsUserKey": data.get("id"),
     }
 
 
@@ -205,7 +212,7 @@ def _validate_profile_name(profile_name: str):
         )
 
 
-def _list_all_profiles(profiles: dict[str, types.UserItem]) -> None:
+def _list_all_profiles(profiles: dict[str, config.UserItem]) -> None:
     _echo("Existing Mapillary profiles:")
 
     # Header
@@ -256,7 +263,7 @@ def _is_login_retryable(ex: requests.HTTPError) -> bool:
 def _prompt_login(
     user_email: str | None = None,
     user_password: str | None = None,
-) -> types.UserItem:
+) -> config.UserItem:
     _enabled = _prompt_enabled()
 
     if user_email is None:
@@ -275,20 +282,21 @@ def _prompt_login(
             if user_password:
                 break
 
-    try:
-        resp = api_v4.get_upload_token(user_email, user_password)
-    except requests.HTTPError as ex:
-        if not _enabled:
-            raise ex
+    with api_v4.create_client_session() as session:
+        try:
+            resp = api_v4.get_upload_token(session, user_email, user_password)
+        except requests.HTTPError as ex:
+            if not _enabled:
+                raise ex
 
-        if _is_login_retryable(ex):
-            return _prompt_login()
+            if _is_login_retryable(ex):
+                return _prompt_login()
 
-        raise ex
+            raise ex
 
-    data = resp.json()
+    data = api_v4.jsonify_response(resp)
 
-    user_items: types.UserItem = {
+    user_items: config.UserItem = {
         "user_upload_token": str(data["access_token"]),
         "MAPSettingsUserKey": str(data["user_id"]),
     }