mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.13.3a1"
+VERSION = "0.14.0a2"

mapillary_tools/api_v4.py

@@ -1,7 +1,11 @@
+from __future__ import annotations
+
+import enum
 import logging
 import os
 import ssl
 import typing as T
+from json import dumps
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -17,6 +21,12 @@ REQUESTS_TIMEOUT = 60  # 1 minutes
 USE_SYSTEM_CERTS: bool = False
 
 
+class ClusterFileType(enum.Enum):
+    ZIP = "zip"
+    BLACKVUE = "mly_blackvue_video"
+    CAMM = "mly_camm_video"
+
+
 class HTTPSystemCertsAdapter(HTTPAdapter):
     """
     This adapter uses the system's certificate store instead of the certifi module.
@@ -46,22 +56,132 @@ class HTTPSystemCertsAdapter(HTTPAdapter):
         conn.ca_certs = None
 
 
+@T.overload
+def _truncate(s: bytes, limit: int = 512) -> bytes: ...
+
+
+@T.overload
+def _truncate(s: str, limit: int = 512) -> str: ...
+
+
+def _truncate(s, limit=512):
+    if limit < len(s):
+        remaining = len(s) - limit
+        if isinstance(s, bytes):
+            return (
+                s[:limit]
+                + b"..."
+                + f"({remaining} more bytes truncated)".encode("utf-8")
+            )
+        else:
+            return str(s[:limit]) + f"...({remaining} more chars truncated)"
+    else:
+        return s
+
+
+def _sanitize(headers: T.Mapping[T.Any, T.Any]) -> T.Mapping[T.Any, T.Any]:
+    new_headers = {}
+
+    for k, v in headers.items():
+        if k.lower() in [
+            "authorization",
+            "cookie",
+            "x-fb-access-token",
+            "access-token",
+            "access_token",
+            "password",
+            "user_upload_token",
+        ]:
+            new_headers[k] = "[REDACTED]"
+        else:
+            new_headers[k] = _truncate(v)
+
+    return new_headers
+
+
+def _log_debug_request(
+    method: str,
+    url: str,
+    json: dict | None = None,
+    params: dict | None = None,
+    headers: dict | None = None,
+    timeout: T.Any = None,
+):
+    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
+        return
+
+    msg = f"HTTP {method} {url}"
+
+    if USE_SYSTEM_CERTS:
+        msg += " (w/sys_certs)"
+
+    if json:
+        t = _truncate(dumps(_sanitize(json)))
+        msg += f" JSON={t}"
+
+    if params:
+        msg += f" PARAMS={_sanitize(params)}"
+
+    if headers:
+        msg += f" HEADERS={_sanitize(headers)}"
+
+    if timeout is not None:
+        msg += f" TIMEOUT={timeout}"
+
+    LOG.debug(msg)
+
+
+def _log_debug_response(resp: requests.Response):
+    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
+        return
+
+    data: str | bytes
+    try:
+        data = _truncate(dumps(_sanitize(resp.json())))
+    except Exception:
+        data = _truncate(resp.content)
+
+    LOG.debug(f"HTTP {resp.status_code} ({resp.reason}): %s", data)
+
+
+def readable_http_error(ex: requests.HTTPError) -> str:
+    req = ex.request
+    resp = ex.response
+
+    data: str | bytes
+    try:
+        data = _truncate(dumps(_sanitize(resp.json())))
+    except Exception:
+        data = _truncate(resp.content)
+
+    return f"{req.method} {resp.url} => {resp.status_code} ({resp.reason}): {str(data)}"
+
+
 def request_post(
     url: str,
-    data: T.Optional[T.Any] = None,
-    json: T.Optional[dict] = None,
+    data: T.Any | None = None,
+    json: dict | None = None,
     **kwargs,
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
+    _log_debug_request(
+        "POST",
+        url,
+        json=json,
+        params=kwargs.get("params"),
+        headers=kwargs.get("headers"),
+        timeout=kwargs.get("timeout"),
+    )
+
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
             session.mount("https://", HTTPSystemCertsAdapter())
-            return session.post(url, data=data, json=json, **kwargs)
+            resp = session.post(url, data=data, json=json, **kwargs)
 
     else:
         try:
-            return requests.post(url, data=data, json=json, **kwargs)
+            resp = requests.post(url, data=data, json=json, **kwargs)
         except requests.exceptions.SSLError as ex:
             if "SSLCertVerificationError" not in str(ex):
                 raise ex
@@ -70,25 +190,35 @@ def request_post(
             LOG.warning(
                 "SSL error occurred, falling back to system SSL certificates: %s", ex
             )
-            with requests.Session() as session:
-                session.mount("https://", HTTPSystemCertsAdapter())
-                return session.post(url, data=data, json=json, **kwargs)
+            return request_post(url, data=data, json=json, **kwargs)
+
+    _log_debug_response(resp)
+
+    return resp
 
 
 def request_get(
     url: str,
-    params: T.Optional[dict] = None,
+    params: dict | None = None,
     **kwargs,
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
+    _log_debug_request(
+        "GET",
+        url,
+        params=kwargs.get("params"),
+        headers=kwargs.get("headers"),
+        timeout=kwargs.get("timeout"),
+    )
+
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
             session.mount("https://", HTTPSystemCertsAdapter())
-            return session.get(url, params=params, **kwargs)
+            resp = session.get(url, params=params, **kwargs)
     else:
         try:
-            return requests.get(url, params=params, **kwargs)
+            resp = requests.get(url, params=params, **kwargs)
         except requests.exceptions.SSLError as ex:
             if "SSLCertVerificationError" not in str(ex):
                 raise ex
@@ -97,15 +227,55 @@ def request_get(
             LOG.warning(
                 "SSL error occurred, falling back to system SSL certificates: %s", ex
             )
-            with requests.Session() as session:
-                session.mount("https://", HTTPSystemCertsAdapter())
-                return session.get(url, params=params, **kwargs)
+            resp = request_get(url, params=params, **kwargs)
+
+    _log_debug_response(resp)
+
+    return resp
+
+
+def is_auth_error(resp: requests.Response) -> bool:
+    if resp.status_code in [401, 403]:
+        return True
+
+    if resp.status_code in [400]:
+        try:
+            error_body = resp.json()
+        except Exception:
+            error_body = {}
+
+        type = error_body.get("debug_info", {}).get("type")
+        if type in ["NotAuthorizedError"]:
+            return True
+
+    return False
+
+
+def extract_auth_error_message(resp: requests.Response) -> str:
+    assert is_auth_error(resp), "has to be an auth error"
+
+    try:
+        error_body = resp.json()
+    except Exception:
+        error_body = {}
+
+    # from Graph APIs
+    message = error_body.get("error", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    # from upload service
+    message = error_body.get("debug_info", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    return resp.text
 
 
 def get_upload_token(email: str, password: str) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
-        params={"access_token": MAPILLARY_CLIENT_TOKEN},
+        headers={"Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}"},
         json={"email": email, "password": password, "locale": "en_US"},
         timeout=REQUESTS_TIMEOUT,
     )
@@ -114,7 +284,7 @@ def get_upload_token(email: str, password: str) -> requests.Response:
 
 
 def fetch_organization(
-    user_access_token: str, organization_id: T.Union[int, str]
+    user_access_token: str, organization_id: int | str
 ) -> requests.Response:
     resp = request_get(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}",
@@ -130,12 +300,36 @@ def fetch_organization(
     return resp
 
 
+def fetch_user_or_me(
+    user_access_token: str,
+    user_id: int | str | None = None,
+) -> requests.Response:
+    if user_id is None:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
+    else:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
+
+    resp = request_get(
+        url,
+        params={
+            "fields": ",".join(["id", "username"]),
+        },
+        headers={
+            "Authorization": f"OAuth {user_access_token}",
+        },
+        timeout=REQUESTS_TIMEOUT,
+    )
+
+    resp.raise_for_status()
+    return resp
+
+
 ActionType = T.Literal[
     "upload_started_upload", "upload_finished_upload", "upload_failed_upload"
 ]
 
 
-def log_event(action_type: ActionType, properties: T.Dict) -> requests.Response:
+def log_event(action_type: ActionType, properties: dict) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging",
         json={
@@ -149,3 +343,30 @@ def log_event(action_type: ActionType, properties: T.Dict) -> requests.Response:
     )
     resp.raise_for_status()
     return resp
+
+
+def finish_upload(
+    user_access_token: str,
+    file_handle: str,
+    cluster_filetype: ClusterFileType,
+    organization_id: int | str | None = None,
+) -> requests.Response:
+    data: dict[str, str | int] = {
+        "file_handle": file_handle,
+        "file_type": cluster_filetype.value,
+    }
+    if organization_id is not None:
+        data["organization_id"] = organization_id
+
+    resp = request_post(
+        f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload",
+        headers={
+            "Authorization": f"OAuth {user_access_token}",
+        },
+        json=data,
+        timeout=REQUESTS_TIMEOUT,
+    )
+
+    resp.raise_for_status()
+
+    return resp