mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.13.3"
+VERSION = "0.14.0"

mapillary_tools/api_v4.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+import enum
 import logging
 import os
 import ssl
@@ -18,6 +21,24 @@ REQUESTS_TIMEOUT = 60  # 1 minutes
 USE_SYSTEM_CERTS: bool = False
 
 
+class HTTPContentError(Exception):
+    """
+    Raised when the HTTP response is ok (200) but the content is not as expected
+    e.g. not JSON or not a valid response.
+    """
+
+    def __init__(self, message: str, response: requests.Response):
+        self.response = response
+        super().__init__(message)
+
+
+class ClusterFileType(enum.Enum):
+    ZIP = "zip"
+    BLACKVUE = "mly_blackvue_video"
+    CAMM = "mly_camm_video"
+    MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
+
+
 class HTTPSystemCertsAdapter(HTTPAdapter):
     """
     This adapter uses the system's certificate store instead of the certifi module.
@@ -48,29 +69,30 @@ class HTTPSystemCertsAdapter(HTTPAdapter):
 
 
 @T.overload
-def _truncate(s: bytes, limit: int = 512) -> bytes: ...
+def _truncate(s: bytes, limit: int = 256) -> bytes | str: ...
 
 
 @T.overload
-def _truncate(s: str, limit: int = 512) -> str: ...
+def _truncate(s: str, limit: int = 256) -> str: ...
 
 
-def _truncate(s, limit=512):
+def _truncate(s, limit=256):
     if limit < len(s):
+        if isinstance(s, bytes):
+            try:
+                s = s.decode("utf-8")
+            except UnicodeDecodeError:
+                pass
         remaining = len(s) - limit
         if isinstance(s, bytes):
-            return (
-                s[:limit]
-                + b"..."
-                + f"({remaining} more bytes truncated)".encode("utf-8")
-            )
+            return s[:limit] + f"...({remaining} bytes truncated)".encode("utf-8")
         else:
-            return str(s[:limit]) + f"...({remaining} more chars truncated)"
+            return str(s[:limit]) + f"...({remaining} chars truncated)"
     else:
         return s
 
 
-def _sanitize(headers: T.Dict):
+def _sanitize(headers: T.Mapping[T.Any, T.Any]) -> T.Mapping[T.Any, T.Any]:
     new_headers = {}
 
     for k, v in headers.items():
@@ -81,10 +103,14 @@ def _sanitize(headers: T.Dict):
             "access-token",
             "access_token",
             "password",
+            "user_upload_token",
         ]:
             new_headers[k] = "[REDACTED]"
         else:
-            new_headers[k] = _truncate(v)
+            if isinstance(v, (str, bytes)):
+                new_headers[k] = T.cast(T.Any, _truncate(v))
+            else:
+                new_headers[k] = v
 
     return new_headers
 
@@ -92,9 +118,9 @@ def _sanitize(headers: T.Dict):
 def _log_debug_request(
     method: str,
     url: str,
-    json: T.Optional[T.Dict] = None,
-    params: T.Optional[T.Dict] = None,
-    headers: T.Optional[T.Dict] = None,
+    json: dict | None = None,
+    params: dict | None = None,
+    headers: dict | None = None,
     timeout: T.Any = None,
 ):
     if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
@@ -118,6 +144,8 @@ def _log_debug_request(
     if timeout is not None:
         msg += f" TIMEOUT={timeout}"
 
+    msg = msg.replace("\n", "\\n")
+
     LOG.debug(msg)
 
 
@@ -125,44 +153,61 @@ def _log_debug_response(resp: requests.Response):
     if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
         return
 
-    data: T.Union[str, bytes]
+    elapsed = resp.elapsed.total_seconds() * 1000  # Convert to milliseconds
+    msg = f"HTTP {resp.status_code} {resp.reason} ({elapsed:.0f} ms): {str(_truncate_response_content(resp))}"
+
+    LOG.debug(msg)
+
+
+def _truncate_response_content(resp: requests.Response) -> str | bytes:
     try:
-        data = _truncate(dumps(_sanitize(resp.json())))
-    except Exception:
-        data = _truncate(resp.content)
+        json_data = resp.json()
+    except requests.JSONDecodeError:
+        if resp.content is not None:
+            data = _truncate(resp.content)
+        else:
+            data = ""
+    else:
+        if isinstance(json_data, dict):
+            data = _truncate(dumps(_sanitize(json_data)))
+        else:
+            data = _truncate(str(json_data))
+
+    if isinstance(data, bytes):
+        return data.replace(b"\n", b"\\n")
 
-    LOG.debug(f"HTTP {resp.status_code} ({resp.reason}): %s", data)
+    elif isinstance(data, str):
+        return data.replace("\n", "\\n")
+
+    return data
 
 
 def readable_http_error(ex: requests.HTTPError) -> str:
-    req = ex.request
-    resp = ex.response
+    return readable_http_response(ex.response)
 
-    data: T.Union[str, bytes]
-    try:
-        data = _truncate(dumps(_sanitize(resp.json())))
-    except Exception:
-        data = _truncate(resp.content)
 
-    return f"{req.method} {resp.url} => {resp.status_code} ({resp.reason}): {str(data)}"
+def readable_http_response(resp: requests.Response) -> str:
+    return f"{resp.request.method} {resp.url} => {resp.status_code} {resp.reason}: {str(_truncate_response_content(resp))}"
 
 
 def request_post(
     url: str,
-    data: T.Optional[T.Any] = None,
-    json: T.Optional[dict] = None,
+    data: T.Any | None = None,
+    json: dict | None = None,
+    disable_debug=False,
     **kwargs,
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
-    _log_debug_request(
-        "POST",
-        url,
-        json=json,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
+    if not disable_debug:
+        _log_debug_request(
+            "POST",
+            url,
+            json=json,
+            params=kwargs.get("params"),
+            headers=kwargs.get("headers"),
+            timeout=kwargs.get("timeout"),
+        )
 
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
@@ -182,25 +227,26 @@ def request_post(
             )
             return request_post(url, data=data, json=json, **kwargs)
 
-    _log_debug_response(resp)
+    if not disable_debug:
+        _log_debug_response(resp)
 
     return resp
 
 
 def request_get(
-    url: str,
-    params: T.Optional[dict] = None,
-    **kwargs,
+    url: str, params: dict | None = None, disable_debug=False, **kwargs
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
-    _log_debug_request(
-        "GET",
-        url,
-        params=kwargs.get("params"),
-        headers=kwargs.get("headers"),
-        timeout=kwargs.get("timeout"),
-    )
+    if not disable_debug:
+        _log_debug_request(
+            "GET",
+            url,
+            params=kwargs.get("params"),
+            headers=kwargs.get("headers"),
+            # Do not log timeout here as it's always set to REQUESTS_TIMEOUT
+            timeout=None,
+        )
 
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
@@ -219,11 +265,50 @@ def request_get(
             )
             resp = request_get(url, params=params, **kwargs)
 
-    _log_debug_response(resp)
+    if not disable_debug:
+        _log_debug_response(resp)
 
     return resp
 
 
+def is_auth_error(resp: requests.Response) -> bool:
+    if resp.status_code in [401, 403]:
+        return True
+
+    if resp.status_code in [400]:
+        try:
+            error_body = resp.json()
+        except Exception:
+            error_body = {}
+
+        type = error_body.get("debug_info", {}).get("type")
+        if type in ["NotAuthorizedError"]:
+            return True
+
+    return False
+
+
+def extract_auth_error_message(resp: requests.Response) -> str:
+    assert is_auth_error(resp), "has to be an auth error"
+
+    try:
+        error_body = resp.json()
+    except Exception:
+        error_body = {}
+
+    # from Graph APIs
+    message = error_body.get("error", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    # from upload service
+    message = error_body.get("debug_info", {}).get("message")
+    if message is not None:
+        return str(message)
+
+    return resp.text
+
+
 def get_upload_token(email: str, password: str) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
@@ -236,7 +321,7 @@ def get_upload_token(email: str, password: str) -> requests.Response:
 
 
 def fetch_organization(
-    user_access_token: str, organization_id: T.Union[int, str]
+    user_access_token: str, organization_id: int | str
 ) -> requests.Response:
     resp = request_get(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}",
@@ -252,22 +337,80 @@ def fetch_organization(
     return resp
 
 
+def fetch_user_or_me(
+    user_access_token: str, user_id: int | str | None = None
+) -> requests.Response:
+    if user_id is None:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
+    else:
+        url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
+
+    resp = request_get(
+        url,
+        params={
+            "fields": ",".join(["id", "username"]),
+        },
+        headers={
+            "Authorization": f"OAuth {user_access_token}",
+        },
+        timeout=REQUESTS_TIMEOUT,
+    )
+
+    resp.raise_for_status()
+    return resp
+
+
 ActionType = T.Literal[
     "upload_started_upload", "upload_finished_upload", "upload_failed_upload"
 ]
 
 
-def log_event(action_type: ActionType, properties: T.Dict) -> requests.Response:
+def log_event(action_type: ActionType, properties: dict) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging",
-        json={
-            "action_type": action_type,
-            "properties": properties,
-        },
+        json={"action_type": action_type, "properties": properties},
         headers={
             "Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}",
         },
         timeout=REQUESTS_TIMEOUT,
+        disable_debug=True,
+    )
+    resp.raise_for_status()
+    return resp
+
+
+def finish_upload(
+    user_access_token: str,
+    file_handle: str,
+    cluster_filetype: ClusterFileType,
+    organization_id: int | str | None = None,
+) -> requests.Response:
+    data: dict[str, str | int] = {
+        "file_handle": file_handle,
+        "file_type": cluster_filetype.value,
+    }
+    if organization_id is not None:
+        data["organization_id"] = organization_id
+
+    resp = request_post(
+        f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload",
+        headers={
+            "Authorization": f"OAuth {user_access_token}",
+        },
+        json=data,
+        timeout=REQUESTS_TIMEOUT,
     )
+
     resp.raise_for_status()
+
     return resp
+
+
+def jsonify_response(resp: requests.Response) -> T.Any:
+    """
+    Convert the response to JSON, raising HTTPContentError if the response is not JSON.
+    """
+    try:
+        return resp.json()
+    except requests.JSONDecodeError as ex:
+        raise HTTPContentError("Invalid JSON response", resp) from ex