mapillary-tools 0.13.2__tar.gz → 0.13.3__tar.gz

{mapillary_tools-0.13.2/mapillary_tools.egg-info → mapillary_tools-0.13.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mapillary_tools
-Version: 0.13.2
+Version: 0.13.3
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary

mapillary_tools-0.13.3/mapillary_tools/__init__.py

@@ -0,0 +1 @@
+VERSION = "0.13.3"

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3}/mapillary_tools/api_v4.py

@@ -2,6 +2,7 @@ import logging
 import os
 import ssl
 import typing as T
+from json import dumps
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -46,6 +47,106 @@ class HTTPSystemCertsAdapter(HTTPAdapter):
         conn.ca_certs = None
 
 
+@T.overload
+def _truncate(s: bytes, limit: int = 512) -> bytes: ...
+
+
+@T.overload
+def _truncate(s: str, limit: int = 512) -> str: ...
+
+
+def _truncate(s, limit=512):
+    if limit < len(s):
+        remaining = len(s) - limit
+        if isinstance(s, bytes):
+            return (
+                s[:limit]
+                + b"..."
+                + f"({remaining} more bytes truncated)".encode("utf-8")
+            )
+        else:
+            return str(s[:limit]) + f"...({remaining} more chars truncated)"
+    else:
+        return s
+
+
+def _sanitize(headers: T.Dict):
+    new_headers = {}
+
+    for k, v in headers.items():
+        if k.lower() in [
+            "authorization",
+            "cookie",
+            "x-fb-access-token",
+            "access-token",
+            "access_token",
+            "password",
+        ]:
+            new_headers[k] = "[REDACTED]"
+        else:
+            new_headers[k] = _truncate(v)
+
+    return new_headers
+
+
+def _log_debug_request(
+    method: str,
+    url: str,
+    json: T.Optional[T.Dict] = None,
+    params: T.Optional[T.Dict] = None,
+    headers: T.Optional[T.Dict] = None,
+    timeout: T.Any = None,
+):
+    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
+        return
+
+    msg = f"HTTP {method} {url}"
+
+    if USE_SYSTEM_CERTS:
+        msg += " (w/sys_certs)"
+
+    if json:
+        t = _truncate(dumps(_sanitize(json)))
+        msg += f" JSON={t}"
+
+    if params:
+        msg += f" PARAMS={_sanitize(params)}"
+
+    if headers:
+        msg += f" HEADERS={_sanitize(headers)}"
+
+    if timeout is not None:
+        msg += f" TIMEOUT={timeout}"
+
+    LOG.debug(msg)
+
+
+def _log_debug_response(resp: requests.Response):
+    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
+        return
+
+    data: T.Union[str, bytes]
+    try:
+        data = _truncate(dumps(_sanitize(resp.json())))
+    except Exception:
+        data = _truncate(resp.content)
+
+    LOG.debug(f"HTTP {resp.status_code} ({resp.reason}): %s", data)
+
+
+def readable_http_error(ex: requests.HTTPError) -> str:
+    req = ex.request
+    resp = ex.response
+
+    data: T.Union[str, bytes]
+    try:
+        data = _truncate(dumps(_sanitize(resp.json())))
+    except Exception:
+        data = _truncate(resp.content)
+
+    return f"{req.method} {resp.url} => {resp.status_code} ({resp.reason}): {str(data)}"
+
+
 def request_post(
     url: str,
     data: T.Optional[T.Any] = None,
@@ -54,14 +155,23 @@ def request_post(
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
+    _log_debug_request(
+        "POST",
+        url,
+        json=json,
+        params=kwargs.get("params"),
+        headers=kwargs.get("headers"),
+        timeout=kwargs.get("timeout"),
+    )
+
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
             session.mount("https://", HTTPSystemCertsAdapter())
-            return session.post(url, data=data, json=json, **kwargs)
+            resp = session.post(url, data=data, json=json, **kwargs)
 
     else:
         try:
-            return requests.post(url, data=data, json=json, **kwargs)
+            resp = requests.post(url, data=data, json=json, **kwargs)
         except requests.exceptions.SSLError as ex:
             if "SSLCertVerificationError" not in str(ex):
                 raise ex
@@ -70,9 +180,11 @@ def request_post(
             LOG.warning(
                 "SSL error occurred, falling back to system SSL certificates: %s", ex
             )
-            with requests.Session() as session:
-                session.mount("https://", HTTPSystemCertsAdapter())
-                return session.post(url, data=data, json=json, **kwargs)
+            return request_post(url, data=data, json=json, **kwargs)
+
+    _log_debug_response(resp)
+
+    return resp
 
 
 def request_get(
@@ -82,13 +194,21 @@ def request_get(
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
+    _log_debug_request(
+        "GET",
+        url,
+        params=kwargs.get("params"),
+        headers=kwargs.get("headers"),
+        timeout=kwargs.get("timeout"),
+    )
+
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
             session.mount("https://", HTTPSystemCertsAdapter())
-            return session.get(url, params=params, **kwargs)
+            resp = session.get(url, params=params, **kwargs)
     else:
         try:
-            return requests.get(url, params=params, **kwargs)
+            resp = requests.get(url, params=params, **kwargs)
         except requests.exceptions.SSLError as ex:
             if "SSLCertVerificationError" not in str(ex):
                 raise ex
@@ -97,15 +217,17 @@ def request_get(
             LOG.warning(
                 "SSL error occurred, falling back to system SSL certificates: %s", ex
             )
-            with requests.Session() as session:
-                session.mount("https://", HTTPSystemCertsAdapter())
-                return session.get(url, params=params, **kwargs)
+            resp = request_get(url, params=params, **kwargs)
+
+    _log_debug_response(resp)
+
+    return resp
 
 
 def get_upload_token(email: str, password: str) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
-        params={"access_token": MAPILLARY_CLIENT_TOKEN},
+        headers={"Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}"},
         json={"email": email, "password": password, "locale": "en_US"},
         timeout=REQUESTS_TIMEOUT,
     )

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3}/mapillary_tools/commands/__main__.py

@@ -5,7 +5,9 @@ import sys
 import typing as T
 from pathlib import Path
 
-from .. import constants, exceptions, VERSION
+import requests
+
+from .. import api_v4, constants, exceptions, VERSION
 from . import (
     authenticate,
     process,
@@ -160,11 +162,14 @@ def main():
 
     try:
         args.func(argvars)
-    except exceptions.MapillaryUserError as exc:
+    except requests.HTTPError as ex:
+        LOG.error("%s: %s", ex.__class__.__name__, api_v4.readable_http_error(ex))
+
+    except exceptions.MapillaryUserError as ex:
         LOG.error(
-            "%s: %s", exc.__class__.__name__, exc, exc_info=log_level == logging.DEBUG
+            "%s: %s", ex.__class__.__name__, ex, exc_info=log_level == logging.DEBUG
         )
-        sys.exit(exc.exit_code)
+        sys.exit(ex.exit_code)
 
 
 if __name__ == "__main__":

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3}/mapillary_tools/commands/process_and_upload.py

@@ -15,5 +15,6 @@ class Command:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
             args["desc_path"] = "\x00"
+
         ProcessCommand().run(args)
         UploadCommand().run(args)

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3}/mapillary_tools/constants.py

@@ -45,6 +45,6 @@ GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
 # Max number of images per sequence
 MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
 # Max file size per sequence (sum of image filesizes in the sequence)
-MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "10G")
+MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
 # Max number of pixels per sequence (sum of image pixels in the sequence)
 MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3}/mapillary_tools/upload.py

@@ -47,25 +47,6 @@ class UploadError(Exception):
         super().__init__(str(inner_ex))
 
 
-class UploadHTTPError(Exception):
-    pass
-
-
-def wrap_http_exception(ex: requests.HTTPError):
-    req = ex.request
-    resp = ex.response
-    if isinstance(resp, requests.Response) and isinstance(req, requests.Request):
-        lines = [
-            f"{req.method} {resp.url}",
-            f"> HTTP Status: {resp.status_code}",
-            str(resp.content),
-        ]
-    else:
-        lines = []
-
-    return UploadHTTPError("\n".join(lines))
-
-
 def _load_validate_metadatas_from_desc_path(
     desc_path: T.Optional[str], import_paths: T.Sequence[Path]
 ) -> T.List[types.Metadata]:
@@ -175,18 +156,12 @@ def fetch_user_items(
                 "Found multiple Mapillary accounts. Please specify one with --user_name"
             )
     else:
-        try:
-            user_items = authenticate.authenticate_user(user_name)
-        except requests.HTTPError as exc:
-            raise wrap_http_exception(exc) from exc
+        user_items = authenticate.authenticate_user(user_name)
 
     if organization_key is not None:
-        try:
-            resp = api_v4.fetch_organization(
-                user_items["user_upload_token"], organization_key
-            )
-        except requests.HTTPError as ex:
-            raise wrap_http_exception(ex) from ex
+        resp = api_v4.fetch_organization(
+            user_items["user_upload_token"], organization_key
+        )
         org = resp.json()
         LOG.info("Uploading to organization: %s", json.dumps(org))
         user_items = T.cast(
@@ -430,15 +405,12 @@ def _api_logging_finished(summary: T.Dict):
     action: api_v4.ActionType = "upload_finished_upload"
     LOG.debug("API Logging for action %s: %s", action, summary)
     try:
-        api_v4.log_event(
-            action,
-            summary,
-        )
+        api_v4.log_event(action, summary)
     except requests.HTTPError as exc:
         LOG.warning(
-            "Error from API Logging for action %s",
+            "HTTPError from API Logging for action %s: %s",
             action,
-            exc_info=wrap_http_exception(exc),
+            api_v4.readable_http_error(exc),
         )
     except Exception:
         LOG.warning("Error from API Logging for action %s", action, exc_info=True)
@@ -452,16 +424,12 @@ def _api_logging_failed(payload: T.Dict, exc: Exception):
     action: api_v4.ActionType = "upload_failed_upload"
     LOG.debug("API Logging for action %s: %s", action, payload)
     try:
-        api_v4.log_event(
-            action,
-            payload_with_reason,
-        )
+        api_v4.log_event(action, payload_with_reason)
     except requests.HTTPError as exc:
-        wrapped_exc = wrap_http_exception(exc)
         LOG.warning(
-            "Error from API Logging for action %s",
+            "HTTPError from API Logging for action %s: %s",
             action,
-            exc_info=wrapped_exc,
+            api_v4.readable_http_error(exc),
         )
     except Exception:
         LOG.warning("Error from API Logging for action %s", action, exc_info=True)
@@ -678,7 +646,7 @@ def upload(
                     raise exceptions.MapillaryUploadUnauthorizedError(
                         debug_info.get("message")
                     ) from inner_ex
-            raise wrap_http_exception(inner_ex) from inner_ex
+            raise inner_ex
 
         raise inner_ex
 

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3}/mapillary_tools/upload_api_v4.py

@@ -1,16 +1,19 @@
 import enum
 import io
-import json
-import logging
 import os
 import random
 import typing as T
+import uuid
 
 import requests
 
-from .api_v4 import MAPILLARY_GRAPH_API_ENDPOINT, request_get, request_post
+from .api_v4 import (
+    MAPILLARY_GRAPH_API_ENDPOINT,
+    request_get,
+    request_post,
+    REQUESTS_TIMEOUT,
+)
 
-LOG = logging.getLogger(__name__)
 MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
     "MAPILLARY_UPLOAD_ENDPOINT", "https://rupload.facebook.com/mapillary_public_uploads"
 )
@@ -21,7 +24,6 @@ DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16  # 16MB
 # i.e. if your the server does not respond within this timeout, it will throw:
 # ConnectionError: ('Connection aborted.', timeout('The write operation timed out'))
 # So let us make sure the largest possible chunks can be uploaded before this timeout for now,
-REQUESTS_TIMEOUT = (20, 20)  # 20 seconds
 UPLOAD_REQUESTS_TIMEOUT = (30 * 60, 30 * 60)  # 30 minutes
 
 
@@ -31,55 +33,33 @@ class ClusterFileType(enum.Enum):
     CAMM = "mly_camm_video"
 
 
-def _sanitize_headers(headers: T.Dict):
-    return {
-        k: v
-        for k, v in headers.items()
-        if k.lower() not in ["authorization", "cookie", "x-fb-access-token"]
-    }
-
-
-_S = T.TypeVar("_S", str, bytes)
-
-
-def _truncate_end(s: _S) -> _S:
-    MAX_LENGTH = 512
-    if MAX_LENGTH < len(s):
-        if isinstance(s, bytes):
-            return s[:MAX_LENGTH] + b"..."
-        else:
-            return str(s[:MAX_LENGTH]) + "..."
-    else:
-        return s
-
-
 class UploadService:
     user_access_token: str
-    entity_size: int
     session_key: str
     callbacks: T.List[T.Callable[[bytes, T.Optional[requests.Response]], None]]
     cluster_filetype: ClusterFileType
     organization_id: T.Optional[T.Union[str, int]]
     chunk_size: int
 
+    MIME_BY_CLUSTER_TYPE: T.Dict[ClusterFileType, str] = {
+        ClusterFileType.ZIP: "application/zip",
+        ClusterFileType.BLACKVUE: "video/mp4",
+        ClusterFileType.CAMM: "video/mp4",
+    }
+
     def __init__(
         self,
         user_access_token: str,
         session_key: str,
-        entity_size: int,
         organization_id: T.Optional[T.Union[str, int]] = None,
         cluster_filetype: ClusterFileType = ClusterFileType.ZIP,
         chunk_size: int = DEFAULT_CHUNK_SIZE,
     ):
-        if entity_size <= 0:
-            raise ValueError(f"Expect positive entity size but got {entity_size}")
-
         if chunk_size <= 0:
             raise ValueError("Expect positive chunk size")
 
         self.user_access_token = user_access_token
         self.session_key = session_key
-        self.entity_size = entity_size
         self.organization_id = organization_id
         #  validate the input
         self.cluster_filetype = ClusterFileType(cluster_filetype)
@@ -91,13 +71,11 @@ class UploadService:
             "Authorization": f"OAuth {self.user_access_token}",
         }
         url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-        LOG.debug("GET %s", url)
         resp = request_get(
             url,
             headers=headers,
             timeout=REQUESTS_TIMEOUT,
         )
-        LOG.debug("HTTP response %s: %s", resp.status_code, resp.content)
         resp.raise_for_status()
         data = resp.json()
         return data["offset"]
@@ -107,54 +85,63 @@ class UploadService:
         data: T.IO[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
-        if offset is None:
-            offset = self.fetch_offset()
-
-        entity_type_map: T.Dict[ClusterFileType, str] = {
-            ClusterFileType.ZIP: "application/zip",
-            ClusterFileType.BLACKVUE: "video/mp4",
-            ClusterFileType.CAMM: "video/mp4",
-        }
-
-        entity_type = entity_type_map[self.cluster_filetype]
-
-        data.seek(offset, io.SEEK_CUR)
+        chunks = self._chunkize_byte_stream(data)
+        return self.upload_chunks(chunks, offset=offset)
 
+    def _chunkize_byte_stream(
+        self, stream: T.IO[bytes]
+    ) -> T.Generator[bytes, None, None]:
         while True:
-            chunk = data.read(self.chunk_size)
-            # it is possible to upload an empty chunk here
-            # in order to return the handle
-            headers = {
-                "Authorization": f"OAuth {self.user_access_token}",
-                "Offset": f"{offset}",
-                "X-Entity-Length": str(self.entity_size),
-                "X-Entity-Name": self.session_key,
-                "X-Entity-Type": entity_type,
-            }
-            url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-            LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-            resp = request_post(
-                url,
-                headers=headers,
-                data=chunk,
-                timeout=UPLOAD_REQUESTS_TIMEOUT,
-            )
-            LOG.debug(
-                "HTTP response %s: %s", resp.status_code, _truncate_end(resp.content)
-            )
-            resp.raise_for_status()
-            offset += len(chunk)
-            LOG.debug("The next offset will be: %s", offset)
+            data = stream.read(self.chunk_size)
+            if not data:
+                break
+            yield data
+
+    def _offset_chunks(
+        self, chunks: T.Iterable[bytes], offset: int
+    ) -> T.Generator[bytes, None, None]:
+        assert offset >= 0, f"Expect non-negative offset but got {offset}"
+
+        for chunk in chunks:
+            if offset:
+                if offset < len(chunk):
+                    yield chunk[offset:]
+                    offset = 0
+                else:
+                    offset -= len(chunk)
+            else:
+                yield chunk
+
+    def _attach_callbacks(
+        self, chunks: T.Iterable[bytes]
+    ) -> T.Generator[bytes, None, None]:
+        for chunk in chunks:
+            yield chunk
             for callback in self.callbacks:
-                callback(chunk, resp)
-            # we can assert that offset == self.fetch_offset(session_key)
-            # otherwise, server will throw
+                callback(chunk, None)
 
-            if not chunk:
-                break
+    def upload_chunks(
+        self,
+        chunks: T.Iterable[bytes],
+        offset: T.Optional[int] = None,
+    ) -> str:
+        if offset is None:
+            offset = self.fetch_offset()
+
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
 
-        assert offset == self.entity_size, (
-            f"Offset ends at {offset} but the entity size is {self.entity_size}"
+        headers = {
+            "Authorization": f"OAuth {self.user_access_token}",
+            "Offset": f"{offset}",
+            "X-Entity-Name": self.session_key,
+            "X-Entity-Type": self.MIME_BY_CLUSTER_TYPE[self.cluster_filetype],
+        }
+        url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
+        resp = request_post(
+            url,
+            headers=headers,
+            data=chunks,
+            timeout=UPLOAD_REQUESTS_TIMEOUT,
         )
 
         payload = resp.json()
@@ -178,14 +165,12 @@ class UploadService:
 
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
 
-        LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
         resp = request_post(
             url,
             headers=headers,
             json=data,
             timeout=REQUESTS_TIMEOUT,
         )
-        LOG.debug("HTTP response %s: %s", resp.status_code, _truncate_end(resp.content))
 
         resp.raise_for_status()
 
@@ -209,35 +194,30 @@ class FakeUploadService(UploadService):
         )
         self._error_ratio = 0.1
 
-    def upload(
+    def upload_chunks(
         self,
-        data: T.IO[bytes],
+        chunks: T.Iterable[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
         if offset is None:
             offset = self.fetch_offset()
+
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
+
         os.makedirs(self._upload_path, exist_ok=True)
         filename = os.path.join(self._upload_path, self.session_key)
         with open(filename, "ab") as fp:
-            data.seek(offset, io.SEEK_CUR)
-            while True:
-                chunk = data.read(self.chunk_size)
-                if not chunk:
-                    break
-                # fail here means nothing uploaded
+            for chunk in chunks:
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Failed to upload with error ratio {self._error_ratio}"
                     )
                 fp.write(chunk)
-                # fail here means patially uploaded
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Partially uploaded with error ratio {self._error_ratio}"
                     )
-                for callback in self.callbacks:
-                    callback(chunk, None)
-        return self.session_key
+        return uuid.uuid4().hex
 
     def finish(self, _: str) -> str:
         return "0"

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3}/mapillary_tools/uploader.py

@@ -195,7 +195,6 @@ class Uploader:
                 upload_api_v4.FakeUploadService(
                     user_access_token=self.user_items["user_upload_token"],
                     session_key=session_key,
-                    entity_size=entity_size,
                     organization_id=self.user_items.get("MAPOrganizationKey"),
                     cluster_filetype=cluster_filetype,
                     chunk_size=self.chunk_size,
@@ -205,7 +204,6 @@ class Uploader:
             upload_service = upload_api_v4.UploadService(
                 user_access_token=self.user_items["user_upload_token"],
                 session_key=session_key,
-                entity_size=entity_size,
                 organization_id=self.user_items.get("MAPOrganizationKey"),
                 cluster_filetype=cluster_filetype,
                 chunk_size=self.chunk_size,

{mapillary_tools-0.13.2 → mapillary_tools-0.13.3/mapillary_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mapillary_tools
-Version: 0.13.2
+Version: 0.13.3
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary

mapillary_tools-0.13.2/mapillary_tools/__init__.py

@@ -1 +0,0 @@
-VERSION = "0.13.2"