mapillary-tools 0.14.0b1__py3-none-any.whl → 0.14.2__py3-none-any.whl

mapillary_tools/upload_api_v4.py

@@ -17,18 +17,11 @@ import tempfile
 
 import requests
 
-from .api_v4 import request_get, request_post, REQUESTS_TIMEOUT
+from .api_v4 import HTTPContentError, jsonify_response, REQUESTS_TIMEOUT
 
 MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
     "MAPILLARY_UPLOAD_ENDPOINT", "https://rupload.facebook.com/mapillary_public_uploads"
 )
-# According to the docs, UPLOAD_REQUESTS_TIMEOUT can be a tuple of
-# (connection_timeout, read_timeout): https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
-# In my test, however, the connection_timeout rules both connection timeout and read timeout.
-# i.e. if your the server does not respond within this timeout, it will throw:
-# ConnectionError: ('Connection aborted.', timeout('The write operation timed out'))
-# So let us make sure the largest possible chunks can be uploaded before this timeout for now,
-UPLOAD_REQUESTS_TIMEOUT = (30 * 60, 30 * 60)  # 30 minutes
 
 
 class UploadService:
@@ -39,19 +32,21 @@ class UploadService:
     user_access_token: str
     session_key: str
 
-    def __init__(self, user_access_token: str, session_key: str):
-        self.user_access_token = user_access_token
+    def __init__(self, user_session: requests.Session, session_key: str):
+        self.user_session = user_session
         self.session_key = session_key
 
     def fetch_offset(self) -> int:
-        headers = {
-            "Authorization": f"OAuth {self.user_access_token}",
-        }
         url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-        resp = request_get(url, headers=headers, timeout=REQUESTS_TIMEOUT)
+
+        resp = self.user_session.get(url, timeout=REQUESTS_TIMEOUT)
         resp.raise_for_status()
-        data = resp.json()
-        return data["offset"]
+
+        data = jsonify_response(resp)
+        try:
+            return data["offset"]
+        except KeyError:
+            raise HTTPContentError("Offset not found in the response", resp)
 
     @classmethod
     def chunkize_byte_stream(
@@ -120,47 +115,58 @@ class UploadService:
         stream: T.IO[bytes],
         offset: int | None = None,
         chunk_size: int = 2 * 1024 * 1024,  # 2MB
+        read_timeout: float | None = None,
     ) -> str:
         if offset is None:
             offset = self.fetch_offset()
-        return self.upload_chunks(self.chunkize_byte_stream(stream, chunk_size), offset)
+        return self.upload_chunks(
+            self.chunkize_byte_stream(stream, chunk_size),
+            offset,
+            read_timeout=read_timeout,
+        )
 
     def upload_chunks(
         self,
         chunks: T.Iterable[bytes],
         offset: int | None = None,
+        read_timeout: float | None = None,
     ) -> str:
         if offset is None:
             offset = self.fetch_offset()
         shifted_chunks = self.shift_chunks(chunks, offset)
-        return self.upload_shifted_chunks(shifted_chunks, offset)
+        return self.upload_shifted_chunks(
+            shifted_chunks, offset, read_timeout=read_timeout
+        )
 
     def upload_shifted_chunks(
-        self, shifted_chunks: T.Iterable[bytes], offset: int
+        self,
+        shifted_chunks: T.Iterable[bytes],
+        offset: int,
+        read_timeout: float | None = None,
     ) -> str:
         """
         Upload the chunks that must already be shifted by the offset (e.g. fp.seek(offset, io.SEEK_SET))
         """
 
+        url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
         headers = {
-            "Authorization": f"OAuth {self.user_access_token}",
             "Offset": f"{offset}",
             "X-Entity-Name": self.session_key,
         }
-        url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-        resp = request_post(
-            url, headers=headers, data=shifted_chunks, timeout=UPLOAD_REQUESTS_TIMEOUT
-        )
 
+        resp = self.user_session.post(
+            url,
+            headers=headers,
+            data=shifted_chunks,
+            timeout=(REQUESTS_TIMEOUT, read_timeout),  # type: ignore
+        )
         resp.raise_for_status()
 
-        payload = resp.json()
+        data = jsonify_response(resp)
         try:
-            return payload["h"]
+            return data["h"]
         except KeyError:
-            raise RuntimeError(
-                f"Upload server error: File handle not found in the upload response {resp.text}"
-            )
+            raise HTTPContentError("File handle not found in the response", resp)
 
 
 # A mock class for testing only
@@ -174,9 +180,9 @@ class FakeUploadService(UploadService):
 
     def __init__(
         self,
+        *args,
         upload_path: Path | None = None,
         transient_error_ratio: float = 0.0,
-        *args,
         **kwargs,
     ):
         super().__init__(*args, **kwargs)
@@ -187,13 +193,12 @@ class FakeUploadService(UploadService):
         self._upload_path = upload_path
         self._transient_error_ratio = transient_error_ratio
 
-    @property
-    def upload_path(self) -> Path:
-        return self._upload_path
-
     @override
     def upload_shifted_chunks(
-        self, shifted_chunks: T.Iterable[bytes], offset: int
+        self,
+        shifted_chunks: T.Iterable[bytes],
+        offset: int,
+        read_timeout: float | None = None,
     ) -> str:
         expected_offset = self.fetch_offset()
         if offset != expected_offset:
@@ -205,15 +210,9 @@ class FakeUploadService(UploadService):
         filename = self._upload_path.joinpath(self.session_key)
         with filename.open("ab") as fp:
             for chunk in shifted_chunks:
-                if random.random() <= self._transient_error_ratio:
-                    raise requests.ConnectionError(
-                        f"TEST ONLY: Failed to upload with error ratio {self._transient_error_ratio}"
-                    )
+                self._randomly_raise_transient_error()
                 fp.write(chunk)
-                if random.random() <= self._transient_error_ratio:
-                    raise requests.ConnectionError(
-                        f"TEST ONLY: Partially uploaded with error ratio {self._transient_error_ratio}"
-                    )
+                self._randomly_raise_transient_error()
 
         file_handle_dir = self._upload_path.joinpath(self.FILE_HANDLE_DIR)
         file_handle_path = file_handle_dir.joinpath(self.session_key)
@@ -226,13 +225,24 @@ class FakeUploadService(UploadService):
 
     @override
     def fetch_offset(self) -> int:
-        if random.random() <= self._transient_error_ratio:
-            raise requests.ConnectionError(
-                f"TEST ONLY: Partially uploaded with error ratio {self._transient_error_ratio}"
-            )
+        self._randomly_raise_transient_error()
         filename = self._upload_path.joinpath(self.session_key)
         if not filename.exists():
             return 0
         with open(filename, "rb") as fp:
             fp.seek(0, io.SEEK_END)
             return fp.tell()
+
+    @property
+    def upload_path(self) -> Path:
+        return self._upload_path
+
+    def _randomly_raise_transient_error(self):
+        """
+        Randomly raise a transient error based on the configured error ratio.
+        This is for testing purposes only.
+        """
+        if random.random() <= self._transient_error_ratio:
+            raise requests.ConnectionError(
+                f"[TEST ONLY]: Transient error with ratio {self._transient_error_ratio}"
+            )