mapillary-tools 0.13.2__py3-none-any.whl → 0.13.3a1__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.13.2"
+VERSION = "0.13.3a1"

mapillary_tools/constants.py

@@ -45,6 +45,6 @@ GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
 # Max number of images per sequence
 MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
 # Max file size per sequence (sum of image filesizes in the sequence)
-MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "10G")
+MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
 # Max number of pixels per sequence (sum of image pixels in the sequence)
 MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")

mapillary_tools/upload_api_v4.py

@@ -5,6 +5,7 @@ import logging
 import os
 import random
 import typing as T
+import uuid
 
 import requests
 
@@ -55,31 +56,31 @@ def _truncate_end(s: _S) -> _S:
 
 class UploadService:
     user_access_token: str
-    entity_size: int
     session_key: str
     callbacks: T.List[T.Callable[[bytes, T.Optional[requests.Response]], None]]
     cluster_filetype: ClusterFileType
     organization_id: T.Optional[T.Union[str, int]]
     chunk_size: int
 
+    MIME_BY_CLUSTER_TYPE: T.Dict[ClusterFileType, str] = {
+        ClusterFileType.ZIP: "application/zip",
+        ClusterFileType.BLACKVUE: "video/mp4",
+        ClusterFileType.CAMM: "video/mp4",
+    }
+
     def __init__(
         self,
         user_access_token: str,
         session_key: str,
-        entity_size: int,
         organization_id: T.Optional[T.Union[str, int]] = None,
         cluster_filetype: ClusterFileType = ClusterFileType.ZIP,
         chunk_size: int = DEFAULT_CHUNK_SIZE,
     ):
-        if entity_size <= 0:
-            raise ValueError(f"Expect positive entity size but got {entity_size}")
-
         if chunk_size <= 0:
             raise ValueError("Expect positive chunk size")
 
         self.user_access_token = user_access_token
         self.session_key = session_key
-        self.entity_size = entity_size
         self.organization_id = organization_id
         #  validate the input
         self.cluster_filetype = ClusterFileType(cluster_filetype)
@@ -107,55 +108,66 @@ class UploadService:
         data: T.IO[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
-        if offset is None:
-            offset = self.fetch_offset()
-
-        entity_type_map: T.Dict[ClusterFileType, str] = {
-            ClusterFileType.ZIP: "application/zip",
-            ClusterFileType.BLACKVUE: "video/mp4",
-            ClusterFileType.CAMM: "video/mp4",
-        }
-
-        entity_type = entity_type_map[self.cluster_filetype]
-
-        data.seek(offset, io.SEEK_CUR)
+        chunks = self._chunkize_byte_stream(data)
+        return self.upload_chunks(chunks, offset=offset)
 
+    def _chunkize_byte_stream(
+        self, stream: T.IO[bytes]
+    ) -> T.Generator[bytes, None, None]:
         while True:
-            chunk = data.read(self.chunk_size)
-            # it is possible to upload an empty chunk here
-            # in order to return the handle
-            headers = {
-                "Authorization": f"OAuth {self.user_access_token}",
-                "Offset": f"{offset}",
-                "X-Entity-Length": str(self.entity_size),
-                "X-Entity-Name": self.session_key,
-                "X-Entity-Type": entity_type,
-            }
-            url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-            LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-            resp = request_post(
-                url,
-                headers=headers,
-                data=chunk,
-                timeout=UPLOAD_REQUESTS_TIMEOUT,
-            )
-            LOG.debug(
-                "HTTP response %s: %s", resp.status_code, _truncate_end(resp.content)
-            )
-            resp.raise_for_status()
-            offset += len(chunk)
-            LOG.debug("The next offset will be: %s", offset)
+            data = stream.read(self.chunk_size)
+            if not data:
+                break
+            yield data
+
+    def _offset_chunks(
+        self, chunks: T.Iterable[bytes], offset: int
+    ) -> T.Generator[bytes, None, None]:
+        assert offset >= 0, f"Expect non-negative offset but got {offset}"
+
+        for chunk in chunks:
+            if offset:
+                if offset < len(chunk):
+                    yield chunk[offset:]
+                    offset = 0
+                else:
+                    offset -= len(chunk)
+            else:
+                yield chunk
+
+    def _attach_callbacks(
+        self, chunks: T.Iterable[bytes]
+    ) -> T.Generator[bytes, None, None]:
+        for chunk in chunks:
+            yield chunk
             for callback in self.callbacks:
-                callback(chunk, resp)
-            # we can assert that offset == self.fetch_offset(session_key)
-            # otherwise, server will throw
+                callback(chunk, None)
 
-            if not chunk:
-                break
+    def upload_chunks(
+        self,
+        chunks: T.Iterable[bytes],
+        offset: T.Optional[int] = None,
+    ) -> str:
+        if offset is None:
+            offset = self.fetch_offset()
 
-        assert offset == self.entity_size, (
-            f"Offset ends at {offset} but the entity size is {self.entity_size}"
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
+
+        headers = {
+            "Authorization": f"OAuth {self.user_access_token}",
+            "Offset": f"{offset}",
+            "X-Entity-Name": self.session_key,
+            "X-Entity-Type": self.MIME_BY_CLUSTER_TYPE[self.cluster_filetype],
+        }
+        url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
+        LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
+        resp = request_post(
+            url,
+            headers=headers,
+            data=chunks,
+            timeout=UPLOAD_REQUESTS_TIMEOUT,
         )
+        LOG.debug("HTTP response %s: %s", resp.status_code, _truncate_end(resp.content))
 
         payload = resp.json()
         try:
@@ -209,35 +221,30 @@ class FakeUploadService(UploadService):
         )
         self._error_ratio = 0.1
 
-    def upload(
+    def upload_chunks(
         self,
-        data: T.IO[bytes],
+        chunks: T.Iterable[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
         if offset is None:
             offset = self.fetch_offset()
+
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
+
         os.makedirs(self._upload_path, exist_ok=True)
         filename = os.path.join(self._upload_path, self.session_key)
         with open(filename, "ab") as fp:
-            data.seek(offset, io.SEEK_CUR)
-            while True:
-                chunk = data.read(self.chunk_size)
-                if not chunk:
-                    break
-                # fail here means nothing uploaded
+            for chunk in chunks:
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Failed to upload with error ratio {self._error_ratio}"
                     )
                 fp.write(chunk)
-                # fail here means patially uploaded
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Partially uploaded with error ratio {self._error_ratio}"
                     )
-                for callback in self.callbacks:
-                    callback(chunk, None)
-        return self.session_key
+        return uuid.uuid4().hex
 
     def finish(self, _: str) -> str:
         return "0"

mapillary_tools/uploader.py

@@ -195,7 +195,6 @@ class Uploader:
                 upload_api_v4.FakeUploadService(
                     user_access_token=self.user_items["user_upload_token"],
                     session_key=session_key,
-                    entity_size=entity_size,
                     organization_id=self.user_items.get("MAPOrganizationKey"),
                     cluster_filetype=cluster_filetype,
                     chunk_size=self.chunk_size,
@@ -205,7 +204,6 @@ class Uploader:
             upload_service = upload_api_v4.UploadService(
                 user_access_token=self.user_items["user_upload_token"],
                 session_key=session_key,
-                entity_size=entity_size,
                 organization_id=self.user_items.get("MAPOrganizationKey"),
                 cluster_filetype=cluster_filetype,
                 chunk_size=self.chunk_size,

{mapillary_tools-0.13.2.dist-info → mapillary_tools-0.13.3a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mapillary_tools
-Version: 0.13.2
+Version: 0.13.3a1
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary

{mapillary_tools-0.13.2.dist-info → mapillary_tools-0.13.3a1.dist-info}/RECORD

@@ -1,8 +1,8 @@
-mapillary_tools/__init__.py,sha256=0qmTgs4HlmMGWxh_Upl_6EAbqBbUzX-Mw8rXWm5EO-U,19
+mapillary_tools/__init__.py,sha256=j5y1cIo14OOuS27ROUABVHN9119rEXV-bdoM-z117o8,21
 mapillary_tools/api_v4.py,sha256=zhRtgx3EnzgqtjziRhvFq3ONvsPaB9hROsuKFcf_pFo,5197
 mapillary_tools/authenticate.py,sha256=LCFcs6LqZmXaYkTUEKgGfmqytWdh5v_L3KXB48ojOZ4,3090
 mapillary_tools/config.py,sha256=jCjaK4jJaTY4AV4qf_b_tcxn5LA_uPsEWlGIdm2zw6g,2103
-mapillary_tools/constants.py,sha256=CZpKd9wUxlTC3lq7h3Mi5J1NpPlLYvgp1r0_sFUJKGk,2452
+mapillary_tools/constants.py,sha256=KKQMwzpN2z7wka6lhZv-agdxNldaEQqvQXu3Kg5XuUk,2453
 mapillary_tools/exceptions.py,sha256=Mh1tgVEFTSMnYEzrl9x7b95fW9Z3SPVD_YMEl7r8I0I,2693
 mapillary_tools/exif_read.py,sha256=F60A0-T8XSBHvFKgVIrUz_ZWKQrTFWrtj3c6siB0IMg,28707
 mapillary_tools/exif_write.py,sha256=3PawLnBOY8Z86TYiA_F4LxRhe5Ui6CTNhxYm9yeJNX8,8786
@@ -18,8 +18,8 @@ mapillary_tools/sample_video.py,sha256=dpdX7bUNEmcrz-3gh3Y3awnTDX66pChbTKuF8qGfe
 mapillary_tools/telemetry.py,sha256=WpBGPF_GMPjM_EFqXIutFtpDFL9wj7yEzGNGnfQZUo8,1255
 mapillary_tools/types.py,sha256=6kww2UdKM6YzabYbc862BYzEWtxL2hhxCRFfeDiUtF0,22074
 mapillary_tools/upload.py,sha256=8dQ3ZWsjau1_xZN3ssjGGkBnLKbKIhjC91-zWstYlD8,24439
-mapillary_tools/upload_api_v4.py,sha256=1WvoUis92KDXbqfoyvyyDmiCqwXezYkMJZhnYaVm3BA,8560
-mapillary_tools/uploader.py,sha256=VieDKi51wdXTIhN7x_mcuQeHESUyFlF5cgB-TAnF4g0,14093
+mapillary_tools/upload_api_v4.py,sha256=VXIAA_lar4y4RgvNuKpkE7CVl4uWa6kNT59hCVFClSk,8490
+mapillary_tools/uploader.py,sha256=DBHso4QIP5nsZFDynLjkUvytzwpEOPnOlfeDyA6rTBk,14007
 mapillary_tools/utils.py,sha256=VNtK1tAb3Hh8y3P5e5Y3iewREkIoLDa3C2myRYcF2lY,5970
 mapillary_tools/camm/camm_builder.py,sha256=TXZfhu3xGjtrLEWnB14D7aSOrHOoSJef24YSLApiIfY,10631
 mapillary_tools/camm/camm_parser.py,sha256=RaCWeLvS_AyHD6B6wDUu9DAsdfByVHMAPTqEqjtFibE,9734
@@ -67,9 +67,9 @@ mapillary_tools/video_data_extraction/extractors/generic_video_parser.py,sha256=
 mapillary_tools/video_data_extraction/extractors/gopro_parser.py,sha256=IVnTyquSraTUaG9rxbJfVWc1-drdY5PaHn5urh3IBk4,1325
 mapillary_tools/video_data_extraction/extractors/gpx_parser.py,sha256=FNrdnXl48k8I1I5fGwYsClhfFEHVsooRLRboUYECv3I,3811
 mapillary_tools/video_data_extraction/extractors/nmea_parser.py,sha256=raSXavBvP-0LJCB_TwLL0mOv2uHSsB744igTsaKAaGc,658
-mapillary_tools-0.13.2.dist-info/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
-mapillary_tools-0.13.2.dist-info/METADATA,sha256=2tmLjwx7V2QhrErmOK9uL1McwGYAaHqXDw-wTIS1N3w,19758
-mapillary_tools-0.13.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-mapillary_tools-0.13.2.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
-mapillary_tools-0.13.2.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
-mapillary_tools-0.13.2.dist-info/RECORD,,
+mapillary_tools-0.13.3a1.dist-info/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
+mapillary_tools-0.13.3a1.dist-info/METADATA,sha256=3wyt_Iv96Md1JNY358MhdTqO02bb0XKdEoDBMiP3_yw,19760
+mapillary_tools-0.13.3a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mapillary_tools-0.13.3a1.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
+mapillary_tools-0.13.3a1.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
+mapillary_tools-0.13.3a1.dist-info/RECORD,,

{mapillary_tools-0.13.2.dist-info → mapillary_tools-0.13.3a1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any