mapillary-tools 0.13.1a1__tar.gz → 0.13.3a1__tar.gz

{mapillary_tools-0.13.1a1/mapillary_tools.egg-info → mapillary_tools-0.13.3a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mapillary_tools
-Version: 0.13.1a1
+Version: 0.13.3a1
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary

mapillary_tools-0.13.3a1/mapillary_tools/__init__.py

@@ -0,0 +1 @@
+VERSION = "0.13.3a1"

{mapillary_tools-0.13.1a1 → mapillary_tools-0.13.3a1}/mapillary_tools/constants.py

@@ -45,6 +45,6 @@ GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
 # Max number of images per sequence
 MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
 # Max file size per sequence (sum of image filesizes in the sequence)
-MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "10G")
+MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
 # Max number of pixels per sequence (sum of image pixels in the sequence)
 MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")

{mapillary_tools-0.13.1a1 → mapillary_tools-0.13.3a1}/mapillary_tools/geotag/geotag_images_from_gpx_file.py

@@ -25,7 +25,13 @@ class GeotagImagesFromGPXFile(GeotagImagesFromGeneric):
         num_processes: T.Optional[int] = None,
     ):
         super().__init__()
-        tracks = parse_gpx(source_path)
+        try:
+            tracks = parse_gpx(source_path)
+        except Exception as ex:
+            raise RuntimeError(
+                f"Error parsing GPX {source_path}: {ex.__class__.__name__}: {ex}"
+            )
+
         if 1 < len(tracks):
             LOG.warning(
                 "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",

{mapillary_tools-0.13.1a1 → mapillary_tools-0.13.3a1}/mapillary_tools/upload_api_v4.py

@@ -5,6 +5,7 @@ import logging
 import os
 import random
 import typing as T
+import uuid
 
 import requests
 
@@ -55,31 +56,31 @@ def _truncate_end(s: _S) -> _S:
 
 class UploadService:
     user_access_token: str
-    entity_size: int
     session_key: str
     callbacks: T.List[T.Callable[[bytes, T.Optional[requests.Response]], None]]
     cluster_filetype: ClusterFileType
     organization_id: T.Optional[T.Union[str, int]]
     chunk_size: int
 
+    MIME_BY_CLUSTER_TYPE: T.Dict[ClusterFileType, str] = {
+        ClusterFileType.ZIP: "application/zip",
+        ClusterFileType.BLACKVUE: "video/mp4",
+        ClusterFileType.CAMM: "video/mp4",
+    }
+
     def __init__(
         self,
         user_access_token: str,
         session_key: str,
-        entity_size: int,
         organization_id: T.Optional[T.Union[str, int]] = None,
         cluster_filetype: ClusterFileType = ClusterFileType.ZIP,
         chunk_size: int = DEFAULT_CHUNK_SIZE,
     ):
-        if entity_size <= 0:
-            raise ValueError(f"Expect positive entity size but got {entity_size}")
-
         if chunk_size <= 0:
             raise ValueError("Expect positive chunk size")
 
         self.user_access_token = user_access_token
         self.session_key = session_key
-        self.entity_size = entity_size
         self.organization_id = organization_id
         #  validate the input
         self.cluster_filetype = ClusterFileType(cluster_filetype)
@@ -107,55 +108,66 @@ class UploadService:
         data: T.IO[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
-        if offset is None:
-            offset = self.fetch_offset()
-
-        entity_type_map: T.Dict[ClusterFileType, str] = {
-            ClusterFileType.ZIP: "application/zip",
-            ClusterFileType.BLACKVUE: "video/mp4",
-            ClusterFileType.CAMM: "video/mp4",
-        }
-
-        entity_type = entity_type_map[self.cluster_filetype]
-
-        data.seek(offset, io.SEEK_CUR)
+        chunks = self._chunkize_byte_stream(data)
+        return self.upload_chunks(chunks, offset=offset)
 
+    def _chunkize_byte_stream(
+        self, stream: T.IO[bytes]
+    ) -> T.Generator[bytes, None, None]:
         while True:
-            chunk = data.read(self.chunk_size)
-            # it is possible to upload an empty chunk here
-            # in order to return the handle
-            headers = {
-                "Authorization": f"OAuth {self.user_access_token}",
-                "Offset": f"{offset}",
-                "X-Entity-Length": str(self.entity_size),
-                "X-Entity-Name": self.session_key,
-                "X-Entity-Type": entity_type,
-            }
-            url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-            LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-            resp = request_post(
-                url,
-                headers=headers,
-                data=chunk,
-                timeout=UPLOAD_REQUESTS_TIMEOUT,
-            )
-            LOG.debug(
-                "HTTP response %s: %s", resp.status_code, _truncate_end(resp.content)
-            )
-            resp.raise_for_status()
-            offset += len(chunk)
-            LOG.debug("The next offset will be: %s", offset)
+            data = stream.read(self.chunk_size)
+            if not data:
+                break
+            yield data
+
+    def _offset_chunks(
+        self, chunks: T.Iterable[bytes], offset: int
+    ) -> T.Generator[bytes, None, None]:
+        assert offset >= 0, f"Expect non-negative offset but got {offset}"
+
+        for chunk in chunks:
+            if offset:
+                if offset < len(chunk):
+                    yield chunk[offset:]
+                    offset = 0
+                else:
+                    offset -= len(chunk)
+            else:
+                yield chunk
+
+    def _attach_callbacks(
+        self, chunks: T.Iterable[bytes]
+    ) -> T.Generator[bytes, None, None]:
+        for chunk in chunks:
+            yield chunk
             for callback in self.callbacks:
-                callback(chunk, resp)
-            # we can assert that offset == self.fetch_offset(session_key)
-            # otherwise, server will throw
+                callback(chunk, None)
 
-            if not chunk:
-                break
+    def upload_chunks(
+        self,
+        chunks: T.Iterable[bytes],
+        offset: T.Optional[int] = None,
+    ) -> str:
+        if offset is None:
+            offset = self.fetch_offset()
 
-        assert offset == self.entity_size, (
-            f"Offset ends at {offset} but the entity size is {self.entity_size}"
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
+
+        headers = {
+            "Authorization": f"OAuth {self.user_access_token}",
+            "Offset": f"{offset}",
+            "X-Entity-Name": self.session_key,
+            "X-Entity-Type": self.MIME_BY_CLUSTER_TYPE[self.cluster_filetype],
+        }
+        url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
+        LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
+        resp = request_post(
+            url,
+            headers=headers,
+            data=chunks,
+            timeout=UPLOAD_REQUESTS_TIMEOUT,
         )
+        LOG.debug("HTTP response %s: %s", resp.status_code, _truncate_end(resp.content))
 
         payload = resp.json()
         try:
@@ -209,35 +221,30 @@ class FakeUploadService(UploadService):
         )
         self._error_ratio = 0.1
 
-    def upload(
+    def upload_chunks(
         self,
-        data: T.IO[bytes],
+        chunks: T.Iterable[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
         if offset is None:
             offset = self.fetch_offset()
+
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
+
         os.makedirs(self._upload_path, exist_ok=True)
         filename = os.path.join(self._upload_path, self.session_key)
         with open(filename, "ab") as fp:
-            data.seek(offset, io.SEEK_CUR)
-            while True:
-                chunk = data.read(self.chunk_size)
-                if not chunk:
-                    break
-                # fail here means nothing uploaded
+            for chunk in chunks:
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Failed to upload with error ratio {self._error_ratio}"
                     )
                 fp.write(chunk)
-                # fail here means patially uploaded
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Partially uploaded with error ratio {self._error_ratio}"
                     )
-                for callback in self.callbacks:
-                    callback(chunk, None)
-        return self.session_key
+        return uuid.uuid4().hex
 
     def finish(self, _: str) -> str:
         return "0"

{mapillary_tools-0.13.1a1 → mapillary_tools-0.13.3a1}/mapillary_tools/uploader.py

@@ -195,7 +195,6 @@ class Uploader:
                 upload_api_v4.FakeUploadService(
                     user_access_token=self.user_items["user_upload_token"],
                     session_key=session_key,
-                    entity_size=entity_size,
                     organization_id=self.user_items.get("MAPOrganizationKey"),
                     cluster_filetype=cluster_filetype,
                     chunk_size=self.chunk_size,
@@ -205,7 +204,6 @@ class Uploader:
             upload_service = upload_api_v4.UploadService(
                 user_access_token=self.user_items["user_upload_token"],
                 session_key=session_key,
-                entity_size=entity_size,
                 organization_id=self.user_items.get("MAPOrganizationKey"),
                 cluster_filetype=cluster_filetype,
                 chunk_size=self.chunk_size,

{mapillary_tools-0.13.1a1 → mapillary_tools-0.13.3a1}/mapillary_tools/video_data_extraction/extractors/camm_parser.py

@@ -13,8 +13,12 @@ class CammParser(BaseParser):
     parser_label = "camm"
 
     @functools.cached_property
-    def __camera_info(self) -> T.Tuple[str, str]:
-        with self.videoPath.open("rb") as fp:
+    def _camera_info(self) -> T.Tuple[str, str]:
+        source_path = self.geotag_source_path
+        if not source_path:
+            return "", ""
+
+        with source_path.open("rb") as fp:
             return camm_parser.extract_camera_make_and_model(fp)
 
     def extract_points(self) -> T.Sequence[geo.Point]:
@@ -28,15 +32,7 @@ class CammParser(BaseParser):
                 return []
 
     def extract_make(self) -> T.Optional[str]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return None
-        with source_path.open("rb") as _fp:
-            return self.__camera_info[0] or None
+        return self._camera_info[0] or None
 
     def extract_model(self) -> T.Optional[str]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return None
-        with source_path.open("rb") as _fp:
-            return self.__camera_info[1] or None
+        return self._camera_info[1] or None

mapillary_tools-0.13.3a1/mapillary_tools/video_data_extraction/extractors/gpx_parser.py

@@ -0,0 +1,108 @@
+import datetime
+import logging
+import typing as T
+
+from ... import geo, telemetry
+from ...geotag import geotag_images_from_gpx_file
+from .base_parser import BaseParser
+from .generic_video_parser import GenericVideoParser
+
+
+LOG = logging.getLogger(__name__)
+
+
+class GpxParser(BaseParser):
+    default_source_pattern = "%g.gpx"
+    parser_label = "gpx"
+
+    def extract_points(self) -> T.Sequence[geo.Point]:
+        path = self.geotag_source_path
+        if not path:
+            return []
+
+        try:
+            gpx_tracks = geotag_images_from_gpx_file.parse_gpx(path)
+        except Exception as ex:
+            raise RuntimeError(
+                f"Error parsing GPX {path}: {ex.__class__.__name__}: {ex}"
+            )
+
+        if 1 < len(gpx_tracks):
+            LOG.warning(
+                "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",
+                len(gpx_tracks),
+                self.videoPath,
+            )
+
+        gpx_points: T.Sequence[geo.Point] = sum(gpx_tracks, [])
+        if not gpx_points:
+            return gpx_points
+
+        offset = self._synx_gpx_by_first_gps_timestamp(gpx_points)
+
+        self._rebase_times(gpx_points, offset=offset)
+
+        return gpx_points
+
+    def _synx_gpx_by_first_gps_timestamp(
+        self, gpx_points: T.Sequence[geo.Point]
+    ) -> float:
+        offset: float = 0.0
+
+        if not gpx_points:
+            return offset
+
+        first_gpx_dt = datetime.datetime.fromtimestamp(
+            gpx_points[0].time, tz=datetime.timezone.utc
+        )
+        LOG.info("First GPX timestamp: %s", first_gpx_dt)
+
+        # Extract first GPS timestamp (if found) for synchronization
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
+        gps_points = parser.extract_points()
+
+        if not gps_points:
+            LOG.warning(
+                "Skip GPX synchronization because no GPS found in video %s",
+                self.videoPath,
+            )
+            return offset
+
+        first_gps_point = gps_points[0]
+        if isinstance(first_gps_point, telemetry.GPSPoint):
+            if first_gps_point.epoch_time is not None:
+                first_gps_dt = datetime.datetime.fromtimestamp(
+                    first_gps_point.epoch_time, tz=datetime.timezone.utc
+                )
+                LOG.info("First GPS timestamp: %s", first_gps_dt)
+                offset = gpx_points[0].time - first_gps_point.epoch_time
+                if offset:
+                    LOG.warning(
+                        "Found offset between GPX %s and video GPS timestamps %s: %s seconds",
+                        first_gpx_dt,
+                        first_gps_dt,
+                        offset,
+                    )
+                else:
+                    LOG.info(
+                        "GPX and GPS are perfectly synchronized (all starts from %s)",
+                        first_gpx_dt,
+                    )
+            else:
+                LOG.warning(
+                    "Skip GPX synchronization because no GPS epoch time found in video %s",
+                    self.videoPath,
+                )
+
+        return offset
+
+    def extract_make(self) -> T.Optional[str]:
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
+        return parser.extract_make()
+
+    def extract_model(self) -> T.Optional[str]:
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
+        return parser.extract_model()

{mapillary_tools-0.13.1a1 → mapillary_tools-0.13.3a1/mapillary_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mapillary_tools
-Version: 0.13.1a1
+Version: 0.13.3a1
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary

mapillary_tools-0.13.1a1/mapillary_tools/__init__.py

@@ -1 +0,0 @@
-VERSION = "0.13.1a1"

mapillary_tools-0.13.1a1/mapillary_tools/video_data_extraction/extractors/gpx_parser.py

@@ -1,71 +0,0 @@
-import datetime
-import logging
-import typing as T
-
-from ... import geo, telemetry
-from ...geotag import geotag_images_from_gpx_file
-from .base_parser import BaseParser
-from .generic_video_parser import GenericVideoParser
-
-
-LOG = logging.getLogger(__name__)
-
-
-class GpxParser(BaseParser):
-    default_source_pattern = "%g.gpx"
-    parser_label = "gpx"
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        path = self.geotag_source_path
-        if not path:
-            return []
-
-        gpx_tracks = geotag_images_from_gpx_file.parse_gpx(path)
-        if 1 < len(gpx_tracks):
-            LOG.warning(
-                "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",
-                len(gpx_tracks),
-                self.videoPath,
-            )
-
-        gpx_points: T.Sequence[geo.Point] = sum(gpx_tracks, [])
-        if not gpx_points:
-            return gpx_points
-
-        first_gpx_dt = datetime.datetime.fromtimestamp(
-            gpx_points[0].time, tz=datetime.timezone.utc
-        )
-        LOG.info("First GPX timestamp: %s", first_gpx_dt)
-
-        # Extract first GPS timestamp (if found) for synchronization
-        offset: float = 0.0
-        parser = GenericVideoParser(self.videoPath, self.options, self.parserOptions)
-        gps_points = parser.extract_points()
-        if gps_points:
-            first_gps_point = gps_points[0]
-            if isinstance(first_gps_point, telemetry.GPSPoint):
-                if first_gps_point.epoch_time is not None:
-                    first_gps_dt = datetime.datetime.fromtimestamp(
-                        first_gps_point.epoch_time, tz=datetime.timezone.utc
-                    )
-                    LOG.info("First GPS timestamp: %s", first_gps_dt)
-                    offset = gpx_points[0].time - first_gps_point.epoch_time
-                    if offset:
-                        LOG.warning(
-                            "Found offset between GPX %s and video GPS timestamps %s: %s seconds",
-                            first_gpx_dt,
-                            first_gps_dt,
-                            offset,
-                        )
-
-        self._rebase_times(gpx_points, offset=offset)
-
-        return gpx_points
-
-    def extract_make(self) -> T.Optional[str]:
-        parser = GenericVideoParser(self.videoPath, self.options, self.parserOptions)
-        return parser.extract_make()
-
-    def extract_model(self) -> T.Optional[str]:
-        parser = GenericVideoParser(self.videoPath, self.options, self.parserOptions)
-        return parser.extract_model()