mapillary-tools 0.13.1a1__py3-none-any.whl → 0.13.3__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1 @@
-VERSION = "0.13.1a1"
+VERSION = "0.13.3"

mapillary_tools/api_v4.py

@@ -2,6 +2,7 @@ import logging
 import os
 import ssl
 import typing as T
+from json import dumps
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -46,6 +47,106 @@ class HTTPSystemCertsAdapter(HTTPAdapter):
         conn.ca_certs = None
 
 
+@T.overload
+def _truncate(s: bytes, limit: int = 512) -> bytes: ...
+
+
+@T.overload
+def _truncate(s: str, limit: int = 512) -> str: ...
+
+
+def _truncate(s, limit=512):
+    if limit < len(s):
+        remaining = len(s) - limit
+        if isinstance(s, bytes):
+            return (
+                s[:limit]
+                + b"..."
+                + f"({remaining} more bytes truncated)".encode("utf-8")
+            )
+        else:
+            return str(s[:limit]) + f"...({remaining} more chars truncated)"
+    else:
+        return s
+
+
+def _sanitize(headers: T.Dict):
+    new_headers = {}
+
+    for k, v in headers.items():
+        if k.lower() in [
+            "authorization",
+            "cookie",
+            "x-fb-access-token",
+            "access-token",
+            "access_token",
+            "password",
+        ]:
+            new_headers[k] = "[REDACTED]"
+        else:
+            new_headers[k] = _truncate(v)
+
+    return new_headers
+
+
+def _log_debug_request(
+    method: str,
+    url: str,
+    json: T.Optional[T.Dict] = None,
+    params: T.Optional[T.Dict] = None,
+    headers: T.Optional[T.Dict] = None,
+    timeout: T.Any = None,
+):
+    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
+        return
+
+    msg = f"HTTP {method} {url}"
+
+    if USE_SYSTEM_CERTS:
+        msg += " (w/sys_certs)"
+
+    if json:
+        t = _truncate(dumps(_sanitize(json)))
+        msg += f" JSON={t}"
+
+    if params:
+        msg += f" PARAMS={_sanitize(params)}"
+
+    if headers:
+        msg += f" HEADERS={_sanitize(headers)}"
+
+    if timeout is not None:
+        msg += f" TIMEOUT={timeout}"
+
+    LOG.debug(msg)
+
+
+def _log_debug_response(resp: requests.Response):
+    if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
+        return
+
+    data: T.Union[str, bytes]
+    try:
+        data = _truncate(dumps(_sanitize(resp.json())))
+    except Exception:
+        data = _truncate(resp.content)
+
+    LOG.debug(f"HTTP {resp.status_code} ({resp.reason}): %s", data)
+
+
+def readable_http_error(ex: requests.HTTPError) -> str:
+    req = ex.request
+    resp = ex.response
+
+    data: T.Union[str, bytes]
+    try:
+        data = _truncate(dumps(_sanitize(resp.json())))
+    except Exception:
+        data = _truncate(resp.content)
+
+    return f"{req.method} {resp.url} => {resp.status_code} ({resp.reason}): {str(data)}"
+
+
 def request_post(
     url: str,
     data: T.Optional[T.Any] = None,
@@ -54,14 +155,23 @@ def request_post(
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
+    _log_debug_request(
+        "POST",
+        url,
+        json=json,
+        params=kwargs.get("params"),
+        headers=kwargs.get("headers"),
+        timeout=kwargs.get("timeout"),
+    )
+
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
             session.mount("https://", HTTPSystemCertsAdapter())
-            return session.post(url, data=data, json=json, **kwargs)
+            resp = session.post(url, data=data, json=json, **kwargs)
 
     else:
         try:
-            return requests.post(url, data=data, json=json, **kwargs)
+            resp = requests.post(url, data=data, json=json, **kwargs)
         except requests.exceptions.SSLError as ex:
             if "SSLCertVerificationError" not in str(ex):
                 raise ex
@@ -70,9 +180,11 @@ def request_post(
             LOG.warning(
                 "SSL error occurred, falling back to system SSL certificates: %s", ex
             )
-            with requests.Session() as session:
-                session.mount("https://", HTTPSystemCertsAdapter())
-                return session.post(url, data=data, json=json, **kwargs)
+            return request_post(url, data=data, json=json, **kwargs)
+
+    _log_debug_response(resp)
+
+    return resp
 
 
 def request_get(
@@ -82,13 +194,21 @@ def request_get(
 ) -> requests.Response:
     global USE_SYSTEM_CERTS
 
+    _log_debug_request(
+        "GET",
+        url,
+        params=kwargs.get("params"),
+        headers=kwargs.get("headers"),
+        timeout=kwargs.get("timeout"),
+    )
+
     if USE_SYSTEM_CERTS:
         with requests.Session() as session:
             session.mount("https://", HTTPSystemCertsAdapter())
-            return session.get(url, params=params, **kwargs)
+            resp = session.get(url, params=params, **kwargs)
     else:
         try:
-            return requests.get(url, params=params, **kwargs)
+            resp = requests.get(url, params=params, **kwargs)
         except requests.exceptions.SSLError as ex:
             if "SSLCertVerificationError" not in str(ex):
                 raise ex
@@ -97,15 +217,17 @@ def request_get(
             LOG.warning(
                 "SSL error occurred, falling back to system SSL certificates: %s", ex
             )
-            with requests.Session() as session:
-                session.mount("https://", HTTPSystemCertsAdapter())
-                return session.get(url, params=params, **kwargs)
+            resp = request_get(url, params=params, **kwargs)
+
+    _log_debug_response(resp)
+
+    return resp
 
 
 def get_upload_token(email: str, password: str) -> requests.Response:
     resp = request_post(
         f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
-        params={"access_token": MAPILLARY_CLIENT_TOKEN},
+        headers={"Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}"},
         json={"email": email, "password": password, "locale": "en_US"},
         timeout=REQUESTS_TIMEOUT,
     )

mapillary_tools/commands/__main__.py

@@ -5,7 +5,9 @@ import sys
 import typing as T
 from pathlib import Path
 
-from .. import constants, exceptions, VERSION
+import requests
+
+from .. import api_v4, constants, exceptions, VERSION
 from . import (
     authenticate,
     process,
@@ -160,11 +162,14 @@ def main():
 
     try:
         args.func(argvars)
-    except exceptions.MapillaryUserError as exc:
+    except requests.HTTPError as ex:
+        LOG.error("%s: %s", ex.__class__.__name__, api_v4.readable_http_error(ex))
+
+    except exceptions.MapillaryUserError as ex:
         LOG.error(
-            "%s: %s", exc.__class__.__name__, exc, exc_info=log_level == logging.DEBUG
+            "%s: %s", ex.__class__.__name__, ex, exc_info=log_level == logging.DEBUG
         )
-        sys.exit(exc.exit_code)
+        sys.exit(ex.exit_code)
 
 
 if __name__ == "__main__":

mapillary_tools/commands/process_and_upload.py

@@ -15,5 +15,6 @@ class Command:
             # \x00 is a special path similiar to /dev/null
             # it tells process command do not write anything
             args["desc_path"] = "\x00"
+
         ProcessCommand().run(args)
         UploadCommand().run(args)

mapillary_tools/constants.py

@@ -45,6 +45,6 @@ GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
 # Max number of images per sequence
 MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
 # Max file size per sequence (sum of image filesizes in the sequence)
-MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "10G")
+MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
 # Max number of pixels per sequence (sum of image pixels in the sequence)
 MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")

mapillary_tools/geotag/geotag_images_from_gpx_file.py

@@ -25,7 +25,13 @@ class GeotagImagesFromGPXFile(GeotagImagesFromGeneric):
         num_processes: T.Optional[int] = None,
     ):
         super().__init__()
-        tracks = parse_gpx(source_path)
+        try:
+            tracks = parse_gpx(source_path)
+        except Exception as ex:
+            raise RuntimeError(
+                f"Error parsing GPX {source_path}: {ex.__class__.__name__}: {ex}"
+            )
+
         if 1 < len(tracks):
             LOG.warning(
                 "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",

mapillary_tools/upload.py

@@ -47,25 +47,6 @@ class UploadError(Exception):
         super().__init__(str(inner_ex))
 
 
-class UploadHTTPError(Exception):
-    pass
-
-
-def wrap_http_exception(ex: requests.HTTPError):
-    req = ex.request
-    resp = ex.response
-    if isinstance(resp, requests.Response) and isinstance(req, requests.Request):
-        lines = [
-            f"{req.method} {resp.url}",
-            f"> HTTP Status: {resp.status_code}",
-            str(resp.content),
-        ]
-    else:
-        lines = []
-
-    return UploadHTTPError("\n".join(lines))
-
-
 def _load_validate_metadatas_from_desc_path(
     desc_path: T.Optional[str], import_paths: T.Sequence[Path]
 ) -> T.List[types.Metadata]:
@@ -175,18 +156,12 @@ def fetch_user_items(
                 "Found multiple Mapillary accounts. Please specify one with --user_name"
             )
     else:
-        try:
-            user_items = authenticate.authenticate_user(user_name)
-        except requests.HTTPError as exc:
-            raise wrap_http_exception(exc) from exc
+        user_items = authenticate.authenticate_user(user_name)
 
     if organization_key is not None:
-        try:
-            resp = api_v4.fetch_organization(
-                user_items["user_upload_token"], organization_key
-            )
-        except requests.HTTPError as ex:
-            raise wrap_http_exception(ex) from ex
+        resp = api_v4.fetch_organization(
+            user_items["user_upload_token"], organization_key
+        )
         org = resp.json()
         LOG.info("Uploading to organization: %s", json.dumps(org))
         user_items = T.cast(
@@ -430,15 +405,12 @@ def _api_logging_finished(summary: T.Dict):
     action: api_v4.ActionType = "upload_finished_upload"
     LOG.debug("API Logging for action %s: %s", action, summary)
     try:
-        api_v4.log_event(
-            action,
-            summary,
-        )
+        api_v4.log_event(action, summary)
     except requests.HTTPError as exc:
         LOG.warning(
-            "Error from API Logging for action %s",
+            "HTTPError from API Logging for action %s: %s",
             action,
-            exc_info=wrap_http_exception(exc),
+            api_v4.readable_http_error(exc),
         )
     except Exception:
         LOG.warning("Error from API Logging for action %s", action, exc_info=True)
@@ -452,16 +424,12 @@ def _api_logging_failed(payload: T.Dict, exc: Exception):
     action: api_v4.ActionType = "upload_failed_upload"
     LOG.debug("API Logging for action %s: %s", action, payload)
     try:
-        api_v4.log_event(
-            action,
-            payload_with_reason,
-        )
+        api_v4.log_event(action, payload_with_reason)
     except requests.HTTPError as exc:
-        wrapped_exc = wrap_http_exception(exc)
         LOG.warning(
-            "Error from API Logging for action %s",
+            "HTTPError from API Logging for action %s: %s",
             action,
-            exc_info=wrapped_exc,
+            api_v4.readable_http_error(exc),
         )
     except Exception:
         LOG.warning("Error from API Logging for action %s", action, exc_info=True)
@@ -678,7 +646,7 @@ def upload(
                     raise exceptions.MapillaryUploadUnauthorizedError(
                         debug_info.get("message")
                     ) from inner_ex
-            raise wrap_http_exception(inner_ex) from inner_ex
+            raise inner_ex
 
         raise inner_ex
 

mapillary_tools/upload_api_v4.py

@@ -1,16 +1,19 @@
 import enum
 import io
-import json
-import logging
 import os
 import random
 import typing as T
+import uuid
 
 import requests
 
-from .api_v4 import MAPILLARY_GRAPH_API_ENDPOINT, request_get, request_post
+from .api_v4 import (
+    MAPILLARY_GRAPH_API_ENDPOINT,
+    request_get,
+    request_post,
+    REQUESTS_TIMEOUT,
+)
 
-LOG = logging.getLogger(__name__)
 MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
     "MAPILLARY_UPLOAD_ENDPOINT", "https://rupload.facebook.com/mapillary_public_uploads"
 )
@@ -21,7 +24,6 @@ DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16  # 16MB
 # i.e. if your the server does not respond within this timeout, it will throw:
 # ConnectionError: ('Connection aborted.', timeout('The write operation timed out'))
 # So let us make sure the largest possible chunks can be uploaded before this timeout for now,
-REQUESTS_TIMEOUT = (20, 20)  # 20 seconds
 UPLOAD_REQUESTS_TIMEOUT = (30 * 60, 30 * 60)  # 30 minutes
 
 
@@ -31,55 +33,33 @@ class ClusterFileType(enum.Enum):
     CAMM = "mly_camm_video"
 
 
-def _sanitize_headers(headers: T.Dict):
-    return {
-        k: v
-        for k, v in headers.items()
-        if k.lower() not in ["authorization", "cookie", "x-fb-access-token"]
-    }
-
-
-_S = T.TypeVar("_S", str, bytes)
-
-
-def _truncate_end(s: _S) -> _S:
-    MAX_LENGTH = 512
-    if MAX_LENGTH < len(s):
-        if isinstance(s, bytes):
-            return s[:MAX_LENGTH] + b"..."
-        else:
-            return str(s[:MAX_LENGTH]) + "..."
-    else:
-        return s
-
-
 class UploadService:
     user_access_token: str
-    entity_size: int
     session_key: str
     callbacks: T.List[T.Callable[[bytes, T.Optional[requests.Response]], None]]
     cluster_filetype: ClusterFileType
     organization_id: T.Optional[T.Union[str, int]]
     chunk_size: int
 
+    MIME_BY_CLUSTER_TYPE: T.Dict[ClusterFileType, str] = {
+        ClusterFileType.ZIP: "application/zip",
+        ClusterFileType.BLACKVUE: "video/mp4",
+        ClusterFileType.CAMM: "video/mp4",
+    }
+
     def __init__(
         self,
         user_access_token: str,
         session_key: str,
-        entity_size: int,
         organization_id: T.Optional[T.Union[str, int]] = None,
         cluster_filetype: ClusterFileType = ClusterFileType.ZIP,
         chunk_size: int = DEFAULT_CHUNK_SIZE,
     ):
-        if entity_size <= 0:
-            raise ValueError(f"Expect positive entity size but got {entity_size}")
-
         if chunk_size <= 0:
             raise ValueError("Expect positive chunk size")
 
         self.user_access_token = user_access_token
         self.session_key = session_key
-        self.entity_size = entity_size
         self.organization_id = organization_id
         #  validate the input
         self.cluster_filetype = ClusterFileType(cluster_filetype)
@@ -91,13 +71,11 @@ class UploadService:
             "Authorization": f"OAuth {self.user_access_token}",
         }
         url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-        LOG.debug("GET %s", url)
         resp = request_get(
             url,
             headers=headers,
             timeout=REQUESTS_TIMEOUT,
         )
-        LOG.debug("HTTP response %s: %s", resp.status_code, resp.content)
         resp.raise_for_status()
         data = resp.json()
         return data["offset"]
@@ -107,54 +85,63 @@ class UploadService:
         data: T.IO[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
-        if offset is None:
-            offset = self.fetch_offset()
-
-        entity_type_map: T.Dict[ClusterFileType, str] = {
-            ClusterFileType.ZIP: "application/zip",
-            ClusterFileType.BLACKVUE: "video/mp4",
-            ClusterFileType.CAMM: "video/mp4",
-        }
-
-        entity_type = entity_type_map[self.cluster_filetype]
-
-        data.seek(offset, io.SEEK_CUR)
+        chunks = self._chunkize_byte_stream(data)
+        return self.upload_chunks(chunks, offset=offset)
 
+    def _chunkize_byte_stream(
+        self, stream: T.IO[bytes]
+    ) -> T.Generator[bytes, None, None]:
         while True:
-            chunk = data.read(self.chunk_size)
-            # it is possible to upload an empty chunk here
-            # in order to return the handle
-            headers = {
-                "Authorization": f"OAuth {self.user_access_token}",
-                "Offset": f"{offset}",
-                "X-Entity-Length": str(self.entity_size),
-                "X-Entity-Name": self.session_key,
-                "X-Entity-Type": entity_type,
-            }
-            url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
-            LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-            resp = request_post(
-                url,
-                headers=headers,
-                data=chunk,
-                timeout=UPLOAD_REQUESTS_TIMEOUT,
-            )
-            LOG.debug(
-                "HTTP response %s: %s", resp.status_code, _truncate_end(resp.content)
-            )
-            resp.raise_for_status()
-            offset += len(chunk)
-            LOG.debug("The next offset will be: %s", offset)
+            data = stream.read(self.chunk_size)
+            if not data:
+                break
+            yield data
+
+    def _offset_chunks(
+        self, chunks: T.Iterable[bytes], offset: int
+    ) -> T.Generator[bytes, None, None]:
+        assert offset >= 0, f"Expect non-negative offset but got {offset}"
+
+        for chunk in chunks:
+            if offset:
+                if offset < len(chunk):
+                    yield chunk[offset:]
+                    offset = 0
+                else:
+                    offset -= len(chunk)
+            else:
+                yield chunk
+
+    def _attach_callbacks(
+        self, chunks: T.Iterable[bytes]
+    ) -> T.Generator[bytes, None, None]:
+        for chunk in chunks:
+            yield chunk
             for callback in self.callbacks:
-                callback(chunk, resp)
-            # we can assert that offset == self.fetch_offset(session_key)
-            # otherwise, server will throw
+                callback(chunk, None)
 
-            if not chunk:
-                break
+    def upload_chunks(
+        self,
+        chunks: T.Iterable[bytes],
+        offset: T.Optional[int] = None,
+    ) -> str:
+        if offset is None:
+            offset = self.fetch_offset()
+
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
 
-        assert offset == self.entity_size, (
-            f"Offset ends at {offset} but the entity size is {self.entity_size}"
+        headers = {
+            "Authorization": f"OAuth {self.user_access_token}",
+            "Offset": f"{offset}",
+            "X-Entity-Name": self.session_key,
+            "X-Entity-Type": self.MIME_BY_CLUSTER_TYPE[self.cluster_filetype],
+        }
+        url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
+        resp = request_post(
+            url,
+            headers=headers,
+            data=chunks,
+            timeout=UPLOAD_REQUESTS_TIMEOUT,
         )
 
         payload = resp.json()
@@ -178,14 +165,12 @@ class UploadService:
 
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
 
-        LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
         resp = request_post(
             url,
             headers=headers,
             json=data,
             timeout=REQUESTS_TIMEOUT,
         )
-        LOG.debug("HTTP response %s: %s", resp.status_code, _truncate_end(resp.content))
 
         resp.raise_for_status()
 
@@ -209,35 +194,30 @@ class FakeUploadService(UploadService):
         )
         self._error_ratio = 0.1
 
-    def upload(
+    def upload_chunks(
         self,
-        data: T.IO[bytes],
+        chunks: T.Iterable[bytes],
         offset: T.Optional[int] = None,
     ) -> str:
         if offset is None:
             offset = self.fetch_offset()
+
+        chunks = self._attach_callbacks(self._offset_chunks(chunks, offset))
+
         os.makedirs(self._upload_path, exist_ok=True)
         filename = os.path.join(self._upload_path, self.session_key)
         with open(filename, "ab") as fp:
-            data.seek(offset, io.SEEK_CUR)
-            while True:
-                chunk = data.read(self.chunk_size)
-                if not chunk:
-                    break
-                # fail here means nothing uploaded
+            for chunk in chunks:
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Failed to upload with error ratio {self._error_ratio}"
                     )
                 fp.write(chunk)
-                # fail here means patially uploaded
                 if random.random() <= self._error_ratio:
                     raise requests.ConnectionError(
                         f"TEST ONLY: Partially uploaded with error ratio {self._error_ratio}"
                     )
-                for callback in self.callbacks:
-                    callback(chunk, None)
-        return self.session_key
+        return uuid.uuid4().hex
 
     def finish(self, _: str) -> str:
         return "0"

mapillary_tools/uploader.py

@@ -195,7 +195,6 @@ class Uploader:
                 upload_api_v4.FakeUploadService(
                     user_access_token=self.user_items["user_upload_token"],
                     session_key=session_key,
-                    entity_size=entity_size,
                     organization_id=self.user_items.get("MAPOrganizationKey"),
                     cluster_filetype=cluster_filetype,
                     chunk_size=self.chunk_size,
@@ -205,7 +204,6 @@ class Uploader:
             upload_service = upload_api_v4.UploadService(
                 user_access_token=self.user_items["user_upload_token"],
                 session_key=session_key,
-                entity_size=entity_size,
                 organization_id=self.user_items.get("MAPOrganizationKey"),
                 cluster_filetype=cluster_filetype,
                 chunk_size=self.chunk_size,

mapillary_tools/video_data_extraction/extractors/camm_parser.py

@@ -13,8 +13,12 @@ class CammParser(BaseParser):
     parser_label = "camm"
 
     @functools.cached_property
-    def __camera_info(self) -> T.Tuple[str, str]:
-        with self.videoPath.open("rb") as fp:
+    def _camera_info(self) -> T.Tuple[str, str]:
+        source_path = self.geotag_source_path
+        if not source_path:
+            return "", ""
+
+        with source_path.open("rb") as fp:
             return camm_parser.extract_camera_make_and_model(fp)
 
     def extract_points(self) -> T.Sequence[geo.Point]:
@@ -28,15 +32,7 @@ class CammParser(BaseParser):
                 return []
 
     def extract_make(self) -> T.Optional[str]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return None
-        with source_path.open("rb") as _fp:
-            return self.__camera_info[0] or None
+        return self._camera_info[0] or None
 
     def extract_model(self) -> T.Optional[str]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return None
-        with source_path.open("rb") as _fp:
-            return self.__camera_info[1] or None
+        return self._camera_info[1] or None

mapillary_tools/video_data_extraction/extractors/gpx_parser.py

@@ -20,7 +20,13 @@ class GpxParser(BaseParser):
         if not path:
             return []
 
-        gpx_tracks = geotag_images_from_gpx_file.parse_gpx(path)
+        try:
+            gpx_tracks = geotag_images_from_gpx_file.parse_gpx(path)
+        except Exception as ex:
+            raise RuntimeError(
+                f"Error parsing GPX {path}: {ex.__class__.__name__}: {ex}"
+            )
+
         if 1 < len(gpx_tracks):
             LOG.warning(
                 "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",
@@ -32,40 +38,71 @@ class GpxParser(BaseParser):
         if not gpx_points:
             return gpx_points
 
+        offset = self._synx_gpx_by_first_gps_timestamp(gpx_points)
+
+        self._rebase_times(gpx_points, offset=offset)
+
+        return gpx_points
+
+    def _synx_gpx_by_first_gps_timestamp(
+        self, gpx_points: T.Sequence[geo.Point]
+    ) -> float:
+        offset: float = 0.0
+
+        if not gpx_points:
+            return offset
+
         first_gpx_dt = datetime.datetime.fromtimestamp(
             gpx_points[0].time, tz=datetime.timezone.utc
         )
         LOG.info("First GPX timestamp: %s", first_gpx_dt)
 
         # Extract first GPS timestamp (if found) for synchronization
-        offset: float = 0.0
-        parser = GenericVideoParser(self.videoPath, self.options, self.parserOptions)
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
         gps_points = parser.extract_points()
-        if gps_points:
-            first_gps_point = gps_points[0]
-            if isinstance(first_gps_point, telemetry.GPSPoint):
-                if first_gps_point.epoch_time is not None:
-                    first_gps_dt = datetime.datetime.fromtimestamp(
-                        first_gps_point.epoch_time, tz=datetime.timezone.utc
-                    )
-                    LOG.info("First GPS timestamp: %s", first_gps_dt)
-                    offset = gpx_points[0].time - first_gps_point.epoch_time
-                    if offset:
-                        LOG.warning(
-                            "Found offset between GPX %s and video GPS timestamps %s: %s seconds",
-                            first_gpx_dt,
-                            first_gps_dt,
-                            offset,
-                        )
 
-        self._rebase_times(gpx_points, offset=offset)
+        if not gps_points:
+            LOG.warning(
+                "Skip GPX synchronization because no GPS found in video %s",
+                self.videoPath,
+            )
+            return offset
 
-        return gpx_points
+        first_gps_point = gps_points[0]
+        if isinstance(first_gps_point, telemetry.GPSPoint):
+            if first_gps_point.epoch_time is not None:
+                first_gps_dt = datetime.datetime.fromtimestamp(
+                    first_gps_point.epoch_time, tz=datetime.timezone.utc
+                )
+                LOG.info("First GPS timestamp: %s", first_gps_dt)
+                offset = gpx_points[0].time - first_gps_point.epoch_time
+                if offset:
+                    LOG.warning(
+                        "Found offset between GPX %s and video GPS timestamps %s: %s seconds",
+                        first_gpx_dt,
+                        first_gps_dt,
+                        offset,
+                    )
+                else:
+                    LOG.info(
+                        "GPX and GPS are perfectly synchronized (all starts from %s)",
+                        first_gpx_dt,
+                    )
+            else:
+                LOG.warning(
+                    "Skip GPX synchronization because no GPS epoch time found in video %s",
+                    self.videoPath,
+                )
+
+        return offset
 
     def extract_make(self) -> T.Optional[str]:
-        parser = GenericVideoParser(self.videoPath, self.options, self.parserOptions)
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
         return parser.extract_make()
 
     def extract_model(self) -> T.Optional[str]:
-        parser = GenericVideoParser(self.videoPath, self.options, self.parserOptions)
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
         return parser.extract_model()

{mapillary_tools-0.13.1a1.dist-info → mapillary_tools-0.13.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mapillary_tools
-Version: 0.13.1a1
+Version: 0.13.3
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary

{mapillary_tools-0.13.1a1.dist-info → mapillary_tools-0.13.3.dist-info}/RECORD

@@ -1,8 +1,8 @@
-mapillary_tools/__init__.py,sha256=Qvs9oGT6nkWrVfCfFMNlCwq4PM3sc2U_WryNkThbhUU,21
-mapillary_tools/api_v4.py,sha256=zhRtgx3EnzgqtjziRhvFq3ONvsPaB9hROsuKFcf_pFo,5197
+mapillary_tools/__init__.py,sha256=tOdTWrFVTLaGb-_qr6fgRv_nOA5NidkflFCzgUk9sdQ,19
+mapillary_tools/api_v4.py,sha256=bbu2VuzkUysLOBcjyGD_qQqnS1zsYgYDJAqBhQEVESw,7786
 mapillary_tools/authenticate.py,sha256=LCFcs6LqZmXaYkTUEKgGfmqytWdh5v_L3KXB48ojOZ4,3090
 mapillary_tools/config.py,sha256=jCjaK4jJaTY4AV4qf_b_tcxn5LA_uPsEWlGIdm2zw6g,2103
-mapillary_tools/constants.py,sha256=CZpKd9wUxlTC3lq7h3Mi5J1NpPlLYvgp1r0_sFUJKGk,2452
+mapillary_tools/constants.py,sha256=KKQMwzpN2z7wka6lhZv-agdxNldaEQqvQXu3Kg5XuUk,2453
 mapillary_tools/exceptions.py,sha256=Mh1tgVEFTSMnYEzrl9x7b95fW9Z3SPVD_YMEl7r8I0I,2693
 mapillary_tools/exif_read.py,sha256=F60A0-T8XSBHvFKgVIrUz_ZWKQrTFWrtj3c6siB0IMg,28707
 mapillary_tools/exif_write.py,sha256=3PawLnBOY8Z86TYiA_F4LxRhe5Ui6CTNhxYm9yeJNX8,8786
@@ -17,17 +17,17 @@ mapillary_tools/process_sequence_properties.py,sha256=5oYEjz9crnLVQtCkxbwn57Tkeu
 mapillary_tools/sample_video.py,sha256=dpdX7bUNEmcrz-3gh3Y3awnTDX66pChbTKuF8qGfeCI,14400
 mapillary_tools/telemetry.py,sha256=WpBGPF_GMPjM_EFqXIutFtpDFL9wj7yEzGNGnfQZUo8,1255
 mapillary_tools/types.py,sha256=6kww2UdKM6YzabYbc862BYzEWtxL2hhxCRFfeDiUtF0,22074
-mapillary_tools/upload.py,sha256=8dQ3ZWsjau1_xZN3ssjGGkBnLKbKIhjC91-zWstYlD8,24439
-mapillary_tools/upload_api_v4.py,sha256=1WvoUis92KDXbqfoyvyyDmiCqwXezYkMJZhnYaVm3BA,8560
-mapillary_tools/uploader.py,sha256=VieDKi51wdXTIhN7x_mcuQeHESUyFlF5cgB-TAnF4g0,14093
+mapillary_tools/upload.py,sha256=C8sWMCYOVzjXpvMsEcx6dpZ47ezKaKV61yFwZs286oo,23639
+mapillary_tools/upload_api_v4.py,sha256=qN6yRpi-qXycRLgIOJqixjXqkYvgt-LJX7NXbQKXZaA,7502
+mapillary_tools/uploader.py,sha256=DBHso4QIP5nsZFDynLjkUvytzwpEOPnOlfeDyA6rTBk,14007
 mapillary_tools/utils.py,sha256=VNtK1tAb3Hh8y3P5e5Y3iewREkIoLDa3C2myRYcF2lY,5970
 mapillary_tools/camm/camm_builder.py,sha256=TXZfhu3xGjtrLEWnB14D7aSOrHOoSJef24YSLApiIfY,10631
 mapillary_tools/camm/camm_parser.py,sha256=RaCWeLvS_AyHD6B6wDUu9DAsdfByVHMAPTqEqjtFibE,9734
 mapillary_tools/commands/__init__.py,sha256=41CFrPLGlG3566uhxssEF3TGAtSpADFPPcDMHbViU0E,171
-mapillary_tools/commands/__main__.py,sha256=VdWkx1ekPH-88Ybe78IcO9FWpZ5cUhsbGRw7LuzQObU,4832
+mapillary_tools/commands/__main__.py,sha256=iDehfT38k4D283BIqy8TDD4K_q42jNhkrUNzWEXuZP4,4974
 mapillary_tools/commands/authenticate.py,sha256=4aVvAQal_mqtm2NEMBt5aKLahi0iRdO8b7WSBf6jokA,1136
 mapillary_tools/commands/process.py,sha256=VxcvQpYHPw7QfT9dNwBLV1jWQ-1w4GtVNVPpmu4Sx9s,10578
-mapillary_tools/commands/process_and_upload.py,sha256=osJv1TVHYAG5E-EsA0nB1F3RkKXMadLR2t1EGK0Ifqw,654
+mapillary_tools/commands/process_and_upload.py,sha256=cbYr6g4sDxZ3A9jLr-GZdHZObJ8ZKS43jAdZgvqAkWk,655
 mapillary_tools/commands/sample_video.py,sha256=bTJmlDsajkC-QJ_ZO_scdD4R664zs-r_dh-x2PlOgyY,3281
 mapillary_tools/commands/upload.py,sha256=JIWgxupV3ppLvPi1iE7UVaE1302JGcIOvnuNt1Y7YEw,1671
 mapillary_tools/commands/video_process.py,sha256=-wQeeIwWXPmy81HQHam5A0huMLRHknkEFa_V1OwElU4,890
@@ -40,7 +40,7 @@ mapillary_tools/geotag/geotag_images_from_exif.py,sha256=hCgBwZABk2tbBQC3cHQBV5p
 mapillary_tools/geotag/geotag_images_from_exiftool.py,sha256=a-c4H8VIyPdJkfUIvJho0phR0QU0zN8-lSyiCz0wc4s,3981
 mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py,sha256=nRVAjgTJwx_eCaSBpPCgcIaZs3EYgGueYxSS9XhKv40,3350
 mapillary_tools/geotag/geotag_images_from_gpx.py,sha256=S9Pw6FvP5kRSpHUnKUYKXmw0CHa9V92UmrS_MJfbjS4,9053
-mapillary_tools/geotag/geotag_images_from_gpx_file.py,sha256=zEbC0kVf_iw9ioIyJLL-gYN_QvAOEdAoEczCBkizl38,5122
+mapillary_tools/geotag/geotag_images_from_gpx_file.py,sha256=-vTbZ1HufZzJCd8VvukdTjsJRcymtfld2W5t65VSG5E,5300
 mapillary_tools/geotag/geotag_images_from_nmea_file.py,sha256=dDdHnJInQ_WN3ZRf-w44NSBElDLPs7XYBiimvE2iCNo,1651
 mapillary_tools/geotag/geotag_images_from_video.py,sha256=XsaWOFChGItl-j1UbKM4hNjUqN29pVNbMpGT_BvI-o8,3306
 mapillary_tools/geotag/geotag_videos_from_exiftool_video.py,sha256=fkkWou1WFt3ft024399vis9No2cxrwot7Pg5HBw7o7s,5225
@@ -60,16 +60,16 @@ mapillary_tools/video_data_extraction/extract_video_data.py,sha256=_2BBdSYeYKR4B
 mapillary_tools/video_data_extraction/video_data_parser_factory.py,sha256=qaJHvLgwI5lukJncMd8ggxeSxXOiVzBSJO5GlGQYiXY,1134
 mapillary_tools/video_data_extraction/extractors/base_parser.py,sha256=s7Xuwg4I5JZ27oL4ebMSdo093plAXfZ-6uDQ_h97WHY,2134
 mapillary_tools/video_data_extraction/extractors/blackvue_parser.py,sha256=jAcGyF6PML2EdJ4zle8cR12QeTRZc5qxlz8_4gcTZPU,1089
-mapillary_tools/video_data_extraction/extractors/camm_parser.py,sha256=sr8oDbC9lGyHHp8qkpQBmWOf4U_umlokaJM0kPVKgzw,1314
+mapillary_tools/video_data_extraction/extractors/camm_parser.py,sha256=YMiViocXSVlfn8_qm1jcwSJhnnEaK8v5ADHwo2YXe10,1117
 mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py,sha256=PFNCRk9pGrPIfVwLMcnzmVNMITVjNHhbrOOMwxaSstg,2270
 mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py,sha256=Tt0h4TiCKocERWMlRXzlpoaA_WJ_4b20MgMLGYNl4AM,1734
 mapillary_tools/video_data_extraction/extractors/generic_video_parser.py,sha256=34O6Km5kNDoJNJtIUOwtAzzMntuqkSZJfeli7caWSkA,1693
 mapillary_tools/video_data_extraction/extractors/gopro_parser.py,sha256=IVnTyquSraTUaG9rxbJfVWc1-drdY5PaHn5urh3IBk4,1325
-mapillary_tools/video_data_extraction/extractors/gpx_parser.py,sha256=FRysFhN2aa0MGnzLO7pcZkwo8790GUelIZg9pddZvic,2594
+mapillary_tools/video_data_extraction/extractors/gpx_parser.py,sha256=FNrdnXl48k8I1I5fGwYsClhfFEHVsooRLRboUYECv3I,3811
 mapillary_tools/video_data_extraction/extractors/nmea_parser.py,sha256=raSXavBvP-0LJCB_TwLL0mOv2uHSsB744igTsaKAaGc,658
-mapillary_tools-0.13.1a1.dist-info/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
-mapillary_tools-0.13.1a1.dist-info/METADATA,sha256=wc2o92Vqxo1fZ_sjWlV5lvJZByclruFL2h20jK0LxKI,19760
-mapillary_tools-0.13.1a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-mapillary_tools-0.13.1a1.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
-mapillary_tools-0.13.1a1.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
-mapillary_tools-0.13.1a1.dist-info/RECORD,,
+mapillary_tools-0.13.3.dist-info/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
+mapillary_tools-0.13.3.dist-info/METADATA,sha256=qJo2HTakj1kI0l5wl_CMb8pcRP68REFJkO1oJVPdtu4,19758
+mapillary_tools-0.13.3.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+mapillary_tools-0.13.3.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
+mapillary_tools-0.13.3.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
+mapillary_tools-0.13.3.dist-info/RECORD,,

{mapillary_tools-0.13.1a1.dist-info → mapillary_tools-0.13.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (76.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any