mapillary-tools 0.14.5__py3-none-any.whl → 0.14.6__py3-none-any.whl

mapillary_tools/__init__.py

@@ -1 +1,6 @@
-VERSION = "0.14.5"
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
+VERSION = "0.14.6"

mapillary_tools/api_v4.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import enum

mapillary_tools/authenticate.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import getpass
@@ -7,7 +12,6 @@ import sys
 import typing as T
 
 import jsonschema
-
 import requests
 
 from . import api_v4, config, constants, exceptions, http

mapillary_tools/blackvue_parser.py

@@ -1,7 +1,12 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import dataclasses
-
+import datetime
 import json
 import logging
 import re
@@ -9,7 +14,7 @@ import typing as T
 
 import pynmea2
 
-from . import geo
+from . import telemetry
 from .mp4 import simple_mp4_parser as sparser
 
 
@@ -32,7 +37,7 @@ NMEA_LINE_REGEX = re.compile(
 class BlackVueInfo:
     # None and [] are equivalent here. Use None as default because:
     # ValueError: mutable default <class 'list'> for field gps is not allowed: use default_factory
-    gps: list[geo.Point] | None = None
+    gps: list[telemetry.GPSPoint] | None = None
     make: str = "BlackVue"
     model: str = ""
 
@@ -50,9 +55,11 @@ def extract_blackvue_info(fp: T.BinaryIO) -> BlackVueInfo | None:
     points.sort(key=lambda p: p.time)
 
     if points:
+        # Convert the time field to relative time to the first point
+        # epoch_time stays as the original time in seconds
         first_point_time = points[0].time
         for p in points:
-            p.time = (p.time - first_point_time) / 1000
+            p.time = p.time - first_point_time
 
     # Camera model
     try:
@@ -114,25 +121,64 @@ def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
         return ""
 
 
-def _parse_gps_box(gps_data: bytes) -> list[geo.Point]:
+def _compute_timezone_offset_from_rmc(
+    epoch_sec: float, message: pynmea2.NMEASentence
+) -> float | None:
     """
-    >>> list(_parse_gps_box(b"[1623057074211]$GPGGA,202530.00,5109.0262,N,11401.8407,W,5,40,0.5,1097.36,M,-17.00,M,18,TSTR*61"))
-    [Point(time=1623057074211, lat=51.150436666666664, lon=-114.03067833333333, alt=1097.36, angle=None)]
+    Compute timezone offset from an RMC message which has full date+time.
 
-    >>> list(_parse_gps_box(b"[1629874404069]$GNGGA,175322.00,3244.53126,N,11710.97811,W,1,12,0.84,17.4,M,-34.0,M,,*45"))
-    [Point(time=1629874404069, lat=32.742187666666666, lon=-117.1829685, alt=17.4, angle=None)]
+    Returns the offset to add to camera epoch to get correct UTC time,
+    or None if this message doesn't have the required datetime.
+    """
+    if (
+        message.sentence_type != "RMC"
+        or not hasattr(message, "datetime")
+        or not message.datetime
+    ):
+        return None
 
-    >>> list(_parse_gps_box(b"[1629874404069]$GNGLL,4404.14012,N,12118.85993,W,001037.00,A,A*67"))
-    [Point(time=1629874404069, lat=44.069002, lon=-121.31433216666667, alt=None, angle=None)]
+    correct_epoch = message.datetime.replace(tzinfo=datetime.timezone.utc).timestamp()
+    # Rounding needed to avoid floating point precision issues
+    return round(correct_epoch - epoch_sec, 3)
 
-    >>> list(_parse_gps_box(b"[1629874404069]$GNRMC,001031.00,A,4404.13993,N,12118.86023,W,0.146,,100117,,,A*7B"))
-    [Point(time=1629874404069, lat=44.06899883333333, lon=-121.31433716666666, alt=None, angle=None)]
 
-    >>> list(_parse_gps_box(b"[1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]"))
-    []
+def _compute_timezone_offset_from_time_only(
+    epoch_sec: float, message: pynmea2.NMEASentence
+) -> float | None:
+    """
+    Compute timezone offset from GGA/GLL which only have time (no date).
+
+    Uses the date from camera epoch and replaces the time with NMEA time assuming camera date is correct.
+    Handles day boundary when camera and GPS times differ by more than 12 hours.
     """
-    points_by_sentence_type: dict[str, list[geo.Point]] = {}
+    if not hasattr(message, "timestamp") or not message.timestamp:
+        return None
 
+    camera_dt = datetime.datetime.fromtimestamp(epoch_sec, tz=datetime.timezone.utc)
+
+    nmea_time = message.timestamp
+    corrected_dt = camera_dt.replace(
+        hour=nmea_time.hour,
+        minute=nmea_time.minute,
+        second=nmea_time.second,
+        microsecond=getattr(nmea_time, "microsecond", 0),
+    )
+    # Handle day boundary e.g. camera time is 23:00, GPS time is 01:00 or vice versa
+    camera_secs = camera_dt.hour * 3600 + camera_dt.minute * 60 + camera_dt.second
+    nmea_secs = nmea_time.hour * 3600 + nmea_time.minute * 60 + nmea_time.second
+    if camera_secs - nmea_secs > 12 * 3600:
+        corrected_dt += datetime.timedelta(days=1)
+    elif nmea_secs - camera_secs > 12 * 3600:
+        corrected_dt -= datetime.timedelta(days=1)
+
+    # Rounding needed to avoid floating point precision issues
+    return round(corrected_dt.timestamp() - epoch_sec, 3)
+
+
+def _parse_nmea_lines(
+    gps_data: bytes,
+) -> T.Iterator[tuple[int, pynmea2.NMEASentence]]:
+    """Parse NMEA lines from GPS data, yielding (epoch_ms, message) tuples."""
     for line_bytes in gps_data.splitlines():
         match = NMEA_LINE_REGEX.match(line_bytes)
         if match is None:
@@ -156,29 +202,101 @@ def _parse_gps_box(gps_data: bytes) -> list[geo.Point]:
             continue
 
         epoch_ms = int(match.group(1))
+        yield epoch_ms, message
+
+
+def _parse_gps_box(gps_data: bytes) -> list[telemetry.GPSPoint]:
+    """
+    >>> list(_parse_gps_box(b"[1623057074211]$GPGGA,202530.00,5109.0262,N,11401.8407,W,5,40,0.5,1097.36,M,-17.00,M,18,TSTR*61"))
+    [GPSPoint(time=1623097530.0, lat=51.150436666666664, lon=-114.03067833333333, alt=1097.36, angle=None, epoch_time=1623097530.0, fix=<GPSFix.FIX_3D: 3>, precision=None, ground_speed=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGGA,175322.00,3244.53126,N,11710.97811,W,1,12,0.84,17.4,M,-34.0,M,,*45"))
+    [GPSPoint(time=1629914002.0, lat=32.742187666666666, lon=-117.1829685, alt=17.4, angle=None, epoch_time=1629914002.0, fix=<GPSFix.FIX_3D: 3>, precision=None, ground_speed=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGLL,4404.14012,N,12118.85993,W,001037.00,A,A*67"))
+    [GPSPoint(time=1629850237.0, lat=44.069002, lon=-121.31433216666667, alt=None, angle=None, epoch_time=1629850237.0, fix=None, precision=None, ground_speed=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNRMC,001031.00,A,4404.13993,N,12118.86023,W,0.146,,100117,,,A*7B"))
+    [GPSPoint(time=1484007031.0, lat=44.06899883333333, lon=-121.31433716666666, alt=None, angle=None, epoch_time=1484007031.0, fix=None, precision=None, ground_speed=None)]
+
+    >>> list(_parse_gps_box(b"[1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]"))
+    []
+    """
+    timezone_offset: float | None = None
+    parsed_lines: list[tuple[float, pynmea2.NMEASentence]] = []
+    first_valid_gga_gll: tuple[float, pynmea2.NMEASentence] | None = None
+
+    # First pass: collect parsed_lines and compute timezone offset from the first valid RMC message
+    for epoch_ms, message in _parse_nmea_lines(gps_data):
+        # Rounding needed to avoid floating point precision issues
+        epoch_sec = round(epoch_ms / 1000, 3)
+        parsed_lines.append((epoch_sec, message))
+        if timezone_offset is None and message.sentence_type == "RMC":
+            if hasattr(message, "is_valid") and message.is_valid:
+                timezone_offset = _compute_timezone_offset_from_rmc(epoch_sec, message)
+                if timezone_offset is not None:
+                    LOG.debug(
+                        "Computed timezone offset %.1fs from RMC (%s %s)",
+                        timezone_offset,
+                        message.datestamp,
+                        message.timestamp,
+                    )
+        # Track first valid GGA/GLL for fallback
+        if first_valid_gga_gll is None and message.sentence_type in ["GGA", "GLL"]:
+            if hasattr(message, "is_valid") and message.is_valid:
+                first_valid_gga_gll = (epoch_sec, message)
+
+    # Fallback: if no RMC found, try GGA/GLL (less reliable - no date info)
+    if timezone_offset is None and first_valid_gga_gll is not None:
+        epoch_sec, message = first_valid_gga_gll
+        timezone_offset = _compute_timezone_offset_from_time_only(epoch_sec, message)
+        if timezone_offset is not None:
+            LOG.debug(
+                "Computed timezone offset %.1fs from %s (fallback, no date info)",
+                timezone_offset,
+                message.sentence_type,
+            )
+
+    # If no offset could be determined, use 0 (camera clock assumed correct)
+    if timezone_offset is None:
+        timezone_offset = 0.0
+
+    points_by_sentence_type: dict[str, list[telemetry.GPSPoint]] = {}
+
+    # Second pass: apply offset to all GPS points
+    for epoch_sec, message in parsed_lines:
+        corrected_epoch = round(epoch_sec + timezone_offset, 3)
 
         # https://tavotech.com/gps-nmea-sentence-structure/
         if message.sentence_type in ["GGA"]:
             if not message.is_valid:
                 continue
-            point = geo.Point(
-                time=epoch_ms,
+            point = telemetry.GPSPoint(
+                time=corrected_epoch,
                 lat=message.latitude,
                 lon=message.longitude,
                 alt=message.altitude,
                 angle=None,
+                epoch_time=corrected_epoch,
+                fix=telemetry.GPSFix.FIX_3D if message.gps_qual >= 1 else None,
+                precision=None,
+                ground_speed=None,
             )
             points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
 
         elif message.sentence_type in ["RMC", "GLL"]:
             if not message.is_valid:
                 continue
-            point = geo.Point(
-                time=epoch_ms,
+            point = telemetry.GPSPoint(
+                time=corrected_epoch,
                 lat=message.latitude,
                 lon=message.longitude,
                 alt=None,
                 angle=None,
+                epoch_time=corrected_epoch,
+                fix=None,
+                precision=None,
+                ground_speed=None,
             )
             points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
 

mapillary_tools/camm/camm_builder.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import io
@@ -9,7 +14,6 @@ from ..mp4 import (
     mp4_sample_parser as sample_parser,
     simple_mp4_builder as builder,
 )
-
 from . import camm_parser
 
 

mapillary_tools/camm/camm_parser.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 # pyre-ignore-all-errors[5, 11, 16, 21, 24, 58]
 from __future__ import annotations
 

mapillary_tools/commands/__init__.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 # ruff: noqa: F401
 from . import (
     authenticate,

mapillary_tools/commands/__main__.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import argparse
 import enum
 import logging

mapillary_tools/commands/authenticate.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import argparse
 import inspect
 

mapillary_tools/commands/process.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import argparse
@@ -203,6 +208,13 @@ class Command:
             default=constants.DUPLICATE_ANGLE,
             required=False,
         )
+        group_sequence.add_argument(
+            "--skip_zigzag_check",
+            help="Skip the GPS zig-zag pattern detection check.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
 
     def run(self, vars_args: dict):
         metadatas = process_geotag_properties(

mapillary_tools/commands/process_and_upload.py

@@ -1,7 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import inspect
 
 from ..authenticate import fetch_user_items
-
 from .process import Command as ProcessCommand
 from .upload import Command as UploadCommand
 

mapillary_tools/commands/sample_video.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import argparse
 import inspect
 from pathlib import Path

mapillary_tools/commands/upload.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import inspect
 
 from .. import constants

mapillary_tools/commands/video_process.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import logging
 
 from ..types import FileType

mapillary_tools/commands/video_process_and_upload.py

@@ -1,7 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import inspect
 
 from ..authenticate import fetch_user_items
-
 from .upload import Command as UploadCommand
 from .video_process import Command as VideoProcessCommand
 

mapillary_tools/commands/zip.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 import argparse
 import inspect
 from pathlib import Path

mapillary_tools/config.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import configparser

mapillary_tools/constants.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import functools
@@ -115,7 +120,7 @@ MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
 ###### SEQUENCE PROCESSING ######
 #################################
 # In meters
-CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
+CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 100))
 # In seconds
 CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
 DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
@@ -136,6 +141,13 @@ MAX_SEQUENCE_FILESIZE: int | None = _parse_filesize(
 MAX_SEQUENCE_PIXELS: int | None = _parse_pixels(
     os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
 )
+# Zig-zag detection parameters
+ZIGZAG_WINDOW_SIZE = int(os.getenv(_ENV_PREFIX + "ZIGZAG_WINDOW_SIZE", 5))
+ZIGZAG_DEVIATION_THRESHOLD = float(
+    os.getenv(_ENV_PREFIX + "ZIGZAG_DEVIATION_THRESHOLD", 0.8)
+)
+ZIGZAG_MIN_DEVIATIONS = int(os.getenv(_ENV_PREFIX + "ZIGZAG_MIN_DEVIATIONS", 1))
+ZIGZAG_MIN_DISTANCE = float(os.getenv(_ENV_PREFIX + "ZIGZAG_MIN_DISTANCE", 30))
 
 
 ##################

mapillary_tools/exceptions.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import typing as T
@@ -107,6 +112,10 @@ class MapillaryNullIslandError(MapillaryDescriptionError):
     pass
 
 
+class MapillaryZigZagError(MapillaryDescriptionError):
+    pass
+
+
 class MapillaryUploadConnectionError(MapillaryUserError):
     exit_code = 12
 

mapillary_tools/exif_read.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 from __future__ import annotations
 
 import abc
@@ -14,6 +19,8 @@ from pathlib import Path
 import exifread
 from exifread.utils import Ratio
 
+from .utils import sanitize_serial
+
 
 LOG = logging.getLogger(__name__)
 XMP_NAMESPACES = {
@@ -24,6 +31,7 @@ XMP_NAMESPACES = {
     "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
     "x": "adobe:ns:meta/",
     "GPano": "http://ns.google.com/photos/1.0/panorama/",
+    "aux": "http://ns.adobe.com/exif/1.0/aux/",
 }
 # https://github.com/ianare/exif-py/issues/167
 EXIFREAD_LOG = logging.getLogger("exifread")
@@ -329,6 +337,10 @@ class ExifReadABC(abc.ABC):
     def extract_orientation(self) -> int:
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def extract_camera_uuid(self) -> str | None:
+        raise NotImplementedError
+
 
 class ExifReadFromXMP(ExifReadABC):
     def __init__(self, etree: et.ElementTree):
@@ -477,6 +489,39 @@ class ExifReadFromXMP(ExifReadABC):
             return 1
         return orientation
 
+    def extract_camera_uuid(self) -> str | None:
+        """
+        Extract camera unique identifier from serial number tags in XMP.
+        Builds a composite ID from body and lens serial numbers.
+        """
+        body_serial = self._extract_alternative_fields(
+            [
+                "exif:SerialNumber",
+                "exif:BodySerialNumber",
+                "exif:CameraSerialNumber",
+                "exifEX:SerialNumber",
+                "exifEX:BodySerialNumber",
+                "aux:SerialNumber",
+            ],
+            str,
+        )
+        lens_serial = self._extract_alternative_fields(
+            [
+                "exif:LensSerialNumber",
+                "exifEX:LensSerialNumber",
+                "aux:LensSerialNumber",
+            ],
+            str,
+        )
+
+        parts = [
+            s for s in [sanitize_serial(body_serial), sanitize_serial(lens_serial)] if s
+        ]
+
+        if parts:
+            return "_".join(parts)
+        return None
+
     def _extract_alternative_fields(
         self,
         fields: T.Iterable[str],
@@ -811,6 +856,38 @@ class ExifReadFromEXIF(ExifReadABC):
             return 1
         return orientation
 
+    def extract_camera_uuid(self) -> str | None:
+        """
+        Extract camera unique identifier from serial number EXIF tags.
+        Builds a composite ID from body and lens serial numbers.
+        """
+        body_serial = self._extract_alternative_fields(
+            [
+                "EXIF BodySerialNumber",
+                "EXIF SerialNumber",
+                "EXIF CameraSerialNumber",
+                "Image BodySerialNumber",
+                "MakerNote SerialNumber",
+                "MakerNote InternalSerialNumber",
+            ],
+            str,
+        )
+        lens_serial = self._extract_alternative_fields(
+            [
+                "EXIF LensSerialNumber",
+                "Image LensSerialNumber",
+            ],
+            str,
+        )
+
+        parts = [
+            s for s in [sanitize_serial(body_serial), sanitize_serial(lens_serial)] if s
+        ]
+
+        if parts:
+            return "_".join(parts)
+        return None
+
     def _extract_alternative_fields(
         self,
         fields: T.Iterable[str],
@@ -982,3 +1059,15 @@ class ExifRead(ExifReadFromEXIF):
         if val is not None:
             return val
         return None
+
+    def extract_camera_uuid(self) -> str | None:
+        val = super().extract_camera_uuid()
+        if val is not None:
+            return val
+        xmp = self._xmp_with_reason("camera_uuid")
+        if xmp is None:
+            return None
+        val = xmp.extract_camera_uuid()
+        if val is not None:
+            return val
+        return None

mapillary_tools/exif_write.py

@@ -1,3 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
 # pyre-ignore-all-errors[5, 21, 24]
 from __future__ import annotations
 
@@ -214,6 +219,18 @@ class ExifEdit:
                     else:
                         del self._ef[ifd][tag]
                         # retry later
+                elif "thumbnail is too large" in message.lower():
+                    # Handle oversized thumbnails (max 64kB per EXIF spec)
+                    if thumbnail_removed:
+                        raise exc
+                    LOG.debug(
+                        "Thumbnail too large (max 64kB) -- removing thumbnail and 1st: %s",
+                        exc,
+                    )
+                    del self._ef["thumbnail"]
+                    del self._ef["1st"]
+                    thumbnail_removed = True
+                    # retry later
                 else:
                     raise exc
             except Exception as exc: