mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

mapillary_tools/sample_video.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import logging
 import os
@@ -11,14 +13,13 @@ from . import constants, exceptions, ffmpeg as ffmpeglib, geo, types, utils
 from .exif_write import ExifEdit
 from .geotag import geotag_videos_from_video
 from .mp4 import mp4_sample_parser
-from .process_geotag_properties import GeotagSource
 
 LOG = logging.getLogger(__name__)
 
 
 def _normalize_path(
     video_import_path: Path, skip_subfolders: bool
-) -> T.Tuple[Path, T.List[Path]]:
+) -> tuple[Path, list[Path]]:
     if video_import_path.is_dir():
         video_list = utils.find_videos(
             [video_import_path], skip_subfolders=skip_subfolders
@@ -46,12 +47,11 @@ def sample_video(
     video_import_path: Path,
     import_path: Path,
     # None if called from the sample_video command
-    geotag_source: T.Optional[GeotagSource] = None,
     skip_subfolders=False,
     video_sample_distance=constants.VIDEO_SAMPLE_DISTANCE,
     video_sample_interval=constants.VIDEO_SAMPLE_INTERVAL,
     video_duration_ratio=constants.VIDEO_DURATION_RATIO,
-    video_start_time: T.Optional[str] = None,
+    video_start_time: str | None = None,
     skip_sample_errors: bool = False,
     rerun: bool = False,
 ) -> None:
@@ -62,7 +62,7 @@ def sample_video(
             f"Expect either non-negative video_sample_distance or positive video_sample_interval but got {video_sample_distance} and {video_sample_interval} respectively"
         )
 
-    video_start_time_dt: T.Optional[datetime.datetime] = None
+    video_start_time_dt: datetime.datetime | None = None
     if video_start_time is not None:
         try:
             video_start_time_dt = types.map_capture_time_to_datetime(video_start_time)
@@ -86,16 +86,6 @@ def sample_video(
             elif sample_dir.is_file():
                 os.remove(sample_dir)
 
-    if geotag_source is None:
-        geotag_source = "exif"
-
-    # If it is not exif, then we use the legacy interval-based sample and geotag them in "process" for backward compatibility
-    if geotag_source not in ["exif"]:
-        if 0 <= video_sample_distance:
-            raise exceptions.MapillaryBadParameterError(
-                f'Geotagging from "{geotag_source}" works with the legacy interval-based sampling only. To switch back, rerun the command with "--video_sample_distance -1 --video_sample_interval 2"'
-            )
-
     for video_path in video_list:
         # need to resolve video_path because video_dir might be absolute
         sample_dir = Path(import_path).joinpath(
@@ -189,7 +179,7 @@ def _sample_single_video_by_interval(
     sample_dir: Path,
     sample_interval: float,
     duration_ratio: float,
-    start_time: T.Optional[datetime.datetime] = None,
+    start_time: datetime.datetime | None = None,
 ) -> None:
     ffmpeg = ffmpeglib.FFMPEG(constants.FFMPEG_PATH, constants.FFPROBE_PATH)
 
@@ -229,7 +219,7 @@ def _sample_video_stream_by_distance(
     points: T.Sequence[geo.Point],
     video_track_parser: mp4_sample_parser.TrackBoxParser,
     sample_distance: float,
-) -> T.Dict[int, T.Tuple[mp4_sample_parser.Sample, geo.Point]]:
+) -> dict[int, tuple[mp4_sample_parser.Sample, geo.Point]]:
     """
     Locate video frames along the track (points), then resample them by the minimal sample_distance, and return the sparse frames.
     """
@@ -285,7 +275,7 @@ def _sample_single_video_by_distance(
     video_path: Path,
     sample_dir: Path,
     sample_distance: float,
-    start_time: T.Optional[datetime.datetime] = None,
+    start_time: datetime.datetime | None = None,
 ) -> None:
     ffmpeg = ffmpeglib.FFMPEG(constants.FFMPEG_PATH, constants.FFPROBE_PATH)
 
@@ -299,9 +289,12 @@ def _sample_single_video_by_distance(
             )
 
     LOG.info("Extracting video metdata")
-    video_metadata = geotag_videos_from_video.GeotagVideosFromVideo.geotag_video(
-        video_path
+
+    video_metadatas = geotag_videos_from_video.GeotagVideosFromVideo().to_description(
+        [video_path]
     )
+    assert len(video_metadatas) == 1, "expect 1 video metadata"
+    video_metadata = video_metadatas[0]
     if isinstance(video_metadata, types.ErrorMetadata):
         LOG.warning(str(video_metadata.error))
         return

mapillary_tools/telemetry.py

@@ -1,5 +1,6 @@
+from __future__ import annotations
+
 import dataclasses
-import typing as T
 from enum import Enum, unique
 
 from .geo import Point
@@ -12,16 +13,8 @@ class GPSFix(Enum):
     FIX_3D = 3
 
 
-@dataclasses.dataclass
-class GPSPoint(Point):
-    epoch_time: T.Optional[float]
-    fix: T.Optional[GPSFix]
-    precision: T.Optional[float]
-    ground_speed: T.Optional[float]
-
-
 @dataclasses.dataclass(order=True)
-class TelemetryMeasurement:
+class TimestampedMeasurement:
     """Base class for all telemetry measurements.
 
     All telemetry measurements must have a timestamp in seconds.
@@ -32,8 +25,28 @@ class TelemetryMeasurement:
     time: float
 
 
+@dataclasses.dataclass
+class GPSPoint(TimestampedMeasurement, Point):
+    epoch_time: float | None
+    fix: GPSFix | None
+    precision: float | None
+    ground_speed: float | None
+
+
+@dataclasses.dataclass
+class CAMMGPSPoint(TimestampedMeasurement, Point):
+    time_gps_epoch: float
+    gps_fix_type: int
+    horizontal_accuracy: float
+    vertical_accuracy: float
+    velocity_east: float
+    velocity_north: float
+    velocity_up: float
+    speed_accuracy: float
+
+
 @dataclasses.dataclass(order=True)
-class GyroscopeData(TelemetryMeasurement):
+class GyroscopeData(TimestampedMeasurement):
     """Gyroscope signal in radians/seconds around XYZ axes of the camera."""
 
     x: float
@@ -42,7 +55,7 @@ class GyroscopeData(TelemetryMeasurement):
 
 
 @dataclasses.dataclass(order=True)
-class AccelerationData(TelemetryMeasurement):
+class AccelerationData(TimestampedMeasurement):
     """Accelerometer reading in meters/second^2 along XYZ axes of the camera."""
 
     x: float
@@ -51,7 +64,7 @@ class AccelerationData(TelemetryMeasurement):
 
 
 @dataclasses.dataclass(order=True)
-class MagnetometerData(TelemetryMeasurement):
+class MagnetometerData(TimestampedMeasurement):
     """Ambient magnetic field."""
 
     x: float

mapillary_tools/types.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import datetime
 import enum
@@ -31,35 +33,40 @@ _ANGLE_PRECISION = 3
 
 
 class FileType(enum.Enum):
+    IMAGE = "image"
+    ZIP = "zip"
+    # VIDEO is a superset of all NATIVE_VIDEO_FILETYPES below.
+    # It also contains the videos that external geotag source (e.g. exiftool) supports
+    VIDEO = "video"
     BLACKVUE = "blackvue"
     CAMM = "camm"
     GOPRO = "gopro"
-    IMAGE = "image"
-    VIDEO = "video"
-    ZIP = "zip"
+
+
+NATIVE_VIDEO_FILETYPES = {
+    FileType.BLACKVUE,
+    FileType.CAMM,
+    FileType.GOPRO,
+}
 
 
 @dataclasses.dataclass
 class ImageMetadata(geo.Point):
     filename: Path
-    # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
-    # filetype: is always FileType.IMAGE
-    width: T.Optional[int] = None
-    height: T.Optional[int] = None
-    MAPSequenceUUID: T.Optional[str] = None
-    MAPDeviceMake: T.Optional[str] = None
-    MAPDeviceModel: T.Optional[str] = None
-    MAPGPSAccuracyMeters: T.Optional[float] = None
-    MAPCameraUUID: T.Optional[str] = None
-    MAPOrientation: T.Optional[int] = None
-    # deprecated since v0.10.0; keep here for compatibility
-    MAPMetaTags: T.Optional[T.Dict] = None
-    # deprecated since v0.10.0; keep here for compatibility
-    MAPFilename: T.Optional[str] = None
-    filesize: T.Optional[int] = None
-
-    def update_md5sum(self, image_data: T.Optional[T.BinaryIO] = None) -> None:
+    md5sum: str | None = None
+    width: int | None = None
+    height: int | None = None
+    MAPSequenceUUID: str | None = None
+    MAPDeviceMake: str | None = None
+    MAPDeviceModel: str | None = None
+    MAPGPSAccuracyMeters: float | None = None
+    MAPCameraUUID: str | None = None
+    MAPOrientation: int | None = None
+    MAPMetaTags: dict | None = None
+    MAPFilename: str | None = None
+    filesize: int | None = None
+
+    def update_md5sum(self, image_data: T.BinaryIO | None = None) -> None:
         if self.md5sum is None:
             if image_data is None:
                 with self.filename.open("rb") as fp:
@@ -77,13 +84,12 @@ class ImageMetadata(geo.Point):
 @dataclasses.dataclass
 class VideoMetadata:
     filename: Path
-    # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
     filetype: FileType
     points: T.Sequence[geo.Point]
-    make: T.Optional[str] = None
-    model: T.Optional[str] = None
-    filesize: T.Optional[int] = None
+    md5sum: str | None = None
+    make: str | None = None
+    model: str | None = None
+    filesize: int | None = None
 
     def update_md5sum(self) -> None:
         if self.md5sum is None:
@@ -94,7 +100,7 @@ class VideoMetadata:
 @dataclasses.dataclass
 class ErrorMetadata:
     filename: Path
-    filetype: T.Optional[FileType]
+    filetype: FileType
     error: Exception
 
 
@@ -104,8 +110,37 @@ Metadata = T.Union[ImageMetadata, VideoMetadata]
 MetadataOrError = T.Union[Metadata, ErrorMetadata]
 
 
+# Assume {GOPRO, VIDEO} are the NATIVE_VIDEO_FILETYPES:
+# a             | b               = result
+# {CAMM}        | {GOPRO}         = {}
+# {CAMM}        | {GOPRO, VIDEO}  = {CAMM}
+# {GOPRO}       | {GOPRO, VIDEO}  = {GOPRO}
+# {GOPRO}       | {VIDEO}         = {GOPRO}
+# {CAMM, GOPRO} | {VIDEO}         = {CAMM, GOPRO}
+# {VIDEO}       | {VIDEO}         = {CAMM, GOPRO, VIDEO}
+def combine_filetype_filters(
+    a: set[FileType] | None, b: set[FileType] | None
+) -> set[FileType] | None:
+    if a is None:
+        return b
+
+    if b is None:
+        return a
+
+    # VIDEO is a superset of NATIVE_VIDEO_FILETYPES,
+    # so we add NATIVE_VIDEO_FILETYPES to each set for intersection later
+
+    if FileType.VIDEO in a:
+        a = a | NATIVE_VIDEO_FILETYPES
+
+    if FileType.VIDEO in b:
+        b = b | NATIVE_VIDEO_FILETYPES
+
+    return a.intersection(b)
+
+
 class UserItem(TypedDict, total=False):
-    MAPOrganizationKey: T.Union[int, str]
+    MAPOrganizationKey: int | str
     # Not in use. Keep here for back-compatibility
     MAPSettingsUsername: str
     MAPSettingsUserKey: str
@@ -144,23 +179,22 @@ class ImageDescription(_SequenceOnly, _Image, MetaProperties, total=True):
     # filename is required
     filename: str
     # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
+    md5sum: str | None
     filetype: Literal["image"]
-    filesize: T.Optional[int]
+    filesize: int | None
 
 
 class _VideoDescriptionRequired(TypedDict, total=True):
     filename: str
-    # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
+    md5sum: str | None
     filetype: str
-    MAPGPSTrack: T.List[T.Sequence[T.Union[float, int, None]]]
+    MAPGPSTrack: list[T.Sequence[float | int | None]]
 
 
 class VideoDescription(_VideoDescriptionRequired, total=False):
     MAPDeviceMake: str
     MAPDeviceModel: str
-    filesize: T.Optional[int]
+    filesize: int | None
 
 
 class _ErrorDescription(TypedDict, total=False):
@@ -168,7 +202,7 @@ class _ErrorDescription(TypedDict, total=False):
     type: str
     message: str
     # vars is optional
-    vars: T.Dict
+    vars: dict
 
 
 class _ImageDescriptionErrorRequired(TypedDict, total=True):
@@ -180,8 +214,26 @@ class ImageDescriptionError(_ImageDescriptionErrorRequired, total=False):
     filetype: str
 
 
+M = T.TypeVar("M")
+
+
+def separate_errors(
+    metadatas: T.Iterable[M | ErrorMetadata],
+) -> tuple[list[M], list[ErrorMetadata]]:
+    good: list[M] = []
+    bad: list[ErrorMetadata] = []
+
+    for metadata in metadatas:
+        if isinstance(metadata, ErrorMetadata):
+            bad.append(metadata)
+        else:
+            good.append(metadata)
+
+    return good, bad
+
+
 def _describe_error_desc(
-    exc: Exception, filename: Path, filetype: T.Optional[FileType]
+    exc: Exception, filename: Path, filetype: FileType | None
 ) -> ImageDescriptionError:
     err: _ErrorDescription = {
         "type": exc.__class__.__name__,
@@ -210,7 +262,7 @@ def _describe_error_desc(
 
 
 def describe_error_metadata(
-    exc: Exception, filename: Path, filetype: T.Optional[FileType]
+    exc: Exception, filename: Path, filetype: FileType
 ) -> ErrorMetadata:
     return ErrorMetadata(filename=filename, filetype=filetype, error=exc)
 
@@ -278,7 +330,6 @@ ImageDescriptionEXIFSchema = {
         "MAPDeviceModel": {"type": "string"},
         "MAPGPSAccuracyMeters": {"type": "number"},
         "MAPCameraUUID": {"type": "string"},
-        # deprecated since v0.10.0; keep here for compatibility
         "MAPFilename": {
             "type": "string",
             "description": "The base filename of the image",
@@ -341,7 +392,7 @@ VideoDescriptionSchema = {
 }
 
 
-def merge_schema(*schemas: T.Dict) -> T.Dict:
+def merge_schema(*schemas: dict) -> dict:
     for s in schemas:
         assert s.get("type") == "object", "must be all object schemas"
     properties = {}
@@ -436,11 +487,11 @@ def validate_image_desc(desc: T.Any) -> None:
         jsonschema.validate(instance=desc, schema=ImageDescriptionFileSchema)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
-        raise exceptions.MapillaryMetadataValidationError(ex.message)
+        raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
     try:
         map_capture_time_to_datetime(desc["MAPCaptureTime"])
     except ValueError as ex:
-        raise exceptions.MapillaryMetadataValidationError(str(ex))
+        raise exceptions.MapillaryMetadataValidationError(str(ex)) from ex
 
 
 def validate_video_desc(desc: T.Any) -> None:
@@ -448,12 +499,12 @@ def validate_video_desc(desc: T.Any) -> None:
         jsonschema.validate(instance=desc, schema=VideoDescriptionFileSchema)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
-        raise exceptions.MapillaryMetadataValidationError(ex.message)
+        raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
 
 
-def datetime_to_map_capture_time(time: T.Union[datetime.datetime, int, float]) -> str:
+def datetime_to_map_capture_time(time: datetime.datetime | int | float) -> str:
     if isinstance(time, (float, int)):
-        dt = datetime.datetime.utcfromtimestamp(time)
+        dt = datetime.datetime.fromtimestamp(time, datetime.timezone.utc)
         # otherwise it will be assumed to be in local time
         dt = dt.replace(tzinfo=datetime.timezone.utc)
     else:
@@ -552,7 +603,7 @@ def from_desc(desc):
 
 
 def _from_image_desc(desc) -> ImageMetadata:
-    kwargs: T.Dict = {}
+    kwargs: dict = {}
     for k, v in desc.items():
         if k not in [
             "filename",
@@ -582,7 +633,7 @@ def _from_image_desc(desc) -> ImageMetadata:
     )
 
 
-def _encode_point(p: geo.Point) -> T.Sequence[T.Union[float, int, None]]:
+def _encode_point(p: geo.Point) -> T.Sequence[float | int | None]:
     entry = [
         int(p.time * 1000),
         round(p.lon, _COORDINATES_PRECISION),
@@ -644,15 +695,16 @@ def validate_and_fail_metadata(metadata: MetadataOrError) -> MetadataOrError:
     if isinstance(metadata, ErrorMetadata):
         return metadata
 
-    filetype: T.Optional[FileType] = None
+    if isinstance(metadata, ImageMetadata):
+        filetype = FileType.IMAGE
+        validate = validate_image_desc
+    else:
+        assert isinstance(metadata, VideoMetadata)
+        filetype = metadata.filetype
+        validate = validate_video_desc
+
     try:
-        if isinstance(metadata, ImageMetadata):
-            filetype = FileType.IMAGE
-            validate_image_desc(as_desc(metadata))
-        else:
-            assert isinstance(metadata, VideoMetadata)
-            filetype = metadata.filetype
-            validate_video_desc(as_desc(metadata))
+        validate(as_desc(metadata))
     except exceptions.MapillaryMetadataValidationError as ex:
         # rethrow because the original error is too verbose
         return describe_error_metadata(
@@ -686,10 +738,10 @@ def desc_file_to_exif(
 
 
 def group_and_sort_images(
-    metadatas: T.Sequence[ImageMetadata],
-) -> T.Dict[str, T.List[ImageMetadata]]:
+    metadatas: T.Iterable[ImageMetadata],
+) -> dict[str, list[ImageMetadata]]:
     # group metadatas by uuid
-    sequences_by_uuid: T.Dict[str, T.List[ImageMetadata]] = {}
+    sequences_by_uuid: dict[str, list[ImageMetadata]] = {}
     missing_sequence_uuid = str(uuid.uuid4())
     for metadata in metadatas:
         if metadata.MAPSequenceUUID is None:
@@ -709,9 +761,10 @@ def group_and_sort_images(
     return sorted_sequences_by_uuid
 
 
-def sequence_md5sum(sequence: T.Iterable[ImageMetadata]) -> str:
+def update_sequence_md5sum(sequence: T.Iterable[ImageMetadata]) -> str:
     md5 = hashlib.md5()
     for metadata in sequence:
+        metadata.update_md5sum()
         assert isinstance(metadata.md5sum, str), "md5sum should be calculated"
         md5.update(metadata.md5sum.encode("utf-8"))
     return md5.hexdigest()