mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0a1__py3-none-any.whl

mapillary_tools/sample_video.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import logging
 import os
@@ -11,7 +13,6 @@ from . import constants, exceptions, ffmpeg as ffmpeglib, geo, types, utils
 from .exif_write import ExifEdit
 from .geotag import geotag_videos_from_video
 from .mp4 import mp4_sample_parser
-from .process_geotag_properties import GeotagSource
 
 LOG = logging.getLogger(__name__)
 
@@ -46,7 +47,6 @@ def sample_video(
     video_import_path: Path,
     import_path: Path,
     # None if called from the sample_video command
-    geotag_source: T.Optional[GeotagSource] = None,
     skip_subfolders=False,
     video_sample_distance=constants.VIDEO_SAMPLE_DISTANCE,
     video_sample_interval=constants.VIDEO_SAMPLE_INTERVAL,
@@ -86,16 +86,6 @@ def sample_video(
             elif sample_dir.is_file():
                 os.remove(sample_dir)
 
-    if geotag_source is None:
-        geotag_source = "exif"
-
-    # If it is not exif, then we use the legacy interval-based sample and geotag them in "process" for backward compatibility
-    if geotag_source not in ["exif"]:
-        if 0 <= video_sample_distance:
-            raise exceptions.MapillaryBadParameterError(
-                f'Geotagging from "{geotag_source}" works with the legacy interval-based sampling only. To switch back, rerun the command with "--video_sample_distance -1 --video_sample_interval 2"'
-            )
-
     for video_path in video_list:
         # need to resolve video_path because video_dir might be absolute
         sample_dir = Path(import_path).joinpath(
@@ -299,9 +289,12 @@ def _sample_single_video_by_distance(
             )
 
     LOG.info("Extracting video metdata")
-    video_metadata = geotag_videos_from_video.GeotagVideosFromVideo.geotag_video(
-        video_path
-    )
+
+    video_metadatas = geotag_videos_from_video.GeotagVideosFromVideo(
+        [video_path]
+    ).to_description()
+    assert len(video_metadatas) == 1, "expect 1 video metadata"
+    video_metadata = video_metadatas[0]
     if isinstance(video_metadata, types.ErrorMetadata):
         LOG.warning(str(video_metadata.error))
         return

mapillary_tools/telemetry.py

@@ -12,16 +12,8 @@ class GPSFix(Enum):
     FIX_3D = 3
 
 
-@dataclasses.dataclass
-class GPSPoint(Point):
-    epoch_time: T.Optional[float]
-    fix: T.Optional[GPSFix]
-    precision: T.Optional[float]
-    ground_speed: T.Optional[float]
-
-
 @dataclasses.dataclass(order=True)
-class TelemetryMeasurement:
+class TimestampedMeasurement:
     """Base class for all telemetry measurements.
 
     All telemetry measurements must have a timestamp in seconds.
@@ -32,8 +24,28 @@ class TelemetryMeasurement:
     time: float
 
 
+@dataclasses.dataclass
+class GPSPoint(TimestampedMeasurement, Point):
+    epoch_time: T.Optional[float]
+    fix: T.Optional[GPSFix]
+    precision: T.Optional[float]
+    ground_speed: T.Optional[float]
+
+
+@dataclasses.dataclass
+class CAMMGPSPoint(TimestampedMeasurement, Point):
+    time_gps_epoch: float
+    gps_fix_type: int
+    horizontal_accuracy: float
+    vertical_accuracy: float
+    velocity_east: float
+    velocity_north: float
+    velocity_up: float
+    speed_accuracy: float
+
+
 @dataclasses.dataclass(order=True)
-class GyroscopeData(TelemetryMeasurement):
+class GyroscopeData(TimestampedMeasurement):
     """Gyroscope signal in radians/seconds around XYZ axes of the camera."""
 
     x: float
@@ -42,7 +54,7 @@ class GyroscopeData(TelemetryMeasurement):
 
 
 @dataclasses.dataclass(order=True)
-class AccelerationData(TelemetryMeasurement):
+class AccelerationData(TimestampedMeasurement):
     """Accelerometer reading in meters/second^2 along XYZ axes of the camera."""
 
     x: float
@@ -51,7 +63,7 @@ class AccelerationData(TelemetryMeasurement):
 
 
 @dataclasses.dataclass(order=True)
-class MagnetometerData(TelemetryMeasurement):
+class MagnetometerData(TimestampedMeasurement):
     """Ambient magnetic field."""
 
     x: float

mapillary_tools/types.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import datetime
 import enum
@@ -31,19 +33,28 @@ _ANGLE_PRECISION = 3
 
 
 class FileType(enum.Enum):
+    IMAGE = "image"
+    ZIP = "zip"
+    # VIDEO is a superset of all NATIVE_VIDEO_FILETYPES below.
+    # It also contains the videos that external geotag source (e.g. exiftool) supports
+    VIDEO = "video"
     BLACKVUE = "blackvue"
     CAMM = "camm"
     GOPRO = "gopro"
-    IMAGE = "image"
-    VIDEO = "video"
-    ZIP = "zip"
+
+
+NATIVE_VIDEO_FILETYPES = {
+    FileType.BLACKVUE,
+    FileType.CAMM,
+    FileType.GOPRO,
+}
 
 
 @dataclasses.dataclass
 class ImageMetadata(geo.Point):
     filename: Path
     # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
+    md5sum: T.Optional[str] = None
     # filetype: is always FileType.IMAGE
     width: T.Optional[int] = None
     height: T.Optional[int] = None
@@ -55,7 +66,6 @@ class ImageMetadata(geo.Point):
     MAPOrientation: T.Optional[int] = None
     # deprecated since v0.10.0; keep here for compatibility
     MAPMetaTags: T.Optional[T.Dict] = None
-    # deprecated since v0.10.0; keep here for compatibility
     MAPFilename: T.Optional[str] = None
     filesize: T.Optional[int] = None
 
@@ -78,9 +88,9 @@ class ImageMetadata(geo.Point):
 class VideoMetadata:
     filename: Path
     # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
     filetype: FileType
     points: T.Sequence[geo.Point]
+    md5sum: T.Optional[str] = None
     make: T.Optional[str] = None
     model: T.Optional[str] = None
     filesize: T.Optional[int] = None
@@ -94,7 +104,7 @@ class VideoMetadata:
 @dataclasses.dataclass
 class ErrorMetadata:
     filename: Path
-    filetype: T.Optional[FileType]
+    filetype: FileType
     error: Exception
 
 
@@ -104,6 +114,35 @@ Metadata = T.Union[ImageMetadata, VideoMetadata]
 MetadataOrError = T.Union[Metadata, ErrorMetadata]
 
 
+# Assume {GOPRO, VIDEO} are the NATIVE_VIDEO_FILETYPES:
+# a             | b               = result
+# {CAMM}        | {GOPRO}         = {}
+# {CAMM}        | {GOPRO, VIDEO}  = {CAMM}
+# {GOPRO}       | {GOPRO, VIDEO}  = {GOPRO}
+# {GOPRO}       | {VIDEO}         = {GOPRO}
+# {CAMM, GOPRO} | {VIDEO}         = {CAMM, GOPRO}
+# {VIDEO}       | {VIDEO}         = {CAMM, GOPRO, VIDEO}
+def combine_filetype_filters(
+    a: set[FileType] | None, b: set[FileType] | None
+) -> set[FileType] | None:
+    if a is None:
+        return b
+
+    if b is None:
+        return a
+
+    # VIDEO is a superset of NATIVE_VIDEO_FILETYPES,
+    # so we add NATIVE_VIDEO_FILETYPES to each set for intersection later
+
+    if FileType.VIDEO in a:
+        a = a | NATIVE_VIDEO_FILETYPES
+
+    if FileType.VIDEO in b:
+        b = b | NATIVE_VIDEO_FILETYPES
+
+    return a.intersection(b)
+
+
 class UserItem(TypedDict, total=False):
     MAPOrganizationKey: T.Union[int, str]
     # Not in use. Keep here for back-compatibility
@@ -180,6 +219,24 @@ class ImageDescriptionError(_ImageDescriptionErrorRequired, total=False):
     filetype: str
 
 
+M = T.TypeVar("M")
+
+
+def separate_errors(
+    metadatas: T.Iterable[M | ErrorMetadata],
+) -> tuple[list[M], list[ErrorMetadata]]:
+    good: list[M] = []
+    bad: list[ErrorMetadata] = []
+
+    for metadata in metadatas:
+        if isinstance(metadata, ErrorMetadata):
+            bad.append(metadata)
+        else:
+            good.append(metadata)
+
+    return good, bad
+
+
 def _describe_error_desc(
     exc: Exception, filename: Path, filetype: T.Optional[FileType]
 ) -> ImageDescriptionError:
@@ -210,7 +267,7 @@ def _describe_error_desc(
 
 
 def describe_error_metadata(
-    exc: Exception, filename: Path, filetype: T.Optional[FileType]
+    exc: Exception, filename: Path, filetype: FileType
 ) -> ErrorMetadata:
     return ErrorMetadata(filename=filename, filetype=filetype, error=exc)
 
@@ -278,7 +335,6 @@ ImageDescriptionEXIFSchema = {
         "MAPDeviceModel": {"type": "string"},
         "MAPGPSAccuracyMeters": {"type": "number"},
         "MAPCameraUUID": {"type": "string"},
-        # deprecated since v0.10.0; keep here for compatibility
         "MAPFilename": {
             "type": "string",
             "description": "The base filename of the image",
@@ -436,11 +492,11 @@ def validate_image_desc(desc: T.Any) -> None:
         jsonschema.validate(instance=desc, schema=ImageDescriptionFileSchema)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
-        raise exceptions.MapillaryMetadataValidationError(ex.message)
+        raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
     try:
         map_capture_time_to_datetime(desc["MAPCaptureTime"])
     except ValueError as ex:
-        raise exceptions.MapillaryMetadataValidationError(str(ex))
+        raise exceptions.MapillaryMetadataValidationError(str(ex)) from ex
 
 
 def validate_video_desc(desc: T.Any) -> None:
@@ -448,12 +504,12 @@ def validate_video_desc(desc: T.Any) -> None:
         jsonschema.validate(instance=desc, schema=VideoDescriptionFileSchema)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
-        raise exceptions.MapillaryMetadataValidationError(ex.message)
+        raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
 
 
 def datetime_to_map_capture_time(time: T.Union[datetime.datetime, int, float]) -> str:
     if isinstance(time, (float, int)):
-        dt = datetime.datetime.utcfromtimestamp(time)
+        dt = datetime.datetime.fromtimestamp(time, datetime.timezone.utc)
         # otherwise it will be assumed to be in local time
         dt = dt.replace(tzinfo=datetime.timezone.utc)
     else:
@@ -644,15 +700,16 @@ def validate_and_fail_metadata(metadata: MetadataOrError) -> MetadataOrError:
     if isinstance(metadata, ErrorMetadata):
         return metadata
 
-    filetype: T.Optional[FileType] = None
+    if isinstance(metadata, ImageMetadata):
+        filetype = FileType.IMAGE
+        validate = validate_image_desc
+    else:
+        assert isinstance(metadata, VideoMetadata)
+        filetype = metadata.filetype
+        validate = validate_video_desc
+
     try:
-        if isinstance(metadata, ImageMetadata):
-            filetype = FileType.IMAGE
-            validate_image_desc(as_desc(metadata))
-        else:
-            assert isinstance(metadata, VideoMetadata)
-            filetype = metadata.filetype
-            validate_video_desc(as_desc(metadata))
+        validate(as_desc(metadata))
     except exceptions.MapillaryMetadataValidationError as ex:
         # rethrow because the original error is too verbose
         return describe_error_metadata(
@@ -709,9 +766,10 @@ def group_and_sort_images(
     return sorted_sequences_by_uuid
 
 
-def sequence_md5sum(sequence: T.Iterable[ImageMetadata]) -> str:
+def update_sequence_md5sum(sequence: T.Iterable[ImageMetadata]) -> str:
     md5 = hashlib.md5()
     for metadata in sequence:
+        metadata.update_md5sum()
         assert isinstance(metadata.md5sum, str), "md5sum should be calculated"
         md5.update(metadata.md5sum.encode("utf-8"))
     return md5.hexdigest()