mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

mapillary_tools/geotag/geotag_images_from_gpx.py

@@ -2,11 +2,17 @@ from __future__ import annotations
 
 import dataclasses
 import logging
+import sys
 import typing as T
 from pathlib import Path
 
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
+
 from .. import exceptions, geo, types
-from .geotag_from_generic import GeotagImagesFromGeneric
+from .base import GeotagImagesFromGeneric
 from .geotag_images_from_exif import ImageEXIFExtractor
 
 
@@ -16,14 +22,13 @@ LOG = logging.getLogger(__name__)
 class GeotagImagesFromGPX(GeotagImagesFromGeneric):
     def __init__(
         self,
-        image_paths: T.Sequence[Path],
         points: T.Sequence[geo.Point],
         use_gpx_start_time: bool = False,
         use_image_start_time: bool = False,
         offset_time: float = 0.0,
         num_processes: int | None = None,
     ):
-        super().__init__(image_paths, num_processes=num_processes)
+        super().__init__(num_processes=num_processes)
         self.points = points
         self.use_gpx_start_time = use_gpx_start_time
         self.use_image_start_time = use_image_start_time
@@ -73,16 +78,21 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
             time=interpolated.time,
         )
 
-    def _generate_image_extractors(self) -> T.Sequence[ImageEXIFExtractor]:
+    @override
+    def _generate_image_extractors(
+        self, image_paths: T.Sequence[Path]
+    ) -> T.Sequence[ImageEXIFExtractor]:
         return [
-            ImageEXIFExtractor(path, skip_lonlat_error=True)
-            for path in self.image_paths
+            ImageEXIFExtractor(path, skip_lonlat_error=True) for path in image_paths
         ]
 
-    def to_description(self) -> list[types.ImageMetadataOrError]:
+    @override
+    def to_description(
+        self, image_paths: T.Sequence[Path]
+    ) -> list[types.ImageMetadataOrError]:
         final_metadatas: list[types.ImageMetadataOrError] = []
 
-        image_metadata_or_errors = super().to_description()
+        image_metadata_or_errors = super().to_description(image_paths)
 
         image_metadatas, error_metadatas = types.separate_errors(
             image_metadata_or_errors
@@ -90,7 +100,7 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
         final_metadatas.extend(error_metadatas)
 
         if not image_metadatas:
-            assert len(self.image_paths) == len(final_metadatas)
+            assert len(image_paths) == len(final_metadatas)
             return final_metadatas
 
         # Do not use point itself for comparison because point.angle or point.alt could be None
@@ -145,6 +155,6 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
                 )
                 final_metadatas.append(error_metadata)
 
-        assert len(self.image_paths) == len(final_metadatas)
+        assert len(image_paths) == len(final_metadatas)
 
         return final_metadatas

mapillary_tools/geotag/geotag_images_from_gpx_file.py

@@ -1,12 +1,9 @@
 from __future__ import annotations
 
 import logging
-import typing as T
 from pathlib import Path
 
-import gpxpy
-
-from .. import geo
+from . import utils
 from .geotag_images_from_gpx import GeotagImagesFromGPX
 
 
@@ -16,14 +13,13 @@ LOG = logging.getLogger(__name__)
 class GeotagImagesFromGPXFile(GeotagImagesFromGPX):
     def __init__(
         self,
-        image_paths: T.Sequence[Path],
         source_path: Path,
         use_gpx_start_time: bool = False,
         offset_time: float = 0.0,
         num_processes: int | None = None,
     ):
         try:
-            tracks = parse_gpx(source_path)
+            tracks = utils.parse_gpx(source_path)
         except Exception as ex:
             raise RuntimeError(
                 f"Error parsing GPX {source_path}: {ex.__class__.__name__}: {ex}"
@@ -37,36 +33,8 @@ class GeotagImagesFromGPXFile(GeotagImagesFromGPX):
             )
         points = sum(tracks, [])
         super().__init__(
-            image_paths,
             points,
             use_gpx_start_time=use_gpx_start_time,
             offset_time=offset_time,
             num_processes=num_processes,
         )
-
-
-Track = T.List[geo.Point]
-
-
-def parse_gpx(gpx_file: Path) -> list[Track]:
-    with gpx_file.open("r") as f:
-        gpx = gpxpy.parse(f)
-
-    tracks: list[Track] = []
-
-    for track in gpx.tracks:
-        for segment in track.segments:
-            tracks.append([])
-            for point in segment.points:
-                if point.time is not None:
-                    tracks[-1].append(
-                        geo.Point(
-                            time=geo.as_unix_time(point.time),
-                            lat=point.latitude,
-                            lon=point.longitude,
-                            alt=point.elevation,
-                            angle=None,
-                        )
-                    )
-
-    return tracks

mapillary_tools/geotag/geotag_images_from_nmea_file.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import datetime
-import typing as T
 from pathlib import Path
 
 import pynmea2
@@ -13,7 +12,6 @@ from .geotag_images_from_gpx import GeotagImagesFromGPX
 class GeotagImagesFromNMEAFile(GeotagImagesFromGPX):
     def __init__(
         self,
-        image_paths: T.Sequence[Path],
         source_path: Path,
         use_gpx_start_time: bool = False,
         offset_time: float = 0.0,
@@ -21,7 +19,6 @@ class GeotagImagesFromNMEAFile(GeotagImagesFromGPX):
     ):
         points = get_lat_lon_time_from_nmea(source_path)
         super().__init__(
-            image_paths,
             points,
             use_gpx_start_time=use_gpx_start_time,
             offset_time=offset_time,

mapillary_tools/geotag/geotag_images_from_video.py

@@ -1,12 +1,17 @@
 from __future__ import annotations
 
 import logging
+import sys
 import typing as T
 from pathlib import Path
 
-from .. import types, utils
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
 
-from .geotag_from_generic import GeotagImagesFromGeneric
+from .. import types, utils
+from .base import GeotagImagesFromGeneric
 from .geotag_images_from_gpx import GeotagImagesFromGPX
 
 
@@ -16,16 +21,18 @@ LOG = logging.getLogger(__name__)
 class GeotagImagesFromVideo(GeotagImagesFromGeneric):
     def __init__(
         self,
-        image_paths: T.Sequence[Path],
         video_metadatas: T.Sequence[types.VideoMetadataOrError],
         offset_time: float = 0.0,
         num_processes: int | None = None,
     ):
-        super().__init__(image_paths, num_processes=num_processes)
+        super().__init__(num_processes=num_processes)
         self.video_metadatas = video_metadatas
         self.offset_time = offset_time
 
-    def to_description(self) -> list[types.ImageMetadataOrError]:
+    @override
+    def to_description(
+        self, image_paths: T.Sequence[Path]
+    ) -> list[types.ImageMetadataOrError]:
         # Will return this list
         final_image_metadatas: list[types.ImageMetadataOrError] = []
 
@@ -35,9 +42,7 @@ class GeotagImagesFromVideo(GeotagImagesFromGeneric):
 
         for video_error_metadata in video_error_metadatas:
             video_path = video_error_metadata.filename
-            sample_paths = list(
-                utils.filter_video_samples(self.image_paths, video_path)
-            )
+            sample_paths = list(utils.filter_video_samples(image_paths, video_path))
             LOG.debug(
                 "Found %d sample images from video %s with error: %s",
                 len(sample_paths),
@@ -55,9 +60,7 @@ class GeotagImagesFromVideo(GeotagImagesFromGeneric):
         for video_metadata in video_metadatas:
             video_path = video_metadata.filename
 
-            sample_paths = list(
-                utils.filter_video_samples(self.image_paths, video_path)
-            )
+            sample_paths = list(utils.filter_video_samples(image_paths, video_path))
             LOG.debug(
                 "Found %d sample images from video %s",
                 len(sample_paths),
@@ -65,7 +68,6 @@ class GeotagImagesFromVideo(GeotagImagesFromGeneric):
             )
 
             geotag = GeotagImagesFromGPX(
-                sample_paths,
                 video_metadata.points,
                 use_gpx_start_time=False,
                 use_image_start_time=True,
@@ -73,7 +75,7 @@ class GeotagImagesFromVideo(GeotagImagesFromGeneric):
                 num_processes=self.num_processes,
             )
 
-            image_metadatas = geotag.to_description()
+            image_metadatas = geotag.to_description(image_paths)
 
             for metadata in image_metadatas:
                 if isinstance(metadata, types.ImageMetadata):
@@ -85,6 +87,6 @@ class GeotagImagesFromVideo(GeotagImagesFromGeneric):
         # NOTE: this method only geotags images that have a corresponding video,
         # so the number of image metadata objects returned might be less than
         # the number of the input image_paths
-        assert len(final_image_metadatas) <= len(self.image_paths)
+        assert len(final_image_metadatas) <= len(image_paths)
 
         return final_image_metadatas

mapillary_tools/geotag/geotag_videos_from_exiftool.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import logging
+import sys
+import typing as T
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
+
+from .. import constants, exceptions, exiftool_read, types
+from ..exiftool_runner import ExiftoolRunner
+from .base import GeotagVideosFromGeneric
+from .utils import index_rdf_description_by_path
+from .video_extractors.exiftool import VideoExifToolExtractor
+
+LOG = logging.getLogger(__name__)
+
+
+class GeotagVideosFromExifToolXML(GeotagVideosFromGeneric):
+    def __init__(
+        self,
+        xml_path: Path,
+        num_processes: int | None = None,
+    ):
+        super().__init__(num_processes=num_processes)
+        self.xml_path = xml_path
+
+    @classmethod
+    def build_image_extractors(
+        cls,
+        rdf_by_path: dict[str, ET.Element],
+        video_paths: T.Iterable[Path],
+    ) -> list[VideoExifToolExtractor | types.ErrorMetadata]:
+        results: list[VideoExifToolExtractor | types.ErrorMetadata] = []
+
+        for path in video_paths:
+            rdf = rdf_by_path.get(exiftool_read.canonical_path(path))
+            if rdf is None:
+                ex = exceptions.MapillaryExifToolXMLNotFoundError(
+                    "Cannot find the video in the ExifTool XML"
+                )
+                results.append(
+                    types.describe_error_metadata(
+                        ex, path, filetype=types.FileType.VIDEO
+                    )
+                )
+            else:
+                results.append(VideoExifToolExtractor(path, rdf))
+
+        return results
+
+    @override
+    def _generate_video_extractors(
+        self, video_paths: T.Sequence[Path]
+    ) -> T.Sequence[VideoExifToolExtractor | types.ErrorMetadata]:
+        rdf_by_path = index_rdf_description_by_path([self.xml_path])
+        return self.build_image_extractors(rdf_by_path, video_paths)
+
+
+class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
+    @override
+    def _generate_video_extractors(
+        self, video_paths: T.Sequence[Path]
+    ) -> T.Sequence[VideoExifToolExtractor | types.ErrorMetadata]:
+        runner = ExiftoolRunner(constants.EXIFTOOL_PATH)
+
+        LOG.debug(
+            "Extracting XML from %d videos with ExifTool command: %s",
+            len(video_paths),
+            " ".join(runner._build_args_read_stdin()),
+        )
+        try:
+            xml = runner.extract_xml(video_paths)
+        except FileNotFoundError as ex:
+            raise exceptions.MapillaryExiftoolNotFoundError(ex) from ex
+
+        try:
+            xml_element = ET.fromstring(xml)
+        except ET.ParseError as ex:
+            LOG.warning(
+                "Failed to parse ExifTool XML: %s",
+                str(ex),
+                exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
+            )
+            rdf_by_path = {}
+        else:
+            rdf_by_path = exiftool_read.index_rdf_description_by_path_from_xml_element(
+                xml_element
+            )
+
+        return GeotagVideosFromExifToolXML.build_image_extractors(
+            rdf_by_path, video_paths
+        )

mapillary_tools/geotag/geotag_videos_from_gpx.py

@@ -1,140 +1,39 @@
 from __future__ import annotations
 
-import dataclasses
-import datetime
 import logging
-
+import sys
 import typing as T
 from pathlib import Path
 
-from .. import geo, telemetry, types
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
+
 from . import options
-from .geotag_from_generic import GenericVideoExtractor, GeotagVideosFromGeneric
-from .geotag_images_from_gpx_file import parse_gpx
-from .geotag_videos_from_video import NativeVideoExtractor
+from .base import GeotagVideosFromGeneric
+from .video_extractors.gpx import GPXVideoExtractor
 
 
 LOG = logging.getLogger(__name__)
 
 
-class GPXVideoExtractor(GenericVideoExtractor):
-    def __init__(self, video_path: Path, gpx_path: Path):
-        self.video_path = video_path
-        self.gpx_path = gpx_path
-
-    def extract(self) -> types.VideoMetadataOrError:
-        try:
-            gpx_tracks = parse_gpx(self.gpx_path)
-        except Exception as ex:
-            raise RuntimeError(
-                f"Error parsing GPX {self.gpx_path}: {ex.__class__.__name__}: {ex}"
-            )
-
-        if 1 < len(gpx_tracks):
-            LOG.warning(
-                "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",
-                len(gpx_tracks),
-                self.gpx_path,
-            )
-
-        gpx_points: T.Sequence[geo.Point] = sum(gpx_tracks, [])
-
-        native_extractor = NativeVideoExtractor(self.video_path)
-
-        video_metadata_or_error = native_extractor.extract()
-
-        if isinstance(video_metadata_or_error, types.ErrorMetadata):
-            self._rebase_times(gpx_points)
-            return types.VideoMetadata(
-                filename=video_metadata_or_error.filename,
-                filetype=video_metadata_or_error.filetype or types.FileType.VIDEO,
-                points=gpx_points,
-            )
-
-        video_metadata = video_metadata_or_error
-
-        offset = self._synx_gpx_by_first_gps_timestamp(
-            gpx_points, video_metadata.points
-        )
-
-        self._rebase_times(gpx_points, offset=offset)
-
-        return dataclasses.replace(video_metadata_or_error, points=gpx_points)
-
-    @staticmethod
-    def _rebase_times(points: T.Sequence[geo.Point], offset: float = 0.0):
-        """
-        Make point times start from 0
-        """
-        if points:
-            first_timestamp = points[0].time
-            for p in points:
-                p.time = (p.time - first_timestamp) + offset
-        return points
-
-    def _synx_gpx_by_first_gps_timestamp(
-        self, gpx_points: T.Sequence[geo.Point], video_gps_points: T.Sequence[geo.Point]
-    ) -> float:
-        offset: float = 0.0
-
-        if not gpx_points:
-            return offset
-
-        first_gpx_dt = datetime.datetime.fromtimestamp(
-            gpx_points[0].time, tz=datetime.timezone.utc
-        )
-        LOG.info("First GPX timestamp: %s", first_gpx_dt)
-
-        if not video_gps_points:
-            LOG.warning(
-                "Skip GPX synchronization because no GPS found in video %s",
-                self.video_path,
-            )
-            return offset
-
-        first_gps_point = video_gps_points[0]
-        if isinstance(first_gps_point, telemetry.GPSPoint):
-            if first_gps_point.epoch_time is not None:
-                first_gps_dt = datetime.datetime.fromtimestamp(
-                    first_gps_point.epoch_time, tz=datetime.timezone.utc
-                )
-                LOG.info("First GPS timestamp: %s", first_gps_dt)
-                offset = gpx_points[0].time - first_gps_point.epoch_time
-                if offset:
-                    LOG.warning(
-                        "Found offset between GPX %s and video GPS timestamps %s: %s seconds",
-                        first_gpx_dt,
-                        first_gps_dt,
-                        offset,
-                    )
-                else:
-                    LOG.info(
-                        "GPX and GPS are perfectly synchronized (all starts from %s)",
-                        first_gpx_dt,
-                    )
-            else:
-                LOG.warning(
-                    "Skip GPX synchronization because no GPS epoch time found in video %s",
-                    self.video_path,
-                )
-
-        return offset
-
-
 class GeotagVideosFromGPX(GeotagVideosFromGeneric):
     def __init__(
         self,
-        video_paths: T.Sequence[Path],
         option: options.SourcePathOption | None = None,
         num_processes: int | None = None,
     ):
-        super().__init__(video_paths, num_processes=num_processes)
+        super().__init__(num_processes=num_processes)
         if option is None:
             option = options.SourcePathOption(pattern="%f.gpx")
         self.option = option
 
-    def _generate_image_extractors(self) -> T.Sequence[GPXVideoExtractor]:
+    @override
+    def _generate_video_extractors(
+        self, video_paths: T.Sequence[Path]
+    ) -> T.Sequence[GPXVideoExtractor]:
         return [
             GPXVideoExtractor(video_path, self.option.resolve(video_path))
-            for video_path in self.video_paths
+            for video_path in video_paths
         ]

mapillary_tools/geotag/geotag_videos_from_video.py

@@ -1,165 +1,32 @@
 from __future__ import annotations
 
+import sys
 import typing as T
 from pathlib import Path
 
-from .. import blackvue_parser, exceptions, geo, telemetry, types, utils
-from ..camm import camm_parser
-from ..gpmf import gpmf_gps_filter, gpmf_parser
-from ..types import FileType
-from .geotag_from_generic import GenericVideoExtractor, GeotagVideosFromGeneric
-
-
-class GoProVideoExtractor(GenericVideoExtractor):
-    def extract(self) -> types.VideoMetadataOrError:
-        with self.video_path.open("rb") as fp:
-            gopro_info = gpmf_parser.extract_gopro_info(fp)
-
-        if gopro_info is None:
-            raise exceptions.MapillaryVideoGPSNotFoundError(
-                "No GPS data found from the video"
-            )
-
-        gps_points = gopro_info.gps
-        assert gps_points is not None, "must have GPS data extracted"
-        if not gps_points:
-            # Instead of raising an exception, return error metadata to tell the file type
-            ex: exceptions.MapillaryDescriptionError = (
-                exceptions.MapillaryGPXEmptyError("Empty GPS data found")
-            )
-            return types.describe_error_metadata(
-                ex, self.video_path, filetype=FileType.GOPRO
-            )
-
-        gps_points = T.cast(
-            T.List[telemetry.GPSPoint], gpmf_gps_filter.remove_noisy_points(gps_points)
-        )
-        if not gps_points:
-            # Instead of raising an exception, return error metadata to tell the file type
-            ex = exceptions.MapillaryGPSNoiseError("GPS is too noisy")
-            return types.describe_error_metadata(
-                ex, self.video_path, filetype=FileType.GOPRO
-            )
-
-        video_metadata = types.VideoMetadata(
-            filename=self.video_path,
-            filesize=utils.get_file_size(self.video_path),
-            filetype=FileType.GOPRO,
-            points=T.cast(T.List[geo.Point], gps_points),
-            make=gopro_info.make,
-            model=gopro_info.model,
-        )
-
-        return video_metadata
-
-
-class CAMMVideoExtractor(GenericVideoExtractor):
-    def extract(self) -> types.VideoMetadataOrError:
-        with self.video_path.open("rb") as fp:
-            camm_info = camm_parser.extract_camm_info(fp)
-
-        if camm_info is None:
-            raise exceptions.MapillaryVideoGPSNotFoundError(
-                "No GPS data found from the video"
-            )
-
-        if not camm_info.gps and not camm_info.mini_gps:
-            # Instead of raising an exception, return error metadata to tell the file type
-            ex: exceptions.MapillaryDescriptionError = (
-                exceptions.MapillaryGPXEmptyError("Empty GPS data found")
-            )
-            return types.describe_error_metadata(
-                ex, self.video_path, filetype=FileType.CAMM
-            )
-
-        return types.VideoMetadata(
-            filename=self.video_path,
-            filesize=utils.get_file_size(self.video_path),
-            filetype=FileType.CAMM,
-            points=T.cast(T.List[geo.Point], camm_info.gps or camm_info.mini_gps),
-            make=camm_info.make,
-            model=camm_info.model,
-        )
-
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
 
-class BlackVueVideoExtractor(GenericVideoExtractor):
-    def extract(self) -> types.VideoMetadataOrError:
-        with self.video_path.open("rb") as fp:
-            blackvue_info = blackvue_parser.extract_blackvue_info(fp)
-
-        if blackvue_info is None:
-            raise exceptions.MapillaryVideoGPSNotFoundError(
-                "No GPS data found from the video"
-            )
-
-        if not blackvue_info.gps:
-            # Instead of raising an exception, return error metadata to tell the file type
-            ex: exceptions.MapillaryDescriptionError = (
-                exceptions.MapillaryGPXEmptyError("Empty GPS data found")
-            )
-            return types.describe_error_metadata(
-                ex, self.video_path, filetype=FileType.BLACKVUE
-            )
-
-        video_metadata = types.VideoMetadata(
-            filename=self.video_path,
-            filesize=utils.get_file_size(self.video_path),
-            filetype=FileType.BLACKVUE,
-            points=blackvue_info.gps or [],
-            make=blackvue_info.make,
-            model=blackvue_info.model,
-        )
-
-        return video_metadata
-
-
-class NativeVideoExtractor(GenericVideoExtractor):
-    def __init__(self, video_path: Path, filetypes: set[FileType] | None = None):
-        super().__init__(video_path)
-        self.filetypes = filetypes
-
-    def extract(self) -> types.VideoMetadataOrError:
-        ft = self.filetypes
-        extractor: GenericVideoExtractor
-
-        if ft is None or FileType.VIDEO in ft or FileType.GOPRO in ft:
-            extractor = GoProVideoExtractor(self.video_path)
-            try:
-                return extractor.extract()
-            except exceptions.MapillaryVideoGPSNotFoundError:
-                pass
-
-        if ft is None or FileType.VIDEO in ft or FileType.CAMM in ft:
-            extractor = CAMMVideoExtractor(self.video_path)
-            try:
-                return extractor.extract()
-            except exceptions.MapillaryVideoGPSNotFoundError:
-                pass
-
-        if ft is None or FileType.VIDEO in ft or FileType.BLACKVUE in ft:
-            extractor = BlackVueVideoExtractor(self.video_path)
-            try:
-                return extractor.extract()
-            except exceptions.MapillaryVideoGPSNotFoundError:
-                pass
-
-        raise exceptions.MapillaryVideoGPSNotFoundError(
-            "No GPS data found from the video"
-        )
+from ..types import FileType
+from .base import GeotagVideosFromGeneric
+from .video_extractors.native import NativeVideoExtractor
 
 
 class GeotagVideosFromVideo(GeotagVideosFromGeneric):
     def __init__(
         self,
-        video_paths: T.Sequence[Path],
         filetypes: set[FileType] | None = None,
         num_processes: int | None = None,
     ):
-        super().__init__(video_paths, num_processes=num_processes)
+        super().__init__(num_processes=num_processes)
         self.filetypes = filetypes
 
-    def _generate_video_extractors(self) -> T.Sequence[GenericVideoExtractor]:
+    @override
+    def _generate_video_extractors(
+        self, video_paths: T.Sequence[Path]
+    ) -> T.Sequence[NativeVideoExtractor]:
         return [
-            NativeVideoExtractor(path, filetypes=self.filetypes)
-            for path in self.video_paths
+            NativeVideoExtractor(path, filetypes=self.filetypes) for path in video_paths
         ]