mapillary-tools 0.14.0a2__py3-none-any.whl → 0.14.1__py3-none-any.whl

mapillary_tools/geo.py

@@ -51,6 +51,22 @@ def gps_distance(latlon_1: tuple[float, float], latlon_2: tuple[float, float]) -
     return math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2 + (z1 - z2) ** 2)
 
 
+def avg_speed(sequence: T.Sequence[PointLike]) -> float:
+    total_distance = 0.0
+    for cur, nxt in pairwise(sequence):
+        total_distance += gps_distance((cur.lat, cur.lon), (nxt.lat, nxt.lon))
+
+    if sequence:
+        time_diff = sequence[-1].time - sequence[0].time
+    else:
+        time_diff = 0.0
+
+    if time_diff == 0.0:
+        return float("inf")
+
+    return total_distance / time_diff
+
+
 def compute_bearing(
     latlon_1: tuple[float, float],
     latlon_2: tuple[float, float],

mapillary_tools/geotag/base.py

@@ -46,12 +46,12 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
                 map_results,
                 desc="Extracting images",
                 unit="images",
-                disable=LOG.getEffectiveLevel() <= logging.DEBUG,
+                disable=LOG.isEnabledFor(logging.DEBUG),
                 total=len(extractors),
             )
         )
 
-        return results + error_metadatas
+        return T.cast(list[types.ImageMetadataOrError], results + error_metadatas)
 
     # This method is passed to multiprocessing
     # so it has to be classmethod or staticmethod to avoid pickling the instance
@@ -62,6 +62,8 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
         try:
             return extractor.extract()
         except exceptions.MapillaryDescriptionError as ex:
+            if LOG.isEnabledFor(logging.DEBUG):
+                LOG.error(f"{cls.__name__}({image_path.name}): {ex}")
             return types.describe_error_metadata(
                 ex, image_path, filetype=types.FileType.IMAGE
             )
@@ -112,12 +114,12 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
                 map_results,
                 desc="Extracting videos",
                 unit="videos",
-                disable=LOG.getEffectiveLevel() <= logging.DEBUG,
+                disable=LOG.isEnabledFor(logging.DEBUG),
                 total=len(extractors),
             )
         )
 
-        return results + error_metadatas
+        return T.cast(list[types.VideoMetadataOrError], results + error_metadatas)
 
     # This method is passed to multiprocessing
     # so it has to be classmethod or staticmethod to avoid pickling the instance
@@ -128,6 +130,8 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
         try:
             return extractor.extract()
         except exceptions.MapillaryDescriptionError as ex:
+            if LOG.isEnabledFor(logging.DEBUG):
+                LOG.error(f"{cls.__name__}({video_path.name}): {ex}")
             return types.describe_error_metadata(
                 ex, video_path, filetype=types.FileType.VIDEO
             )

mapillary_tools/geotag/factory.py

@@ -6,7 +6,6 @@ import typing as T
 from pathlib import Path
 
 from .. import exceptions, types, utils
-from ..types import FileType
 from . import (
     base,
     geotag_images_from_exif,
@@ -68,10 +67,34 @@ def process(
     reprocessable_paths = set(paths)
 
     for idx, option in enumerate(options):
-        LOG.debug("Processing %d files with %s", len(reprocessable_paths), option)
+        if LOG.isEnabledFor(logging.DEBUG):
+            LOG.info(
+                f"==> Processing {len(reprocessable_paths)} files with source {option}..."
+            )
+        else:
+            LOG.info(
+                f"==> Processing {len(reprocessable_paths)} files with source {option.source.value}..."
+            )
+
+        image_videos, video_paths = _filter_images_and_videos(
+            reprocessable_paths, option.filetypes
+        )
 
-        image_metadata_or_errors = _geotag_images(reprocessable_paths, option)
-        video_metadata_or_errors = _geotag_videos(reprocessable_paths, option)
+        if image_videos:
+            image_geotag = _build_image_geotag(option)
+            image_metadata_or_errors = (
+                image_geotag.to_description(image_videos) if image_geotag else []
+            )
+        else:
+            image_metadata_or_errors = []
+
+        if video_paths:
+            video_geotag = _build_video_geotag(option)
+            video_metadata_or_errors = (
+                video_geotag.to_description(video_paths) if video_geotag else []
+            )
+        else:
+            video_metadata_or_errors = []
 
         more_option = idx < len(options) - 1
 
@@ -97,6 +120,8 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
             (
                 exceptions.MapillaryGeoTaggingError,
                 exceptions.MapillaryVideoGPSNotFoundError,
+                exceptions.MapillaryExiftoolNotFoundError,
+                exceptions.MapillaryExifToolXMLNotFoundError,
             ),
         ):
             return True
@@ -140,152 +165,144 @@ def _ensure_source_path(option: SourceOption) -> Path:
     return option.source_path.source_path
 
 
-def _geotag_images(
-    paths: T.Iterable[Path], option: SourceOption
-) -> list[types.ImageMetadataOrError]:
-    image_paths, _ = _filter_images_and_videos(paths, option.filetypes)
-
-    if not image_paths:
-        return []
-
+def _build_image_geotag(option: SourceOption) -> base.GeotagImagesFromGeneric | None:
+    """
+    Build a GeotagImagesFromGeneric object based on the provided option.
+    """
     if option.interpolation is None:
         interpolation = InterpolationOption()
     else:
         interpolation = option.interpolation
 
-    geotag: base.GeotagImagesFromGeneric
-
-    if option.source is SourceType.NATIVE:
-        geotag = geotag_images_from_exif.GeotagImagesFromEXIF(
+    if option.source in [SourceType.EXIF, SourceType.NATIVE]:
+        return geotag_images_from_exif.GeotagImagesFromEXIF(
             num_processes=option.num_processes
         )
-        return geotag.to_description(image_paths)
 
     if option.source is SourceType.EXIFTOOL_RUNTIME:
-        geotag = geotag_images_from_exiftool.GeotagImagesFromExifToolRunner(
+        return geotag_images_from_exiftool.GeotagImagesFromExifToolRunner(
             num_processes=option.num_processes
         )
-        try:
-            return geotag.to_description(image_paths)
-        except exceptions.MapillaryExiftoolNotFoundError as ex:
-            LOG.warning('Skip "%s" because: %s', option.source.value, ex)
-            return []
 
     elif option.source is SourceType.EXIFTOOL_XML:
         # This is to ensure 'video_process --geotag={"source": "exiftool_xml", "source_path": "/tmp/xml_path"}'
         # to work
-        geotag = geotag_images_from_exiftool.GeotagImagesFromExifToolWithSamples(
-            xml_path=_ensure_source_path(option),
+        if option.source_path is None:
+            raise exceptions.MapillaryBadParameterError(
+                "source_path must be provided for EXIFTOOL_XML source"
+            )
+        return geotag_images_from_exiftool.GeotagImagesFromExifToolWithSamples(
+            source_path=option.source_path,
             num_processes=option.num_processes,
         )
-        return geotag.to_description(image_paths)
 
     elif option.source is SourceType.GPX:
-        geotag = geotag_images_from_gpx_file.GeotagImagesFromGPXFile(
+        return geotag_images_from_gpx_file.GeotagImagesFromGPXFile(
             source_path=_ensure_source_path(option),
             use_gpx_start_time=interpolation.use_gpx_start_time,
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
-        return geotag.to_description(image_paths)
 
     elif option.source is SourceType.NMEA:
-        geotag = geotag_images_from_nmea_file.GeotagImagesFromNMEAFile(
+        return geotag_images_from_nmea_file.GeotagImagesFromNMEAFile(
             source_path=_ensure_source_path(option),
             use_gpx_start_time=interpolation.use_gpx_start_time,
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
 
-        return geotag.to_description(image_paths)
-
-    elif option.source is SourceType.EXIF:
-        geotag = geotag_images_from_exif.GeotagImagesFromEXIF(
-            num_processes=option.num_processes
-        )
-        return geotag.to_description(image_paths)
-
-    elif option.source in [
-        SourceType.GOPRO,
-        SourceType.BLACKVUE,
-        SourceType.CAMM,
-    ]:
-        map_geotag_source_to_filetype: dict[SourceType, FileType] = {
-            SourceType.GOPRO: FileType.GOPRO,
-            SourceType.BLACKVUE: FileType.BLACKVUE,
-            SourceType.CAMM: FileType.CAMM,
-        }
-        video_paths = utils.find_videos([_ensure_source_path(option)])
-        image_samples_by_video_path = utils.find_all_image_samples(
-            image_paths, video_paths
-        )
-        video_paths_with_image_samples = list(image_samples_by_video_path.keys())
-        video_metadatas = geotag_videos_from_video.GeotagVideosFromVideo(
-            filetypes={map_geotag_source_to_filetype[option.source]},
-            num_processes=option.num_processes,
-        ).to_description(video_paths_with_image_samples)
-        geotag = geotag_images_from_video.GeotagImagesFromVideo(
-            video_metadatas,
+    elif option.source in [SourceType.GOPRO, SourceType.BLACKVUE, SourceType.CAMM]:
+        return geotag_images_from_video.GeotagImageSamplesFromVideo(
+            _ensure_source_path(option),
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
-        return geotag.to_description(image_paths)
 
     else:
         raise ValueError(f"Invalid geotag source {option.source}")
 
 
-def _geotag_videos(
-    paths: T.Iterable[Path], option: SourceOption
-) -> list[types.VideoMetadataOrError]:
-    _, video_paths = _filter_images_and_videos(paths, option.filetypes)
-
-    if not video_paths:
-        return []
-
-    geotag: base.GeotagVideosFromGeneric
+def _build_video_geotag(option: SourceOption) -> base.GeotagVideosFromGeneric | None:
+    """
+    Build a GeotagVideosFromGeneric object based on the provided option.
 
+    Examples:
+        >>> from pathlib import Path
+        >>> from mapillary_tools.geotag.options import SourceOption, SourceType
+        >>> opt = SourceOption(SourceType.NATIVE)
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromVideo'
+
+        >>> opt = SourceOption(SourceType.EXIFTOOL_RUNTIME)
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromExifToolRunner'
+
+        >>> opt = SourceOption(SourceType.EXIFTOOL_XML, source_path=Path("/tmp/test.xml"))
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromExifToolXML'
+
+        >>> opt = SourceOption(SourceType.GPX, source_path=Path("/tmp/test.gpx"))
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromGPX'
+
+        >>> opt = SourceOption(SourceType.NMEA, source_path=Path("/tmp/test.nmea"))
+        >>> _build_video_geotag(opt) is None
+        True
+
+        >>> opt = SourceOption(SourceType.EXIF)
+        >>> _build_video_geotag(opt) is None
+        True
+
+        >>> opt = SourceOption(SourceType.GOPRO)
+        >>> _build_video_geotag(opt) is None
+        True
+
+        >>> try:
+        ...     _build_video_geotag(SourceOption("invalid"))
+        ... except ValueError as e:
+        ...     "Invalid geotag source" in str(e)
+        True
+    """
     if option.source is SourceType.NATIVE:
-        geotag = geotag_videos_from_video.GeotagVideosFromVideo(
+        return geotag_videos_from_video.GeotagVideosFromVideo(
             num_processes=option.num_processes, filetypes=option.filetypes
         )
-        return geotag.to_description(video_paths)
 
     if option.source is SourceType.EXIFTOOL_RUNTIME:
-        geotag = geotag_videos_from_exiftool.GeotagVideosFromExifToolRunner(
+        return geotag_videos_from_exiftool.GeotagVideosFromExifToolRunner(
             num_processes=option.num_processes
         )
-        try:
-            return geotag.to_description(video_paths)
-        except exceptions.MapillaryExiftoolNotFoundError as ex:
-            LOG.warning('Skip "%s" because: %s', option.source.value, ex)
-            return []
 
     elif option.source is SourceType.EXIFTOOL_XML:
-        geotag = geotag_videos_from_exiftool.GeotagVideosFromExifToolXML(
-            xml_path=_ensure_source_path(option),
+        if option.source_path is None:
+            raise exceptions.MapillaryBadParameterError(
+                "source_path must be provided for EXIFTOOL_XML source"
+            )
+        return geotag_videos_from_exiftool.GeotagVideosFromExifToolXML(
+            source_path=option.source_path,
         )
-        return geotag.to_description(video_paths)
 
     elif option.source is SourceType.GPX:
-        geotag = geotag_videos_from_gpx.GeotagVideosFromGPX()
-        return geotag.to_description(video_paths)
+        return geotag_videos_from_gpx.GeotagVideosFromGPX(
+            source_path=option.source_path, num_processes=option.num_processes
+        )
 
     elif option.source is SourceType.NMEA:
         # TODO: geotag videos from NMEA
-        return []
+        return None
 
     elif option.source is SourceType.EXIF:
         # Legacy image-specific geotag types
-        return []
+        return None
 
-    elif option.source in [
-        SourceType.GOPRO,
-        SourceType.BLACKVUE,
-        SourceType.CAMM,
-    ]:
+    elif option.source in [SourceType.GOPRO, SourceType.BLACKVUE, SourceType.CAMM]:
         # Legacy image-specific geotag types
-        return []
+        return None
 
     else:
         raise ValueError(f"Invalid geotag source {option.source}")

mapillary_tools/geotag/geotag_images_from_exiftool.py

@@ -13,29 +13,25 @@ else:
 
 from .. import constants, exceptions, exiftool_read, types, utils
 from ..exiftool_runner import ExiftoolRunner
+from . import options
 from .base import GeotagImagesFromGeneric
 from .geotag_images_from_video import GeotagImagesFromVideo
 from .geotag_videos_from_exiftool import GeotagVideosFromExifToolXML
 from .image_extractors.exiftool import ImageExifToolExtractor
-from .utils import index_rdf_description_by_path
 
 LOG = logging.getLogger(__name__)
 
 
 class GeotagImagesFromExifToolXML(GeotagImagesFromGeneric):
     def __init__(
-        self,
-        xml_path: Path,
-        num_processes: int | None = None,
+        self, source_path: options.SourcePathOption, num_processes: int | None = None
     ):
-        self.xml_path = xml_path
+        self.source_path = source_path
         super().__init__(num_processes=num_processes)
 
     @classmethod
     def build_image_extractors(
-        cls,
-        rdf_by_path: dict[str, ET.Element],
-        image_paths: T.Iterable[Path],
+        cls, rdf_by_path: dict[str, ET.Element], image_paths: T.Iterable[Path]
     ) -> list[ImageExifToolExtractor | types.ErrorMetadata]:
         results: list[ImageExifToolExtractor | types.ErrorMetadata] = []
 
@@ -59,7 +55,9 @@ class GeotagImagesFromExifToolXML(GeotagImagesFromGeneric):
     def _generate_image_extractors(
         self, image_paths: T.Sequence[Path]
     ) -> T.Sequence[ImageExifToolExtractor | types.ErrorMetadata]:
-        rdf_by_path = index_rdf_description_by_path([self.xml_path])
+        rdf_by_path = GeotagVideosFromExifToolXML.find_rdf_by_path(
+            self.source_path, image_paths
+        )
         return self.build_image_extractors(rdf_by_path, image_paths)
 
 
@@ -68,7 +66,10 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
     def _generate_image_extractors(
         self, image_paths: T.Sequence[Path]
     ) -> T.Sequence[ImageExifToolExtractor | types.ErrorMetadata]:
-        runner = ExiftoolRunner(constants.EXIFTOOL_PATH)
+        if constants.EXIFTOOL_PATH is None:
+            runner = ExiftoolRunner()
+        else:
+            runner = ExiftoolRunner(constants.EXIFTOOL_PATH)
 
         LOG.debug(
             "Extracting XML from %d images with ExifTool command: %s",
@@ -78,7 +79,13 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
         try:
             xml = runner.extract_xml(image_paths)
         except FileNotFoundError as ex:
-            raise exceptions.MapillaryExiftoolNotFoundError(ex) from ex
+            exiftool_ex = exceptions.MapillaryExiftoolNotFoundError(ex)
+            return [
+                types.describe_error_metadata(
+                    exiftool_ex, image_path, filetype=types.FileType.IMAGE
+                )
+                for image_path in image_paths
+            ]
 
         try:
             xml_element = ET.fromstring(xml)
@@ -86,7 +93,7 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
             LOG.warning(
                 "Failed to parse ExifTool XML: %s",
                 str(ex),
-                exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
             rdf_by_path = {}
         else:
@@ -102,29 +109,30 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
 class GeotagImagesFromExifToolWithSamples(GeotagImagesFromGeneric):
     def __init__(
         self,
-        xml_path: Path,
+        source_path: options.SourcePathOption,
         offset_time: float = 0.0,
         num_processes: int | None = None,
     ):
         super().__init__(num_processes=num_processes)
-        self.xml_path = xml_path
+        self.source_path = source_path
         self.offset_time = offset_time
 
     def geotag_samples(
         self, image_paths: T.Sequence[Path]
     ) -> list[types.ImageMetadataOrError]:
         # Find all video paths in self.xml_path
-        rdf_by_path = index_rdf_description_by_path([self.xml_path])
+        rdf_by_path = GeotagVideosFromExifToolXML.find_rdf_by_path(
+            self.source_path, image_paths
+        )
         video_paths = utils.find_videos(
-            [Path(pathstr) for pathstr in rdf_by_path.keys()],
+            [Path(canonical_path) for canonical_path in rdf_by_path.keys()],
             skip_subfolders=True,
         )
         # Find all video paths that have sample images
         samples_by_video = utils.find_all_image_samples(image_paths, video_paths)
 
         video_metadata_or_errors = GeotagVideosFromExifToolXML(
-            self.xml_path,
-            num_processes=self.num_processes,
+            self.source_path, num_processes=self.num_processes
         ).to_description(list(samples_by_video.keys()))
         sample_paths = sum(samples_by_video.values(), [])
         sample_metadata_or_errors = GeotagImagesFromVideo(
@@ -146,8 +154,7 @@ class GeotagImagesFromExifToolWithSamples(GeotagImagesFromGeneric):
         non_sample_paths = [path for path in image_paths if path not in sample_paths]
 
         non_sample_metadata_or_errors = GeotagImagesFromExifToolXML(
-            self.xml_path,
-            num_processes=self.num_processes,
+            self.source_path, num_processes=self.num_processes
         ).to_description(non_sample_paths)
 
         return sample_metadata_or_errors + non_sample_metadata_or_errors

mapillary_tools/geotag/geotag_images_from_gpx.py

@@ -12,6 +12,7 @@ else:
     from typing_extensions import override
 
 from .. import exceptions, geo, types
+from ..serializer.description import build_capture_time
 from .base import GeotagImagesFromGeneric
 from .geotag_images_from_exif import ImageEXIFExtractor
 
@@ -43,26 +44,26 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
 
         if image_metadata.time < sorted_points[0].time:
             delta = sorted_points[0].time - image_metadata.time
-            gpx_start_time = types.datetime_to_map_capture_time(sorted_points[0].time)
-            gpx_end_time = types.datetime_to_map_capture_time(sorted_points[-1].time)
+            gpx_start_time = build_capture_time(sorted_points[0].time)
+            gpx_end_time = build_capture_time(sorted_points[-1].time)
             # with the tolerance of 1ms
             if 0.001 < delta:
                 raise exceptions.MapillaryOutsideGPXTrackError(
                     f"The image date time is {round(delta, 3)} seconds behind the GPX start point",
-                    image_time=types.datetime_to_map_capture_time(image_metadata.time),
+                    image_time=build_capture_time(image_metadata.time),
                     gpx_start_time=gpx_start_time,
                     gpx_end_time=gpx_end_time,
                 )
 
         if sorted_points[-1].time < image_metadata.time:
             delta = image_metadata.time - sorted_points[-1].time
-            gpx_start_time = types.datetime_to_map_capture_time(sorted_points[0].time)
-            gpx_end_time = types.datetime_to_map_capture_time(sorted_points[-1].time)
+            gpx_start_time = build_capture_time(sorted_points[0].time)
+            gpx_end_time = build_capture_time(sorted_points[-1].time)
             # with the tolerance of 1ms
             if 0.001 < delta:
                 raise exceptions.MapillaryOutsideGPXTrackError(
                     f"The image time is {round(delta, 3)} seconds beyond the GPX end point",
-                    image_time=types.datetime_to_map_capture_time(image_metadata.time),
+                    image_time=build_capture_time(image_metadata.time),
                     gpx_start_time=gpx_start_time,
                     gpx_end_time=gpx_end_time,
                 )

mapillary_tools/geotag/geotag_images_from_video.py

@@ -13,6 +13,7 @@ else:
 from .. import types, utils
 from .base import GeotagImagesFromGeneric
 from .geotag_images_from_gpx import GeotagImagesFromGPX
+from .geotag_videos_from_video import GeotagVideosFromVideo
 
 
 LOG = logging.getLogger(__name__)
@@ -90,3 +91,37 @@ class GeotagImagesFromVideo(GeotagImagesFromGeneric):
         assert len(final_image_metadatas) <= len(image_paths)
 
         return final_image_metadatas
+
+
+class GeotagImageSamplesFromVideo(GeotagImagesFromGeneric):
+    def __init__(
+        self,
+        source_path: Path,
+        filetypes: set[types.FileType] | None = None,
+        offset_time: float = 0.0,
+        num_processes: int | None = None,
+    ):
+        super().__init__(num_processes=num_processes)
+        self.source_path = source_path
+        self.filetypes = filetypes
+        self.offset_time = offset_time
+
+    @override
+    def to_description(
+        self, image_paths: T.Sequence[Path]
+    ) -> list[types.ImageMetadataOrError]:
+        video_paths = utils.find_videos([self.source_path])
+        image_samples_by_video_path = utils.find_all_image_samples(
+            image_paths, video_paths
+        )
+        video_paths_with_image_samples = list(image_samples_by_video_path.keys())
+        video_metadatas = GeotagVideosFromVideo(
+            filetypes=self.filetypes,
+            num_processes=self.num_processes,
+        ).to_description(video_paths_with_image_samples)
+        geotag = GeotagImagesFromVideo(
+            video_metadatas,
+            offset_time=self.offset_time,
+            num_processes=self.num_processes,
+        )
+        return geotag.to_description(image_paths)