mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0b1__py3-none-any.whl

mapillary_tools/geo.py

@@ -244,14 +244,14 @@ class Interpolator:
         return interpolated
 
 
-_PointAbstract = T.TypeVar("_PointAbstract")
+_T = T.TypeVar("_T")
 
 
 def sample_points_by_distance(
-    samples: T.Iterable[_PointAbstract],
+    samples: T.Iterable[_T],
     min_distance: float,
-    point_func: T.Callable[[_PointAbstract], Point],
-) -> T.Generator[_PointAbstract, None, None]:
+    point_func: T.Callable[[_T], Point],
+) -> T.Generator[_T, None, None]:
     prevp: Point | None = None
     for sample in samples:
         if prevp is None:
@@ -281,23 +281,6 @@ def interpolate_directions_if_none(sequence: T.Sequence[PointLike]) -> None:
             sequence[-1].angle = prev_angle
 
 
-def extend_deduplicate_points(
-    sequence: T.Iterable[PointLike],
-    to_extend: list[PointLike] | None = None,
-) -> list[PointLike]:
-    if to_extend is None:
-        to_extend = []
-    for point in sequence:
-        if to_extend:
-            prev = to_extend[-1].lon, to_extend[-1].lat
-            cur = (point.lon, point.lat)
-            if cur != prev:
-                to_extend.append(point)
-        else:
-            to_extend.append(point)
-    return to_extend
-
-
 def _ecef_from_lla2(lat: float, lon: float) -> tuple[float, float, float]:
     """
     Compute ECEF XYZ from latitude and longitude.

mapillary_tools/geotag/__init__.py

@@ -1 +0,0 @@
-from .. import geo  # noqa: F401

mapillary_tools/geotag/{geotag_from_generic.py → base.py}

@@ -8,24 +8,14 @@ from pathlib import Path
 from tqdm import tqdm
 
 from .. import exceptions, types, utils
+from .image_extractors.base import BaseImageExtractor
+from .video_extractors.base import BaseVideoExtractor
 
 
 LOG = logging.getLogger(__name__)
 
 
-class GenericImageExtractor(abc.ABC):
-    """
-    Extracts metadata from an image file.
-    """
-
-    def __init__(self, image_path: Path):
-        self.image_path = image_path
-
-    def extract(self) -> types.ImageMetadataOrError:
-        raise NotImplementedError
-
-
-TImageExtractor = T.TypeVar("TImageExtractor", bound=GenericImageExtractor)
+TImageExtractor = T.TypeVar("TImageExtractor", bound=BaseImageExtractor)
 
 
 class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
@@ -33,16 +23,15 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
     Extracts metadata from a list of image files with multiprocessing.
     """
 
-    def __init__(
-        self, image_paths: T.Sequence[Path], num_processes: int | None = None
-    ) -> None:
-        self.image_paths = image_paths
+    def __init__(self, num_processes: int | None = None) -> None:
         self.num_processes = num_processes
 
-    def to_description(self) -> list[types.ImageMetadataOrError]:
-        extractor_or_errors = self._generate_image_extractors()
+    def to_description(
+        self, image_paths: T.Sequence[Path]
+    ) -> list[types.ImageMetadataOrError]:
+        extractor_or_errors = self._generate_image_extractors(image_paths)
 
-        assert len(extractor_or_errors) == len(self.image_paths)
+        assert len(extractor_or_errors) == len(image_paths)
 
         extractors, error_metadatas = types.separate_errors(extractor_or_errors)
 
@@ -62,12 +51,7 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
             )
         )
 
-        return results + error_metadatas
-
-    def _generate_image_extractors(
-        self,
-    ) -> T.Sequence[TImageExtractor | types.ErrorMetadata]:
-        raise NotImplementedError
+        return T.cast(list[types.ImageMetadataOrError], results + error_metadatas)
 
     # This method is passed to multiprocessing
     # so it has to be classmethod or staticmethod to avoid pickling the instance
@@ -81,26 +65,23 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
             return types.describe_error_metadata(
                 ex, image_path, filetype=types.FileType.IMAGE
             )
+        except exceptions.MapillaryUserError as ex:
+            # Considered as fatal error if not MapillaryDescriptionError
+            raise ex
         except Exception as ex:
+            # TODO: hide details if not verbose mode
             LOG.exception("Unexpected error extracting metadata from %s", image_path)
             return types.describe_error_metadata(
                 ex, image_path, filetype=types.FileType.IMAGE
             )
 
-
-class GenericVideoExtractor(abc.ABC):
-    """
-    Extracts metadata from a video file.
-    """
-
-    def __init__(self, video_path: Path):
-        self.video_path = video_path
-
-    def extract(self) -> types.VideoMetadataOrError:
+    def _generate_image_extractors(
+        self, image_paths: T.Sequence[Path]
+    ) -> T.Sequence[TImageExtractor | types.ErrorMetadata]:
         raise NotImplementedError
 
 
-TVideoExtractor = T.TypeVar("TVideoExtractor", bound=GenericVideoExtractor)
+TVideoExtractor = T.TypeVar("TVideoExtractor", bound=BaseVideoExtractor)
 
 
 class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
@@ -108,16 +89,15 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
     Extracts metadata from a list of video files with multiprocessing.
     """
 
-    def __init__(
-        self, video_paths: T.Sequence[Path], num_processes: int | None = None
-    ) -> None:
-        self.video_paths = video_paths
+    def __init__(self, num_processes: int | None = None) -> None:
         self.num_processes = num_processes
 
-    def to_description(self) -> list[types.VideoMetadataOrError]:
-        extractor_or_errors = self._generate_video_extractors()
+    def to_description(
+        self, video_paths: T.Sequence[Path]
+    ) -> list[types.VideoMetadataOrError]:
+        extractor_or_errors = self._generate_video_extractors(video_paths)
 
-        assert len(extractor_or_errors) == len(self.video_paths)
+        assert len(extractor_or_errors) == len(video_paths)
 
         extractors, error_metadatas = types.separate_errors(extractor_or_errors)
 
@@ -137,12 +117,7 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
             )
         )
 
-        return results + error_metadatas
-
-    def _generate_video_extractors(
-        self,
-    ) -> T.Sequence[TVideoExtractor | types.ErrorMetadata]:
-        raise NotImplementedError
+        return T.cast(list[types.VideoMetadataOrError], results + error_metadatas)
 
     # This method is passed to multiprocessing
     # so it has to be classmethod or staticmethod to avoid pickling the instance
@@ -156,8 +131,17 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
             return types.describe_error_metadata(
                 ex, video_path, filetype=types.FileType.VIDEO
             )
+        except exceptions.MapillaryUserError as ex:
+            # Considered as fatal error if not MapillaryDescriptionError
+            raise ex
         except Exception as ex:
+            # TODO: hide details if not verbose mode
             LOG.exception("Unexpected error extracting metadata from %s", video_path)
             return types.describe_error_metadata(
                 ex, video_path, filetype=types.FileType.VIDEO
             )
+
+    def _generate_video_extractors(
+        self, video_paths: T.Sequence[Path]
+    ) -> T.Sequence[TVideoExtractor | types.ErrorMetadata]:
+        raise NotImplementedError

mapillary_tools/geotag/factory.py

@@ -6,16 +6,14 @@ import typing as T
 from pathlib import Path
 
 from .. import exceptions, types, utils
-from ..types import FileType
 from . import (
-    geotag_from_generic,
+    base,
     geotag_images_from_exif,
     geotag_images_from_exiftool,
-    geotag_images_from_exiftool_both_image_and_video,
     geotag_images_from_gpx_file,
     geotag_images_from_nmea_file,
     geotag_images_from_video,
-    geotag_videos_from_exiftool_video,
+    geotag_videos_from_exiftool,
     geotag_videos_from_gpx,
     geotag_videos_from_video,
 )
@@ -71,8 +69,25 @@ def process(
     for idx, option in enumerate(options):
         LOG.debug("Processing %d files with %s", len(reprocessable_paths), option)
 
-        image_metadata_or_errors = _geotag_images(reprocessable_paths, option)
-        video_metadata_or_errors = _geotag_videos(reprocessable_paths, option)
+        image_videos, video_paths = _filter_images_and_videos(
+            reprocessable_paths, option.filetypes
+        )
+
+        if image_videos:
+            image_geotag = _build_image_geotag(option)
+            image_metadata_or_errors = (
+                image_geotag.to_description(image_videos) if image_geotag else []
+            )
+        else:
+            image_metadata_or_errors = []
+
+        if video_paths:
+            video_geotag = _build_video_geotag(option)
+            video_metadata_or_errors = (
+                video_geotag.to_description(video_paths) if video_geotag else []
+            )
+        else:
+            video_metadata_or_errors = []
 
         more_option = idx < len(options) - 1
 
@@ -98,6 +113,7 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
             (
                 exceptions.MapillaryGeoTaggingError,
                 exceptions.MapillaryVideoGPSNotFoundError,
+                exceptions.MapillaryExiftoolNotFoundError,
             ),
         ):
             return True
@@ -106,7 +122,7 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
 
 
 def _filter_images_and_videos(
-    file_paths: T.Iterable[Path],
+    paths: T.Iterable[Path],
     filetypes: set[types.FileType] | None = None,
 ) -> tuple[list[Path], list[Path]]:
     image_paths = []
@@ -121,7 +137,7 @@ def _filter_images_and_videos(
         include_images = types.FileType.IMAGE in filetypes
         include_videos = bool(filetypes & ALL_VIDEO_TYPES)
 
-    for path in file_paths:
+    for path in paths:
         if utils.is_image_file(path):
             if include_images:
                 image_paths.append(path)
@@ -141,158 +157,144 @@ def _ensure_source_path(option: SourceOption) -> Path:
     return option.source_path.source_path
 
 
-def _geotag_images(
-    paths: T.Iterable[Path], option: SourceOption
-) -> list[types.ImageMetadataOrError]:
-    image_paths, _ = _filter_images_and_videos(paths, option.filetypes)
-
-    if not image_paths:
-        return []
-
+def _build_image_geotag(option: SourceOption) -> base.GeotagImagesFromGeneric | None:
+    """
+    Build a GeotagImagesFromGeneric object based on the provided option.
+    """
     if option.interpolation is None:
         interpolation = InterpolationOption()
     else:
         interpolation = option.interpolation
 
-    geotag: geotag_from_generic.GeotagImagesFromGeneric
-
-    if option.source is SourceType.NATIVE:
-        geotag = geotag_images_from_exif.GeotagImagesFromEXIF(
-            image_paths, num_processes=option.num_processes
+    if option.source in [SourceType.EXIF, SourceType.NATIVE]:
+        return geotag_images_from_exif.GeotagImagesFromEXIF(
+            num_processes=option.num_processes
         )
-        return geotag.to_description()
 
     if option.source is SourceType.EXIFTOOL_RUNTIME:
-        geotag = geotag_images_from_exiftool.GeotagImagesFromExifToolRunner(
-            image_paths, num_processes=option.num_processes
+        return geotag_images_from_exiftool.GeotagImagesFromExifToolRunner(
+            num_processes=option.num_processes
         )
-        try:
-            return geotag.to_description()
-        except exceptions.MapillaryExiftoolNotFoundError as ex:
-            LOG.warning('Skip "%s" because: %s', option.source.value, ex)
-            return []
 
     elif option.source is SourceType.EXIFTOOL_XML:
         # This is to ensure 'video_process --geotag={"source": "exiftool_xml", "source_path": "/tmp/xml_path"}'
         # to work
-        geotag = geotag_images_from_exiftool_both_image_and_video.GeotagImagesFromExifToolBothImageAndVideo(
-            image_paths,
-            xml_path=_ensure_source_path(option),
+        if option.source_path is None:
+            raise exceptions.MapillaryBadParameterError(
+                "source_path must be provided for EXIFTOOL_XML source"
+            )
+        return geotag_images_from_exiftool.GeotagImagesFromExifToolWithSamples(
+            source_path=option.source_path,
             num_processes=option.num_processes,
         )
-        return geotag.to_description()
 
     elif option.source is SourceType.GPX:
-        geotag = geotag_images_from_gpx_file.GeotagImagesFromGPXFile(
-            image_paths,
+        return geotag_images_from_gpx_file.GeotagImagesFromGPXFile(
             source_path=_ensure_source_path(option),
             use_gpx_start_time=interpolation.use_gpx_start_time,
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
-        return geotag.to_description()
 
     elif option.source is SourceType.NMEA:
-        geotag = geotag_images_from_nmea_file.GeotagImagesFromNMEAFile(
-            image_paths,
+        return geotag_images_from_nmea_file.GeotagImagesFromNMEAFile(
             source_path=_ensure_source_path(option),
             use_gpx_start_time=interpolation.use_gpx_start_time,
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
 
-        return geotag.to_description()
-
-    elif option.source is SourceType.EXIF:
-        geotag = geotag_images_from_exif.GeotagImagesFromEXIF(
-            image_paths, num_processes=option.num_processes
-        )
-        return geotag.to_description()
-
-    elif option.source in [
-        SourceType.GOPRO,
-        SourceType.BLACKVUE,
-        SourceType.CAMM,
-    ]:
-        map_geotag_source_to_filetype: dict[SourceType, FileType] = {
-            SourceType.GOPRO: FileType.GOPRO,
-            SourceType.BLACKVUE: FileType.BLACKVUE,
-            SourceType.CAMM: FileType.CAMM,
-        }
-        video_paths = utils.find_videos([_ensure_source_path(option)])
-        image_samples_by_video_path = utils.find_all_image_samples(
-            image_paths, video_paths
-        )
-        video_paths_with_image_samples = list(image_samples_by_video_path.keys())
-        video_metadatas = geotag_videos_from_video.GeotagVideosFromVideo(
-            video_paths_with_image_samples,
-            filetypes={map_geotag_source_to_filetype[option.source]},
-            num_processes=option.num_processes,
-        ).to_description()
-        geotag = geotag_images_from_video.GeotagImagesFromVideo(
-            image_paths,
-            video_metadatas,
+    elif option.source in [SourceType.GOPRO, SourceType.BLACKVUE, SourceType.CAMM]:
+        return geotag_images_from_video.GeotagImageSamplesFromVideo(
+            _ensure_source_path(option),
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
-        return geotag.to_description()
 
     else:
         raise ValueError(f"Invalid geotag source {option.source}")
 
 
-def _geotag_videos(
-    paths: T.Iterable[Path], option: SourceOption
-) -> list[types.VideoMetadataOrError]:
-    _, video_paths = _filter_images_and_videos(paths, option.filetypes)
-
-    if not video_paths:
-        return []
-
-    geotag: geotag_from_generic.GeotagVideosFromGeneric
+def _build_video_geotag(option: SourceOption) -> base.GeotagVideosFromGeneric | None:
+    """
+    Build a GeotagVideosFromGeneric object based on the provided option.
 
+    Examples:
+        >>> from pathlib import Path
+        >>> from mapillary_tools.geotag.options import SourceOption, SourceType
+        >>> opt = SourceOption(SourceType.NATIVE)
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromVideo'
+
+        >>> opt = SourceOption(SourceType.EXIFTOOL_RUNTIME)
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromExifToolRunner'
+
+        >>> opt = SourceOption(SourceType.EXIFTOOL_XML, source_path=Path("/tmp/test.xml"))
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromExifToolXML'
+
+        >>> opt = SourceOption(SourceType.GPX, source_path=Path("/tmp/test.gpx"))
+        >>> geotagger = _build_video_geotag(opt)
+        >>> geotagger.__class__.__name__
+        'GeotagVideosFromGPX'
+
+        >>> opt = SourceOption(SourceType.NMEA, source_path=Path("/tmp/test.nmea"))
+        >>> _build_video_geotag(opt) is None
+        True
+
+        >>> opt = SourceOption(SourceType.EXIF)
+        >>> _build_video_geotag(opt) is None
+        True
+
+        >>> opt = SourceOption(SourceType.GOPRO)
+        >>> _build_video_geotag(opt) is None
+        True
+
+        >>> try:
+        ...     _build_video_geotag(SourceOption("invalid"))
+        ... except ValueError as e:
+        ...     "Invalid geotag source" in str(e)
+        True
+    """
     if option.source is SourceType.NATIVE:
-        geotag = geotag_videos_from_video.GeotagVideosFromVideo(
-            video_paths, num_processes=option.num_processes, filetypes=option.filetypes
+        return geotag_videos_from_video.GeotagVideosFromVideo(
+            num_processes=option.num_processes, filetypes=option.filetypes
         )
-        return geotag.to_description()
 
     if option.source is SourceType.EXIFTOOL_RUNTIME:
-        geotag = geotag_videos_from_exiftool_video.GeotagVideosFromExifToolRunner(
-            video_paths, num_processes=option.num_processes
+        return geotag_videos_from_exiftool.GeotagVideosFromExifToolRunner(
+            num_processes=option.num_processes
         )
-        try:
-            return geotag.to_description()
-        except exceptions.MapillaryExiftoolNotFoundError as ex:
-            LOG.warning('Skip "%s" because: %s', option.source.value, ex)
-            return []
 
     elif option.source is SourceType.EXIFTOOL_XML:
-        geotag = geotag_videos_from_exiftool_video.GeotagVideosFromExifToolVideo(
-            video_paths,
-            xml_path=_ensure_source_path(option),
+        if option.source_path is None:
+            raise exceptions.MapillaryBadParameterError(
+                "source_path must be provided for EXIFTOOL_XML source"
+            )
+        return geotag_videos_from_exiftool.GeotagVideosFromExifToolXML(
+            source_path=option.source_path,
         )
-        return geotag.to_description()
 
     elif option.source is SourceType.GPX:
-        geotag = geotag_videos_from_gpx.GeotagVideosFromGPX(video_paths)
-        return geotag.to_description()
+        return geotag_videos_from_gpx.GeotagVideosFromGPX(
+            source_path=option.source_path, num_processes=option.num_processes
+        )
 
     elif option.source is SourceType.NMEA:
         # TODO: geotag videos from NMEA
-        return []
+        return None
 
     elif option.source is SourceType.EXIF:
         # Legacy image-specific geotag types
-        return []
+        return None
 
-    elif option.source in [
-        SourceType.GOPRO,
-        SourceType.BLACKVUE,
-        SourceType.CAMM,
-    ]:
+    elif option.source in [SourceType.GOPRO, SourceType.BLACKVUE, SourceType.CAMM]:
         # Legacy image-specific geotag types
-        return []
+        return None
 
     else:
         raise ValueError(f"Invalid geotag source {option.source}")

mapillary_tools/geotag/geotag_images_from_exif.py

@@ -1,60 +1,24 @@
-import contextlib
+from __future__ import annotations
+
 import logging
+import sys
 import typing as T
 from pathlib import Path
 
-from .. import exceptions, geo, types, utils
-from ..exif_read import ExifRead, ExifReadABC
-from .geotag_from_generic import GenericImageExtractor, GeotagImagesFromGeneric
-
-LOG = logging.getLogger(__name__)
-
-
-class ImageEXIFExtractor(GenericImageExtractor):
-    def __init__(self, image_path: Path, skip_lonlat_error: bool = False):
-        super().__init__(image_path)
-        self.skip_lonlat_error = skip_lonlat_error
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
 
-    @contextlib.contextmanager
-    def _exif_context(self) -> T.Generator[ExifReadABC, None, None]:
-        with self.image_path.open("rb") as fp:
-            yield ExifRead(fp)
+from .base import GeotagImagesFromGeneric
+from .image_extractors.exif import ImageEXIFExtractor
 
-    def extract(self) -> types.ImageMetadata:
-        with self._exif_context() as exif:
-            lonlat = exif.extract_lon_lat()
-            if lonlat is None:
-                if not self.skip_lonlat_error:
-                    raise exceptions.MapillaryGeoTaggingError(
-                        "Unable to extract GPS Longitude or GPS Latitude from the image"
-                    )
-                lonlat = (0.0, 0.0)
-            lon, lat = lonlat
-
-            capture_time = exif.extract_capture_time()
-            if capture_time is None:
-                raise exceptions.MapillaryGeoTaggingError(
-                    "Unable to extract timestamp from the image"
-                )
-
-            image_metadata = types.ImageMetadata(
-                filename=self.image_path,
-                filesize=utils.get_file_size(self.image_path),
-                time=geo.as_unix_time(capture_time),
-                lat=lat,
-                lon=lon,
-                alt=exif.extract_altitude(),
-                angle=exif.extract_direction(),
-                width=exif.extract_width(),
-                height=exif.extract_height(),
-                MAPOrientation=exif.extract_orientation(),
-                MAPDeviceMake=exif.extract_make(),
-                MAPDeviceModel=exif.extract_model(),
-            )
-
-        return image_metadata
+LOG = logging.getLogger(__name__)
 
 
 class GeotagImagesFromEXIF(GeotagImagesFromGeneric):
-    def _generate_image_extractors(self) -> T.Sequence[ImageEXIFExtractor]:
-        return [ImageEXIFExtractor(path) for path in self.image_paths]
+    @override
+    def _generate_image_extractors(
+        self, image_paths: T.Sequence[Path]
+    ) -> T.Sequence[ImageEXIFExtractor]:
+        return [ImageEXIFExtractor(path) for path in image_paths]