mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

mapillary_tools/geotag/{geotag_from_generic.py → base.py}

@@ -8,24 +8,14 @@ from pathlib import Path
 from tqdm import tqdm
 
 from .. import exceptions, types, utils
+from .image_extractors.base import BaseImageExtractor
+from .video_extractors.base import BaseVideoExtractor
 
 
 LOG = logging.getLogger(__name__)
 
 
-class GenericImageExtractor(abc.ABC):
-    """
-    Extracts metadata from an image file.
-    """
-
-    def __init__(self, image_path: Path):
-        self.image_path = image_path
-
-    def extract(self) -> types.ImageMetadataOrError:
-        raise NotImplementedError
-
-
-TImageExtractor = T.TypeVar("TImageExtractor", bound=GenericImageExtractor)
+TImageExtractor = T.TypeVar("TImageExtractor", bound=BaseImageExtractor)
 
 
 class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
@@ -33,16 +23,15 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
     Extracts metadata from a list of image files with multiprocessing.
     """
 
-    def __init__(
-        self, image_paths: T.Sequence[Path], num_processes: int | None = None
-    ) -> None:
-        self.image_paths = image_paths
+    def __init__(self, num_processes: int | None = None) -> None:
         self.num_processes = num_processes
 
-    def to_description(self) -> list[types.ImageMetadataOrError]:
-        extractor_or_errors = self._generate_image_extractors()
+    def to_description(
+        self, image_paths: T.Sequence[Path]
+    ) -> list[types.ImageMetadataOrError]:
+        extractor_or_errors = self._generate_image_extractors(image_paths)
 
-        assert len(extractor_or_errors) == len(self.image_paths)
+        assert len(extractor_or_errors) == len(image_paths)
 
         extractors, error_metadatas = types.separate_errors(extractor_or_errors)
 
@@ -64,11 +53,6 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
 
         return results + error_metadatas
 
-    def _generate_image_extractors(
-        self,
-    ) -> T.Sequence[TImageExtractor | types.ErrorMetadata]:
-        raise NotImplementedError
-
     # This method is passed to multiprocessing
     # so it has to be classmethod or staticmethod to avoid pickling the instance
     @classmethod
@@ -81,26 +65,23 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
             return types.describe_error_metadata(
                 ex, image_path, filetype=types.FileType.IMAGE
             )
+        except exceptions.MapillaryUserError as ex:
+            # Considered as fatal error if not MapillaryDescriptionError
+            raise ex
         except Exception as ex:
+            # TODO: hide details if not verbose mode
             LOG.exception("Unexpected error extracting metadata from %s", image_path)
             return types.describe_error_metadata(
                 ex, image_path, filetype=types.FileType.IMAGE
             )
 
-
-class GenericVideoExtractor(abc.ABC):
-    """
-    Extracts metadata from a video file.
-    """
-
-    def __init__(self, video_path: Path):
-        self.video_path = video_path
-
-    def extract(self) -> types.VideoMetadataOrError:
+    def _generate_image_extractors(
+        self, image_paths: T.Sequence[Path]
+    ) -> T.Sequence[TImageExtractor | types.ErrorMetadata]:
         raise NotImplementedError
 
 
-TVideoExtractor = T.TypeVar("TVideoExtractor", bound=GenericVideoExtractor)
+TVideoExtractor = T.TypeVar("TVideoExtractor", bound=BaseVideoExtractor)
 
 
 class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
@@ -108,16 +89,15 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
     Extracts metadata from a list of video files with multiprocessing.
     """
 
-    def __init__(
-        self, video_paths: T.Sequence[Path], num_processes: int | None = None
-    ) -> None:
-        self.video_paths = video_paths
+    def __init__(self, num_processes: int | None = None) -> None:
         self.num_processes = num_processes
 
-    def to_description(self) -> list[types.VideoMetadataOrError]:
-        extractor_or_errors = self._generate_video_extractors()
+    def to_description(
+        self, video_paths: T.Sequence[Path]
+    ) -> list[types.VideoMetadataOrError]:
+        extractor_or_errors = self._generate_video_extractors(video_paths)
 
-        assert len(extractor_or_errors) == len(self.video_paths)
+        assert len(extractor_or_errors) == len(video_paths)
 
         extractors, error_metadatas = types.separate_errors(extractor_or_errors)
 
@@ -139,11 +119,6 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
 
         return results + error_metadatas
 
-    def _generate_video_extractors(
-        self,
-    ) -> T.Sequence[TVideoExtractor | types.ErrorMetadata]:
-        raise NotImplementedError
-
     # This method is passed to multiprocessing
     # so it has to be classmethod or staticmethod to avoid pickling the instance
     @classmethod
@@ -156,8 +131,17 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
             return types.describe_error_metadata(
                 ex, video_path, filetype=types.FileType.VIDEO
             )
+        except exceptions.MapillaryUserError as ex:
+            # Considered as fatal error if not MapillaryDescriptionError
+            raise ex
         except Exception as ex:
+            # TODO: hide details if not verbose mode
             LOG.exception("Unexpected error extracting metadata from %s", video_path)
             return types.describe_error_metadata(
                 ex, video_path, filetype=types.FileType.VIDEO
             )
+
+    def _generate_video_extractors(
+        self, video_paths: T.Sequence[Path]
+    ) -> T.Sequence[TVideoExtractor | types.ErrorMetadata]:
+        raise NotImplementedError

mapillary_tools/geotag/factory.py

@@ -8,14 +8,13 @@ from pathlib import Path
 from .. import exceptions, types, utils
 from ..types import FileType
 from . import (
-    geotag_from_generic,
+    base,
     geotag_images_from_exif,
     geotag_images_from_exiftool,
-    geotag_images_from_exiftool_both_image_and_video,
     geotag_images_from_gpx_file,
     geotag_images_from_nmea_file,
     geotag_images_from_video,
-    geotag_videos_from_exiftool_video,
+    geotag_videos_from_exiftool,
     geotag_videos_from_gpx,
     geotag_videos_from_video,
 )
@@ -106,7 +105,7 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
 
 
 def _filter_images_and_videos(
-    file_paths: T.Iterable[Path],
+    paths: T.Iterable[Path],
     filetypes: set[types.FileType] | None = None,
 ) -> tuple[list[Path], list[Path]]:
     image_paths = []
@@ -121,7 +120,7 @@ def _filter_images_and_videos(
         include_images = types.FileType.IMAGE in filetypes
         include_videos = bool(filetypes & ALL_VIDEO_TYPES)
 
-    for path in file_paths:
+    for path in paths:
         if utils.is_image_file(path):
             if include_images:
                 image_paths.append(path)
@@ -154,20 +153,20 @@ def _geotag_images(
     else:
         interpolation = option.interpolation
 
-    geotag: geotag_from_generic.GeotagImagesFromGeneric
+    geotag: base.GeotagImagesFromGeneric
 
     if option.source is SourceType.NATIVE:
         geotag = geotag_images_from_exif.GeotagImagesFromEXIF(
-            image_paths, num_processes=option.num_processes
+            num_processes=option.num_processes
         )
-        return geotag.to_description()
+        return geotag.to_description(image_paths)
 
     if option.source is SourceType.EXIFTOOL_RUNTIME:
         geotag = geotag_images_from_exiftool.GeotagImagesFromExifToolRunner(
-            image_paths, num_processes=option.num_processes
+            num_processes=option.num_processes
         )
         try:
-            return geotag.to_description()
+            return geotag.to_description(image_paths)
         except exceptions.MapillaryExiftoolNotFoundError as ex:
             LOG.warning('Skip "%s" because: %s', option.source.value, ex)
             return []
@@ -175,39 +174,36 @@ def _geotag_images(
     elif option.source is SourceType.EXIFTOOL_XML:
         # This is to ensure 'video_process --geotag={"source": "exiftool_xml", "source_path": "/tmp/xml_path"}'
         # to work
-        geotag = geotag_images_from_exiftool_both_image_and_video.GeotagImagesFromExifToolBothImageAndVideo(
-            image_paths,
+        geotag = geotag_images_from_exiftool.GeotagImagesFromExifToolWithSamples(
             xml_path=_ensure_source_path(option),
             num_processes=option.num_processes,
         )
-        return geotag.to_description()
+        return geotag.to_description(image_paths)
 
     elif option.source is SourceType.GPX:
         geotag = geotag_images_from_gpx_file.GeotagImagesFromGPXFile(
-            image_paths,
             source_path=_ensure_source_path(option),
             use_gpx_start_time=interpolation.use_gpx_start_time,
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
-        return geotag.to_description()
+        return geotag.to_description(image_paths)
 
     elif option.source is SourceType.NMEA:
         geotag = geotag_images_from_nmea_file.GeotagImagesFromNMEAFile(
-            image_paths,
             source_path=_ensure_source_path(option),
             use_gpx_start_time=interpolation.use_gpx_start_time,
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
 
-        return geotag.to_description()
+        return geotag.to_description(image_paths)
 
     elif option.source is SourceType.EXIF:
         geotag = geotag_images_from_exif.GeotagImagesFromEXIF(
-            image_paths, num_processes=option.num_processes
+            num_processes=option.num_processes
         )
-        return geotag.to_description()
+        return geotag.to_description(image_paths)
 
     elif option.source in [
         SourceType.GOPRO,
@@ -225,17 +221,15 @@ def _geotag_images(
         )
         video_paths_with_image_samples = list(image_samples_by_video_path.keys())
         video_metadatas = geotag_videos_from_video.GeotagVideosFromVideo(
-            video_paths_with_image_samples,
             filetypes={map_geotag_source_to_filetype[option.source]},
             num_processes=option.num_processes,
-        ).to_description()
+        ).to_description(video_paths_with_image_samples)
         geotag = geotag_images_from_video.GeotagImagesFromVideo(
-            image_paths,
             video_metadatas,
             offset_time=interpolation.offset_time,
             num_processes=option.num_processes,
         )
-        return geotag.to_description()
+        return geotag.to_description(image_paths)
 
     else:
         raise ValueError(f"Invalid geotag source {option.source}")
@@ -249,34 +243,33 @@ def _geotag_videos(
     if not video_paths:
         return []
 
-    geotag: geotag_from_generic.GeotagVideosFromGeneric
+    geotag: base.GeotagVideosFromGeneric
 
     if option.source is SourceType.NATIVE:
         geotag = geotag_videos_from_video.GeotagVideosFromVideo(
-            video_paths, num_processes=option.num_processes, filetypes=option.filetypes
+            num_processes=option.num_processes, filetypes=option.filetypes
         )
-        return geotag.to_description()
+        return geotag.to_description(video_paths)
 
     if option.source is SourceType.EXIFTOOL_RUNTIME:
-        geotag = geotag_videos_from_exiftool_video.GeotagVideosFromExifToolRunner(
-            video_paths, num_processes=option.num_processes
+        geotag = geotag_videos_from_exiftool.GeotagVideosFromExifToolRunner(
+            num_processes=option.num_processes
         )
         try:
-            return geotag.to_description()
+            return geotag.to_description(video_paths)
         except exceptions.MapillaryExiftoolNotFoundError as ex:
             LOG.warning('Skip "%s" because: %s', option.source.value, ex)
             return []
 
     elif option.source is SourceType.EXIFTOOL_XML:
-        geotag = geotag_videos_from_exiftool_video.GeotagVideosFromExifToolVideo(
-            video_paths,
+        geotag = geotag_videos_from_exiftool.GeotagVideosFromExifToolXML(
             xml_path=_ensure_source_path(option),
         )
-        return geotag.to_description()
+        return geotag.to_description(video_paths)
 
     elif option.source is SourceType.GPX:
-        geotag = geotag_videos_from_gpx.GeotagVideosFromGPX(video_paths)
-        return geotag.to_description()
+        geotag = geotag_videos_from_gpx.GeotagVideosFromGPX()
+        return geotag.to_description(video_paths)
 
     elif option.source is SourceType.NMEA:
         # TODO: geotag videos from NMEA

mapillary_tools/geotag/geotag_images_from_exif.py

@@ -1,60 +1,24 @@
-import contextlib
+from __future__ import annotations
+
 import logging
+import sys
 import typing as T
 from pathlib import Path
 
-from .. import exceptions, geo, types, utils
-from ..exif_read import ExifRead, ExifReadABC
-from .geotag_from_generic import GenericImageExtractor, GeotagImagesFromGeneric
-
-LOG = logging.getLogger(__name__)
-
-
-class ImageEXIFExtractor(GenericImageExtractor):
-    def __init__(self, image_path: Path, skip_lonlat_error: bool = False):
-        super().__init__(image_path)
-        self.skip_lonlat_error = skip_lonlat_error
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
 
-    @contextlib.contextmanager
-    def _exif_context(self) -> T.Generator[ExifReadABC, None, None]:
-        with self.image_path.open("rb") as fp:
-            yield ExifRead(fp)
+from .base import GeotagImagesFromGeneric
+from .image_extractors.exif import ImageEXIFExtractor
 
-    def extract(self) -> types.ImageMetadata:
-        with self._exif_context() as exif:
-            lonlat = exif.extract_lon_lat()
-            if lonlat is None:
-                if not self.skip_lonlat_error:
-                    raise exceptions.MapillaryGeoTaggingError(
-                        "Unable to extract GPS Longitude or GPS Latitude from the image"
-                    )
-                lonlat = (0.0, 0.0)
-            lon, lat = lonlat
-
-            capture_time = exif.extract_capture_time()
-            if capture_time is None:
-                raise exceptions.MapillaryGeoTaggingError(
-                    "Unable to extract timestamp from the image"
-                )
-
-            image_metadata = types.ImageMetadata(
-                filename=self.image_path,
-                filesize=utils.get_file_size(self.image_path),
-                time=geo.as_unix_time(capture_time),
-                lat=lat,
-                lon=lon,
-                alt=exif.extract_altitude(),
-                angle=exif.extract_direction(),
-                width=exif.extract_width(),
-                height=exif.extract_height(),
-                MAPOrientation=exif.extract_orientation(),
-                MAPDeviceMake=exif.extract_make(),
-                MAPDeviceModel=exif.extract_model(),
-            )
-
-        return image_metadata
+LOG = logging.getLogger(__name__)
 
 
 class GeotagImagesFromEXIF(GeotagImagesFromGeneric):
-    def _generate_image_extractors(self) -> T.Sequence[ImageEXIFExtractor]:
-        return [ImageEXIFExtractor(path) for path in self.image_paths]
+    @override
+    def _generate_image_extractors(
+        self, image_paths: T.Sequence[Path]
+    ) -> T.Sequence[ImageEXIFExtractor]:
+        return [ImageEXIFExtractor(path) for path in image_paths]

mapillary_tools/geotag/geotag_images_from_exiftool.py

@@ -1,105 +1,153 @@
 from __future__ import annotations
 
-import contextlib
 import logging
+import sys
 import typing as T
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from .. import constants, exceptions, exiftool_read, types
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
+
+from .. import constants, exceptions, exiftool_read, types, utils
 from ..exiftool_runner import ExiftoolRunner
-from .geotag_from_generic import GeotagImagesFromGeneric
-from .geotag_images_from_exif import ImageEXIFExtractor
+from .base import GeotagImagesFromGeneric
+from .geotag_images_from_video import GeotagImagesFromVideo
+from .geotag_videos_from_exiftool import GeotagVideosFromExifToolXML
+from .image_extractors.exiftool import ImageExifToolExtractor
+from .utils import index_rdf_description_by_path
 
 LOG = logging.getLogger(__name__)
 
 
-class ImageExifToolExtractor(ImageEXIFExtractor):
-    def __init__(self, image_path: Path, element: ET.Element):
-        super().__init__(image_path)
-        self.element = element
-
-    @contextlib.contextmanager
-    def _exif_context(self):
-        yield exiftool_read.ExifToolRead(ET.ElementTree(self.element))
-
-
-class GeotagImagesFromExifTool(GeotagImagesFromGeneric):
+class GeotagImagesFromExifToolXML(GeotagImagesFromGeneric):
     def __init__(
         self,
-        image_paths: T.Sequence[Path],
         xml_path: Path,
         num_processes: int | None = None,
     ):
         self.xml_path = xml_path
-        super().__init__(image_paths=image_paths, num_processes=num_processes)
-
-    def _generate_image_extractors(
-        self,
-    ) -> T.Sequence[ImageExifToolExtractor | types.ErrorMetadata]:
-        rdf_description_by_path = exiftool_read.index_rdf_description_by_path(
-            [self.xml_path]
-        )
-
+        super().__init__(num_processes=num_processes)
+
+    @classmethod
+    def build_image_extractors(
+        cls,
+        rdf_by_path: dict[str, ET.Element],
+        image_paths: T.Iterable[Path],
+    ) -> list[ImageExifToolExtractor | types.ErrorMetadata]:
         results: list[ImageExifToolExtractor | types.ErrorMetadata] = []
 
-        for path in self.image_paths:
-            rdf_description = rdf_description_by_path.get(
-                exiftool_read.canonical_path(path)
-            )
-            if rdf_description is None:
-                exc = exceptions.MapillaryEXIFNotFoundError(
-                    f"The {exiftool_read._DESCRIPTION_TAG} XML element for the image not found"
+        for path in image_paths:
+            rdf = rdf_by_path.get(exiftool_read.canonical_path(path))
+            if rdf is None:
+                ex = exceptions.MapillaryExifToolXMLNotFoundError(
+                    "Cannot find the image in the ExifTool XML"
                 )
                 results.append(
                     types.describe_error_metadata(
-                        exc, path, filetype=types.FileType.IMAGE
+                        ex, path, filetype=types.FileType.IMAGE
                     )
                 )
             else:
-                results.append(ImageExifToolExtractor(path, rdf_description))
+                results.append(ImageExifToolExtractor(path, rdf))
 
         return results
 
+    @override
+    def _generate_image_extractors(
+        self, image_paths: T.Sequence[Path]
+    ) -> T.Sequence[ImageExifToolExtractor | types.ErrorMetadata]:
+        rdf_by_path = index_rdf_description_by_path([self.xml_path])
+        return self.build_image_extractors(rdf_by_path, image_paths)
+
 
 class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
+    @override
     def _generate_image_extractors(
-        self,
+        self, image_paths: T.Sequence[Path]
     ) -> T.Sequence[ImageExifToolExtractor | types.ErrorMetadata]:
         runner = ExiftoolRunner(constants.EXIFTOOL_PATH)
 
         LOG.debug(
-            "Extracting XML from %d images with exiftool command: %s",
-            len(self.image_paths),
+            "Extracting XML from %d images with ExifTool command: %s",
+            len(image_paths),
             " ".join(runner._build_args_read_stdin()),
         )
         try:
-            xml = runner.extract_xml(self.image_paths)
+            xml = runner.extract_xml(image_paths)
         except FileNotFoundError as ex:
             raise exceptions.MapillaryExiftoolNotFoundError(ex) from ex
 
-        rdf_description_by_path = (
-            exiftool_read.index_rdf_description_by_path_from_xml_element(
-                ET.fromstring(xml)
+        try:
+            xml_element = ET.fromstring(xml)
+        except ET.ParseError as ex:
+            LOG.warning(
+                "Failed to parse ExifTool XML: %s",
+                str(ex),
+                exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
+            )
+            rdf_by_path = {}
+        else:
+            rdf_by_path = exiftool_read.index_rdf_description_by_path_from_xml_element(
+                xml_element
             )
-        )
 
-        results: list[ImageExifToolExtractor | types.ErrorMetadata] = []
+        return GeotagImagesFromExifToolXML.build_image_extractors(
+            rdf_by_path, image_paths
+        )
 
-        for path in self.image_paths:
-            rdf_description = rdf_description_by_path.get(
-                exiftool_read.canonical_path(path)
-            )
-            if rdf_description is None:
-                exc = exceptions.MapillaryEXIFNotFoundError(
-                    f"The {exiftool_read._DESCRIPTION_TAG} XML element for the image not found"
-                )
-                results.append(
-                    types.describe_error_metadata(
-                        exc, path, filetype=types.FileType.IMAGE
-                    )
-                )
-            else:
-                results.append(ImageExifToolExtractor(path, rdf_description))
 
-        return results
+class GeotagImagesFromExifToolWithSamples(GeotagImagesFromGeneric):
+    def __init__(
+        self,
+        xml_path: Path,
+        offset_time: float = 0.0,
+        num_processes: int | None = None,
+    ):
+        super().__init__(num_processes=num_processes)
+        self.xml_path = xml_path
+        self.offset_time = offset_time
+
+    def geotag_samples(
+        self, image_paths: T.Sequence[Path]
+    ) -> list[types.ImageMetadataOrError]:
+        # Find all video paths in self.xml_path
+        rdf_by_path = index_rdf_description_by_path([self.xml_path])
+        video_paths = utils.find_videos(
+            [Path(pathstr) for pathstr in rdf_by_path.keys()],
+            skip_subfolders=True,
+        )
+        # Find all video paths that have sample images
+        samples_by_video = utils.find_all_image_samples(image_paths, video_paths)
+
+        video_metadata_or_errors = GeotagVideosFromExifToolXML(
+            self.xml_path,
+            num_processes=self.num_processes,
+        ).to_description(list(samples_by_video.keys()))
+        sample_paths = sum(samples_by_video.values(), [])
+        sample_metadata_or_errors = GeotagImagesFromVideo(
+            video_metadata_or_errors,
+            offset_time=self.offset_time,
+            num_processes=self.num_processes,
+        ).to_description(sample_paths)
+
+        return sample_metadata_or_errors
+
+    @override
+    def to_description(
+        self, image_paths: T.Sequence[Path]
+    ) -> list[types.ImageMetadataOrError]:
+        sample_metadata_or_errors = self.geotag_samples(image_paths)
+
+        sample_paths = set(metadata.filename for metadata in sample_metadata_or_errors)
+
+        non_sample_paths = [path for path in image_paths if path not in sample_paths]
+
+        non_sample_metadata_or_errors = GeotagImagesFromExifToolXML(
+            self.xml_path,
+            num_processes=self.num_processes,
+        ).to_description(non_sample_paths)
+
+        return sample_metadata_or_errors + non_sample_metadata_or_errors