mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0a1__py3-none-any.whl

mapillary_tools/geotag/geotag_images_from_gpx.py

@@ -1,12 +1,13 @@
+from __future__ import annotations
+
 import dataclasses
 import logging
 import typing as T
-from multiprocessing import Pool
 from pathlib import Path
 
 from .. import exceptions, geo, types
 from .geotag_from_generic import GeotagImagesFromGeneric
-from .geotag_images_from_exif import GeotagImagesFromEXIF
+from .geotag_images_from_exif import ImageEXIFExtractor
 
 
 LOG = logging.getLogger(__name__)
@@ -20,41 +21,19 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
         use_gpx_start_time: bool = False,
         use_image_start_time: bool = False,
         offset_time: float = 0.0,
-        num_processes: T.Optional[int] = None,
+        num_processes: int | None = None,
     ):
-        super().__init__()
-        self.image_paths = image_paths
+        super().__init__(image_paths, num_processes=num_processes)
         self.points = points
         self.use_gpx_start_time = use_gpx_start_time
         self.use_image_start_time = use_image_start_time
         self.offset_time = offset_time
-        self.num_processes = num_processes
-
-    @staticmethod
-    def geotag_image(image_path: Path) -> types.ImageMetadataOrError:
-        return GeotagImagesFromEXIF.geotag_image(image_path, skip_lonlat_error=True)
-
-    def geotag_multiple_images(
-        self, image_paths: T.Sequence[Path]
-    ) -> T.List[types.ImageMetadataOrError]:
-        if self.num_processes is None:
-            num_processes = self.num_processes
-            disable_multiprocessing = False
-        else:
-            num_processes = max(self.num_processes, 1)
-            disable_multiprocessing = self.num_processes <= 0
-
-        if disable_multiprocessing:
-            return list(map(GeotagImagesFromGPX.geotag_image, image_paths))
-        else:
-            with Pool(processes=num_processes) as pool:
-                return pool.map(GeotagImagesFromGPX.geotag_image, image_paths)
 
     def _interpolate_image_metadata_along(
         self,
         image_metadata: types.ImageMetadata,
         sorted_points: T.Sequence[geo.Point],
-    ) -> types.ImageMetadataOrError:
+    ) -> types.ImageMetadata:
         assert sorted_points, "must have at least one point"
 
         if image_metadata.time < sorted_points[0].time:
@@ -63,15 +42,12 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
             gpx_end_time = types.datetime_to_map_capture_time(sorted_points[-1].time)
             # with the tolerance of 1ms
             if 0.001 < delta:
-                exc = exceptions.MapillaryOutsideGPXTrackError(
+                raise exceptions.MapillaryOutsideGPXTrackError(
                     f"The image date time is {round(delta, 3)} seconds behind the GPX start point",
                     image_time=types.datetime_to_map_capture_time(image_metadata.time),
                     gpx_start_time=gpx_start_time,
                     gpx_end_time=gpx_end_time,
                 )
-                return types.describe_error_metadata(
-                    exc, image_metadata.filename, filetype=types.FileType.IMAGE
-                )
 
         if sorted_points[-1].time < image_metadata.time:
             delta = image_metadata.time - sorted_points[-1].time
@@ -79,15 +55,12 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
             gpx_end_time = types.datetime_to_map_capture_time(sorted_points[-1].time)
             # with the tolerance of 1ms
             if 0.001 < delta:
-                exc = exceptions.MapillaryOutsideGPXTrackError(
+                raise exceptions.MapillaryOutsideGPXTrackError(
                     f"The image time is {round(delta, 3)} seconds beyond the GPX end point",
                     image_time=types.datetime_to_map_capture_time(image_metadata.time),
                     gpx_start_time=gpx_start_time,
                     gpx_end_time=gpx_end_time,
                 )
-                return types.describe_error_metadata(
-                    exc, image_metadata.filename, filetype=types.FileType.IMAGE
-                )
 
         interpolated = geo.interpolate(sorted_points, image_metadata.time)
 
@@ -100,34 +73,25 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
             time=interpolated.time,
         )
 
-    def to_description(self) -> T.List[types.ImageMetadataOrError]:
-        metadatas: T.List[types.ImageMetadataOrError] = []
-
-        if not self.points:
-            exc = exceptions.MapillaryGPXEmptyError(
-                "Empty GPS extracted from the geotag source"
-            )
-            for image_path in self.image_paths:
-                metadatas.append(
-                    types.describe_error_metadata(
-                        exc, image_path, filetype=types.FileType.IMAGE
-                    ),
-                )
-            assert len(self.image_paths) == len(metadatas)
-            return metadatas
+    def _generate_image_extractors(self) -> T.Sequence[ImageEXIFExtractor]:
+        return [
+            ImageEXIFExtractor(path, skip_lonlat_error=True)
+            for path in self.image_paths
+        ]
+
+    def to_description(self) -> list[types.ImageMetadataOrError]:
+        final_metadatas: list[types.ImageMetadataOrError] = []
 
-        image_metadata_or_errors = self.geotag_multiple_images(self.image_paths)
+        image_metadata_or_errors = super().to_description()
 
-        image_metadatas = []
-        for metadata_or_error in image_metadata_or_errors:
-            if isinstance(metadata_or_error, types.ErrorMetadata):
-                metadatas.append(metadata_or_error)
-            else:
-                image_metadatas.append(metadata_or_error)
+        image_metadatas, error_metadatas = types.separate_errors(
+            image_metadata_or_errors
+        )
+        final_metadatas.extend(error_metadatas)
 
         if not image_metadatas:
-            assert len(self.image_paths) == len(metadatas)
-            return metadatas
+            assert len(self.image_paths) == len(final_metadatas)
+            return final_metadatas
 
         # Do not use point itself for comparison because point.angle or point.alt could be None
         # when you compare nonnull value with None, it will throw
@@ -162,64 +126,25 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
                 LOG.debug("GPX start time delta: %s", time_delta)
                 image_time_offset += time_delta
 
-        LOG.debug("Final time offset for interpolation: %s", image_time_offset)
+        if image_time_offset:
+            LOG.debug("Final time offset for interpolation: %s", image_time_offset)
+            for image_metadata in sorted_image_metadatas:
+                # TODO: this time modification seems to affect final capture times
+                image_metadata.time += image_time_offset
 
         for image_metadata in sorted_image_metadatas:
-            image_metadata.time += image_time_offset
-            metadatas.append(
-                self._interpolate_image_metadata_along(image_metadata, sorted_points)
-            )
-
-        assert len(self.image_paths) == len(metadatas)
-        return metadatas
-
-
-class GeotagImagesFromGPXWithProgress(GeotagImagesFromGPX):
-    def __init__(
-        self,
-        image_paths: T.Sequence[Path],
-        points: T.Sequence[geo.Point],
-        use_gpx_start_time: bool = False,
-        use_image_start_time: bool = False,
-        offset_time: float = 0.0,
-        num_processes: T.Optional[int] = None,
-        progress_bar=None,
-    ) -> None:
-        super().__init__(
-            image_paths,
-            points,
-            use_gpx_start_time=use_gpx_start_time,
-            use_image_start_time=use_image_start_time,
-            offset_time=offset_time,
-            num_processes=num_processes,
-        )
-        self._progress_bar = progress_bar
-
-    def geotag_multiple_images(
-        self, image_paths: T.Sequence[Path]
-    ) -> T.List[types.ImageMetadataOrError]:
-        if self._progress_bar is None:
-            return super().geotag_multiple_images(image_paths)
-
-        if self.num_processes is None:
-            num_processes = self.num_processes
-            disable_multiprocessing = False
-        else:
-            num_processes = max(self.num_processes, 1)
-            disable_multiprocessing = self.num_processes <= 0
-
-        output = []
-        with Pool(processes=num_processes) as pool:
-            image_metadatas_iter: T.Iterator[types.ImageMetadataOrError]
-            if disable_multiprocessing:
-                image_metadatas_iter = map(
-                    GeotagImagesFromGPX.geotag_image, image_paths
+            try:
+                final_metadatas.append(
+                    self._interpolate_image_metadata_along(
+                        image_metadata, sorted_points
+                    )
                 )
-            else:
-                image_metadatas_iter = pool.imap(
-                    GeotagImagesFromGPX.geotag_image, image_paths
+            except exceptions.MapillaryOutsideGPXTrackError as ex:
+                error_metadata = types.describe_error_metadata(
+                    ex, image_metadata.filename, filetype=types.FileType.IMAGE
                 )
-            for image_metadata_or_error in image_metadatas_iter:
-                self._progress_bar.update(1)
-                output.append(image_metadata_or_error)
-        return output
+                final_metadatas.append(error_metadata)
+
+        assert len(self.image_paths) == len(final_metadatas)
+
+        return final_metadatas

mapillary_tools/geotag/geotag_images_from_gpx_file.py

@@ -1,30 +1,27 @@
-import dataclasses
+from __future__ import annotations
+
 import logging
 import typing as T
-from multiprocessing import Pool
 from pathlib import Path
 
 import gpxpy
-from tqdm import tqdm
 
-from .. import exif_read, geo, types
-from .geotag_from_generic import GeotagImagesFromGeneric
-from .geotag_images_from_gpx import GeotagImagesFromGPXWithProgress
+from .. import geo
+from .geotag_images_from_gpx import GeotagImagesFromGPX
 
 
 LOG = logging.getLogger(__name__)
 
 
-class GeotagImagesFromGPXFile(GeotagImagesFromGeneric):
+class GeotagImagesFromGPXFile(GeotagImagesFromGPX):
     def __init__(
         self,
         image_paths: T.Sequence[Path],
         source_path: Path,
         use_gpx_start_time: bool = False,
         offset_time: float = 0.0,
-        num_processes: T.Optional[int] = None,
+        num_processes: int | None = None,
     ):
-        super().__init__()
         try:
             tracks = parse_gpx(source_path)
         except Exception as ex:
@@ -38,102 +35,24 @@ class GeotagImagesFromGPXFile(GeotagImagesFromGeneric):
                 len(tracks),
                 source_path,
             )
-        self.points: T.List[geo.Point] = sum(tracks, [])
-        self.image_paths = image_paths
-        self.source_path = source_path
-        self.use_gpx_start_time = use_gpx_start_time
-        self.offset_time = offset_time
-        self.num_processes = num_processes
-
-    @staticmethod
-    def _extract_image_metadata(
-        image_metadata: types.ImageMetadata,
-    ) -> types.ImageMetadataOrError:
-        try:
-            exif = exif_read.ExifRead(image_metadata.filename)
-            orientation = exif.extract_orientation()
-            make = exif.extract_make()
-            model = exif.extract_model()
-        except Exception as ex:
-            return types.describe_error_metadata(
-                ex, image_metadata.filename, filetype=types.FileType.IMAGE
-            )
-
-        return dataclasses.replace(
-            image_metadata,
-            MAPOrientation=orientation,
-            MAPDeviceMake=make,
-            MAPDeviceModel=model,
-        )
-
-    def to_description(self) -> T.List[types.ImageMetadataOrError]:
-        with tqdm(
-            total=len(self.image_paths),
-            desc="Interpolating",
-            unit="images",
-            disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-        ) as pbar:
-            geotag = GeotagImagesFromGPXWithProgress(
-                self.image_paths,
-                self.points,
-                use_gpx_start_time=self.use_gpx_start_time,
-                offset_time=self.offset_time,
-                progress_bar=pbar,
-            )
-            image_metadata_or_errors = geotag.to_description()
-
-        image_metadatas: T.List[types.ImageMetadata] = []
-        error_metadatas: T.List[types.ErrorMetadata] = []
-        for metadata in image_metadata_or_errors:
-            if isinstance(metadata, types.ErrorMetadata):
-                error_metadatas.append(metadata)
-            else:
-                image_metadatas.append(metadata)
-
-        if self.num_processes is None:
-            num_processes = self.num_processes
-            disable_multiprocessing = False
-        else:
-            num_processes = max(self.num_processes, 1)
-            disable_multiprocessing = self.num_processes <= 0
-
-        with Pool(processes=num_processes) as pool:
-            image_metadatas_iter: T.Iterator[types.ImageMetadataOrError]
-            if disable_multiprocessing:
-                image_metadatas_iter = map(
-                    GeotagImagesFromGPXFile._extract_image_metadata, image_metadatas
-                )
-            else:
-                # Do not pass error metadatas where the error object can not be pickled for multiprocessing to work
-                # Otherwise we get:
-                # TypeError: __init__() missing 3 required positional arguments: 'image_time', 'gpx_start_time', and 'gpx_end_time'
-                # See https://stackoverflow.com/a/61432070
-                image_metadatas_iter = pool.imap(
-                    GeotagImagesFromGPXFile._extract_image_metadata, image_metadatas
-                )
-            image_metadata_or_errors = list(
-                tqdm(
-                    image_metadatas_iter,
-                    desc="Processing",
-                    unit="images",
-                    disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-                )
-            )
-
-        return (
-            T.cast(T.List[types.ImageMetadataOrError], error_metadatas)
-            + image_metadata_or_errors
+        points = sum(tracks, [])
+        super().__init__(
+            image_paths,
+            points,
+            use_gpx_start_time=use_gpx_start_time,
+            offset_time=offset_time,
+            num_processes=num_processes,
         )
 
 
 Track = T.List[geo.Point]
 
 
-def parse_gpx(gpx_file: Path) -> T.List[Track]:
+def parse_gpx(gpx_file: Path) -> list[Track]:
     with gpx_file.open("r") as f:
         gpx = gpxpy.parse(f)
 
-    tracks: T.List[Track] = []
+    tracks: list[Track] = []
 
     for track in gpx.tracks:
         for segment in track.segments:

mapillary_tools/geotag/geotag_images_from_nmea_file.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import typing as T
 from pathlib import Path
@@ -15,7 +17,7 @@ class GeotagImagesFromNMEAFile(GeotagImagesFromGPX):
         source_path: Path,
         use_gpx_start_time: bool = False,
         offset_time: float = 0.0,
-        num_processes: T.Optional[int] = None,
+        num_processes: int | None = None,
     ):
         points = get_lat_lon_time_from_nmea(source_path)
         super().__init__(
@@ -27,7 +29,7 @@ class GeotagImagesFromNMEAFile(GeotagImagesFromGPX):
         )
 
 
-def get_lat_lon_time_from_nmea(nmea_file: Path) -> T.List[geo.Point]:
+def get_lat_lon_time_from_nmea(nmea_file: Path) -> list[geo.Point]:
     with nmea_file.open("r") as f:
         lines = f.readlines()
         lines = [line.rstrip("\n\r") for line in lines]

mapillary_tools/geotag/geotag_images_from_video.py

@@ -1,13 +1,13 @@
+from __future__ import annotations
+
 import logging
 import typing as T
 from pathlib import Path
 
-from tqdm import tqdm
-
 from .. import types, utils
 
 from .geotag_from_generic import GeotagImagesFromGeneric
-from .geotag_images_from_gpx import GeotagImagesFromGPXWithProgress
+from .geotag_images_from_gpx import GeotagImagesFromGPX
 
 
 LOG = logging.getLogger(__name__)
@@ -19,69 +19,69 @@ class GeotagImagesFromVideo(GeotagImagesFromGeneric):
         image_paths: T.Sequence[Path],
         video_metadatas: T.Sequence[types.VideoMetadataOrError],
         offset_time: float = 0.0,
-        num_processes: T.Optional[int] = None,
+        num_processes: int | None = None,
     ):
-        self.image_paths = image_paths
+        super().__init__(image_paths, num_processes=num_processes)
         self.video_metadatas = video_metadatas
         self.offset_time = offset_time
-        self.num_processes = num_processes
-        super().__init__()
 
-    def to_description(self) -> T.List[types.ImageMetadataOrError]:
-        # will return this list
-        final_image_metadatas: T.List[types.ImageMetadataOrError] = []
+    def to_description(self) -> list[types.ImageMetadataOrError]:
+        # Will return this list
+        final_image_metadatas: list[types.ImageMetadataOrError] = []
 
-        for video_metadata in self.video_metadatas:
+        video_metadatas, video_error_metadatas = types.separate_errors(
+            self.video_metadatas
+        )
+
+        for video_error_metadata in video_error_metadatas:
+            video_path = video_error_metadata.filename
+            sample_paths = list(
+                utils.filter_video_samples(self.image_paths, video_path)
+            )
+            LOG.debug(
+                "Found %d sample images from video %s with error: %s",
+                len(sample_paths),
+                video_path,
+                video_error_metadata.error,
+            )
+            for sample_path in sample_paths:
+                image_error_metadata = types.describe_error_metadata(
+                    video_error_metadata.error,
+                    sample_path,
+                    filetype=types.FileType.IMAGE,
+                )
+                final_image_metadatas.append(image_error_metadata)
+
+        for video_metadata in video_metadatas:
             video_path = video_metadata.filename
-            LOG.debug("Processing video: %s", video_path)
 
-            sample_image_paths = list(
+            sample_paths = list(
                 utils.filter_video_samples(self.image_paths, video_path)
             )
             LOG.debug(
                 "Found %d sample images from video %s",
-                len(sample_image_paths),
+                len(sample_paths),
                 video_path,
             )
 
-            if isinstance(video_metadata, types.ErrorMetadata):
-                for sample_image_path in sample_image_paths:
-                    error_metadata = types.describe_error_metadata(
-                        video_metadata.error,
-                        sample_image_path,
-                        filetype=types.FileType.IMAGE,
-                    )
-                    final_image_metadatas.append(error_metadata)
-                continue
-
-            with tqdm(
-                total=len(sample_image_paths),
-                desc=f"Interpolating {video_path.name}",
-                unit="images",
-                disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-            ) as pbar:
-                image_metadatas = GeotagImagesFromGPXWithProgress(
-                    sample_image_paths,
-                    video_metadata.points,
-                    use_gpx_start_time=False,
-                    use_image_start_time=True,
-                    offset_time=self.offset_time,
-                    num_processes=self.num_processes,
-                    progress_bar=pbar,
-                ).to_description()
-                final_image_metadatas.extend(image_metadatas)
-
-            # update make and model
-            LOG.debug(
-                'Found camera make "%s" and model "%s"',
-                video_metadata.make,
-                video_metadata.model,
+            geotag = GeotagImagesFromGPX(
+                sample_paths,
+                video_metadata.points,
+                use_gpx_start_time=False,
+                use_image_start_time=True,
+                offset_time=self.offset_time,
+                num_processes=self.num_processes,
             )
+
+            image_metadatas = geotag.to_description()
+
             for metadata in image_metadatas:
                 if isinstance(metadata, types.ImageMetadata):
                     metadata.MAPDeviceMake = video_metadata.make
                     metadata.MAPDeviceModel = video_metadata.model
 
+            final_image_metadatas.extend(image_metadatas)
+
         # NOTE: this method only geotags images that have a corresponding video,
         # so the number of image metadata objects returned might be less than
         # the number of the input image_paths