mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0b1__py3-none-any.whl

mapillary_tools/exiftool_read.py

@@ -6,10 +6,10 @@ import typing as T
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from . import exif_read, utils
+from . import exif_read
 
 
-EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
+EXIFTOOL_NAMESPACES: dict[str, str] = {
     "Adobe": "http://ns.exiftool.org/APP14/Adobe/1.0/",
     "Apple": "http://ns.exiftool.org/MakerNotes/Apple/1.0/",
     "Composite": "http://ns.exiftool.org/Composite/1.0/",
@@ -53,11 +53,11 @@ EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
 
 
 LOG = logging.getLogger(__name__)
+DESCRIPTION_TAG = "rdf:Description"
 _FIELD_TYPE = T.TypeVar("_FIELD_TYPE", int, float, str)
-_DESCRIPTION_TAG = "rdf:Description"
 
 
-def expand_tag(ns_tag: str, namespaces: T.Dict[str, str]) -> str:
+def expand_tag(ns_tag: str, namespaces: dict[str, str]) -> str:
     try:
         ns, tag = ns_tag.split(":", maxsplit=2)
     except ValueError:
@@ -72,42 +72,19 @@ def canonical_path(path: Path) -> str:
     return str(path.resolve().as_posix())
 
 
-def find_rdf_description_path(element: ET.Element) -> T.Optional[Path]:
+def find_rdf_description_path(element: ET.Element) -> Path | None:
     about = element.get(_EXPANDED_ABOUT_TAG)
     if about is None:
         return None
     return Path(about)
 
 
-def index_rdf_description_by_path(
-    xml_paths: T.Sequence[Path],
-) -> T.Dict[str, ET.Element]:
-    rdf_description_by_path: T.Dict[str, ET.Element] = {}
-
-    for xml_path in utils.find_xml_files(xml_paths):
-        try:
-            etree = ET.parse(xml_path)
-        except ET.ParseError as ex:
-            verbose = LOG.getEffectiveLevel() <= logging.DEBUG
-            if verbose:
-                LOG.warning(f"Failed to parse {xml_path}", exc_info=verbose)
-            else:
-                LOG.warning(f"Failed to parse {xml_path}: {ex}", exc_info=verbose)
-            continue
-
-        rdf_description_by_path.update(
-            index_rdf_description_by_path_from_xml_element(etree.getroot())
-        )
-
-    return rdf_description_by_path
-
-
 def index_rdf_description_by_path_from_xml_element(
     element: ET.Element,
 ) -> dict[str, ET.Element]:
     rdf_description_by_path: dict[str, ET.Element] = {}
 
-    elements = element.iterfind(_DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
+    elements = element.iterfind(DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
     for element in elements:
         path = find_rdf_description_path(element)
         if path is not None:
@@ -127,7 +104,7 @@ class ExifToolRead(exif_read.ExifReadABC):
     ) -> None:
         self.etree = etree
 
-    def extract_altitude(self) -> T.Optional[float]:
+    def extract_altitude(self) -> float | None:
         """
         Extract altitude
         """
@@ -143,7 +120,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
     def _extract_gps_datetime(
         self, date_tags: T.Sequence[str], time_tags: T.Sequence[str]
-    ) -> T.Optional[datetime.datetime]:
+    ) -> datetime.datetime | None:
         """
         Extract timestamp from GPS field.
         """
@@ -157,13 +134,13 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return exif_read.parse_gps_datetime_separately(gpsdate, gpstimestamp)
 
-    def extract_gps_datetime(self) -> T.Optional[datetime.datetime]:
+    def extract_gps_datetime(self) -> datetime.datetime | None:
         """
         Extract timestamp from GPS field.
         """
         return self._extract_gps_datetime(["GPS:GPSDateStamp"], ["GPS:GPSTimeStamp"])
 
-    def extract_gps_datetime_from_xmp(self) -> T.Optional[datetime.datetime]:
+    def extract_gps_datetime_from_xmp(self) -> datetime.datetime | None:
         """
         Extract timestamp from XMP GPS field.
         """
@@ -180,7 +157,7 @@ class ExifToolRead(exif_read.ExifReadABC):
         dt_tags: T.Sequence[str],
         subsec_tags: T.Sequence[str],
         offset_tags: T.Sequence[str],
-    ) -> T.Optional[datetime.datetime]:
+    ) -> datetime.datetime | None:
         dtstr = self._extract_alternative_fields(dt_tags, str)
         if dtstr is None:
             return None
@@ -195,7 +172,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return dt
 
-    def extract_exif_datetime_from_xmp(self) -> T.Optional[datetime.datetime]:
+    def extract_exif_datetime_from_xmp(self) -> datetime.datetime | None:
         # EXIF DateTimeOriginal: 0x9003 (date/time when original image was taken)
         # EXIF SubSecTimeOriginal: 0x9291 (fractional seconds for DateTimeOriginal)
         # EXIF OffsetTimeOriginal: 0x9011 (time zone for DateTimeOriginal)
@@ -234,7 +211,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_exif_datetime(self) -> T.Optional[datetime.datetime]:
+    def extract_exif_datetime(self) -> datetime.datetime | None:
         # EXIF DateTimeOriginal: 0x9003 (date/time when original image was taken)
         # EXIF SubSecTimeOriginal: 0x9291 (fractional seconds for DateTimeOriginal)
         # EXIF OffsetTimeOriginal: 0x9011 (time zone for DateTimeOriginal)
@@ -270,7 +247,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_capture_time(self) -> T.Optional[datetime.datetime]:
+    def extract_capture_time(self) -> datetime.datetime | None:
         """
         Extract capture time from EXIF DateTime tags
         """
@@ -300,7 +277,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_direction(self) -> T.Optional[float]:
+    def extract_direction(self) -> float | None:
         """
         Extract image direction (i.e. compass, heading, bearing)
         """
@@ -313,7 +290,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             float,
         )
 
-    def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]:
+    def extract_lon_lat(self) -> tuple[float, float] | None:
         lon_lat = self._extract_lon_lat("GPS:GPSLongitude", "GPS:GPSLatitude")
         if lon_lat is not None:
             return lon_lat
@@ -332,7 +309,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
     def _extract_lon_lat(
         self, lon_tag: str, lat_tag: str
-    ) -> T.Optional[T.Tuple[float, float]]:
+    ) -> tuple[float, float] | None:
         lon = self._extract_alternative_fields(
             [lon_tag],
             float,
@@ -355,7 +332,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return lon, lat
 
-    def extract_make(self) -> T.Optional[str]:
+    def extract_make(self) -> str | None:
         """
         Extract camera make
         """
@@ -374,7 +351,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return make.strip()
 
-    def extract_model(self) -> T.Optional[str]:
+    def extract_model(self) -> str | None:
         """
         Extract camera model
         """
@@ -394,7 +371,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return model.strip()
 
-    def extract_width(self) -> T.Optional[int]:
+    def extract_width(self) -> int | None:
         """
         Extract image width in pixels
         """
@@ -409,7 +386,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             int,
         )
 
-    def extract_height(self) -> T.Optional[int]:
+    def extract_height(self) -> int | None:
         """
         Extract image height in pixels
         """
@@ -447,8 +424,8 @@ class ExifToolRead(exif_read.ExifReadABC):
     def _extract_alternative_fields(
         self,
         fields: T.Sequence[str],
-        field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+        field_type: type[_FIELD_TYPE],
+    ) -> _FIELD_TYPE | None:
         for field in fields:
             value = self.etree.findtext(field, namespaces=EXIFTOOL_NAMESPACES)
             if value is None:

mapillary_tools/exiftool_read_video.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import functools
 import logging
@@ -9,7 +11,7 @@ from .telemetry import GPSFix, GPSPoint
 
 
 MAX_TRACK_ID = 10
-EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
+EXIFTOOL_NAMESPACES: dict[str, str] = {
     "Keys": "http://ns.exiftool.org/QuickTime/Keys/1.0/",
     "IFD0": "http://ns.exiftool.org/EXIF/IFD0/1.0/",
     "QuickTime": "http://ns.exiftool.org/QuickTime/QuickTime/1.0/",
@@ -28,7 +30,7 @@ _FIELD_TYPE = T.TypeVar("_FIELD_TYPE", int, float, str, T.List[str])
 expand_tag = functools.partial(exiftool_read.expand_tag, namespaces=EXIFTOOL_NAMESPACES)
 
 
-def _maybe_float(text: T.Optional[str]) -> T.Optional[float]:
+def _maybe_float(text: str | None) -> float | None:
     if text is None:
         return None
     try:
@@ -37,8 +39,8 @@ def _maybe_float(text: T.Optional[str]) -> T.Optional[float]:
         return None
 
 
-def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> T.Dict[str, T.List[str]]:
-    texts_by_tag: T.Dict[str, T.List[str]] = {}
+def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> dict[str, list[str]]:
+    texts_by_tag: dict[str, list[str]] = {}
     for element in elements:
         tag = element.tag
         if element.text is not None:
@@ -47,10 +49,10 @@ def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> T.Dict[str, T.List[s
 
 
 def _extract_alternative_fields(
-    texts_by_tag: T.Dict[str, T.List[str]],
+    texts_by_tag: dict[str, list[str]],
     fields: T.Sequence[str],
     field_type: T.Type[_FIELD_TYPE],
-) -> T.Optional[_FIELD_TYPE]:
+) -> _FIELD_TYPE | None:
     for field in fields:
         values = texts_by_tag.get(expand_tag(field))
         if values is None:
@@ -80,16 +82,42 @@ def _extract_alternative_fields(
     return None
 
 
+def _same_gps_point(left: GPSPoint, right: GPSPoint) -> bool:
+    """
+    >>> left =  GPSPoint(time=56.0, lat=36.741385, lon=29.021274, alt=141.6, angle=1.54, epoch_time=None, fix=None, precision=None, ground_speed=None)
+    >>> right = GPSPoint(time=56.0, lat=36.741385, lon=29.021274, alt=142.4, angle=1.54, epoch_time=None, fix=None, precision=None, ground_speed=None)
+    >>> _same_gps_point(left, right)
+    True
+    """
+    return (
+        left.time == right.time
+        and left.lon == right.lon
+        and left.lat == right.lat
+        and left.epoch_time == right.epoch_time
+        and left.angle == right.angle
+    )
+
+
+def _deduplicate_gps_points(
+    track: list[GPSPoint], same_gps_point: T.Callable[[GPSPoint, GPSPoint], bool]
+) -> list[GPSPoint]:
+    deduplicated_track: list[GPSPoint] = []
+    for point in track:
+        if not deduplicated_track or not same_gps_point(deduplicated_track[-1], point):
+            deduplicated_track.append(point)
+    return deduplicated_track
+
+
 def _aggregate_gps_track(
-    texts_by_tag: T.Dict[str, T.List[str]],
-    time_tag: T.Optional[str],
+    texts_by_tag: dict[str, list[str]],
+    time_tag: str | None,
     lon_tag: str,
     lat_tag: str,
-    alt_tag: T.Optional[str] = None,
-    gps_time_tag: T.Optional[str] = None,
-    direction_tag: T.Optional[str] = None,
-    ground_speed_tag: T.Optional[str] = None,
-) -> T.List[GPSPoint]:
+    alt_tag: str | None = None,
+    gps_time_tag: str | None = None,
+    direction_tag: str | None = None,
+    ground_speed_tag: str | None = None,
+) -> list[GPSPoint]:
     """
     Aggregate all GPS data by the tags.
     It requires lat, lon to be present, and their lengths must match.
@@ -140,8 +168,8 @@ def _aggregate_gps_track(
     assert len(timestamps) == expected_length
 
     def _aggregate_float_values_same_length(
-        tag: T.Optional[str],
-    ) -> T.List[T.Optional[float]]:
+        tag: str | None,
+    ) -> list[float | None]:
         if tag is not None:
             vals = [
                 _maybe_float(val)
@@ -172,7 +200,7 @@ def _aggregate_gps_track(
                 epoch_time = geo.as_unix_time(dt)
 
     # build track
-    track = []
+    track: list[GPSPoint] = []
     for timestamp, lon, lat, alt, direction, ground_speed in zip(
         timestamps,
         lons,
@@ -183,22 +211,26 @@ def _aggregate_gps_track(
     ):
         if timestamp is None or lon is None or lat is None:
             continue
-        track.append(
-            GPSPoint(
-                time=timestamp,
-                lon=lon,
-                lat=lat,
-                alt=alt,
-                angle=direction,
-                epoch_time=epoch_time,
-                fix=None,
-                precision=None,
-                ground_speed=ground_speed,
-            )
+
+        point = GPSPoint(
+            time=timestamp,
+            lon=lon,
+            lat=lat,
+            alt=alt,
+            angle=direction,
+            epoch_time=epoch_time,
+            fix=None,
+            precision=None,
+            ground_speed=ground_speed,
         )
 
+        if not track or not _same_gps_point(track[-1], point):
+            track.append(point)
+
     track.sort(key=lambda point: point.time)
 
+    track = _deduplicate_gps_points(track, same_gps_point=_same_gps_point)
+
     if time_tag is not None:
         if track:
             first_time = track[0].time
@@ -212,11 +244,11 @@ def _aggregate_samples(
     elements: T.Iterable[ET.Element],
     sample_time_tag: str,
     sample_duration_tag: str,
-) -> T.Generator[T.Tuple[float, float, T.List[ET.Element]], None, None]:
+) -> T.Generator[tuple[float, float, list[ET.Element]], None, None]:
     expanded_sample_time_tag = expand_tag(sample_time_tag)
     expanded_sample_duration_tag = expand_tag(sample_duration_tag)
 
-    accumulated_elements: T.List[ET.Element] = []
+    accumulated_elements: list[ET.Element] = []
     sample_time = None
     sample_duration = None
     for element in elements:
@@ -234,17 +266,17 @@ def _aggregate_samples(
 
 
 def _aggregate_gps_track_by_sample_time(
-    sample_iterator: T.Iterable[T.Tuple[float, float, T.List[ET.Element]]],
+    sample_iterator: T.Iterable[tuple[float, float, list[ET.Element]]],
     lon_tag: str,
     lat_tag: str,
-    alt_tag: T.Optional[str] = None,
-    gps_time_tag: T.Optional[str] = None,
-    direction_tag: T.Optional[str] = None,
-    ground_speed_tag: T.Optional[str] = None,
-    gps_fix_tag: T.Optional[str] = None,
-    gps_precision_tag: T.Optional[str] = None,
-) -> T.List[GPSPoint]:
-    track: T.List[GPSPoint] = []
+    alt_tag: str | None = None,
+    gps_time_tag: str | None = None,
+    direction_tag: str | None = None,
+    ground_speed_tag: str | None = None,
+    gps_fix_tag: str | None = None,
+    gps_precision_tag: str | None = None,
+) -> list[GPSPoint]:
+    track: list[GPSPoint] = []
 
     expanded_gps_fix_tag = None
     if gps_fix_tag is not None:
@@ -308,10 +340,13 @@ class ExifToolReadVideo:
         etree: ET.ElementTree,
     ) -> None:
         self.etree = etree
-        self._texts_by_tag = _index_text_by_tag(self.etree.getroot())
+        root = self.etree.getroot()
+        if root is None:
+            raise ValueError("ElementTree root is None")
+        self._texts_by_tag = _index_text_by_tag(root)
         self._all_tags = set(self._texts_by_tag.keys())
 
-    def extract_gps_track(self) -> T.List[geo.Point]:
+    def extract_gps_track(self) -> list[geo.Point]:
         # blackvue and many other cameras
         track_with_fix = self._extract_gps_track_from_quicktime()
         if track_with_fix:
@@ -329,7 +364,7 @@ class ExifToolReadVideo:
 
         return []
 
-    def _extract_make_and_model(self) -> T.Tuple[T.Optional[str], T.Optional[str]]:
+    def _extract_make_and_model(self) -> tuple[str | None, str | None]:
         make = self._extract_alternative_fields(["GoPro:Make"], str)
         model = self._extract_alternative_fields(["GoPro:Model"], str)
         if model is not None:
@@ -360,15 +395,19 @@ class ExifToolReadVideo:
             model = model.strip()
         return make, model
 
-    def extract_make(self) -> T.Optional[str]:
+    def extract_make(self) -> str | None:
         make, _ = self._extract_make_and_model()
         return make
 
-    def extract_model(self) -> T.Optional[str]:
+    def extract_model(self) -> str | None:
         _, model = self._extract_make_and_model()
         return model
 
-    def _extract_gps_track_from_track(self) -> T.List[GPSPoint]:
+    def _extract_gps_track_from_track(self) -> list[GPSPoint]:
+        root = self.etree.getroot()
+        if root is None:
+            raise ValueError("ElementTree root is None")
+
         for track_id in range(1, MAX_TRACK_ID + 1):
             track_ns = f"Track{track_id}"
             if self._all_tags_exists(
@@ -380,7 +419,7 @@ class ExifToolReadVideo:
                 }
             ):
                 sample_iterator = _aggregate_samples(
-                    self.etree.getroot(),
+                    root,
                     f"{track_ns}:SampleTime",
                     f"{track_ns}:SampleDuration",
                 )
@@ -402,15 +441,15 @@ class ExifToolReadVideo:
         self,
         fields: T.Sequence[str],
         field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+    ) -> _FIELD_TYPE | None:
         return _extract_alternative_fields(self._texts_by_tag, fields, field_type)
 
-    def _all_tags_exists(self, tags: T.Set[str]) -> bool:
+    def _all_tags_exists(self, tags: set[str]) -> bool:
         return self._all_tags.issuperset(tags)
 
     def _extract_gps_track_from_quicktime(
         self, namespace: str = "QuickTime"
-    ) -> T.List[GPSPoint]:
+    ) -> list[GPSPoint]:
         if not self._all_tags_exists(
             {
                 expand_tag(f"{namespace}:GPSDateTime"),

mapillary_tools/exiftool_runner.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-import platform
-import shutil
 import subprocess
 import typing as T
 from pathlib import Path
@@ -12,32 +10,14 @@ class ExiftoolRunner:
     Wrapper around ExifTool to run it in a subprocess
     """
 
-    def __init__(self, exiftool_path: str | None = None, recursive: bool = False):
-        if exiftool_path is None:
-            exiftool_path = self._search_preferred_exiftool_path()
-        self.exiftool_path = exiftool_path
+    def __init__(self, exiftool_executable: str = "exiftool", recursive: bool = False):
+        self.exiftool_executable = exiftool_executable
         self.recursive = recursive
 
-    def _search_preferred_exiftool_path(self) -> str:
-        system = platform.system()
-
-        if system and system.lower() == "windows":
-            exiftool_paths = ["exiftool.exe", "exiftool"]
-        else:
-            exiftool_paths = ["exiftool", "exiftool.exe"]
-
-        for path in exiftool_paths:
-            full_path = shutil.which(path)
-            if full_path:
-                return path
-
-        # Always return the prefered one, even if it is not found,
-        # and let the subprocess.run figure out the error later
-        return exiftool_paths[0]
-
     def _build_args_read_stdin(self) -> list[str]:
         args: list[str] = [
-            self.exiftool_path,
+            self.exiftool_executable,
+            "-fast",
             "-q",
             "-n",  # Disable print conversion
             "-X",  # XML output